Rework build tool installation

Makefile

@@ -18,28 +18,15 @@ GOARCH 	?= $(shell go env GOARCH)
 GOOS 	?= $(shell go env GOOS)
 GOPROXY ?= $(shell go env GOPROXY)
 
-# Runnable tools
-GO 				?= go
-BUF 			:= $(GO) run github.com/bufbuild/buf/cmd/[email protected]
-CONTROLLER_GEN 	:= $(GO) run sigs.k8s.io/controller-tools/cmd/[email protected]
-GOFUMPT 		:= $(GO) run mvdan.cc/[email protected]
-KUSTOMIZE 		:= $(GO) run sigs.k8s.io/kustomize/kustomize/[email protected]
-SETUP_ENVTEST   := $(GO) run sigs.k8s.io/controller-runtime/tools/[email protected]
-GOLANGCI_LINT	:= $(GO) run github.com/golangci/golangci-lint/cmd/[email protected]
-YAMLFMT			:= $(GO) run github.com/google/yamlfmt/cmd/[email protected]
-MOQ				:= $(GO) run github.com/matryer/[email protected]
-
-# Installed tools
-PROTOC_GEN_GO_GRPC 	:= google.golang.org/grpc/cmd/[email protected]
-PROTOC_GEN_GO 		:= google.golang.org/protobuf/cmd/[email protected]
+include Tools.mk
 
 .PHONY: help
 help: ## Print this help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[%\/0-9A-Za-z_-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 	@echo
 	@echo Individual binaries can be built with their name. For example, \`make tink-server\`.
 	@echo
-	@echo Individual images can be built with their name appended with -image. For example, 
+	@echo Individual images can be built with their name appended with -image. For example,
 	@echo \`make tink-server-image\`.
 
 # Version defines the string injected into binaries that indicates the version of the build.
@@ -55,15 +42,15 @@ BINARIES := tink-server tink-agent tink-worker tink-controller virtual-worker
 build: $(BINARIES) ## Build all tink binaries. Cross build by setting GOOS and GOARCH.
 
 # Create targets for all the binaries we build. They can be individually invoked with `make <binary>`.
-# For example, `make tink-server`. Callers can cross build by defining the GOOS and GOARCH 
+# For example, `make tink-server`. Callers can cross build by defining the GOOS and GOARCH
 # variables. For example, `GOOS=linux GOARCH=arm64 make tink-server`.
 # See https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html.
 .PHONY: $(BINARIES)
 $(BINARIES):
 	CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build $(LDFLAGS) -o ./bin/$@-$(GOOS)-$(GOARCH) ./cmd/$@
 
-# IMAGE_ARGS is resolved when its used in the `%-image` targets. Consequently, the $* automatic 
-# variable isn't evaluated until the target is called.  
+# IMAGE_ARGS is resolved when its used in the `%-image` targets. Consequently, the $* automatic
+# variable isn't evaluated until the target is called.
 # See https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html.
 IMAGE_ARGS ?= -t $*
 
@@ -77,8 +64,8 @@ images: $(addsuffix -image,$(BINARIES)) ## Build all tink container images. All
 # We only build Linux images so we need to force binaries to be built for Linux. Exporting the
 # GOOS variable ensures the recipe's binary dependency is built for Linux.
 #
-# The $$* leverages .SECONDEXPANSION to specify the matched part of the target name as a 
-# dependency. In doing so, we ensure the binary is built so it can be copied into the image. For 
+# The $$* leverages .SECONDEXPANSION to specify the matched part of the target name as a
+# dependency. In doing so, we ensure the binary is built so it can be copied into the image. For
 # example, `make tink-server-image` will depend on `tink-server`.
 # See https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html.
 # See https://www.gnu.org/software/make/manual/html_node/Secondary-Expansion.html.
@@ -92,17 +79,18 @@ test: ## Run tests
 
 .PHONY: e2e-test
 e2e-test: ## Run e2e tests
+e2e-test: $(SETUP_ENVTEST)
 	$(SETUP_ENVTEST) use
 	source <($(SETUP_ENVTEST) use -p env) && $(GO) test -v ./internal/e2e/... -tags=e2e
 
-mocks:
+mocks: $(MOQ)
 	$(MOQ) -fmt goimpots -rm -out ./internal/proto/workflow/v2/mock.go ./internal/proto/workflow/v2 WorkflowServiceClient WorkflowService_GetWorkflowsClient
 	$(MOQ) -fmt goimports -rm -out ./internal/agent/transport/mock.go ./internal/agent/transport WorkflowHandler
 	$(MOQ) -fmt goimports -rm -out ./internal/agent/mock.go ./internal/agent Transport ContainerRuntime
 	$(MOQ) -fmt goimports -rm -out ./internal/agent/event/mock.go ./internal/agent/event Recorder
 
 .PHONY: generate-proto
-generate-proto: buf.gen.yaml buf.lock $(shell git ls-files '**/*.proto') _protoc
+generate-proto: buf.gen.yaml buf.lock $(shell git ls-files '**/*.proto') $(BUF) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(GOFUMPT)
 	$(BUF) mod update
 	$(BUF) generate
 	$(GOFUMPT) -w internal/proto/*.pb.*
@@ -112,15 +100,15 @@ generate-proto: buf.gen.yaml buf.lock $(shell git ls-files '**/*.proto') _protoc
 generate: generate-proto generate-go generate-manifests ## Generate code, manifests etc.
 
 .PHONY: generate-go
-generate-go:
+generate-go: $(CONTROLLER_GEN) $(GOFUMPT)
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.generatego.txt" paths="./api/..."
 	$(GOFUMPT) -w ./api
 
 .PHONY: generate-manifests
 generate-manifests: generate-crds generate-rbacs generate-server-rbacs ## Generate manifests e.g. CRD, RBAC etc.
 
 .PHONY: generate-crds
-generate-crds:
+generate-crds: $(CONTROLLER_GEN) $(YAMLFMT)
 	$(CONTROLLER_GEN) \
 		paths=./api/... \
 		crd:crdVersions=v1 \
@@ -131,15 +119,15 @@ generate-crds:
 	$(YAMLFMT) ./config/crd/bases/* ./config/webhook/*
 
 .PHONY: generate-rbacs
-generate-rbacs:
+generate-rbacs: $(CONTROLLER_GEN) $(YAMLFMT)
 	$(CONTROLLER_GEN) \
 		paths=./internal/controller/... \
 		output:rbac:dir=./config/rbac/ \
 		rbac:roleName=manager-role
 	$(YAMLFMT) ./config/rbac/*
 
 .PHONY: generate-server-rbacs
-generate-server-rbacs:
+generate-server-rbacs: $(CONTROLLER_GEN) $(YAMLFMT)
 	$(CONTROLLER_GEN) \
 		paths=./internal/server/... \
 		output:rbac:dir=./config/server-rbac \
@@ -159,7 +147,7 @@ out/release/default/kustomization.yaml: config/default/kustomization.yaml
 	mkdir -p out/
 	cp -a config/ out/release/
 
-out/release/tink.yaml: generate-manifests out/release/default/kustomization.yaml
+out/release/tink.yaml: generate-manifests out/release/default/kustomization.yaml $(KUSTOMIZE)
 	(
 		cd out/release/default && \
 		$(KUSTOMIZE) edit set image server=$(TINK_SERVER_IMAGE):$(TINK_CONTROLLER_TAG) controller=$(TINK_CONTROLLER_IMAGE):$(TINK_CONTROLLER_TAG) && \
@@ -169,10 +157,12 @@ out/release/tink.yaml: generate-manifests out/release/default/kustomization.yaml
 	prettier --write $@
 
 .PHONY: release-manifests
-release-manifests: out/release/tink.yaml ## Builds the manifests to publish with a release.
+release-manifests: ## Builds the manifests to publish with a release.
+release-manifests: out/release/tink.yaml
 
 .PHONY: check-generated
-check-generated: check-proto ## Check if generated files are up to date.
+check-generated: ## Check if generated files are up to date.
+check-generated: check-proto
 
 .PHONY: check-proto
 check-proto: generate-proto
@@ -182,7 +172,8 @@ check-proto: generate-proto
 	)
 
 .PHONY: verify
-verify: lint check-generated ## Verify code style, is lint free, freshness ...
+verify: ## Verify code style, is lint free, freshness ...
+verify: lint check-generated $(GOFUMPT)
 	$(GOFUMPT) -d .
 
 .PHONY: ci-checks
@@ -194,59 +185,21 @@ ci-checks: ## Run ci-checks.sh script
 	fi
 
 .PHONY: lint
-lint: shellcheck hadolint golangci-lint yamllint ## Lint code
-
-LINT_ARCH := $(shell uname -m)
-LINT_OS := $(shell uname)
-LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
-
-SHELLCHECK_VERSION ?= v0.8.0
-SHELLCHECK_BIN := out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
-$(SHELLCHECK_BIN):
-	mkdir -p out/linters
-	curl -sSfL -o [email protected] https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_OS_LOWER).$(LINT_ARCH).tar.xz \
-		|| echo "Unable to fetch shellcheck for $(LINT_OS)/$(LINT_ARCH): falling back to locally install"
-	test -f [email protected] \
-		&& tar -C out/linters -xJf [email protected] \
-		&& mv out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck $@ \
-		|| printf "#!/usr/bin/env shellcheck\n" > $@
-	chmod u+x $@
+lint: ## Lint code.
+lint: shellcheck hadolint golangci-lint yamllint
 
 .PHONY: shellcheck
-shellcheck: $(SHELLCHECK_BIN)
-	$(SHELLCHECK_BIN) $(shell find . -name "*.sh")
-
-HADOLINT_VERSION ?= v2.12.1-beta
-HADOLINT_BIN := out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
-$(HADOLINT_BIN):
-	mkdir -p out/linters
-	curl -sSfL -o [email protected] https://github.com/hadolint/hadolint/releases/download/$(HADOLINT_VERSION)/hadolint-$(LINT_OS)-$(LINT_ARCH) \
-		|| echo "Unable to fetch hadolint for $(LINT_OS)/$(LINT_ARCH), falling back to local install"
-	test -f [email protected] && mv $(HADOLINT_BIN).dl $@ || printf "#!/usr/bin/env hadolint\n" > $@
-	chmod u+x $@
+shellcheck: $(SHELLCHECK)
+	$(SHELLCHECK) $(shell find . -name "*.sh")
 
 .PHONY: hadolint
-hadolint: $(HADOLINT_BIN)
-	$(HADOLINT_BIN) --no-fail $(shell find . -name "*Dockerfile")
+hadolint: $(HADOLINT)
+	$(HADOLINT) --no-fail $(shell find . -name "*Dockerfile")
 
 .PHONY: golangci-lint
-golangci-lint:
+golangci-lint: $(GOLANGCI_LINT)
 	$(GOLANGCI_LINT) run
 
-YAMLLINT_VERSION ?= 1.26.3
-YAMLLINT_ROOT := out/linters/yamllint-$(YAMLLINT_VERSION)
-YAMLLINT_BIN := $(YAMLLINT_ROOT)/dist/bin/yamllint
-$(YAMLLINT_BIN):
-	mkdir -p out/linters
-	rm -rf out/linters/yamllint-*
-	curl -sSfL https://github.com/adrienverge/yamllint/archive/refs/tags/v$(YAMLLINT_VERSION).tar.gz | tar -C out/linters -zxf -
-	cd $(YAMLLINT_ROOT) && pip3 install --target dist . || pip install --target dist .
-
 .PHONY: yamllint
 yamllint: $(YAMLLINT_BIN)
-	PYTHONPATH=$(YAMLLINT_ROOT)/dist $(YAMLLINT_ROOT)/dist/bin/yamllint .
-
-.PHONY: _protoc ## Install all required tools for use with this Makefile.
-_protoc:
-	GOBIN=$${PWD}/bin $(GO) install $(PROTOC_GEN_GO)
-	GOBIN=$${PWD}/bin $(GO) install $(PROTOC_GEN_GO_GRPC)
+	$(YAMLLINT) .

Tools.mk

@@ -0,0 +1,138 @@
+# Define the directory tools are installed to.
+TOOLS_DIR := $(PWD)/out/tools
+
+# Some tools rely on other tools being on the PATH (protoc).
+PATH := $(PATH):$(TOOLS_DIR)
+
+# Define a variable for Go so we can easily change the version we use.
+GO ?= go
+
+LINT_ARCH := $(shell uname -m)
+LINT_OS := $(shell uname)
+LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
+
+BUF_VER := v1.11
+BUF     := $(TOOLS_DIR)/buf-$(BUF_VER)
+
+GOFUMPT_VER := v0.4
+GOFUMPT     := $(TOOLS_DIR)/gofumpt-$(GOFUMPT_VER)
+
+PROTOC_GEN_GO_GRPC_VER := v1.2
+PROTOC_GEN_GO_GRPC     := $(TOOLS_DIR)/protoc-gen-go-grpc
+
+PROTOC_GEN_GO_VER := v1.28
+PROTOC_GEN_GO     := $(TOOLS_DIR)/protoc-gen-go
+
+CONTROLLER_GEN_VER := v0.11
+CONTROLLER_GEN     := $(TOOLS_DIR)/controller-gen-$(CONTROLLER_GEN_VER)
+
+KUSTOMIZE_VER := v4.5
+KUSTOMIZE     := $(TOOLS_DIR)/kustomize-$(KUSTOMIZE_VER)
+
+SETUP_ENVTEST_VER := v0.0.0-20220304125252-9ee63fc65a97
+SETUP_ENVTEST     := $(TOOLS_DIR)/setup-envtest-$(SETUP_ENVTEST_VER)
+
+GOLANGCI_LINT_VER := v1.52
+GOLANGCI_LINT     := $(TOOLS_DIR)/golangci-lint-$(GOLANGCI_LINT_VER)
+
+YAMLFMT_VER := v0.6
+YAMLFMT     := $(TOOLS_DIR)/yamlfmt-$(YAMLFMT_VER)
+
+MOQ_VER := v0.3
+MOQ     := $(TOOLS_DIR)/moq-$(MOQ_VER)
+
+SHELLCHECK_VER := v0.8.0
+SHELLCHECK     := $(TOOLS_DIR)/shellcheck-$(SHELLCHECK_VER)
+
+HADOLINT_VER := v2.12.1-beta
+HADOLINT     := $(TOOLS_DIR)/hadolint-$(HADOLINT_VER)
+
+YAMLLINT_VER  := 1.26.3
+YAMLLINT_ROOT := $(TOOLS_DIR)/yamllint-$(YAMLLINT_VER)
+YAMLLINT_BIN  := $(YAMLLINT_ROOT)/dist/bin/yamllint
+YAMLLINT      := PYTHONPATH=$(YAMLLINT_ROOT)/dist $(YAMLLINT_ROOT)/dist/bin/yamllint
+
+tools: ## Install all tool depedencies.
+tools: $(BUF) $(GOFUMPT) $(PROTOC_GEN_GO_GRPC) $(PROTOC_GEN_GO) $(CONTROLLER_GEN) $(KUSTOMIZE)
+tools: $(SETUP_ENVTEST) $(GOLANGCI_LINT) $(YAMLFMT) $(MOQ) $(SHELLCHECK) $(HADOLINT) $(YAMLLINT_BIN)
+
+$(BUF):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing buf at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install github.com/bufbuild/buf/cmd/buf@$(BUF_VER)
+	@mv $(TOOLS_DIR)/buf $@
+
+$(GOFUMPT):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing gofumpt at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install mvdan.cc/gofumpt@$(GOFUMPT_VER)
+	@mv $(TOOLS_DIR)/gofumpt $@
+
+$(PROTOC_GEN_GO_GRPC):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing protoc-gen-go-grpc at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$(PROTOC_GEN_GO_GRPC_VER)
+
+$(PROTOC_GEN_GO):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing protoc-gen-go at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@$(PROTOC_GEN_GO_VER)
+
+$(CONTROLLER_GEN):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing controller-gen at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VER)
+	@mv $(TOOLS_DIR)/controller-gen $@
+
+$(KUSTOMIZE):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing kustomize at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install sigs.k8s.io/kustomize/kustomize/v4@$(KUSTOMIZE_VER)
+	@mv $(TOOLS_DIR)/kustomize $@
+
+$(SETUP_ENVTEST):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing setup-envtest at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VER)
+	@mv $(TOOLS_DIR)/setup-envtest $@
+
+$(GOLANGCI_LINT):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing golangci-lint at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VER)
+	@mv $(TOOLS_DIR)/golangci-lint $@
+
+$(YAMLFMT):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing yamlfmt at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install github.com/google/yamlfmt/cmd/yamlfmt@$(YAMLFMT_VER)
+	@mv $(TOOLS_DIR)/yamlfmt $@
+
+$(MOQ):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing moq at $@"
+	@GOBIN=$(TOOLS_DIR) $(GO) install github.com/matryer/moq@$(MOQ_VER)
+	@mv $(TOOLS_DIR)/moq $@
+
+$(SHELLCHECK):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing shellcheck at $@"
+	@mkdir -p _tmp
+	@curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VER)/shellcheck-$(SHELLCHECK_VER).$(LINT_OS_LOWER).$(LINT_ARCH).tar.xz | \
+		tar -C _tmp -xJ --strip-components 1 -f -
+	@mv _tmp/shellcheck $@
+	@rm -rf _tmp
+	@chmod u+x $@
+
+$(HADOLINT):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing hadolint at $@"
+	@curl -sSfL -o $@ https://github.com/hadolint/hadolint/releases/download/$(HADOLINT_VER)/hadolint-$(LINT_OS)-$(LINT_ARCH)
+	@chmod u+x $@
+
+$(YAMLLINT_BIN):
+	@mkdir -p $(TOOLS_DIR)
+	@echo "Installing yamllint at $@"
+	@curl -sSfL https://github.com/adrienverge/yamllint/archive/refs/tags/v$(YAMLLINT_VER).tar.gz | \
+		tar -C $(TOOLS_DIR) -zxf -
+	@cd $(YAMLLINT_ROOT) && pip3 install --target dist . > /dev/null || pip install --target dist .

buf.lock

@@ -4,4 +4,4 @@ deps:
   - remote: buf.build
     owner: googleapis
     repository: googleapis
-    commit: a86849a25cc04f4dbe9b15ddddfbc488
+    commit: e874a0be2bf140a5a4c7d4122c635823

config/crd/bases/tinkerbell.org_workflows.yaml

@@ -157,7 +157,7 @@ spec:
             spec:
               properties:
                 hardwareRef:
-                  description: HardwareRef is a reference to a Hardware resource this workflow will execute on. If no namespace is specified the Workflow's namespace is assumed.
+                  description: HardwareRef is a reference to a Hardware resource this workflow will execute on.
                   properties:
                     name:
                       description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
@@ -170,7 +170,7 @@ spec:
                   description: "TemplateParams are a list of key-value pairs that are injected into templates at render time. TemplateParams are exposed to templates using a top level .Params key. \n For example, TemplateParams = {\"foo\": \"bar\"}, the foo key can be accessed via .Params.foo."
                   type: object
                 templateRef:
-                  description: TemplateRef is a reference to a Template resource used to render workflow actions. If no namespace is specified the Workflow's namespace is assumed.
+                  description: TemplateRef is a reference to a Template resource used to render workflow actions.
                   properties:
                     name:
                       description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
@@ -294,7 +294,7 @@ spec:
                   format: date-time
                   type: string
                 state:
-                  description: State describes the current state of the workflow. For the workflow to enter the WorkflowStateSucceeded state all actions must be in ActionStateSucceeded. The Workflow will enter a WorkflowStateFailed if 1 or more Actions fails.
+                  description: State describes the current state of the Workflow.
                   type: string
               required:
                 - actions