diff --git a/Makefile b/Makefile index 19a4c4b9a141..497c98cb2593 100644 --- a/Makefile +++ b/Makefile @@ -34,8 +34,6 @@ DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock IMG ?= gcr.io/istio-testing/build-tools:2019-09-20T15-04-58 UID = $(shell id -u) PWD = $(shell pwd) -GOBIN_SOURCE ?= $(GOPATH)/bin -GOBIN ?= /work/out/bin LOCAL_ARCH := $(shell uname -m) ifeq ($(LOCAL_ARCH),x86_64) @@ -72,7 +70,6 @@ GOOS ?= $(GOOS_LOCAL) RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \ -e GOOS="$(GOOS)" \ -e GOARCH="$(GOARCH)" \ - -e GOBIN="$(GOBIN)" \ -e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \ -e TZ="$(TIMEZONE)" \ -v /etc/passwd:/etc/passwd:ro \ @@ -81,7 +78,6 @@ RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \ --mount type=bind,source="$(PWD)",destination="/work" \ --mount type=volume,source=istio-go-mod,destination="/go/pkg/mod" \ --mount type=volume,source=istio-go-cache,destination="/gocache" \ - --mount type=bind,source="$(GOBIN_SOURCE)",destination="/go/out/bin" \ -w /work $(IMG) else RUN = diff --git a/Makefile.core.mk b/Makefile.core.mk index fdddc7b6db03..eb72f9ffec67 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -51,7 +51,7 @@ GO_TOP := $(shell echo ${GOPATH} | cut -d ':' -f1) export GO_TOP # Note that disabling cgo here adversely affects go get. Instead we'll rely on this -# to be handled in bin/gobuild.sh +# to be handled in common/scripts/gobuild.sh # export CGO_ENABLED=0 # It's more concise to use GO?=$(shell which go) @@ -310,7 +310,7 @@ $(OUTPUT_DIRS): .PHONY: ${GEN_CERT} ${GEN_CERT}: - GOOS=$(GOOS_LOCAL) && GOARCH=$(GOARCH_LOCAL) && CGO_ENABLED=1 bin/gobuild.sh $@ ./security/tools/generate_cert + GOOS=$(GOOS_LOCAL) && GOARCH=$(GOARCH_LOCAL) && CGO_ENABLED=1 common/scripts/gobuild.sh $@ ./security/tools/generate_cert #----------------------------------------------------------------------------- # Target: precommit @@ -357,11 +357,11 @@ RELEASE_BINARIES:=pilot-discovery pilot-agent sidecar-injector mixc mixs mixgen .PHONY: build build: depend - STATIC=0 GOOS=$(GOOS) GOARCH=$(GOARCH) LDFLAGS='-extldflags -static -s -w' bin/gobuild.sh $(ISTIO_OUT)/ $(BINARIES) + STATIC=0 GOOS=$(GOOS) GOARCH=$(GOARCH) LDFLAGS='-extldflags -static -s -w' common/scripts/gobuild.sh $(ISTIO_OUT)/ $(BINARIES) .PHONY: build-linux build-linux: depend - STATIC=0 GOOS=linux GOARCH=amd64 LDFLAGS='-extldflags -static -s -w' bin/gobuild.sh $(ISTIO_OUT_LINUX)/ $(BINARIES) + STATIC=0 GOOS=linux GOARCH=amd64 LDFLAGS='-extldflags -static -s -w' common/scripts/gobuild.sh $(ISTIO_OUT_LINUX)/ $(BINARIES) # Create targets for ISTIO_OUT_LINUX/binary $(foreach bin,$(BINARIES),$(ISTIO_OUT_LINUX)/$(shell basename $(bin))): build-linux @@ -375,9 +375,6 @@ $(foreach bin,$(BINARIES),$(shell basename $(bin))): build lint: buildcache SKIP_INIT=1 bin/linters.sh -shellcheck: - bin/check_shell_scripts.sh - MARKDOWN_LINT_WHITELIST=localhost:8080,storage.googleapis.com/istio-artifacts/pilot/,http://ratings.default.svc.cluster.local:9080/ratings lint_modern: lint-python lint-copyright-banner lint-scripts lint-dockerfiles lint-markdown lint-yaml @@ -411,11 +408,11 @@ DEBUG_LDFLAGS='-extldflags "-static"' # Non-static istioctl targets. These are typically a build artifact. ${ISTIO_OUT}/istioctl-linux: depend - STATIC=0 GOOS=linux LDFLAGS=$(RELEASE_LDFLAGS) bin/gobuild.sh $@ ./istioctl/cmd/istioctl + STATIC=0 GOOS=linux LDFLAGS=$(RELEASE_LDFLAGS) common/scripts/gobuild.sh $@ ./istioctl/cmd/istioctl ${ISTIO_OUT}/istioctl-osx: depend - STATIC=0 GOOS=darwin LDFLAGS=$(RELEASE_LDFLAGS) bin/gobuild.sh $@ ./istioctl/cmd/istioctl + STATIC=0 GOOS=darwin LDFLAGS=$(RELEASE_LDFLAGS) common/scripts/gobuild.sh $@ ./istioctl/cmd/istioctl ${ISTIO_OUT}/istioctl-win.exe: depend - STATIC=0 GOOS=windows LDFLAGS=$(RELEASE_LDFLAGS) bin/gobuild.sh $@ ./istioctl/cmd/istioctl + STATIC=0 GOOS=windows LDFLAGS=$(RELEASE_LDFLAGS) common/scripts/gobuild.sh $@ ./istioctl/cmd/istioctl # generate the istioctl completion files ${ISTIO_OUT}/istioctl.bash: istioctl diff --git a/bin/check_shell_scripts.sh b/bin/check_shell_scripts.sh deleted file mode 100755 index 3be39c130aa5..000000000000 --- a/bin/check_shell_scripts.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# Copyright 2018 Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Runs shellcheck on all shell scripts in the istio repository. - -if ! command -v shellcheck > /dev/null; then - echo 'error: ShellCheck is not installed' - echo 'Visit https://github.com/koalaman/shellcheck#installing' - exit 1 -fi - -BASE_DIR="$(cd "$(dirname "${0}")" && pwd -P)" -ISTIO_ROOT="$(cd "$(dirname "${BASE_DIR}")" && pwd -P)" - -# All files ending in .sh. -SH_FILES=$( \ - find "${ISTIO_ROOT}" \ - -name '*.sh' -type f \ - -not -path '*/vendor/*' \ - -not -path '*/.git/*') -# All files not ending in .sh but starting with a shebang. -SHEBANG_FILES=$( \ - find "${ISTIO_ROOT}" \ - -not -name '*.sh' -type f \ - -not -path '*/vendor/*' \ - -not -path '*/.git/*' | \ - while read -r f; do - head -n 1 "$f" | grep -q '^#!.*sh' && echo "$f"; - done) - -# Set global rule exclusions with the "exclude" flag, separated by comma (e.g. -# "--exclude=SC1090,SC1091"). See https://github.com/koalaman/shellcheck/wiki -# for details on each code's corresponding rule. - -echo "${SH_FILES}" "${SHEBANG_FILES}" \ - | xargs shellcheck diff --git a/bin/get_workspace_status.sh b/bin/get_workspace_status.sh deleted file mode 100755 index 85ad351e3d25..000000000000 --- a/bin/get_workspace_status.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if BUILD_GIT_REVISION=$(git rev-parse HEAD 2> /dev/null); then - if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then - BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty" - fi -else - BUILD_GIT_REVISION=unknown -fi - -# Check for local changes -if git diff-index --quiet HEAD --; then - tree_status="Clean" -else - tree_status="Modified" -fi - -# security wanted VERSION='unknown' -VERSION="${BUILD_GIT_REVISION}" -if [[ -n ${ISTIO_VERSION} ]]; then - VERSION="${ISTIO_VERSION}" -fi - -GIT_DESCRIBE_TAG=$(git describe --tags) - -# used by bin/gobuild.sh -echo "istio.io/pkg/version.buildVersion=${VERSION}" -echo "istio.io/pkg/version.buildGitRevision=${BUILD_GIT_REVISION}" -echo "istio.io/pkg/version.buildStatus=${tree_status}" -echo "istio.io/pkg/version.buildTag=${GIT_DESCRIBE_TAG}" diff --git a/bin/ghdata.py b/bin/ghdata.py deleted file mode 100755 index 5ea030562783..000000000000 --- a/bin/ghdata.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function -import requests -import os -import sys -import datetime - -# get_rawdata uses github releases API to fetch latest releases info -# download_counts are only available to admin users - - -def get_rawdata(token, repo="istio"): - release = "https://api.github.com/repos/istio/{}/releases".format(repo) - headers = {"Authorization": "Bearer {}".format(token)} - resp = requests.get(release, headers=headers) - if not resp.ok: - raise Exception(resp.content) - - return resp.json() - - -# Name of the env var -GITHUB_TOKEN = "GITHUB_TOKEN" - - -def usage(): - print("Visit https://github.com/settings/tokens to generate a token") - print("You must have admin access on the repository get download counts") - - -def main(args): - token = os.environ.get(GITHUB_TOKEN) - if token is None and len(args) > 0: - token = args[0] - # if 1st arg starts with @ read the file - if token.startsWith("@"): - token = open(token).read() - - if token is None: - print("Unable to get GITHUB_TOKEN as env var, first argument or @file") - usage() - return -1 - - try: - data = get_rawdata(token) - except Exception as ex: - print(ex) - usage() - return -1 - - print("# Report created at UTC:", str(datetime.datetime.utcnow())) - for d in ["{}, {}, {}".format( - q['created_at'], - q['download_count'], - q['browser_download_url'].split('/')[-1]) - for j in data for q in j['assets']]: - print(d) - - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/bin/gobuild.sh b/bin/gobuild.sh deleted file mode 100755 index 7f740dd4e382..000000000000 --- a/bin/gobuild.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This script builds and link stamps the output - -VERBOSE=${VERBOSE:-"0"} -V="" -if [[ "${VERBOSE}" == "1" ]];then - V="-x" - set -x -fi - -ROOTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -OUT=${1:?"output path"} -shift - -set -e - -BUILD_GOOS=${GOOS:-linux} -BUILD_GOARCH=${GOARCH:-amd64} -GOBINARY=${GOBINARY:-go} -GOPKG="$GOPATH/pkg" -BUILDINFO=${BUILDINFO:-""} -STATIC=${STATIC:-1} -LDFLAGS=${LDFLAGS:--extldflags -static} -GOBUILDFLAGS=${GOBUILDFLAGS:-""} -# Split GOBUILDFLAGS by spaces into an array called GOBUILDFLAGS_ARRAY. -IFS=' ' read -r -a GOBUILDFLAGS_ARRAY <<< "$GOBUILDFLAGS" - -GCFLAGS=${GCFLAGS:-} -export CGO_ENABLED=0 - -if [[ "${STATIC}" != "1" ]];then - LDFLAGS="" -fi - -# gather buildinfo if not already provided -# For a release build BUILDINFO should be produced -# at the beginning of the build and used throughout -if [[ -z ${BUILDINFO} ]];then - BUILDINFO=$(mktemp) - "${ROOTDIR}/bin/get_workspace_status.sh" > "${BUILDINFO}" -fi - -# BUILD LD_EXTRAFLAGS -LD_EXTRAFLAGS="" - -while read -r line; do - LD_EXTRAFLAGS="${LD_EXTRAFLAGS} -X ${line}" -done < "${BUILDINFO}" - -# forgoing -i (incremental build) because it will be deprecated by tool chain. -time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \ - ${V} "${GOBUILDFLAGS_ARRAY[@]}" ${GCFLAGS:+-gcflags "${GCFLAGS}"} \ - -o "${OUT}" \ - -trimpath \ - -pkgdir="${GOPKG}/${BUILD_GOOS}_${BUILD_GOARCH}" \ - -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${@}" diff --git a/common-protos/.commonfiles.sha b/common-protos/.commonfiles.sha index beae79d3c6fb..a4b5f286d854 100644 --- a/common-protos/.commonfiles.sha +++ b/common-protos/.commonfiles.sha @@ -1 +1 @@ -26f9010dc0dd119451e2e32fb7f704121af55104 +8f666ddd60790734388e89a29932014a632ebada diff --git a/common-protos/google/protobuf/compiler/ruby/ruby_generated_code.proto b/common-protos/google/protobuf/compiler/ruby/ruby_generated_code.proto deleted file mode 100644 index 42d82a6bac7e..000000000000 --- a/common-protos/google/protobuf/compiler/ruby/ruby_generated_code.proto +++ /dev/null @@ -1,67 +0,0 @@ -syntax = "proto3"; - -package A.B.C; - -message TestMessage { - int32 optional_int32 = 1; - int64 optional_int64 = 2; - uint32 optional_uint32 = 3; - uint64 optional_uint64 = 4; - bool optional_bool = 5; - double optional_double = 6; - float optional_float = 7; - string optional_string = 8; - bytes optional_bytes = 9; - TestEnum optional_enum = 10; - TestMessage optional_msg = 11; - - repeated int32 repeated_int32 = 21; - repeated int64 repeated_int64 = 22; - repeated uint32 repeated_uint32 = 23; - repeated uint64 repeated_uint64 = 24; - repeated bool repeated_bool = 25; - repeated double repeated_double = 26; - repeated float repeated_float = 27; - repeated string repeated_string = 28; - repeated bytes repeated_bytes = 29; - repeated TestEnum repeated_enum = 30; - repeated TestMessage repeated_msg = 31; - - oneof my_oneof { - int32 oneof_int32 = 41; - int64 oneof_int64 = 42; - uint32 oneof_uint32 = 43; - uint64 oneof_uint64 = 44; - bool oneof_bool = 45; - double oneof_double = 46; - float oneof_float = 47; - string oneof_string = 48; - bytes oneof_bytes = 49; - TestEnum oneof_enum = 50; - TestMessage oneof_msg = 51; - } - - map map_int32_string = 61; - map map_int64_string = 62; - map map_uint32_string = 63; - map map_uint64_string = 64; - map map_bool_string = 65; - map map_string_string = 66; - map map_string_msg = 67; - map map_string_enum = 68; - map map_string_int32 = 69; - map map_string_bool = 70; - - message NestedMessage { - int32 foo = 1; - } - - NestedMessage nested_message = 80; -} - -enum TestEnum { - Default = 0; - A = 1; - B = 2; - C = 3; -} diff --git a/common-protos/google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto b/common-protos/google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto deleted file mode 100644 index 8d3cc13e03d2..000000000000 --- a/common-protos/google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto +++ /dev/null @@ -1,68 +0,0 @@ -syntax = "proto2"; - -package A.B.C; - -message TestMessage { - optional int32 optional_int32 = 1 [default = 1]; - optional int64 optional_int64 = 2 [default = 2]; - optional uint32 optional_uint32 = 3 [default = 3]; - optional uint64 optional_uint64 = 4 [default = 4]; - optional bool optional_bool = 5 [default = true]; - optional double optional_double = 6 [default = 6.0]; - optional float optional_float = 7 [default = 7.0]; - optional string optional_string = 8 [default = "default str"]; - optional bytes optional_bytes = 9 [default = "\0\1\2\100fubar"]; - optional TestEnum optional_enum = 10 [default = A]; - optional TestMessage optional_msg = 11; - - repeated int32 repeated_int32 = 21; - repeated int64 repeated_int64 = 22; - repeated uint32 repeated_uint32 = 23; - repeated uint64 repeated_uint64 = 24; - repeated bool repeated_bool = 25; - repeated double repeated_double = 26; - repeated float repeated_float = 27; - repeated string repeated_string = 28; - repeated bytes repeated_bytes = 29; - repeated TestEnum repeated_enum = 30; - repeated TestMessage repeated_msg = 31; - - required int32 required_int32 = 41; - required int64 required_int64 = 42; - required uint32 required_uint32 = 43; - required uint64 required_uint64 = 44; - required bool required_bool = 45; - required double required_double = 46; - required float required_float = 47; - required string required_string = 48; - required bytes required_bytes = 49; - required TestEnum required_enum = 50; - required TestMessage required_msg = 51; - - oneof my_oneof { - int32 oneof_int32 = 61; - int64 oneof_int64 = 62; - uint32 oneof_uint32 = 63; - uint64 oneof_uint64 = 64; - bool oneof_bool = 65; - double oneof_double = 66; - float oneof_float = 67; - string oneof_string = 68; - bytes oneof_bytes = 69; - TestEnum oneof_enum = 70; - TestMessage oneof_msg = 71; - } - - message NestedMessage { - optional int32 foo = 1; - } - - optional NestedMessage nested_message = 80; -} - -enum TestEnum { - Default = 0; - A = 1; - B = 2; - C = 3; -} diff --git a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit.proto b/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit.proto deleted file mode 100644 index 8d7c948a180b..000000000000 --- a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; - -package one.two.a_three; - -option ruby_package = "A::B::C"; - -message Four { - string a_string = 1; -} diff --git a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit_legacy.proto b/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit_legacy.proto deleted file mode 100644 index 7a0d26086544..000000000000 --- a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_explicit_legacy.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; - -package one.two.a_three.and; - -option ruby_package = "AA.BB.CC"; - -message Four { - string another_string = 1; -} diff --git a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_implicit.proto b/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_implicit.proto deleted file mode 100644 index 544db64d94e9..000000000000 --- a/common-protos/google/protobuf/compiler/ruby/ruby_generated_pkg_implicit.proto +++ /dev/null @@ -1,7 +0,0 @@ -syntax = "proto3"; - -package one.two.a_three; - -message Four { - string a_string = 1; -} diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha index 2271b46c5421..a4b5f286d854 100644 --- a/common/.commonfiles.sha +++ b/common/.commonfiles.sha @@ -1 +1 @@ -ceedeea7179dc6dc721be783ac1b58e20a7fefe4 +8f666ddd60790734388e89a29932014a632ebada diff --git a/common/scripts/report_build_info.sh b/common/scripts/report_build_info.sh index 1ab7d070d910..06f9afcf5e9e 100755 --- a/common/scripts/report_build_info.sh +++ b/common/scripts/report_build_info.sh @@ -36,18 +36,16 @@ else tree_status="Modified" fi -# XXX This needs to be updated to accommodate tags added after building, rather than prior to builds -RELEASE_TAG=$(git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --exact-match 2> /dev/null || echo "") - # security wanted VERSION='unknown' VERSION="${BUILD_GIT_REVISION}" -if [[ -n "${RELEASE_TAG}" ]]; then - VERSION="${RELEASE_TAG}" -elif [[ -n ${ISTIO_VERSION} ]]; then +if [[ -n ${ISTIO_VERSION} ]]; then VERSION="${ISTIO_VERSION}" fi +GIT_DESCRIBE_TAG=$(git describe --tags) + # used by common/scripts/gobuild.sh echo "istio.io/pkg/version.buildVersion=${VERSION}" echo "istio.io/pkg/version.buildGitRevision=${BUILD_GIT_REVISION}" echo "istio.io/pkg/version.buildStatus=${tree_status}" +echo "istio.io/pkg/version.buildTag=${GIT_DESCRIBE_TAG}" diff --git a/mixer/test/listbackend/cmd/Makefile b/mixer/test/listbackend/cmd/Makefile index c9ae08c88ce6..939cf86ea030 100644 --- a/mixer/test/listbackend/cmd/Makefile +++ b/mixer/test/listbackend/cmd/Makefile @@ -15,7 +15,7 @@ .PHONY: deploy docker listadapter: main.go - ../../../../bin/gobuild.sh listadapter istio.io/istio/pkg/version . + ../../../../common/scripts/gobuild.sh listadapter istio.io/istio/pkg/version . docker: listadapter Dockerfile docker build . -t gcr.io/mixologist-142215/listadapter:latest diff --git a/mixer/test/prometheus/cmd/Makefile b/mixer/test/prometheus/cmd/Makefile index 55536113adde..f49e27c8f994 100644 --- a/mixer/test/prometheus/cmd/Makefile +++ b/mixer/test/prometheus/cmd/Makefile @@ -15,7 +15,7 @@ .PHONY: deploy docker prometheusadapter: main.go - ../../../../bin/gobuild.sh prometheusadapter istio.io/istio/pkg/version . + ../../../../common/scripts/gobuild.sh prometheusadapter istio.io/istio/pkg/version . docker: prometheusadapter docker build . -t gcr.io/mixologist-142215/prometheusadapter:latest