From d8a5a4a35f758e38a30ffab65e0e51b46a496af6 Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Fri, 9 Nov 2018 14:11:27 +0100 Subject: [PATCH] bazel: Allow to distdir all dependencies To use --distdir option of Bazel (which allows to use previously fetched tarballs instead of downloading dependencies during build), all dependencies should use http instead of git and need to have sha256 sums specified. Signed-off-by: Michal Rostecki --- Makefile | 1 + WORKSPACE | 14 ++++++++++---- cc_gogo_protobuf.bzl | 11 +++++++---- googleapis.bzl | 12 ++++++++---- protobuf.bzl | 13 +++++++++---- repositories.bzl | 30 +++++++++++++++++++---------- script/check-repositories | 40 +++++++++++++++++++++++++++++++++++++++ x_tools_imports.bzl | 13 +++++++++---- 8 files changed, 104 insertions(+), 30 deletions(-) create mode 100755 script/check-repositories diff --git a/Makefile b/Makefile index 501d2ebb14f..a73ee832f09 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ test_tsan: check: @script/check-license-headers + @script/check-repositories @script/check-style artifacts: build diff --git a/WORKSPACE b/WORKSPACE index 0cc27fae06b..a86fb9fb121 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,11 +31,13 @@ bind( # When updating envoy sha manually please update the sha in istio.deps file also ENVOY_SHA = "74de08a0d4d31bd466639d25d681df5d290bb770" +ENVOY_SHA256 = "12905d063be326a2f6e60b9d3448772e5a163794e7d7e46276b29c9565fe0365" http_archive( name = "envoy", strip_prefix = "envoy-" + ENVOY_SHA, url = "https://github.com/envoyproxy/envoy/archive/" + ENVOY_SHA + ".zip", + sha256 = ENVOY_SHA256, ) load("@envoy//bazel:repositories.bzl", "envoy_dependencies") @@ -51,9 +53,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_too go_rules_dependencies() go_register_toolchains() -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -git_repository( +# Nov 28, 2017 (bazel 0.8.0 support) +RULES_PROTOBUF_SHA = "563b674a2ce6650d459732932ea2bc98c9c9a9bf" +RULES_PROTOBUF_SHA256 = "338e0d65cd709c6a6f9b5702466e641d536479be8b564d1e12a5d1de22a5cff6" + +http_archive( name = "org_pubref_rules_protobuf", - commit = "563b674a2ce6650d459732932ea2bc98c9c9a9bf", # Nov 28, 2017 (bazel 0.8.0 support) - remote = "https://github.com/pubref/rules_protobuf", + strip_prefix = "rules_protobuf-" + RULES_PROTOBUF_SHA, + url = "https://github.com/pubref/rules_protobuf/archive/" + RULES_PROTOBUF_SHA + ".tar.gz", + sha256 = RULES_PROTOBUF_SHA256, ) diff --git a/cc_gogo_protobuf.bzl b/cc_gogo_protobuf.bzl index 4f549fd2e29..18e5563ca23 100644 --- a/cc_gogo_protobuf.bzl +++ b/cc_gogo_protobuf.bzl @@ -14,8 +14,10 @@ # ################################################################################ # +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +GOGO_PROTO_SHA = "100ba4e885062801d56799d78530b73b178a78f3" +GOGO_PROTO_SHA256 = "b04eb8eddd2d15d8b12d111d4ef7816fca6e5c5d495adf45fb8478278aa80f79" def cc_gogoproto_repositories(bind=True): BUILD = """ @@ -56,10 +58,11 @@ cc_proto_library( ], ) """ - new_git_repository( + http_archive( name = "gogoproto_git", - commit = "100ba4e885062801d56799d78530b73b178a78f3", - remote = "https://github.com/gogo/protobuf", + strip_prefix = "protobuf-" + GOGO_PROTO_SHA, + url = "https://github.com/gogo/protobuf/archive/" + GOGO_PROTO_SHA + ".tar.gz", + sha256 = GOGO_PROTO_SHA256, build_file_content = BUILD, ) diff --git a/googleapis.bzl b/googleapis.bzl index 62cbd0e33ab..22ca9128793 100644 --- a/googleapis.bzl +++ b/googleapis.bzl @@ -14,8 +14,11 @@ # ################################################################################ # +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +# Oct 21, 2016 (only release pre-dates sha) +GOOGLEAPIS_SHA = "13ac2436c5e3d568bd0e938f6ed58b77a48aba15" +GOOGLEAPIS_SHA256 = "f48956fb8c55617ed052c20884465f06b9a57b807164431185be397ea46993ca" def googleapis_repositories(bind=True): GOOGLEAPIS_BUILD_FILE = """ @@ -37,11 +40,12 @@ cc_proto_library( ) """ - new_git_repository( + http_archive( name = "com_github_googleapis_googleapis", build_file_content = GOOGLEAPIS_BUILD_FILE, - commit = "13ac2436c5e3d568bd0e938f6ed58b77a48aba15", # Oct 21, 2016 (only release pre-dates sha) - remote = "https://github.com/googleapis/googleapis.git", + strip_prefix = "googleapis-" + GOOGLEAPIS_SHA, + url = "https://github.com/googleapis/googleapis/archive/" + GOOGLEAPIS_SHA + ".tar.gz", + sha256 = GOOGLEAPIS_SHA256, ) if bind: diff --git a/protobuf.bzl b/protobuf.bzl index 860a3b42e09..b556cdbfb91 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -14,14 +14,19 @@ # ################################################################################ # -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Match SHA used by Envoy +PROTOBUF_SHA = "fa252ec2a54acb24ddc87d48fed1ecfd458445fd" +PROTOBUF_SHA256 = "3d610ac90f8fa16e12490088605c248b85fdaf23114ce4b3605cdf81f7823604" def protobuf_repositories(load_repo=True, bind=True): if load_repo: - git_repository( + http_archive( name = "com_google_protobuf", - commit = "fa252ec2a54acb24ddc87d48fed1ecfd458445fd", # Match SHA used by Envoy - remote = "https://github.com/google/protobuf.git", + strip_prefix = "protobuf-" + PROTOBUF_SHA, + url = "https://github.com/google/protobuf/archive/" + PROTOBUF_SHA + ".tar.gz", + sha256 = PROTOBUF_SHA256, ) if bind: diff --git a/repositories.bzl b/repositories.bzl index c6f017dbec4..8f125128009 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -14,13 +14,17 @@ # ################################################################################ # -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +BORINGSSL = "12c35d69008ae6b8486e435447445240509f7662" +BORINGSSL_SHA256 = "fc668864f0b4ec825e889c048b7debda7f2327468672cdb5ac4dc5f5f0ee365a" def boringssl_repositories(bind=True): - git_repository( + http_archive( name = "boringssl", - commit = "12c35d69008ae6b8486e435447445240509f7662", # 2016-10-24 - remote = "https://boringssl.googlesource.com/boringssl", + strip_prefix = "boringssl-" + BORINGSSL, + url = "https://github.com/google/boringssl/archive/" + BORINGSSL + ".tar.gz", + sha256 = BORINGSSL_SHA256, ) if bind: @@ -34,6 +38,9 @@ def boringssl_repositories(bind=True): actual = "@boringssl//:ssl", ) +GOOGLETEST = "d225acc90bc3a8c420a9bcd1f033033c1ccd7fe0" +GOOGLETEST_SHA256 = "01508c8f47c99509130f128924f07f3a60be05d039cff571bb11d60bb11a3581" + def googletest_repositories(bind=True): BUILD = """ # Copyright 2017 Istio Authors. All Rights Reserved. @@ -90,11 +97,12 @@ cc_library( visibility = ["//visibility:public"], ) """ - native.new_git_repository( + http_archive( name = "googletest_git", build_file_content = BUILD, - commit = "d225acc90bc3a8c420a9bcd1f033033c1ccd7fe0", - remote = "https://github.com/google/googletest.git", + strip_prefix = "googletest-" + GOOGLETEST, + url = "https://github.com/google/googletest/archive/" + GOOGLETEST + ".tar.gz", + sha256 = GOOGLETEST_SHA256, ) if bind: @@ -114,6 +122,7 @@ cc_library( ) ISTIO_API = "6b9e3a501e6ef254958bf82f7b74c37d64a57a15" +ISTIO_API_SHA256 = "ce43fcc51bd7c653d39b810e50df68e32ed95991919c1d1f2f56b331d79e674e" def mixerapi_repositories(bind=True): BUILD = """ @@ -212,11 +221,12 @@ filegroup( ) """ - native.new_git_repository( + http_archive( name = "mixerapi_git", build_file_content = BUILD, - commit = ISTIO_API, - remote = "https://github.com/istio/api.git", + strip_prefix = "api-" + ISTIO_API, + url = "https://github.com/istio/api/archive/" + ISTIO_API + ".tar.gz", + sha256 = ISTIO_API_SHA256, ) if bind: native.bind( diff --git a/script/check-repositories b/script/check-repositories new file mode 100755 index 00000000000..e8c4cca0162 --- /dev/null +++ b/script/check-repositories @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright 2018 Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +set -eu + +# Check whether any git repositories are defined. +# Git repository definition contains `commit` and `remote` fields. +if grep -nr "commit =\|remote =" --include=WORKSPACE --include=*.bzl .; then + echo "Using git repositories is not allowed." + echo "To ensure that all dependencies can be stored offline in distdir, only HTTP repositories are allowed." + exit 1 +fi + +# Check whether number of defined `url =` and `sha256 =` kwargs in repository +# definitions is equal. +urls_count=$(grep -nr "url =" --include=WORKSPACE --include=*.bzl | wc -l) +sha256sums_count=$(grep -nr "sha256 =" --include=WORKSPACE --include=*.bzl | wc -l) + +if [[ $urls_count != $sha256sums_count ]]; then + echo "Found more defined repository URLs than SHA256 sums, which means that there are some repositories without sums." + echo "Dependencies without SHA256 sums cannot be stored in distdir." + echo "Please ensure that every repository has a SHA256 sum." + echo "Repositories are defined in the WORKSPACE file." + exit 1 +fi diff --git a/x_tools_imports.bzl b/x_tools_imports.bzl index 3f354a67b94..3f7ec45ac91 100644 --- a/x_tools_imports.bzl +++ b/x_tools_imports.bzl @@ -14,7 +14,11 @@ # ################################################################################ # -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Jun 23, 2017 (no releases) +TOOLS_SHA = "e6cb469339aef5b7be0c89de730d5f3cc8e47e50" +TOOLS_SHA256 = "fe9489eabcb598e13137d0641525ff3813d8af151e1418e6940e611850d90136" def go_x_tools_imports_repositories(): BUILD_FILE = """ @@ -46,9 +50,10 @@ go_binary( # simple build rule that will build the binary for usage (and avoid # the need to project a more complicated BUILD file over the entire # tools repo.) - new_git_repository( + http_archive( name = "org_golang_x_tools_imports", build_file_content = BUILD_FILE, - commit = "e6cb469339aef5b7be0c89de730d5f3cc8e47e50", # Jun 23, 2017 (no releases) - remote = "https://github.com/golang/tools.git", + strip_prefix = "tools-" + TOOLS_SHA, + url = "https://github.com/golang/tools/archives/" + TOOLS_SHA + ".tar.gz", + sha256 = TOOLS_SHA256, )