Merge pull request kubernetes#39898 from ixdy/bazel-release-tars

Automatic merge from submit-queue

Build release tars using bazel

**What this PR does / why we need it**: builds equivalents of the various kubernetes release tarballs, solely using bazel.

For example, you can now do
```console
$ make bazel-release
$ hack/e2e.go -v -up -test -down
```

**Special notes for your reviewer**: this is currently dependent on ixdy/bazel@3b29803, which I have yet to turn into a pull request, since I'm still trying to figure out if this is the best approach.

Basically, the issue comes up with the way we generate the various server docker image tarfiles and load them on nodes:
* we `md5sum` the binary being encapsulated (e.g. kube-proxy) and save that to `$binary.docker_tag` in the server tarball
* we then build the docker image and tag using that md5sum (e.g. `gcr.io/google_containers/kube-proxy:$MD5SUM`)
* we `docker save` this image, which embeds the full tag in the `$binary.tar` file.
* on cluster startup, we `docker load` these tarballs, which are loaded with the tag that we'd created at build time. the nodes then use the `$binary.docker_tag` file to find the right image.

With the current bazel `docker_build` rule, the tag isn't saved in the docker image tar, so the node is unable to find the image after `docker load`ing it.

My changes to the rule save the tag in the docker image tar, though I don't know if there are subtle issues with it. (Maybe we want to only tag when `--stamp` is given?)

Also, the docker images produced by bazel have the timestamp set to the unix epoch, which is not great for debugging. Might be another thing to change with a `--stamp`.

Long story short, we probably need to follow up with bazel folks on the best way to solve this problem.

**Release note**:

```release-note
NONE
```

BUILD.bazel

@@ -12,6 +12,7 @@ gcs_upload(
     name = "ci-artifacts",
     data = [
         "//build/debs",
+        "//build/release-tars",
     ],
 )
 
@@ -51,13 +52,3 @@ filegroup(
     ],
     tags = ["automanaged"],
 )
-
-pkg_tar(
-    name = "kubernetes-src",
-    extension = "tar.gz",
-    files = [
-        ":all-srcs",
-    ],
-    package_dir = "kubernetes",
-    strip_prefix = "//",
-)

Makefile

@@ -470,7 +470,7 @@ help:
 endif
 
 # Non-dockerized bazel rules.
-.PHONY: bazel-build bazel-test
+.PHONY: bazel-build bazel-test bazel-release
 
 ifeq ($(PRINT_HELP),y)
 define BAZEL_BUILD_HELP_INFO
@@ -500,3 +500,17 @@ else
 bazel-test:
 	bazel test  --test_output=errors //cmd/... //pkg/... //federation/... //plugin/... //build/... //third_party/... //hack/...
 endif
+
+ifeq ($(PRINT_HELP),y)
+define BAZEL_BUILD_HELP_INFO
+# Build release tars with bazel
+#
+# Example:
+# make bazel-release
+endef
+bazel-release:
+	@echo "$$BAZEL_BUILD_HELP_INFO"
+else
+bazel-release:
+	bazel build //build/release-tars
+endif

WORKSPACE

@@ -10,6 +10,12 @@ git_repository(
     remote = "https://github.com/kubernetes/release.git",
 )
 
+git_repository(
+    name = "io_bazel",
+    commit = "3b29803eb528ff525c7024190ffbf4b08c598cf2",
+    remote = "https://github.com/ixdy/bazel.git",
+)
+
 load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
 
 go_repositories()

build/BUILD

@@ -1,6 +1,22 @@
 package(default_visibility = ["//visibility:public"])
 
-load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
+load("@io_bazel//tools/build_defs/docker:docker.bzl", "docker_build")
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//build/debs:all-srcs",
+        "//build/release-tars:all-srcs",
+    ],
+    tags = ["automanaged"],
+)
 
 docker_build(
     name = "busybox",
@@ -34,43 +50,59 @@ docker_build(
     ],
 )
 
+DOCKERIZED_BINARIES = {
+    "kube-apiserver": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-apiserver:kube-apiserver",
+    },
+    "kube-controller-manager": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-controller-manager:kube-controller-manager",
+    },
+    "kube-scheduler": {
+        "base": ":busybox-libc",
+        "target": "//plugin/cmd/kube-scheduler:kube-scheduler",
+    },
+    "kube-aggregator": {
+        "base": ":busybox-libc",
+        "target": "//cmd/kube-aggregator:kube-aggregator",
+    },
+    "kube-proxy": {
+        "base": ":busybox-net",
+        "target": "//cmd/kube-proxy:kube-proxy",
+    },
+}
+
+[genrule(
+    name = binary + "_docker_tag",
+    srcs = [meta["target"]],
+    outs = [binary + ".docker_tag"],
+    # Currently each target has two outputs, the binary and its library, so hash only the first item (the binary).
+    # This can be made less hacky when we have static linking working.
+    cmd = "md5sum $(locations " + meta["target"] + ") | grep '" + binary + "'$$ | cut -f1 -d' ' | tr -d '\n' > $@",
+) for binary, meta in DOCKERIZED_BINARIES.items()]
+
 [docker_build(
     name = binary,
-    base = ":busybox-libc",
+    base = meta["base"],
     cmd = ["/usr/bin/" + binary],
     debs = [
         "//build/debs:%s.deb" % binary,
     ],
-    repository = "gcr.io/google-containers",
-) for binary in [
-    "kube-apiserver",
-    "kube-controller-manager",
-    "kube-scheduler",
-    "kube-aggregator",
-]]
-
-docker_build(
-    name = "kube-proxy",
-    base = ":busybox-net",
-    cmd = ["/usr/bin/kube-proxy"],
-    debs = [
-        "//build/debs:kube-proxy.deb",
+    image_tags = [
+        "@%s.docker_tag" % binary,
     ],
-    repository = "gcr.io/google-containers",
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
+    repository = "gcr.io/google_containers/" + binary,
+    repository_append_package = False,
+    symlinks = {
+        # Some cluster startup scripts expect to find the binaries in /usr/local/bin,
+        # but the debs install the binaries into /usr/bin.
+        "/usr/local/bin/" + binary: "/usr/bin/" + binary,
+    },
+) for binary, meta in DOCKERIZED_BINARIES.items()]
 
 filegroup(
-    name = "all-srcs",
-    srcs = [
-        ":package-srcs",
-        "//build/debs:all-srcs",
-    ],
-    tags = ["automanaged"],
+    name = "docker-artifacts",
+    srcs = [":%s.tar" % binary for binary in DOCKERIZED_BINARIES.keys()] +
+           [":%s.docker_tag" % binary for binary in DOCKERIZED_BINARIES.keys()],
 )

build/lib/release.sh

@@ -356,34 +356,37 @@ function kube::release::package_salt_tarball() {
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
+  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
+
   local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
   rm -rf "${release_stage}"
-  local dst_dir="${release_stage}/gci-trusty"
-  mkdir -p "${dst_dir}"
 
-  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
-  cp "${salt_dir}/cluster-autoscaler/cluster-autoscaler.manifest" "${dst_dir}/"
+  mkdir -p "${release_stage}"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
-  cp "${salt_dir}/etcd/etcd.manifest" "${dst_dir}"
-  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${dst_dir}"
-  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}"
-  cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}"
-  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
-  cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}"
-  cp "${salt_dir}/l7-gcp/glbc.manifest" "${dst_dir}"
-  cp "${salt_dir}/rescheduler/rescheduler.manifest" "${dst_dir}/"
-  cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/"
-  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
-  cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
-  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
-  cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
-  cp "${KUBE_ROOT}/cluster/gce/container-linux/configure-helper.sh" "${dst_dir}/container-linux-configure-helper.sh"
-  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
+
+  local gci_dst_dir="${release_stage}/gci-trusty"
+  mkdir -p "${gci_dst_dir}"
+  cp "${salt_dir}/cluster-autoscaler/cluster-autoscaler.manifest" "${gci_dst_dir}/"
+  cp "${salt_dir}/etcd/etcd.manifest" "${gci_dst_dir}"
+  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${gci_dst_dir}"
+  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${gci_dst_dir}"
+  cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${gci_dst_dir}"
+  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${gci_dst_dir}"
+  cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${gci_dst_dir}"
+  cp "${salt_dir}/l7-gcp/glbc.manifest" "${gci_dst_dir}"
+  cp "${salt_dir}/rescheduler/rescheduler.manifest" "${gci_dst_dir}/"
+  cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${gci_dst_dir}/"
+  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${gci_dst_dir}/trusty-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${gci_dst_dir}/gci-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${gci_dst_dir}/gci-mounter"
+  cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${gci_dst_dir}/health-monitor.sh"
+  cp "${KUBE_ROOT}/cluster/gce/container-linux/configure-helper.sh" "${gci_dst_dir}/container-linux-configure-helper.sh"
+  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${gci_dst_dir}"
   local objects
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
-  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
+  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${gci_dst_dir}"
 
   kube::release::clean_cruft