gh2gcs: Initial commit of utility for uploading GH releases to GCS

Signed-off-by: Stephen Augustus <[email protected]>

BUILD.bazel

@@ -29,6 +29,7 @@ filegroup(
         ":package-srcs",
         "//cmd/blocking-testgrid-tests:all-srcs",
         "//cmd/gcbuilder:all-srcs",
+        "//cmd/gh2gcs:all-srcs",
         "//cmd/krel:all-srcs",
         "//cmd/kubepkg:all-srcs",
         "//cmd/patch-announce:all-srcs",

cmd/gh2gcs/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "k8s.io/release/cmd/gh2gcs",
+    visibility = ["//visibility:private"],
+    deps = ["//cmd/gh2gcs/cmd:go_default_library"],
+)
+
+go_binary(
+    name = "kubepkg",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cmd/gh2gcs/cmd:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/gh2gcs/cmd/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["root.go"],
+    importpath = "k8s.io/release/cmd/gh2gcs/cmd",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/github:go_default_library",
+        "//pkg/log:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_spf13_cobra//:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/gh2gcs/cmd/root.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"k8s.io/release/pkg/github"
+	"k8s.io/release/pkg/log"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:               "debs [--arch <architectures>] [--channels <channels>]",
+	Short:             "debs creates Debian-based packages for Kubernetes components",
+	Example:           "kubepkg debs --arch amd64 --channels nightly",
+	SilenceUsage:      true,
+	SilenceErrors:     true,
+	PersistentPreRunE: initLogging,
+	RunE: func(*cobra.Command, []string) error {
+		return run(opts)
+	},
+}
+
+type options struct {
+	githubToken string
+	org         string
+	repo        string
+	tag         string
+	gcsEndpoint string
+	logLevel    string
+}
+
+var opts = &options{}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(
+		&opts.org,
+		"org",
+		// TODO: Remove test value
+		"containernetworking",
+		"org",
+	)
+
+	rootCmd.PersistentFlags().StringVar(
+		&opts.repo,
+		"repo",
+		// TODO: Remove test value
+		"plugins",
+		"repo",
+	)
+
+	rootCmd.PersistentFlags().StringVar(
+		&opts.tag,
+		"tag",
+		// TODO: Remove test value
+		"",
+		"tag",
+	)
+
+	rootCmd.PersistentFlags().StringVar(
+		&opts.gcsEndpoint,
+		"gcs-endpoint",
+		// TODO: Remove test value
+		"k8s-staging-release-test/augustus/cni",
+		"org",
+	)
+
+	rootCmd.PersistentFlags().StringVar(
+		&opts.logLevel,
+		"log-level",
+		"info",
+		"the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'",
+	)
+}
+
+func initLogging(*cobra.Command, []string) error {
+	return log.SetupGlobalLogger(opts.logLevel)
+}
+
+func run(opts *options) error {
+	// Create a real GitHub API client
+	gh := github.New()
+
+	if err := gh.DownloadReleaseAssets(opts.org, opts.repo, opts.tag); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Validate verifies if all set options are valid
+func (o *options) Validate() error {
+	// TODO: Add validation logic for options
+	return nil
+}

cmd/gh2gcs/main.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import "k8s.io/release/cmd/gh2gcs/cmd"
+
+func main() {
+	cmd.Execute()
+}

compile-release-tools

@@ -20,6 +20,7 @@ set -o pipefail
 
 RELEASE_TOOLS=(
   blocking-testgrid-tests
+  gh2gcs
   kubepkg
   krel
 )

pkg/github/github.go

@@ -19,8 +19,11 @@ package github
 import (
 	"context"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/go-github/v29/github"
@@ -74,6 +77,14 @@ type Client interface {
 		context.Context, string, string, *github.ListOptions,
 	) ([]*github.RepositoryRelease, *github.Response, error)
 
+	GetReleaseByTag(
+		context.Context, string, string, string,
+	) (*github.RepositoryRelease, *github.Response, error)
+
+	DownloadReleaseAsset(
+		context.Context, string, string, int64,
+	) (io.ReadCloser, string, error)
+
 	ListTags(
 		context.Context, string, string, *github.ListOptions,
 	) ([]*github.RepositoryTag, *github.Response, error)
@@ -190,6 +201,30 @@ func (g *githubClient) ListReleases(
 	}
 }
 
+func (g *githubClient) GetReleaseByTag(
+	ctx context.Context, owner, repo, tag string,
+) (*github.RepositoryRelease, *github.Response, error) {
+	for shouldRetry := internal.DefaultGithubErrChecker(); ; {
+		release, resp, err := g.Repositories.GetReleaseByTag(ctx, owner, repo, tag)
+		if !shouldRetry(err) {
+			return release, resp, err
+		}
+	}
+}
+
+func (g *githubClient) DownloadReleaseAsset(
+	ctx context.Context, owner, repo string, assetID int64,
+) (io.ReadCloser, string, error) {
+	// TODO: Should we be getting this http client from somewhere else?
+	httpClient := http.DefaultClient
+	for shouldRetry := internal.DefaultGithubErrChecker(); ; {
+		assetBody, redirectURL, err := g.Repositories.DownloadReleaseAsset(ctx, owner, repo, assetID, httpClient)
+		if !shouldRetry(err) {
+			return assetBody, redirectURL, err
+		}
+	}
+}
+
 func (g *githubClient) ListTags(
 	ctx context.Context, owner, repo string, opt *github.ListOptions,
 ) ([]*github.RepositoryTag, *github.Response, error) {
@@ -348,6 +383,66 @@ func (g *GitHub) Releases(owner, repo string, includePrereleases bool) ([]*githu
 	return releases, nil
 }
 
+// TODO: Support multiple tags
+func (g *GitHub) DownloadReleaseAssets(owner, repo, tag string) error {
+	var releases []*github.RepositoryRelease
+	var err error
+
+	if tag != "" {
+		release, _, err := g.client.GetReleaseByTag(context.Background(), owner, repo, tag)
+		if err != nil {
+			return err
+		}
+
+		releases = append(releases, release)
+	} else {
+		releases, err = g.Releases(owner, repo, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	// TODO: Move temp directory naming logic out of package
+	tmpDir, err := ioutil.TempDir("", "gh2gcs")
+	if err != nil {
+		return err
+	}
+
+	for _, release := range releases {
+		releaseTag := release.GetTagName()
+		logrus.Infof("Download assets for %s/%s@%s", owner, repo, releaseTag)
+
+		releaseDir := filepath.Join(tmpDir, releaseTag)
+		if err := os.MkdirAll(releaseDir, os.FileMode(0777)); err != nil {
+			return err
+		}
+
+		logrus.Infof("Writing assets to %s", releaseDir)
+
+		assets := release.Assets
+		for _, asset := range assets {
+			logrus.Infof("GitHub asset ID: %v, download URL: %s", *asset.ID, *asset.BrowserDownloadURL)
+			assetBody, _, err := g.client.DownloadReleaseAsset(context.Background(), owner, repo, *asset.ID)
+			if err != nil {
+				return err
+			}
+
+			filename := *asset.Name
+			absFile := filepath.Join(releaseDir, filename)
+			defer assetBody.Close()
+			assetFile, err := os.Create(absFile)
+			if err != nil {
+				return err
+			}
+
+			defer assetFile.Close()
+			io.Copy(assetFile, assetBody)
+		}
+	}
+
+	return nil
+}
+
 // CreatePullRequest Creates a new pull request in owner/repo:baseBranch to merge changes from headBranchName
 // which is a string containing a branch in the same repository or a user:branch pair
 func (g *GitHub) CreatePullRequest(

pkg/github/github_test.go

@@ -289,14 +289,14 @@ func TestCreatePullRequest(t *testing.T) {
 	require.Equal(t, fakeID, pr.GetID())
 }
 
-func TestGetRespository(t *testing.T) {
+func TestGetRepository(t *testing.T) {
 	// Given
 	sut, client := newSUT()
 	fakeRepositoryID := int64(54596517) // k/release
 	kubernetesUserID := int64(13629408)
 	kubernetesLogin := "kubernetes"
 	repoName := "release"
-	client.GetRespositoryReturns(&gogithub.Repository{
+	client.GetRepositoryReturns(&gogithub.Repository{
 		ID:   &fakeRepositoryID,
 		Name: &repoName,
 		Owner: &gogithub.User{
@@ -329,7 +329,7 @@ func TestRepoIsForkOf(t *testing.T) {
 
 	trueVal := true
 
-	client.GetRespositoryReturns(&gogithub.Repository{
+	client.GetRepositoryReturns(&gogithub.Repository{
 		Name: &repoName,
 		Fork: &trueVal,
 		Owner: &gogithub.User{
@@ -364,7 +364,7 @@ func TestRepoIsNotForkOf(t *testing.T) {
 
 	trueVal := true
 
-	client.GetRespositoryReturns(&gogithub.Repository{
+	client.GetRepositoryReturns(&gogithub.Repository{
 		Name: &repoName,
 		Fork: &trueVal,
 		Owner: &gogithub.User{