From 39670fc06f5b59f5902ca3eee3bd2fd30aa7cf1d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 17:07:49 -0800 Subject: [PATCH 1/7] Add download package & support URL fallback --- cmd/minikube/cmd/start.go | 22 ++- go.mod | 3 - go.sum | 6 - pkg/minikube/bootstrapper/bsutil/binaries.go | 3 +- pkg/minikube/cluster/iso.go | 30 ---- pkg/minikube/config/types.go | 14 +- pkg/minikube/constants/constants.go | 6 - pkg/minikube/driver/install.go | 30 +--- pkg/minikube/machine/cache_binaries.go | 77 +-------- pkg/minikube/machine/cache_binaries_test.go | 150 ------------------ pkg/minikube/machine/cluster_test.go | 14 +- pkg/minikube/node/cache.go | 3 +- .../registry/drvs/hyperkit/hyperkit.go | 3 +- pkg/minikube/registry/drvs/hyperv/hyperv.go | 2 +- pkg/minikube/registry/drvs/kvm2/kvm2.go | 3 +- .../registry/drvs/parallels/parallels.go | 3 +- .../registry/drvs/virtualbox/virtualbox.go | 3 +- pkg/minikube/registry/drvs/vmware/vmware.go | 3 +- .../drvs/vmwarefusion/vmwarefusion.go | 3 +- pkg/util/downloader.go | 131 --------------- pkg/util/downloader_test.go | 117 -------------- pkg/util/progressbar.go | 74 --------- pkg/version/version.go | 4 - .../en/docs/Contributing/drivers.en.md | 2 +- 24 files changed, 42 insertions(+), 664 deletions(-) delete mode 100644 pkg/minikube/cluster/iso.go delete mode 100644 pkg/util/downloader.go delete mode 100644 pkg/util/downloader_test.go delete mode 100644 pkg/util/progressbar.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index c05d715ec4f7..e29b5f041835 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -43,10 +43,10 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/kubeconfig" @@ -154,7 +154,7 @@ func initMinikubeFlags() { startCmd.Flags().String(humanReadableDiskSize, defaultDiskSize, "Disk size allocated to the minikube VM (format: [], where unit = b, k, m or g).") startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.") startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.") - startCmd.Flags().String(isoURL, constants.DefaultISOURL, "Location of the minikube iso.") + startCmd.Flags().StringSlice(isoURL, download.DefaultISOURLs(), "Locations to fetch the minikube ISO from.") startCmd.Flags().Bool(keepContext, false, "This will keep the existing kubectl context and will create a minikube context.") startCmd.Flags().Bool(embedCerts, false, "if true, will embed the certs in kubeconfig.") startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).") @@ -320,7 +320,13 @@ func runStart(cmd *cobra.Command, args []string) { return } - cacheISO(&mc, driverName) + if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { + url, err := download.ISO(viper.GetStringSlice(isoURL)) + if err != nil { + exit.WithError("Failed to cache ISO", err) + } + mc.MinikubeISO = url + } if viper.GetBool(nativeSSH) { ssh.SetDefaultClient(ssh.Native) @@ -354,14 +360,6 @@ func updateDriver(driverName string) { } } -func cacheISO(cfg *config.MachineConfig, driverName string) { - if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) { - if err := cluster.CacheISO(*cfg); err != nil { - exit.WithError("Failed to cache ISO", err) - } - } -} - func displayVersion(version string) { prefix := "" if viper.GetString(config.MachineProfile) != constants.DefaultMachineName { @@ -782,7 +780,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) Name: viper.GetString(config.MachineProfile), KeepContext: viper.GetBool(keepContext), EmbedCerts: viper.GetBool(embedCerts), - MinikubeISO: viper.GetString(isoURL), Memory: pkgutil.CalculateSizeInMB(viper.GetString(memory)), CPUs: viper.GetInt(cpus), DiskSize: pkgutil.CalculateSizeInMB(viper.GetString(humanReadableDiskSize)), @@ -803,7 +800,6 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) KVMQemuURI: viper.GetString(kvmQemuURI), KVMGPU: viper.GetBool(kvmGPU), KVMHidden: viper.GetBool(kvmHidden), - Downloader: pkgutil.DefaultDownloader{}, DisableDriverMounts: viper.GetBool(disableDriverMounts), UUID: viper.GetString(uuid), NoVTXCheck: viper.GetBool(noVTXCheck), diff --git a/go.mod b/go.mod index e4798d896c40..3618988a2a4f 100644 --- a/go.mod +++ b/go.mod @@ -23,15 +23,12 @@ require ( github.com/google/go-cmp v0.3.0 github.com/google/go-containerregistry v0.0.0-20200131185320-aec8da010de2 github.com/googleapis/gnostic v0.3.0 // indirect - github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect github.com/hashicorp/go-getter v1.4.0 - github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect github.com/hashicorp/go-retryablehttp v0.5.4 github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8 github.com/imdario/mergo v0.3.8 // indirect github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect - github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect diff --git a/go.sum b/go.sum index a5032159c7db..42c9980d01c4 100644 --- a/go.sum +++ b/go.sum @@ -346,12 +346,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 h1:VIq8E7fMiC4h3agg0ya56L0jHn7QisZZcWZXVKJb9jQ= -github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -386,8 +382,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 h1:XboatR7lasl05yel5hNXF7kQBw2oFUGdMztcgisfhNU= github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6/go.mod h1:RmeVYf9XrPRbRc3XIx0gLYA8qOFvNoPOfaEZduRlEp4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b h1:3TknJxYSK1eDe21QorC3C2Yz8jylk6vlJG9YABnFzkU= -github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b/go.mod h1:I3WsAhNNoG7a/d8HMlYUywJJlfOs/+/83NEUjuDp4lc= github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index 532be8a7ad75..6778599db732 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -27,6 +27,7 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -43,7 +44,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { for _, name := range constants.KubernetesReleaseBinaries { name := name g.Go(func() error { - src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) + src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) if err != nil { return errors.Wrapf(err, "downloading %s", name) } diff --git a/pkg/minikube/cluster/iso.go b/pkg/minikube/cluster/iso.go deleted file mode 100644 index 253acbd37081..000000000000 --- a/pkg/minikube/cluster/iso.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cluster - -import ( - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/driver" -) - -// CacheISO downloads and caches ISO. -func CacheISO(cfg config.MachineConfig) error { - if driver.BareMetal(cfg.Driver) { - return nil - } - return cfg.Downloader.CacheMinikubeISOFromURL(cfg.MinikubeISO) -} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 6513efd2d99c..b953c514c19d 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -20,7 +20,6 @@ import ( "net" "github.com/blang/semver" - "k8s.io/minikube/pkg/util" ) // Profile represents a minikube profile @@ -49,13 +48,12 @@ type MachineConfig struct { HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string - KVMNetwork string // Only used by the KVM driver - KVMQemuURI string // Only used by kvm2 - KVMGPU bool // Only used by kvm2 - KVMHidden bool // Only used by kvm2 - Downloader util.ISODownloader `json:"-"` - DockerOpt []string // Each entry is formatted as KEY=VALUE. - DisableDriverMounts bool // Only used by virtualbox + KVMNetwork string // Only used by the KVM driver + KVMQemuURI string // Only used by kvm2 + KVMGPU bool // Only used by kvm2 + KVMHidden bool // Only used by kvm2 + DockerOpt []string // Each entry is formatted as KEY=VALUE. + DisableDriverMounts bool // Only used by virtualbox NFSShare []string NFSSharesRoot string UUID string // Only used by hyperkit to restore the mac address diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index dbf74c4747bd..0bc95661f9d5 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -17,13 +17,11 @@ limitations under the License. package constants import ( - "fmt" "path/filepath" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" "k8s.io/minikube/pkg/minikube/localpath" - minikubeVersion "k8s.io/minikube/pkg/version" ) const ( @@ -78,10 +76,6 @@ var ( var ( SHASuffix = ".sha256" - // DefaultISOURL is the default location of the minikube.iso file - DefaultISOURL = fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s.iso", minikubeVersion.GetISOPath(), minikubeVersion.GetISOVersion()) - // DefaultISOSHAURL is the default location of the minikube.iso.sha256 file - DefaultISOSHAURL = DefaultISOURL + SHASuffix // DockerDaemonEnvs is list of docker-daemon related environment variables. DockerDaemonEnvs = [3]string{DockerHostEnv, DockerTLSVerifyEnv, DockerCertPathEnv} diff --git a/pkg/minikube/driver/install.go b/pkg/minikube/driver/install.go index 8a49e833faed..ae17a917cdba 100644 --- a/pkg/minikube/driver/install.go +++ b/pkg/minikube/driver/install.go @@ -27,12 +27,11 @@ import ( "github.com/blang/semver" "github.com/golang/glog" - "github.com/hashicorp/go-getter" "github.com/juju/mutex" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/lock" ) @@ -59,7 +58,7 @@ func InstallOrUpdate(name string, directory string, v semver.Version, interactiv if !exists || (err != nil && autoUpdate) { glog.Warningf("%s: %v", executable, err) path = filepath.Join(directory, executable) - derr := download(executable, path, v) + derr := download.Driver(executable, path, v) if derr != nil { return derr } @@ -139,31 +138,6 @@ func validateDriver(executable string, v semver.Version) (string, error) { return path, nil } -func driverWithChecksumURL(name string, v semver.Version) string { - base := fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/v%s/%s", v, name) - return fmt.Sprintf("%s?checksum=file:%s.sha256", base, base) -} - -// download an arbitrary driver -func download(name string, destination string, v semver.Version) error { - out.T(out.FileDownload, "Downloading driver {{.driver}}:", out.V{"driver": name}) - os.Remove(destination) - url := driverWithChecksumURL(name, v) - client := &getter.Client{ - Src: url, - Dst: destination, - Mode: getter.ClientModeFile, - Options: []getter.ClientOption{getter.WithProgress(util.DefaultProgressBar)}, - } - - glog.Infof("Downloading: %+v", client) - if err := client.Get(); err != nil { - return errors.Wrapf(err, "download failed: %s", url) - } - // Give downloaded drivers a baseline decent file permission - return os.Chmod(destination, 0755) -} - // extractDriverVersion extracts the driver version. // KVM and Hyperkit drivers support the 'version' command, that display the information as: // version: vX.X.X diff --git a/pkg/minikube/machine/cache_binaries.go b/pkg/minikube/machine/cache_binaries.go index 898540eab388..f702ee950cc9 100644 --- a/pkg/minikube/machine/cache_binaries.go +++ b/pkg/minikube/machine/cache_binaries.go @@ -17,22 +17,15 @@ limitations under the License. package machine import ( - "crypto" - "fmt" - "os" "path" "runtime" - "github.com/blang/semver" - "github.com/golang/glog" - "github.com/jimmidyson/go-download" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/download" ) // CacheBinariesForBootstrapper will cache binaries for a bootstrapper @@ -43,7 +36,7 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er for _, bin := range binaries { bin := bin // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { - if _, err := CacheBinary(bin, version, "linux", runtime.GOARCH); err != nil { + if _, err := download.Binary(bin, version, "linux", runtime.GOARCH); err != nil { return errors.Wrapf(err, "caching binary %s", bin) } return nil @@ -52,72 +45,6 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er return g.Wait() } -// releaseURL gets the location of a Kubernetes binary -func releaseURL(binaryName, version, osName, archName string) string { - return fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/%s/%s/%s", version, osName, archName, binaryName) -} - -// downloadOptions returns appropriate download options for a -func downloadOptions(url string, version string) (download.FileOptions, error) { - fo := download.FileOptions{ - Mkdirs: download.MkdirAll, - Options: download.Options{ - ChecksumHash: crypto.SHA1, - Checksum: url + ".sha1", - }, - } - - v, err := semver.Make(version[1:]) - if err != nil { - return fo, err - } - - if v.GTE(semver.MustParse("1.17.0")) { - fo.ChecksumHash = crypto.SHA256 - fo.Checksum = url + ".sha256" - } - return fo, nil -} - -// CacheBinary will cache a binary on the host -func CacheBinary(binary, version, osName, archName string) (string, error) { - - targetDir := localpath.MakeMiniPath("cache", osName, version) - targetFilepath := path.Join(targetDir, binary) - - url := releaseURL(binary, version, osName, archName) - - _, err := os.Stat(targetFilepath) - // If it exists, do no verification and continue - if err == nil { - glog.Infof("Not caching binary, using %s", url) - return targetFilepath, nil - } - if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat %s version %s at %s", binary, version, targetDir) - } - - if err = os.MkdirAll(targetDir, 0777); err != nil { - return "", errors.Wrapf(err, "mkdir %s", targetDir) - } - - options, err := downloadOptions(url, version) - if err != nil { - return "", errors.Wrap(err, "options") - } - glog.Infof("Downloading %s: options: %+v", url, options) - - out.T(out.FileDownload, "Downloading {{.name}} {{.version}}", out.V{"name": binary, "version": version}) - if err := download.ToFile(url, targetFilepath, options); err != nil { - return "", errors.Wrapf(err, url) - } - if osName == runtime.GOOS && archName == runtime.GOARCH { - if err = os.Chmod(targetFilepath, 0755); err != nil { - return "", errors.Wrapf(err, "chmod +x %s", targetFilepath) - } - } - return targetFilepath, nil -} // CopyBinary copies a locally cached binary to the guest VM func CopyBinary(cr command.Runner, src string, dest string) error { diff --git a/pkg/minikube/machine/cache_binaries_test.go b/pkg/minikube/machine/cache_binaries_test.go index b490974eac1e..500c0ea74daf 100644 --- a/pkg/minikube/machine/cache_binaries_test.go +++ b/pkg/minikube/machine/cache_binaries_test.go @@ -17,15 +17,11 @@ limitations under the License. package machine import ( - "crypto" "fmt" "io/ioutil" "os" - "runtime" "testing" - "github.com/google/go-cmp/cmp" - "github.com/jimmidyson/go-download" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" "k8s.io/minikube/pkg/minikube/command" @@ -126,149 +122,3 @@ func TestCacheBinariesForBootstrapper(t *testing.T) { }) } } -func TestCacheBinary(t *testing.T) { - oldMinikubeHome := os.Getenv("MINIKUBE_HOME") - defer os.Setenv("MINIKUBE_HOME", oldMinikubeHome) - - minikubeHome, err := ioutil.TempDir("/tmp", "") - if err != nil { - t.Fatalf("error during creating tmp dir: %v", err) - } - defer os.RemoveAll(minikubeHome) - noWritePermDir, err := ioutil.TempDir("/tmp", "") - if err != nil { - t.Fatalf("error during creating tmp dir: %v", err) - } - defer os.RemoveAll(noWritePermDir) - err = os.Chmod(noWritePermDir, 0000) - if err != nil { - t.Fatalf("error (%v) during changing permissions of dir %v", err, noWritePermDir) - } - - var tc = []struct { - desc, version, osName, archName string - minikubeHome, binary, description string - err bool - }{ - { - desc: "ok kubeadm", - version: "v1.16.0", - osName: "linux", - archName: runtime.GOARCH, - binary: "kubeadm", - err: false, - minikubeHome: minikubeHome, - }, - { - desc: "minikube home in dir without perms and arm runtime", - version: "v1.16.0", - osName: runtime.GOOS, - archName: "arm", - binary: "kubectl", - err: true, - minikubeHome: noWritePermDir, - }, - { - desc: "minikube home in dir without perms", - version: "v1.16.0", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "kubectl", - err: true, - minikubeHome: noWritePermDir, - }, - { - desc: "binary foo", - version: "v1.16.0", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "foo", - err: true, - minikubeHome: minikubeHome, - }, - { - desc: "version 9000", - version: "v9000", - osName: runtime.GOOS, - archName: runtime.GOARCH, - binary: "foo", - err: true, - minikubeHome: minikubeHome, - }, - { - desc: "bad os", - version: "v1.16.0", - osName: "no-such-os", - archName: runtime.GOARCH, - binary: "kubectl", - err: true, - minikubeHome: minikubeHome, - }, - } - for _, test := range tc { - t.Run(test.desc, func(t *testing.T) { - os.Setenv("MINIKUBE_HOME", test.minikubeHome) - _, err := CacheBinary(test.binary, test.version, test.osName, test.archName) - if err != nil && !test.err { - t.Fatalf("Got unexpected error %v", err) - } - if err == nil && test.err { - t.Fatalf("Expected error but got %v", err) - } - }) - } -} - -func TestDownloadOptions(t *testing.T) { - var tc = []struct { - url string - version string - want download.FileOptions - }{ - { - url: "https://s/kubernetes-release/release/v1.16.0/bin/amd64/kubectl", - version: "v1.16.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.16.0/bin/amd64/kubectl.sha1", - ChecksumHash: crypto.SHA1, - }, - Mkdirs: download.MkdirAll, - }, - }, - { - url: "https://s/kubernetes-release/release/v1.10.0/bin/hp9k/kubeadm", - version: "v1.10.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.10.0/bin/hp9k/kubeadm.sha1", - ChecksumHash: crypto.SHA1, - }, - Mkdirs: download.MkdirAll, - }, - }, - { - url: "https://s/kubernetes-release/release/v1.18.0/bin/arm64/kubelet", - version: "v1.18.0", - want: download.FileOptions{ - Options: download.Options{ - Checksum: "https://s/kubernetes-release/release/v1.18.0/bin/arm64/kubelet.sha256", - ChecksumHash: crypto.SHA256, - }, - Mkdirs: download.MkdirAll, - }, - }, - } - for _, test := range tc { - t.Run(test.version, func(t *testing.T) { - got, err := downloadOptions(test.url, test.version) - if err != nil { - t.Fatalf("unexpected error %v", err) - } - - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("unexpected options(-want +got):\n%s", diff) - } - }) - } -} diff --git a/pkg/minikube/machine/cluster_test.go b/pkg/minikube/machine/cluster_test.go index bf38656062a3..84c23ae889c0 100644 --- a/pkg/minikube/machine/cluster_test.go +++ b/pkg/minikube/machine/cluster_test.go @@ -30,7 +30,6 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/registry" "k8s.io/minikube/pkg/minikube/tests" @@ -61,10 +60,8 @@ func RegisterMockDriver(t *testing.T) { } var defaultMachineConfig = config.MachineConfig{ - Driver: driver.Mock, - MinikubeISO: constants.DefaultISOURL, - Downloader: MockDownloader{}, - DockerEnv: []string{"MOCK_MAKE_IT_PROVISION=true"}, + Driver: driver.Mock, + DockerEnv: []string{"MOCK_MAKE_IT_PROVISION=true"}, } func TestCreateHost(t *testing.T) { @@ -262,10 +259,9 @@ func TestStartHostConfig(t *testing.T) { provision.SetDetector(md) config := config.MachineConfig{ - Driver: driver.Mock, - DockerEnv: []string{"FOO=BAR"}, - DockerOpt: []string{"param=value"}, - Downloader: MockDownloader{}, + Driver: driver.Mock, + DockerEnv: []string{"FOO=BAR"}, + DockerOpt: []string{"param=value"}, } h, err := StartHost(api, config) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index b1d5ad8d128e..e030b9942ec4 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -26,6 +26,7 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/image" "k8s.io/minikube/pkg/minikube/localpath" @@ -71,7 +72,7 @@ func CacheKubectlBinary(k8sVerison string) (string, error) { binary = "kubectl.exe" } - return machine.CacheBinary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH) + return download.Binary(binary, k8sVerison, runtime.GOOS, runtime.GOARCH) } // doCacheBinaries caches Kubernetes binaries in the foreground diff --git a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go index 50a4e5a40817..edde1da359a7 100644 --- a/pkg/minikube/registry/drvs/hyperkit/hyperkit.go +++ b/pkg/minikube/registry/drvs/hyperkit/hyperkit.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/drivers/hyperkit" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -69,7 +70,7 @@ func configure(config cfg.MachineConfig) (interface{}, error) { StorePath: localpath.MiniPath(), SSHUser: "docker", }, - Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO), + Boot2DockerURL: download.LocalISOResource(config.MinikubeISO), DiskSize: config.DiskSize, Memory: config.Memory, CPU: config.CPUs, diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 9f15d0c47007..db2f3c014486 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -54,7 +54,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := hyperv.NewDriver(config.Name, localpath.MiniPath()) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.VSwitch = config.HypervVirtualSwitch if d.VSwitch == "" && config.HypervUseExternalSwitch { switchName, adapter, err := chooseSwitch(config.HypervExternalAdapter) diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index dedad73bfb80..0ae7d89754f7 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -30,6 +30,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -79,7 +80,7 @@ func configure(mc config.MachineConfig) (interface{}, error) { CPU: mc.CPUs, Network: mc.KVMNetwork, PrivateNetwork: "minikube-net", - Boot2DockerURL: mc.Downloader.GetISOFileURI(mc.MinikubeISO), + Boot2DockerURL: download.LocalISOResource(mc.MinikubeISO), DiskSize: mc.DiskSize, DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.rawdisk", name)), ISO: filepath.Join(localpath.MiniPath(), "machines", name, "boot2docker.iso"), diff --git a/pkg/minikube/registry/drvs/parallels/parallels.go b/pkg/minikube/registry/drvs/parallels/parallels.go index de319ec8fbac..d5a6ef71696f 100644 --- a/pkg/minikube/registry/drvs/parallels/parallels.go +++ b/pkg/minikube/registry/drvs/parallels/parallels.go @@ -25,6 +25,7 @@ import ( parallels "github.com/Parallels/docker-machine-parallels" "github.com/docker/machine/libmachine/drivers" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -46,7 +47,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := parallels.NewDriver(config.Name, localpath.MiniPath()).(*parallels.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize diff --git a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go index bfba0e42db89..9fb264b8f687 100644 --- a/pkg/minikube/registry/drvs/virtualbox/virtualbox.go +++ b/pkg/minikube/registry/drvs/virtualbox/virtualbox.go @@ -27,6 +27,7 @@ import ( "github.com/docker/machine/libmachine/drivers" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -51,7 +52,7 @@ func init() { func configure(mc config.MachineConfig) (interface{}, error) { d := virtualbox.NewDriver(mc.Name, localpath.MiniPath()) - d.Boot2DockerURL = mc.Downloader.GetISOFileURI(mc.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(mc.MinikubeISO) d.Memory = mc.Memory d.CPU = mc.CPUs d.DiskSize = mc.DiskSize diff --git a/pkg/minikube/registry/drvs/vmware/vmware.go b/pkg/minikube/registry/drvs/vmware/vmware.go index 885063cde28a..f432916211f1 100644 --- a/pkg/minikube/registry/drvs/vmware/vmware.go +++ b/pkg/minikube/registry/drvs/vmware/vmware.go @@ -22,6 +22,7 @@ import ( vmwcfg "github.com/machine-drivers/docker-machine-driver-vmware/pkg/drivers/vmware/config" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -41,7 +42,7 @@ func init() { func configure(mc config.MachineConfig) (interface{}, error) { d := vmwcfg.NewConfig(mc.Name, localpath.MiniPath()) - d.Boot2DockerURL = mc.Downloader.GetISOFileURI(mc.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(mc.MinikubeISO) d.Memory = mc.Memory d.CPU = mc.CPUs d.DiskSize = mc.DiskSize diff --git a/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go b/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go index bb5ed4196bca..5c371d0dadd7 100644 --- a/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go +++ b/pkg/minikube/registry/drvs/vmwarefusion/vmwarefusion.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" @@ -46,7 +47,7 @@ func init() { func configure(config cfg.MachineConfig) (interface{}, error) { d := vmwarefusion.NewDriver(config.Name, localpath.MiniPath()).(*vmwarefusion.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize diff --git a/pkg/util/downloader.go b/pkg/util/downloader.go deleted file mode 100644 index 72b478256589..000000000000 --- a/pkg/util/downloader.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "net/url" - "os" - "path/filepath" - "time" - - "github.com/golang/glog" - "github.com/hashicorp/go-getter" - "github.com/juju/mutex" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/util/lock" -) - -const fileScheme = "file" - -// ISODownloader downloads an ISO -type ISODownloader interface { - GetISOFileURI(isoURL string) string - CacheMinikubeISOFromURL(isoURL string) error -} - -// DefaultDownloader is the default ISODownloader -type DefaultDownloader struct{} - -// GetISOFileURI gets the local destination for a remote source -func (f DefaultDownloader) GetISOFileURI(isoURL string) string { - urlObj, err := url.Parse(isoURL) - if err != nil { - return isoURL - } - if urlObj.Scheme == fileScheme { - return isoURL - } - isoPath := filepath.Join(localpath.MiniPath(), "cache", "iso", filepath.Base(isoURL)) - // As this is a file URL there should be no backslashes regardless of platform running on. - return "file://" + filepath.ToSlash(isoPath) -} - -// CacheMinikubeISOFromURL downloads the ISO, if it doesn't exist in cache -func (f DefaultDownloader) CacheMinikubeISOFromURL(url string) error { - dst := f.GetISOCacheFilepath(url) - - // Lock before we check for existence to avoid thundering herd issues - spec := lock.PathMutexSpec(dst) - spec.Timeout = 10 * time.Minute - glog.Infof("acquiring lock: %+v", spec) - releaser, err := mutex.Acquire(spec) - if err != nil { - return errors.Wrapf(err, "unable to acquire lock for %+v", spec) - } - defer releaser.Release() - - if !f.ShouldCacheMinikubeISO(url) { - glog.Infof("Not caching ISO, using %s", url) - return nil - } - - urlWithChecksum := url - if url == constants.DefaultISOURL { - urlWithChecksum = url + "?checksum=file:" + constants.DefaultISOSHAURL - } - - // Predictable temp destination so that resume can function - tmpDst := dst + ".download" - - opts := []getter.ClientOption{getter.WithProgress(DefaultProgressBar)} - client := &getter.Client{ - Src: urlWithChecksum, - Dst: tmpDst, - Mode: getter.ClientModeFile, - Options: opts, - } - - glog.Infof("full url: %s", urlWithChecksum) - out.T(out.ISODownload, "Downloading VM boot image ...") - if err := client.Get(); err != nil { - return errors.Wrap(err, url) - } - return os.Rename(tmpDst, dst) -} - -// ShouldCacheMinikubeISO returns if we need to download the ISO -func (f DefaultDownloader) ShouldCacheMinikubeISO(isoURL string) bool { - // store the minikube-iso inside the .minikube dir - - urlObj, err := url.Parse(isoURL) - if err != nil { - return false - } - if urlObj.Scheme == fileScheme { - return false - } - if f.IsMinikubeISOCached(isoURL) { - return false - } - return true -} - -// GetISOCacheFilepath returns the path of an ISO in the local cache -func (f DefaultDownloader) GetISOCacheFilepath(isoURL string) string { - return filepath.Join(localpath.MiniPath(), "cache", "iso", filepath.Base(isoURL)) -} - -// IsMinikubeISOCached returns if an ISO exists in the local cache -func (f DefaultDownloader) IsMinikubeISOCached(isoURL string) bool { - if _, err := os.Stat(f.GetISOCacheFilepath(isoURL)); os.IsNotExist(err) { - return false - } - return true -} diff --git a/pkg/util/downloader_test.go b/pkg/util/downloader_test.go deleted file mode 100644 index 3db859329697..000000000000 --- a/pkg/util/downloader_test.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/tests" -) - -func TestGetISOFileURI(t *testing.T) { - dler := DefaultDownloader{} - - tests := map[string]string{ - "file:///test/path/minikube-test.iso": "file:///test/path/minikube-test.iso", - "https://storage.googleapis.com/minikube/iso/minikube-test.iso": "file://" + filepath.ToSlash(filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso")), - } - - for input, expected := range tests { - if isoFileURI := dler.GetISOFileURI(input); isoFileURI != expected { - t.Fatalf("Expected GetISOFileURI with input %s to return %s but instead got: %s", input, expected, isoFileURI) - } - } - -} - -var testISOString = "hello" - -func TestCacheMinikubeISOFromURL(t *testing.T) { - tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) - dler := DefaultDownloader{} - isoPath := filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso") - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, err := io.WriteString(w, testISOString); err != nil { - t.Fatalf("WriteString: %v", err) - } - })) - isoURL := server.URL + "/minikube-test.iso" - if err := dler.CacheMinikubeISOFromURL(isoURL); err != nil { - t.Fatalf("Unexpected error from CacheMinikubeISOFromURL: %v", err) - } - - transferred, err := ioutil.ReadFile(filepath.Join(isoPath)) - if err != nil { - t.Fatalf("File not copied. Could not open file at path: %s", isoPath) - } - - // test that the ISO is transferred properly - contents := []byte(testISOString) - if !bytes.Contains(transferred, contents) { - t.Fatalf("Expected transfers to contain: %s. It was: %s", contents, transferred) - } - -} - -func TestShouldCacheMinikubeISO(t *testing.T) { - dler := DefaultDownloader{} - - tests := map[string]bool{ - "file:///test/path/minikube-test.iso": false, - "https://storage.googleapis.com/minikube/iso/minikube-test.iso": true, - } - - for input, expected := range tests { - if out := dler.ShouldCacheMinikubeISO(input); out != expected { - t.Fatalf("Expected ShouldCacheMinikubeISO with input %s to return %t but instead got: %t", input, expected, out) - } - } -} - -func TestIsMinikubeISOCached(t *testing.T) { - tempDir := tests.MakeTempDir() - defer os.RemoveAll(tempDir) - - dler := DefaultDownloader{} - - testFileURI := "file:///test/path/minikube-test.iso" - expected := false - - if out := dler.IsMinikubeISOCached(testFileURI); out != expected { - t.Fatalf("Expected IsMinikubeISOCached with input %s to return %t but instead got: %t", testFileURI, expected, out) - } - - if err := ioutil.WriteFile(filepath.Join(localpath.MiniPath(), "cache", "iso", "minikube-test.iso"), []byte(testISOString), os.FileMode(int(0644))); err != nil { - t.Fatalf("WriteFile: %v", err) - } - - expected = true - if out := dler.IsMinikubeISOCached(testFileURI); out != expected { - t.Fatalf("Expected IsMinikubeISOCached with input %s to return %t but instead got: %t", testFileURI, expected, out) - } - -} diff --git a/pkg/util/progressbar.go b/pkg/util/progressbar.go deleted file mode 100644 index fb16f6773ce6..000000000000 --- a/pkg/util/progressbar.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file implements a go-getter wrapper for cheggaaa progress bar -// based on: -// https://github.com/hashicorp/go-getter/blob/master/cmd/go-getter/progress_tracking.go - -package util - -import ( - "io" - "path/filepath" - "sync" - - "github.com/cheggaaa/pb/v3" - "github.com/hashicorp/go-getter" -) - -// DefaultProgressBar is the default cheggaaa progress bar -var DefaultProgressBar getter.ProgressTracker = &progressBar{} - -type progressBar struct { - lock sync.Mutex - progress *pb.ProgressBar -} - -// TrackProgress instantiates a new progress bar that will -// display the progress of stream until closed. -// total can be 0. -func (cpb *progressBar) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser { - cpb.lock.Lock() - defer cpb.lock.Unlock() - if cpb.progress == nil { - cpb.progress = pb.New64(totalSize) - } - p := pb.Full.Start64(totalSize) - p.Set("prefix", " > "+filepath.Base(src+": ")) - p.SetCurrent(currentSize) - p.Set(pb.Bytes, true) - - // Just a hair less than 80 (standard terminal width) for aesthetics & pasting into docs - p.SetWidth(79) - barReader := p.NewProxyReader(stream) - - return &readCloser{ - Reader: barReader, - close: func() error { - cpb.lock.Lock() - defer cpb.lock.Unlock() - p.Finish() - return nil - }, - } -} - -type readCloser struct { - io.Reader - close func() error -} - -func (c *readCloser) Close() error { return c.close() } diff --git a/pkg/version/version.go b/pkg/version/version.go index 727035115b1b..6587f3f7223c 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -53,10 +53,6 @@ func GetISOVersion() string { return isoVersion } -// GetISOPath returns the remote path to the minikube.iso -func GetISOPath() string { - return isoPath -} // GetSemverVersion returns the current minikube semantic version (semver) func GetSemverVersion() (semver.Version, error) { diff --git a/site/content/en/docs/Contributing/drivers.en.md b/site/content/en/docs/Contributing/drivers.en.md index aecd0e1179b1..3641e6b76638 100644 --- a/site/content/en/docs/Contributing/drivers.en.md +++ b/site/content/en/docs/Contributing/drivers.en.md @@ -87,7 +87,7 @@ func init() { func createVMwareFusionHost(config cfg.MachineConfig) interface{} { d := vmwarefusion.NewDriver(config.Name, localpath.MiniPath()).(*vmwarefusion.Driver) - d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Boot2DockerURL = download.LocalISOResource(config.MinikubeISO) d.Memory = config.Memory d.CPU = config.CPUs d.DiskSize = config.DiskSize From 27cca764b57d89252f4a4184a3498845f4dc8cd6 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 18:57:12 -0800 Subject: [PATCH 2/7] Fix integration test --- test/integration/aaa_download_only_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index 08e42ed07bf6..2b9393c72224 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -181,15 +182,15 @@ func TestDownloadOnlyDocker(t *testing.T) { t.Errorf("%s failed: %v:\n%s", args, err, rr.Output()) } - // Make sure the preloaded image tarball exists - tarball := download.TarballFilepath(constants.DefaultKubernetesVersion) + // Make sure the downloaded image tarball exists + tarball := download.TarballPath(constants.DefaultKubernetesVersion) contents, err := ioutil.ReadFile(tarball) if err != nil { t.Errorf("reading tarball: %v", err) } // Make sure it has the correct checksum checksum := md5.Sum(contents) - remoteChecksum, err := ioutil.ReadFile(preload.ChecksumFilepath(constants.DefaultKubernetesVersion)) + remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion)) if err != nil { t.Errorf("reading checksum file: %v", err) } From fc7218424960bacb0d68615608621549026421ad Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:08:49 -0800 Subject: [PATCH 3/7] go.mod update --- go.mod | 6 ++++-- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f5a6c5170aed..83b96bb6d431 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/cheggaaa/pb/v3 v3.0.1 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect + github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect github.com/docker/docker v1.13.1 github.com/docker/go-units v0.4.0 github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5 @@ -63,7 +64,8 @@ require ( github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097 golang.org/x/build v0.0.0-20190927031335-2835ba2e683f - golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 + golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20191010194322-b09406accb47 golang.org/x/text v0.3.2 @@ -74,7 +76,7 @@ require ( k8s.io/client-go v0.17.3 k8s.io/kubectl v0.0.0 k8s.io/kubernetes v1.17.3 - k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect + k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab // indirect sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible ) diff --git a/go.sum b/go.sum index 1c873b120941..fa0a2efcd5c4 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304 h1:A7SYzidcyuQ/yS4wezWGYeUioUFJQk8HYWY9aMYTF4I= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 h1:Cvj7S8I4Xpx78KAl6TwTmMHuHlZ/0SM60NUneGJQ7IE= @@ -729,6 +731,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -774,6 +778,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= @@ -992,6 +998,8 @@ k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCui k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 h1:3tT5mBZNurtd5BoYrPBII3Sa8n7T2w405qdTQvr3vmY= k8s.io/utils v0.0.0-20200122174043-1e243dd1a584/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= +k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= From 832fb6b4ab56b09f099eaa61079763570f94f189 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:42:05 -0800 Subject: [PATCH 4/7] preload: check local cache and status codes --- pkg/minikube/download/preload.go | 26 ++++++++++++++++++++++++-- pkg/minikube/node/cache.go | 1 + pkg/minikube/node/start.go | 11 +++++++---- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 8161ec68204f..05c0b4c490db 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -77,9 +77,31 @@ func PreloadExists(k8sVersion, containerRuntime string) bool { if containerRuntime != "docker" { return false } + + // Omit remote check if tarball exists locally + targetPath := TarballPath(k8sVersion) + if _, err := os.Stat(targetPath); err == nil { + if err := verifyChecksum(k8sVersion); err == nil { + glog.Infof("Found %s in cache, no need to check remotely", targetPath) + return true + } + } + url := remoteTarballURL(k8sVersion) - _, err := http.Head(url) - return err == nil + resp, err := http.Head(url) + if err != nil { + glog.Warningf("%s fetch error: %v", url, err) + return false + } + + // note: err won't be set if it's a 404 + if resp.StatusCode != 200 { + glog.Warningf("%s status code: %d", url, resp.StatusCode) + return false + } + + glog.Infof("Goody! %s exists!", url) + return true } // Preload caches the preloaded images tarball on the host machine diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 4336049af8fa..15dc7e40186c 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -98,6 +98,7 @@ func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) { func waitDownloadKicArtifacts(g *errgroup.Group) { if err := g.Wait(); err != nil { glog.Errorln("Error downloading kic artifacts: ", err) + return } glog.Info("Successfully downloaded all kic artifacts") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 7c148243cc62..b1e72acd66b8 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -41,13 +41,16 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons // See if we can create a volume of preloaded images // If not, pull images in the background while the VM boots. var kicGroup errgroup.Group + downloadImages := true if driver.IsKIC(driverName) { - beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) + if download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) { + beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) + downloadImages = false + } } - // Now that the ISO is downloaded, pull images in the background while the VM boots. + var cacheGroup errgroup.Group - skipCacheImages := driver.IsKIC(driverName) && download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) - if !skipCacheImages { + if downloadImages { beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) } From 9adf54388ddccf73d75328dbfc54f6247866fa6e Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 20:49:49 -0800 Subject: [PATCH 5/7] lint cleanup --- pkg/minikube/machine/cache_binaries.go | 1 - pkg/version/version.go | 5 ----- 2 files changed, 6 deletions(-) diff --git a/pkg/minikube/machine/cache_binaries.go b/pkg/minikube/machine/cache_binaries.go index f702ee950cc9..3bb9d4d998da 100644 --- a/pkg/minikube/machine/cache_binaries.go +++ b/pkg/minikube/machine/cache_binaries.go @@ -45,7 +45,6 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string) er return g.Wait() } - // CopyBinary copies a locally cached binary to the guest VM func CopyBinary(cr command.Runner, src string, dest string) error { f, err := assets.NewFileAsset(src, path.Dir(dest), path.Base(dest), "0755") diff --git a/pkg/version/version.go b/pkg/version/version.go index 6587f3f7223c..a3b921c5caae 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -25,8 +25,6 @@ import ( // VersionPrefix is the prefix of the git tag for a version const VersionPrefix = "v" -// The current version of the minikube - // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.version=vX.Y.Z" var version = "v0.0.0-unset" @@ -36,8 +34,6 @@ var gitCommitID = "" // isoVersion is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.isoVersion=vX.Y.Z" var isoVersion = "v0.0.0-unset" -var isoPath = "minikube/iso" - // GetVersion returns the current minikube version func GetVersion() string { return version @@ -53,7 +49,6 @@ func GetISOVersion() string { return isoVersion } - // GetSemverVersion returns the current minikube semantic version (semver) func GetSemverVersion() (semver.Version, error) { return semver.Make(strings.TrimPrefix(GetVersion(), VersionPrefix)) From 883e108dce222e45e0a7a25f6895b6b5a6eaecfc Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 4 Mar 2020 21:05:38 -0800 Subject: [PATCH 6/7] Fix hyperv import --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index b452c63d4964..7b96d041d39c 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" cfg "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/registry" From 1a01b9450cc12dbe664d89f676a441c4762635df Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 5 Mar 2020 10:23:51 -0800 Subject: [PATCH 7/7] Clarify logic around beginDownloadKicArtifacts --- pkg/minikube/node/cache.go | 19 ++++++++++++------- pkg/minikube/node/start.go | 13 ++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 15dc7e40186c..67bf646ce747 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -35,8 +35,8 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) -// beginCacheRequiredImages caches images required for kubernetes version in the background -func beginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVersion string) { +// beginCacheKubernetesImages caches images required for kubernetes version in the background +func beginCacheKubernetesImages(g *errgroup.Group, imageRepository string, k8sVersion string) { if !viper.GetBool("cache-images") { return } @@ -82,17 +82,22 @@ func doCacheBinaries(k8sVersion string) error { return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper)) } -func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) { +// beginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available +func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) bool { glog.Info("Beginning downloading kic artifacts") g.Go(func() error { glog.Infof("Downloading %s to local daemon", kic.BaseImage) return image.WriteImageToDaemon(kic.BaseImage) }) - g.Go(func() error { - glog.Info("Caching tarball of preloaded images") - return download.Preload(k8sVersion, cRuntime) - }) + if download.PreloadExists(k8sVersion, cRuntime) { + g.Go(func() error { + glog.Info("Caching tarball of preloaded images") + return download.Preload(k8sVersion, cRuntime) + }) + return true + } + return false } func waitDownloadKicArtifacts(g *errgroup.Group) { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index f2a5507fc42f..44a1667c469f 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -23,7 +23,6 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/minikube/pkg/addons" "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/kubeconfig" @@ -41,17 +40,17 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons // See if we can create a volume of preloaded images // If not, pull images in the background while the VM boots. var kicGroup errgroup.Group - downloadImages := true + needKubernetesImages := true if driver.IsKIC(driverName) { - if download.PreloadExists(k8sVersion, mc.KubernetesConfig.ContainerRuntime) { - beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) - downloadImages = false + // If we can download a preload tarball, it isn't necessary to pull Kubernetes images + if beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) { + needKubernetesImages = false } } var cacheGroup errgroup.Group - if downloadImages { - beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) + if needKubernetesImages { + beginCacheKubernetesImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) } // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.