Mirror keystore & secretstore

This allows us to have our API objects in kops-server, but our
configuration on S3 or GCS.

cmd/kops/create.go

@@ -27,7 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kops/cmd/kops/util"
 	kopsapi "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/kops/v1alpha1"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/util/pkg/vfs"
@@ -208,7 +207,7 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
 					return err
 				}
 
-				keyStore, err := registry.KeyStore(cluster)
+				keyStore, err := clientset.KeyStore(cluster)
 				if err != nil {
 					return err
 				}

cmd/kops/create_cluster.go

@@ -119,6 +119,9 @@ type CreateClusterOptions struct {
 	// We need VSphereDatastore to support Kubernetes vSphere Cloud Provider (v1.5.3)
 	// We can remove this once we support higher versions.
 	VSphereDatastore string
+
+	// ConfigBase is the location where we will store the configuration, it defaults to the state store
+	ConfigBase string
 }
 
 func (o *CreateClusterOptions) InitDefaults() {
@@ -225,6 +228,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform, cloudformation")
 	cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)")
 
+	// Configuration / state location
+	cmd.Flags().StringVar(&options.ConfigBase, "config-base", options.ConfigBase, "The location where the cluster state files should be stored")
+
 	cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws, vsphere")
 
 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
@@ -369,6 +375,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 	}
 	cluster.Spec.Channel = c.Channel
 
+	cluster.Spec.ConfigBase = c.ConfigBase
 	configBase, err := clientset.ConfigBaseFor(cluster)
 	if err != nil {
 		return fmt.Errorf("error building ConfigBase for cluster: %v", err)
@@ -896,7 +903,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 	}
 
 	assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-	fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+	fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 	if err != nil {
 		return err
 	}
@@ -921,7 +928,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_dockerconfig.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -97,7 +96,12 @@ func RunCreateSecretDockerConfig(f *util.Factory, out io.Writer, options *Create
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_encryptionconfig.go

@@ -25,7 +25,6 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -98,7 +97,12 @@ func RunCreateSecretEncryptionConfig(f *util.Factory, out io.Writer, options *Cr
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_sshpublickey.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
 )
@@ -99,7 +98,12 @@ func RunCreateSecretPublicKey(f *util.Factory, out io.Writer, options *CreateSec
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/delete_cluster.go

@@ -24,15 +24,13 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
 	api "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/pkg/resources"
 	"k8s.io/kops/pkg/resources/tracker"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/util/pkg/tables"
-	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
 )
@@ -96,8 +94,6 @@ func NewCmdDeleteCluster(f *util.Factory, out io.Writer) *cobra.Command {
 type getter func(o interface{}) interface{}
 
 func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOptions) error {
-	var configBase vfs.Path
-
 	clusterName := options.ClusterName
 	if clusterName == "" {
 		return fmt.Errorf("--name is required (for safety)")
@@ -123,11 +119,6 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti
 		if err != nil {
 			return err
 		}
-
-		configBase, err = registry.ConfigBase(cluster)
-		if err != nil {
-			return err
-		}
 	}
 
 	wouldDeleteCloudResources := false
@@ -205,7 +196,11 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti
 			}
 			return nil
 		}
-		err := registry.DeleteAllClusterState(configBase)
+		clientset, err := f.Clientset()
+		if err != nil {
+			return err
+		}
+		err = clientset.DeleteCluster(cluster)
 		if err != nil {
 			return fmt.Errorf("error removing cluster from state store: %v", err)
 		}

cmd/kops/delete_secret.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -88,17 +87,22 @@ func RunDeleteSecret(f *util.Factory, out io.Writer, options *DeleteSecretOption
 		return fmt.Errorf("SecretName is required")
 	}
 
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
 	cluster, err := GetCluster(f, options.ClusterName)
 	if err != nil {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/describe_secrets.go

@@ -17,17 +17,15 @@ limitations under the License.
 package main
 
 import (
-	"fmt"
-
 	"bytes"
 	"crypto/rsa"
+	"fmt"
 	"os"
 	"sort"
 	"strings"
 	"text/tabwriter"
 
 	"github.com/spf13/cobra"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -77,12 +75,17 @@ func (c *DescribeSecretsCommand) Run(args []string) error {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := rootCommand.Clientset()
+	if err != nil {
+		return err
+	}
+
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/edit_cluster.go

@@ -211,7 +211,7 @@ func RunEditCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.W
 		}
 
 		assetBuilder := assets.NewAssetBuilder(newCluster.Spec.Assets)
-		fullCluster, err := cloudup.PopulateClusterSpec(newCluster, assetBuilder)
+		fullCluster, err := cloudup.PopulateClusterSpec(clientset, newCluster, assetBuilder)
 		if err != nil {
 			results = editResults{
 				file: file,

cmd/kops/edit_instancegroup.go

@@ -168,7 +168,7 @@ func RunEditInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, ou
 	}
 
 	assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-	fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+	fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 	if err != nil {
 		return err
 	}

cmd/kops/export_kubecfg.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@@ -73,17 +72,22 @@ func RunExportKubecfg(f *util.Factory, out io.Writer, options *ExportKubecfgOpti
 		return err
 	}
 
+	clientset, err := rootCommand.Clientset()
+	if err != nil {
+		return err
+	}
+
 	cluster, err := rootCommand.Cluster()
 	if err != nil {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/get_secrets.go

@@ -18,14 +18,12 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"os"
-
 	"strings"
 
 	"github.com/spf13/cobra"
-	"io"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/util/pkg/tables"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@@ -148,12 +146,17 @@ func RunGetSecrets(options *GetSecretsOptions, args []string) error {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := rootCommand.Clientset()
+	if err != nil {
+		return err
+	}
+
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/update_cluster.go

@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
@@ -144,17 +143,17 @@ func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *Upd
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := f.Clientset()
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	clientset, err := f.Clientset()
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/upgrade_cluster.go

@@ -290,7 +290,7 @@ func (c *UpgradeClusterCmd) Run(args []string) error {
 		}
 
 		assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-		fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+		fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 		if err != nil {
 			return err
 		}