From 1a249105498bbcf6055d868d4940b6ff898a947a Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Tue, 20 Nov 2018 15:59:44 -0800 Subject: [PATCH 01/11] commands/.../test,pkg/test: add --up-local flag to test local --- CHANGELOG.md | 1 + commands/operator-sdk/cmd/test/local.go | 52 +++++++++++------- doc/sdk-cli-reference.md | 1 + doc/test-framework/writing-e2e-tests.md | 20 +++++-- hack/tests/test-subcommand.sh | 4 ++ pkg/test/e2eutil/wait_util.go | 16 ++++++ pkg/test/framework.go | 4 +- pkg/test/main_entry.go | 55 ++++++++++++++++++- pkg/test/resource_creator.go | 4 ++ .../memcached_test.go.tmpl | 2 +- test/test-framework/memcached_test.go | 2 +- 11 files changed, 132 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fad4095d6d7..a2ecc7e23f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Added - A new command [`operator-sdk print-deps`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#print-deps) which prints Golang packages and versions expected by the current Operator SDK version. Supplying `--as-file` prints packages and versions in Gopkg.toml format. ([#772](https://github.com/operator-framework/operator-sdk/pull/772)) +- Add [`up-local`](https://github.com/operator-framework/operator-sdk/blob/master/doc/sdk-cli-reference.md#flags-9) flag to `test local` subcommand ([#781](https://github.com/operator-framework/operator-sdk/pull/781)) ### Bug fixes diff --git a/commands/operator-sdk/cmd/test/local.go b/commands/operator-sdk/cmd/test/local.go index bbf3f5ed04f..683edbd66a3 100644 --- a/commands/operator-sdk/cmd/test/local.go +++ b/commands/operator-sdk/cmd/test/local.go @@ -39,6 +39,7 @@ type testLocalConfig struct { namespacedManPath string goTestFlags string namespace string + upLocal bool } var tlConfig testLocalConfig @@ -54,6 +55,7 @@ func NewTestLocalCmd() *cobra.Command { testCmd.Flags().StringVar(&tlConfig.namespacedManPath, "namespaced-manifest", "", "Path to manifest for per-test, namespaced resources (e.g. RBAC and Operator manifest)") testCmd.Flags().StringVar(&tlConfig.goTestFlags, "go-test-flags", "", "Additional flags to pass to go test") testCmd.Flags().StringVar(&tlConfig.namespace, "namespace", "", "If non-empty, single namespace to run tests in") + testCmd.Flags().BoolVar(&tlConfig.upLocal, "up-local", false, "Enable running operator locally with go run instead of as an image in the cluster") return testCmd } @@ -63,6 +65,10 @@ func testLocalFunc(cmd *cobra.Command, args []string) { log.Fatal("operator-sdk test local requires exactly 1 argument") } + if tlConfig.upLocal && tlConfig.namespace == "" { + log.Fatal("must specify a namespace to run in when -up-local flag is set") + } + log.Info("Testing operator locally.") // if no namespaced manifest path is given, combine deploy/service_account.yaml, deploy/role.yaml, deploy/role_binding.yaml and deploy/operator.yaml @@ -72,28 +78,29 @@ func testLocalFunc(cmd *cobra.Command, args []string) { log.Fatalf("could not create %s: (%v)", deployTestDir, err) } tlConfig.namespacedManPath = filepath.Join(deployTestDir, "namespace-manifests.yaml") - - sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile)) - if err != nil { - log.Warnf("could not find the serviceaccount manifest: (%v)", err) - } - role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile)) - if err != nil { - log.Warnf("could not find role manifest: (%v)", err) - } - roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile)) - if err != nil { - log.Warnf("could not find role_binding manifest: (%v)", err) - } - operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile)) - if err != nil { - log.Fatalf("could not find operator manifest: (%v)", err) - } combined := []byte{} - combined = combineManifests(combined, sa) - combined = combineManifests(combined, role) - combined = combineManifests(combined, roleBinding) - combined = append(combined, operator...) + if !tlConfig.upLocal { + sa, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.ServiceAccountYamlFile)) + if err != nil { + log.Warnf("could not find the serviceaccount manifest: (%v)", err) + } + role, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleYamlFile)) + if err != nil { + log.Warnf("could not find role manifest: (%v)", err) + } + roleBinding, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.RoleBindingYamlFile)) + if err != nil { + log.Warnf("could not find role_binding manifest: (%v)", err) + } + operator, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, scaffold.OperatorYamlFile)) + if err != nil { + log.Fatalf("could not find operator manifest: (%v)", err) + } + combined = combineManifests(combined, sa) + combined = combineManifests(combined, role) + combined = combineManifests(combined, roleBinding) + combined = append(combined, operator...) + } err = ioutil.WriteFile(tlConfig.namespacedManPath, combined, os.FileMode(fileutil.DefaultFileMode)) if err != nil { log.Fatalf("could not create temporary namespaced manifest file: (%v)", err) @@ -154,6 +161,9 @@ func testLocalFunc(cmd *cobra.Command, args []string) { if tlConfig.namespace != "" { testArgs = append(testArgs, "-"+test.SingleNamespaceFlag, "-parallel=1") } + if tlConfig.upLocal { + testArgs = append(testArgs, "-"+test.LocalOperatorFlag) + } dc := exec.Command("go", testArgs...) dc.Env = append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace)) dc.Dir = projutil.MustGetwd() diff --git a/doc/sdk-cli-reference.md b/doc/sdk-cli-reference.md index 2d230554d6c..05916b2ae1b 100644 --- a/doc/sdk-cli-reference.md +++ b/doc/sdk-cli-reference.md @@ -257,6 +257,7 @@ Runs the tests locally * `--namespaced-manifest` string - path to manifest for per-test, namespaced resources (default: combines deploy/service_account.yaml, deploy/rbac.yaml, and deploy/operator.yaml) * `--namespace` string - if non-empty, single namespace to run tests in (e.g. "operator-test") (default: "") * `--go-test-flags` string - extra arguments to pass to `go test` (e.g. -f "-v -parallel=2") +* `--up-local` - Enable running operator locally with go run instead of as an image in the cluster * `-h, --help` - help for local ##### Use diff --git a/doc/test-framework/writing-e2e-tests.md b/doc/test-framework/writing-e2e-tests.md index 6b3bbf5af45..51e450d3517 100644 --- a/doc/test-framework/writing-e2e-tests.md +++ b/doc/test-framework/writing-e2e-tests.md @@ -121,10 +121,9 @@ in your namespaced manifest. The custom `Create` function use the controller-run creates a cleanup function that is called by `ctx.Cleanup` which deletes the resource and then waits for the resource to be fully deleted before returning. This is configurable with `CleanupOptions`. For info on how to use `CleanupOptions` see [this section](#how-to-use-cleanup). - If you want to make sure the operator's deployment is fully ready before moving onto the next part of the -test, the `WaitForDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used: +test, the `WaitForOperatorDeployment` function from [e2eutil][e2eutil-link] (in the sdk under `pkg/test/e2eutil`) can be used: ```go // get namespace @@ -135,7 +134,7 @@ if err != nil { // get global framework variables f := framework.Global // wait for memcached-operator to be ready -err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30) +err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, time.Second*5, time.Second*30) if err != nil { t.Fatal(err) } @@ -188,7 +187,10 @@ if err != nil { ``` Now we can check if the operator successfully worked. In the case of the memcached operator, it should have -created a deployment called "example-memcached" with 3 replicas: +created a deployment called "example-memcached" with 3 replicas. To check, we use the `WaitForDeployment` function, which +is the same as `WaitForOperatorDeployment` with the exception that `WaitForOperatorDeployment` will skip waiting +for the deployment if the test is run locally and the `--up-local` flag is set; the `WaitForDeployment` function always +waits for the deployment: ```go // wait for example-memcached to reach 3 replicas @@ -243,6 +245,16 @@ $ kubectl create namespace operator-test $ operator-sdk test local ./test/e2e --namespace operator-test ``` +To run the operator itself locally during the tests instead of starting a deployment in the cluster, you can use the +`--up-local` flag. This mode will still create global resources, but by default will not create any in-cluster namespaced +resources unless the user specifies one through the `--namespaced-manifest` flag. (Note: the `--up-local` flag requires +the `--namespace` flag): + +```shell +$ kubectl create namespace operator-test +$ operator-sdk test local ./test/e2e --namespace operator-test --up-local +``` + For more documentation on the `operator-sdk test local` command, see the [SDK CLI Reference][sdk-cli-ref] doc. For advanced use cases, it is possible to run the tests via `go test` directly. As long as all flags defined diff --git a/hack/tests/test-subcommand.sh b/hack/tests/test-subcommand.sh index 092412cd278..0051f85280f 100755 --- a/hack/tests/test-subcommand.sh +++ b/hack/tests/test-subcommand.sh @@ -10,3 +10,7 @@ operator-sdk test local . --global-manifest deploy/crds/cache_v1alpha1_memcached kubectl create namespace test-memcached operator-sdk test local . --namespace=test-memcached kubectl delete namespace test-memcached +# test operator in up local mode +kubectl create namespace test-memcached +operator-sdk test local . --up-local --namespace=test-memcached +kubectl delete namespace test-memcached diff --git a/pkg/test/e2eutil/wait_util.go b/pkg/test/e2eutil/wait_util.go index eca45a978d9..bbf44a81d0c 100644 --- a/pkg/test/e2eutil/wait_util.go +++ b/pkg/test/e2eutil/wait_util.go @@ -18,6 +18,8 @@ import ( "testing" "time" + "github.com/operator-framework/operator-sdk/pkg/test" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -29,6 +31,20 @@ import ( // This can be used in multiple ways, like verifying that a required resource is ready before trying to use it, or to test // failure handling, like simulated in SimulatePodFail. func WaitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration) error { + return waitForDeployment(t, kubeclient, namespace, name, replicas, retryInterval, timeout, false) +} + +// WaitForOperatorDeployment has the same functionality as WaitForDeployment but will no wait for the deployment if the +// test was run with a locally run operator (--up-local flag) +func WaitForOperatorDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration) error { + return waitForDeployment(t, kubeclient, namespace, name, replicas, retryInterval, timeout, true) +} + +func waitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration, isOperator bool) error { + if isOperator && test.Global.LocalOperator { + t.Logf("Operator is running locally; skip waitForDeployment\n") + return nil + } err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { deployment, err := kubeclient.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{IncludeUninitialized: true}) if err != nil { diff --git a/pkg/test/framework.go b/pkg/test/framework.go index e5c4473d50f..52d667d4d7a 100644 --- a/pkg/test/framework.go +++ b/pkg/test/framework.go @@ -54,9 +54,10 @@ type Framework struct { Scheme *runtime.Scheme NamespacedManPath *string Namespace string + LocalOperator bool } -func setup(kubeconfigPath, namespacedManPath *string) error { +func setup(kubeconfigPath, namespacedManPath *string, localOperator bool) error { var err error var kubeconfig *rest.Config if *kubeconfigPath == "incluster" { @@ -105,6 +106,7 @@ func setup(kubeconfigPath, namespacedManPath *string) error { Scheme: scheme, NamespacedManPath: namespacedManPath, Namespace: namespace, + LocalOperator: localOperator, } return nil } diff --git a/pkg/test/main_entry.go b/pkg/test/main_entry.go index 16f01c7493a..a621203169d 100644 --- a/pkg/test/main_entry.go +++ b/pkg/test/main_entry.go @@ -15,11 +15,21 @@ package test import ( + "bytes" "flag" + "fmt" "io/ioutil" "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" "testing" + "k8s.io/client-go/tools/clientcmd" + + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "github.com/operator-framework/operator-sdk/pkg/scaffold" log "github.com/sirupsen/logrus" ) @@ -30,6 +40,7 @@ const ( GlobalManPathFlag = "globalMan" SingleNamespaceFlag = "singleNamespace" TestNamespaceEnv = "TEST_NAMESPACE" + LocalOperatorFlag = "localOperator" ) func MainEntry(m *testing.M) { @@ -38,21 +49,63 @@ func MainEntry(m *testing.M) { globalManPath := flag.String(GlobalManPathFlag, "", "path to operator manifest") namespacedManPath := flag.String(NamespacedManPathFlag, "", "path to rbac manifest") singleNamespace = flag.Bool(SingleNamespaceFlag, false, "enable single namespace mode") + localOperator := flag.Bool(LocalOperatorFlag, false, "enable if operator is running locally (not in cluster)") flag.Parse() // go test always runs from the test directory; change to project root err := os.Chdir(*projRoot) if err != nil { log.Fatalf("failed to change directory to project root: %v", err) } - if err := setup(kubeconfigPath, namespacedManPath); err != nil { + if err := setup(kubeconfigPath, namespacedManPath, *localOperator); err != nil { log.Fatalf("failed to set up framework: %v", err) } + // setup local operator command, but don't start it yet + var localCmd *exec.Cmd + var localCmdOutBuf, localCmdErrBuf bytes.Buffer + if *localOperator { + // taken from commands/operator-sdk/cmd/up/local.go + runArgs := append([]string{"run"}, []string{filepath.Join(scaffold.ManagerDir, scaffold.CmdFile)}...) + localCmd = exec.Command("go", runArgs...) + localCmd.Stdout = &localCmdOutBuf + localCmd.Stderr = &localCmdErrBuf + c := make(chan os.Signal) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + err := localCmd.Process.Signal(os.Interrupt) + if err != nil { + log.Fatalf("failed to terminate the operator: (%v)", err) + } + os.Exit(0) + }() + if *kubeconfigPath != "" { + localCmd.Env = append(os.Environ(), fmt.Sprintf("%v=%v", k8sutil.KubeConfigEnvVar, kubeconfigPath)) + } else { + // we can hardcode index 0 as that is the highest priority kubeconfig to be loaded and will always + // be populated by NewDefaultClientConfigLoadingRules() + localCmd.Env = append(os.Environ(), fmt.Sprintf("%v=%v", k8sutil.KubeConfigEnvVar, clientcmd.NewDefaultClientConfigLoadingRules().Precedence[0])) + } + localCmd.Env = append(localCmd.Env, fmt.Sprintf("%v=%v", k8sutil.WatchNamespaceEnvVar, Global.Namespace)) + } // setup context to use when setting up crd ctx := NewTestCtx(nil) // os.Exit stops the program before the deferred functions run // to fix this, we put the exit in the defer as well defer func() { + // start local operator before running tests + if *localOperator { + err := localCmd.Start() + if err != nil { + log.Fatalf("failed to run operator locally: (%v)", err) + } + log.Info("started local operator") + } exitCode := m.Run() + if *localOperator { + localCmd.Process.Kill() + log.Infof("local operator stdout: %s", string(localCmdOutBuf.Bytes())) + log.Infof("local operator stderr: %s", string(localCmdErrBuf.Bytes())) + } ctx.CleanupNoT() os.Exit(exitCode) }() diff --git a/pkg/test/resource_creator.go b/pkg/test/resource_creator.go index a5756a96dc5..ddcfbbadd6a 100644 --- a/pkg/test/resource_creator.go +++ b/pkg/test/resource_creator.go @@ -66,6 +66,10 @@ func (ctx *TestCtx) createFromYAML(yamlFile []byte, skipIfExists bool, cleanupOp } yamlSplit := bytes.Split(yamlFile, []byte("\n---\n")) for _, yamlSpec := range yamlSplit { + // some autogenerated files may include an extra `---` at the end of the file or be empty + if string(yamlSpec) == "" { + continue + } yamlSpec, err = setNamespaceYAML(yamlSpec, namespace) if err != nil { return err diff --git a/test/e2e/incluster-test-code/memcached_test.go.tmpl b/test/e2e/incluster-test-code/memcached_test.go.tmpl index 6f43c81b742..616e5c6a988 100644 --- a/test/e2e/incluster-test-code/memcached_test.go.tmpl +++ b/test/e2e/incluster-test-code/memcached_test.go.tmpl @@ -114,7 +114,7 @@ func MemcachedCluster(t *testing.T) { // get global framework variables f := framework.Global // wait for memcached-operator to be ready - err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout) + err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout) if err != nil { t.Fatal(err) } diff --git a/test/test-framework/memcached_test.go b/test/test-framework/memcached_test.go index 21df1dae43e..c2fcdcd2583 100644 --- a/test/test-framework/memcached_test.go +++ b/test/test-framework/memcached_test.go @@ -114,7 +114,7 @@ func MemcachedCluster(t *testing.T) { // get global framework variables f := framework.Global // wait for memcached-operator to be ready - err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout) + err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, "memcached-operator", 1, retryInterval, timeout) if err != nil { t.Fatal(err) } From 782404032b6770eb86c8b58496a47c4d5d217e0b Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Tue, 27 Nov 2018 15:06:52 -0500 Subject: [PATCH 02/11] pkg/scaffold: fix an import cycle --- pkg/scaffold/test_pod_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/scaffold/test_pod_test.go b/pkg/scaffold/test_pod_test.go index 3f0b358b3b3..3debc6e6c58 100644 --- a/pkg/scaffold/test_pod_test.go +++ b/pkg/scaffold/test_pod_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/operator-framework/operator-sdk/pkg/scaffold/internal/testutil" - "github.com/operator-framework/operator-sdk/pkg/test" ) func TestPodTest(t *testing.T) { @@ -26,7 +25,7 @@ func TestPodTest(t *testing.T) { err := s.Execute(appConfig, &TestPod{ Image: "quay.io/app/operator:v1.0.0", - TestNamespaceEnv: test.TestNamespaceEnv, + TestNamespaceEnv: "TEST_NAMESPACE", }) if err != nil { t.Fatalf("failed to execute the scaffold: (%v)", err) From 2090c8628f57ac23143d9277c9e714543ddfcadc Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Tue, 27 Nov 2018 16:09:13 -0500 Subject: [PATCH 03/11] test/test-framework: add missing files --- Gopkg.lock | 50 ++++ hack/tests/test-subcommand.sh | 8 +- test/test-framework/cmd/manager/main.go | 65 +++++ .../pkg/controller/add_memcached.go | 10 + .../pkg/controller/controller.go | 18 ++ .../memcached/memcached_controller.go | 229 ++++++++++++++++++ .../memcached/memcached_controller_test.go | 126 ++++++++++ .../{ => test/e2e}/main_test.go | 0 .../{ => test/e2e}/memcached_test.go | 0 9 files changed, 502 insertions(+), 4 deletions(-) create mode 100644 test/test-framework/cmd/manager/main.go create mode 100644 test/test-framework/pkg/controller/add_memcached.go create mode 100644 test/test-framework/pkg/controller/controller.go create mode 100644 test/test-framework/pkg/controller/memcached/memcached_controller.go create mode 100644 test/test-framework/pkg/controller/memcached/memcached_controller_test.go rename test/test-framework/{ => test/e2e}/main_test.go (100%) rename test/test-framework/{ => test/e2e}/memcached_test.go (100%) diff --git a/Gopkg.lock b/Gopkg.lock index f18c468c0a4..b9755f96a82 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,14 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:dd029f90457e0bc0fc4a549a241327bdcac1f2a36692041a19ef02aec4c36103" + name = "cloud.google.com/go" + packages = ["compute/metadata"] + pruneopts = "" + revision = "74b12019e2aa53ec27882158f59192d7cd6d1998" + version = "v0.33.1" + [[projects]] digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474" name = "github.com/BurntSushi/toml" @@ -488,6 +496,7 @@ name = "golang.org/x/net" packages = [ "context", + "context/ctxhttp", "html", "html/atom", "http/httpguts", @@ -498,6 +507,20 @@ pruneopts = "" revision = "04a2e542c03f1d053ab3e4d6e5abcd4b66e2be8e" +[[projects]] + branch = "master" + digest = "1:76df884b1ac08579aff75a621a538800759808f000bb4a1b08c91b485bfb3522" + name = "golang.org/x/oauth2" + packages = [ + ".", + "google", + "internal", + "jws", + "jwt", + ] + pruneopts = "" + revision = "8f65e3013ebad444f13bc19536f7865efc793816" + [[projects]] branch = "master" digest = "1:9bbe878c5cef3e193360515704d01ddb34c756e8cfd4abf7166f3f6a2059c553" @@ -554,6 +577,25 @@ pruneopts = "" revision = "6adeb8aab2ded9eb693b831d5fd090c10a6ebdfa" +[[projects]] + digest = "1:77d3cff3a451d50be4b52db9c7766c0d8570ba47593f0c9dc72173adb208e788" + name = "google.golang.org/appengine" + packages = [ + ".", + "internal", + "internal/app_identity", + "internal/base", + "internal/datastore", + "internal/log", + "internal/modules", + "internal/remote_api", + "internal/urlfetch", + "urlfetch", + ] + pruneopts = "" + revision = "4a4468ece617fc8205e99368fa2200e9d1fad421" + version = "v1.3.0" + [[projects]] digest = "1:75fb3fcfc73a8c723efde7777b40e8e8ff9babf30d8c56160d01beffea8a95a6" name = "gopkg.in/inf.v0" @@ -719,10 +761,12 @@ "pkg/apis/clientauthentication/v1beta1", "pkg/version", "plugin/pkg/client/auth/exec", + "plugin/pkg/client/auth/gcp", "rest", "rest/watch", "restmapper", "testing", + "third_party/forked/golang/template", "tools/auth", "tools/cache", "tools/clientcmd", @@ -742,6 +786,7 @@ "util/flowcontrol", "util/homedir", "util/integer", + "util/jsonpath", "util/retry", "util/workqueue", ] @@ -771,6 +816,7 @@ "pkg/client/config", "pkg/client/fake", "pkg/controller", + "pkg/controller/controllerutil", "pkg/event", "pkg/handler", "pkg/internal/controller", @@ -811,6 +857,7 @@ "github.com/spf13/cobra", "golang.org/x/tools/imports", "gopkg.in/yaml.v2", + "k8s.io/api/apps/v1", "k8s.io/api/core/v1", "k8s.io/api/rbac/v1", "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme", @@ -818,6 +865,7 @@ "k8s.io/apimachinery/pkg/api/meta", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", + "k8s.io/apimachinery/pkg/labels", "k8s.io/apimachinery/pkg/runtime", "k8s.io/apimachinery/pkg/runtime/schema", "k8s.io/apimachinery/pkg/runtime/serializer", @@ -829,6 +877,7 @@ "k8s.io/client-go/discovery/cached", "k8s.io/client-go/kubernetes", "k8s.io/client-go/kubernetes/scheme", + "k8s.io/client-go/plugin/pkg/client/auth/gcp", "k8s.io/client-go/rest", "k8s.io/client-go/restmapper", "k8s.io/client-go/tools/clientcmd", @@ -838,6 +887,7 @@ "sigs.k8s.io/controller-runtime/pkg/client/config", "sigs.k8s.io/controller-runtime/pkg/client/fake", "sigs.k8s.io/controller-runtime/pkg/controller", + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil", "sigs.k8s.io/controller-runtime/pkg/handler", "sigs.k8s.io/controller-runtime/pkg/manager", "sigs.k8s.io/controller-runtime/pkg/reconcile", diff --git a/hack/tests/test-subcommand.sh b/hack/tests/test-subcommand.sh index 0051f85280f..7621e18d95f 100755 --- a/hack/tests/test-subcommand.sh +++ b/hack/tests/test-subcommand.sh @@ -3,14 +3,14 @@ set -ex cd test/test-framework # test framework with defaults -operator-sdk test local . +operator-sdk test local ./test/e2e # test operator-sdk test flags -operator-sdk test local . --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config +operator-sdk test local ./test/e2e --global-manifest deploy/crds/cache_v1alpha1_memcached_crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config # test operator-sdk test local single namespace mode kubectl create namespace test-memcached -operator-sdk test local . --namespace=test-memcached +operator-sdk test local ./test/e2e --namespace=test-memcached kubectl delete namespace test-memcached # test operator in up local mode kubectl create namespace test-memcached -operator-sdk test local . --up-local --namespace=test-memcached +operator-sdk test local ./test/e2e --up-local --namespace=test-memcached kubectl delete namespace test-memcached diff --git a/test/test-framework/cmd/manager/main.go b/test/test-framework/cmd/manager/main.go new file mode 100644 index 00000000000..f05eecb1f31 --- /dev/null +++ b/test/test-framework/cmd/manager/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "flag" + "log" + "runtime" + + k8sutil "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis" + "github.com/operator-framework/operator-sdk/test/test-framework/pkg/controller" + sdkVersion "github.com/operator-framework/operator-sdk/version" + + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/signals" +) + +func printVersion() { + log.Printf("Go Version: %s", runtime.Version()) + log.Printf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) + log.Printf("operator-sdk Version: %v", sdkVersion.Version) +} + +func main() { + printVersion() + flag.Parse() + + namespace, err := k8sutil.GetWatchNamespace() + if err != nil { + log.Fatalf("failed to get watch namespace: %v", err) + } + + // TODO: Expose metrics port after SDK uses controller-runtime's dynamic client + // sdk.ExposeMetricsPort() + + // Get a config to talk to the apiserver + cfg, err := config.GetConfig() + if err != nil { + log.Fatal(err) + } + + // Create a new Cmd to provide shared dependencies and start components + mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + if err != nil { + log.Fatal(err) + } + + log.Print("Registering Components.") + + // Setup Scheme for all resources + if err := apis.AddToScheme(mgr.GetScheme()); err != nil { + log.Fatal(err) + } + + // Setup all Controllers + if err := controller.AddToManager(mgr); err != nil { + log.Fatal(err) + } + + log.Print("Starting the Cmd.") + + // Start the Cmd + log.Fatal(mgr.Start(signals.SetupSignalHandler())) +} diff --git a/test/test-framework/pkg/controller/add_memcached.go b/test/test-framework/pkg/controller/add_memcached.go new file mode 100644 index 00000000000..4cee66e709d --- /dev/null +++ b/test/test-framework/pkg/controller/add_memcached.go @@ -0,0 +1,10 @@ +package controller + +import ( + "github.com/operator-framework/operator-sdk/test/test-framework/pkg/controller/memcached" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, memcached.Add) +} diff --git a/test/test-framework/pkg/controller/controller.go b/test/test-framework/pkg/controller/controller.go new file mode 100644 index 00000000000..7c069f3ee6e --- /dev/null +++ b/test/test-framework/pkg/controller/controller.go @@ -0,0 +1,18 @@ +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// AddToManagerFuncs is a list of functions to add all Controllers to the Manager +var AddToManagerFuncs []func(manager.Manager) error + +// AddToManager adds all Controllers to the Manager +func AddToManager(m manager.Manager) error { + for _, f := range AddToManagerFuncs { + if err := f(m); err != nil { + return err + } + } + return nil +} diff --git a/test/test-framework/pkg/controller/memcached/memcached_controller.go b/test/test-framework/pkg/controller/memcached/memcached_controller.go new file mode 100644 index 00000000000..b6b6fd4ddac --- /dev/null +++ b/test/test-framework/pkg/controller/memcached/memcached_controller.go @@ -0,0 +1,229 @@ +package memcached + +import ( + "context" + "reflect" + + cachev1alpha1 "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis/cache/v1alpha1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("controller_memcached") + +// Add creates a new Memcached Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileMemcached{client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("memcached-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to primary resource Memcached + err = c.Watch(&source.Kind{Type: &cachev1alpha1.Memcached{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // TODO(user): Modify this to be the types you create that are owned by the primary resource + // Watch for changes to secondary resource Pods and requeue the owner Memcached + err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &cachev1alpha1.Memcached{}, + }) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileMemcached{} + +// ReconcileMemcached reconciles a Memcached object +type ReconcileMemcached struct { + // TODO: Clarify the split client + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + client client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Memcached object and makes changes based on the state read +// and what is in the Memcached.Spec +// TODO(user): Modify this Reconcile function to implement your Controller logic. This example creates +// a Memcached Deployment for each Memcached CR +// Note: +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Result, error) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Reconciling Memcached.") + + // Fetch the Memcached instance + memcached := &cachev1alpha1.Memcached{} + err := r.client.Get(context.TODO(), request.NamespacedName, memcached) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + reqLogger.Info("Memcached resource not found. Ignoring since object must be deleted.") + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + reqLogger.Error(err, "Failed to get Memcached.") + return reconcile.Result{}, err + } + + // Check if the deployment already exists, if not create a new one + found := &appsv1.Deployment{} + err = r.client.Get(context.TODO(), types.NamespacedName{Name: memcached.Name, Namespace: memcached.Namespace}, found) + if err != nil && errors.IsNotFound(err) { + // Define a new deployment + dep := r.deploymentForMemcached(memcached) + reqLogger.Info("Creating a new Deployment.", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name) + err = r.client.Create(context.TODO(), dep) + if err != nil { + reqLogger.Error(err, "Failed to create new Deployment.", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name) + return reconcile.Result{}, err + } + // Deployment created successfully - return and requeue + return reconcile.Result{Requeue: true}, nil + } else if err != nil { + reqLogger.Error(err, "Failed to get Deployment.") + return reconcile.Result{}, err + } + + // Ensure the deployment size is the same as the spec + size := memcached.Spec.Size + if *found.Spec.Replicas != size { + found.Spec.Replicas = &size + err = r.client.Update(context.TODO(), found) + if err != nil { + reqLogger.Error(err, "Failed to update Deployment.", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name) + return reconcile.Result{}, err + } + // Spec updated - return and requeue + return reconcile.Result{Requeue: true}, nil + } + + // Update the Memcached status with the pod names + // List the pods for this memcached's deployment + podList := &corev1.PodList{} + labelSelector := labels.SelectorFromSet(labelsForMemcached(memcached.Name)) + listOps := &client.ListOptions{ + Namespace: memcached.Namespace, + LabelSelector: labelSelector, + // HACK: due to a fake client bug, ListOptions.Raw.TypeMeta must be + // explicitly populated for testing. + // + // See https://github.com/kubernetes-sigs/controller-runtime/issues/168 + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Memcached", + APIVersion: cachev1alpha1.SchemeGroupVersion.Version, + }, + }, + } + err = r.client.List(context.TODO(), listOps, podList) + if err != nil { + reqLogger.Error(err, "Failed to list pods.", "Memcached.Namespace", memcached.Namespace, "Memcached.Name", memcached.Name) + return reconcile.Result{}, err + } + podNames := getPodNames(podList.Items) + + // Update status.Nodes if needed + if !reflect.DeepEqual(podNames, memcached.Status.Nodes) { + memcached.Status.Nodes = podNames + err := r.client.Update(context.TODO(), memcached) + if err != nil { + reqLogger.Error(err, "Failed to update Memcached status.") + return reconcile.Result{}, err + } + } + + return reconcile.Result{}, nil +} + +// deploymentForMemcached returns a memcached Deployment object +func (r *ReconcileMemcached) deploymentForMemcached(m *cachev1alpha1.Memcached) *appsv1.Deployment { + ls := labelsForMemcached(m.Name) + replicas := m.Spec.Size + + dep := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: m.Name, + Namespace: m.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: ls, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ls, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Image: "memcached:1.4.36-alpine", + Name: "memcached", + Command: []string{"memcached", "-m=64", "-o", "modern", "-v"}, + Ports: []corev1.ContainerPort{{ + ContainerPort: 11211, + Name: "memcached", + }}, + }}, + }, + }, + }, + } + // Set Memcached instance as the owner and controller + controllerutil.SetControllerReference(m, dep, r.scheme) + return dep +} + +// labelsForMemcached returns the labels for selecting the resources +// belonging to the given memcached CR name. +func labelsForMemcached(name string) map[string]string { + return map[string]string{"app": "memcached", "memcached_cr": name} +} + +// getPodNames returns the pod names of the array of pods passed in +func getPodNames(pods []corev1.Pod) []string { + var podNames []string + for _, pod := range pods { + podNames = append(podNames, pod.Name) + } + return podNames +} diff --git a/test/test-framework/pkg/controller/memcached/memcached_controller_test.go b/test/test-framework/pkg/controller/memcached/memcached_controller_test.go new file mode 100644 index 00000000000..b2cce77f23a --- /dev/null +++ b/test/test-framework/pkg/controller/memcached/memcached_controller_test.go @@ -0,0 +1,126 @@ +package memcached + +import ( + "context" + "math/rand" + "reflect" + "strconv" + "testing" + + cachev1alpha1 "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis/cache/v1alpha1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" +) + +// TestMemcachedController runs ReconcileMemcached.Reconcile() against a +// fake client that tracks a Memcached object. +func TestMemcachedController(t *testing.T) { + // Set the logger to development mode for verbose logs. + logf.SetLogger(logf.ZapLogger(true)) + + var ( + name = "memcached-operator" + namespace = "memcached" + replicas int32 = 3 + ) + + // A Memcached resource with metadata and spec. + memcached := &cachev1alpha1.Memcached{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: cachev1alpha1.MemcachedSpec{ + Size: replicas, // Set desired number of Memcached replicas. + }, + } + // Objects to track in the fake client. + objs := []runtime.Object{ + memcached, + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(cachev1alpha1.SchemeGroupVersion, memcached) + // Create a fake client to mock API calls. + cl := fake.NewFakeClient(objs...) + // Create a ReconcileMemcached object with the scheme and fake client. + r := &ReconcileMemcached{client: cl, scheme: s} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + res, err := r.Reconcile(req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + // Check the result of reconciliation to make sure it has the desired state. + if !res.Requeue { + t.Error("reconcile did not requeue request as expected") + } + + // Check if deployment has been created and has the correct size. + dep := &appsv1.Deployment{} + err = cl.Get(context.TODO(), req.NamespacedName, dep) + if err != nil { + t.Fatalf("get deployment: (%v)", err) + } + dsize := *dep.Spec.Replicas + if dsize != replicas { + t.Errorf("dep size (%d) is not the expected size (%d)", dsize, replicas) + } + + // Create the 3 expected pods in namespace and collect their names to check + // later. + podLabels := labelsForMemcached(name) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Labels: podLabels, + }, + } + podNames := make([]string, 3) + for i := 0; i < 3; i++ { + pod.ObjectMeta.Name = name + ".pod." + strconv.Itoa(rand.Int()) + podNames[i] = pod.ObjectMeta.Name + if err = cl.Create(context.TODO(), pod.DeepCopy()); err != nil { + t.Fatalf("create pod %d: (%v)", i, err) + } + } + + // Reconcile again so Reconcile() checks pods and updates the Memcached + // resources' Status. + res, err = r.Reconcile(req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + if res != (reconcile.Result{}) { + t.Error("reconcile did not return an empty Result") + } + + // Get the updated Memcached object. + memcached = &cachev1alpha1.Memcached{} + err = r.client.Get(context.TODO(), req.NamespacedName, memcached) + if err != nil { + t.Errorf("get memcached: (%v)", err) + } + + // Ensure Reconcile() updated the Memcached's Status as expected. + nodes := memcached.Status.Nodes + if !reflect.DeepEqual(podNames, nodes) { + t.Errorf("pod names %v did not match expected %v", nodes, podNames) + } +} diff --git a/test/test-framework/main_test.go b/test/test-framework/test/e2e/main_test.go similarity index 100% rename from test/test-framework/main_test.go rename to test/test-framework/test/e2e/main_test.go diff --git a/test/test-framework/memcached_test.go b/test/test-framework/test/e2e/memcached_test.go similarity index 100% rename from test/test-framework/memcached_test.go rename to test/test-framework/test/e2e/memcached_test.go From 40f7a97926cd222d318583db2fef9cd48da3c638 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Tue, 27 Nov 2018 16:37:10 -0500 Subject: [PATCH 04/11] test/test-framework: fix sanity tests --- test/test-framework/cmd/manager/main.go | 14 ++++++++++ .../pkg/controller/add_memcached.go | 14 ++++++++++ .../pkg/controller/controller.go | 14 ++++++++++ .../memcached/memcached_controller.go | 26 ++++++++++++++----- .../memcached/memcached_controller_test.go | 14 ++++++++++ 5 files changed, 76 insertions(+), 6 deletions(-) diff --git a/test/test-framework/cmd/manager/main.go b/test/test-framework/cmd/manager/main.go index f05eecb1f31..a7ea1309259 100644 --- a/test/test-framework/cmd/manager/main.go +++ b/test/test-framework/cmd/manager/main.go @@ -1,3 +1,17 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/test/test-framework/pkg/controller/add_memcached.go b/test/test-framework/pkg/controller/add_memcached.go index 4cee66e709d..4747af143a7 100644 --- a/test/test-framework/pkg/controller/add_memcached.go +++ b/test/test-framework/pkg/controller/add_memcached.go @@ -1,3 +1,17 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package controller import ( diff --git a/test/test-framework/pkg/controller/controller.go b/test/test-framework/pkg/controller/controller.go index 7c069f3ee6e..c678b7f9c68 100644 --- a/test/test-framework/pkg/controller/controller.go +++ b/test/test-framework/pkg/controller/controller.go @@ -1,3 +1,17 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package controller import ( diff --git a/test/test-framework/pkg/controller/memcached/memcached_controller.go b/test/test-framework/pkg/controller/memcached/memcached_controller.go index b6b6fd4ddac..098074540b2 100644 --- a/test/test-framework/pkg/controller/memcached/memcached_controller.go +++ b/test/test-framework/pkg/controller/memcached/memcached_controller.go @@ -1,3 +1,17 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package memcached import ( @@ -97,7 +111,7 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res return reconcile.Result{}, nil } // Error reading the object - requeue the request. - reqLogger.Error(err, "Failed to get Memcached.") + reqLogger.Error(err, "failed to get Memcached.") return reconcile.Result{}, err } @@ -110,13 +124,13 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res reqLogger.Info("Creating a new Deployment.", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name) err = r.client.Create(context.TODO(), dep) if err != nil { - reqLogger.Error(err, "Failed to create new Deployment.", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name) + reqLogger.Error(err, "failed to create new Deployment.", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name) return reconcile.Result{}, err } // Deployment created successfully - return and requeue return reconcile.Result{Requeue: true}, nil } else if err != nil { - reqLogger.Error(err, "Failed to get Deployment.") + reqLogger.Error(err, "failed to get Deployment.") return reconcile.Result{}, err } @@ -126,7 +140,7 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res found.Spec.Replicas = &size err = r.client.Update(context.TODO(), found) if err != nil { - reqLogger.Error(err, "Failed to update Deployment.", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name) + reqLogger.Error(err, "failed to update Deployment.", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name) return reconcile.Result{}, err } // Spec updated - return and requeue @@ -153,7 +167,7 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res } err = r.client.List(context.TODO(), listOps, podList) if err != nil { - reqLogger.Error(err, "Failed to list pods.", "Memcached.Namespace", memcached.Namespace, "Memcached.Name", memcached.Name) + reqLogger.Error(err, "failed to list pods.", "Memcached.Namespace", memcached.Namespace, "Memcached.Name", memcached.Name) return reconcile.Result{}, err } podNames := getPodNames(podList.Items) @@ -163,7 +177,7 @@ func (r *ReconcileMemcached) Reconcile(request reconcile.Request) (reconcile.Res memcached.Status.Nodes = podNames err := r.client.Update(context.TODO(), memcached) if err != nil { - reqLogger.Error(err, "Failed to update Memcached status.") + reqLogger.Error(err, "failed to update Memcached status.") return reconcile.Result{}, err } } diff --git a/test/test-framework/pkg/controller/memcached/memcached_controller_test.go b/test/test-framework/pkg/controller/memcached/memcached_controller_test.go index b2cce77f23a..b2428cc8a9b 100644 --- a/test/test-framework/pkg/controller/memcached/memcached_controller_test.go +++ b/test/test-framework/pkg/controller/memcached/memcached_controller_test.go @@ -1,3 +1,17 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package memcached import ( From b19c4d651937fde830794757ec2598a635db9cd1 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 10:48:39 -0800 Subject: [PATCH 05/11] pkg/test/main_entry.go: check an error --- pkg/test/main_entry.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/test/main_entry.go b/pkg/test/main_entry.go index a621203169d..85dd0458db4 100644 --- a/pkg/test/main_entry.go +++ b/pkg/test/main_entry.go @@ -102,7 +102,10 @@ func MainEntry(m *testing.M) { } exitCode := m.Run() if *localOperator { - localCmd.Process.Kill() + err := localCmd.Process.Kill() + if err != nil { + log.Warn("failed to stop local operator process") + } log.Infof("local operator stdout: %s", string(localCmdOutBuf.Bytes())) log.Infof("local operator stderr: %s", string(localCmdErrBuf.Bytes())) } From 5ea8dbc32356f2403a23514ac380b4803859bd83 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 11:11:01 -0800 Subject: [PATCH 06/11] pkg/test/main_entry.go: add a TODO --- pkg/test/main_entry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/test/main_entry.go b/pkg/test/main_entry.go index 85dd0458db4..a8552ab9552 100644 --- a/pkg/test/main_entry.go +++ b/pkg/test/main_entry.go @@ -63,6 +63,7 @@ func MainEntry(m *testing.M) { var localCmd *exec.Cmd var localCmdOutBuf, localCmdErrBuf bytes.Buffer if *localOperator { + // TODO: make a generic 'up-local' function to deduplicate shared code between this and cmd/up/local // taken from commands/operator-sdk/cmd/up/local.go runArgs := append([]string{"run"}, []string{filepath.Join(scaffold.ManagerDir, scaffold.CmdFile)}...) localCmd = exec.Command("go", runArgs...) From 34db11b4afc581b7e72996fd96f94c61f44e91dc Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 14:05:03 -0800 Subject: [PATCH 07/11] hack/tests/test-subcommand.sh: fix test --- hack/tests/test-subcommand.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/tests/test-subcommand.sh b/hack/tests/test-subcommand.sh index cb4eb7e4d02..55e173a2c4c 100755 --- a/hack/tests/test-subcommand.sh +++ b/hack/tests/test-subcommand.sh @@ -30,6 +30,6 @@ kubectl create -f deploy/service_account.yaml --namespace test-memcached kubectl create -f deploy/role.yaml --namespace test-memcached kubectl create -f deploy/role_binding.yaml --namespace test-memcached kubectl create -f deploy/operator.yaml --namespace test-memcached -operator-sdk test local . --namespace=test-memcached --no-setup +operator-sdk test local ./test/e2e --namespace=test-memcached --no-setup kubectl delete namespace test-memcached popd From 21fbdc1f3212539f2ee634c7a60d749c3b058e39 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 15:27:23 -0800 Subject: [PATCH 08/11] doc/test-framework: add more subsections to test local doc --- doc/test-framework/writing-e2e-tests.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/test-framework/writing-e2e-tests.md b/doc/test-framework/writing-e2e-tests.md index 6782257de5b..34fdbc2a1c3 100644 --- a/doc/test-framework/writing-e2e-tests.md +++ b/doc/test-framework/writing-e2e-tests.md @@ -230,7 +230,9 @@ default test settings, such as locations of your global resource manifest file ( `deploy/crd.yaml`) and your namespaced resource manifest file (by default `deploy/service_account.yaml` concatenated with `deploy/rbac.yaml` and `deploy/operator.yaml`), and allows the user to configure runtime options. There are 2 ways to use the subcommand: local and cluster. + ### Local + To run the tests locally, run the `operator-sdk test local` command in your project root and pass the location of the tests as an argument. You can use `--help` to view the other configuration options and use `--go-test-flags` to pass in arguments to `go test`. Here is an example command: @@ -238,6 +240,8 @@ as an argument. You can use `--help` to view the other configuration options and $ operator-sdk test local ./test/e2e --go-test-flags "-v -parallel=2" ``` +#### Image Flag + If you wish to specify a different operator image than specified in your `operator.yaml` file (or a user-specified namespaced manifest file), you can use the `--image` flag: @@ -245,6 +249,8 @@ namespaced manifest file), you can use the `--image` flag: $ operator-sdk test local ./test/e2e --image quay.io/example/my-operator:v0.0.2 ``` +#### Namespace Flag + If you wish to run all the tests in 1 namespace (which also forces `-parallel=1`), you can use the `--namespace` flag: ```shell @@ -252,6 +258,8 @@ $ kubectl create namespace operator-test $ operator-sdk test local ./test/e2e --namespace operator-test ``` +#### Up-Local Flag + To run the operator itself locally during the tests instead of starting a deployment in the cluster, you can use the `--up-local` flag. This mode will still create global resources, but by default will not create any in-cluster namespaced resources unless the user specifies one through the `--namespaced-manifest` flag. (Note: the `--up-local` flag requires @@ -262,6 +270,8 @@ $ kubectl create namespace operator-test $ operator-sdk test local ./test/e2e --namespace operator-test --up-local ``` +#### No-Setup Flag + If you would prefer to create the resources yourself and skip resource creation, you can use the `--no-setup` flag: ```shell $ kubectl create namespace operator-test @@ -275,6 +285,8 @@ $ operator-sdk test local ./test/e2e --namespace operator-test --no-setup For more documentation on the `operator-sdk test local` command, see the [SDK CLI Reference][sdk-cli-ref] doc. +#### Running Go Test Directly (Not Recommended) + For advanced use cases, it is possible to run the tests via `go test` directly. As long as all flags defined in [MainEntry][main-entry-link] are declared, the tests will run correctly. Running the tests directly with missing flags will result in undefined behavior. This is an example `go test` equivalent to the `operator-sdk test local` example above: From 1c97ddec67c4e129c449ffcc7b806cbf3181bc74 Mon Sep 17 00:00:00 2001 From: Haseeb Tariq Date: Thu, 29 Nov 2018 15:38:00 -0800 Subject: [PATCH 09/11] Update pkg/test/e2eutil/wait_util.go Co-Authored-By: AlexNPavel --- pkg/test/e2eutil/wait_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/test/e2eutil/wait_util.go b/pkg/test/e2eutil/wait_util.go index bbf44a81d0c..e0d0879a161 100644 --- a/pkg/test/e2eutil/wait_util.go +++ b/pkg/test/e2eutil/wait_util.go @@ -42,7 +42,7 @@ func WaitForOperatorDeployment(t *testing.T, kubeclient kubernetes.Interface, na func waitForDeployment(t *testing.T, kubeclient kubernetes.Interface, namespace, name string, replicas int, retryInterval, timeout time.Duration, isOperator bool) error { if isOperator && test.Global.LocalOperator { - t.Logf("Operator is running locally; skip waitForDeployment\n") + t.Log("Operator is running locally; skip waitForDeployment") return nil } err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { From f51a01029fb1e60f9a3ff8ff9221828a75fa5839 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 16:13:20 -0800 Subject: [PATCH 10/11] test/test-framework: update to latest managaer/main scaffold --- test/test-framework/cmd/manager/main.go | 65 ++++++++++++++++++------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/test/test-framework/cmd/manager/main.go b/test/test-framework/cmd/manager/main.go index a7ea1309259..bb9da753efc 100644 --- a/test/test-framework/cmd/manager/main.go +++ b/test/test-framework/cmd/manager/main.go @@ -15,65 +15,94 @@ package main import ( + "context" "flag" - "log" + "fmt" + "os" "runtime" - k8sutil "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/ready" "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis" "github.com/operator-framework/operator-sdk/test/test-framework/pkg/controller" sdkVersion "github.com/operator-framework/operator-sdk/version" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +var log = logf.Log.WithName("cmd") + func printVersion() { - log.Printf("Go Version: %s", runtime.Version()) - log.Printf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) - log.Printf("operator-sdk Version: %v", sdkVersion.Version) + log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) + log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)) + log.Info(fmt.Sprintf("operator-sdk Version: %v", sdkVersion.Version)) } func main() { - printVersion() flag.Parse() + // The logger instantiated here can be changed to any logger + // implementing the logr.Logger interface. This logger will + // be propagated through the whole operator, generating + // uniform and structured logs. + logf.SetLogger(logf.ZapLogger(false)) + + printVersion() + namespace, err := k8sutil.GetWatchNamespace() if err != nil { - log.Fatalf("failed to get watch namespace: %v", err) + log.Error(err, "failed to get watch namespace") + os.Exit(1) } - // TODO: Expose metrics port after SDK uses controller-runtime's dynamic client - // sdk.ExposeMetricsPort() - // Get a config to talk to the apiserver cfg, err := config.GetConfig() if err != nil { - log.Fatal(err) + log.Error(err, "") + os.Exit(1) + } + + // Become the leader before proceeding + leader.Become(context.TODO(), "memcached-operator-lock") + + r := ready.NewFileReady() + err = r.Set() + if err != nil { + log.Error(err, "") + os.Exit(1) } + defer r.Unset() // Create a new Cmd to provide shared dependencies and start components mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) if err != nil { - log.Fatal(err) + log.Error(err, "") + os.Exit(1) } - log.Print("Registering Components.") + log.Info("Registering Components.") // Setup Scheme for all resources if err := apis.AddToScheme(mgr.GetScheme()); err != nil { - log.Fatal(err) + log.Error(err, "") + os.Exit(1) } // Setup all Controllers if err := controller.AddToManager(mgr); err != nil { - log.Fatal(err) + log.Error(err, "") + os.Exit(1) } - log.Print("Starting the Cmd.") + log.Info("Starting the Cmd.") // Start the Cmd - log.Fatal(mgr.Start(signals.SetupSignalHandler())) + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + log.Error(err, "manager exited non-zero") + os.Exit(1) + } } From e30aee5fefbeb4aa0f9803f02e19c382c19f6f0a Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 29 Nov 2018 16:14:07 -0800 Subject: [PATCH 11/11] test/test-framework: remove unnecessary file --- .../memcached/memcached_controller_test.go | 140 ------------------ 1 file changed, 140 deletions(-) delete mode 100644 test/test-framework/pkg/controller/memcached/memcached_controller_test.go diff --git a/test/test-framework/pkg/controller/memcached/memcached_controller_test.go b/test/test-framework/pkg/controller/memcached/memcached_controller_test.go deleted file mode 100644 index b2428cc8a9b..00000000000 --- a/test/test-framework/pkg/controller/memcached/memcached_controller_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memcached - -import ( - "context" - "math/rand" - "reflect" - "strconv" - "testing" - - cachev1alpha1 "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis/cache/v1alpha1" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" -) - -// TestMemcachedController runs ReconcileMemcached.Reconcile() against a -// fake client that tracks a Memcached object. -func TestMemcachedController(t *testing.T) { - // Set the logger to development mode for verbose logs. - logf.SetLogger(logf.ZapLogger(true)) - - var ( - name = "memcached-operator" - namespace = "memcached" - replicas int32 = 3 - ) - - // A Memcached resource with metadata and spec. - memcached := &cachev1alpha1.Memcached{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: cachev1alpha1.MemcachedSpec{ - Size: replicas, // Set desired number of Memcached replicas. - }, - } - // Objects to track in the fake client. - objs := []runtime.Object{ - memcached, - } - - // Register operator types with the runtime scheme. - s := scheme.Scheme - s.AddKnownTypes(cachev1alpha1.SchemeGroupVersion, memcached) - // Create a fake client to mock API calls. - cl := fake.NewFakeClient(objs...) - // Create a ReconcileMemcached object with the scheme and fake client. - r := &ReconcileMemcached{client: cl, scheme: s} - - // Mock request to simulate Reconcile() being called on an event for a - // watched resource . - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: name, - Namespace: namespace, - }, - } - res, err := r.Reconcile(req) - if err != nil { - t.Fatalf("reconcile: (%v)", err) - } - // Check the result of reconciliation to make sure it has the desired state. - if !res.Requeue { - t.Error("reconcile did not requeue request as expected") - } - - // Check if deployment has been created and has the correct size. - dep := &appsv1.Deployment{} - err = cl.Get(context.TODO(), req.NamespacedName, dep) - if err != nil { - t.Fatalf("get deployment: (%v)", err) - } - dsize := *dep.Spec.Replicas - if dsize != replicas { - t.Errorf("dep size (%d) is not the expected size (%d)", dsize, replicas) - } - - // Create the 3 expected pods in namespace and collect their names to check - // later. - podLabels := labelsForMemcached(name) - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Labels: podLabels, - }, - } - podNames := make([]string, 3) - for i := 0; i < 3; i++ { - pod.ObjectMeta.Name = name + ".pod." + strconv.Itoa(rand.Int()) - podNames[i] = pod.ObjectMeta.Name - if err = cl.Create(context.TODO(), pod.DeepCopy()); err != nil { - t.Fatalf("create pod %d: (%v)", i, err) - } - } - - // Reconcile again so Reconcile() checks pods and updates the Memcached - // resources' Status. - res, err = r.Reconcile(req) - if err != nil { - t.Fatalf("reconcile: (%v)", err) - } - if res != (reconcile.Result{}) { - t.Error("reconcile did not return an empty Result") - } - - // Get the updated Memcached object. - memcached = &cachev1alpha1.Memcached{} - err = r.client.Get(context.TODO(), req.NamespacedName, memcached) - if err != nil { - t.Errorf("get memcached: (%v)", err) - } - - // Ensure Reconcile() updated the Memcached's Status as expected. - nodes := memcached.Status.Nodes - if !reflect.DeepEqual(podNames, nodes) { - t.Errorf("pod names %v did not match expected %v", nodes, podNames) - } -}