From 33f5f0fc47bd076d8e8c54d93b375e55beb08870 Mon Sep 17 00:00:00 2001 From: Matous Jobanek Date: Wed, 2 Jan 2019 10:00:41 +0000 Subject: [PATCH] fix(#710): when creation/update of a namespace fails,create new ones with new base-name (#714) --- .make/test.mk | 2 +- controller/tenant.go | 180 +++------------------------------- controller/tenants.go | 30 ++---- controller/tenants_test.go | 34 ++----- controller/update_test.go | 5 +- jsonapi/jsonapi_utility.go | 7 +- main.go | 5 +- openshift/apply.go | 9 +- openshift/clean_tenant.go | 34 +++++++ openshift/init_tenant.go | 170 ++++++++++++++++++++++++++++---- openshift/init_tenant_test.go | 124 +++++++++++++++++++++-- openshift/update.go | 15 ++- tenant/repository.go | 13 +-- tenant/repository_test.go | 47 ++++++++- test/update/executor.go | 6 +- update/update.go | 2 +- 16 files changed, 409 insertions(+), 274 deletions(-) diff --git a/.make/test.mk b/.make/test.mk index f4079e0..46b9b58 100644 --- a/.make/test.mk +++ b/.make/test.mk @@ -117,7 +117,7 @@ ALL_PKGS_EXCLUDE_PATTERN = 'vendor\|app\|tool\/cli\|design\|client\|test' GOANALYSIS_PKGS_EXCLUDE_PATTERN="vendor|app|client|tool/cli" GOANALYSIS_DIRS=$(shell go list -f {{.Dir}} ./... | grep -v -E $(GOANALYSIS_PKGS_EXCLUDE_PATTERN)) -TEST_FLAGS?=-v -p 1 -vet off +TEST_FLAGS ?= -v -p 1 -vet off MINISHIFT_URL ?= https://$(shell minishift ip):8443 MINISHIFT_USER_NAME ?= tenant.minishift.test.$(shell date +'%H.%M.%S') diff --git a/controller/tenant.go b/controller/tenant.go index 1ba2c82..9b6261f 100644 --- a/controller/tenant.go +++ b/controller/tenant.go @@ -3,10 +3,6 @@ package controller import ( "context" "fmt" - "net/http" - "sync/atomic" - "time" - "github.com/fabric8-services/fabric8-common/log" "github.com/fabric8-services/fabric8-tenant/app" "github.com/fabric8-services/fabric8-tenant/auth" @@ -22,7 +18,6 @@ import ( "github.com/goadesign/goa" goajwt "github.com/goadesign/goa/middleware/security/jwt" errs "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) // TenantController implements the status resource. @@ -112,13 +107,7 @@ func (c *TenantController) Setup(ctx *app.SetupTenantContext) error { go func() { ctx := ctx t := tenant - err = openshift.RawInitTenant( - ctx, - openshiftConfig, - InitTenant(ctx, openshiftConfig.MasterURL, c.tenantService, t), - user.OpenShiftUsername, - nsBaseName, - user.OpenShiftUserToken) + _, err = openshift.RawInitTenant(ctx, openshiftConfig, t, user.OpenShiftUserToken, c.tenantService, true) if err != nil { sentry.LogError(ctx, map[string]interface{}{ @@ -178,7 +167,7 @@ func (c *TenantController) Update(ctx *app.UpdateTenantContext) error { return jsonapi.JSONErrorResponse(ctx, errors.NewInternalError(ctx, fmt.Errorf("unable to update tenant configuration: %v", err))) } - err = UpdateTenant(&TenantUpdater{}, ctx, c.tenantService, openshiftConfig, tenant, env.DefaultEnvTypes...) + err = openshift.UpdateTenant(&TenantUpdater{}, ctx, c.tenantService, openshiftConfig, tenant, user.OpenShiftUserToken, true, env.DefaultEnvTypes...) if err != nil { log.Error(ctx, map[string]interface{}{ "err": err, @@ -192,59 +181,13 @@ func (c *TenantController) Update(ctx *app.UpdateTenantContext) error { return ctx.Accepted() } -func UpdateTenant(updateExecutor UpdateExecutor, ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, envTypes ...env.Type) error { - versionMapping, err := updateExecutor.Update(ctx, tenantService, openshiftConfig, t, envTypes) - if err != nil { - er := updateNamespaceEntities(tenantService, t, versionMapping, true) - if er != nil { - return fmt.Errorf("there occured two errors when doing update: \n1.[%s]\n2.[%s]", err, er) - } - return err - } - - return updateNamespaceEntities(tenantService, t, versionMapping, false) -} - -func updateNamespaceEntities(tenantService tenant.Service, t *tenant.Tenant, versionMapping map[env.Type]string, failed bool) error { - namespaces, err := tenantService.GetNamespaces(t.ID) - if err != nil { - return errs.Wrapf(err, "unable to get tenant namespaces") - } - var found bool - var nsVersion string - for _, ns := range namespaces { - if nsVersion, found = versionMapping[ns.Type]; found { - if failed { - ns.State = tenant.Failed - } else { - ns.State = tenant.Ready - ns.Version = nsVersion - } - ns.UpdatedBy = configuration.Commit - err := tenantService.SaveNamespace(ns) - if err != nil { - return errs.Wrapf(err, "unable to save tenant namespace %+v", ns) - } - } - } - return nil -} - -type UpdateExecutor interface { - Update(ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, envTypes []env.Type) (map[env.Type]string, error) -} - type TenantUpdater struct { } -func (u TenantUpdater) Update(ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, envTypes []env.Type) (map[env.Type]string, error) { - return openshift.RawUpdateTenant( - ctx, - openshiftConfig, - InitTenant(ctx, openshiftConfig.MasterURL, tenantService, t), - t.OSUsername, - t.NsBaseName, - envTypes) +func (u TenantUpdater) Update(ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, + envTypes []env.Type, usertoken string, allowSelfHealing bool) (map[env.Type]string, error) { + + return openshift.RawUpdateTenant(ctx, openshiftConfig, t, envTypes, usertoken, tenantService, allowSelfHealing) } // Clean runs the setup action for the tenant namespaces. @@ -294,7 +237,11 @@ func (c *TenantController) Clean(ctx *app.CleanTenantContext) error { return jsonapi.JSONErrorResponse(ctx, err) } if removeFromCluster { - err = c.tenantService.DeleteAll(ttoken.Subject()) + err = c.tenantService.DeleteNamespaces(ttoken.Subject()) + if err != nil { + return jsonapi.JSONErrorResponse(ctx, err) + } + err = c.tenantService.DeleteTenant(ttoken.Subject()) if err != nil { return jsonapi.JSONErrorResponse(ctx, err) } @@ -325,103 +272,6 @@ func (c *TenantController) Show(ctx *app.ShowTenantContext) error { return ctx.OK(result) } -// InitTenant is a Callback that assumes a new tenant is being created -func InitTenant(ctx context.Context, masterURL string, service tenant.Service, currentTenant *tenant.Tenant) openshift.Callback { - var maxResourceQuotaStatusCheck int32 = 50 // technically a global retry count across all ResourceQuota on all Tenant Namespaces - var currentResourceQuotaStatusCheck int32 // default is 0 - return func(statusCode int, method string, request, response map[interface{}]interface{}, versionMapping map[env.Type]string) (string, map[interface{}]interface{}) { - log.Info(ctx, map[string]interface{}{ - "status": statusCode, - "method": method, - "cluster_url": masterURL, - "namespace": env.GetNamespace(request), - "name": env.GetName(request), - "kind": env.GetKind(request), - "request": yamlString(request), - "response": yamlString(response), - }, "resource requested") - if statusCode == http.StatusConflict { - if env.GetKind(request) == env.ValKindNamespace { - return "", nil - } - if env.GetKind(request) == env.ValKindProjectRequest { - return "", nil - } - if env.GetKind(request) == env.ValKindPersistenceVolumeClaim { - return "", nil - } - if env.GetKind(request) == env.ValKindServiceAccount { - return "", nil - } - return "DELETE", request - } else if statusCode == http.StatusCreated { - if env.GetKind(request) == env.ValKindProjectRequest { - name := env.GetName(request) - envType := tenant.GetNamespaceType(name, currentTenant.NsBaseName) - templatesVersion := versionMapping[envType] - service.SaveNamespace(&tenant.Namespace{ - TenantID: currentTenant.ID, - Name: name, - State: tenant.Ready, - Version: templatesVersion, - Type: envType, - MasterURL: masterURL, - UpdatedBy: configuration.Commit, - }) - - // HACK to workaround osio applying some dsaas-user permissions async - // Should loop on a Check if allowed type of call instead - time.Sleep(time.Second * 5) - - } else if env.GetKind(request) == env.ValKindNamespace { - name := env.GetName(request) - envType := tenant.GetNamespaceType(name, currentTenant.NsBaseName) - templatesVersion := versionMapping[envType] - service.SaveNamespace(&tenant.Namespace{ - TenantID: currentTenant.ID, - Name: name, - State: tenant.Ready, - Version: templatesVersion, - Type: envType, - MasterURL: masterURL, - UpdatedBy: configuration.Commit, - }) - } else if env.GetKind(request) == env.ValKindResourceQuota { - // trigger a check status loop - time.Sleep(time.Millisecond * 50) - return "GET", response - } - return "", nil - } else if statusCode == http.StatusOK { - if method == "DELETE" { - return "POST", request - } else if method == "GET" { - if env.GetKind(request) == env.ValKindResourceQuota { - - if env.HasValidStatus(response) || atomic.LoadInt32(¤tResourceQuotaStatusCheck) >= maxResourceQuotaStatusCheck { - return "", nil - } - atomic.AddInt32(¤tResourceQuotaStatusCheck, 1) - time.Sleep(time.Millisecond * 50) - return "GET", response - } - } - return "", nil - } - log.Info(ctx, map[string]interface{}{ - "status": statusCode, - "method": method, - "namespace": env.GetNamespace(request), - "cluster_url": masterURL, - "name": env.GetName(request), - "kind": env.GetKind(request), - "request": yamlString(request), - "response": yamlString(response), - }, "unhandled resource response") - return "", nil - } -} - func convertTenant(ctx context.Context, tenant *tenant.Tenant, namespaces []*tenant.Namespace, resolveCluster cluster.GetCluster) *app.Tenant { result := app.Tenant{ ID: &tenant.ID, @@ -463,11 +313,3 @@ func convertTenant(ctx context.Context, tenant *tenant.Tenant, namespaces []*ten } return &result } - -func yamlString(data map[interface{}]interface{}) string { - b, err := yaml.Marshal(data) - if err != nil { - return fmt.Sprintf("Could not marshal yaml %v", data) - } - return string(b) -} diff --git a/controller/tenants.go b/controller/tenants.go index d1955b2..daa0395 100644 --- a/controller/tenants.go +++ b/controller/tenants.go @@ -5,6 +5,7 @@ import ( commonauth "github.com/fabric8-services/fabric8-common/auth" "github.com/fabric8-services/fabric8-common/errors" + errs "github.com/fabric8-services/fabric8-common/errors" "github.com/fabric8-services/fabric8-common/log" "github.com/fabric8-services/fabric8-tenant/app" "github.com/fabric8-services/fabric8-tenant/auth" @@ -21,7 +22,6 @@ var SERVICE_ACCOUNTS = []string{"fabric8-jenkins-idler", "rh-che"} type TenantsController struct { *goa.Controller tenantService tenant.Service - openshiftService openshift.Service clusterService cluster.Service authClientService auth.Service } @@ -31,13 +31,11 @@ func NewTenantsController(service *goa.Service, tenantService tenant.Service, clusterService cluster.Service, authClientService auth.Service, - openshiftService openshift.Service, ) *TenantsController { return &TenantsController{ Controller: service.NewController("TenantsController"), tenantService: tenantService, clusterService: clusterService, - openshiftService: openshiftService, authClientService: authClientService, } } @@ -100,38 +98,28 @@ func (c *TenantsController) Delete(ctx *app.DeleteTenantsContext) error { if err != nil { return jsonapi.JSONErrorResponse(ctx, err) } - for _, namespace := range namespaces { + if len(namespaces) > 0 { // fetch the cluster info - clustr, err := c.clusterService.GetCluster(ctx, namespace.MasterURL) + clustr, err := c.clusterService.GetCluster(ctx, namespaces[0].MasterURL) if err != nil { log.Error(ctx, map[string]interface{}{ "err": err, - "cluster_url": namespace.MasterURL, + "cluster_url": namespaces[0].MasterURL, "tenant_id": tenantID, }, "unable to fetch cluster for user") - return jsonapi.JSONErrorResponse(ctx, errors.NewInternalError(ctx, err)) + return jsonapi.JSONErrorResponse(ctx, errs.NewInternalError(ctx, err)) } - openshiftConfig := openshift.Config{ - MasterURL: namespace.MasterURL, + MasterURL: clustr.APIURL, Token: clustr.Token, } - log.Info(ctx, map[string]interface{}{"tenant_id": tenantID, "namespace": namespace.Name}, "deleting namespace...") - // delete the namespace in the cluster - err = c.openshiftService.DeleteNamespace(ctx, openshiftConfig, namespace.Name) + err = openshift.DeleteNamespaces(ctx, tenantID, openshiftConfig, c.tenantService) if err != nil { - log.Error(ctx, map[string]interface{}{ - "err": err, - "cluster_url": namespace.MasterURL, - "namespace": namespace.Name, - "tenant_id": tenantID, - }, "failed to delete namespace") - return jsonapi.JSONErrorResponse(ctx, err) + return err } - // then delete the corresponding record in the DB } // finally, delete the tenant record (all NS were already deleted, but that's fine) - err = c.tenantService.DeleteAll(tenantID) + err = c.tenantService.DeleteTenant(tenantID) if err != nil { return jsonapi.JSONErrorResponse(ctx, err) } diff --git a/controller/tenants_test.go b/controller/tenants_test.go index b95d570..0ea0199 100644 --- a/controller/tenants_test.go +++ b/controller/tenants_test.go @@ -11,13 +11,13 @@ import ( "github.com/fabric8-services/fabric8-tenant/cluster" "github.com/fabric8-services/fabric8-tenant/configuration" "github.com/fabric8-services/fabric8-tenant/controller" - "github.com/fabric8-services/fabric8-tenant/openshift" + "github.com/fabric8-services/fabric8-tenant/environment" "github.com/fabric8-services/fabric8-tenant/tenant" "github.com/fabric8-services/fabric8-tenant/test" "github.com/fabric8-services/fabric8-tenant/test/doubles" "github.com/fabric8-services/fabric8-tenant/test/gormsupport" "github.com/fabric8-services/fabric8-tenant/test/recorder" - "github.com/fabric8-services/fabric8-tenant/test/testfixture" + tf "github.com/fabric8-services/fabric8-tenant/test/testfixture" "github.com/goadesign/goa" goajwt "github.com/goadesign/goa/middleware/security/jwt" "github.com/satori/go.uuid" @@ -56,7 +56,7 @@ func (s *TenantsControllerTestSuite) TestShowTenants() { s.T().Run("OK", func(t *testing.T) { // given - fxt := testfixture.NewTestFixture(t, s.DB, testfixture.Tenants(1), testfixture.Namespaces(1)) + fxt := tf.NewTestFixture(t, s.DB, tf.Tenants(1), tf.Namespaces(1)) // when _, tenant := goatest.ShowTenantsOK(t, createValidSAContext("fabric8-jenkins-idler"), svc, ctrl, fxt.Tenants[0].ID) // then @@ -98,7 +98,7 @@ func (s *TenantsControllerTestSuite) TestSearchTenants() { s.T().Run("OK", func(t *testing.T) { // given - fxt := testfixture.NewTestFixture(t, s.DB, testfixture.Tenants(1), testfixture.Namespaces(1)) + fxt := tf.NewTestFixture(t, s.DB, tf.Tenants(1), tf.Namespaces(1)) // when _, tenant := goatest.SearchTenantsOK(t, createValidSAContext("fabric8-jenkins-idler"), svc, ctrl, fxt.Namespaces[0].MasterURL, fxt.Namespaces[0].Name) // then @@ -181,28 +181,7 @@ func (s *TenantsControllerTestSuite) TestFailedDeleteTenants() { svc, ctrl, reset := s.newTestTenantsController() defer reset() - fxt := testfixture.NewTestFixture(t, s.DB, testfixture.Tenants(1, func(fxt *testfixture.TestFixture, idx int) error { - id, err := uuid.FromString("5a95c51b-120a-4d03-b529-98bd7d4a5689") // force the ID to match the go-vcr cassette in the `delete-tenants.yaml` file - if err != nil { - return err - } - fxt.Tenants[0].ID = id - fxt.Tenants[0].OSUsername = "baz" - fxt.Tenants[0].NsBaseName = "baz" - return nil - }), testfixture.Namespaces(2, func(fxt *testfixture.TestFixture, idx int) error { - fxt.Namespaces[idx].TenantID = fxt.Tenants[0].ID - fxt.Namespaces[idx].MasterURL = "https://api.cluster1" - if idx == 0 { - fxt.Namespaces[idx].Name = "baz" - fxt.Namespaces[idx].Type = "user" - } else if idx == 1 { - fxt.Namespaces[idx].TenantID = fxt.Tenants[0].ID - fxt.Namespaces[idx].Name = "baz-che" - fxt.Namespaces[idx].Type = "che" - } - return nil - })) + fxt := tf.FillDB(t, s.DB, tf.AddTenantsNamed("baz"), true, tf.AddNamespaces(environment.TypeUser, environment.TypeChe).State(tenant.Ready)) // when goatest.DeleteTenantsInternalServerError(t, createValidSAContext("fabric8-auth"), svc, ctrl, fxt.Tenants[0].ID) @@ -266,8 +245,7 @@ func prepareConfigClusterAndAuthService(t *testing.T) (cluster.Service, auth.Ser } func (s *TenantsControllerTestSuite) newTestTenantsController() (*goa.Service, *controller.TenantsController, func()) { clusterService, authService, _, reset := prepareConfigClusterAndAuthService(s.T()) - openshiftService := openshift.NewService() svc := goa.New("Tenants-service") - ctrl := controller.NewTenantsController(svc, tenant.NewDBService(s.DB), clusterService, authService, openshiftService) + ctrl := controller.NewTenantsController(svc, tenant.NewDBService(s.DB), clusterService, authService) return svc, ctrl, reset } diff --git a/controller/update_test.go b/controller/update_test.go index 8b73bdc..4ec327d 100644 --- a/controller/update_test.go +++ b/controller/update_test.go @@ -308,7 +308,7 @@ func (s *UpdateControllerTestSuite) TestStopUpdateOk() { goatest.StartUpdateAccepted(s.T(), createValidSAContext("fabric8-tenant-update"), svc, ctrl, nil, nil) // then - test.WaitWithTimeout(5 * time.Second).Until(func() error { + err := test.WaitWithTimeout(5 * time.Second).Until(func() error { var tenantsUpdate *update.TenantsUpdate testupdate.Tx(s.T(), s.DB, func(repo update.Repository) error { var err error @@ -320,12 +320,13 @@ func (s *UpdateControllerTestSuite) TestStopUpdateOk() { } return nil }) + require.NoError(s.T(), err) testupdate.Tx(s.T(), s.DB, func(repo update.Repository) error { return repo.Stop() }) var tenantsUpdate *update.TenantsUpdate - err := test.WaitWithTimeout(10 * time.Second).Until(func() error { + err = test.WaitWithTimeout(10 * time.Second).Until(func() error { err := update.Transaction(s.DB, func(tx *gorm.DB) error { var err error tenantsUpdate, err = update.NewRepository(tx).GetTenantsUpdate() diff --git a/jsonapi/jsonapi_utility.go b/jsonapi/jsonapi_utility.go index 53819b7..df699ea 100644 --- a/jsonapi/jsonapi_utility.go +++ b/jsonapi/jsonapi_utility.go @@ -142,8 +142,13 @@ type Conflict interface { // JSONErrorResponse auto maps the provided error to the correct response type // If all else fails, InternalServerError is returned func JSONErrorResponse(obj interface{}, err error) error { - x := obj.(InternalServerError) c := obj.(context.Context) + x, ok := obj.(InternalServerError) + + if !ok { + sentry.Sentry().CaptureError(c, err) + return errs.WithStack(errors.NewInternalError(c, err)) + } jsonErr, status := ErrorToJSONAPIErrors(c, err) switch status { diff --git a/main.go b/main.go index 13a63f2..a4ff472 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,6 @@ import ( "github.com/fabric8-services/fabric8-tenant/environment" "github.com/fabric8-services/fabric8-tenant/jsonapi" "github.com/fabric8-services/fabric8-tenant/migration" - "github.com/fabric8-services/fabric8-tenant/openshift" "github.com/fabric8-services/fabric8-tenant/sentry" "github.com/fabric8-services/fabric8-tenant/tenant" "github.com/fabric8-services/fabric8-tenant/toggles" @@ -104,8 +103,6 @@ func main() { } defer clusterService.Stop() - openshiftService := openshift.NewService() - haltSentry, err := sentry.InitializeLogger(config, configuration.Commit) if err != nil { log.Panic(nil, map[string]interface{}{ @@ -132,7 +129,7 @@ func main() { tenantCtrl := controller.NewTenantController(service, tenantService, clusterService, authService, config) app.MountTenantController(service, tenantCtrl) - tenantsCtrl := controller.NewTenantsController(service, tenantService, clusterService, authService, openshiftService) + tenantsCtrl := controller.NewTenantsController(service, tenantService, clusterService, authService) app.MountTenantsController(service, tenantsCtrl) // Mount "update" controller diff --git a/openshift/apply.go b/openshift/apply.go index 4996c77..8f11d53 100644 --- a/openshift/apply.go +++ b/openshift/apply.go @@ -120,8 +120,8 @@ metadata: ) // Callback is called after initial action -type Callback func(statusCode int, method string, request, response map[interface{}]interface{}, versionMapping map[env.Type]string) (string, map[interface{}]interface{}) -type CallbackWithVersionMapping func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}) +type Callback func(statusCode int, method string, request, response map[interface{}]interface{}, versionMapping map[env.Type]string) (string, map[interface{}]interface{}, error) +type CallbackWithVersionMapping func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}, error) // ApplyOptions contains options for connecting to the target API type ApplyOptions struct { @@ -229,7 +229,10 @@ func Apply(object env.Object, action string, opts ApplyOptions) (env.Object, err } if opts.Callback != nil { - act, newObject := opts.Callback(resp.StatusCode, action, object, respType) + act, newObject, err := opts.Callback(resp.StatusCode, action, object, respType) + if err != nil { + return nil, err + } if act != "" { return Apply(newObject, act, opts) } diff --git a/openshift/clean_tenant.go b/openshift/clean_tenant.go index 6081c99..e4f04d4 100644 --- a/openshift/clean_tenant.go +++ b/openshift/clean_tenant.go @@ -5,10 +5,14 @@ import ( "fmt" "sync" + errs "github.com/fabric8-services/fabric8-common/errors" "github.com/fabric8-services/fabric8-common/log" "github.com/fabric8-services/fabric8-tenant/environment" + "github.com/fabric8-services/fabric8-tenant/jsonapi" "github.com/fabric8-services/fabric8-tenant/sentry" + "github.com/fabric8-services/fabric8-tenant/tenant" "github.com/pkg/errors" + "github.com/satori/go.uuid" ) // CleanTenant clean or remove @@ -76,3 +80,33 @@ func executeCleanNamespaceCMD(namespace string, opt ApplyOptions) (string, error func executeDeleteNamespaceCMD(namespace string, opt ApplyOptions) (string, error) { return executeCMD(nil, []string{"-c", fmt.Sprintf("oc delete project %v --server=%v --token=%v", namespace, opt.MasterURL, opt.Token)}) } + +func DeleteNamespaces(ctx context.Context, tenantID uuid.UUID, openshiftConfig Config, tenantService tenant.Service) error { + namespaces, err := tenantService.GetNamespaces(tenantID) + if err != nil { + return jsonapi.JSONErrorResponse(ctx, err) + } + for _, namespace := range namespaces { + log.Info(ctx, map[string]interface{}{"tenant_id": tenantID, "namespace": namespace.Name}, "deleting namespace...") + // delete the namespace in the cluster + openshiftService := NewService() + err = openshiftService.DeleteNamespace(ctx, openshiftConfig, namespace.Name) + if err != nil { + log.Error(ctx, map[string]interface{}{ + "err": err, + "cluster_url": namespace.MasterURL, + "namespace": namespace.Name, + "tenant_id": tenantID, + }, "failed to delete namespace") + return jsonapi.JSONErrorResponse(ctx, err) + } + } + if err := tenantService.DeleteNamespaces(tenantID); err != nil { + log.Error(ctx, map[string]interface{}{ + "err": err, + "tenant_id": tenantID, + }, "failed to delete namespaces entities from DB") + return jsonapi.JSONErrorResponse(ctx, errs.NewInternalError(ctx, err)) + } + return nil +} diff --git a/openshift/init_tenant.go b/openshift/init_tenant.go index 43f8ab9..9538029 100644 --- a/openshift/init_tenant.go +++ b/openshift/init_tenant.go @@ -7,34 +7,40 @@ import ( "fmt" "github.com/fabric8-services/fabric8-common/log" + "github.com/fabric8-services/fabric8-tenant/configuration" env "github.com/fabric8-services/fabric8-tenant/environment" "github.com/fabric8-services/fabric8-tenant/sentry" "github.com/fabric8-services/fabric8-tenant/tenant" "github.com/pkg/errors" "gopkg.in/yaml.v2" + "net/http" + "sync/atomic" + "time" ) -func RawInitTenant(ctx context.Context, config Config, callback Callback, openshiftUsername, nsBaseName, usertoken string) error { - templs, versionMapping, err := LoadProcessedTemplates(ctx, config, openshiftUsername, nsBaseName, env.DefaultEnvTypes) +func RawInitTenant(ctx context.Context, config Config, tnnt *tenant.Tenant, usertoken string, repo tenant.Service, allowSelfHealing bool) (map[env.Type]string, error) { + + templs, versionMapping, err := LoadProcessedTemplates(ctx, config, tnnt.OSUsername, tnnt.NsBaseName, env.DefaultEnvTypes) if err != nil { - return err + return versionMapping, err } mapped, err := MapByNamespaceAndSort(templs) if err != nil { - return err + return versionMapping, err } - callbackWithVersionMapping := func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}) { - return callback(statusCode, method, request, response, versionMapping) + callbackWithVersionMapping := func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}, error) { + return InitTenant(ctx, config.MasterURL, repo, tnnt)(statusCode, method, request, response, versionMapping) } masterOpts := ApplyOptions{Config: config, Callback: callbackWithVersionMapping} userOpts := ApplyOptions{Config: config.WithToken(usertoken), Callback: callbackWithVersionMapping} var wg sync.WaitGroup wg.Add(len(mapped)) + errorChan := make(chan error, len(mapped)) for key, val := range mapped { - namespaceType := tenant.GetNamespaceType(key, nsBaseName) + namespaceType := tenant.GetNamespaceType(key, tnnt.NsBaseName) if namespaceType == env.TypeUser { go func(namespace string, objects env.Objects, opts, userOpts ApplyOptions) { @@ -44,6 +50,8 @@ func RawInitTenant(ctx context.Context, config Config, callback Callback, opensh sentry.LogError(ctx, map[string]interface{}{ "namespace": namespace, }, err, "error init user project, ProjectRequest") + errorChan <- err + return } err = ApplyProcessed(Filter(objects, IsOfKind(env.ValKindRoleBindingRestriction)), opts) if err != nil { @@ -56,12 +64,14 @@ func RawInitTenant(ctx context.Context, config Config, callback Callback, opensh sentry.LogError(ctx, map[string]interface{}{ "namespace": namespace, }, err, "error init user project, Other") + errorChan <- err + return } _, err = Apply( CreateAdminRoleBinding(namespace), "DELETE", opts.WithCallback( - func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}) { + func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}, error) { log.Info(ctx, map[string]interface{}{ "status": statusCode, "method": method, @@ -69,7 +79,7 @@ func RawInitTenant(ctx context.Context, config Config, callback Callback, opensh "name": env.GetName(request), "kind": env.GetKind(request), }, "resource requested") - return "", nil + return "", nil, nil }, ), ) @@ -87,24 +97,29 @@ func RawInitTenant(ctx context.Context, config Config, callback Callback, opensh sentry.LogError(ctx, map[string]interface{}{ "namespace": namespace, }, err, "error dsaas project") + errorChan <- err + return } }(key, val, masterOpts) } } wg.Wait() - return nil + vm, err := handleErrors(errorChan, "creation", ctx, config, tnnt, usertoken, repo, allowSelfHealing) + if vm != nil { + return vm, err + } + return versionMapping, err } -func RawUpdateTenant(ctx context.Context, config Config, callback Callback, osUsername, nsBaseName string, envTypes []env.Type) (map[env.Type]string, error) { - templs, versionMapping, err := LoadProcessedTemplates(ctx, config, osUsername, nsBaseName, envTypes) +func RawUpdateTenant(ctx context.Context, config Config, tnnt *tenant.Tenant, envTypes []env.Type, usertoken string, + repo tenant.Service, allowSelfHealing bool) (map[env.Type]string, error) { + + templs, versionMapping, err := LoadProcessedTemplates(ctx, config, tnnt.OSUsername, tnnt.NsBaseName, envTypes) if err != nil { return versionMapping, err } - callbackWithVersionMapping := func(statusCode int, method string, request, response map[interface{}]interface{}) (string, map[interface{}]interface{}) { - return callback(statusCode, method, request, response, versionMapping) - } - masterOpts := ApplyOptions{Config: config, Callback: callbackWithVersionMapping} + masterOpts := ApplyOptions{Config: config} mapped, err := MapByNamespaceAndSort(templs) if err != nil { @@ -141,6 +156,16 @@ func RawUpdateTenant(ctx context.Context, config Config, callback Callback, osUs }(key, val, masterOpts) } wg.Wait() + vm, err := handleErrors(errorChan, "update", ctx, config, tnnt, usertoken, repo, allowSelfHealing) + if vm != nil { + return vm, err + } + return versionMapping, err +} + +func handleErrors(errorChan chan error, action string, ctx context.Context, config Config, tnnt *tenant.Tenant, + usertoken string, repo tenant.Service, allowSelfHealing bool) (map[env.Type]string, error) { + var errs []error close(errorChan) for er := range errorChan { @@ -149,9 +174,25 @@ func RawUpdateTenant(ctx context.Context, config Config, callback Callback, osUs } } if len(errs) > 0 { - return versionMapping, fmt.Errorf("update of namespaces failed with one or more errors %s", errs) + log.Error(ctx, map[string]interface{}{ + "errs": errs, + }, "%s of namespaces failed with one or more errors", action) + if allowSelfHealing && usertoken != "" { + err := DeleteNamespaces(ctx, tnnt.ID, config, repo) + if err != nil { + return nil, errors.Wrapf(err, "unable to cleanup namespaces in order to create a new ones") + } + newNsBaseName, err := tenant.ConstructNsBaseName(repo, env.RetrieveUserName(tnnt.OSUsername)) + if err != nil { + return nil, errors.Wrapf(err, "unable to construct namespace base name for user wit OSname %s", tnnt.OSUsername) + } + tnnt.NsBaseName = newNsBaseName + repo.SaveTenant(tnnt) + return RawInitTenant(ctx, config, tnnt, usertoken, repo, false) + } + return nil, fmt.Errorf("%s of namespaces failed with one or more errors %s", action, errs) } - return versionMapping, nil + return nil, nil } func listToTemplate(objects env.Objects) string { @@ -164,3 +205,96 @@ func listToTemplate(objects env.Objects) string { b, _ := yaml.Marshal(template) return string(b) } + +// InitTenant is a Callback that assumes a new tenant is being created +func InitTenant(ctx context.Context, masterURL string, service tenant.Service, currentTenant *tenant.Tenant) Callback { + var maxResourceQuotaStatusCheck int32 = 50 // technically a global retry count across all ResourceQuota on all Tenant Namespaces + var currentResourceQuotaStatusCheck int32 // default is 0 + return func(statusCode int, method string, request, response map[interface{}]interface{}, versionMapping map[env.Type]string) (string, map[interface{}]interface{}, error) { + log.Info(ctx, map[string]interface{}{ + "status": statusCode, + "method": method, + "cluster_url": masterURL, + "namespace": env.GetNamespace(request), + "name": env.GetName(request), + "kind": env.GetKind(request), + "request": yamlString(request), + "response": yamlString(response), + }, "resource requested") + if statusCode == http.StatusConflict || statusCode == http.StatusForbidden { + return "", nil, fmt.Errorf("unable to create - should create with other base-name") + } else if statusCode == http.StatusCreated { + if env.GetKind(request) == env.ValKindProjectRequest { + name := env.GetName(request) + envType := tenant.GetNamespaceType(name, currentTenant.NsBaseName) + templatesVersion := versionMapping[envType] + service.SaveNamespace(&tenant.Namespace{ + TenantID: currentTenant.ID, + Name: name, + State: tenant.Ready, + Version: templatesVersion, + Type: envType, + MasterURL: masterURL, + UpdatedBy: configuration.Commit, + }) + + // HACK to workaround osio applying some dsaas-user permissions async + // Should loop on a Check if allowed type of call instead + time.Sleep(time.Second * 5) + + } else if env.GetKind(request) == env.ValKindNamespace { + name := env.GetName(request) + envType := tenant.GetNamespaceType(name, currentTenant.NsBaseName) + templatesVersion := versionMapping[envType] + service.SaveNamespace(&tenant.Namespace{ + TenantID: currentTenant.ID, + Name: name, + State: tenant.Ready, + Version: templatesVersion, + Type: envType, + MasterURL: masterURL, + UpdatedBy: configuration.Commit, + }) + } else if env.GetKind(request) == env.ValKindResourceQuota { + // trigger a check status loop + time.Sleep(time.Millisecond * 50) + return "GET", response, nil + } + return "", nil, nil + } else if statusCode == http.StatusOK { + if method == "DELETE" { + return "POST", request, nil + } else if method == "GET" { + if env.GetKind(request) == env.ValKindResourceQuota { + + if env.HasValidStatus(response) || atomic.LoadInt32(¤tResourceQuotaStatusCheck) >= maxResourceQuotaStatusCheck { + return "", nil, nil + } + atomic.AddInt32(¤tResourceQuotaStatusCheck, 1) + time.Sleep(time.Millisecond * 50) + return "GET", response, nil + } + } + return "", nil, nil + } + log.Info(ctx, map[string]interface{}{ + "status": statusCode, + "method": method, + "namespace": env.GetNamespace(request), + "cluster_url": masterURL, + "name": env.GetName(request), + "kind": env.GetKind(request), + "request": yamlString(request), + "response": yamlString(response), + }, "unhandled resource response") + return "", nil, nil + } +} + +func yamlString(data map[interface{}]interface{}) string { + b, err := yaml.Marshal(data) + if err != nil { + return fmt.Sprintf("Could not marshal yaml %v", data) + } + return string(b) +} diff --git a/openshift/init_tenant_test.go b/openshift/init_tenant_test.go index 81daf93..c424231 100644 --- a/openshift/init_tenant_test.go +++ b/openshift/init_tenant_test.go @@ -5,21 +5,34 @@ import ( "github.com/fabric8-services/fabric8-tenant/auth/client" "github.com/fabric8-services/fabric8-tenant/environment" "github.com/fabric8-services/fabric8-tenant/openshift" + "github.com/fabric8-services/fabric8-tenant/tenant" "github.com/fabric8-services/fabric8-tenant/test" + "github.com/fabric8-services/fabric8-tenant/test/gormsupport" + tf "github.com/fabric8-services/fabric8-tenant/test/testfixture" + "github.com/satori/go.uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "gopkg.in/h2non/gock.v1" "net/http" "testing" ) +type InitTenantTestSuite struct { + gormsupport.DBTestSuite +} + +func TestInitTenant(t *testing.T) { + suite.Run(t, &InitTenantTestSuite{DBTestSuite: gormsupport.NewDBTestSuite("../config.yaml")}) +} + var emptyCallback = func(statusCode int, method string, request, response map[interface{}]interface{}, versionMapping map[environment.Type]string) (string, map[interface{}]interface{}) { return "", nil } -func TestNumberOfCallsToCluster(t *testing.T) { +func (s *InitTenantTestSuite) TestNumberOfCallsToCluster() { // given - data, reset := test.LoadTestConfig(t) + data, reset := test.LoadTestConfig(s.T()) defer func() { gock.OffAll() reset() @@ -35,13 +48,112 @@ func TestNumberOfCallsToCluster(t *testing.T) { user := &client.UserDataAttributes{} config := openshift.NewConfigForUser(data, user, "clusterUser", "clusterToken", "http://my.cluster") config.HTTPTransport = http.DefaultTransport - objectsInTemplates := tmplObjects(t, data) + objectsInTemplates := tmplObjects(s.T(), data) + + tnnt := &tenant.Tenant{ID: uuid.NewV4(), OSUsername: "developer", NsBaseName: "developer"} + + // when + _, err := openshift.RawInitTenant(context.Background(), config, tnnt, "12345", tenant.NewDBService(s.DB), true) + + // then + require.NoError(s.T(), err) + // the number of calls should be equal to the number of parsed objects plus one call that removes admin role from user's namespace + assert.Equal(s.T(), len(objectsInTemplates)+1, calls) +} + +func (s *InitTenantTestSuite) TestCreateNewNamespacesWithBaseNameEnding2WhenFails() { + // given + data, reset := test.LoadTestConfig(s.T()) + defer func() { + gock.OffAll() + reset() + }() + fxt := tf.FillDB(s.T(), s.DB, tf.AddTenantsNamed("johndoe"), true, tf.AddDefaultNamespaces().State(tenant.Provisioning)) + johndoeCalls := 0 + projectRequestCalls := 0 + deleteCalls := 0 + johndoe2Calls := 2 + + gock.New("http://api.cluster1"). + Post("/api/v1/namespaces/johndoe-jenkins/persistentvolumeclaims"). + Reply(409). + BodyString("{}") + gock.New("http://api.cluster1"). + Delete("/apis/project.openshift.io/v1/projects/.*"). + SetMatcher(test.SpyOnCalls(&deleteCalls)). + Times(5). + Reply(200). + BodyString("{}") + gock.New("http://api.cluster1"). + Path(`.*johndoe2.*`). + SetMatcher(test.SpyOnCalls(&johndoe2Calls)). + Persist(). + Reply(200). + BodyString("{}") + gock.New("http://api.cluster1"). + Path(`.*johndoe[^2].*`). + SetMatcher(test.SpyOnCalls(&johndoeCalls)). + Persist(). + Reply(200). + BodyString("{}") + gock.New("http://api.cluster1"). + Post("/oapi/v1/projectrequests"). + SetMatcher(test.SpyOnCalls(&projectRequestCalls)). + Times(10). + Reply(200). + BodyString("{}") + + user := &client.UserDataAttributes{} + config := openshift.NewConfigForUser(data, user, "clusterUser", "clusterToken", "http://api.cluster1/") + config.HTTPTransport = http.DefaultTransport + objsNumber := len(tmplObjects(s.T(), data)) + repo := tenant.NewDBService(s.DB) + + // when + _, err := openshift.RawInitTenant(context.Background(), config, fxt.Tenants[0], "12345", repo, true) + + // then + require.NoError(s.T(), err) + // the number of calls should be equal to the number of parsed objects plus one call that removes admin role from user's namespace + assert.Equal(s.T(), objsNumber-10, johndoeCalls) + assert.Equal(s.T(), 10, projectRequestCalls) + assert.Equal(s.T(), 5, deleteCalls) + assert.Equal(s.T(), objsNumber-2, johndoe2Calls) + updatedTnnt, err := repo.GetTenant(fxt.Tenants[0].ID) + require.NoError(s.T(), err) + assert.Equal(s.T(), "johndoe2", updatedTnnt.NsBaseName) +} + +func (s *InitTenantTestSuite) TestCreateNewNamespacesWithBaseNameEnding3WhenFailsAnd2Exists() { + // given + data, reset := test.LoadTestConfig(s.T()) + defer func() { + gock.OffAll() + reset() + }() + fxt := tf.FillDB(s.T(), s.DB, tf.AddTenantsNamed("johndoe", "johndoe2"), true, tf.AddDefaultNamespaces().State(tenant.Provisioning)) + + gock.New("http://api.cluster1"). + Post("/api/v1/namespaces/johndoe-jenkins/persistentvolumeclaims"). + Reply(403). + BodyString("{}") + gock.New("http://api.cluster1"). + Persist(). + Reply(200). + BodyString("{}") + + user := &client.UserDataAttributes{} + config := openshift.NewConfigForUser(data, user, "clusterUser", "clusterToken", "http://api.cluster1/") + config.HTTPTransport = http.DefaultTransport + repo := tenant.NewDBService(s.DB) // when - err := openshift.RawInitTenant(context.Background(), config, emptyCallback, "developer", "developer", "12345") + _, err := openshift.RawInitTenant(context.Background(), config, fxt.Tenants[0], "12345", repo, true) // then - require.NoError(t, err) + require.NoError(s.T(), err) // the number of calls should be equal to the number of parsed objects plus one call that removes admin role from user's namespace - assert.Equal(t, len(objectsInTemplates)+1, calls) + updatedTnnt, err := repo.GetTenant(fxt.Tenants[0].ID) + require.NoError(s.T(), err) + assert.Equal(s.T(), "johndoe3", updatedTnnt.NsBaseName) } diff --git a/openshift/update.go b/openshift/update.go index ef85eb2..4557cd6 100644 --- a/openshift/update.go +++ b/openshift/update.go @@ -2,6 +2,7 @@ package openshift import ( "context" + "fmt" "github.com/fabric8-services/fabric8-tenant/configuration" "github.com/fabric8-services/fabric8-tenant/environment" "github.com/fabric8-services/fabric8-tenant/tenant" @@ -9,13 +10,19 @@ import ( ) type UpdateExecutor interface { - Update(ctx context.Context, tenantService tenant.Service, openshiftConfig Config, t *tenant.Tenant, envTypes []environment.Type) (map[environment.Type]string, error) + Update(ctx context.Context, tenantService tenant.Service, openshiftConfig Config, t *tenant.Tenant, + envTypes []environment.Type, usertoken string, allowSelfHealing bool) (map[environment.Type]string, error) } -func UpdateTenant(updateExecutor UpdateExecutor, ctx context.Context, tenantService tenant.Service, openshiftConfig Config, t *tenant.Tenant, envTypes ...environment.Type) error { - versionMapping, err := updateExecutor.Update(ctx, tenantService, openshiftConfig, t, envTypes) +func UpdateTenant(updateExecutor UpdateExecutor, ctx context.Context, tenantService tenant.Service, openshiftConfig Config, + t *tenant.Tenant, usertoken string, allowSelfHealing bool, envTypes ...environment.Type) error { + + versionMapping, err := updateExecutor.Update(ctx, tenantService, openshiftConfig, t, envTypes, usertoken, allowSelfHealing) if err != nil { - updateNamespaceEntities(tenantService, t, versionMapping, true) + er := updateNamespaceEntities(tenantService, t, versionMapping, true) + if er != nil { + return fmt.Errorf("there occured two errors when doing update: \n1.[%s]\n2.[%s]", err, er) + } return err } diff --git a/tenant/repository.go b/tenant/repository.go index 2ba7cc6..f256817 100644 --- a/tenant/repository.go +++ b/tenant/repository.go @@ -21,7 +21,8 @@ type Service interface { // SaveTenant will update on dupliate 'insert' SaveTenant(tenant *Tenant) error SaveNamespace(namespace *Namespace) error - DeleteAll(tenantID uuid.UUID) error + DeleteTenant(tenantID uuid.UUID) error + DeleteNamespaces(tenantID uuid.UUID) error NamespaceExists(nsName string) (bool, error) ExistsWithNsBaseName(nsBaseName string) (bool, error) GetTenantsToUpdate(typeWithVersion map[environment.Type]string, count int, commit string, masterURL string) ([]*Tenant, error) @@ -147,20 +148,14 @@ func (s DBService) GetTenantsToUpdate(typeWithVersion map[environment.Type]strin return tenants, err } -func (s DBService) DeleteAll(tenantID uuid.UUID) error { - err := s.deleteNamespaces(tenantID) - err = s.deleteTenant(tenantID) - return err -} - -func (s DBService) deleteNamespaces(tenantID uuid.UUID) error { +func (s DBService) DeleteNamespaces(tenantID uuid.UUID) error { if tenantID == uuid.Nil { return nil } return s.db.Unscoped().Delete(&Namespace{}, "tenant_id = ?", tenantID).Error } -func (s DBService) deleteTenant(tenantID uuid.UUID) error { +func (s DBService) DeleteTenant(tenantID uuid.UUID) error { if tenantID == uuid.Nil { return nil } diff --git a/tenant/repository_test.go b/tenant/repository_test.go index b542a22..fdd5c01 100644 --- a/tenant/repository_test.go +++ b/tenant/repository_test.go @@ -322,7 +322,7 @@ func (s *TenantServiceTestSuite) TestGetSubsetOfTenantsThatMatchesRequiredCluste }) } -func (s *TenantServiceTestSuite) TestDelete() { +func (s *TenantServiceTestSuite) TestDeleteNamespaces() { s.T().Run("all info", func(t *testing.T) { // given fxt := tf.NewTestFixture(t, s.DB, tf.Tenants(2), tf.Namespaces(10, func(fxt *tf.TestFixture, idx int) error { @@ -337,15 +337,52 @@ func (s *TenantServiceTestSuite) TestDelete() { tenant1 := fxt.Tenants[0] tenant2 := fxt.Tenants[1] // when - svc.DeleteAll(tenant1.ID) + svc.DeleteNamespaces(tenant1.ID) // then // should be deleted - ten1, _ := svc.GetTenant(tenant1.ID) - require.Nil(t, ten1) - ns1, _ := svc.GetNamespaces(tenant1.ID) + ns1, err := svc.GetNamespaces(tenant1.ID) + require.NoError(t, err) require.Len(t, ns1, 0) // should not be deleted + ten1, err := svc.GetTenant(tenant1.ID) + require.NoError(t, err) + require.NotNil(t, ten1) + ten2, err := svc.GetTenant(tenant2.ID) + require.NotNil(t, ten2) + require.NoError(t, err) + ns2, err := svc.GetNamespaces(tenant2.ID) + require.NoError(t, err) + require.Len(t, ns2, 5) + }) +} + +func (s *TenantServiceTestSuite) TestDeleteTenant() { + s.T().Run("all info", func(t *testing.T) { + // given + fxt := tf.NewTestFixture(t, s.DB, tf.Tenants(2), tf.Namespaces(10, func(fxt *tf.TestFixture, idx int) error { + if idx < 5 { + fxt.Namespaces[idx].TenantID = fxt.Tenants[0].ID + } else { + fxt.Namespaces[idx].TenantID = fxt.Tenants[1].ID + } + return nil + })) + svc := tenant.NewDBService(s.DB) + tenant1 := fxt.Tenants[0] + tenant2 := fxt.Tenants[1] + // when + svc.DeleteTenant(tenant1.ID) + // then + // should be deleted + ten1, err := svc.GetTenant(tenant1.ID) + test.AssertError(t, err) + require.Nil(t, ten1) + + // should not be deleted + ns1, err := svc.GetNamespaces(tenant1.ID) + require.NoError(t, err) + require.Len(t, ns1, 5) ten2, err := svc.GetTenant(tenant2.ID) require.NotNil(t, ten2) require.NoError(t, err) diff --git a/test/update/executor.go b/test/update/executor.go index f4de9bd..6783321 100644 --- a/test/update/executor.go +++ b/test/update/executor.go @@ -26,7 +26,9 @@ func NewDummyUpdateExecutor() *DummyUpdateExecutor { return &DummyUpdateExecutor{NumberOfCalls: ptr.Uint64(0)} } -func (e *DummyUpdateExecutor) Update(ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, envTypes []environment.Type) (map[environment.Type]string, error) { +func (e *DummyUpdateExecutor) Update(ctx context.Context, tenantService tenant.Service, openshiftConfig openshift.Config, t *tenant.Tenant, + envTypes []environment.Type, usertoken string, allowSelfHealing bool) (map[environment.Type]string, error) { + atomic.AddUint64(e.NumberOfCalls, 1) time.Sleep(e.TimeToSleep) @@ -34,7 +36,7 @@ func (e *DummyUpdateExecutor) Update(ctx context.Context, tenantService tenant.S e.waitGroup.Wait() } if e.ShouldCallOriginalUpdater { - return controller.TenantUpdater{}.Update(ctx, tenantService, openshiftConfig, t, envTypes) + return controller.TenantUpdater{}.Update(ctx, tenantService, openshiftConfig, t, envTypes, usertoken, allowSelfHealing) } if e.ShouldFail { return testdoubles.GetMappedVersions(envTypes...), fmt.Errorf("failing") diff --git a/update/update.go b/update/update.go index b003a97..50b052d 100644 --- a/update/update.go +++ b/update/update.go @@ -341,7 +341,7 @@ func updateTenant(wg *sync.WaitGroup, tnnt *tenant.Tenant, tenantRepo tenant.Ser }, "starting update of tenant for outdated namespace") osConfig := openshift.NewConfig(updater.config, emptyTemplateRepoInfoSetter, userCluster.User, userCluster.Token, userCluster.APIURL) - err = openshift.UpdateTenant(updater.updateExecutor, nil, tenantRepo, osConfig, tnnt, envType) + err = openshift.UpdateTenant(updater.updateExecutor, nil, tenantRepo, osConfig, tnnt, "", false, envTypes...) if err != nil { err = Transaction(updater.db, lock(func(repo Repository) error { return repo.IncrementFailedCount()