From b734d71dee81029d70b9c2a7298b97fd5889f8c3 Mon Sep 17 00:00:00 2001 From: anotherchrisberry Date: Wed, 13 Sep 2017 14:31:55 -0700 Subject: [PATCH] fix(canary): target canary cleanup server groups by create date --- .../orca/mine/pipeline/CanaryStage.groovy | 61 ++++++++++----- .../orca/mine/pipeline/CanaryStageSpec.groovy | 75 +++++++++++++++++++ 2 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStageSpec.groovy diff --git a/orca-mine/src/main/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStage.groovy b/orca-mine/src/main/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStage.groovy index 4e39c93cef..c3da202617 100644 --- a/orca-mine/src/main/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStage.groovy +++ b/orca-mine/src/main/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStage.groovy @@ -16,10 +16,13 @@ package com.netflix.spinnaker.orca.mine.pipeline +import com.netflix.spinnaker.orca.CancellableStage.Result +import com.netflix.spinnaker.orca.clouddriver.tasks.servergroup.DestroyServerGroupTask +import com.netflix.spinnaker.orca.clouddriver.utils.OortHelper + import java.util.concurrent.TimeUnit import com.netflix.frigga.autoscaling.AutoScalingGroupNameBuilder import com.netflix.spinnaker.orca.CancellableStage -import com.netflix.spinnaker.orca.clouddriver.tasks.cluster.ShrinkClusterTask import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder import com.netflix.spinnaker.orca.pipeline.model.Execution import com.netflix.spinnaker.orca.pipeline.model.Stage @@ -35,7 +38,8 @@ class CanaryStage implements StageDefinitionBuilder, CancellableStage { @Autowired DeployCanaryStage deployCanaryStage @Autowired MonitorCanaryStage monitorCanaryStage - @Autowired ShrinkClusterTask shrinkClusterTask + @Autowired DestroyServerGroupTask destroyServerGroupTask + @Autowired OortHelper oortHelper @Override def > List> aroundStages(Stage stage) { @@ -55,13 +59,18 @@ class CanaryStage implements StageDefinitionBuilder, CancellableStage { } @Override - CancellableStage.Result cancel(Stage stage) { + Result cancel(Stage stage) { log.info("Cancelling stage (stageId: ${stage.id}, executionId: ${stage.execution.id}, context: ${stage.context as Map})") // it's possible the server groups haven't been created yet, allow a grace period before cleanup Thread.sleep(TimeUnit.MINUTES.toMillis(2)) - Collection> shrinkContexts = [] + return cleanupCanary(stage) + } + + protected Result cleanupCanary(Stage stage) { + Collection> destroyContexts = [] + stage.context.clusterPairs.each { Map clusterPair -> [clusterPair.baseline, clusterPair.canary].each { Map cluster -> @@ -70,28 +79,40 @@ class CanaryStage implements StageDefinitionBuilder, CancellableStage { builder.stack = cluster.stack builder.detail = cluster.freeFormDetails - String region = cluster.region ?: (cluster.availabilityZones as Map).keySet().first() + Map deployedCluster = oortHelper.getCluster(cluster.application, cluster.account, builder.buildGroupName(), cluster.cloudProvider ?: 'aws').orElse([:]) + Long start = stage.startTime + // add a small buffer to deal with latency between the cloud provider and Orca + Long createdTimeCutoff = stage.endTime + 5000 + + List serverGroups = deployedCluster.serverGroups ?: [] + + String clusterRegion = cluster.region ?: (cluster.availabilityZones as Map).keySet().first() + List matches = serverGroups.findAll { + it.region == clusterRegion && it.createdTime > start && it.createdTime < createdTimeCutoff + } ?: [] - shrinkContexts << [ - cluster : builder.buildGroupName(), - region : region, - shrinkToSize : 0, - allowDeleteActive: true, - credentials : cluster.account, - cloudProvider : cluster.cloudProvider ?: 'aws' - ] + // really hope they're not running concurrent canaries in the same cluster + matches.each { + destroyContexts << [ + serverGroupName: it.name, + region : it.region, + credentials : cluster.account, + cloudProvider : it.cloudProvider ?: 'aws' + ] + } } } - def shrinkResults = shrinkContexts.collect { - def shrinkStage = new Stage<>() - shrinkStage.context.putAll(it) - shrinkClusterTask.execute(shrinkStage) + def destroyResults = destroyContexts.collect { + def destroyStage = new Stage<>() + destroyStage.execution = stage.execution + destroyStage.context.putAll(it) + destroyServerGroupTask.execute(destroyStage) } - return new CancellableStage.Result(stage, [ - shrinkContexts: shrinkContexts, - shrinkResults : shrinkResults + return new Result(stage, [ + destroyContexts: destroyContexts, + destroyResults : destroyResults ]) } } diff --git a/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStageSpec.groovy b/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStageSpec.groovy new file mode 100644 index 0000000000..b0d8a0cc20 --- /dev/null +++ b/orca-mine/src/test/groovy/com/netflix/spinnaker/orca/mine/pipeline/CanaryStageSpec.groovy @@ -0,0 +1,75 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.mine.pipeline + +import com.netflix.spinnaker.orca.CancellableStage.Result +import com.netflix.spinnaker.orca.clouddriver.tasks.servergroup.DestroyServerGroupTask +import com.netflix.spinnaker.orca.clouddriver.utils.OortHelper +import com.netflix.spinnaker.orca.pipeline.model.Stage +import spock.lang.Specification +import spock.lang.Unroll +import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage + +class CanaryStageSpec extends Specification { + + @Unroll + void "cancel destroys canary/baseline if found and were deployed during canary stage"() { + given: + Map stageContext = [ + clusterPairs: [ + [ + baseline: [application: "app", stack: "stack1", freeFormDetails: "baseline", region: "us-east-1", account: "test"], + canary: [application: "app", stack: "stack1", freeFormDetails: "canary", region: "us-east-1", account: "test"] + ] + ] + ] + + Stage canceledStage = stage { + context = stageContext + startTime = 5 + endTime = 10 + } + + OortHelper oortHelper = Mock(OortHelper) + DestroyServerGroupTask destroyServerGroupTask = Mock(DestroyServerGroupTask) + + CanaryStage canaryStage = new CanaryStage(oortHelper: oortHelper, destroyServerGroupTask: destroyServerGroupTask) + + when: + Result result = canaryStage.cleanupCanary(canceledStage) + + then: + result.details.destroyContexts.size() == destroyedServerGroups + 1 * oortHelper.getCluster("app", "test", "app-stack1-baseline", "aws") >> [ + serverGroups: [[region: "us-east-1", createdTime: createdTime, name: "app-stack1-baseline-v003"]] + ] + 1 * oortHelper.getCluster("app", "test", "app-stack1-canary", "aws") >> [ + serverGroups: [[region: "us-east-1", createdTime: createdTime, name: "app-stack1-canary-v003"]] + ] + + where: + createdTime || destroyedServerGroups + 4 || 0 + 5 || 0 + 6 || 2 + 10 || 2 + 5010 || 0 + 5011 || 0 + + } + +}