From 873ca27f1f156bca577b2a05acfbcc75b60cc62e Mon Sep 17 00:00:00 2001 From: brujoand Date: Thu, 26 Oct 2017 08:15:17 +0200 Subject: [PATCH] feat(orca) Place produced artifacts in stage output --- .../spinnaker/orca/bakery/api/Bake.groovy | 2 + .../orca/bakery/pipeline/BakeStage.groovy | 8 ++-- .../bakery/tasks/CompletedBakeTask.groovy | 4 +- .../bakery/tasks/CompletedBakeTaskSpec.groovy | 7 +++- .../cluster/FindImageFromClusterTask.groovy | 40 ++++++++++++++++++- .../tasks/image/FindImageFromTagsTask.java | 34 +++++++++++++++- .../clouddriver/tasks/image/ImageFinder.java | 1 + .../providers/aws/AmazonImageFinder.java | 5 +++ .../providers/gce/GoogleImageFinder.java | 5 +++ .../image/FindImageFromTagTaskSpec.groovy | 11 +++-- 10 files changed, 106 insertions(+), 11 deletions(-) diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy index 78407e52719..a9b67e56cf9 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy @@ -18,6 +18,7 @@ package com.netflix.spinnaker.orca.bakery.api +import com.netflix.spinnaker.kork.artifacts.model.Artifact import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString @@ -35,5 +36,6 @@ class Bake { String ami String amiName String imageName + Artifact artifact // TODO(duftler): Add a cloudProviderType property here? Will be straightforward once rosco is backed by redis. } diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy index acc06da7f76..4cb45d8b6a2 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy @@ -127,10 +127,12 @@ class BakeStage implements StageDefinitionBuilder, RestartableStage { it.parentStageId == stage.parentStageId && it.status == ExecutionStatus.RUNNING } + def relatedBakeStages = stage.execution.stages.findAll { + it.type == PIPELINE_CONFIG_TYPE && bakeInitializationStages*.id.contains(it.parentStageId) + } + def globalContext = [ - deploymentDetails: stage.execution.stages.findAll { - it.type == PIPELINE_CONFIG_TYPE && bakeInitializationStages*.id.contains(it.parentStageId) && (it.context.ami || it.context.imageId) - }.collect { Stage bakeStage -> + deploymentDetails: relatedBakeStages.findAll{it.context.ami || it.context.imageId}.collect { Stage bakeStage -> def deploymentDetails = [:] ["ami", "imageId", "amiSuffix", "baseLabel", "baseOs", "refId", "storeType", "vmType", "region", "package", "cloudProviderType", "cloudProvider"].each { if (bakeStage.context.containsKey(it)) { diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy index ae58e9779a3..a2f586fba87 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy @@ -16,6 +16,7 @@ package com.netflix.spinnaker.orca.bakery.tasks +import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.orca.ExecutionStatus import com.netflix.spinnaker.orca.Task import com.netflix.spinnaker.orca.TaskResult @@ -39,7 +40,7 @@ class CompletedBakeTask implements Task { def bakeStatus = stage.context.status as BakeStatus def bake = bakery.lookupBake(region, bakeStatus.resourceId).toBlocking().first() // This treatment of ami allows both the aws and gce bake results to be propagated. - def results = [ami: bake.ami ?: bake.imageName, imageId: bake.ami ?: bake.imageName] + def results = [ami: bake.ami ?: bake.imageName, imageId: bake.ami ?: bake.imageName, artifact: bake.artifact ?: new Artifact()] /** * TODO: * It would be good to standardize on the key here. "imageId" works for all providers. @@ -49,6 +50,7 @@ class CompletedBakeTask implements Task { if (bake.imageName || bake.amiName) { results.imageName = bake.imageName ?: bake.amiName } + new TaskResult(ExecutionStatus.SUCCEEDED, results) } } diff --git a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy index 03914aa54b6..5a4dfde6047 100644 --- a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy +++ b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy @@ -16,6 +16,7 @@ package com.netflix.spinnaker.orca.bakery.tasks +import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.orca.ExecutionStatus import com.netflix.spinnaker.orca.bakery.api.Bake import com.netflix.spinnaker.orca.bakery.api.BakeStatus @@ -44,10 +45,10 @@ class CompletedBakeTaskSpec extends Specification { null ) - def "finds the AMI created by a bake"() { + def "finds the AMI and artifact created by a bake"() { given: task.bakery = Stub(BakeryService) { - lookupBake(region, bakeId) >> Observable.from(new Bake(id: bakeId, ami: ami)) + lookupBake(region, bakeId) >> Observable.from(new Bake(id: bakeId, ami: ami, artifact: artifact)) } and: @@ -59,11 +60,13 @@ class CompletedBakeTaskSpec extends Specification { then: result.status == ExecutionStatus.SUCCEEDED result.context.ami == ami + result.context.artifact.reference == ami where: region = "us-west-1" bakeId = "b-5af233wjj78mwt2f420wt8ey3w" ami = "ami-280c3b6d" + artifact = new Artifact(reference: ami) } def "fails if the bake is not found"() { diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy index 05c308a384f..1e65d84e97b 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy @@ -19,6 +19,7 @@ package com.netflix.spinnaker.orca.clouddriver.tasks.cluster import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.frigga.Names +import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.moniker.Moniker import com.netflix.spinnaker.orca.ExecutionStatus import com.netflix.spinnaker.orca.RetryableTask @@ -229,8 +230,45 @@ class FindImageFromClusterTask extends AbstractCloudProviderAwareTask implements } }.flatten() + List artifacts = imageSummaries.collect { placement, summaries -> + Artifact artifact = new Artifact() + summaries.findResults { summary -> + if (config.imageNamePattern && !(summary.imageName ==~ config.imageNamePattern)) { + return null + } + def location = "global" + + if (placement.type == Location.Type.REGION) { + location = placement.value + } else if (placement.type == Location.Type.ZONE) { + location = placement.value + } + + def metadata = [ + sourceServerGroup: summary.serverGroupName, + refId: stage.refId + ] + + try { + metadata.putAll(summary.image ?: [:]) + metadata.putAll(summary.buildInfo ?: [:]) + } catch (Exception e) { + log.error("Unable to merge server group image/build info (summary: ${summary})", e) + } + + artifact.metadata = metadata + artifact.name = summary.imageName + artifact.location = location + artifact.type = "${cloudProvider}/image" + artifact.reference = "${summary.imageId}" + artifact.uuid = UUID.randomUUID().toString() + } + return artifact + }.flatten() + return new TaskResult(ExecutionStatus.SUCCEEDED, [ - amiDetails: deploymentDetails + amiDetails: deploymentDetails, + artifacts: artifacts ], [ deploymentDetails: deploymentDetails ]) diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java index 0c0b98ef2f3..7f0a583c83e 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java @@ -16,12 +16,17 @@ package com.netflix.spinnaker.orca.clouddriver.tasks.image; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.orca.ExecutionStatus; import com.netflix.spinnaker.orca.RetryableTask; import com.netflix.spinnaker.orca.TaskResult; @@ -53,13 +58,40 @@ public TaskResult execute(Stage stage) { throw new IllegalStateException("Could not find tagged image for package: " + stageData.packageName + " and tags: " + stageData.tags); } + List artifacts = new ArrayList<>(); + imageDetails.forEach(imageDetail -> artifacts.add(generateArtifactFrom(imageDetail, cloudProvider))); + + Map stageOutputs = new HashMap<>(); + stageOutputs.put("amiDetails", imageDetails); + stageOutputs.put("artifacts", artifacts); + return new TaskResult( ExecutionStatus.SUCCEEDED, - Collections.singletonMap("amiDetails", imageDetails), + stageOutputs, Collections.singletonMap("deploymentDetails", imageDetails) ); + } + + private Artifact generateArtifactFrom(ImageFinder.ImageDetails imageDetails, String cloudProvider) { + Map metadata = new HashMap<>(); + try { + ImageFinder.JenkinsDetails jenkinsDetails = imageDetails.getJenkins(); + metadata.put("build_info_url", jenkinsDetails.get("host")); + metadata.put("build_number", jenkinsDetails.get("number")); + } catch (Exception e) { + // This is either all or nothing + } + + Artifact artifact = new Artifact(); + artifact.setName(imageDetails.getImageName()); + artifact.setReference(imageDetails.getImageId()); + artifact.setLocation(imageDetails.getRegion()); + artifact.setType(cloudProvider + "/image"); + artifact.setMetadata(metadata); + artifact.setUuid(UUID.randomUUID().toString()); + return artifact; } @Override diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java index 412a1393b6e..38b23ba8adc 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java @@ -29,6 +29,7 @@ public interface ImageFinder { interface ImageDetails { String getImageId(); String getImageName(); + String getRegion(); JenkinsDetails getJenkins(); } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java index dca92aa740b..ba10ad93a34 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java @@ -144,6 +144,11 @@ public String getImageName() { return (String) super.get("imageName"); } + @Override + public String getRegion() { + return (String) super.get("region"); + } + @Override public JenkinsDetails getJenkins() { return (JenkinsDetails) super.get("jenkins"); diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java index 72e01c969be..2b3b340d288 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java @@ -132,6 +132,11 @@ public String getImageName() { return (String) super.get("imageName"); } + @Override + public String getRegion() { + return (String) super.get("region"); + } + @Override public JenkinsDetails getJenkins() { return (JenkinsDetails) super.get("jenkins"); diff --git a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy b/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy index 183ca6e8342..ed69aa14c50 100644 --- a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy +++ b/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy @@ -9,12 +9,13 @@ import spock.lang.Subject class FindImageFromTagTaskSpec extends Specification { def imageFinder = Mock(ImageFinder) + def imageDetails = Mock(ImageFinder.ImageDetails) Stage stage = new Stage<>(new Pipeline("orca"), "", [packageName: 'myPackage', tags: ['foo': 'bar']]) @Subject def task = new FindImageFromTagsTask(imageFinders: [imageFinder]) - def "Not finding an images should throw IllegalStateException"() { + def "Not finding images should throw IllegalStateException"() { when: task.execute(stage) @@ -25,7 +26,7 @@ class FindImageFromTagTaskSpec extends Specification { 1 * imageFinder.getCloudProvider() >> 'aws' } - def "Finding an images should set task state to SUCCEEDED"() { + def "Finding images should set task state to SUCCEEDED"() { when: def result = task.execute(stage) @@ -33,6 +34,10 @@ class FindImageFromTagTaskSpec extends Specification { result.status == ExecutionStatus.SUCCEEDED 1 * imageFinder.getCloudProvider() >> 'aws' - 1 * imageFinder.byTags(stage, stage.context.packageName, stage.context.tags) >> [[:]] + 1 * imageFinder.byTags(stage, stage.context.packageName, stage.context.tags) >> [imageDetails] + 1 * imageDetails.getImageName() >> "somename" + 1 * imageDetails.getImageId() >> "someId" + 1 * imageDetails.getJenkins() >> new ImageFinder.JenkinsDetails("somehost", "somename", "42") } + }