From 834758bacea631c4ea5f19af26f7b09b41206e48 Mon Sep 17 00:00:00 2001 From: brujoand Date: Thu, 12 Oct 2017 14:40:49 +0200 Subject: [PATCH] feat(orca) Place produced artifacts in global context --- .../spinnaker/orca/bakery/api/Bake.groovy | 2 + .../orca/bakery/pipeline/BakeStage.groovy | 11 +- .../bakery/tasks/CompletedBakeTask.groovy | 4 +- .../orca/bakery/pipeline/BakeStageSpec.groovy | 28 +++++ .../bakery/tasks/CompletedBakeTaskSpec.groovy | 7 +- .../cluster/FindImageFromClusterTask.groovy | 38 +++++- .../tasks/image/FindImageFromTagsTask.java | 30 ++++- .../clouddriver/tasks/image/ImageFinder.java | 1 + .../providers/aws/AmazonImageFinder.java | 5 + .../providers/gce/GoogleImageFinder.java | 5 + .../image/FindImageFromTagTaskSpec.groovy | 11 +- .../orca/pipeline/model/Artifact.java | 115 ++++++++++++++++++ 12 files changed, 246 insertions(+), 11 deletions(-) create mode 100644 orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Artifact.java diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy index 78407e52719..9300a618954 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/Bake.groovy @@ -18,6 +18,7 @@ package com.netflix.spinnaker.orca.bakery.api +import com.netflix.spinnaker.orca.pipeline.model.Artifact import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString @@ -35,5 +36,6 @@ class Bake { String ami String amiName String imageName + Artifact artifact // TODO(duftler): Add a cloudProviderType property here? Will be straightforward once rosco is backed by redis. } diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy index acc06da7f76..4531ae287fc 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStage.groovy @@ -127,10 +127,12 @@ class BakeStage implements StageDefinitionBuilder, RestartableStage { it.parentStageId == stage.parentStageId && it.status == ExecutionStatus.RUNNING } + def relatedBakeStages = stage.execution.stages.findAll { + it.type == PIPELINE_CONFIG_TYPE && bakeInitializationStages*.id.contains(it.parentStageId) + } + def globalContext = [ - deploymentDetails: stage.execution.stages.findAll { - it.type == PIPELINE_CONFIG_TYPE && bakeInitializationStages*.id.contains(it.parentStageId) && (it.context.ami || it.context.imageId) - }.collect { Stage bakeStage -> + deploymentDetails: relatedBakeStages.findAll{it.context.ami || it.context.imageId}.collect { Stage bakeStage -> def deploymentDetails = [:] ["ami", "imageId", "amiSuffix", "baseLabel", "baseOs", "refId", "storeType", "vmType", "region", "package", "cloudProviderType", "cloudProvider"].each { if (bakeStage.context.containsKey(it)) { @@ -139,6 +141,9 @@ class BakeStage implements StageDefinitionBuilder, RestartableStage { } return deploymentDetails + }, + artifacts: relatedBakeStages.findAll{it.context.artifact}.collect { Stage bakeStage -> + return bakeStage.context.get("artifact") ?: [:] } ] new TaskResult(ExecutionStatus.SUCCEEDED, [:], globalContext) diff --git a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy index ae58e9779a3..5cc4ab42437 100644 --- a/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy +++ b/orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTask.groovy @@ -21,6 +21,7 @@ import com.netflix.spinnaker.orca.Task import com.netflix.spinnaker.orca.TaskResult import com.netflix.spinnaker.orca.bakery.api.BakeStatus import com.netflix.spinnaker.orca.bakery.api.BakeryService +import com.netflix.spinnaker.orca.pipeline.model.Artifact import com.netflix.spinnaker.orca.pipeline.model.Stage import groovy.transform.CompileStatic import org.springframework.beans.factory.annotation.Autowired @@ -39,7 +40,7 @@ class CompletedBakeTask implements Task { def bakeStatus = stage.context.status as BakeStatus def bake = bakery.lookupBake(region, bakeStatus.resourceId).toBlocking().first() // This treatment of ami allows both the aws and gce bake results to be propagated. - def results = [ami: bake.ami ?: bake.imageName, imageId: bake.ami ?: bake.imageName] + def results = [ami: bake.ami ?: bake.imageName, imageId: bake.ami ?: bake.imageName, artifact: bake.artifact ?: new Artifact()] /** * TODO: * It would be good to standardize on the key here. "imageId" works for all providers. @@ -49,6 +50,7 @@ class CompletedBakeTask implements Task { if (bake.imageName || bake.amiName) { results.imageName = bake.imageName ?: bake.amiName } + new TaskResult(ExecutionStatus.SUCCEEDED, results) } } diff --git a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStageSpec.groovy b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStageSpec.groovy index 510b3b27d3c..0db8ec58cd6 100644 --- a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStageSpec.groovy +++ b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/pipeline/BakeStageSpec.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.orca.bakery.pipeline import com.netflix.spinnaker.orca.ExecutionStatus +import com.netflix.spinnaker.orca.pipeline.model.Artifact import com.netflix.spinnaker.orca.pipeline.model.Stage import groovy.time.TimeCategory import spock.lang.Specification @@ -108,6 +109,33 @@ class BakeStageSpec extends Specification { } } + def "should include provided artifacts"() { + given: + def pipeline = pipeline { + stage { + id = "1" + type = "bake" + context = [ + "region": "global", + ] + status = ExecutionStatus.RUNNING + } + } + + def bakeStage = pipeline.stageById("1") + def parallelStages = new BakeStage().parallelStages(bakeStage) + parallelStages.each { it.context.artifact = new Artifact(name: "myArtifact") } + pipeline.stages.addAll(parallelStages) + + when: + def taskResult = new BakeStage.CompleteParallelBakeTask().execute(pipeline.stageById("1")) + + then: + with(taskResult.outputs) { + artifacts[0].name == "myArtifact" + } + } + private static List deployAz(String cloudProvider, String prefix, String... regions) { if (prefix == "clusters") { diff --git a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy index 03914aa54b6..547b2514f49 100644 --- a/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy +++ b/orca-bakery/src/test/groovy/com/netflix/spinnaker/orca/bakery/tasks/CompletedBakeTaskSpec.groovy @@ -20,6 +20,7 @@ import com.netflix.spinnaker.orca.ExecutionStatus import com.netflix.spinnaker.orca.bakery.api.Bake import com.netflix.spinnaker.orca.bakery.api.BakeStatus import com.netflix.spinnaker.orca.bakery.api.BakeryService +import com.netflix.spinnaker.orca.pipeline.model.Artifact import com.netflix.spinnaker.orca.pipeline.model.Pipeline import com.netflix.spinnaker.orca.pipeline.model.Stage import retrofit.RetrofitError @@ -44,10 +45,10 @@ class CompletedBakeTaskSpec extends Specification { null ) - def "finds the AMI created by a bake"() { + def "finds the AMI and artifact created by a bake"() { given: task.bakery = Stub(BakeryService) { - lookupBake(region, bakeId) >> Observable.from(new Bake(id: bakeId, ami: ami)) + lookupBake(region, bakeId) >> Observable.from(new Bake(id: bakeId, ami: ami, artifact: artifact)) } and: @@ -59,11 +60,13 @@ class CompletedBakeTaskSpec extends Specification { then: result.status == ExecutionStatus.SUCCEEDED result.context.ami == ami + result.context.artifact.reference == ami where: region = "us-west-1" bakeId = "b-5af233wjj78mwt2f420wt8ey3w" ami = "ami-280c3b6d" + artifact = new Artifact(reference: ami) } def "fails if the bake is not found"() { diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy index 05c308a384f..7715da9e449 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/FindImageFromClusterTask.groovy @@ -26,6 +26,7 @@ import com.netflix.spinnaker.orca.TaskResult import com.netflix.spinnaker.orca.clouddriver.OortService import com.netflix.spinnaker.orca.clouddriver.pipeline.servergroup.support.Location import com.netflix.spinnaker.orca.clouddriver.tasks.AbstractCloudProviderAwareTask +import com.netflix.spinnaker.orca.pipeline.model.Artifact import com.netflix.spinnaker.orca.pipeline.model.Stage import groovy.transform.Canonical import groovy.util.logging.Slf4j @@ -229,10 +230,45 @@ class FindImageFromClusterTask extends AbstractCloudProviderAwareTask implements } }.flatten() + List artifacts = imageSummaries.collect { placement, summaries -> + Artifact artifact = new Artifact() + summaries.findResults { summary -> + if (config.imageNamePattern && !(summary.imageName ==~ config.imageNamePattern)) { + return null + } + def location = "global" + + if (placement.type == Location.Type.REGION) { + location = placement.value + } else if (placement.type == Location.Type.ZONE) { + location = placement.value + } + + def metadata = [ + sourceServerGroup: summary.serverGroupName, + refId: stage.refId + ] + + try { + metadata.putAll(summary.image ?: [:]) + metadata.putAll(summary.buildInfo ?: [:]) + } catch (Exception e) { + log.error("Unable to merge server group image/build info (summary: ${summary})", e) + } + + artifact.metadata = metadata + artifact.name = summary.imageName + artifact.type = "${cloudProvider}/image/${location}" + artifact.reference = "${summary.imageId}" + } + return artifact + }.flatten() + return new TaskResult(ExecutionStatus.SUCCEEDED, [ amiDetails: deploymentDetails ], [ - deploymentDetails: deploymentDetails + deploymentDetails: deploymentDetails, + artifacts: artifacts ]) } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java index 0c0b98ef2f3..e41acbcf7f1 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagsTask.java @@ -16,8 +16,10 @@ package com.netflix.spinnaker.orca.clouddriver.tasks.image; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonProperty; @@ -26,6 +28,7 @@ import com.netflix.spinnaker.orca.RetryableTask; import com.netflix.spinnaker.orca.TaskResult; import com.netflix.spinnaker.orca.clouddriver.tasks.AbstractCloudProviderAwareTask; +import com.netflix.spinnaker.orca.pipeline.model.Artifact; import com.netflix.spinnaker.orca.pipeline.model.Stage; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -53,13 +56,38 @@ public TaskResult execute(Stage stage) { throw new IllegalStateException("Could not find tagged image for package: " + stageData.packageName + " and tags: " + stageData.tags); } + List artifacts = new ArrayList<>(); + imageDetails.forEach(imageDetail -> artifacts.add(generateArtifactFrom(imageDetail, cloudProvider))); + + Map globalOutputs = new HashMap<>(); + globalOutputs.put("deploymentDetails", imageDetails); + globalOutputs.put("artifacts", artifacts); + return new TaskResult( ExecutionStatus.SUCCEEDED, Collections.singletonMap("amiDetails", imageDetails), - Collections.singletonMap("deploymentDetails", imageDetails) + globalOutputs ); + } + + private Artifact generateArtifactFrom(ImageFinder.ImageDetails imageDetails, String cloudProvider) { + Map metadata = new HashMap<>(); + try { + ImageFinder.JenkinsDetails jenkinsDetails = imageDetails.getJenkins(); + metadata.put("build_info_url", jenkinsDetails.get("host")); + metadata.put("build_number", jenkinsDetails.get("number")); + } catch (Exception e) { + // This is either all or nothing + } + + Artifact artifact = new Artifact(); + artifact.setName(imageDetails.getImageName()); + artifact.setReference(imageDetails.getImageId()); + artifact.setType(cloudProvider + "/image/" + imageDetails.getRegion()); + artifact.setMetadata(metadata); + return artifact; } @Override diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java index 412a1393b6e..38b23ba8adc 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/ImageFinder.java @@ -29,6 +29,7 @@ public interface ImageFinder { interface ImageDetails { String getImageId(); String getImageName(); + String getRegion(); JenkinsDetails getJenkins(); } diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java index dca92aa740b..ba10ad93a34 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonImageFinder.java @@ -144,6 +144,11 @@ public String getImageName() { return (String) super.get("imageName"); } + @Override + public String getRegion() { + return (String) super.get("region"); + } + @Override public JenkinsDetails getJenkins() { return (JenkinsDetails) super.get("jenkins"); diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java index 72e01c969be..2b3b340d288 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/gce/GoogleImageFinder.java @@ -132,6 +132,11 @@ public String getImageName() { return (String) super.get("imageName"); } + @Override + public String getRegion() { + return (String) super.get("region"); + } + @Override public JenkinsDetails getJenkins() { return (JenkinsDetails) super.get("jenkins"); diff --git a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy b/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy index 183ca6e8342..ed69aa14c50 100644 --- a/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy +++ b/orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/image/FindImageFromTagTaskSpec.groovy @@ -9,12 +9,13 @@ import spock.lang.Subject class FindImageFromTagTaskSpec extends Specification { def imageFinder = Mock(ImageFinder) + def imageDetails = Mock(ImageFinder.ImageDetails) Stage stage = new Stage<>(new Pipeline("orca"), "", [packageName: 'myPackage', tags: ['foo': 'bar']]) @Subject def task = new FindImageFromTagsTask(imageFinders: [imageFinder]) - def "Not finding an images should throw IllegalStateException"() { + def "Not finding images should throw IllegalStateException"() { when: task.execute(stage) @@ -25,7 +26,7 @@ class FindImageFromTagTaskSpec extends Specification { 1 * imageFinder.getCloudProvider() >> 'aws' } - def "Finding an images should set task state to SUCCEEDED"() { + def "Finding images should set task state to SUCCEEDED"() { when: def result = task.execute(stage) @@ -33,6 +34,10 @@ class FindImageFromTagTaskSpec extends Specification { result.status == ExecutionStatus.SUCCEEDED 1 * imageFinder.getCloudProvider() >> 'aws' - 1 * imageFinder.byTags(stage, stage.context.packageName, stage.context.tags) >> [[:]] + 1 * imageFinder.byTags(stage, stage.context.packageName, stage.context.tags) >> [imageDetails] + 1 * imageDetails.getImageName() >> "somename" + 1 * imageDetails.getImageId() >> "someId" + 1 * imageDetails.getJenkins() >> new ImageFinder.JenkinsDetails("somehost", "somename", "42") } + } diff --git a/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Artifact.java b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Artifact.java new file mode 100644 index 00000000000..94f448bf222 --- /dev/null +++ b/orca-core/src/main/groovy/com/netflix/spinnaker/orca/pipeline/model/Artifact.java @@ -0,0 +1,115 @@ +/* + * Copyright 2017 Schibsted ASA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca.pipeline.model; + +import java.io.Serializable; +import java.util.Map; + +public class Artifact implements Serializable { + private String name; + private String type; + private String version; + private String reference; + private Map metadata; + private String auth; + private String provenance; + private String uuid; + + public void setName(String name) { + this.name = name; + } + + public void setType(String type) { + this.type = type; + } + + public void setVersion(String version) { + this.version = version; + } + + public void setReference(String reference) { + this.reference = reference; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public void setAuth(String auth) { + this.auth = auth; + } + + public void setProvenance(String provenance) { + this.provenance = provenance; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public String getVersion() { + return version; + } + + public String getReference() { + return reference; + } + + public Map getMetadata() { + return metadata; + } + + public String getAuth() { + return auth; + } + + public String getProvenance() { + return provenance; + } + + public String getUuid() { + return uuid; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Artifact artifact = (Artifact) o; + + if (!name.equals(artifact.name)) return false; + if (!type.equals(artifact.type)) return false; + return version.equals(artifact.version); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + type.hashCode(); + result = 31 * result + version.hashCode(); + return result; + } +}