From 005d0fdde53bc2144e88ebf3db280312d8509983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Jervidalo?= Date: Tue, 17 Oct 2017 12:51:19 +0200 Subject: [PATCH] feat(pipeline_template): Render configuration for templated pipelines with dynamic source * Will render using a specific execution, or the latest if a specific one is not set. --- .../PipelineTemplateService.java | 69 ++++++++++++++++++- .../TemplatedPipelineRequest.java | 9 +++ .../v1schema/V1SchemaExecutionGenerator.java | 6 ++ .../handler/V1TemplateLoaderHandler.kt | 1 + .../v1schema/example-expected.json | 1 + .../v1schema/inheritance-expected.json | 1 + .../controllers/OperationsController.groovy | 26 +++++-- .../PipelineTemplateController.groovy | 9 +-- .../OperationsControllerSpec.groovy | 47 ++++++++++++- 9 files changed, 157 insertions(+), 12 deletions(-) diff --git a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/PipelineTemplateService.java b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/PipelineTemplateService.java index 7c0a654323c..d0c71dadb1b 100644 --- a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/PipelineTemplateService.java +++ b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/PipelineTemplateService.java @@ -15,27 +15,92 @@ */ package com.netflix.spinnaker.orca.pipelinetemplate; +import com.netflix.spinnaker.orca.pipeline.model.Pipeline; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException; +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository; import com.netflix.spinnaker.orca.pipelinetemplate.loader.TemplateLoader; import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.TemplateMerge; import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.PipelineTemplate; import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.TemplateConfiguration.TemplateSource; +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.DefaultRenderContext; +import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.Renderer; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.annotation.Nullable; import java.util.List; +import java.util.NoSuchElementException; @Component public class PipelineTemplateService { private final TemplateLoader templateLoader; + private final ExecutionRepository executionRepository; + + private final Renderer renderer; + @Autowired - public PipelineTemplateService(TemplateLoader templateLoader) { + public PipelineTemplateService(TemplateLoader templateLoader, ExecutionRepository executionRepository, Renderer renderer) { this.templateLoader = templateLoader; + this.executionRepository = executionRepository; + this.renderer = renderer; } - public PipelineTemplate resolveTemplate(TemplateSource templateSource) { + public PipelineTemplate resolveTemplate(TemplateSource templateSource, @Nullable String executionId, @Nullable String pipelineConfigId) { + if (containsJinja(templateSource.getSource()) && !(executionId == null && pipelineConfigId == null)) { + try { + Pipeline pipeline = retrievePipelineOrNewestExecution(executionId, pipelineConfigId); + String renderedSource = render(templateSource.getSource(), pipeline); + if (StringUtils.isNotBlank(renderedSource)) { + templateSource.setSource(renderedSource); + } + } catch (NoSuchElementException e) { + // Do nothing + } + } List templates = templateLoader.load(templateSource); return TemplateMerge.merge(templates); } + + /** + * If {@code executionId} is set, it will be retrieved. Otherwise, {@code pipelineConfigId} will be used to find the + * newest pipeline execution for that configuration. + * @param executionId An explicit pipeline execution id. + * @param pipelineConfigId A pipeline configuration id. Ignored if {@code executionId} is set. + * @return The pipeline + * @throws IllegalArgumentException if neither executionId or pipelineConfigId are provided + * @throws ExecutionNotFoundException if no execution could be found + */ + public Pipeline retrievePipelineOrNewestExecution(@Nullable String executionId, @Nullable String pipelineConfigId) throws ExecutionNotFoundException { + if (executionId != null) { + // Use an explicit execution + return executionRepository.retrievePipeline(executionId); + } else if (pipelineConfigId != null) { + // No executionId set - use last execution + ExecutionRepository.ExecutionCriteria criteria = new ExecutionRepository.ExecutionCriteria().setLimit(1); + try { + return executionRepository.retrievePipelinesForPipelineConfigId(pipelineConfigId, criteria) + .toSingle() + .toBlocking() + .value(); + } catch (NoSuchElementException e) { + throw new ExecutionNotFoundException("No pipeline execution could be found for config id " + + pipelineConfigId + ": " + e.getMessage()); + } + } else { + throw new IllegalArgumentException("Either executionId or pipelineConfigId have to be set."); + } + } + + private String render(String templateString, Pipeline pipeline) { + DefaultRenderContext rc = new DefaultRenderContext(pipeline.getApplication(), null, pipeline.getTrigger()); + return renderer.render(templateString, rc); + } + + private static boolean containsJinja(String string) { + return string.contains("{%") || string.contains("{{"); + } + } diff --git a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/TemplatedPipelineRequest.java b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/TemplatedPipelineRequest.java index ef5a9add2b2..8e5eaf7b07b 100644 --- a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/TemplatedPipelineRequest.java +++ b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/TemplatedPipelineRequest.java @@ -27,6 +27,7 @@ public class TemplatedPipelineRequest { Map trigger = new HashMap<>(); Map config; Map template; + String executionId; Boolean plan = false; boolean limitConcurrent = true; boolean keepWaitingPipelines = false; @@ -92,6 +93,14 @@ public void setTemplate(Map template) { this.template = template; } + public String getExecutionId() { + return executionId; + } + + public void setExecutionId(String executionId) { + this.executionId = executionId; + } + public Boolean getPlan() { return plan; } diff --git a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/V1SchemaExecutionGenerator.java b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/V1SchemaExecutionGenerator.java index 37750b94244..c9768a48c3d 100644 --- a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/V1SchemaExecutionGenerator.java +++ b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/V1SchemaExecutionGenerator.java @@ -35,6 +35,12 @@ public Map generate(PipelineTemplate template, TemplateConfigura Map pipeline = new HashMap<>(); pipeline.put("id", Optional.ofNullable(request.getId()).orElse(Optional.ofNullable(configuration.getPipeline().getPipelineConfigId()).orElse("unknown"))); pipeline.put("application", configuration.getPipeline().getApplication()); + if (template.getSource() != null) { + pipeline.put("source", template.getSource()); + } + if (request.getExecutionId() != null) { + pipeline.put("executionId", request.getExecutionId()); + } pipeline.put("name", Optional.ofNullable(configuration.getPipeline().getName()).orElse("Unnamed Execution")); if (configuration.getPipeline().getExecutionEngine() != null) { diff --git a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/handler/V1TemplateLoaderHandler.kt b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/handler/V1TemplateLoaderHandler.kt index 70e78f214f8..2710fe37bbb 100644 --- a/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/handler/V1TemplateLoaderHandler.kt +++ b/orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/handler/V1TemplateLoaderHandler.kt @@ -84,6 +84,7 @@ class V1TemplateLoaderHandler( val value = v.defaultValue if (value != null && value is String) { v.defaultValue = renderer.renderGraph(value.toString(), renderContext) + renderContext.variables.putIfAbsent(v.name, v.defaultValue) } } } diff --git a/orca-pipelinetemplate/src/test/resources/integration/v1schema/example-expected.json b/orca-pipelinetemplate/src/test/resources/integration/v1schema/example-expected.json index 24fa5c105c2..7251f3a107a 100644 --- a/orca-pipelinetemplate/src/test/resources/integration/v1schema/example-expected.json +++ b/orca-pipelinetemplate/src/test/resources/integration/v1schema/example-expected.json @@ -3,6 +3,7 @@ "limitConcurrent": true, "application": "myApp", "name": "My super awesome pipeline", + "source": "example-root.yml,example-child.yml", "stages": [ { "requisiteStageRefIds": [], diff --git a/orca-pipelinetemplate/src/test/resources/integration/v1schema/inheritance-expected.json b/orca-pipelinetemplate/src/test/resources/integration/v1schema/inheritance-expected.json index 528c521220a..bfdd4cb90ed 100644 --- a/orca-pipelinetemplate/src/test/resources/integration/v1schema/inheritance-expected.json +++ b/orca-pipelinetemplate/src/test/resources/integration/v1schema/inheritance-expected.json @@ -3,6 +3,7 @@ "limitConcurrent": true, "application": "orca", "name": "MPT Inheritance Test", + "source": "inheritance-root.yml", "stages": [ { "requisiteStageRefIds": [], diff --git a/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy b/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy index b61bb378dc4..7b472d84b31 100644 --- a/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy +++ b/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy @@ -16,9 +16,6 @@ package com.netflix.spinnaker.orca.controllers -import com.netflix.spinnaker.orca.pipeline.util.ArtifactResolver - -import javax.servlet.http.HttpServletResponse import com.fasterxml.jackson.databind.ObjectMapper import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException import com.netflix.spinnaker.kork.web.exceptions.ValidationException @@ -28,8 +25,11 @@ import com.netflix.spinnaker.orca.igor.BuildService import com.netflix.spinnaker.orca.pipeline.OrchestrationLauncher import com.netflix.spinnaker.orca.pipeline.PipelineLauncher import com.netflix.spinnaker.orca.pipeline.model.Pipeline +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository +import com.netflix.spinnaker.orca.pipeline.util.ArtifactResolver import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor +import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplateService import com.netflix.spinnaker.orca.webhook.service.WebhookService import com.netflix.spinnaker.security.AuthenticatedRequest import groovy.util.logging.Slf4j @@ -40,6 +40,8 @@ import org.springframework.web.bind.annotation.RequestMethod import org.springframework.web.bind.annotation.RestController import static net.logstash.logback.argument.StructuredArguments.value +import javax.servlet.http.HttpServletResponse + @RestController @Slf4j class OperationsController { @@ -58,6 +60,9 @@ class OperationsController { @Autowired ExecutionRepository executionRepository + @Autowired + PipelineTemplateService pipelineTemplateService + @Autowired ContextParameterProcessor contextParameterProcessor @@ -72,7 +77,7 @@ class OperationsController { @RequestMapping(value = "/orchestrate", method = RequestMethod.POST) Map orchestrate(@RequestBody Map pipeline, HttpServletResponse response) { - parsePipelineTrigger(executionRepository, buildService, pipeline) + parsePipelineTrigger(executionRepository, buildService, pipelineTemplateService, pipeline) Map trigger = pipeline.trigger boolean plan = pipeline.plan ?: false @@ -112,9 +117,20 @@ class OperationsController { startPipeline(processedPipeline) } - private void parsePipelineTrigger(ExecutionRepository executionRepository, BuildService buildService, Map pipeline) { + private void parsePipelineTrigger(ExecutionRepository executionRepository, BuildService buildService, PipelineTemplateService pipelineTemplateService, Map pipeline) { if (!(pipeline.trigger instanceof Map)) { pipeline.trigger = [:] + if (pipeline.plan && pipeline.type == "templatedPipeline") { + // If possible, initialize the config with a previous execution trigger context, to be able to resolve + // dynamic parameters in jinja expressions + try { + def previousExecution = pipelineTemplateService.retrievePipelineOrNewestExecution(pipeline.executionId, pipeline.id) + pipeline.trigger = previousExecution.trigger + pipeline.executionId = previousExecution.id + } catch (ExecutionNotFoundException ignore) { + // Do nothing + } + } } if (!pipeline.trigger.type) { diff --git a/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/PipelineTemplateController.groovy b/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/PipelineTemplateController.groovy index c45e4b3f101..bb1e58be03f 100644 --- a/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/PipelineTemplateController.groovy +++ b/orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/PipelineTemplateController.groovy @@ -38,12 +38,13 @@ class PipelineTemplateController { PipelineTemplateService pipelineTemplateService @RequestMapping(value = "/pipelineTemplate", method = RequestMethod.GET) - PipelineTemplate getPipelineTemplate(@RequestParam("source") String source) { - if (source == null || source?.empty) { + PipelineTemplate getPipelineTemplate(@RequestParam String source, + @RequestParam(required = false) String executionId, + @RequestParam(required = false) String pipelineConfigId) { + if (!source) { throw new InvalidRequestException("template source must not be empty") } - - pipelineTemplateService.resolveTemplate(new TemplateSource(source: source)) + pipelineTemplateService.resolveTemplate(new TemplateSource(source: source), executionId, pipelineConfigId) } @RequestMapping(value = "/convertPipelineToTemplate", method = RequestMethod.POST, produces = 'text/x-yaml') diff --git a/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy b/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy index d247fabcf81..fcc4485d8f9 100644 --- a/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy +++ b/orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy @@ -16,15 +16,16 @@ package com.netflix.spinnaker.orca.controllers -import javax.servlet.http.HttpServletResponse import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException import com.netflix.spinnaker.orca.igor.BuildArtifactFilter import com.netflix.spinnaker.orca.igor.BuildService import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper import com.netflix.spinnaker.orca.pipeline.PipelineLauncher import com.netflix.spinnaker.orca.pipeline.model.Pipeline +import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor +import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplateService import com.netflix.spinnaker.orca.webhook.config.PreconfiguredWebhookProperties import com.netflix.spinnaker.orca.webhook.service.WebhookService import com.netflix.spinnaker.security.AuthenticatedRequest @@ -38,7 +39,11 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders import spock.lang.Specification import spock.lang.Subject import spock.lang.Unroll + +import javax.servlet.http.HttpServletResponse + import static com.netflix.spinnaker.orca.ExecutionStatus.CANCELED +import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.pipeline import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post @@ -52,6 +57,7 @@ class OperationsControllerSpec extends Specification { def buildService = Stub(BuildService) def mapper = OrcaObjectMapper.newInstance() def executionRepository = Mock(ExecutionRepository) + def pipelineTemplateService = Mock(PipelineTemplateService) def webhookService = Mock(WebhookService) def env = new MockEnvironment() @@ -63,6 +69,7 @@ class OperationsControllerSpec extends Specification { buildService: buildService, buildArtifactFilter: buildArtifactFilter, executionRepository: executionRepository, + pipelineTemplateService: pipelineTemplateService, pipelineLauncher: pipelineLauncher, contextParameterProcessor: new ContextParameterProcessor(), webhookService: webhookService @@ -197,6 +204,41 @@ class OperationsControllerSpec extends Specification { ] } + def "should get pipeline execution context from a previous execution if not provided and attribute plan is truthy"() { + given: + Pipeline startedPipeline = null + pipelineLauncher.start(_) >> { String json -> + startedPipeline = mapper.readValue(json, Pipeline) + startedPipeline.id = UUID.randomUUID().toString() + startedPipeline + } + + Pipeline previousExecution = pipeline { + name = "Last executed pipeline" + status = SUCCEEDED + id = "12345" + application = "covfefe" + trigger << [ + type: "travis" + ] + } + + when: + def orchestration = controller.orchestrate(requestedPipeline, Mock(HttpServletResponse)) + + then: + 1 * pipelineTemplateService.retrievePipelineOrNewestExecution("12345", _) >> previousExecution + orchestration.trigger.type == "travis" + + where: + requestedPipeline = [ + id: "54321", + plan: true, + type: "templatedPipeline", + executionId: "12345" + ] + } + def "trigger user takes precedence over query parameter"() { given: Pipeline startedPipeline = null @@ -492,6 +534,8 @@ class OperationsControllerSpec extends Specification { given: def pipelineConfig = [ plan : true, + type : "templatedPipeline", + executionId: "12345", errors: [ 'things broke': 'because of the way it is' ] @@ -503,6 +547,7 @@ class OperationsControllerSpec extends Specification { then: thrown(InvalidRequestException) + 1 * pipelineTemplateService.retrievePipelineOrNewestExecution("12345", null) >> { throw new ExecutionNotFoundException("Not found") } 0 * pipelineLauncher.start(_) }