feat(pipeline_template): Render configuration for templated pipelines with dynamic source

orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/PipelineTemplateService.java

@@ -15,27 +15,92 @@
  */
 package com.netflix.spinnaker.orca.pipelinetemplate;
 
+import com.netflix.spinnaker.orca.pipeline.model.Pipeline;
+import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException;
+import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository;
 import com.netflix.spinnaker.orca.pipelinetemplate.loader.TemplateLoader;
 import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.TemplateMerge;
 import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.PipelineTemplate;
 import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.model.TemplateConfiguration.TemplateSource;
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.DefaultRenderContext;
+import com.netflix.spinnaker.orca.pipelinetemplate.v1schema.render.Renderer;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import javax.annotation.Nullable;
 import java.util.List;
+import java.util.NoSuchElementException;
 
 @Component
 public class PipelineTemplateService {
 
   private final TemplateLoader templateLoader;
 
+  private final ExecutionRepository executionRepository;
+
+  private final Renderer renderer;
+
   @Autowired
-  public PipelineTemplateService(TemplateLoader templateLoader) {
+  public PipelineTemplateService(TemplateLoader templateLoader, ExecutionRepository executionRepository, Renderer renderer) {
     this.templateLoader = templateLoader;
+    this.executionRepository = executionRepository;
+    this.renderer = renderer;
   }
 
-  public PipelineTemplate resolveTemplate(TemplateSource templateSource) {
+  public PipelineTemplate resolveTemplate(TemplateSource templateSource, @Nullable String executionId, @Nullable String pipelineConfigId) {
+    if (containsJinja(templateSource.getSource()) && !(executionId == null && pipelineConfigId == null)) {
+      try {
+        Pipeline pipeline = retrievePipelineOrNewestExecution(executionId, pipelineConfigId);
+        String renderedSource = render(templateSource.getSource(), pipeline);
+        if (StringUtils.isNotBlank(renderedSource)) {
+          templateSource.setSource(renderedSource);
+        }
+      } catch (NoSuchElementException e) {
+        // Do nothing
+      }
+    }
     List<PipelineTemplate> templates = templateLoader.load(templateSource);
     return TemplateMerge.merge(templates);
   }
+
+  /**
+   * If {@code executionId} is set, it will be retrieved. Otherwise, {@code pipelineConfigId} will be used to find the
+   * newest pipeline execution for that configuration.
+   * @param executionId An explicit pipeline execution id.
+   * @param pipelineConfigId A pipeline configuration id. Ignored if {@code executionId} is set.
+   * @return The pipeline
+   * @throws IllegalArgumentException if neither executionId or pipelineConfigId are provided
+   * @throws ExecutionNotFoundException if no execution could be found
+   */
+  public Pipeline retrievePipelineOrNewestExecution(@Nullable String executionId, @Nullable String pipelineConfigId) throws ExecutionNotFoundException {
+    if (executionId != null) {
+      // Use an explicit execution
+      return executionRepository.retrievePipeline(executionId);
+    } else if (pipelineConfigId != null) {
+      // No executionId set - use last execution
+      ExecutionRepository.ExecutionCriteria criteria = new ExecutionRepository.ExecutionCriteria().setLimit(1);
+      try {
+        return executionRepository.retrievePipelinesForPipelineConfigId(pipelineConfigId, criteria)
+          .toSingle()
+          .toBlocking()
+          .value();
+      } catch (NoSuchElementException e) {
+        throw new ExecutionNotFoundException("No pipeline execution could be found for config id " +
+          pipelineConfigId + ": " + e.getMessage());
+      }
+    } else {
+      throw new IllegalArgumentException("Either executionId or pipelineConfigId have to be set.");
+    }
+  }
+
+  private String render(String templateString, Pipeline pipeline) {
+    DefaultRenderContext rc = new DefaultRenderContext(pipeline.getApplication(), null, pipeline.getTrigger());
+    return renderer.render(templateString, rc);
+  }
+
+  private static boolean containsJinja(String string) {
+    return string.contains("{%") || string.contains("{{");
+  }
+
 }

orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/TemplatedPipelineRequest.java

@@ -27,6 +27,7 @@ public class TemplatedPipelineRequest {
   Map<String, Object> trigger = new HashMap<>();
   Map<String, Object> config;
   Map<String, Object> template;
+  String executionId;
   Boolean plan = false;
   boolean limitConcurrent = true;
   boolean keepWaitingPipelines = false;
@@ -92,6 +93,14 @@ public void setTemplate(Map<String, Object> template) {
     this.template = template;
   }
 
+  public String getExecutionId() {
+    return executionId;
+  }
+
+  public void setExecutionId(String executionId) {
+    this.executionId = executionId;
+  }
+
   public Boolean getPlan() {
     return plan;
   }

orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/V1SchemaExecutionGenerator.java

@@ -35,6 +35,12 @@ public Map<String, Object> generate(PipelineTemplate template, TemplateConfigura
     Map<String, Object> pipeline = new HashMap<>();
     pipeline.put("id", Optional.ofNullable(request.getId()).orElse(Optional.ofNullable(configuration.getPipeline().getPipelineConfigId()).orElse("unknown")));
     pipeline.put("application", configuration.getPipeline().getApplication());
+    if (template.getSource() != null) {
+      pipeline.put("source", template.getSource());
+    }
+    if (request.getExecutionId() != null) {
+      pipeline.put("executionId", request.getExecutionId());
+    }
     pipeline.put("name", Optional.ofNullable(configuration.getPipeline().getName()).orElse("Unnamed Execution"));
 
     if (configuration.getPipeline().getExecutionEngine() != null) {

orca-pipelinetemplate/src/main/java/com/netflix/spinnaker/orca/pipelinetemplate/v1schema/handler/V1TemplateLoaderHandler.kt

@@ -84,6 +84,7 @@ class V1TemplateLoaderHandler(
       val value = v.defaultValue
       if (value != null && value is String) {
         v.defaultValue = renderer.renderGraph(value.toString(), renderContext)
+        renderContext.variables.putIfAbsent(v.name, v.defaultValue)
       }
     }
   }

orca-pipelinetemplate/src/test/resources/integration/v1schema/example-expected.json

@@ -3,6 +3,7 @@
   "limitConcurrent": true,
   "application": "myApp",
   "name": "My super awesome pipeline",
+  "source": "example-root.yml,example-child.yml",
   "stages": [
     {
       "requisiteStageRefIds": [],

orca-pipelinetemplate/src/test/resources/integration/v1schema/inheritance-expected.json

@@ -3,6 +3,7 @@
   "limitConcurrent": true,
   "application": "orca",
   "name": "MPT Inheritance Test",
+  "source": "inheritance-root.yml",
   "stages": [
     {
       "requisiteStageRefIds": [],

orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy

@@ -25,9 +25,11 @@ import com.netflix.spinnaker.orca.igor.BuildService
 import com.netflix.spinnaker.orca.pipeline.OrchestrationLauncher
 import com.netflix.spinnaker.orca.pipeline.PipelineLauncher
 import com.netflix.spinnaker.orca.pipeline.model.Pipeline
+import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException
 import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository
 import com.netflix.spinnaker.orca.pipeline.util.ArtifactResolver
 import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor
+import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplateService
 import com.netflix.spinnaker.orca.webhook.service.WebhookService
 import com.netflix.spinnaker.security.AuthenticatedRequest
 import groovy.util.logging.Slf4j
@@ -41,6 +43,8 @@ import javax.servlet.http.HttpServletResponse
 
 import static net.logstash.logback.argument.StructuredArguments.value
 
+import javax.servlet.http.HttpServletResponse
+
 @RestController
 @Slf4j
 class OperationsController {
@@ -59,6 +63,9 @@ class OperationsController {
   @Autowired
   ExecutionRepository executionRepository
 
+  @Autowired
+  PipelineTemplateService pipelineTemplateService
+
   @Autowired
   ContextParameterProcessor contextParameterProcessor
 
@@ -73,7 +80,7 @@ class OperationsController {
 
   @RequestMapping(value = "/orchestrate", method = RequestMethod.POST)
   Map<String, Object> orchestrate(@RequestBody Map pipeline, HttpServletResponse response) {
-    parsePipelineTrigger(executionRepository, buildService, pipeline)
+    parsePipelineTrigger(executionRepository, buildService, pipelineTemplateService, pipeline)
     Map trigger = pipeline.trigger
 
     boolean plan = pipeline.plan ?: false
@@ -113,9 +120,20 @@ class OperationsController {
     startPipeline(processedPipeline)
   }
 
-  private void parsePipelineTrigger(ExecutionRepository executionRepository, BuildService buildService, Map pipeline) {
+  private void parsePipelineTrigger(ExecutionRepository executionRepository, BuildService buildService, PipelineTemplateService pipelineTemplateService, Map pipeline) {
     if (!(pipeline.trigger instanceof Map)) {
       pipeline.trigger = [:]
+      if (pipeline.plan && pipeline.type == "templatedPipeline") {
+        // If possible, initialize the config with a previous execution trigger context, to be able to resolve
+        // dynamic parameters in jinja expressions
+        try {
+          def previousExecution = pipelineTemplateService.retrievePipelineOrNewestExecution(pipeline.executionId, pipeline.id)
+          pipeline.trigger = previousExecution.trigger
+          pipeline.executionId = previousExecution.id
+        } catch (ExecutionNotFoundException ignore) {
+          // Do nothing
+        }
+      }
     }
 
     if (!pipeline.trigger.type) {

orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/PipelineTemplateController.groovy

@@ -38,12 +38,13 @@ class PipelineTemplateController {
   PipelineTemplateService pipelineTemplateService
 
   @RequestMapping(value = "/pipelineTemplate", method = RequestMethod.GET)
-  PipelineTemplate getPipelineTemplate(@RequestParam("source") String source) {
-    if (source == null || source?.empty) {
+  PipelineTemplate getPipelineTemplate(@RequestParam String source,
+                                       @RequestParam(required = false) String executionId,
+                                       @RequestParam(required = false) String pipelineConfigId) {
+    if (!source) {
       throw new InvalidRequestException("template source must not be empty")
     }
-
-    pipelineTemplateService.resolveTemplate(new TemplateSource(source: source))
+    pipelineTemplateService.resolveTemplate(new TemplateSource(source: source), executionId, pipelineConfigId)
   }
 
   @RequestMapping(value = "/convertPipelineToTemplate", method = RequestMethod.POST, produces = 'text/x-yaml')

orca-web/src/test/groovy/com/netflix/spinnaker/orca/controllers/OperationsControllerSpec.groovy

@@ -16,15 +16,16 @@
 
 package com.netflix.spinnaker.orca.controllers
 
-import javax.servlet.http.HttpServletResponse
 import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException
 import com.netflix.spinnaker.orca.igor.BuildArtifactFilter
 import com.netflix.spinnaker.orca.igor.BuildService
 import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper
 import com.netflix.spinnaker.orca.pipeline.PipelineLauncher
 import com.netflix.spinnaker.orca.pipeline.model.Pipeline
+import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionNotFoundException
 import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository
 import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor
+import com.netflix.spinnaker.orca.pipelinetemplate.PipelineTemplateService
 import com.netflix.spinnaker.orca.webhook.config.PreconfiguredWebhookProperties
 import com.netflix.spinnaker.orca.webhook.service.WebhookService
 import com.netflix.spinnaker.security.AuthenticatedRequest
@@ -38,7 +39,11 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders
 import spock.lang.Specification
 import spock.lang.Subject
 import spock.lang.Unroll
+
+import javax.servlet.http.HttpServletResponse
+
 import static com.netflix.spinnaker.orca.ExecutionStatus.CANCELED
+import static com.netflix.spinnaker.orca.ExecutionStatus.SUCCEEDED
 import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.pipeline
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post
 
@@ -52,6 +57,7 @@ class OperationsControllerSpec extends Specification {
   def buildService = Stub(BuildService)
   def mapper = OrcaObjectMapper.newInstance()
   def executionRepository = Mock(ExecutionRepository)
+  def pipelineTemplateService = Mock(PipelineTemplateService)
   def webhookService = Mock(WebhookService)
 
   def env = new MockEnvironment()
@@ -63,6 +69,7 @@ class OperationsControllerSpec extends Specification {
       buildService: buildService,
       buildArtifactFilter: buildArtifactFilter,
       executionRepository: executionRepository,
+      pipelineTemplateService: pipelineTemplateService,
       pipelineLauncher: pipelineLauncher,
       contextParameterProcessor: new ContextParameterProcessor(),
       webhookService: webhookService
@@ -197,6 +204,41 @@ class OperationsControllerSpec extends Specification {
     ]
   }
 
+  def "should get pipeline execution context from a previous execution if not provided and attribute plan is truthy"() {
+    given:
+    Pipeline startedPipeline = null
+    pipelineLauncher.start(_) >> { String json ->
+      startedPipeline = mapper.readValue(json, Pipeline)
+      startedPipeline.id = UUID.randomUUID().toString()
+      startedPipeline
+    }
+
+    Pipeline previousExecution = pipeline {
+      name = "Last executed pipeline"
+      status = SUCCEEDED
+      id = "12345"
+      application = "covfefe"
+      trigger << [
+        type: "travis"
+      ]
+    }
+
+    when:
+    def orchestration = controller.orchestrate(requestedPipeline, Mock(HttpServletResponse))
+
+    then:
+    1 * pipelineTemplateService.retrievePipelineOrNewestExecution("12345", _) >> previousExecution
+    orchestration.trigger.type == "travis"
+
+    where:
+    requestedPipeline = [
+      id: "54321",
+      plan: true,
+      type: "templatedPipeline",
+      executionId: "12345"
+    ]
+  }
+
   def "trigger user takes precedence over query parameter"() {
     given:
     Pipeline startedPipeline = null
@@ -492,6 +534,8 @@ class OperationsControllerSpec extends Specification {
     given:
     def pipelineConfig = [
       plan  : true,
+      type  : "templatedPipeline",
+      executionId: "12345",
       errors: [
         'things broke': 'because of the way it is'
       ]
@@ -503,6 +547,7 @@ class OperationsControllerSpec extends Specification {
 
     then:
     thrown(InvalidRequestException)
+    1 * pipelineTemplateService.retrievePipelineOrNewestExecution("12345", null) >> { throw new ExecutionNotFoundException("Not found") }
     0 * pipelineLauncher.start(_)
   }