diff --git a/.argo/test-yamls/fixtures-dynamic.yaml b/.argo/test-yamls/fixtures-dynamic.yaml index c98c3b1a2b10..21c30a0598f9 100644 --- a/.argo/test-yamls/fixtures-dynamic.yaml +++ b/.argo/test-yamls/fixtures-dynamic.yaml @@ -65,3 +65,45 @@ args: [" resources: cpu_cores: 0.05 mem_mib: 64 + +--- +type: workflow +version: 1 +name: test-fixtures-dynamic-outputs +description: Workflow which exports artifacts produced by dynamic fixtures +inputs: + parameters: + COMMIT: + default: "%%session.commit%%" + REPO: + default: "%%session.repo%%" +fixtures: +- DYN_FIX_WITH_OUTPUTS: + template: test-dynamic-fixture-container-with-outputs +steps: +- WEB-CLIENT-INLINED: + image: alpine:latest + command: [sh, -c] + args: ["sleep 60"] + resources: + cpu_cores: 0.05 + mem_mib: 64 +outputs: + artifacts: + WF_OUTPUTS: + from: "%%fixtures.DYN_FIX_WITH_OUTPUTS.outputs.artifacts.BIN-DIR%%" + +--- +type: container +version: 1 +name: test-dynamic-fixture-container-with-outputs +image: alpine:latest +command: [sh, -c] +args: ["sleep 999999"] +resources: + cpu_cores: 0.05 + mem_mib: 64 +outputs: + artifacts: + BIN-DIR: + path: /bin \ No newline at end of file diff --git a/saas/axops/src/applatix.io/axops/service/preprocess.go b/saas/axops/src/applatix.io/axops/service/preprocess.go index e82db273bae8..355d2adc8d34 100644 --- a/saas/axops/src/applatix.io/axops/service/preprocess.go +++ b/saas/axops/src/applatix.io/axops/service/preprocess.go @@ -221,10 +221,27 @@ func buildReplaceMap(tmpl EmbeddedTemplateIf, arguments template.Arguments) (map } } - // For any immediate children of this template, replace usages of %%steps.STEPNAME.outputs.artifacts.ARTNAME%% with concrete services IDs. - // This is only needed with workflows, since containers don't have children, and deployments do not use outputs from child containers + // For any immediate children of this template, replace usages of %%steps.STEPNAME.outputs.artifacts.ARTNAME%% or + // %%fixtures.FIXNAME.outputs.artifacts.ARTNAME%% with concrete services IDs. This is only needed with workflows, + // since containers don't have children, and deployments do not use outputs from child containers if tmpl.GetType() == template.TemplateTypeWorkflow { wft := tmpl.(*EmbeddedWorkflowTemplate) + for _, parallelFixtures := range wft.Fixtures { + for fixRefName, ftr := range parallelFixtures { + if !ftr.IsDynamicFixture() { + continue + } + outputs := ftr.Template.GetOutputs() + if outputs == nil { + continue + } + // The following converts %%fixtures.STEP1.outputs.artifacts.ARTNAME%% to %%service.service_id.outputs.artifacts.ARTNAME%% + for artName := range outputs.Artifacts { + serviceOutputRef := fmt.Sprintf("%%%%service.%s.outputs.artifacts.%s%%%%", ftr.Id, artName) + replaceMap[fmt.Sprintf("%%%%fixtures.%s.outputs.artifacts.%s%%%%", fixRefName, artName)] = &serviceOutputRef + } + } + } for _, parallelSteps := range wft.Steps { for stepName, step := range parallelSteps { if step.Template == nil { diff --git a/saas/common/src/applatix.io/template/param.go b/saas/common/src/applatix.io/template/param.go index 00bbcf7bac5b..2d06e49d2469 100644 --- a/saas/common/src/applatix.io/template/param.go +++ b/saas/common/src/applatix.io/template/param.go @@ -20,7 +20,7 @@ const ( var ( paramNameRegexStr = "[-0-9A-Za-z_]+" paramNameRegex = regexp.MustCompile("^[-0-9A-Za-z_]+$") - ouputArtifactRegexp = regexp.MustCompile(`^%%steps\.` + paramNameRegexStr + `\.outputs\.artifacts\.` + paramNameRegexStr + `%%$`) + ouputArtifactRegexp = regexp.MustCompile(`^%%(steps|fixtures)\.` + paramNameRegexStr + `\.outputs\.artifacts\.` + paramNameRegexStr + `%%$`) VarRegex = regexp.MustCompile("%%[-0-9A-Za-z_]+(\\.[-0-9A-Za-z_]+)*%%") varRegexExact = regexp.MustCompile("^%%[-0-9A-Za-z_]+(\\.[-0-9A-Za-z_]+)*%%$") listExpansionParamRegex = regexp.MustCompile("\\$\\$\\[(.*)\\]\\$\\$") diff --git a/saas/common/src/applatix.io/template/workflow.go b/saas/common/src/applatix.io/template/workflow.go index c9780cf0184e..0389d8f2c094 100644 --- a/saas/common/src/applatix.io/template/workflow.go +++ b/saas/common/src/applatix.io/template/workflow.go @@ -171,6 +171,10 @@ func (tmpl *WorkflowTemplate) ValidateContext(context *TemplateBuildContext) *ax // the previous step group to the current scope scopedParams := tmpl.parametersInScope() + // dynFixtureOutputs keeps track of any outputs produced by dynamic fixtures. + // We will add this to the scope after all steps complete + dynFixtureOutputs := paramMap{} + // Verify any dynamic fixtures. Template references must be container type for i, parallelFixtures := range tmpl.Fixtures { for fixRefName, ftr := range parallelFixtures { @@ -190,6 +194,15 @@ func (tmpl *WorkflowTemplate) ValidateContext(context *TemplateBuildContext) *ax if axErr != nil { return axerror.ERR_AXDB_INVALID_PARAM.NewWithMessagef("fixtures[%d].%s: %v", i, fixRefName, axErr) } + if ctrTemplate.Outputs != nil { + for artRefName := range ctrTemplate.Outputs.Artifacts { + p := param{ + name: fmt.Sprintf("fixtures.%s.outputs.artifacts.%s", fixRefName, artRefName), + paramType: paramTypeArtifact, + } + dynFixtureOutputs[p.name] = p + } + } } } @@ -271,6 +284,12 @@ func (tmpl *WorkflowTemplate) ValidateContext(context *TemplateBuildContext) *ax } } + // Add dynamic fixtures outputs to the scope + axErr := scopedParams.merge(dynFixtureOutputs) + if axErr != nil { + return axErr + } + // Do one last validation of all parameters used in the template usedParams, axErr := tmpl.usedParameters() if axErr != nil { @@ -281,8 +300,8 @@ func (tmpl *WorkflowTemplate) ValidateContext(context *TemplateBuildContext) *ax return axErr } - // See if this workflow is exporting any artifacs. If so, verify that 'from' is - // referencing a valid step, and that step has expected artifact with the name. + // See if this workflow is exporting any artifacts. If so, verify that 'from' is + // referencing a valid step or fixture, and that step has expected artifact with the name. if tmpl.Outputs != nil && tmpl.Outputs.Artifacts != nil { for refName, art := range tmpl.Outputs.Artifacts { if art.Path != "" { @@ -292,7 +311,7 @@ func (tmpl *WorkflowTemplate) ValidateContext(context *TemplateBuildContext) *ax return axerror.ERR_API_INVALID_PARAM.NewWithMessagef("outputs.artifacts.%s.from is required", refName) } if !ouputArtifactRegexp.MatchString(art.From) { - return axerror.ERR_API_INVALID_PARAM.NewWithMessagef("outputs.artifacts.%s.from invalid format '%s'. expected format: '%%%%steps..outputs.artifacts.%%%%'", refName, art.From) + return axerror.ERR_API_INVALID_PARAM.NewWithMessagef("outputs.artifacts.%s.from invalid format '%s'. expected format: '%%%%(steps|fixtures)..outputs.artifacts.%%%%'", refName, art.From) } pName := strings.Trim(art.From, "%") _, ok := scopedParams[pName]