From f86fae150644464c9801e72f897704e48340ab05 Mon Sep 17 00:00:00 2001 From: "Chang, Hui-Tang" Date: Thu, 11 Jan 2024 02:53:42 +0800 Subject: [PATCH] feat(recipe): use `${}` as reference syntax (#358) Because - Originally, we had two syntax options: `{}` and `{{}}` for reference and string literals (Liquid template). There were two main problems: 1. Users were easily confused by these two, and Liquid template is actually not a must-have feature in component input. 2. We would encounter syntax conflicts when trying to input a JSON string. This commit - Uses `${}` as the reference syntax to avoid conflict with JSON string. - Updates the condition syntax to use `${}` as well. --- go.mod | 2 +- go.sum | 4 ++-- integration-test/pipeline/const.js | 14 +++++++------- pkg/service/openapi.go | 4 ++-- pkg/utils/dag.go | 17 +++++++++++------ pkg/worker/workflow.go | 5 ++++- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index a65b52458..b128f8c39 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 github.com/iancoleman/strcase v0.2.0 github.com/influxdata/influxdb-client-go/v2 v2.12.3 - github.com/instill-ai/component v0.8.0-beta.0.20240109070207-eb76043417a3 + github.com/instill-ai/component v0.8.0-beta.0.20240110183225-60b21171a6ab github.com/instill-ai/connector v0.9.0-beta.0.20240110040755-9bc404800d3e github.com/instill-ai/operator v0.6.0-beta.0.20240108023644-8644208adde4 github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240105094938-3a71d8f7a812 diff --git a/go.sum b/go.sum index fbf2ef7b9..e36dfade7 100644 --- a/go.sum +++ b/go.sum @@ -1185,8 +1185,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0 github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/instill-ai/component v0.8.0-beta.0.20240109070207-eb76043417a3 h1:00WIhLXUP3mq39w/P0T/82jex+FUMkoOnBcwMU29/to= -github.com/instill-ai/component v0.8.0-beta.0.20240109070207-eb76043417a3/go.mod h1:fWyVPJVJ4WyFSQMgWCc7KvcS7m9QpdS3VXCL2CZE8OY= +github.com/instill-ai/component v0.8.0-beta.0.20240110183225-60b21171a6ab h1:HeP0cDDmgvHcBq08NIruXJ36pVtThBaahI1MFUSjunU= +github.com/instill-ai/component v0.8.0-beta.0.20240110183225-60b21171a6ab/go.mod h1:fWyVPJVJ4WyFSQMgWCc7KvcS7m9QpdS3VXCL2CZE8OY= github.com/instill-ai/connector v0.9.0-beta.0.20240110040755-9bc404800d3e h1:0s6UEpn8f5eIcmbyJVtimbLP7pHvRt6duWI2k1+q2Y4= github.com/instill-ai/connector v0.9.0-beta.0.20240110040755-9bc404800d3e/go.mod h1:5KQ7C4E8t9KPr3GmpWVPIyy0i16JwZgaYclDcLCVNBo= github.com/instill-ai/operator v0.6.0-beta.0.20240108023644-8644208adde4 h1:gNxhkOnemxPMpN4vUkOXf1TilyN1kJUK6QQijoaboHg= diff --git a/integration-test/pipeline/const.js b/integration-test/pipeline/const.js index 7a7001728..bcc8643d6 100644 --- a/integration-test/pipeline/const.js +++ b/integration-test/pipeline/const.js @@ -106,7 +106,7 @@ export const simpleRecipe = { } }, input: { - answer: "{ start.input }" + answer: "${start.input}" } } }, @@ -117,7 +117,7 @@ export const simpleRecipe = { configuration: { input: { data: { - text: "{ start.input }" + text: "${start.input}" } } } @@ -129,7 +129,7 @@ export const simpleRecipe = { configuration: { input: { data: { - text: "{ start.input }" + text: "${start.input}" } } } @@ -165,7 +165,7 @@ export const simpleRecipeWithoutCSV = { } }, input: { - answer: "{ start.input }" + answer: "${start.input}" } } }, @@ -200,7 +200,7 @@ export const simpleRecipeDupId = { } }, input: { - answer: "{ start.input }" + answer: "${start.input}" } } }, @@ -211,7 +211,7 @@ export const simpleRecipeDupId = { configuration: { input: { data: { - text: "{ start.input }" + text: "${start.input}" } } } @@ -223,7 +223,7 @@ export const simpleRecipeDupId = { configuration: { input: { data: { - text: "{ start.input }" + text: "${start.input}" } } } diff --git a/pkg/service/openapi.go b/pkg/service/openapi.go index 28e4879f9..b000f874a 100644 --- a/pkg/service/openapi.go +++ b/pkg/service/openapi.go @@ -198,9 +198,9 @@ func (s *service) GenerateOpenApiSpec(startCompOrigin *pipelinePB.Component, end switch inputFields[k].AsInterface().(type) { case string: str := inputFields[k].GetStringValue() - if strings.HasPrefix(str, "{") && strings.HasSuffix(str, "}") && !strings.HasPrefix(str, "{{") && !strings.HasSuffix(str, "}}") { + if strings.HasPrefix(str, "${") && strings.HasSuffix(str, "}") && strings.Count(str, "${") == 1 { // TODO - str = str[1:] + str = str[2:] str = str[:len(str)-1] str = strings.ReplaceAll(str, " ", "") isArrayReference := false diff --git a/pkg/utils/dag.go b/pkg/utils/dag.go index ae6e8a991..2c38a6f09 100644 --- a/pkg/utils/dag.go +++ b/pkg/utils/dag.go @@ -169,8 +169,8 @@ func RenderInput(input interface{}, bindings map[string]interface{}) (interface{ switch input := input.(type) { case string: - if strings.HasPrefix(input, "{") && strings.HasSuffix(input, "}") && !strings.HasPrefix(input, "{{") && !strings.HasSuffix(input, "}}") { - input = input[1:] + if strings.HasPrefix(input, "${") && strings.HasSuffix(input, "}") && strings.Count(input, "${") == 1 { + input = input[2:] input = input[:len(input)-1] input = strings.ReplaceAll(input, " ", "") if input[0] == '[' && input[len(input)-1] == ']' { @@ -194,7 +194,8 @@ func RenderInput(input interface{}, bindings map[string]interface{}) (interface{ } } - engine := liquid.NewEngine() + // TODO: we should retire Liquid instead of changing the delimiters + engine := liquid.NewEngine().Delims("${", "}", "{%", "%}") out, err := engine.ParseAndRenderString(input, bindings) if err != nil { return nil, err @@ -476,7 +477,9 @@ func GenerateDAG(components []*datamodel.Component) (*dag, error) { } graph := NewDAG(components) for _, component := range components { - engine := liquid.NewEngine() + + // TODO: we should retire Liquid instead of changing the delimiters + engine := liquid.NewEngine().Delims("${", "}", "{%", "%}") configuration := proto.Clone(component.Configuration) template, _ := protojson.Marshal(configuration) out, err := engine.ParseTemplate(template) @@ -486,6 +489,8 @@ func GenerateDAG(components []*datamodel.Component) (*dag, error) { condUpstreams := []string{} if cond := component.Configuration.Fields["condition"].GetStringValue(); cond != "" { + cond = strings.ReplaceAll(cond, "${", "") + cond = strings.ReplaceAll(cond, "}", "") expr, err := parser.ParseExpr(cond) if err != nil { return nil, err @@ -540,9 +545,9 @@ func FindReferenceParent(input string) []string { switch parsed := parsed.(type) { case string: - if strings.HasPrefix(parsed, "{") && strings.HasSuffix(parsed, "}") && !strings.HasPrefix(parsed, "{{") && !strings.HasSuffix(parsed, "}}") { + if strings.HasPrefix(parsed, "${") && strings.HasSuffix(parsed, "}") && strings.Count(parsed, "${") == 1 { - parsed = parsed[1:] + parsed = parsed[2:] parsed = parsed[:len(parsed)-1] parsed = strings.ReplaceAll(parsed, " ", "") if parsed[0] == '[' && parsed[len(parsed)-1] == ']' { diff --git a/pkg/worker/workflow.go b/pkg/worker/workflow.go index 96139acc1..d7e8e3b7a 100644 --- a/pkg/worker/workflow.go +++ b/pkg/worker/workflow.go @@ -283,7 +283,10 @@ func (w *worker) TriggerPipelineWorkflow(ctx workflow.Context, param *TriggerPip if !statuses[idx][comp.Id].Skipped { if comp.Configuration.Fields["condition"].GetStringValue() != "" { - expr, err := parser.ParseExpr(comp.Configuration.Fields["condition"].GetStringValue()) + condStr := comp.Configuration.Fields["condition"].GetStringValue() + condStr = strings.ReplaceAll(condStr, "${", "") + condStr = strings.ReplaceAll(condStr, "}", "") + expr, err := parser.ParseExpr(condStr) if err != nil { return nil, err }