diff --git a/Makefile b/Makefile index c605f2eee..5ca38157f 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ export .PHONY: dev dev: ## Run dev container - @docker compose ls -q | grep -q "instill-vdp" && true || \ - (echo "Error: Run \"make latest PROFILE=pipeline\" in vdp repository (https://github.com/instill-ai/vdp) in your local machine first." && exit 1) + @docker compose ls -q | grep -q "instill-core" && true || \ + (echo "Error: Run \"make latest PROFILE=pipeline\" in vdp repository (https://github.com/instill-ai/instill-core) in your local machine first." && exit 1) @docker inspect --type container ${SERVICE_NAME} >/dev/null 2>&1 && echo "A container named ${SERVICE_NAME} is already running." || \ echo "Run dev container ${SERVICE_NAME}. To stop it, run \"make stop\"." @docker run -d --rm \ diff --git a/go.mod b/go.mod index 07352141e..ab8c4f8dc 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,10 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 github.com/iancoleman/strcase v0.2.0 github.com/influxdata/influxdb-client-go/v2 v2.12.3 - github.com/instill-ai/component v0.12.0-beta.0.20240304105959-85bbc223c1df - github.com/instill-ai/connector v0.13.0-beta.0.20240304112007-00178f471ffd + github.com/instill-ai/component v0.12.0-beta.0.20240307023544-7c27d15e4e01 + github.com/instill-ai/connector v0.13.0-beta.0.20240307024046-140fe8f50ded github.com/instill-ai/operator v0.9.0-beta.0.20240304112014-043d5e3b0ebd - github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240305055434-f65d3c697ec8 + github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73 github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a github.com/instill-ai/x v0.4.0-alpha github.com/knadh/koanf v1.5.0 diff --git a/go.sum b/go.sum index d57f6a238..fa12ea9c7 100644 --- a/go.sum +++ b/go.sum @@ -1190,14 +1190,14 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0 github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/instill-ai/component v0.12.0-beta.0.20240304105959-85bbc223c1df h1:T8xczmtdyZXrHW7mVwNXTi/WaBYgPw8wUE1Pm+ptkuc= -github.com/instill-ai/component v0.12.0-beta.0.20240304105959-85bbc223c1df/go.mod h1:mvySqiBVNAy5wpDZnPdUZ/P1Iaj2BZJmi328hRMS818= -github.com/instill-ai/connector v0.13.0-beta.0.20240304112007-00178f471ffd h1:om3/KN5x0jNO825kTwFl0TRooZN5rR19TrcGFq9/MZU= -github.com/instill-ai/connector v0.13.0-beta.0.20240304112007-00178f471ffd/go.mod h1:eFgXeM+JhpUbZbTy+KGfVeMgHrjqKauh0OYOInWVE5w= +github.com/instill-ai/component v0.12.0-beta.0.20240307023544-7c27d15e4e01 h1:sfbQZD/vun9+UUusZqtCWMpWxvdc7rAD/UsibNrRpZg= +github.com/instill-ai/component v0.12.0-beta.0.20240307023544-7c27d15e4e01/go.mod h1:Zr3ej9EbkCe+lgSyzKhaqq8+mq84LGGP2F3OWOr/l3c= +github.com/instill-ai/connector v0.13.0-beta.0.20240307024046-140fe8f50ded h1:3RP7caIe2w6rlY9gV4JGpLjiT9gvG2BvAEF8auBDbBI= +github.com/instill-ai/connector v0.13.0-beta.0.20240307024046-140fe8f50ded/go.mod h1:cnvYWlUm35+hHkCh1EhNUfpTMqn6ZzlN5SsgxHC8+Gk= github.com/instill-ai/operator v0.9.0-beta.0.20240304112014-043d5e3b0ebd h1:DnTC/0IiHDDSPE8idE3LYJFtEsK6O1kl8cRZroNkMNg= github.com/instill-ai/operator v0.9.0-beta.0.20240304112014-043d5e3b0ebd/go.mod h1:69LCxn3s4nl7+sUwsHx8hI6kkR0MyyQ59MUKhWVPw7Y= -github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240305055434-f65d3c697ec8 h1:3Nw+/qThtXY5sySXpS6nymNVMkwB8JkkWhtnq27muiY= -github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240305055434-f65d3c697ec8/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73 h1:3YT3WV9F1eltn5x3AhtOuRDO2zY3Aud6nk/som9YQvs= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240306151355-4398dad0ba73/go.mod h1:jhEL0SauySMoPLVvx105DWyThju9sYTbsXIySVCArmM= github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw= github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw= github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg= diff --git a/pkg/service/convert.go b/pkg/service/convert.go index 6b5b61f92..6f2bfcd52 100644 --- a/pkg/service/convert.go +++ b/pkg/service/convert.go @@ -262,9 +262,7 @@ func (s *service) includeOperatorComponentDetail(comp *pipelinePB.OperatorCompon if err != nil { return err } - conf := &structpb.Struct{Fields: map[string]*structpb.Value{}} - conf.Fields["input"] = structpb.NewStructValue(comp.Input) - def, err := s.operator.GetOperatorDefinitionByUID(uid, conf) + def, err := s.operator.GetOperatorDefinitionByUID(uid, comp) if err != nil { return err } @@ -309,9 +307,7 @@ func (s *service) includeConnectorComponentDetail(comp *pipelinePB.ConnectorComp str.Fields["instill_model_backend"] = structpb.NewStringValue(fmt.Sprintf("%s:%d", config.Config.ModelBackend.Host, config.Config.ModelBackend.PublicPort)) str.Fields["instill_mgmt_backend"] = structpb.NewStringValue(fmt.Sprintf("%s:%d", config.Config.MgmtBackend.Host, config.Config.MgmtBackend.PublicPort)) - conf := &structpb.Struct{Fields: map[string]*structpb.Value{}} - conf.Fields["input"] = structpb.NewStructValue(comp.Input) - d, err := s.connector.GetConnectorDefinitionByID(pbConnector.ConnectorDefinition.Id, &str, conf) + d, err := s.connector.GetConnectorDefinitionByID(pbConnector.ConnectorDefinition.Id, &str, comp) if err != nil { return err } @@ -323,9 +319,7 @@ func (s *service) includeConnectorComponentDetail(comp *pipelinePB.ConnectorComp if err != nil { return err } - conf := &structpb.Struct{Fields: map[string]*structpb.Value{}} - conf.Fields["input"] = structpb.NewStructValue(comp.Input) - def, err := s.connector.GetConnectorDefinitionByUID(uid, nil, conf) + def, err := s.connector.GetConnectorDefinitionByUID(uid, nil, comp) if err != nil { return err } @@ -346,6 +340,140 @@ func (s *service) includeConnectorComponentDetail(comp *pipelinePB.ConnectorComp return nil } +func (s *service) includeIteratorComponentDetail(comp *pipelinePB.IteratorComponent, userUID uuid.UUID) error { + + for nestIdx := range comp.Components { + switch comp.Components[nestIdx].Component.(type) { + case *pipelinePB.NestedComponent_ConnectorComponent: + err := s.includeConnectorComponentDetail(comp.Components[nestIdx].GetConnectorComponent(), userUID) + if err != nil { + return err + } + case *pipelinePB.NestedComponent_OperatorComponent: + err := s.includeOperatorComponentDetail(comp.Components[nestIdx].GetOperatorComponent()) + if err != nil { + return err + } + } + } + + dataOutput := &structpb.Struct{Fields: make(map[string]*structpb.Value)} + dataOutput.Fields["type"] = structpb.NewStringValue("object") + dataOutput.Fields["properties"] = structpb.NewStructValue(&structpb.Struct{Fields: make(map[string]*structpb.Value)}) + + for k, v := range comp.OutputElements { + var m *structpb.Value + + var err error + + str := v + if strings.HasPrefix(str, "${") && strings.HasSuffix(str, "}") && strings.Count(str, "${") == 1 { + // TODO + str = str[2:] + str = str[:len(str)-1] + str = strings.ReplaceAll(str, " ", "") + + compID := strings.Split(str, ".")[0] + str = str[len(strings.Split(str, ".")[0]):] + upstreamCompIdx := -1 + for compIdx := range comp.Components { + if comp.Components[compIdx].Id == compID { + upstreamCompIdx = compIdx + } + } + if upstreamCompIdx != -1 { + comp := proto.Clone(comp.Components[upstreamCompIdx]).(*pipelinePB.NestedComponent) + + var walk *structpb.Value + switch comp.Component.(type) { + case *pipelinePB.NestedComponent_ConnectorComponent: + task := comp.GetConnectorComponent().GetTask() + + splits := strings.Split(str, ".") + + if splits[1] == "output" { + walk = structpb.NewStructValue(comp.GetConnectorComponent().GetDefinition().Spec.DataSpecifications[task].Output) + } else if splits[1] == "input" { + walk = structpb.NewStructValue(comp.GetConnectorComponent().GetDefinition().Spec.DataSpecifications[task].Input) + } else { + return fmt.Errorf("generate OpenAPI spec error") + } + str = str[len(splits[1])+1:] + case *pipelinePB.NestedComponent_OperatorComponent: + task := comp.GetOperatorComponent().GetTask() + splits := strings.Split(str, ".") + + if splits[1] == "output" { + walk = structpb.NewStructValue(comp.GetOperatorComponent().GetDefinition().Spec.DataSpecifications[task].Output) + } else if splits[1] == "input" { + walk = structpb.NewStructValue(comp.GetOperatorComponent().GetDefinition().Spec.DataSpecifications[task].Input) + } else { + return fmt.Errorf("generate OpenAPI spec error") + } + str = str[len(splits[1])+1:] + } + + for { + if len(str) == 0 { + break + } + + splits := strings.Split(str, ".") + curr := splits[1] + + if strings.Contains(curr, "[") && strings.Contains(curr, "]") { + target := strings.Split(curr, "[")[0] + if _, ok := walk.GetStructValue().Fields["properties"]; ok { + if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { + break + } + } else { + break + } + walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target].GetStructValue().Fields["items"] + } else { + target := curr + + if _, ok := walk.GetStructValue().Fields["properties"]; ok { + if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { + break + } + } else { + break + } + + walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target] + + } + + str = str[len(curr)+1:] + } + m = structpb.NewStructValue(walk.GetStructValue()) + + } else { + return fmt.Errorf("generate data spec error") + } + + } + + if err != nil { + + } else { + s := &structpb.Struct{Fields: map[string]*structpb.Value{}} + s.Fields["type"] = structpb.NewStringValue("array") + s.Fields["items"] = m + dataOutput.Fields["properties"].GetStructValue().Fields[k] = structpb.NewStructValue(s) + + } + } + + comp.DataSpecification = &pipelinePB.DataSpecification{ + Output: dataOutput, + } + + return nil +} + func (s *service) includeDetailInRecipe(recipe *pipelinePB.Recipe, userUID uuid.UUID) error { for idx := range recipe.Components { @@ -361,20 +489,9 @@ func (s *service) includeDetailInRecipe(recipe *pipelinePB.Recipe, userUID uuid. return err } case *pipelinePB.Component_IteratorComponent: - comps := recipe.Components[idx].GetIteratorComponent().Components - for nestIdx := range recipe.Components[idx].GetIteratorComponent().Components { - switch comps[nestIdx].Component.(type) { - case *pipelinePB.NestedComponent_ConnectorComponent: - err := s.includeConnectorComponentDetail(comps[nestIdx].GetConnectorComponent(), userUID) - if err != nil { - return err - } - case *pipelinePB.NestedComponent_OperatorComponent: - err := s.includeOperatorComponentDetail(comps[nestIdx].GetOperatorComponent()) - if err != nil { - return err - } - } + err := s.includeIteratorComponentDetail(recipe.Components[idx].GetIteratorComponent(), userUID) + if err != nil { + return err } } @@ -605,9 +722,9 @@ func (s *service) DBToPBPipeline(ctx context.Context, dbPipeline *datamodel.Pipe } if pbRecipe != nil && view == ViewFull && startComp != nil && endComp != nil { - spec, err := s.GenerateOpenAPISpec(startComp, endComp, pbRecipe.Components) + spec, err := s.GeneratePipelineDataSpec(startComp, endComp, pbRecipe.Components) if err == nil { - pbPipeline.OpenapiSchema = spec + pbPipeline.DataSpecification = spec } } releases := []*datamodel.PipelineRelease{} @@ -821,9 +938,9 @@ func (s *service) DBToPBPipelineRelease(ctx context.Context, dbPipelineRelease * } if pbRecipe != nil && view == ViewFull && startComp != nil && endComp != nil { - spec, err := s.GenerateOpenAPISpec(startComp, endComp, pbRecipe.Components) + spec, err := s.GeneratePipelineDataSpec(startComp, endComp, pbRecipe.Components) if err == nil { - pbPipelineRelease.OpenapiSchema = spec + pbPipelineRelease.DataSpecification = spec } } if pbPipelineRelease.Uid == latestUUID.String() { @@ -1039,3 +1156,171 @@ func (s *service) convertDatamodelArrayToProtoArray( return pbConnectors, nil } + +// TODO: refactor these codes +func (s *service) GeneratePipelineDataSpec(startCompOrigin *pipelinePB.Component, endCompOrigin *pipelinePB.Component, compsOrigin []*pipelinePB.Component) (*pipelinePB.DataSpecification, error) { + success := true + pipelineDataSpec := &pipelinePB.DataSpecification{} + + dataInput := &structpb.Struct{Fields: make(map[string]*structpb.Value)} + dataInput.Fields["type"] = structpb.NewStringValue("object") + dataInput.Fields["properties"] = structpb.NewStructValue(&structpb.Struct{Fields: make(map[string]*structpb.Value)}) + + startComp := proto.Clone(startCompOrigin).(*pipelinePB.Component) + for k, v := range startComp.GetStartComponent().GetFields() { + b, _ := protojson.Marshal(v) + p := &structpb.Struct{} + _ = protojson.Unmarshal(b, p) + dataInput.Fields["properties"].GetStructValue().Fields[k] = structpb.NewStructValue(p) + } + + // output + dataOutput := &structpb.Struct{Fields: make(map[string]*structpb.Value)} + dataOutput.Fields["type"] = structpb.NewStringValue("object") + dataOutput.Fields["properties"] = structpb.NewStructValue(&structpb.Struct{Fields: make(map[string]*structpb.Value)}) + + endComp := proto.Clone(endCompOrigin).(*pipelinePB.Component) + + for k, v := range endComp.GetEndComponent().Fields { + var m *structpb.Value + + var err error + + str := v.Value + if strings.HasPrefix(str, "${") && strings.HasSuffix(str, "}") && strings.Count(str, "${") == 1 { + // TODO + str = str[2:] + str = str[:len(str)-1] + str = strings.ReplaceAll(str, " ", "") + + compID := strings.Split(str, ".")[0] + str = str[len(strings.Split(str, ".")[0]):] + upstreamCompIdx := -1 + for compIdx := range compsOrigin { + if compsOrigin[compIdx].Id == compID { + upstreamCompIdx = compIdx + } + } + + if upstreamCompIdx != -1 { + comp := proto.Clone(compsOrigin[upstreamCompIdx]).(*pipelinePB.Component) + + var walk *structpb.Value + switch comp.Component.(type) { + case *pipelinePB.Component_IteratorComponent: + + splits := strings.Split(str, ".") + + if splits[1] == "output" { + walk = structpb.NewStructValue(comp.GetIteratorComponent().DataSpecification.Output) + } else { + return nil, fmt.Errorf("generate OpenAPI spec error") + } + str = str[len(splits[1])+1:] + case *pipelinePB.Component_ConnectorComponent: + task := comp.GetConnectorComponent().GetTask() + + splits := strings.Split(str, ".") + + if splits[1] == "output" { + walk = structpb.NewStructValue(comp.GetConnectorComponent().GetDefinition().Spec.DataSpecifications[task].Output) + } else if splits[1] == "input" { + walk = structpb.NewStructValue(comp.GetConnectorComponent().GetDefinition().Spec.DataSpecifications[task].Input) + } else { + return nil, fmt.Errorf("generate OpenAPI spec error") + } + str = str[len(splits[1])+1:] + case *pipelinePB.Component_StartComponent: + walk = structpb.NewStructValue(dataInput) + case *pipelinePB.Component_OperatorComponent: + task := comp.GetOperatorComponent().GetTask() + + splits := strings.Split(str, ".") + + if splits[1] == "output" { + walk = structpb.NewStructValue(comp.GetOperatorComponent().GetDefinition().Spec.DataSpecifications[task].Output) + } else if splits[1] == "input" { + walk = structpb.NewStructValue(comp.GetOperatorComponent().GetDefinition().Spec.DataSpecifications[task].Input) + } else { + return nil, fmt.Errorf("generate OpenAPI spec error") + } + str = str[len(splits[1])+1:] + } + + for { + if len(str) == 0 { + break + } + + splits := strings.Split(str, ".") + curr := splits[1] + + if strings.Contains(curr, "[") && strings.Contains(curr, "]") { + target := strings.Split(curr, "[")[0] + if _, ok := walk.GetStructValue().Fields["properties"]; ok { + if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { + break + } + } else { + break + } + walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target].GetStructValue().Fields["items"] + } else { + target := curr + + if _, ok := walk.GetStructValue().Fields["properties"]; ok { + if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { + break + } + } else { + break + } + + walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target] + + } + + str = str[len(curr)+1:] + } + m = structpb.NewStructValue(walk.GetStructValue()) + + } else { + return nil, fmt.Errorf("generate data spec error") + } + + if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { + m.GetStructValue().Fields["title"] = structpb.NewStringValue(v.Title) + } + if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { + m.GetStructValue().Fields["description"] = structpb.NewStringValue(v.Description) + } + if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { + m.GetStructValue().Fields["instillUIOrder"] = structpb.NewNumberValue(float64(v.InstillUiOrder)) + } + + } else { + m, err = structpb.NewValue(map[string]interface{}{ + "title": v.Title, + "description": v.Description, + "instillUIOrder": v.InstillUiOrder, + "type": "string", + "instillFormat": "string", + }) + } + + if err != nil { + success = false + } else { + dataOutput.Fields["properties"].GetStructValue().Fields[k] = m + } + + } + + if success { + pipelineDataSpec.Input = dataInput + pipelineDataSpec.Output = dataOutput + return pipelineDataSpec, nil + } + return nil, fmt.Errorf("generate data spec error") + +} diff --git a/pkg/service/openapi.go b/pkg/service/openapi.go deleted file mode 100644 index 381fca83d..000000000 --- a/pkg/service/openapi.go +++ /dev/null @@ -1,363 +0,0 @@ -package service - -import ( - "fmt" - "strings" - - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/structpb" - - pipelinePB "github.com/instill-ai/protogen-go/vdp/pipeline/v1beta" -) - -const openAPISchemaTemplate = `{ - "openapi": "3.0.0", - "info": { - "version": "dev", - "title": "Pipeline Trigger" - }, - "paths": { - "/trigger": { - "post": { - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "inputs": { - "type": "array", - "items": {} - } - } - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "outputs": { - "type": "array", - "items": {} - } - } - } - } - } - } - } - } - }, - "/triggerAsync": { - "post": { - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "inputs": { - "type": "array", - "items": {} - } - } - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "operation": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "@type": { - "type": "string" - } - }, - "additionalProperties": {} - }, - "done": { - "type": "boolean" - }, - "error": { - "type": "object", - "properties": { - "code": { - "type": "integer", - "format": "int32" - }, - "message": { - "type": "string" - }, - "details": { - "type": "array", - "items": { - "type": "object", - "properties": { - "@type": { - "type": "string" - } - }, - "additionalProperties": {} - } - } - } - }, - "response": { - "type": "object", - "properties": { - "@type": { - "type": "string" - } - }, - "additionalProperties": {} - } - } - } - } - } - } - } - } - } - } - } - } - }` - -// TODO: refactor these messy code -func (s *service) GenerateOpenAPISpec(startCompOrigin *pipelinePB.Component, endCompOrigin *pipelinePB.Component, compsOrigin []*pipelinePB.Component) (*structpb.Struct, error) { - success := true - template := &structpb.Struct{} - err := protojson.Unmarshal([]byte(openAPISchemaTemplate), template) - if err != nil { - return nil, err - } - - var templateWalk *structpb.Value - - openAPIInput := &structpb.Struct{Fields: make(map[string]*structpb.Value)} - openAPIInput.Fields["type"] = structpb.NewStringValue("object") - openAPIInput.Fields["properties"] = structpb.NewStructValue(&structpb.Struct{Fields: make(map[string]*structpb.Value)}) - - startComp := proto.Clone(startCompOrigin).(*pipelinePB.Component) - for k, v := range startComp.GetStartComponent().GetFields() { - b, _ := protojson.Marshal(v) - p := &structpb.Struct{} - _ = protojson.Unmarshal(b, p) - openAPIInput.Fields["properties"].GetStructValue().Fields[k] = structpb.NewStructValue(p) - } - - templateWalk = template.GetFields()["paths"] - for _, key := range []string{"/trigger", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs", "items"} { - templateWalk = templateWalk.GetStructValue().Fields[key] - } - *templateWalk = *structpb.NewStructValue(openAPIInput) - templateWalk = template.GetFields()["paths"] - for _, key := range []string{"/triggerAsync", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs", "items"} { - templateWalk = templateWalk.GetStructValue().Fields[key] - } - *templateWalk = *structpb.NewStructValue(openAPIInput) - - // output - - openAPIOutput := &structpb.Struct{Fields: make(map[string]*structpb.Value)} - openAPIOutput.Fields["type"] = structpb.NewStringValue("object") - openAPIOutput.Fields["properties"] = structpb.NewStructValue(&structpb.Struct{Fields: make(map[string]*structpb.Value)}) - - endComp := proto.Clone(endCompOrigin).(*pipelinePB.Component) - - for k, v := range endComp.GetEndComponent().Fields { - var m *structpb.Value - - var err error - - str := v.Value - if strings.HasPrefix(str, "${") && strings.HasSuffix(str, "}") && strings.Count(str, "${") == 1 { - // TODO - str = str[2:] - str = str[:len(str)-1] - str = strings.ReplaceAll(str, " ", "") - - compID := strings.Split(str, ".")[0] - str = str[len(strings.Split(str, ".")[0]):] - upstreamCompIdx := -1 - for compIdx := range compsOrigin { - if compsOrigin[compIdx].Id == compID { - upstreamCompIdx = compIdx - } - } - - if upstreamCompIdx != -1 { - comp := proto.Clone(compsOrigin[upstreamCompIdx]).(*pipelinePB.Component) - - var walk *structpb.Value - switch comp.Component.(type) { - case *pipelinePB.Component_IteratorComponent: - // TODO: implement this - continue - case *pipelinePB.Component_ConnectorComponent: - task := comp.GetConnectorComponent().GetTask() - if task == "" { - keys := make([]string, 0, len(comp.GetConnectorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields())) - if len(keys) != 1 { - return nil, fmt.Errorf("must specify a task") - } - task = keys[0] - } - - if _, ok := comp.GetConnectorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields()[task]; !ok { - return nil, fmt.Errorf("generate OpenAPI spec error") - } - walk = comp.GetConnectorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields()[task] - - splits := strings.Split(str, ".") - - if splits[1] == "output" { - for _, key := range []string{"paths", "/execute", "post", "responses", "200", "content", "application/json", "schema", "properties", "outputs", "items"} { - walk = walk.GetStructValue().Fields[key] - } - } else if splits[1] == "input" { - for _, key := range []string{"paths", "/execute", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs", "items"} { - walk = walk.GetStructValue().Fields[key] - } - } else { - return nil, fmt.Errorf("generate OpenAPI spec error") - } - str = str[len(splits[1])+1:] - case *pipelinePB.Component_StartComponent: - walk = structpb.NewStructValue(openAPIInput) - case *pipelinePB.Component_OperatorComponent: - task := comp.GetOperatorComponent().GetTask() - if task == "" { - keys := make([]string, 0, len(comp.GetOperatorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields())) - if len(keys) != 1 { - return nil, fmt.Errorf("must specify a task") - } - task = keys[0] - } - - if _, ok := comp.GetOperatorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields()[task]; !ok { - return nil, fmt.Errorf("generate OpenAPI spec error") - } - - walk = comp.GetOperatorComponent().GetDefinition().Spec.OpenapiSpecifications.GetFields()[task] - - splits := strings.Split(str, ".") - - if splits[1] == "output" { - for _, key := range []string{"paths", "/execute", "post", "responses", "200", "content", "application/json", "schema", "properties", "outputs", "items"} { - walk = walk.GetStructValue().Fields[key] - } - } else if splits[1] == "input" { - for _, key := range []string{"paths", "/execute", "post", "requestBody", "content", "application/json", "schema", "properties", "inputs", "items"} { - walk = walk.GetStructValue().Fields[key] - } - } else { - return nil, fmt.Errorf("generate OpenAPI spec error") - } - str = str[len(splits[1])+1:] - } - - for { - if len(str) == 0 { - break - } - - splits := strings.Split(str, ".") - curr := splits[1] - - if strings.Contains(curr, "[") && strings.Contains(curr, "]") { - target := strings.Split(curr, "[")[0] - if _, ok := walk.GetStructValue().Fields["properties"]; ok { - if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { - break - } - } else { - break - } - walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target].GetStructValue().Fields["items"] - } else { - target := curr - - if _, ok := walk.GetStructValue().Fields["properties"]; ok { - if _, ok := walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target]; !ok { - break - } - } else { - break - } - - walk = walk.GetStructValue().Fields["properties"].GetStructValue().Fields[target] - - } - - str = str[len(curr)+1:] - } - m = structpb.NewStructValue(walk.GetStructValue()) - - } else { - return nil, fmt.Errorf("generate OpenAPI spec error") - } - - if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { - m.GetStructValue().Fields["title"] = structpb.NewStringValue(v.Title) - } - if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { - m.GetStructValue().Fields["description"] = structpb.NewStringValue(v.Description) - } - if m.GetStructValue() != nil && m.GetStructValue().Fields != nil { - m.GetStructValue().Fields["instillUIOrder"] = structpb.NewNumberValue(float64(v.InstillUiOrder)) - } - - } else { - m, err = structpb.NewValue(map[string]interface{}{ - "title": v.Title, - "description": v.Description, - "instillUIOrder": v.InstillUiOrder, - "type": "string", - "instillFormat": "string", - }) - } - - if err != nil { - success = false - } else { - openAPIOutput.Fields["properties"].GetStructValue().Fields[k] = m - } - - } - - templateWalk = template.GetFields()["paths"] - for _, key := range []string{"/trigger", "post", "responses", "200", "content", "application/json", "schema", "properties", "outputs", "items"} { - templateWalk = templateWalk.GetStructValue().Fields[key] - } - *templateWalk = *structpb.NewStructValue(openAPIOutput) - - if success { - return template, nil - } - return nil, fmt.Errorf("generate OpenAPI spec error") - -}