From 16afda749f34f607563edcce37c2412f399b14c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:11:10 +0100 Subject: [PATCH] feat(vdp): add component definition list endpoint (#274) Because - Component page requires unified filtering and pagination. This commit - Adds a "component list" endpoint to the public VDP API. - Unifies the component view and removes connector and operator view enums. - Connector and operator definition messages are unified in `component_definition.proto` - Replaces `CONNECTOR_BLOCKCHAIN` by `CONNECTOR_APPLICATION` ([INS-3739](https://linear.app/instill-ai/issue/INS-3739/rename-blockhain-type-to-application)) - Removes operator / connector definition lookup endpoints (unused). --------- Co-authored-by: droplet-bot --- openapiv2/vdp/service.swagger.yaml | 175 ++++++--- vdp/pipeline/v1beta/common.proto | 18 + .../v1beta/component_definition.proto | 337 ++++++++++++++++++ vdp/pipeline/v1beta/connector.proto | 2 +- .../v1beta/connector_definition.proto | 194 ---------- vdp/pipeline/v1beta/operator_definition.proto | 165 --------- vdp/pipeline/v1beta/pipeline.proto | 17 +- .../v1beta/pipeline_private_service.proto | 20 -- .../v1beta/pipeline_public_service.proto | 12 +- 9 files changed, 484 insertions(+), 456 deletions(-) create mode 100644 vdp/pipeline/v1beta/component_definition.proto delete mode 100644 vdp/pipeline/v1beta/connector_definition.proto delete mode 100644 vdp/pipeline/v1beta/operator_definition.proto diff --git a/openapiv2/vdp/service.swagger.yaml b/openapiv2/vdp/service.swagger.yaml index a08ab0d6..f0edd5dd 100644 --- a/openapiv2/vdp/service.swagger.yaml +++ b/openapiv2/vdp/service.swagger.yaml @@ -2145,7 +2145,7 @@ paths: - name: page_size description: |- The maximum number of connector definitions to return. If this parameter - is unspecified, at most 10 pipelines will be returned. The cap value for + is unspecified, at most 10 definitions will be returned. The cap value for this parameter is 100 (i.e. any value above that will be coerced to 100). in: query required: false @@ -2156,6 +2156,14 @@ paths: in: query required: false type: string + - name: filter + description: |- + Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + expression. + - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. + in: query + required: false + type: string - name: view description: |- View allows clients to specify the desired resource view in the response. @@ -2169,14 +2177,6 @@ paths: enum: - VIEW_BASIC - VIEW_FULL - - name: filter - description: |- - Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter - expression. - - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. - in: query - required: false - type: string tags: - PipelinePublicService /v1beta/{connector_definition_name}: @@ -2253,6 +2253,64 @@ paths: in: query required: false type: string + - name: filter + description: |- + Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + expression. + - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. + in: query + required: false + type: string + - name: view + description: |- + View allows clients to specify the desired resource view in the response. + + - VIEW_BASIC: Default view, only includes basic information (removes the `spec` + field). + - VIEW_FULL: Full representation. + in: query + required: false + type: string + enum: + - VIEW_BASIC + - VIEW_FULL + tags: + - PipelinePublicService + /v1beta/component-definitions: + get: + summary: List component definitions + description: |- + Returns a paginated list of component definitions, regardless their type. + This offers a single source of truth, with pagination and filter + capabilities, for the components that might be used in a VDP pipeline. + operationId: PipelinePublicService_ListComponentDefinitions + responses: + "200": + description: A successful response. + schema: + $ref: '#/definitions/v1betaListComponentDefinitionsResponse' + "401": + description: Returned when the client credentials are not valid. + schema: {} + default: + description: An unexpected error response. + schema: + $ref: '#/definitions/googlerpcStatus' + parameters: + - name: page_size + description: |- + The maximum number of component definitions to return. If this parameter + is unspecified, at most 10 definitions will be returned. The cap value for + this parameter is 100 (i.e. any value above that will be coerced to 100). + in: query + required: false + type: integer + format: int32 + - name: page_token + description: Page token. + in: query + required: false + type: string - name: view description: |- View allows clients to specify the desired resource view in the response. @@ -2270,7 +2328,8 @@ paths: description: |- Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter expression. - - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. + - Example: `component_type="COMPONENT_TYPE_CONNECTOR_AI"`. + - Example: `tasks:"TASK_TEXT_GENERATION"`. in: query required: false type: string @@ -4068,6 +4127,33 @@ definitions: required: - id - definition_name + v1betaComponentDefinition: + type: object + properties: + type: + $ref: '#/definitions/v1betaComponentType' + description: Defines the type of task the component will perform. + readOnly: true + operator_definition: + $ref: '#/definitions/v1betaOperatorDefinition' + title: operator definition detail + readOnly: true + connector_definition: + $ref: '#/definitions/v1betaConnectorDefinition' + title: connector definition detail + readOnly: true + description: ComponentDefinition describes a certain type of Component. + v1betaComponentDefinitionView: + type: string + enum: + - VIEW_BASIC + - VIEW_FULL + description: |- + View defines how a component definition is presented. + + - VIEW_BASIC: Default view, only includes basic information (removes the `spec` + field). + - VIEW_FULL: Full representation. v1betaComponentTask: type: object properties: @@ -4091,15 +4177,15 @@ definitions: enum: - COMPONENT_TYPE_CONNECTOR_AI - COMPONENT_TYPE_CONNECTOR_DATA - - COMPONENT_TYPE_CONNECTOR_BLOCKCHAIN - COMPONENT_TYPE_OPERATOR + - COMPONENT_TYPE_CONNECTOR_APPLICATION description: |- ComponentType defines the component type based on its task features. - COMPONENT_TYPE_CONNECTOR_AI: Connect with an AI model. - COMPONENT_TYPE_CONNECTOR_DATA: Connect with a remote data source. - - COMPONENT_TYPE_CONNECTOR_BLOCKCHAIN: Connect with a blockchain service. - COMPONENT_TYPE_OPERATOR: Manipulate data. + - COMPONENT_TYPE_CONNECTOR_APPLICATION: Connect with an external application. v1betaConnectOrganizationConnectorResponse: type: object properties: @@ -4295,17 +4381,6 @@ definitions: For more information, see [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) in the official documentation. - v1betaConnectorDefinitionView: - type: string - enum: - - VIEW_BASIC - - VIEW_FULL - description: |- - View defines how a connector definition is presented. - - - VIEW_BASIC: Default view, only includes basic information (removes the `spec` - field). - - VIEW_FULL: Full representation. v1betaConnectorSpec: type: object properties: @@ -4338,21 +4413,17 @@ definitions: v1betaConnectorType: type: string enum: - - CONNECTOR_TYPE_SOURCE - - CONNECTOR_TYPE_DESTINATION - CONNECTOR_TYPE_AI - - CONNECTOR_TYPE_BLOCKCHAIN - CONNECTOR_TYPE_DATA - CONNECTOR_TYPE_OPERATOR + - CONNECTOR_TYPE_APPLICATION description: |- ConnectorType defines the connector type based on its task features. - - CONNECTOR_TYPE_SOURCE: Source connector. - - CONNECTOR_TYPE_DESTINATION: Destination connector. - CONNECTOR_TYPE_AI: AI connector. - - CONNECTOR_TYPE_BLOCKCHAIN: Blockchain connector. - CONNECTOR_TYPE_DATA: Data connector. - CONNECTOR_TYPE_OPERATOR: Operator connector. + - CONNECTOR_TYPE_APPLICATION: Application connector. v1betaConnectorView: type: string enum: @@ -4531,6 +4602,23 @@ definitions: $ref: '#/definitions/v1betaPipeline' description: The pipeline resource. description: GetUserPipelineResponse contains the requested pipeline. + v1betaListComponentDefinitionsResponse: + type: object + properties: + component_definitions: + type: array + items: + type: object + $ref: '#/definitions/v1betaComponentDefinition' + description: A list of component definition resources. + next_page_token: + type: string + description: Next page token. + total_size: + type: integer + format: int32 + description: Total number of connector definitions. + description: ListComponentDefinitionsResponse contains a list of component definitions. v1betaListConnectorDefinitionsResponse: type: object properties: @@ -4773,15 +4861,6 @@ definitions: description: |- LookUpConnectorAdminResponse represents a response for a connector. - v1betaLookUpConnectorDefinitionAdminResponse: - type: object - properties: - connector_definition: - $ref: '#/definitions/v1betaConnectorDefinition' - description: The requested connector. - description: |- - LookUpConnectorDefinitionAdminResponse represents a response for a - connector definition. v1betaLookUpConnectorResponse: type: object properties: @@ -4789,15 +4868,6 @@ definitions: $ref: '#/definitions/v1betaConnector' description: The requested connector. description: LookUpConnectorResponse represents a response for a connector. - v1betaLookUpOperatorDefinitionAdminResponse: - type: object - properties: - operator_definition: - $ref: '#/definitions/v1betaOperatorDefinition' - description: The requested operator definition. - description: |- - LookUpOperatorDefinitionAdminResponse represents a response for an operator - definition. v1betaLookUpPipelineAdminResponse: type: object properties: @@ -4893,17 +4963,6 @@ definitions: For more information, see [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) in the official documentation. - v1betaOperatorDefinitionView: - type: string - enum: - - VIEW_BASIC - - VIEW_FULL - description: |- - View defines how an operator definition is presented. - - - VIEW_BASIC: Default view, only includes basic information (removes the `spec` - field). - - VIEW_FULL: Full representation. v1betaOperatorSpec: type: object properties: diff --git a/vdp/pipeline/v1beta/common.proto b/vdp/pipeline/v1beta/common.proto index 581f8ff9..88266e3b 100644 --- a/vdp/pipeline/v1beta/common.proto +++ b/vdp/pipeline/v1beta/common.proto @@ -101,3 +101,21 @@ message ComponentTask { // Description contains information about the task. string description = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } + +// ComponentType defines the component type based on its task features. +enum ComponentType { + // Unspecified. + COMPONENT_TYPE_UNSPECIFIED = 0; + // Connect with an AI model. + COMPONENT_TYPE_CONNECTOR_AI = 1; + // Connect with a remote data source. + COMPONENT_TYPE_CONNECTOR_DATA = 2; + // 3 Is reserved for the connector blockchain type, deprecated by + // COMPONENT_TYPE_CONNECTOR_APPLICATION. + reserved 3; + // Manipulate data. + COMPONENT_TYPE_OPERATOR = 4; + // Connect with an external application. + COMPONENT_TYPE_CONNECTOR_APPLICATION = 5; + +} diff --git a/vdp/pipeline/v1beta/component_definition.proto b/vdp/pipeline/v1beta/component_definition.proto new file mode 100644 index 00000000..fc44a23b --- /dev/null +++ b/vdp/pipeline/v1beta/component_definition.proto @@ -0,0 +1,337 @@ +syntax = "proto3"; + +package vdp.pipeline.v1beta; + +// Google API +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +// Protocol Buffers Well-Known Types +import "google/protobuf/struct.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +// VDP definitions +import "vdp/pipeline/v1beta/common.proto"; + +/////////////////////////////////////////////////////////////////////// +// Domain types +/////////////////////////////////////////////////////////////////////// + +// ComponentDefinition describes a certain type of Component. +message ComponentDefinition { + // View defines how a component definition is presented. + enum View { + // Unspecified, equivalent to BASIC. + VIEW_UNSPECIFIED = 0; + // Default view, only includes basic information (removes the `spec` + // field). + VIEW_BASIC = 1; + // Full representation. + VIEW_FULL = 2; + } + + // Defines the type of task the component will perform. + ComponentType type = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The component definition details. + oneof definition { + // operator definition detail + OperatorDefinition operator_definition = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // connector definition detail + ConnectorDefinition connector_definition = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + } +} + +// ConnectorSpec represents a specification data model. +message ConnectorSpec { + // Resource specification. + google.protobuf.Struct resource_specification = 2 [(google.api.field_behavior) = REQUIRED]; + // Component specification. + google.protobuf.Struct component_specification = 3 [(google.api.field_behavior) = REQUIRED]; + // OpenAPI specification. + google.protobuf.Struct openapi_specifications = 4 [(google.api.field_behavior) = REQUIRED]; +} + +// ConnectorType defines the connector type based on its task features. +enum ConnectorType { + // Unspecified. + CONNECTOR_TYPE_UNSPECIFIED = 0; + // 1 Is reserved for the source connector. + reserved 1; + // 2 Is reserved for the destination connector. + reserved 2; + // AI connector. + CONNECTOR_TYPE_AI = 3; + // 4 is reserved for the blockchain connector (deprecated by + // CONNECTOR_TYPE_APPLICATION). + reserved 4; + // Data connector. + CONNECTOR_TYPE_DATA = 5; + // Operator connector. + CONNECTOR_TYPE_OPERATOR = 6; + // Application connector. + CONNECTOR_TYPE_APPLICATION = 7; +} + +// A Connector is a type of pipeline component that queries, processes or sends +// the ingested unstructured data to a service or app. Users need to configure +// their connectors (e.g. by providing an API token to a remote service). A +// ConnectorDefinition describes a certain type of Connector. +// +// For more information, see +// [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) +// in the official documentation. +message ConnectorDefinition { + option (google.api.resource) = { +type: "api.instill.tech/ConnectorDefinition" + pattern: "connector-definitions/{id}" + pattern: "connector-definitions/{uid}" + }; + + // The name of the connector definition, defined by its ID. + // - Format: `connector-definitions/{id} + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition UUID. + string uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition resource ID (used in `name` as the last segment). This + // conforms to RFC-1034, which restricts to letters, numbers, and hyphen, + // with the first character a letter, the last a letter or a number, and a 63 + // character maximum. + string id = 3 [(google.api.field_behavior) = IMMUTABLE]; + // Connector definition title. + string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition documentation URL. + string documentation_url = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition icon. This is a path that's relative to the root of + // the connector implementation (see `source_url`) and that allows clients + // frontend applications to pull and locate the icons. + string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition specification. + ConnectorSpec spec = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition type. + ConnectorType type = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition tombstone. If true, this configuration is permanently + // off. Otherwise, the configuration is active. + bool tombstone = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The public flag determines whether this connector definition is available + // to all workspaces. + bool public = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition custom flag, i.e., whether this is a custom + // connector definition. + bool custom = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; + // field 12 is reserved for 'icon_url'. + reserved 12; + // Connector definition vendor name. + string vendor = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Vendor-specific attributes. + google.protobuf.Struct vendor_attributes = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Source code URL. This points to the source code where the connector is + // implemented. + string source_url = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connector definition version. This is a string that fulfills the SemVer + // specification (e.g. `1.0.0-beta`). + string version = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; + // List of tasks that can be executed by the connector. + repeated ComponentTask tasks = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// OperatorSpec represents a specification data model. +message OperatorSpec { + // Component specification. + google.protobuf.Struct component_specification = 1 [(google.api.field_behavior) = REQUIRED]; + // OpenAPI specification. + google.protobuf.Struct openapi_specifications = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// An Operator is a type of pipeline component that performs data injection and +// manipulation. OperatorDefinition describes a certain type of operator. +// +// For more information, see +// [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) +// in the official documentation. +message OperatorDefinition { + option (google.api.resource) = { +type: "api.instill.tech/OperatorDefinition" + pattern: "operator-definitions/{id}" + pattern: "operator-definitions/{uid}" + }; + + // The name of the operator definition. + // - Format: `operator-definitions/*` + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition UUID. + string uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition resource ID (used in `name` as the last segment). + // This conforms to RFC-1034, which restricts to letters, numbers, and + // hyphen, with the first character a letter, the last a letter or a number, + // and a 63 character maximum. + string id = 3 [(google.api.field_behavior) = IMMUTABLE]; + // Operator definition title. + string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition documentation URL. + string documentation_url = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition icon. This is a path that's relative to the root of + // the operator implementation (see `source_url`) and that allows clients + // frontend applications to pull and locate the icons. + string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition specification. + OperatorSpec spec = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition tombstone. If true, this configuration is permanently + // off. Otherwise, the configuration is active. + bool tombstone = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The public flag determines whether this operator definition is available + // to all workspaces. + bool public = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The custom flag determines whether this is a custom operator definition. + bool custom = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; + // field 11 is reserved for 'icon_url'. + reserved 11; + // Source code URL. This points to the source code where the operator is + // implemented. + string source_url = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Operator definition version. This is a string that fulfills the SemVer + // specification (e.g. `1.0.0-beta`). + string version = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; + // List of tasks that can be executed by the operator. + repeated ComponentTask tasks = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +/////////////////////////////////////////////////////////////////////// +// API messages +/////////////////////////////////////////////////////////////////////// + +// ListComponentDefinitionsRequest represents a request to list component +// definitions. +message ListComponentDefinitionsRequest { + // The maximum number of component definitions to return. If this parameter + // is unspecified, at most 10 definitions will be returned. The cap value for + // this parameter is 100 (i.e. any value above that will be coerced to 100). + optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + // Page token. + optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + // View allows clients to specify the desired resource view in the response. + optional ComponentDefinition.View view = 3 [(google.api.field_behavior) = OPTIONAL]; + // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + // expression. + // - Example: `component_type="COMPONENT_TYPE_CONNECTOR_AI"`. + // - Example: `tasks:"TASK_TEXT_GENERATION"`. + optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// ListComponentDefinitionsResponse contains a list of component definitions. +message ListComponentDefinitionsResponse { + // A list of component definition resources. + repeated ComponentDefinition component_definitions = 1; + // Next page token. + string next_page_token = 2; + // Total number of connector definitions. + int32 total_size = 3; +} + +// ListConnectorDefinitionsRequest represents a request to list connector +// definitions. +message ListConnectorDefinitionsRequest { + // The maximum number of connector definitions to return. If this parameter + // is unspecified, at most 10 definitions will be returned. The cap value for + // this parameter is 100 (i.e. any value above that will be coerced to 100). + optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + // Page token. + optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + // field 3 is reserved for the "view" property of ConnectorDefinition.View + // type, removed in favour of ComponentDefinition.View. + reserved 3; + // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + // expression. + // - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. + optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + // View allows clients to specify the desired resource view in the response. + optional ComponentDefinition.View view = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// ListConnectorDefinitionsResponse contains a list of connector definitions. +message ListConnectorDefinitionsResponse { + // A list of connector definition resources. + repeated ConnectorDefinition connector_definitions = 1; + // Next page token. + string next_page_token = 2; + // Total number of connector definitions. + int32 total_size = 3; +} + +// GetConnectorDefinitionRequest represents a request to fetch the details of a +// connector definition. +message GetConnectorDefinitionRequest { + // The resource name of the connector definition, which allows its access by ID. + // - Format: `connector-definitions/{id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = {type: "api.instill.tech/ConnectorDefinition"}, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { +field_configuration: {path_param_name: "connector_definition_name"} + } + ]; + // field 2 is reserved for the "view" property of ConnectorDefinition.View + // type, removed in favour of ComponentDefinition.View. + reserved 2; + // View allows clients to specify the desired resource view in the response. + optional ComponentDefinition.View view = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// GetConnectorDefinitionResponse contains the requested connector definition. +message GetConnectorDefinitionResponse { + // The connector definition resource. + ConnectorDefinition connector_definition = 1; +} + +// ListOperatorDefinitionsRequest represents a request to list operator +// definitions. +message ListOperatorDefinitionsRequest { + // The maximum number of OperatorDefinitions to return. The + // service may return fewer than this value. If unspecified, at most 10 + // OperatorDefinitions will be returned. The maximum value is 100; + // values above 100 will be coerced to 100. + optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + // Page token. + optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + // field 3 is reserved for the "view" property of OperatorDefinition.View + // type, removed in favour of ComponentDefinition.View. + reserved 3; + // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + // expression. + // - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. + optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; + // View allows clients to specify the desired resource view in the response. + optional ComponentDefinition.View view = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +// ListOperatorDefinitionsResponse contains a list of operator definitions. +message ListOperatorDefinitionsResponse { + // A list of operator definition resources. + repeated OperatorDefinition operator_definitions = 1; + // Next page token. + string next_page_token = 2; + // Total number of operator definitions. + int32 total_size = 3; +} + +// GetOperatorDefinitionRequest represents a request to fetch the details of a +// operator definition. +message GetOperatorDefinitionRequest { + // The resource name of the operator definition, which allows its access by ID. + // - Format: `operator-definitions/{id}`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = {type: "api.instill.tech/OperatorDefinition"}, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { +field_configuration: {path_param_name: "operator_definition_name"} + } + ]; + // field 2 is reserved for the "view" property of OperatorDefinition.View + // type, removed in favour of ComponentDefinition.View. + reserved 2; + // View allows clients to specify the desired resource view in the response. + optional ComponentDefinition.View view = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// GetOperatorDefinitionResponse contains the requested operator definition. +message GetOperatorDefinitionResponse { + // The operator definition resource. + OperatorDefinition operator_definition = 1; +} diff --git a/vdp/pipeline/v1beta/connector.proto b/vdp/pipeline/v1beta/connector.proto index b87e06ad..eb063018 100644 --- a/vdp/pipeline/v1beta/connector.proto +++ b/vdp/pipeline/v1beta/connector.proto @@ -14,7 +14,7 @@ import "google/protobuf/timestamp.proto"; // OpenAPI definition import "protoc-gen-openapiv2/options/annotations.proto"; // VDP definitions -import "vdp/pipeline/v1beta/connector_definition.proto"; +import "vdp/pipeline/v1beta/component_definition.proto"; // A Connector is a type of pipeline component that queries, processes or sends // the ingested unstructured data to a service or app. Users need to configure diff --git a/vdp/pipeline/v1beta/connector_definition.proto b/vdp/pipeline/v1beta/connector_definition.proto deleted file mode 100644 index 9d057fc0..00000000 --- a/vdp/pipeline/v1beta/connector_definition.proto +++ /dev/null @@ -1,194 +0,0 @@ -syntax = "proto3"; - -package vdp.pipeline.v1beta; - -import "google/api/field_behavior.proto"; -// Google API -import "google/api/resource.proto"; -// Protocol Buffers Well-Known Types -import "google/protobuf/struct.proto"; -import "protoc-gen-openapiv2/options/annotations.proto"; -// VDP definitions -import "vdp/pipeline/v1beta/common.proto"; - -//////////////////////////////////// - -// ConnectorSpec represents a specification data model. -message ConnectorSpec { - // Resource specification. - google.protobuf.Struct resource_specification = 2 [(google.api.field_behavior) = REQUIRED]; - // Component specification. - google.protobuf.Struct component_specification = 3 [(google.api.field_behavior) = REQUIRED]; - // OpenAPI specification. - google.protobuf.Struct openapi_specifications = 4 [(google.api.field_behavior) = REQUIRED]; -} - -////////////////////////////////////////////////////////////////////////// - -// ConnectorType defines the connector type based on its task features. -enum ConnectorType { - // Unspecified. - CONNECTOR_TYPE_UNSPECIFIED = 0; - // Source connector. - CONNECTOR_TYPE_SOURCE = 1; - // Destination connector. - CONNECTOR_TYPE_DESTINATION = 2; - // AI connector. - CONNECTOR_TYPE_AI = 3; - // Blockchain connector. - CONNECTOR_TYPE_BLOCKCHAIN = 4; - // Data connector. - CONNECTOR_TYPE_DATA = 5; - // Operator connector. - CONNECTOR_TYPE_OPERATOR = 6; -} - -// A Connector is a type of pipeline component that queries, processes or sends -// the ingested unstructured data to a service or app. Users need to configure -// their connectors (e.g. by providing an API token to a remote service). A -// ConnectorDefinition describes a certain type of Connector. -// -// For more information, see -// [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) -// in the official documentation. -message ConnectorDefinition { - option (google.api.resource) = { - type: "api.instill.tech/ConnectorDefinition" - pattern: "connector-definitions/{id}" - pattern: "connector-definitions/{uid}" - }; - - // View defines how a connector definition is presented. - enum View { - // Unspecified, equivalent to BASIC. - VIEW_UNSPECIFIED = 0; - // Default view, only includes basic information (removes the `spec` - // field). - VIEW_BASIC = 1; - // Full representation. - VIEW_FULL = 2; - } - - // The name of the connector definition, defined by its ID. - // - Format: `connector-definitions/{id} - string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition UUID. - string uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition resource ID (used in `name` as the last segment). This - // conforms to RFC-1034, which restricts to letters, numbers, and hyphen, - // with the first character a letter, the last a letter or a number, and a 63 - // character maximum. - string id = 3 [(google.api.field_behavior) = IMMUTABLE]; - // Connector definition title. - string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition documentation URL. - string documentation_url = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition icon. This is a path that's relative to the root of - // the connector implementation (see `source_url`) and that allows clients - // frontend applications to pull and locate the icons. - string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition specification. - ConnectorSpec spec = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition type. - ConnectorType type = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition tombstone. If true, this configuration is permanently - // off. Otherwise, the configuration is active. - bool tombstone = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - // The public flag determines whether this connector definition is available - // to all workspaces. - bool public = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition custom flag, i.e., whether this is a custom - // connector definition. - bool custom = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; - // field 12 is reserved for 'icon_url'. - reserved 12; - // Connector definition vendor name. - string vendor = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Vendor-specific attributes. - google.protobuf.Struct vendor_attributes = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Source code URL. This points to the source code where the connector is - // implemented. - string source_url = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Connector definition version. This is a string that fulfills the SemVer - // specification (e.g. `1.0.0-beta`). - string version = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; - // List of tasks that can be executed by the connector. - repeated ComponentTask tasks = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -/////////////////////////////////////////////////////////////////////// -// RPC messages -/////////////////////////////////////////////////////////////////////// - -// ListConnectorDefinitionsRequest represents a request to list connector -// definitions. -message ListConnectorDefinitionsRequest { - // The maximum number of connector definitions to return. If this parameter - // is unspecified, at most 10 pipelines will be returned. The cap value for - // this parameter is 100 (i.e. any value above that will be coerced to 100). - optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; - // Page token. - optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; - // View allows clients to specify the desired resource view in the response. - optional ConnectorDefinition.View view = 3 [(google.api.field_behavior) = OPTIONAL]; - // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter - // expression. - // - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. - optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// ListConnectorDefinitionsResponse contains a list of connector definitions. -message ListConnectorDefinitionsResponse { - // A list of connector definition resources. - repeated ConnectorDefinition connector_definitions = 1; - // Next page token. - string next_page_token = 2; - // Total number of connector definitions. - int32 total_size = 3; -} - -// GetConnectorDefinitionRequest represents a request to fetch the details of a -// connector definition. -message GetConnectorDefinitionRequest { - // The resource name of the connector definition, which allows its access by ID. - // - Format: `connector-definitions/{id}`. - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = {type: "api.instill.tech/ConnectorDefinition"}, - (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - field_configuration: {path_param_name: "connector_definition_name"} - } - ]; - // View allows clients to specify the desired resource view in the response. - optional ConnectorDefinition.View view = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// GetConnectorDefinitionResponse contains the requested connector definition. -message GetConnectorDefinitionResponse { - // The connector definition resource. - ConnectorDefinition connector_definition = 1; -} - -// LookUpConnectorDefinitionAdminRequest represents a request by an admin to -// query a connector definition by its UID. -message LookUpConnectorDefinitionAdminRequest { - // The permalink of the connector definition, which allows its access by UID. - // - Format: `connector-definitions/{uid}` - string permalink = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = {type: "api.instill.tech/ConnectorDefinition"}, - (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - field_configuration: {path_param_name: "connector_definition_permalink"} - } - ]; - // View allows clients to specify the desired connector definition view in - // the response. - optional ConnectorDefinition.View view = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// LookUpConnectorDefinitionAdminResponse represents a response for a -// connector definition. -message LookUpConnectorDefinitionAdminResponse { - // The requested connector. - ConnectorDefinition connector_definition = 1; -} diff --git a/vdp/pipeline/v1beta/operator_definition.proto b/vdp/pipeline/v1beta/operator_definition.proto deleted file mode 100644 index 603a3ae1..00000000 --- a/vdp/pipeline/v1beta/operator_definition.proto +++ /dev/null @@ -1,165 +0,0 @@ -syntax = "proto3"; - -package vdp.pipeline.v1beta; - -// Google API -import "google/api/field_behavior.proto"; -import "google/api/resource.proto"; -// Protocol Buffers Well-Known Types -import "google/protobuf/struct.proto"; -// OpenAPI definition -import "protoc-gen-openapiv2/options/annotations.proto"; -// VDP definitions -import "vdp/pipeline/v1beta/common.proto"; - -// OperatorSpec represents a specification data model. -message OperatorSpec { - // Component specification. - google.protobuf.Struct component_specification = 1 [(google.api.field_behavior) = REQUIRED]; - // OpenAPI specification. - google.protobuf.Struct openapi_specifications = 2 [(google.api.field_behavior) = REQUIRED]; -} - -////////////////////////////////////////////////////////////////////////// - -// An Operator is a type of pipeline component that performs data injection and -// manipulation. OperatorDefinition describes a certain type of operator. -// -// For more information, see -// [Component](https://www.instill.tech/docs/latest/core/concepts/pipeline#pipeline-component) -// in the official documentation. -message OperatorDefinition { - option (google.api.resource) = { - type: "api.instill.tech/OperatorDefinition" - pattern: "operator-definitions/{id}" - pattern: "operator-definitions/{uid}" - }; - - // View defines how an operator definition is presented. - enum View { - // Unspecified, equivalent to BASIC. - VIEW_UNSPECIFIED = 0; - // Default view, only includes basic information (removes the `spec` - // field). - VIEW_BASIC = 1; - // Full representation. - VIEW_FULL = 2; - } - - // The name of the operator definition. - // - Format: `operator-definitions/*` - string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition UUID. - string uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition resource ID (used in `name` as the last segment). - // This conforms to RFC-1034, which restricts to letters, numbers, and - // hyphen, with the first character a letter, the last a letter or a number, - // and a 63 character maximum. - string id = 3 [(google.api.field_behavior) = IMMUTABLE]; - // Operator definition title. - string title = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition documentation URL. - string documentation_url = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition icon. This is a path that's relative to the root of - // the operator implementation (see `source_url`) and that allows clients - // frontend applications to pull and locate the icons. - string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition specification. - OperatorSpec spec = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition tombstone. If true, this configuration is permanently - // off. Otherwise, the configuration is active. - bool tombstone = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - // The public flag determines whether this operator definition is available - // to all workspaces. - bool public = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - // The custom flag determines whether this is a custom operator definition. - bool custom = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; - // field 11 is reserved for 'icon_url'. - reserved 11; - // Source code URL. This points to the source code where the operator is - // implemented. - string source_url = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Operator definition version. This is a string that fulfills the SemVer - // specification (e.g. `1.0.0-beta`). - string version = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; - // List of tasks that can be executed by the operator. - repeated ComponentTask tasks = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -/////////////////////////////////////////////////////////////////////// -// RPC messages -/////////////////////////////////////////////////////////////////////// - -// ListOperatorDefinitionsRequest represents a request to list operator -// definitions. -message ListOperatorDefinitionsRequest { - // The maximum number of OperatorDefinitions to return. The - // service may return fewer than this value. If unspecified, at most 10 - // OperatorDefinitions will be returned. The maximum value is 100; - // values above 100 will be coerced to 100. - optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; - // Page token. - optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; - // View allows clients to specify the desired resource view in the response. - optional OperatorDefinition.View view = 3 [(google.api.field_behavior) = OPTIONAL]; - // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter - // expression. - // - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`. - optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// ListOperatorDefinitionsResponse contains a list of operator definitions. -message ListOperatorDefinitionsResponse { - // A list of operator definition resources. - repeated OperatorDefinition operator_definitions = 1; - // Next page token. - string next_page_token = 2; - // Total number of operator definitions. - int32 total_size = 3; -} - -// GetOperatorDefinitionRequest represents a request to fetch the details of a -// operator definition. -message GetOperatorDefinitionRequest { - // The resource name of the operator definition, which allows its access by ID. - // - Format: `operator-definitions/{id}`. - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = {type: "api.instill.tech/OperatorDefinition"}, - (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - field_configuration: {path_param_name: "operator_definition_name"} - } - ]; - // View allows clients to specify the desired resource view in the response. - optional OperatorDefinition.View view = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// GetOperatorDefinitionResponse contains the requested operator definition. -message GetOperatorDefinitionResponse { - // The operator definition resource. - OperatorDefinition operator_definition = 1; -} - -// LookUpOperatorDefinitionAdminRequest represents a request by an admin to -// query an operator definition by its UID. -message LookUpOperatorDefinitionAdminRequest { - // The permalink of the operator definition, which allows its access by UID. - // - Format: `operator-definitions/{uid}`. - string permalink = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = {type: "api.instill.tech/OperatorDefinition"}, - (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - field_configuration: {path_param_name: "operator_definition_permalink"} - } - ]; - // View allows clients to specify the desired operator definition view in the - // response. - optional OperatorDefinition.View view = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// LookUpOperatorDefinitionAdminResponse represents a response for an operator -// definition. -message LookUpOperatorDefinitionAdminResponse { - // The requested operator definition. - OperatorDefinition operator_definition = 1; -} diff --git a/vdp/pipeline/v1beta/pipeline.proto b/vdp/pipeline/v1beta/pipeline.proto index 9a2c97d6..6f5b874b 100644 --- a/vdp/pipeline/v1beta/pipeline.proto +++ b/vdp/pipeline/v1beta/pipeline.proto @@ -17,9 +17,8 @@ import "google/protobuf/timestamp.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; // VDP definitions import "vdp/pipeline/v1beta/common.proto"; +import "vdp/pipeline/v1beta/component_definition.proto"; import "vdp/pipeline/v1beta/connector.proto"; -import "vdp/pipeline/v1beta/connector_definition.proto"; -import "vdp/pipeline/v1beta/operator_definition.proto"; // LivenessRequest represents a request to check a service liveness status message LivenessRequest { @@ -45,20 +44,6 @@ message ReadinessResponse { common.healthcheck.v1beta.HealthCheckResponse health_check_response = 1; } -// ComponentType defines the component type based on its task features. -enum ComponentType { - // Unspecified. - COMPONENT_TYPE_UNSPECIFIED = 0; - // Connect with an AI model. - COMPONENT_TYPE_CONNECTOR_AI = 1; - // Connect with a remote data source. - COMPONENT_TYPE_CONNECTOR_DATA = 2; - // Connect with a blockchain service. - COMPONENT_TYPE_CONNECTOR_BLOCKCHAIN = 3; - // Manipulate data. - COMPONENT_TYPE_OPERATOR = 4; -} - // Component is the fundamental building block in pipelines. // // For more information, see [Pipeline diff --git a/vdp/pipeline/v1beta/pipeline_private_service.proto b/vdp/pipeline/v1beta/pipeline_private_service.proto index c798ebb5..0a7c756f 100644 --- a/vdp/pipeline/v1beta/pipeline_private_service.proto +++ b/vdp/pipeline/v1beta/pipeline_private_service.proto @@ -9,8 +9,6 @@ import "google/api/visibility.proto"; // OpenAPI definition import "protoc-gen-openapiv2/options/annotations.proto"; import "vdp/pipeline/v1beta/connector.proto"; -import "vdp/pipeline/v1beta/connector_definition.proto"; -import "vdp/pipeline/v1beta/operator_definition.proto"; import "vdp/pipeline/v1beta/pipeline.proto"; // PipelinePrivateService defines private methods to interact with Pipeline @@ -35,15 +33,6 @@ service PipelinePrivateService { option (google.api.method_signature) = "permalink"; } - // Get an operator definition by UID (admin only) - // - // This is a *private* method that allows admin users to access an operator - // definition by its UID. - rpc LookUpOperatorDefinitionAdmin(LookUpOperatorDefinitionAdminRequest) returns (LookUpOperatorDefinitionAdminResponse) { - option (google.api.http) = {get: "/v1beta/admin/{permalink=operator-definitions/*}/lookUp"}; - option (google.api.method_signature) = "permalink"; - } - // List pipeline releases (admin only) // // This is a *private* method that allows admin users to list *all* pipeline @@ -52,15 +41,6 @@ service PipelinePrivateService { option (google.api.http) = {get: "/v1beta/admin/releases"}; } - // Get a connector definition by UID (admin only) - // - // This is a *private* method that allows admin users to access a connector - // definition by its UID. - rpc LookUpConnectorDefinitionAdmin(LookUpConnectorDefinitionAdminRequest) returns (LookUpConnectorDefinitionAdminResponse) { - option (google.api.http) = {get: "/v1beta/admin/{permalink=connector-definitions/*}/lookUp"}; - option (google.api.method_signature) = "permalink"; - } - // List connectors (admin only) // // This is a *private* method that allows admin users to list *all* connectors. diff --git a/vdp/pipeline/v1beta/pipeline_public_service.proto b/vdp/pipeline/v1beta/pipeline_public_service.proto index 7c73da6e..25f72a24 100644 --- a/vdp/pipeline/v1beta/pipeline_public_service.proto +++ b/vdp/pipeline/v1beta/pipeline_public_service.proto @@ -10,9 +10,8 @@ import "google/api/visibility.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; // VDP definitions import "vdp/pipeline/v1beta/common.proto"; +import "vdp/pipeline/v1beta/component_definition.proto"; import "vdp/pipeline/v1beta/connector.proto"; -import "vdp/pipeline/v1beta/connector_definition.proto"; -import "vdp/pipeline/v1beta/operator_definition.proto"; import "vdp/pipeline/v1beta/pipeline.proto"; // VDP @@ -625,6 +624,15 @@ service PipelinePublicService { option (google.api.http) = {get: "/v1beta/operator-definitions"}; } + // List component definitions + // + // Returns a paginated list of component definitions, regardless their type. + // This offers a single source of truth, with pagination and filter + // capabilities, for the components that might be used in a VDP pipeline. + rpc ListComponentDefinitions(ListComponentDefinitionsRequest) returns (ListComponentDefinitionsResponse) { + option (google.api.http) = {get: "/v1beta/component-definitions"}; + } + // Get operator definition // // Returns the details of an operator definition.