From 0d48eef1d92b4efaf62e6d4748f1d3467e6b7b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:18:23 +0200 Subject: [PATCH] feat(vdp): add Instill Integration endpoints (#418) Because - We want to let users reference a full configuration for pipeline components and include both form (key-value) and OAuth integration methods. This commit - Adds the endpoints and entities to manage integrations. --------- Co-authored-by: droplet-bot --- openapiv2/vdp/service.swagger.yaml | 2 + vdp/pipeline/v1beta/common.proto | 13 + vdp/pipeline/v1beta/integration.proto | 246 ++++++++++++++++++ vdp/pipeline/v1beta/openapi.proto.templ | 4 + .../v1beta/pipeline_public_service.proto | 109 ++++++++ 5 files changed, 374 insertions(+) create mode 100644 vdp/pipeline/v1beta/integration.proto diff --git a/openapiv2/vdp/service.swagger.yaml b/openapiv2/vdp/service.swagger.yaml index bb261c70..be7e69ea 100644 --- a/openapiv2/vdp/service.swagger.yaml +++ b/openapiv2/vdp/service.swagger.yaml @@ -21,6 +21,8 @@ tags: description: Pipeline Trigger endpoints - name: Secret description: Namespace Secret endpoints + - name: Integration + description: Namespace Integration endpoints - name: Utils description: Utils endpoints host: api.instill.tech diff --git a/vdp/pipeline/v1beta/common.proto b/vdp/pipeline/v1beta/common.proto index d4757a8f..40b004cf 100644 --- a/vdp/pipeline/v1beta/common.proto +++ b/vdp/pipeline/v1beta/common.proto @@ -122,3 +122,16 @@ enum ComponentType { // Generic. COMPONENT_TYPE_GENERIC = 6; } + +// View defines how a resource is presented. Most resources can share this view +// definition, the particular meaning of each value should be defined in the +// resource itself. Certain resources might have their own View definition if +// they need to implement more than 2 (basic / full) views. +enum View { + // Unspecified, equivalent to BASIC. + VIEW_UNSPECIFIED = 0; + // Default view. + VIEW_BASIC = 1; + // Full representation. + VIEW_FULL = 2; +} diff --git a/vdp/pipeline/v1beta/integration.proto b/vdp/pipeline/v1beta/integration.proto new file mode 100644 index 00000000..5187eadc --- /dev/null +++ b/vdp/pipeline/v1beta/integration.proto @@ -0,0 +1,246 @@ +syntax = "proto3"; + +package vdp.pipeline.v1beta; + +// Google API +import "google/api/field_behavior.proto"; +// Protocol Buffers Well-Known types +import "google/protobuf/field_mask.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +// VDP definitions +import "vdp/pipeline/v1beta/common.proto"; + +// Connection contains the parameters to communicate with a 3rd party app. A +// component may reference a connection in their setup. One connection may be +// used by several components and pipelines. +message Connection { + // Method defines how the connection is set up. + enum Method { + // Unspecified. + METHOD_UNSPECIFIED = 0; + // Key-value collection. The user is responsible of fetching the connection + // details from the 3rd party service. + METHOD_DICTIONARY = 1; + // Access token created via OAuth 2.0 authorization. + METHOD_OAUTH = 2; + } + // UUID-formatted unique identifier. + string uid = 1 [ + (google.api.field_behavior) = OUTPUT_ONLY, + (google.api.field_behavior) = IMMUTABLE + ]; + // ID. + string id = 2 [(google.api.field_behavior) = REQUIRED]; + // ID of the namespace owning the connection. + string namespace_id = 3 [(google.api.field_behavior) = REQUIRED]; + // Integration ID. It determines for which type of components can reference + // this connection. + string integration_id = 4 [(google.api.field_behavior) = REQUIRED]; + // Integration title. This helps the console display the results grouped by + // integration ID without needing an extra call to fetch title by integration + // ID. + string integration_title = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The IDs of the pipelines that use this integration. All the pipelines will + // be within the same namespace as the connection. + // The length of the list will be capped to 100. + repeated string pipeline_ids = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Connection method. It references the setup schema provided by the + // integration. + Method method = 7 [(google.api.field_behavior) = REQUIRED]; + // Connection details. This field is required on creation, optional on view. + // When viewing the connection details, the setup values will be redacted. + google.protobuf.Struct setup = 8 [(google.api.field_behavior) = REQUIRED]; + // View defines how the integration is presented. The `setup` field is only + // showed in the FULL view. + View view = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Creation timestamp. + google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Last update timestamp. + google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; + +} + +// ListNamespaceConnectionsRequest represents a request to list the connections +// created by a namespace. +message ListNamespaceConnectionsRequest { + // Namespace ID. + string namespace_id = 1 [(google.api.field_behavior) = REQUIRED]; + // The maximum number of items to return. The default and cap values are 10 + // and 100, respectively. + optional int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + // Page token. By default, the first page will be returned. + optional string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + // expression. + // The following filters are supported: + // - `integration_id` + // - `q` (fuzzy search on connection ID, integration title or vendor) + // Examples: + // - List connections where app name, vendor or connection ID match `googl`: + // `q="googl"`. + // - List connections where the component type is `openai` (e.g. to setup a + // connector within a pipeline): `integration_id="openai"`. + optional string filter = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// ListNamespaceConnectionsResponse contains a paginated list of connections. +message ListNamespaceConnectionsResponse { + // A list of connections matching the request parameters. + repeated Connection connections = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Next page token. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Total number of items. + int32 total_size = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// GetNamespaceConnectionRequest represents a request to view the details of a +// connection. +message GetNamespaceConnectionRequest { + // Namespace ID. + string namespace_id = 1 [(google.api.field_behavior) = REQUIRED]; + // Connection ID. + string connection_id = 2 [(google.api.field_behavior) = REQUIRED]; + // View allows clients to specify the desired view in the response. + optional View view = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// GetNamespaceConnectionResponse contains the requested connection. +message GetNamespaceConnectionResponse { + // The requested connection. + Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// CreateNamespaceConnectionRequest represents a request to create a +// connection. +message CreateNamespaceConnectionRequest { + // Properties of the connection to be created. + Connection connection = 1 [(google.api.field_behavior) = REQUIRED]; +} + +// CreateNamespaceConnectionResponse contains the created connection. +message CreateNamespaceConnectionResponse { + // The created connection. + Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// UpdateNamespaceConnectionRequest represents a request to update a +// connection. +message UpdateNamespaceConnectionRequest { + // Properties of the connection to be updated. + Connection connection = 1 [(google.api.field_behavior) = REQUIRED]; + // The update mask specifies the subset of fields that should be modified. + // + // For more information about this field, see + // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#field-mask. + google.protobuf.FieldMask update_mask = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// UpdateNamespaceConnectionResponse contains the updated connection. +message UpdateNamespaceConnectionResponse { + // The created connection. + Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// DeleteNamespaceConnectionRequest represents a request to delete a +// connection. +message DeleteNamespaceConnectionRequest { + // Namespace ID. + string namespace_id = 1 [(google.api.field_behavior) = REQUIRED]; + // Connection ID. + string connection_id = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// DeleteNamespaceConnectionResponse is an empty response. +message DeleteNamespaceConnectionResponse {} + +// TestNamespaceConnectionRequest represents a request to test a connection. +message TestNamespaceConnectionRequest { + // Namespace ID. + string namespace_id = 1 [(google.api.field_behavior) = REQUIRED]; + // Connection ID. + string connection_id = 2 [(google.api.field_behavior) = REQUIRED]; +} + +// TestNamespaceConnectionResponse is an empty response. +message TestNamespaceConnectionResponse {} + +// Integration contains the parameters to create a connection between +// components and 3rd party apps. +message Integration { + // SetupSchema defines the schema for a connection setup. + message SetupSchema { + // The connection method, which will define the fields in the schema. + Connection.Method method = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // The connection setup field definitions. Each integration will require + // different data to connect to the 3rd party app. + google.protobuf.Struct schema = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + } + // Identifier of the integration, which references a component definition. + // Components with that definition ID will be able to use the connections + // produced by this integration. + string id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Title, reflects the app name. + string title = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Short description of the integrated app. + string description = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Integrated app vendor name. + string vendor = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Integration icon. This is a path that's relative to the root of + // the component implementation and that allows frontend applications to pull + // and locate the icons. + // See the `icon` field in the `ComponentDefinition` entity for more + // information. + string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; + // This field allows requesters to list only shortlisted integrations. + bool featured = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Schemas defines the supported schemas for the connection setup. + repeated SetupSchema schemas = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + // View defines how the integration is presented. The `spec` field is only + // showed in the FULL view. + View view = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// ListIntegrationsRequest represents a request to list the available +// integrations. +message ListIntegrationsRequest { + // The maximum number of items to return. The default and cap values are 10 + // and 100, respectively. + optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + // Page token. By default, the first page will be returned. + optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter + // expression. + // The following filters are supported: + // - `q` (fuzzy search on title or vendor) + // - `featured` + // Examples: + // - List integrations where app name or vendor match `googl`: `q="googl"`. + // - List featured integrations: `featured=true`. + optional string filter = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// ListIntegrationsResponse contains a paginated list of integrations. +message ListIntegrationsResponse { + // A list of integrations matching the request parameters. + repeated Integration integrations = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Next page token. + string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Total number of items. + int32 total_size = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// GetIntegrationRequest represents a request to view the details of an +// integration. +message GetIntegrationRequest { + // Integration ID. + string integration_id = 1 [(google.api.field_behavior) = REQUIRED]; + // View allows clients to specify the desired view in the response. + optional View view = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// GetIntegrationResponse contains the requested integration. +message GetIntegrationResponse { + // The requested integration. + Integration integration = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/vdp/pipeline/v1beta/openapi.proto.templ b/vdp/pipeline/v1beta/openapi.proto.templ index 35597803..54812b92 100644 --- a/vdp/pipeline/v1beta/openapi.proto.templ +++ b/vdp/pipeline/v1beta/openapi.proto.templ @@ -32,6 +32,10 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { name: "Secret" description: "Namespace Secret endpoints" }, + { + name: "Integration" + description: "Namespace Integration endpoints" + }, { name: "Utils" description: "Utils endpoints" diff --git a/vdp/pipeline/v1beta/pipeline_public_service.proto b/vdp/pipeline/v1beta/pipeline_public_service.proto index 989d6fce..1155682f 100644 --- a/vdp/pipeline/v1beta/pipeline_public_service.proto +++ b/vdp/pipeline/v1beta/pipeline_public_service.proto @@ -10,6 +10,7 @@ import "protoc-gen-openapiv2/options/annotations.proto"; // VDP definitions import "vdp/pipeline/v1beta/common.proto"; import "vdp/pipeline/v1beta/component_definition.proto"; +import "vdp/pipeline/v1beta/integration.proto"; import "vdp/pipeline/v1beta/pipeline.proto"; import "vdp/pipeline/v1beta/secret.proto"; @@ -1507,4 +1508,112 @@ service PipelinePublicService { option (google.api.http) = {get: "/v1beta/pipeline-runs/{pipeline_run_id}/component-runs"}; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Trigger"}; } + + // List namespace connections + // + // Returns a paginated list of connections created by a namespace. + rpc ListNamespaceConnections(ListNamespaceConnectionsRequest) returns (ListNamespaceConnectionsResponse) { + option (google.api.http) = {get: "/v1beta/namespaces/{namespace_id}/connections"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Get a namespace connection + // + // Returns the details of a connection. + rpc GetNamespaceConnection(GetNamespaceConnectionRequest) returns (GetNamespaceConnectionResponse) { + option (google.api.http) = {get: "/v1beta/namespaces/{namespace_id}/connections/{connection_id}"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Create a connection + // + // Creates a connection under the ownership of a namespace. + rpc CreateNamespaceConnection(CreateNamespaceConnectionRequest) returns (CreateNamespaceConnectionResponse) { + option (google.api.http) = { + post: "/v1beta/namespaces/{connection.namespace_id}/connections" + body: "connection" + }; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Update a connection + // + // Updates a connection with the supplied connection fields. + rpc UpdateNamespaceConnection(UpdateNamespaceConnectionRequest) returns (UpdateNamespaceConnectionResponse) { + option (google.api.http) = { + patch: "/v1beta/namespaces/{connection.namespace_id}/connections/{connection.id}" + body: "connection" + }; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Delete a connection + // + // Deletes a connection. + rpc DeleteNamespaceConnection(DeleteNamespaceConnectionRequest) returns (DeleteNamespaceConnectionResponse) { + option (google.api.http) = {delete: "/v1beta/namespaces/{namespace_id}/connections/{connection_id}"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Test a connection + // + // Makes a request to the 3rd party app that the connection is configured to + // communicate with, and checks the result of the call. If the test fails, + // the response status and error message will provide more information about + // the failure. + // + // Note that this action might affect the quota or billing of the integrated + // account in the 3rd party app. + rpc TestNamespaceConnection(TestNamespaceConnectionRequest) returns (TestNamespaceConnectionResponse) { + option (google.api.http) = {post: "/v1beta/namespaces/{namespace_id}/connections/{connection_id}/test"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // List integrations + // + // Returns a paginated list of available integrations. + rpc ListIntegrations(ListIntegrationsRequest) returns (ListIntegrationsResponse) { + option (google.api.http) = {get: "/v1beta/integrations"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } + + // Get an integration + // + // Returns the details of an integration. + rpc GetIntegration(GetIntegrationRequest) returns (GetIntegrationResponse) { + option (google.api.http) = {get: "/v1beta/integrations/{integration_id}"}; + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Integration"}; + + // This is hidden while the implementation of the endpoints is under + // development. + option (google.api.method_visibility).restriction = "INTERNAL"; + } }