feat(vdp): add Instill Integration endpoints (#418)

Because

- We want to let users reference a full configuration for pipeline
components and include both form (key-value) and OAuth integration
methods.

This commit

- Adds the endpoints and entities to manage integrations.

---------

Co-authored-by: droplet-bot <[email protected]>

openapiv2/vdp/service.swagger.yaml

@@ -21,6 +21,8 @@ tags:
     description: Pipeline Trigger endpoints
   - name: Secret
     description: Namespace Secret endpoints
+  - name: Integration
+    description: Namespace Integration endpoints
   - name: Utils
     description: Utils endpoints
 host: api.instill.tech

vdp/pipeline/v1beta/common.proto

@@ -122,3 +122,16 @@ enum ComponentType {
   // Generic.
   COMPONENT_TYPE_GENERIC = 6;
 }
+
+// View defines how a resource is presented. Most resources can share this view
+// definition, the particular meaning of each value should be defined in the
+// resource itself. Certain resources might have their own View definition if
+// they need to implement more than 2 (basic / full) views.
+enum View {
+  // Unspecified, equivalent to BASIC.
+  VIEW_UNSPECIFIED = 0;
+  // Default view.
+  VIEW_BASIC = 1;
+  // Full representation.
+  VIEW_FULL = 2;
+}

vdp/pipeline/v1beta/integration.proto

@@ -0,0 +1,246 @@
+syntax = "proto3";
+
+package vdp.pipeline.v1beta;
+
+// Google API
+import "google/api/field_behavior.proto";
+// Protocol Buffers Well-Known types
+import "google/protobuf/field_mask.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+// VDP definitions
+import "vdp/pipeline/v1beta/common.proto";
+
+// Connection contains the parameters to communicate with a 3rd party app. A
+// component may reference a connection in their setup. One connection may be
+// used by several components and pipelines.
+message Connection {
+  // Method defines how the connection is set up.
+  enum Method {
+    // Unspecified.
+    METHOD_UNSPECIFIED = 0;
+    // Key-value collection. The user is responsible of fetching the connection
+    // details from the 3rd party service.
+    METHOD_DICTIONARY = 1;
+    // Access token created via OAuth 2.0 authorization.
+    METHOD_OAUTH = 2;
+  }
+  // UUID-formatted unique identifier.
+  string uid = 1 [
+    (google.api.field_behavior) = OUTPUT_ONLY,
+    (google.api.field_behavior) = IMMUTABLE
+  ];
+  // ID.
+  string id = 2 [(google.api.field_behavior) = REQUIRED];
+  // ID of the namespace owning the connection.
+  string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
+  // Integration ID. It determines for which type of components can reference
+  // this connection.
+  string integration_id = 4 [(google.api.field_behavior) = REQUIRED];
+  // Integration title. This helps the console display the results grouped by
+  // integration ID without needing an extra call to fetch title by integration
+  // ID.
+  string integration_title = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // The IDs of the pipelines that use this integration. All the pipelines will
+  // be within the same namespace as the connection.
+  // The length of the list will be capped to 100.
+  repeated string pipeline_ids = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Connection method. It references the setup schema provided by the
+  // integration.
+  Method method = 7 [(google.api.field_behavior) = REQUIRED];
+  // Connection details. This field is required on creation, optional on view.
+  // When viewing the connection details, the setup values will be redacted.
+  google.protobuf.Struct setup = 8 [(google.api.field_behavior) = REQUIRED];
+  // View defines how the integration is presented. The `setup` field is only
+  // showed in the FULL view.
+  View view = 9  [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Creation timestamp.
+  google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Last update timestamp.
+  google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+}
+
+// ListNamespaceConnectionsRequest represents a request to list the connections
+// created by a namespace.
+message ListNamespaceConnectionsRequest {
+  // Namespace ID.
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // The maximum number of items to return. The default and cap values are 10
+  // and 100, respectively.
+  optional int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL];
+  // Page token. By default, the first page will be returned.
+  optional string page_token = 3 [(google.api.field_behavior) = OPTIONAL];
+  // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
+  // expression.
+  // The following filters are supported:
+  // - `integration_id`
+  // - `q` (fuzzy search on connection ID, integration title or vendor)
+  // Examples:
+  // - List connections where app name, vendor or connection ID match `googl`:
+  // `q="googl"`.
+  // - List connections where the component type is `openai` (e.g. to setup a
+  // connector within a pipeline): `integration_id="openai"`.
+  optional string filter = 4 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// ListNamespaceConnectionsResponse contains a paginated list of connections.
+message ListNamespaceConnectionsResponse {
+  // A list of connections matching the request parameters.
+  repeated Connection connections = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Next page token.
+  string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Total number of items.
+  int32 total_size = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// GetNamespaceConnectionRequest represents a request to view the details of a
+// connection.
+message GetNamespaceConnectionRequest {
+  // Namespace ID.
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // Connection ID.
+  string connection_id = 2 [(google.api.field_behavior) = REQUIRED];
+  // View allows clients to specify the desired view in the response.
+  optional View view = 3 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// GetNamespaceConnectionResponse contains the requested connection.
+message GetNamespaceConnectionResponse {
+  // The requested connection.
+  Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// CreateNamespaceConnectionRequest represents a request to create a
+// connection.
+message CreateNamespaceConnectionRequest {
+  // Properties of the connection to be created.
+  Connection connection = 1 [(google.api.field_behavior) = REQUIRED];
+}
+
+// CreateNamespaceConnectionResponse contains the created connection.
+message CreateNamespaceConnectionResponse {
+  // The created connection.
+  Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// UpdateNamespaceConnectionRequest represents a request to update a
+// connection.
+message UpdateNamespaceConnectionRequest {
+  // Properties of the connection to be updated.
+  Connection connection = 1 [(google.api.field_behavior) = REQUIRED];
+  // The update mask specifies the subset of fields that should be modified.
+  //
+  // For more information about this field, see
+  // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#field-mask.
+  google.protobuf.FieldMask update_mask = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// UpdateNamespaceConnectionResponse contains the updated connection.
+message UpdateNamespaceConnectionResponse {
+  // The created connection.
+  Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// DeleteNamespaceConnectionRequest represents a request to delete a
+// connection.
+message DeleteNamespaceConnectionRequest {
+  // Namespace ID.
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // Connection ID.
+  string connection_id = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// DeleteNamespaceConnectionResponse is an empty response.
+message DeleteNamespaceConnectionResponse {}
+
+// TestNamespaceConnectionRequest represents a request to test a connection.
+message TestNamespaceConnectionRequest {
+  // Namespace ID.
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // Connection ID.
+  string connection_id = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// TestNamespaceConnectionResponse is an empty response.
+message TestNamespaceConnectionResponse {}
+
+// Integration contains the parameters to create a connection between
+// components and 3rd party apps.
+message Integration {
+  // SetupSchema defines the schema for a connection setup.
+  message SetupSchema {
+    // The connection method, which will define the fields in the schema.
+    Connection.Method method = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+    // The connection setup field definitions. Each integration will require
+    // different data to connect to the 3rd party app.
+    google.protobuf.Struct schema = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  }
+  // Identifier of the integration, which references a component definition.
+  // Components with that definition ID will be able to use the connections
+  // produced by this integration.
+  string id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Title, reflects the app name.
+  string title = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Short description of the integrated app.
+  string description = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Integrated app vendor name.
+  string vendor = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Integration icon. This is a path that's relative to the root of
+  // the component implementation and that allows frontend applications to pull
+  // and locate the icons.
+  // See the `icon` field in the `ComponentDefinition` entity for more
+  // information.
+  string icon = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // This field allows requesters to list only shortlisted integrations.
+  bool featured = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Schemas defines the supported schemas for the connection setup.
+  repeated SetupSchema schemas = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // View defines how the integration is presented. The `spec` field is only
+  // showed in the FULL view.
+  View view = 9  [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// ListIntegrationsRequest represents a request to list the available
+// integrations.
+message ListIntegrationsRequest {
+  // The maximum number of items to return. The default and cap values are 10
+  // and 100, respectively.
+  optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
+  // Page token. By default, the first page will be returned.
+  optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
+  // Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
+  // expression.
+  // The following filters are supported:
+  // - `q` (fuzzy search on title or vendor)
+  // - `featured`
+  // Examples:
+  // - List integrations where app name or vendor match `googl`: `q="googl"`.
+  // - List featured integrations: `featured=true`.
+  optional string filter = 3 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// ListIntegrationsResponse contains a paginated list of integrations.
+message ListIntegrationsResponse {
+  // A list of integrations matching the request parameters.
+  repeated Integration integrations = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Next page token.
+  string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Total number of items.
+  int32 total_size = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// GetIntegrationRequest represents a request to view the details of an
+// integration.
+message GetIntegrationRequest {
+  // Integration ID.
+  string integration_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // View allows clients to specify the desired view in the response.
+  optional View view = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// GetIntegrationResponse contains the requested integration.
+message GetIntegrationResponse {
+  // The requested integration.
+  Integration integration = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+}

vdp/pipeline/v1beta/openapi.proto.templ

@@ -32,6 +32,10 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
       name: "Secret"
       description: "Namespace Secret endpoints"
     },
+    {
+        name: "Integration"
+        description: "Namespace Integration endpoints"
+    },
     {
       name: "Utils"
       description: "Utils endpoints"