From 66233cea80fc40adeb32c02058394c167bcbda03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:33:27 +0100 Subject: [PATCH] feat(pipeline): allow connection override in trigger data (#525) Because - When a pipeline recipe references a connection, that connection ID will be fetched from the requester's namespace. - This forces requesters to create connections in order to trigger pipelines from other namesapces. This commit - Adds a `connectionReferences` field in the pipeline trigger data, which allows clients to override the references in the recipe with per-batch references. - Adds a private endpoint to fetch connections by UID, needed in the 1-minute to impact project. --------- Co-authored-by: droplet-bot --- openapi/v2/service.swagger.yaml | 38 +++++++++++++++++-- vdp/pipeline/v1beta/integration.proto | 16 ++++++++ vdp/pipeline/v1beta/pipeline.proto | 24 ++++++++++-- .../v1beta/pipeline_private_service.proto | 18 ++++++--- 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/openapi/v2/service.swagger.yaml b/openapi/v2/service.swagger.yaml index a8365b00..2ce82178 100644 --- a/openapi/v2/service.swagger.yaml +++ b/openapi/v2/service.swagger.yaml @@ -8659,6 +8659,15 @@ definitions: description: Total number of users. readOnly: true description: ListUsersResponse contains a list of users. + LookUpConnectionAdminResponse: + type: object + properties: + connection: + description: The requested connection. + readOnly: true + allOf: + - $ref: '#/definitions/Connection' + description: LookUpConnectionAdminResponse contains the requested connection. LookUpModelAdminResponse: type: object properties: @@ -10407,13 +10416,36 @@ definitions: properties: variable: type: object - title: Variables + description: The values of the pipeline variables. secret: type: object additionalProperties: type: string - title: Variables - title: Data + description: |- + A collection of secrets. By default, if a pipeline references any secret + (`${secret.}`), its value is read from the namespace's secrets. + This object provides a way to override these secret values: if one of its + keys matches the ID of a reference secret, its value will be read from + here instead of from the namespace secret collection. + connectionReferences: + type: object + additionalProperties: + type: string + description: |- + A collection of connection references. By default, connection references + (`${connection.}`) in a pipeline will be resolved by fetching the + value from the requester's connections. Connections contain sensitive data + and can't be shared across namespaces, so this means that, in order to + successfully run a pipeline owned by another namespace, the requester will + need to have a connection with the same ID. + This object provides a way to override the connection references with + connections that the requester owns. Each element in the object maps a + connection ID present in the pipeline (key) to the ID of a connection + owned by the requester (value). + Note that, since only references are accepted (this object shouldn't + contain connection **values**), the reference syntax shouldn't be used + here. + description: TriggerData contains the input data for a pipeline run. TriggerMetadata: type: object properties: diff --git a/vdp/pipeline/v1beta/integration.proto b/vdp/pipeline/v1beta/integration.proto index 4610da2c..f61d6933 100644 --- a/vdp/pipeline/v1beta/integration.proto +++ b/vdp/pipeline/v1beta/integration.proto @@ -306,3 +306,19 @@ message GetIntegrationResponse { // The requested integration. Integration integration = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; } + +// LookUpConnectionAdminRequest represents a request to fetch the details of a +// connection by UID. +message LookUpConnectionAdminRequest { + // Connection UID. + string namespace_id = 1 [(google.api.field_behavior) = REQUIRED]; + // View allows clients to specify the desired view in the response. It + // defaults to `VIEW_BASIC`. + optional View view = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// LookUpConnectionAdminResponse contains the requested connection. +message LookUpConnectionAdminResponse { + // The requested connection. + Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/vdp/pipeline/v1beta/pipeline.proto b/vdp/pipeline/v1beta/pipeline.proto index 56a9a228..9443dc56 100644 --- a/vdp/pipeline/v1beta/pipeline.proto +++ b/vdp/pipeline/v1beta/pipeline.proto @@ -1035,12 +1035,30 @@ message TriggerUserPipelineWithStreamResponse { TriggerMetadata metadata = 2; } -// Data +// TriggerData contains the input data for a pipeline run. message TriggerData { - // Variables + // The values of the pipeline variables. google.protobuf.Struct variable = 1 [(google.api.field_behavior) = OPTIONAL]; - // Variables + // A collection of secrets. By default, if a pipeline references any secret + // (`${secret.}`), its value is read from the namespace's secrets. + // This object provides a way to override these secret values: if one of its + // keys matches the ID of a reference secret, its value will be read from + // here instead of from the namespace secret collection. map secret = 2 [(google.api.field_behavior) = OPTIONAL]; + // A collection of connection references. By default, connection references + // (`${connection.}`) in a pipeline will be resolved by fetching the + // value from the requester's connections. Connections contain sensitive data + // and can't be shared across namespaces, so this means that, in order to + // successfully run a pipeline owned by another namespace, the requester will + // need to have a connection with the same ID. + // This object provides a way to override the connection references with + // connections that the requester owns. Each element in the object maps a + // connection ID present in the pipeline (key) to the ID of a connection + // owned by the requester (value). + // Note that, since only references are accepted (this object shouldn't + // contain connection **values**), the reference syntax shouldn't be used + // here. + map connection_references = 3 [(google.api.field_behavior) = OPTIONAL]; } // TriggerUserPipelineRequest represents a request to trigger a user-owned diff --git a/vdp/pipeline/v1beta/pipeline_private_service.proto b/vdp/pipeline/v1beta/pipeline_private_service.proto index 9baa9720..e74e97b0 100644 --- a/vdp/pipeline/v1beta/pipeline_private_service.proto +++ b/vdp/pipeline/v1beta/pipeline_private_service.proto @@ -4,6 +4,7 @@ package vdp.pipeline.v1beta; // OpenAPI definition import "protoc-gen-openapiv2/options/annotations.proto"; +import "vdp/pipeline/v1beta/integration.proto"; import "vdp/pipeline/v1beta/pipeline.proto"; // PipelinePrivateService defines private methods to interact with Pipeline @@ -13,19 +14,24 @@ service PipelinePrivateService { // List pipelines (admin only) // - // This is a *private* method that allows admin users and internal clients to - // list *all* pipeline resources. + // This *private* method allows internal clients to list *all* pipeline + // resources. rpc ListPipelinesAdmin(ListPipelinesAdminRequest) returns (ListPipelinesAdminResponse) {} // Get a pipeline by UID (admin only) // - // This is a *private* method that allows admin users to access any pipeline - // resource by its UID. + // This *private* method allows internal clients to access any pipeline + // resource by UID. rpc LookUpPipelineAdmin(LookUpPipelineAdminRequest) returns (LookUpPipelineAdminResponse) {} // List pipeline releases (admin only) // - // This is a *private* method that allows admin users to list *all* pipeline - // releases. + // This *private* method allows admin users to list *all* pipeline releases. rpc ListPipelineReleasesAdmin(ListPipelineReleasesAdminRequest) returns (ListPipelineReleasesAdminResponse) {} + + // Get a connection by UID (admin only) + // + // This *private* method allows internal clients to access any connection + // resource by UID. + rpc LookUpConnectionAdmin(LookUpConnectionAdminRequest) returns (LookUpConnectionAdminResponse) {} }