feat(pipeline): allow connection override in trigger data (#525)

Because

- When a pipeline recipe references a connection, that connection ID
will
  be fetched from the requester's namespace.
- This forces requesters to create connections in order to trigger
  pipelines from other namesapces.

This commit

- Adds a `connectionReferences` field in the pipeline trigger data,
which
  allows clients to override the references in the recipe with per-batch
  references.
- Adds a private endpoint to fetch connections by UID, needed in the
  1-minute to impact project.

---------

Co-authored-by: droplet-bot <[email protected]>

openapi/v2/service.swagger.yaml

@@ -8659,6 +8659,15 @@ definitions:
         description: Total number of users.
         readOnly: true
     description: ListUsersResponse contains a list of users.
+  LookUpConnectionAdminResponse:
+    type: object
+    properties:
+      connection:
+        description: The requested connection.
+        readOnly: true
+        allOf:
+          - $ref: '#/definitions/Connection'
+    description: LookUpConnectionAdminResponse contains the requested connection.
   LookUpModelAdminResponse:
     type: object
     properties:
@@ -10407,13 +10416,36 @@ definitions:
     properties:
       variable:
         type: object
-        title: Variables
+        description: The values of the pipeline variables.
       secret:
         type: object
         additionalProperties:
           type: string
-        title: Variables
-    title: Data
+        description: |-
+          A collection of secrets. By default, if a pipeline references any secret
+          (`${secret.<id>}`), its value is read from the namespace's secrets.
+          This object provides a way to override these secret values: if one of its
+          keys matches the ID of a reference secret, its value will be read from
+          here instead of from the namespace secret collection.
+      connectionReferences:
+        type: object
+        additionalProperties:
+          type: string
+        description: |-
+          A collection of connection references. By default, connection references
+          (`${connection.<id>}`) in a pipeline will be resolved by fetching the
+          value from the requester's connections. Connections contain sensitive data
+          and can't be shared across namespaces, so this means that, in order to
+          successfully run a pipeline owned by another namespace, the requester will
+          need to have a connection with the same ID.
+          This object provides a way to override the connection references with
+          connections that the requester owns. Each element in the object maps a
+          connection ID present in the pipeline (key) to the ID of a connection
+          owned by the requester (value).
+          Note that, since only references are accepted (this object shouldn't
+          contain connection **values**), the reference syntax shouldn't be used
+          here.
+    description: TriggerData contains the input data for a pipeline run.
   TriggerMetadata:
     type: object
     properties:

vdp/pipeline/v1beta/integration.proto

@@ -306,3 +306,19 @@ message GetIntegrationResponse {
   // The requested integration.
   Integration integration = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
+
+// LookUpConnectionAdminRequest represents a request to fetch the details of a
+// connection by UID.
+message LookUpConnectionAdminRequest {
+  // Connection UID.
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
+  // View allows clients to specify the desired view in the response. It
+  // defaults to `VIEW_BASIC`.
+  optional View view = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// LookUpConnectionAdminResponse contains the requested connection.
+message LookUpConnectionAdminResponse {
+  // The requested connection.
+  Connection connection = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+}

vdp/pipeline/v1beta/pipeline.proto

@@ -1035,12 +1035,30 @@ message TriggerUserPipelineWithStreamResponse {
   TriggerMetadata metadata = 2;
 }
 
-// Data
+// TriggerData contains the input data for a pipeline run.
 message TriggerData {
-  // Variables
+  // The values of the pipeline variables.
   google.protobuf.Struct variable = 1 [(google.api.field_behavior) = OPTIONAL];
-  // Variables
+  // A collection of secrets. By default, if a pipeline references any secret
+  // (`${secret.<id>}`), its value is read from the namespace's secrets.
+  // This object provides a way to override these secret values: if one of its
+  // keys matches the ID of a reference secret, its value will be read from
+  // here instead of from the namespace secret collection.
   map<string, string> secret = 2 [(google.api.field_behavior) = OPTIONAL];
+  // A collection of connection references. By default, connection references
+  // (`${connection.<id>}`) in a pipeline will be resolved by fetching the
+  // value from the requester's connections. Connections contain sensitive data
+  // and can't be shared across namespaces, so this means that, in order to
+  // successfully run a pipeline owned by another namespace, the requester will
+  // need to have a connection with the same ID.
+  // This object provides a way to override the connection references with
+  // connections that the requester owns. Each element in the object maps a
+  // connection ID present in the pipeline (key) to the ID of a connection
+  // owned by the requester (value).
+  // Note that, since only references are accepted (this object shouldn't
+  // contain connection **values**), the reference syntax shouldn't be used
+  // here.
+  map<string, string> connection_references = 3 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // TriggerUserPipelineRequest represents a request to trigger a user-owned

vdp/pipeline/v1beta/pipeline_private_service.proto

@@ -4,6 +4,7 @@ package vdp.pipeline.v1beta;
 
 // OpenAPI definition
 import "protoc-gen-openapiv2/options/annotations.proto";
+import "vdp/pipeline/v1beta/integration.proto";
 import "vdp/pipeline/v1beta/pipeline.proto";
 
 // PipelinePrivateService defines private methods to interact with Pipeline
@@ -13,19 +14,24 @@ service PipelinePrivateService {
 
   // List pipelines (admin only)
   //
-  // This is a *private* method that allows admin users and internal clients to
-  // list *all* pipeline resources.
+  // This *private* method allows internal clients to list *all* pipeline
+  // resources.
   rpc ListPipelinesAdmin(ListPipelinesAdminRequest) returns (ListPipelinesAdminResponse) {}
 
   // Get a pipeline by UID (admin only)
   //
-  // This is a *private* method that allows admin users to access any pipeline
-  // resource by its UID.
+  // This *private* method allows internal clients to access any pipeline
+  // resource by UID.
   rpc LookUpPipelineAdmin(LookUpPipelineAdminRequest) returns (LookUpPipelineAdminResponse) {}
 
   // List pipeline releases (admin only)
   //
-  // This is a *private* method that allows admin users to list *all* pipeline
-  // releases.
+  // This *private* method allows admin users to list *all* pipeline releases.
   rpc ListPipelineReleasesAdmin(ListPipelineReleasesAdminRequest) returns (ListPipelineReleasesAdminResponse) {}
+
+  // Get a connection by UID (admin only)
+  //
+  // This *private* method allows internal clients to access any connection
+  // resource by UID.
+  rpc LookUpConnectionAdmin(LookUpConnectionAdminRequest) returns (LookUpConnectionAdminResponse) {}
 }