feat(artifact): add tag create / list endpoints (#286)

Because - We want to list the versions of a model. - Registry has a [tags](https://distribution.github.io/distribution/#listing-image-tags) endpoint but it doesn't expose any timestamp. ![CleanShot 2024-03-19 at 20 24 14](https://github.com/instill-ai/protobufs/assets/3977183/38f408d1-0096-4f39-b3fe-92991104fa66) This commit - Adds and documents a private API for the repository tags in artifact. - Only `api-gateway` (when it detects the end of the image push operation) should communicate the creation of a tag. - The tags won't be exposed directly to the client. `model-backend` will request this information to expose the model versions in the same family as the rest of the model endpoints. ## 🗒️ Notes As an initial addition to the Artifact domain, I made some naming choices. Please review and provide feedback about them. ## ⏭️ Next steps - `artifact-backend` will implement a first version of this endpoint that will fetch the tag list from the `registry` instance. This endpoint only returns the tag list so the digest and update time will be empty. - In order to record the update time, `api-gateway` will call the creation endpoint when we intercept a `PUT manifest` request. We'll store the digest and update time in the `artifact` database and we'll aggregate that info to the `registry` tag list (the registry should remain the source of truth). --------- Co-authored-by: droplet-bot <[email protected]>
instill-ai · Mar 26, 2024 · 6678571 · 6678571
1 parent c369167
commit 6678571
Show file tree

Hide file tree

Showing 6 changed files with 188 additions and 7 deletions.
diff --git a/artifact/artifact/v1alpha/artifact.proto b/artifact/artifact/v1alpha/artifact.proto
@@ -5,6 +5,9 @@ package artifact.artifact.v1alpha;
 import "common/healthcheck/v1beta/healthcheck.proto";
 // Google API
 import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+// Protocol Buffers Well-Known Types
+import "google/protobuf/timestamp.proto";
 
 // LivenessRequest represents a request to check a service liveness status
 message LivenessRequest {
@@ -29,3 +32,104 @@ message ReadinessResponse {
   // HealthCheckResponse message
   common.healthcheck.v1beta.HealthCheckResponse health_check_response = 1;
 }
+
+/*
+
+This API is under development and, therefore, some of its entitites and
+entpoints are not implemented yet. This section aims to give context about the
+current interface and how it fits in the Artifact vision.
+
+# Artifact
+
+The Artifact domain is responsible of storing data that will later be used for
+processing unstructured data. Artifact will support the following types of
+data:
+
+- Repositories
+- Objects
+- Vectors
+
+## Repositories
+
+An implementation of the [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec?tab=readme-ov-file)
+is used to manage versioned content. The main use for repositories is storing
+container images that can be used to deploy AI models or VDP pipelines.
+
+The ID of a repository has 2 segments, the owner (an Instill user or
+organization) and the content ID (the AI model or pipeline ID), e.g.
+`curious-wombat/llava-34b`.
+
+## Objects
+
+Raw data is stored in binary blobs. Object storage allows users to upload data
+(e.g. images, audio) that can be used by pipelines or to store the results or a
+pipeline trigger.
+
+## Vectors
+
+Vector embeddings have their own storage, which allows fast retrieval and similarity search.
+
+*/
+
+// RepositoryTag contains information about the version of some content in a
+// repository.
+message RepositoryTag {
+  // The name of the tag, defined by its parent repository and ID.
+  // - Format: `repositories/{repository.id}/tags/{tag.id}`.
+  string name = 1 [(google.api.field_behavior) = IMMUTABLE];
+  // The tag identifier.
+  string id = 2 [(google.api.field_behavior) = IMMUTABLE];
+  // Unique identifier, computed from the manifest the tag refers to.
+  string digest = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Tag update time, i.e. timestamp of the last push.
+  google.protobuf.Timestamp update_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// ListRepositoryTagsRequest represents a request to list the tags of a
+// repository.
+message ListRepositoryTagsRequest {
+  // The maximum number of tags to return. The default and cap values are 10
+  // and 100, respectively.
+  optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
+  // Page number.
+  optional int32 page = 2 [(google.api.field_behavior) = OPTIONAL];
+  // The repository holding the different versions of a given content.
+  // - Format: `repositories/{repository.id}`.
+  // - Example: `repository/flaming-wombat/llama-2-7b`.
+  string parent = 3 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {type: "api.instill.tech/Repository"}
+  ];
+}
+
+// ListRepositoryTagsResponse contains a list of container image tags.
+message ListRepositoryTagsResponse {
+  // A list of repository tags.
+  repeated RepositoryTag tags = 1;
+  // Total number of tags.
+  int32 total_size = 2;
+  // The requested page size.
+  int32 page_size = 3;
+  // The requested page offset.
+  int32 page = 4;
+}
+
+// CreateRepositoryTagRequest represents a request to add a tag to a given
+// repository.
+message CreateRepositoryTagRequest {
+  // The tag information.
+  RepositoryTag tag = 1;
+  // The repository holding the different versions of a given content.
+  // - Format: `repositories/{repository.id}`.
+  // - Example: `repository/flaming-wombat/llama-2-7b`.
+  string parent = 2 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {type: "api.instill.tech/Repository"}
+  ];
+}
+
+// CreateRepositoryTagResponse contains the created tag.
+message CreateRepositoryTagResponse {
+  // The created tag.
+  RepositoryTag tag = 1;
+}
diff --git a/artifact/artifact/v1alpha/artifact_private_service.proto b/artifact/artifact/v1alpha/artifact_private_service.proto
@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package artifact.artifact.v1alpha;
+
+import "google/api/visibility.proto";
+import "artifact/artifact/v1alpha/artifact.proto";
+
+// ArtifactPrivateService exposes the private endpoints that allow clients to
+// manage artifacts.
+service ArtifactPrivateService {
+  // List the tags in a repository.
+  //
+  // Returns a portion of the versions that the specified repository holds.
+  rpc ListRepositoryTags(ListRepositoryTagsRequest) returns (ListRepositoryTagsResponse);
+
+  // Create a new repository tag.
+  //
+  // Adds a tag to a given repository. Note that this operation is only
+  // intended to register the information of an *already created* tag. This
+  // method should be called as part of the content push operation, right after
+  // the [PUT Manifest](https://distribution.github.io/distribution/#put-manifest) has
+  // succeeded. The distribution registry won't hold data such as the push time
+  // or the tag digest, so `artifact-backend` will hold this information locally.
+  rpc CreateRepositoryTag(CreateRepositoryTagRequest) returns (CreateRepositoryTagResponse);
+
+  option (google.api.api_visibility).restriction = "INTERNAL";
+}
diff --git a/artifact/artifact/v1alpha/artifact_public_service.proto b/artifact/artifact/v1alpha/artifact_public_service.proto
@@ -10,8 +10,6 @@ import "google/api/visibility.proto";
 // OpenAPI definition
 import "protoc-gen-openapiv2/options/annotations.proto";
 
-// Artifact
-//
 // ArtifactPublicService exposes the public endpoints that allow clients to
 // manage artifacts.
 service ArtifactPublicService {

diff --git a/openapiv2/artifact/service.swagger.yaml b/openapiv2/artifact/service.swagger.yaml
@@ -30,3 +30,55 @@ definitions:
         items:
           type: object
           $ref: '#/definitions/protobufAny'
+  v1alphaCreateRepositoryTagResponse:
+    type: object
+    properties:
+      tag:
+        $ref: '#/definitions/v1alphaRepositoryTag'
+        description: The created tag.
+    description: CreateRepositoryTagResponse contains the created tag.
+  v1alphaListRepositoryTagsResponse:
+    type: object
+    properties:
+      tags:
+        type: array
+        items:
+          type: object
+          $ref: '#/definitions/v1alphaRepositoryTag'
+        description: A list of repository tags.
+      total_size:
+        type: integer
+        format: int32
+        description: Total number of tags.
+      page_size:
+        type: integer
+        format: int32
+        description: The requested page size.
+      page:
+        type: integer
+        format: int32
+        description: The requested page offset.
+    description: ListRepositoryTagsResponse contains a list of container image tags.
+  v1alphaRepositoryTag:
+    type: object
+    properties:
+      name:
+        type: string
+        description: |-
+          The name of the tag, defined by its parent repository and ID.
+          - Format: `repositories/{repository.id}/tags/{tag.id}`.
+      id:
+        type: string
+        description: The tag identifier.
+      digest:
+        type: string
+        description: Unique identifier, computed from the manifest the tag refers to.
+        readOnly: true
+      update_time:
+        type: string
+        format: date-time
+        description: Tag update time, i.e. timestamp of the last push.
+        readOnly: true
+    description: |-
+      RepositoryTag contains information about the version of some content in a
+      repository.
diff --git a/openapiv2/vdp/service.swagger.yaml b/openapiv2/vdp/service.swagger.yaml
@@ -868,7 +868,7 @@ paths:
         - name: pipeline_release_name
           description: |-
             The name of the release, defined by its parent and ID.
-            - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+            - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
           in: path
           required: true
           type: string
@@ -1836,7 +1836,7 @@ paths:
         - name: pipeline_release_name_1
           description: |-
             The name of the release, defined by its parent and ID.
-            - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+            - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
           in: path
           required: true
           type: string
@@ -5369,9 +5369,9 @@ definitions:
     properties:
       name:
         type: string
-        title: |-
+        description: |-
           The name of the release, defined by its parent and ID.
-          - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+          - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
         readOnly: true
       uid:
         type: string

diff --git a/vdp/pipeline/v1beta/pipeline.proto b/vdp/pipeline/v1beta/pipeline.proto
@@ -345,7 +345,7 @@ message PipelineRelease {
   };
 
   // The name of the release, defined by its parent and ID.
-  // - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}
+  // - Format: `{parent_type}/{parent.id}/pipelines/{pipeline.id}/releases/{release.id}`.
   string name = 1 [
     (google.api.field_behavior) = OUTPUT_ONLY,
     (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {