feat(all): introduce new namespace endpoints (#390)

Because - We'd like to simplify the API endpoints by merging users and organization endpoints into namespace endpoints. This commit - Introduces new namespace endpoints. - Fixes several bugs for OpenAPI generation. - Makes the path parameters explicit. - Uses `rdme` cli tool for CI and validation.
instill-ai · Jul 24, 2024 · 5f4c572 · 5f4c572
1 parent 44f8c68
commit 5f4c572
Show file tree

Hide file tree

Showing 34 changed files with 5,863 additions and 2,750 deletions.
diff --git a/.github/workflows/buf-gen-openapi.yaml b/.github/workflows/buf-gen-openapi.yaml
@@ -28,17 +28,22 @@ jobs:
       - name: Generate OpenAPI definitions
         run: |
           make openapi
-
-          # Pass generated files to linter step
-          echo 'FILE_LIST<<EOF' >> $GITHUB_ENV
-          find openapiv2 -type f >> $GITHUB_ENV
-          echo 'EOF' >> $GITHUB_ENV
-      - name: Lint generated OpenAPI definitions
-        uses: mbowman100/swagger-validator-action@master
+      - name: Lint generated Core OpenAPI definitions
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi:validate openapiv2/core/service.swagger.yaml
+      - name: Lint generated VDP OpenAPI definitions
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi:validate openapiv2/vdp/service.swagger.yaml
+      - name: Lint generated Model OpenAPI definitions
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi:validate openapiv2/model/service.swagger.yaml
+      - name: Lint generated Artifact OpenAPI definitions
+        uses: readmeio/rdme@v8
         with:
-          # Uses the same linter than swagger-cli but spares us the
-          # installation.
-          files: ${{ env.FILE_LIST }}
+          rdme: openapi:validate openapiv2/artifact/service.swagger.yaml
       - name: Commit and push
         run: |
           if [[ `git status --porcelain` ]]; then

diff --git a/.github/workflows/sync-api-docs.yml b/.github/workflows/sync-api-docs.yml
@@ -1,6 +1,7 @@
 name: Sync docs in ReadMe 🦉
 
 on:
+  pull_request:
   push:
     branches:
       # This workflow will run every time changes in the OpenAPI docs are
@@ -14,8 +15,62 @@ on:
       - 'openapiv2/vdp/**'
 
 jobs:
-  sync-openapi-private:
+  sync-openapi-private-dev:
+    name: Keep private (dev) docs in sync with latest pull request
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    outputs:
+      old_release: ${{ steps.check-new-release.outputs.old_release }}
+      new_release: ${{ steps.check-new-release.outputs.new_release }}
+    steps:
+      - name: Check out repo 📚
+        uses: actions/checkout@v3
+        with:
+          # Needed in check-new-release to compare with the previous commit.
+          fetch-depth: 0
+
+      - name: Sync Core 🔮
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi openapiv2/core/service.swagger.yaml --title "🔮 Core (PR ${{ github.ref_name }})" --key=${{ secrets.README_API_KEY }} --id=669fb9c72dd8c500184c652d
+
+      - name: Sync Model ⚗️
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi openapiv2/model/service.swagger.yaml --title "⚗️ Model (PR ${{ github.ref_name }})"--key=${{ secrets.README_API_KEY }} --id=669fb9c72dd8c500184c652e
+
+      - name: Sync VDP 💧
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi openapiv2/vdp/service.swagger.yaml --title "💧 VDP (PR ${{ github.ref_name }})"--key=${{ secrets.README_API_KEY }} --id=669fb9c72dd8c500184c652f
+
+      - name: Sync Artifact 💾
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi openapiv2/artifact/service.swagger.yaml --title "💾 Artifact (PR ${{ github.ref_name }})" --key=${{ secrets.README_API_KEY }} --id=669fb9c72dd8c500184c6531
+
+      - name: Check new release 🔍
+        id: check-new-release
+        run: |
+          # If the version in the OpenAPI configuration has changed, extract
+          # the old and new release versions (without the "v" prefix) to
+          # variables.
+          version_file=common/openapi/v1beta/api_info.conf
+          capture_old='^-\s\+\<version:'
+          capture_new='^+\s\+\<version:'
+          extract_version='s/.*"v\(.*\)".*/\1/'
+
+          old_version=$(git diff ${{ github.event.before }} ${{ github.event.after }} $version_file | grep $capture_old | sed $extract_version)
+          new_version=$(git diff ${{ github.event.before }} ${{ github.event.after }} $version_file | grep $capture_new | sed $extract_version)
+
+          if [[ $new_version ]]; then
+            echo "new_release=$new_version" >> $GITHUB_OUTPUT
+            echo "old_release=$old_version" >> $GITHUB_OUTPUT
+          fi
+
+  sync-openapi-private-staging:
     name: Keep private (staging) docs in sync with `main`
+    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     outputs:
       old_release: ${{ steps.check-new-release.outputs.old_release }}
@@ -42,6 +97,11 @@ jobs:
         with:
           rdme: openapi openapiv2/vdp/service.swagger.yaml --key=${{ secrets.README_API_KEY }} --id=65bcfc78e72e300017c0b164
 
+      - name: Sync Artifact 💾
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi openapiv2/artifact/service.swagger.yaml --key=${{ secrets.README_API_KEY }} --id=669fb97f2071e2004a8f9183
+
       - name: Check new release 🔍
         id: check-new-release
         run: |
@@ -63,7 +123,7 @@ jobs:
 
   sync-openapi-public:
     name: Sync public docs on new release
-    needs: [sync-openapi-private]
+    needs: [sync-openapi-private-staging]
     runs-on: ubuntu-latest
     if: needs.sync-openapi-private.outputs.new_release != ''
     steps:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,15 +4,31 @@ repos:
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
+        exclude: common/openapi
       - id: trailing-whitespace
   - repo: https://github.com/pinglin/conventional-pre-commit
     rev: v1.1.0
     hooks:
       - id: conventional-pre-commit
         stages: [commit-msg]
   - repo: https://github.com/bufbuild/buf.git
-    rev: v1.25.0
+    rev: v1.35.0
     hooks:
       - id: buf-generate
+  - repo: local
+    hooks:
+      - id: generate-openapi
+        name: generate openapi
+        entry: make openapi
+        language: system
+  - repo: https://github.com/bufbuild/buf.git
+    rev: v1.35.0
+    hooks:
       - id: buf-format
       - id: buf-lint
+  - repo: local
+    hooks:
+      - id: validate-openapi
+        name: validate openapi
+        entry: make openapi-lint
+        language: system
diff --git a/Makefile b/Makefile
@@ -5,8 +5,11 @@
 # that matches the one in our CI.
 OS_NAME := $(shell uname -s | tr A-Z a-z)
 SED_IN_PLACE := sed -i
+
+# On macOS, you can install the `gsed` command via `brew install gsed` to
+# achieve the same behavior as the `sed` command on Linux.
 ifeq (${OS_NAME}, darwin)
-	SED_IN_PLACE += ''
+	SED_IN_PLACE = gsed -i
 endif
 openapi:
 	@# Inject common API configuration into each OpenAPI proto template.
@@ -25,4 +28,4 @@ openapi:
 	find openapiv2 -type f | xargs -I '{}' ${SED_IN_PLACE} '/^[[:space:]]*enum: \[\]/,+0d' {}
 openapi-lint:
 	@# Lint each file under openapiv2.
-	find openapiv2 -type f | xargs -I '{}' swagger-cli validate {}
+	find openapiv2 -type f | xargs -I '{}' rdme openapi:validate {}
diff --git a/artifact/artifact/v1alpha/artifact.proto b/artifact/artifact/v1alpha/artifact.proto
@@ -196,7 +196,7 @@ message KnowledgeBase {
 // CreateKnowledgeBaseRequest represents a request to create a knowledge base.
 message CreateKnowledgeBaseRequest {
   // The knowledge base owner(nammespace).
-  string owner_id = 1;
+  string namespace_id = 1;
   // The knowledge base name.
   string name = 2;
   // The knowledge base description.
@@ -214,7 +214,7 @@ message CreateKnowledgeBaseResponse {
 // Request message for ListKnowledgeBases
 message ListKnowledgeBasesRequest {
   // User ID for which to list the knowledge bases
-  string owner_id = 1;
+  string namespace_id = 1;
 }
 
 // GetKnowledgeBasesResponse represents a response for getting all knowledge bases from users.
@@ -232,7 +232,7 @@ message UpdateKnowledgeBaseRequest {
   // The knowledge base tags.
   repeated string tags = 3;
   // The knowledge base owner(namespace).
-  string owner_id = 4;
+  string namespace_id = 4;
 }
 
 // UpdateKnowledgeBaseResponse represents a response for updating a knowledge base.
@@ -244,7 +244,7 @@ message UpdateKnowledgeBaseResponse {
 // DeleteKnowledgeBaseRequest represents a request to delete a knowledge base.
 message DeleteKnowledgeBaseRequest {
   // The owner's id. i.e. namespace.
-  string owner_id = 1;
+  string namespace_id = 1;
   // The knowledge base identifier.
   string kb_id = 2;
 }
@@ -325,14 +325,14 @@ message File {
   int64 size = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
   // total chunks
   int32 total_chunks = 15 [(google.api.field_behavior) = OUTPUT_ONLY];
-  // total tokens 
+  // total tokens
   int32 total_tokens = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // upload file request
 message UploadKnowledgeBaseFileRequest {
   // owenr uid
-  string owner_id = 1 [(google.api.field_behavior) = REQUIRED];
+  string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
   // knowledge base uid
   string kb_id = 2 [(google.api.field_behavior) = REQUIRED];
   // file
@@ -379,7 +379,7 @@ message ListKnowledgeBaseFilesFilter {
 // list files request
 message ListKnowledgeBaseFilesRequest {
   // The owner uid.
-  string owner_id = 1;
+  string namespace_id = 1;
   // The knowledge base uid.
   string kb_id = 2;
   // The page size (default:10; max 100).

diff --git a/artifact/artifact/v1alpha/artifact_private_service.proto b/artifact/artifact/v1alpha/artifact_private_service.proto
@@ -3,7 +3,6 @@ syntax = "proto3";
 package artifact.artifact.v1alpha;
 
 import "artifact/artifact/v1alpha/artifact.proto";
-import "google/api/visibility.proto";
 
 // ArtifactPrivateService exposes the private endpoints that allow clients to
 // manage artifacts.
@@ -28,6 +27,4 @@ service ArtifactPrivateService {
 
   // Delete a repository tag.
   rpc DeleteRepositoryTag(DeleteRepositoryTagRequest) returns (DeleteRepositoryTagResponse);
-
-  option (google.api.api_visibility).restriction = "INTERNAL";
 }
diff --git a/artifact/artifact/v1alpha/artifact_public_service.proto b/artifact/artifact/v1alpha/artifact_public_service.proto
@@ -43,37 +43,37 @@ service ArtifactPublicService {
   // Create a knowledge base
   rpc CreateKnowledgeBase(CreateKnowledgeBaseRequest) returns (CreateKnowledgeBaseResponse) {
     option (google.api.http) = {
-      post: "/v1alpha/namespaces/{owner_id}/knowledge-bases"
+      post: "/v1alpha/namespaces/{namespace_id}/knowledge-bases"
       body: "*"
     };
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
   // Get all knowledge bases info
   rpc ListKnowledgeBases(ListKnowledgeBasesRequest) returns (ListKnowledgeBasesResponse) {
-    option (google.api.http) = {get: "/v1alpha/namespaces/{owner_id}/knowledge-bases"};
+    option (google.api.http) = {get: "/v1alpha/namespaces/{namespace_id}/knowledge-bases"};
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
   // Update a knowledge base info
   rpc UpdateKnowledgeBase(UpdateKnowledgeBaseRequest) returns (UpdateKnowledgeBaseResponse) {
     option (google.api.http) = {
-      put: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}"
+      put: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}"
       body: "*"
     };
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
   // Delete a knowledge base
   rpc DeleteKnowledgeBase(DeleteKnowledgeBaseRequest) returns (DeleteKnowledgeBaseResponse) {
-    option (google.api.http) = {delete: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}"};
+    option (google.api.http) = {delete: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}"};
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
   // Create a file
   rpc UploadKnowledgeBaseFile(UploadKnowledgeBaseFileRequest) returns (UploadKnowledgeBaseFileResponse) {
     option (google.api.http) = {
-      post: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}/files"
+      post: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}/files"
       body: "file"
     };
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
@@ -96,22 +96,18 @@ service ArtifactPublicService {
 
   // list files
   rpc ListKnowledgeBaseFiles(ListKnowledgeBaseFilesRequest) returns (ListKnowledgeBaseFilesResponse) {
-    option (google.api.http) = {get: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}/files"};
+    option (google.api.http) = {get: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}/files"};
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
   // List chunks
   rpc ListChunks(ListChunksRequest) returns (ListChunksResponse) {
-    option (google.api.http) = {
-      get: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}/chunks"
-    };
+    option (google.api.http) = {get: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}/chunks"};
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
   // Get source file
   rpc GetSourceFile(GetSourceFileRequest) returns (GetSourceFileResponse) {
-    option (google.api.http) = {
-      get: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}/files/{file_uid}/source"
-    };
+    option (google.api.http) = {get: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}/files/{file_uid}/source"};
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};
   }
 
@@ -127,7 +123,7 @@ service ArtifactPublicService {
   // Similarity chunks search
   rpc SimilarityChunksSearch(SimilarityChunksSearchRequest) returns (SimilarityChunksSearchResponse) {
     option (google.api.http) = {
-      post: "/v1alpha/namespaces/{owner_id}/knowledge-bases/{kb_id}/chunks/similarity"
+      post: "/v1alpha/namespaces/{namespace_id}/knowledge-bases/{kb_id}/chunks/similarity"
       body: "*"
     };
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "KnowledgeBase"};