Use buf to generate python stubs (flyteorg#346)

* Buf python migration Signed-off-by: Eduardo Apolinario <[email protected]> * Generate pyi files Signed-off-by: Eduardo Apolinario <[email protected]> * Add venv to .gitgnore Signed-off-by: Eduardo Apolinario <[email protected]> * Use buf to generate python stubs Signed-off-by: Eduardo Apolinario <[email protected]> * Use buf docker image to generate stubs Signed-off-by: Eduardo Apolinario <[email protected]> * Add stubs produced by call to `buf generate` using buf's docker image Signed-off-by: Eduardo Apolinario <[email protected]> * Add pyi files Signed-off-by: Eduardo Apolinario <[email protected]> * Use buf locally Signed-off-by: Eduardo Apolinario <[email protected]> * Verify that generated protos by using buf Signed-off-by: Eduardo Apolinario <[email protected]> * Copy generated code to a separate artifact Signed-off-by: Eduardo Apolinario <[email protected]> * Move back to go_generate.yml@master Signed-off-by: Eduardo Apolinario <[email protected]> Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
eapolinario · Nov 29, 2022 · f4ede35 · f4ede35
1 parent 0d0c0ed
commit f4ede35
Show file tree

Hide file tree

Showing 171 changed files with 8,320 additions and 24,163 deletions.
diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml
@@ -51,19 +51,20 @@ jobs:
       - name: Proto-Datacatalog-Go
         run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out
 
+      # Python is going to be generated using buf
       # Python
-      - name: Proto-Service-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
-      - name: Proto-Admin-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
-      - name: Proto-Core-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
-      - name: Proto-Event-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
-      - name: Proto-Plugins-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
-      - name: Proto-Datacatalog-Python
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
+      # - name: Proto-Service-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python
+      # - name: Proto-Admin-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python
+      # - name: Proto-Core-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python
+      # - name: Proto-Event-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python
+      # - name: Proto-Plugins-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python
+      # - name: Proto-Datacatalog-Python
+      #   run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python
 
       # Cpp
       - name: Proto-Service-Cpp
@@ -98,10 +99,31 @@ jobs:
           name: generated
           path: ./gen
 
+  generate-protos-buf:
+    name: Generate python protos using buf
+    runs-on: ubuntu-latest
+    needs: [ generate-protos ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - uses: bufbuild/buf-setup-action@v1
+      - name: "Clean Generated"
+        run: rm -rf ./gen
+      - name: "Run buf generate"
+        run: buf generate
+      - uses: actions/upload-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
+
   generate-admin-swagger-code:
     runs-on: ubuntu-latest
     name: Generate Admin Swagger Code
-    needs: [ generate-protos ]
+    needs:
+      - generate-protos
+      - generate-protos-buf
     container:
       image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2
       options: --cpus 1
@@ -116,6 +138,10 @@ jobs:
         with:
           name: generated
           path: ./gen
+      - uses: actions/download-artifact@master
+        with:
+          name: generated-buf
+          path: ./gen
       - run: ls gen/pb-go/flyteidl/service/
       - run: cp -R gen/pb-go/flyteidl/service/ ../
       - run: ls ../service/

diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,5 @@ docs/build/
 tmp/
 .python-version
 __pycache__/
+
+venv/
diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -0,0 +1,10 @@
+version: v1
+managed:
+  enabled: true
+plugins:
+  - remote: buf.build/grpc/plugins/python:v1.49.1-1
+    out: gen/pb_python
+  - remote: buf.build/protocolbuffers/plugins/python:v21.7.0-1
+    out: gen/pb_python
+  - remote: buf.build/protocolbuffers/plugins/pyi:v21.9.0-1
+    out: gen/pb_python
diff --git a/gen/__init__.py b/gen/__init__.py
diff --git a/gen/pb_python/__init__.py b/gen/pb_python/__init__.py
diff --git a/gen/pb_python/flyteidl/__init__.py b/gen/pb_python/flyteidl/__init__.py
diff --git a/gen/pb_python/flyteidl/admin/__init__.py b/gen/pb_python/flyteidl/admin/__init__.py
diff --git a/gen/pb_python/flyteidl/admin/cluster_assignment_pb2.py b/gen/pb_python/flyteidl/admin/cluster_assignment_pb2.py
diff --git a/gen/pb_python/flyteidl/admin/cluster_assignment_pb2.pyi b/gen/pb_python/flyteidl/admin/cluster_assignment_pb2.pyi
diff --git a/gen/pb_python/flyteidl/admin/cluster_assignment_pb2_grpc.py b/gen/pb_python/flyteidl/admin/cluster_assignment_pb2_grpc.py
-Original file line number
+Diff line change
@@ Expand Up / @@ -16,3 +16,5 @@ docs/build/ @@
     tmp/
     .python-version
     __pycache__/
+    venv/