Merge pull request #102 from caraml-dev/sdk-migrate

Migrate golang SDK from original feast repo
caraml-dev · Jun 5, 2023 · 24119a9 · 24119a9
2 parents fe2fded + 9d3263f
commit 24119a9
Show file tree

Hide file tree

Showing 89 changed files with 12,633 additions and 3,665 deletions.
diff --git a/.github/workflows/python_proto.yaml b/.github/workflows/python_proto.yaml
diff --git a/.github/workflows/sdk_release.yaml b/.github/workflows/sdk_release.yaml
@@ -5,7 +5,7 @@ on:
   # (only if there are changes to relevant paths)
   push:
     tags:
-      - "sdk/v[0-9]+.[0-9]+.[0-9]+*"
+      - "sdk/python/v[0-9]+.[0-9]+.[0-9]+*"
     paths:
       - ".github/workflows/sdk_release.yaml"
       - "caraml-store-sdk/python/**"

diff --git a/Makefile b/Makefile
@@ -15,28 +15,13 @@ test-all:
 image:
 	./gradlew jib
 
-setup-e2e-tests:
-	pip install -r e2e_tests/requirements.txt && \
-	pip install feast==$(PYTHON_SDK_VERSION) feast-spark==$(SPARK_SDK_VERSION)
-
-run-e2e-tests: setup-e2e-tests
-	cd e2e_tests; \
-	pytest --verbose \
-	--color=yes \
-	--registry-url $(CARAML_STORE_REGISTRY_URL) \
-	--serving-url $(CARAML_STORE_SERVING_URL) \
-	--kafka-brokers $(KAFKA_BROKERS) \
-	--bq-project $(GCP_PROJECT) \
-	--historical-feature-output-location $(GCP_BUCKET_PATH) \
-	--store-name $(STORE_NAME) \
-	--store-type $(STORE_TYPE)
-
-# Python SDK
-
-PROTOC_IMAGE_VERSION=latest
-
-build-docker-protoc:
-	docker build -t protoc:${PROTOC_IMAGE_VERSION} -f caraml-store-sdk/python/Dockerfile caraml-store-sdk/python
+# SDK
+
+bufbuild-image:
+	docker build -f bufbuild.Dockerfile . -t caraml-store-bufbuild:build
+
+bufbuild-proto:
+	docker run --volume "${PWD}:/caraml-store" --workdir /caraml-store/caraml-store-protobuf/src/main/proto caraml-store-bufbuild:build generate --template buf.gen.sdk.yaml
 
 compile-protos-py:
 	docker run -v ${PROJECT_ROOT_DIR}:/local protoc

diff --git a/bufbuild.Dockerfile b/bufbuild.Dockerfile
@@ -0,0 +1,15 @@
+FROM bufbuild/buf:1.20.0
+
+RUN apk add --no-cache git make musl-dev go
+
+# Configure Go
+ENV GOROOT /usr/lib/go
+ENV GOPATH /go
+ENV PATH /go/bin:$PATH
+
+RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
+
+ENV PYTHONUNBUFFERED=1
+RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
+RUN python3 -m ensurepip
+RUN pip3 install --no-cache --upgrade pip setuptools
diff --git a/caraml-store-protobuf/src/main/proto/buf.gen.sdk.yaml b/caraml-store-protobuf/src/main/proto/buf.gen.sdk.yaml
@@ -0,0 +1,13 @@
+version: v1
+
+plugins:
+  - plugin: buf.build/protocolbuffers/go:v1.30.0
+    out: ../../../../caraml-store-sdk/go/protos
+    opt: paths=source_relative
+  - plugin: buf.build/grpc/go:v1.3.0
+    out: ../../../../caraml-store-sdk/go/protos
+    opt: paths=source_relative,require_unimplemented_servers=false
+  - plugin: buf.build/protocolbuffers/python:v21.12
+    out: ../../../../caraml-store-sdk/python
+  - plugin: buf.build/grpc/python:v1.50.0
+    out: ../../../../caraml-store-sdk/python
diff --git a/caraml-store-protobuf/src/main/proto/feast/core/CoreService.proto b/caraml-store-protobuf/src/main/proto/feast/core/CoreService.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "CoreServiceProto";
 option java_package = "dev.caraml.store.protobuf.core";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/DataFormat.proto b/caraml-store-protobuf/src/main/proto/feast/core/DataFormat.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "DataFormatProto";
 option java_package = "dev.caraml.store.protobuf.core";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/DataSource.proto b/caraml-store-protobuf/src/main/proto/feast/core/DataSource.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "DataSourceProto";
 option java_package = "dev.caraml.store.protobuf.core";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/Entity.proto b/caraml-store-protobuf/src/main/proto/feast/core/Entity.proto
@@ -1,6 +1,8 @@
 syntax = "proto3";
 
 package feast.core;
+
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_package = "dev.caraml.store.protobuf.core";
 option java_outer_classname = "EntityProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/Feature.proto b/caraml-store-protobuf/src/main/proto/feast/core/Feature.proto
@@ -1,7 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
-
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "FeatureProto";
 option java_package = "dev.caraml.store.protobuf.core";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/FeatureTable.proto b/caraml-store-protobuf/src/main/proto/feast/core/FeatureTable.proto
@@ -1,7 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
-
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "FeatureTableProto";
 option java_package = "dev.caraml.store.protobuf.core";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/LegacyJobService.proto b/caraml-store-protobuf/src/main/proto/feast/core/LegacyJobService.proto
@@ -4,6 +4,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_outer_classname = "JobServiceProto";
 option java_package = "dev.caraml.store.protobuf.compat";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/OnlineStore.proto b/caraml-store-protobuf/src/main/proto/feast/core/OnlineStore.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_package = "dev.caraml.store.protobuf.core";
 option java_outer_classname = "OnlineStoreProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/core/SparkOverride.proto b/caraml-store-protobuf/src/main/proto/feast/core/SparkOverride.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast.core;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/core";
 option java_package = "dev.caraml.store.protobuf.core";
 option java_outer_classname = "SparkOverrideProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/serving/ServingService.proto b/caraml-store-protobuf/src/main/proto/feast/serving/ServingService.proto
@@ -5,6 +5,7 @@ package feast.serving;
 import "google/protobuf/timestamp.proto";
 import "feast/types/Value.proto";
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/serving";
 option java_package = "dev.caraml.store.protobuf.serving";
 option java_outer_classname = "ServingServiceProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/types/Field.proto b/caraml-store-protobuf/src/main/proto/feast/types/Field.proto
@@ -4,6 +4,7 @@ import "feast/types/Value.proto";
 
 package feast.types;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/types";
 option java_package = "dev.caraml.store.protobuf.types";
 option java_outer_classname = "FieldProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast/types/Value.proto b/caraml-store-protobuf/src/main/proto/feast/types/Value.proto
@@ -2,6 +2,7 @@ syntax = "proto3";
 
 package feast.types;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast/types";
 option java_package = "dev.caraml.store.protobuf.types";
 option java_outer_classname = "ValueProto";
 

diff --git a/caraml-store-protobuf/src/main/proto/feast_spark/api/JobService.proto b/caraml-store-protobuf/src/main/proto/feast_spark/api/JobService.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 package feast_spark.api;
 
+option go_package = "github.com/caraml-dev/caraml-store/caraml-store-sdk/go/protos/feast_spark/api";
 option java_outer_classname = "JobServiceProto";
 option java_package = "dev.caraml.store.protobuf.jobservice";
 

diff --git a/caraml-store-sdk/go/go.mod b/caraml-store-sdk/go/go.mod
@@ -0,0 +1,28 @@
+module github.com/caraml-dev/caraml-store/caraml-store-sdk/go
+
+go 1.20
+
+require (
+	github.com/golang/mock v1.1.1
+	github.com/golang/protobuf v1.5.2
+	github.com/google/go-cmp v0.5.9
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
+	go.opencensus.io v0.24.0
+	golang.org/x/oauth2 v0.4.0
+	google.golang.org/api v0.103.0
+	google.golang.org/grpc v1.54.1
+	google.golang.org/protobuf v1.28.1
+)
+
+require (
+	cloud.google.com/go/compute v1.15.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+)