diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9eb561263c..46e1665754 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server-python-aws, feature-server-java, feature-transformation-server] + component: [feature-server-python, feature-server-python-aws, feature-server-java, feature-transformation-server] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6d5b21d4cc..b0da6d424e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,9 @@ - [(Contrib) Running tests for HBase online store](#contrib-running-tests-for-hbase-online-store) - [(Experimental) Feast UI](#experimental-feast-ui) - [Feast Java Serving](#feast-java-serving) +- [Developing the Feast Helm charts](#developing-the-feast-helm-charts) + - [Feast Java Feature Server Helm Chart](#feast-java-feature-server-helm-chart) + - [Feast Python / Go Feature Server Helm Chart](#feast-python--go-feature-server-helm-chart) - [Feast Go Client](#feast-go-client) - [Environment Setup](#environment-setup-1) - [Building](#building) @@ -197,7 +200,6 @@ To test across clouds, on top of setting up Redis, you also need GCP / AWS / Sno > and commenting out tests that are added to `DEFAULT_FULL_REPO_CONFIGS` **GCP** -### Setup your GCP BigQuery Instance 1. You can get free credits [here](https://cloud.google.com/free/docs/free-cloud-features#free-trial). 2. You will need to setup a service account, enable the BigQuery API, and create a staging location for a bucket. * Setup your service account and project using steps 1-5 [here](https://codelabs.developers.google.com/codelabs/cloud-bigquery-python#0). @@ -347,6 +349,35 @@ See [Feast contributing guide](ui/CONTRIBUTING.md) ## Feast Java Serving See [Java contributing guide](java/CONTRIBUTING.md) +See also development instructions related to the helm chart below at [Developing the Feast Helm charts](#developing-the-feast-helm-charts) + +## Developing the Feast Helm charts +There are 3 helm charts: +- Feast Java feature server +- Feast Python / Go feature server +- (deprecated) Feast Python feature server + +Generally, you can override the images in the helm charts with locally built Docker images, and install the local helm +chart. + +All README's for helm charts are generated using [helm-docs](https://github.com/norwoodj/helm-docs). You can install it +(e.g. with `brew install norwoodj/tap/helm-docs`) and then run `make build-helm-docs`. + +### Feast Java Feature Server Helm Chart +See the Java demo example (it has development instructions too using minikube) [here](examples/java-demo/README.md) + +It will: +- run `make build-java-docker-dev` to build local Java feature server binaries +- configure the included `application-override.yaml` to override the image tag to use the locally built dev images. +- install the local chart with `helm install feast-release ../../../infra/charts/feast --values application-override.yaml` + +### Feast Python / Go Feature Server Helm Chart +See the Python demo example (it has development instructions too using minikube) [here](examples/python-helm-demo/README.md) + +It will: +- run `make build-feature-server-dev` to build a local python feature server binary +- install the local chart with `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` + ## Feast Go Client ### Environment Setup Setting up your development environment for Feast Go SDK: diff --git a/Makefile b/Makefile index a2d8bca5e4..4bd1eaed4c 100644 --- a/Makefile +++ b/Makefile @@ -330,7 +330,7 @@ lint-go: compile-protos-go compile-go-lib # Docker -build-docker: build-ci-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-ci-docker build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) @@ -339,13 +339,21 @@ push-ci-docker: build-ci-docker: docker buildx build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile --load . +push-feature-server-python-docker: + docker push $(REGISTRY)/feature-server:$$VERSION + +build-feature-server-python-docker: + docker buildx build --build-arg VERSION=$$VERSION \ + -t $(REGISTRY)/feature-server:$$VERSION \ + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile --load . + push-feature-server-python-aws-docker: - docker push $(REGISTRY)/feature-server-python-aws:$$VERSION + docker push $(REGISTRY)/feature-server-python-aws:$$VERSION build-feature-server-python-aws-docker: - docker buildx build --build-arg VERSION=$$VERSION \ - -t $(REGISTRY)/feature-server-python-aws:$$VERSION \ - -f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile --load . + docker buildx build --build-arg VERSION=$$VERSION \ + -t $(REGISTRY)/feature-server-python-aws:$$VERSION \ + -f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile --load . push-feature-transformation-server-docker: docker push $(REGISTRY)/feature-transformation-server:$(VERSION) @@ -363,6 +371,13 @@ build-feature-server-java-docker: -t $(REGISTRY)/feature-server-java:$(VERSION) \ -f java/infra/docker/feature-server/Dockerfile --load . +# Dev images + +build-feature-server-dev: + docker buildx build --build-arg VERSION=dev \ + -t feastdev/feature-server:dev \ + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . + build-java-docker-dev: make build-java-no-tests REVISION=dev docker buildx build --build-arg VERSION=dev \ @@ -402,6 +417,11 @@ build-sphinx: compile-protos-python build-templates: python infra/scripts/compile-templates.py +build-helm-docs: + cd ${ROOT_DIR}/infra/charts/feast; helm-docs + cd ${ROOT_DIR}/infra/charts/feast-feature-server; helm-docs + cd ${ROOT_DIR}/infra/charts/feast-python-server; helm-docs + # Web UI # Note: requires node and yarn to be installed diff --git a/docs/getting-started/concepts/feature-retrieval.md b/docs/getting-started/concepts/feature-retrieval.md index eb7f7a18b1..0bc8c9e1d6 100644 --- a/docs/getting-started/concepts/feature-retrieval.md +++ b/docs/getting-started/concepts/feature-retrieval.md @@ -6,7 +6,9 @@ Generally, Feast supports several patterns of feature retrieval: 1. Training data generation (via `feature_store.get_historical_features(...)`) 2. Offline feature retrieval for batch scoring (via `feature_store.get_historical_features(...)`) -3. Online feature retrieval for real-time model predictions (via `feature_store.get_online_features(...)`) +3. Online feature retrieval for real-time model predictions + - via the SDK: `feature_store.get_online_features(...)` + - via deployed feature server endpoints: `requests.post('http://localhost:6566/get-online-features', data=json.dumps(online_request))` Each of these retrieval mechanisms accept: @@ -100,7 +102,6 @@ batch_scoring_features = store.get_historical_features( ```python from feast import FeatureStore -import pandas as pd store = FeatureStore(repo_path=".") @@ -124,13 +125,23 @@ batch_scoring_features = store.get_historical_features(
-How to: retrieve online features for real-time model inference +How to: retrieve online features for real-time model inference (Python SDK) Feast will ensure the latest feature values for registered features are available. At retrieval time, you need to supply a list of **entities** and the corresponding **features** to be retrieved. Similar to `get_historical_features`, we recommend using feature services as a mechanism for grouping features in a model version. _Note: unlike `get_historical_features`, the `entity_rows` **do not need timestamps** since you only want one feature value per entity key._ ```python +from feast import RepoConfig, FeatureStore +from feast.repo_config import RegistryConfig + +repo_config = RepoConfig( + registry=RegistryConfig(path="gs://feast-test-gcs-bucket/registry.pb"), + project="feast_demo_gcp", + provider="gcp", +) +store = FeatureStore(config=repo_config) + features = store.get_online_features( features=[ "driver_hourly_stats:conv_rate", @@ -147,6 +158,32 @@ features = store.get_online_features(
+
+ +How to: retrieve online features for real-time model inference (Feature Server) + +Feast will ensure the latest feature values for registered features are available. At retrieval time, you need to supply a list of **entities** and the corresponding **features** to be retrieved. Similar to `get_historical_features`, we recommend using feature services as a mechanism for grouping features in a model version. + +_Note: unlike `get_historical_features`, the `entity_rows` **do not need timestamps** since you only want one feature value per entity key._ + +This approach requires you to deploy a feature server (see [Python feature server](../../reference/feature-servers/python-feature-server)). + +```python +import requests +import json + +online_request = { + "features": [ + "driver_hourly_stats:conv_rate", + ], + "entities": {"driver_id": [1001, 1002]}, +} +r = requests.post('http://localhost:6566/get-online-features', data=json.dumps(online_request)) +print(json.dumps(r.json(), indent=4, sort_keys=True)) +``` + +
+ ## Feature Services A feature service is an object that represents a logical group of features from one or more [feature views](feature-view.md#feature-view). Feature Services allows features from within a feature view to be used as needed by an ML model. Users can expect to create one feature service per model version, allowing for tracking of the features used by models. diff --git a/examples/java-demo/README.md b/examples/java-demo/README.md index ef4960fb4d..0ae085e0a7 100644 --- a/examples/java-demo/README.md +++ b/examples/java-demo/README.md @@ -30,18 +30,21 @@ For this tutorial, we setup Feast with Redis, using the Feast CLI to register an 2. Make a bucket in GCS (or S3) 3. The feature repo is already setup here, so you just need to swap in your GCS bucket and Redis credentials. We need to modify the `feature_store.yaml`, which has two fields for you to replace: - ```yaml - registry: gs://[YOUR BUCKET]/demo-repo/registry.db + ```yaml + registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db project: feast_java_demo provider: gcp online_store: type: redis + # Note: this would normally be using instance URL's to access Redis connection_string: localhost:6379,password=[YOUR PASSWORD] offline_store: type: file + entity_key_serialization_version: 2 ``` 4. Run `feast apply` to apply your local features to the remote registry -5. Materialize features to the online store: + - Note: you may need to authenticate to gcloud first with `gcloud auth login` +6. Materialize features to the online store: ```bash CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") feast materialize-incremental $CURRENT_TIME diff --git a/examples/java-demo/feature_repo/feature_store.yaml b/examples/java-demo/feature_repo/feature_store.yaml index 29fe3c97ce..16d426fc5a 100644 --- a/examples/java-demo/feature_repo/feature_store.yaml +++ b/examples/java-demo/feature_repo/feature_store.yaml @@ -3,6 +3,7 @@ project: feast_java_demo provider: gcp online_store: type: redis + # Note: this would normally be using instance URL's to access Redis connection_string: localhost:6379,password=[YOUR PASSWORD] offline_store: type: file diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md new file mode 100644 index 0000000000..44cd4799d5 --- /dev/null +++ b/examples/python-helm-demo/README.md @@ -0,0 +1,89 @@ + +# Running Feast Python / Go Feature Server with Redis on Kubernetes + +For this tutorial, we set up Feast with Redis. + +We use the Feast CLI to register and materialize features, and then retrieving via a Feast Python feature server deployed in Kubernetes + +## First, let's set up a Redis cluster +1. Start minikube (`minikube start`) +2. Use helm to install a default Redis cluster + ```bash + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo update + helm install my-redis bitnami/redis + ``` + ![](redis-screenshot.png) +3. Port forward Redis so we can materialize features to it + + ```bash + kubectl port-forward --namespace default svc/my-redis-master 6379:6379 + ``` +4. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. + + ```bash + export REDIS_PASSWORD=$(kubectl get secret --namespace default my-redis -o jsonpath="{.data.redis-password}" | base64 --decode) + echo $REDIS_PASSWORD + ``` + +## Next, we setup a local Feast repo +1. Install Feast with Redis dependencies `pip install "feast[redis]"` +2. Make a bucket in GCS (or S3) +3. The feature repo is already setup here, so you just need to swap in your GCS bucket and Redis credentials. + We need to modify the `feature_store.yaml`, which has two fields for you to replace: + ```yaml + registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db + project: feast_python_demo + provider: gcp + online_store: + type: redis + # Note: this would normally be using instance URL's to access Redis + connection_string: localhost:6379,password=[YOUR PASSWORD] + offline_store: + type: file + entity_key_serialization_version: 2 + ``` +4. Run `feast apply` from within the `feature_repo` directory to apply your local features to the remote registry + - Note: you may need to authenticate to gcloud first with `gcloud auth login` +5. Materialize features to the online store: + ```bash + CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") + feast materialize-incremental $CURRENT_TIME + ``` + +## Now let's setup the Feast Server +1. Add the gcp-auth addon to mount GCP credentials: + ```bash + minikube addons enable gcp-auth + ``` +2. Add Feast's Python/Go feature server chart repo + ```bash + helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com + helm repo update + ``` +3. For this tutorial, because we don't have a direct hosted endpoint into Redis, we need to change `feature_store.yaml` to talk to the Kubernetes Redis service + ```bash + sed -i '' 's/localhost:6379/my-redis-master:6379/g' feature_store.yaml + ``` +4. Install the Feast helm chart: `helm install feast-release feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml)` + > **Dev instructions**: if you're changing the java logic or chart, you can do + 1. `eval $(minikube docker-env)` + 2. `make build-feature-server-dev` + 3. `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` +5. (Optional): check logs of the server to make sure it’s working + ```bash + kubectl logs svc/feast-feature-server + ``` +6. Port forward to expose the grpc endpoint: + ```bash + kubectl port-forward svc/feast-feature-server 6566:80 + ``` +7. Run test fetches for online features:8. + - First: change back the Redis connection string to allow localhost connections to Redis + ```bash + sed -i '' 's/my-redis-master:6379/localhost:6379/g' feature_store.yaml + ``` + - Then run the included fetch script, which fetches both via the HTTP endpoint and for comparison, via the Python SDK + ```bash + python test_python_fetch.py + ``` \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/__init__.py b/examples/python-helm-demo/feature_repo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet new file mode 100644 index 0000000000..83b8c31aa5 Binary files /dev/null and b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet differ diff --git a/examples/python-helm-demo/feature_repo/driver_repo.py b/examples/python-helm-demo/feature_repo/driver_repo.py new file mode 100644 index 0000000000..f7dd05afff --- /dev/null +++ b/examples/python-helm-demo/feature_repo/driver_repo.py @@ -0,0 +1,61 @@ +from datetime import timedelta + +import pandas as pd + +from feast.data_source import RequestSource +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64, String +from feast.field import Field + +from feast import Entity, FileSource, FeatureView + +driver_hourly_stats = FileSource( + path="data/driver_stats_with_string.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +driver = Entity(name="driver_id", description="driver id",) +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + Field(name="string_feature", dtype=String), + ], + online=True, + source=driver_hourly_stats, + tags={}, +) + +# Define a request data source which encodes features / information only +# available at request time (e.g. part of the user initiated HTTP request) +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), + ], +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[ + driver_hourly_stats_view, + input_request, + ], + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml b/examples/python-helm-demo/feature_repo/feature_store.yaml new file mode 100644 index 0000000000..d49c0cbd0e --- /dev/null +++ b/examples/python-helm-demo/feature_repo/feature_store.yaml @@ -0,0 +1,10 @@ +registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db +project: feast_python_demo +provider: gcp +online_store: + type: redis + # Note: this would normally be using instance URL's to access Redis + connection_string: localhost:6379,password=[YOUR PASSWORD] +offline_store: + type: file +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/test_python_fetch.py b/examples/python-helm-demo/feature_repo/test_python_fetch.py new file mode 100644 index 0000000000..f9c7c62f4f --- /dev/null +++ b/examples/python-helm-demo/feature_repo/test_python_fetch.py @@ -0,0 +1,43 @@ +from feast import FeatureStore +import requests +import json + + +def run_demo_http(): + print("\n--- Online features with HTTP endpoint ---") + online_request = { + "features": [ + "driver_hourly_stats:conv_rate", + ], + "entities": {"driver_id": [1001, 1002]}, + } + r = requests.post( + "http://localhost:6566/get-online-features", data=json.dumps(online_request) + ) + print(json.dumps(r.json(), indent=4, sort_keys=True)) + + +def run_demo_sdk(): + store = FeatureStore(repo_path=".") + + print("\n--- Online features with SDK ---") + features = store.get_online_features( + features=[ + "driver_hourly_stats:conv_rate", + ], + entity_rows=[ + { + "driver_id": 1001, + }, + { + "driver_id": 1002, + }, + ], + ).to_dict() + for key, value in sorted(features.items()): + print(key, " : ", value) + + +if __name__ == "__main__": + run_demo_sdk() + run_demo_http() diff --git a/examples/python-helm-demo/redis-screenshot.png b/examples/python-helm-demo/redis-screenshot.png new file mode 100644 index 0000000000..489deb699d Binary files /dev/null and b/examples/python-helm-demo/redis-screenshot.png differ diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index c5c2b9d654..8d551c82f7 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -3,59 +3,31 @@ Current chart version is `0.23.0` ## Installation -### Python feature server -Docker repository and tag are required. Helm install example: -``` -helm install feast-feature-server . --set image.repository=REPO --set image.tag=TAG -``` - -Deployment assumes that `feature_store.yaml` exists on docker image. Example docker image: -``` -FROM python:3.8 - -RUN apt update && \ - apt install -y jq - -RUN pip install pip --upgrade -RUN pip install feast +Run the following commands to add the repository -COPY feature_store.yaml /feature_store.yaml ``` - -### Go feature server -> Warning: this is experimental, and only supports a local file registry + Redis - -Furthermore, if you wish to use the Go feature server, then you must install the Apache Arrow C++ libraries, and your `feature_store.yaml` should include `go_feature_server: True`. -For more details, see the [docs](https://docs.feast.dev/reference/feature-servers/go-feature-server). - -The docker image might look like: +helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com +helm repo update ``` - FROM python:3.8 - RUN apt update && \ - apt install -y jq - RUN pip install pip --upgrade - RUN pip install feast - RUN apt update - RUN apt install -y -V ca-certificates lsb-release wget - RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - RUN apt update - RUN apt -y install libarrow-dev +Install Feast - COPY feature_store.yaml /feature_store.yaml - ``` +A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: +``` +helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) +``` ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | | +| feature_store_yaml_base64 | string | `""` | [required] a base64 encoded version of feature_store.yaml | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `""` | [required] The repository for the Docker image | -| image.tag | string | `""` | [required] The Docker image tag | +| image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | +| image.tag | string | `"0.23.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/README.md.gotmpl b/infra/charts/feast-feature-server/README.md.gotmpl index 79a681b3fc..75f2827466 100644 --- a/infra/charts/feast-feature-server/README.md.gotmpl +++ b/infra/charts/feast-feature-server/README.md.gotmpl @@ -3,49 +3,20 @@ Current chart version is `{{ template "chart.version" . }}` ## Installation -### Python feature server -Docker repository and tag are required. Helm install example: -``` -helm install feast-feature-server . --set image.repository=REPO --set image.tag=TAG -``` - -Deployment assumes that `feature_store.yaml` exists on docker image. Example docker image: -``` -FROM python:3.8 - -RUN apt update && \ - apt install -y jq -RUN pip install pip --upgrade +Run the following commands to add the repository -RUN pip install feast - -COPY feature_store.yaml /feature_store.yaml +``` +helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com +helm repo update ``` -### Go feature server -> Warning: this is experimental, and only supports a local file registry + Redis - -Furthermore, if you wish to use the Go feature server, then you must install the Apache Arrow C++ libraries, and your `feature_store.yaml` should include `go_feature_server: True`. -For more details, see the [docs](https://docs.feast.dev/reference/feature-servers/go-feature-server). +Install Feast -The docker image might look like: +A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: +``` +helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` - FROM python:3.8 - - RUN apt update && \ - apt install -y jq - RUN pip install pip --upgrade - RUN pip install feast - RUN apt update - RUN apt install -y -V ca-certificates lsb-release wget - RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - RUN apt update - RUN apt -y install libarrow-dev - - COPY feature_store.yaml /feature_store.yaml - ``` {{ template "chart.requirementsSection" . }} diff --git a/infra/charts/feast-feature-server/templates/deployment.yaml b/infra/charts/feast-feature-server/templates/deployment.yaml index 69cf92f6c0..94c56de9dd 100644 --- a/infra/charts/feast-feature-server/templates/deployment.yaml +++ b/infra/charts/feast-feature-server/templates/deployment.yaml @@ -30,6 +30,9 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: FEATURE_STORE_YAML_BASE64 + value: {{ .Values.feature_store_yaml_base64 }} command: ["feast", "serve", "-h", "0.0.0.0"] ports: - name: http diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 6d0ab9c0ae..9ffcb1485b 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -5,16 +5,19 @@ replicaCount: 1 image: - # image.repository -- [required] The repository for the Docker image - repository: "" + # image.repository -- Docker image for Feature Server repository + repository: feastdev/feature-server pullPolicy: IfNotPresent - # image.tag -- [required] The Docker image tag - tag: "" + # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) + tag: 0.23.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" +# feature_store_yaml_base64 -- [required] a base64 encoded version of feature_store.yaml +feature_store_yaml_base64: "" + podAnnotations: {} podSecurityContext: {} diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index 2c8264c35c..b0bc0be1e6 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -1,4 +1,6 @@ -# Feast Python Feature Server Helm Charts +# Feast Python Feature Server Helm Charts (deprecated) + +> Note: this helm chart is deprecated in favor of [feast-feature-server](../feast-feature-server/README.md) Current chart version is `0.23.0` diff --git a/infra/charts/feast-python-server/README.md.gotmpl b/infra/charts/feast-python-server/README.md.gotmpl index b9c88d11b4..cb264c0066 100644 --- a/infra/charts/feast-python-server/README.md.gotmpl +++ b/infra/charts/feast-python-server/README.md.gotmpl @@ -1,4 +1,6 @@ -# Feast Python Feature Server Helm Charts +# Feast Python Feature Server Helm Charts (deprecated) + +> Note: this helm chart is deprecated in favor of [feast-feature-server](../feast-feature-server/README.md) Current chart version is `{{ template "chart.version" . }}` diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 6a558e04f0..e94ec88db0 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -8,8 +8,9 @@ infra/charts/feast/charts/feature-server/README.md 3 20 infra/charts/feast/charts/feature-server/values.yaml 8 infra/charts/feast/README.md 11 58 59 infra/charts/feast-python-server/Chart.yaml 5 -infra/charts/feast-python-server/README.md 3 +infra/charts/feast-python-server/README.md 5 infra/charts/feast-feature-server/Chart.yaml 5 infra/charts/feast-feature-server/README.md 3 +infra/charts/feast-feature-server/values.yaml 12 java/pom.xml 38 ui/package.json 3 diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index f9d8bf37b6..af9aa51191 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import base64 import json import logging +import os +import tempfile from datetime import datetime from pathlib import Path from typing import List, Optional @@ -25,7 +28,10 @@ from pygments import formatters, highlight, lexers from feast import utils -from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT +from feast.constants import ( + DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + FEATURE_STORE_YAML_ENV_NAME, +) from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError from feast.feature_store import FeatureStore from feast.feature_view import FeatureView @@ -685,9 +691,21 @@ def serve_command( ): """Start a feature server locally on a given port.""" repo = ctx.obj["CHDIR"] - fs_yaml_file = ctx.obj["FS_YAML_FILE"] - cli_check_repo(repo, fs_yaml_file) - store = FeatureStore(repo_path=str(repo), fs_yaml_file=fs_yaml_file) + + # If we received a base64 encoded version of feature_store.yaml, use that + config_base64 = os.getenv(FEATURE_STORE_YAML_ENV_NAME) + if config_base64: + print("Received base64 encoded feature_store.yaml") + config_bytes = base64.b64decode(config_base64) + # Create a new unique directory for writing feature_store.yaml + repo_path = Path(tempfile.mkdtemp()) + with open(repo_path / "feature_store.yaml", "wb") as f: + f.write(config_bytes) + store = FeatureStore(repo_path=str(repo_path.resolve())) + else: + fs_yaml_file = ctx.obj["FS_YAML_FILE"] + cli_check_repo(repo, fs_yaml_file) + store = FeatureStore(repo_path=str(repo), fs_yaml_file=fs_yaml_file) if go: # Turn on Go feature retrieval. diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile new file mode 100644 index 0000000000..b853411e27 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8 + +RUN apt update && \ + apt install -y jq +RUN pip install pip --upgrade +RUN pip install "feast[aws,gcp,snowflake,redis,go]" +RUN apt update +RUN apt install -y -V ca-certificates lsb-release wget +RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +RUN apt update +RUN apt -y install libarrow-dev \ No newline at end of file diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev new file mode 100644 index 0000000000..f1dd7cc390 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -0,0 +1,14 @@ +FROM python:3.8 + +RUN apt update && \ + apt install -y jq +RUN pip install pip --upgrade +COPY . . + +RUN pip install ".[aws,gcp,snowflake,redis,go]" +RUN apt update +RUN apt install -y -V ca-certificates lsb-release wget +RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +RUN apt update +RUN apt -y install libarrow-dev \ No newline at end of file diff --git a/sdk/python/feast/infra/feature_servers/multicloud/__init__.py b/sdk/python/feast/infra/feature_servers/multicloud/__init__.py new file mode 100644 index 0000000000..e69de29bb2