diff --git a/.dockerignore b/.dockerignore
index e9401f0cc9..0e3b22687d 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,3 @@
docs
+!docs/coverage
charts
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index a30c747b13..c02551b014 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,8 +1,8 @@
# Each line is a file pattern followed by one or more owners.
# https://help.github.com/en/articles/about-code-owners
-* @zhilingc @pradithya @woop @davidheryanto @khorshuheng
-/core/ @zhilingc @pradithya
-/ingestion/ @zhilingc @pradithya
-/serving/ @zhilingc @pradithya
-/cli/ @zhilingc @pradithya
+* @zhilingc @woop @davidheryanto @khorshuheng @pyalex
+/core/ @zhilingc
+/ingestion/ @zhilingc
+/serving/ @zhilingc
+/cli/ @zhilingc
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index b9c8cd6dff..7a78437b5d 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,8 +1,8 @@
org.hibernatehibernate-core
- 5.3.6.Final
@@ -187,7 +206,6 @@
org.mockitomockito-core
- 2.23.0test
@@ -207,5 +225,21 @@
jaxb-api
+
+ javax.validation
+ validation-api
+ 2.0.0.Final
+
+
+ org.hibernate.validator
+ hibernate-validator
+ 6.1.2.Final
+
+
+ org.hibernate.validator
+ hibernate-validator-annotation-processor
+ 6.1.2.Final
+
+
diff --git a/core/src/main/java/feast/core/config/FeastProperties.java b/core/src/main/java/feast/core/config/FeastProperties.java
index b9c787b6c7..6dad278242 100644
--- a/core/src/main/java/feast/core/config/FeastProperties.java
+++ b/core/src/main/java/feast/core/config/FeastProperties.java
@@ -16,53 +16,225 @@
*/
package feast.core.config;
-import java.util.Map;
+import feast.core.config.FeastProperties.StreamProperties.FeatureStreamOptions;
+import feast.core.validators.OneOfStrings;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import javax.annotation.PostConstruct;
+import javax.validation.*;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Positive;
import lombok.Getter;
import lombok.Setter;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.info.BuildProperties;
@Getter
@Setter
@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true)
public class FeastProperties {
- private String version;
- private JobProperties jobs;
+ /**
+ * Instantiates a new Feast properties.
+ *
+ * @param buildProperties Feast build properties
+ */
+ @Autowired
+ public FeastProperties(BuildProperties buildProperties) {
+ setVersion(buildProperties.getVersion());
+ }
+
+ /** Instantiates a new Feast properties. */
+ public FeastProperties() {}
+
+ /* Feast Core Build Version */
+ @NotBlank private String version = "unknown";
+
+ /* Population job properties */
+ @NotNull private JobProperties jobs;
+
+ @NotNull
+ /* Feast Kafka stream properties */
private StreamProperties stream;
+ /** Feast job properties. These properties are used for ingestion jobs. */
@Getter
@Setter
public static class JobProperties {
- private String runner;
- private Map options;
+ @NotBlank
+ /* The active Apache Beam runner name. This name references one instance of the Runner class */
+ private String activeRunner;
+
+ /** List of configured job runners. */
+ private List runners = new ArrayList<>();
+
+ /**
+ * Gets a {@link Runner} instance of the active runner
+ *
+ * @return the active runner
+ */
+ public Runner getActiveRunner() {
+ for (Runner runner : getRunners()) {
+ if (activeRunner.equals(runner.getName())) {
+ return runner;
+ }
+ }
+ throw new RuntimeException(
+ String.format(
+ "Active runner is misconfigured. Could not find runner: %s.", activeRunner));
+ }
+
+ /** Job Runner class. */
+ @Getter
+ @Setter
+ public static class Runner {
+
+ /** Job runner name. This must be unique. */
+ String name;
+
+ /** Job runner type DirectRunner, DataflowRunner currently supported */
+ String type;
+
+ /**
+ * Job runner configuration options. See the following for options
+ * https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner
+ */
+ Map options = new HashMap<>();
+
+ /**
+ * Gets the job runner type as an enum.
+ *
+ * @return Returns the job runner type as {@link feast.core.job.Runner}
+ */
+ public feast.core.job.Runner getType() {
+ return feast.core.job.Runner.fromString(type);
+ }
+ }
+
+ @NotNull
+ /* Population job metric properties */
private MetricsProperties metrics;
- private JobUpdatesProperties updates;
- }
- @Getter
- @Setter
- public static class JobUpdatesProperties {
+ /* Timeout in seconds for each attempt to update or submit a new job to the runner */
+ @Positive private long jobUpdateTimeoutSeconds;
- private long timeoutSeconds;
- private long pollingIntervalMillis;
+ /* Job update polling interval in millisecond. How frequently Feast will update running jobs. */
+ @Positive private long pollingIntervalMilliseconds;
}
+ /** Properties used to configure Feast's managed Kafka feature stream. */
@Getter
@Setter
public static class StreamProperties {
+ /* Feature stream type. Only "kafka" is supported. */
+ @OneOfStrings({"kafka"})
+ @NotBlank
private String type;
- private Map options;
+
+ /* Feature stream options */
+ @NotNull private FeatureStreamOptions options;
+
+ /** Feature stream options */
+ @Getter
+ @Setter
+ public static class FeatureStreamOptions {
+
+ /* Kafka topic to use for feature sets without source topics. */
+ @NotBlank private String topic = "feast-features";
+
+ /**
+ * Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined
+ * source.
+ */
+ @NotBlank private String bootstrapServers = "localhost:9092";
+
+ /* Defines the number of copies of managed feature stream Kafka. */
+ @Positive private short replicationFactor = 1;
+
+ /* Number of Kafka partitions to to use for managed feature stream. */
+ @Positive private int partitions = 1;
+ }
}
+ /** Feast population job metrics */
@Getter
@Setter
public static class MetricsProperties {
+ /* Population job metrics enabled */
private boolean enabled;
+
+ /* Metric type. Possible options: statsd */
+ @OneOfStrings({"statsd"})
+ @NotBlank
private String type;
+
+ /* Host of metric sink */
private String host;
- private int port;
+
+ /* Port of metric sink */
+ @Positive private int port;
+ }
+
+ /**
+ * Validates all FeastProperties. This method runs after properties have been initialized and
+ * individually and conditionally validates each class.
+ */
+ @PostConstruct
+ public void validate() {
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Validator validator = factory.getValidator();
+
+ // Validate root fields in FeastProperties
+ Set> violations = validator.validate(this);
+ if (!violations.isEmpty()) {
+ throw new ConstraintViolationException(violations);
+ }
+
+ // Validate Stream properties
+ Set> streamPropertyViolations =
+ validator.validate(getStream());
+ if (!streamPropertyViolations.isEmpty()) {
+ throw new ConstraintViolationException(streamPropertyViolations);
+ }
+
+ // Validate Stream Options
+ Set> featureStreamOptionsViolations =
+ validator.validate(getStream().getOptions());
+ if (!featureStreamOptionsViolations.isEmpty()) {
+ throw new ConstraintViolationException(featureStreamOptionsViolations);
+ }
+
+ // Validate JobProperties
+ Set> jobPropertiesViolations = validator.validate(getJobs());
+ if (!jobPropertiesViolations.isEmpty()) {
+ throw new ConstraintViolationException(jobPropertiesViolations);
+ }
+
+ // Validate MetricsProperties
+ if (getJobs().getMetrics().isEnabled()) {
+ Set> jobMetricViolations =
+ validator.validate(getJobs().getMetrics());
+ if (!jobMetricViolations.isEmpty()) {
+ throw new ConstraintViolationException(jobMetricViolations);
+ }
+ // Additional custom check for hostname value because there is no built-in Spring annotation
+ // to validate the value is a DNS resolvable hostname or an IP address.
+ try {
+ //noinspection ResultOfMethodCallIgnored
+ InetAddress.getByName(getJobs().getMetrics().getHost());
+ } catch (UnknownHostException e) {
+ throw new IllegalArgumentException(
+ "Invalid config value for feast.jobs.metrics.host: "
+ + getJobs().getMetrics().getHost()
+ + ". Make sure it is a valid IP address or DNS hostname e.g. localhost or 10.128.10.40. Error detail: "
+ + e.getMessage());
+ }
+ }
}
}
diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java
index 45de359ac7..c1982604c3 100644
--- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java
+++ b/core/src/main/java/feast/core/config/FeatureStreamConfig.java
@@ -17,10 +17,10 @@
package feast.core.config;
import com.google.common.base.Strings;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
import feast.core.config.FeastProperties.StreamProperties;
import feast.core.model.Source;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.SourceType;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -48,8 +48,8 @@ public Source getDefaultSource(FeastProperties feastProperties) {
SourceType featureStreamType = SourceType.valueOf(streamProperties.getType().toUpperCase());
switch (featureStreamType) {
case KAFKA:
- String bootstrapServers = streamProperties.getOptions().get("bootstrapServers");
- String topicName = streamProperties.getOptions().get("topic");
+ String bootstrapServers = streamProperties.getOptions().getBootstrapServers();
+ String topicName = streamProperties.getOptions().getTopic();
Map map = new HashMap<>();
map.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
map.put(
@@ -59,9 +59,8 @@ public Source getDefaultSource(FeastProperties feastProperties) {
NewTopic newTopic =
new NewTopic(
topicName,
- Integer.valueOf(streamProperties.getOptions().getOrDefault("numPartitions", "1")),
- Short.valueOf(
- streamProperties.getOptions().getOrDefault("replicationFactor", "1")));
+ streamProperties.getOptions().getPartitions(),
+ streamProperties.getOptions().getReplicationFactor());
CreateTopicsResult createTopicsResult =
client.createTopics(Collections.singleton(newTopic));
try {
diff --git a/core/src/main/java/feast/core/config/JobConfig.java b/core/src/main/java/feast/core/config/JobConfig.java
index 728fc0545b..30023de064 100644
--- a/core/src/main/java/feast/core/config/JobConfig.java
+++ b/core/src/main/java/feast/core/config/JobConfig.java
@@ -16,22 +16,16 @@
*/
package feast.core.config;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.dataflow.Dataflow;
-import com.google.api.services.dataflow.DataflowScopes;
-import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.JsonFormat;
import feast.core.config.FeastProperties.JobProperties;
-import feast.core.config.FeastProperties.JobUpdatesProperties;
import feast.core.job.JobManager;
-import feast.core.job.Runner;
import feast.core.job.dataflow.DataflowJobManager;
import feast.core.job.direct.DirectJobRegistry;
import feast.core.job.direct.DirectRunnerJobManager;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.util.HashMap;
+import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions;
+import feast.proto.core.RunnerProto.DirectRunnerConfigOptions;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -42,67 +36,39 @@
@Slf4j
@Configuration
public class JobConfig {
+ private final Gson gson = new Gson();
/**
- * Get a JobManager according to the runner type and dataflow configuration.
+ * Get a JobManager according to the runner type and Dataflow configuration.
*
* @param feastProperties feast config properties
*/
@Bean
@Autowired
- public JobManager getJobManager(
- FeastProperties feastProperties, DirectJobRegistry directJobRegistry) {
+ public JobManager getJobManager(FeastProperties feastProperties)
+ throws InvalidProtocolBufferException {
JobProperties jobProperties = feastProperties.getJobs();
- Runner runner = Runner.fromString(jobProperties.getRunner());
- if (jobProperties.getOptions() == null) {
- jobProperties.setOptions(new HashMap<>());
- }
- Map jobOptions = jobProperties.getOptions();
- switch (runner) {
- case DATAFLOW:
- if (Strings.isNullOrEmpty(jobOptions.getOrDefault("region", null))
- || Strings.isNullOrEmpty(jobOptions.getOrDefault("project", null))) {
- log.error("Project and location of the Dataflow runner is not configured");
- throw new IllegalStateException(
- "Project and location of Dataflow runner must be specified for jobs to be run on Dataflow runner.");
- }
- try {
- GoogleCredential credential =
- GoogleCredential.getApplicationDefault().createScoped(DataflowScopes.all());
- Dataflow dataflow =
- new Dataflow(
- GoogleNetHttpTransport.newTrustedTransport(),
- JacksonFactory.getDefaultInstance(),
- credential);
+ FeastProperties.JobProperties.Runner runner = jobProperties.getActiveRunner();
+ Map runnerConfigOptions = runner.getOptions();
+ String configJson = gson.toJson(runnerConfigOptions);
+
+ FeastProperties.MetricsProperties metrics = jobProperties.getMetrics();
- return new DataflowJobManager(
- dataflow, jobProperties.getOptions(), jobProperties.getMetrics());
- } catch (IOException e) {
- throw new IllegalStateException(
- "Unable to find credential required for Dataflow monitoring API", e);
- } catch (GeneralSecurityException e) {
- throw new IllegalStateException("Security exception while connecting to Dataflow API", e);
- } catch (Exception e) {
- throw new IllegalStateException("Unable to initialize DataflowJobManager", e);
- }
+ switch (runner.getType()) {
+ case DATAFLOW:
+ DataflowRunnerConfigOptions.Builder dataflowRunnerConfigOptions =
+ DataflowRunnerConfigOptions.newBuilder();
+ JsonFormat.parser().merge(configJson, dataflowRunnerConfigOptions);
+ return new DataflowJobManager(dataflowRunnerConfigOptions.build(), metrics);
case DIRECT:
+ DirectRunnerConfigOptions.Builder directRunnerConfigOptions =
+ DirectRunnerConfigOptions.newBuilder();
+ JsonFormat.parser().merge(configJson, directRunnerConfigOptions);
return new DirectRunnerJobManager(
- jobProperties.getOptions(), directJobRegistry, jobProperties.getMetrics());
+ directRunnerConfigOptions.build(), new DirectJobRegistry(), metrics);
default:
- throw new IllegalArgumentException("Unsupported runner: " + jobProperties.getRunner());
+ throw new IllegalArgumentException("Unsupported runner: " + runner);
}
}
-
- /** Get a direct job registry */
- @Bean
- public DirectJobRegistry directJobRegistry() {
- return new DirectJobRegistry();
- }
-
- /** Extracts job update options from feast core options. */
- @Bean
- public JobUpdatesProperties jobUpdatesProperties(FeastProperties feastProperties) {
- return feastProperties.getJobs().getUpdates();
- }
}
diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java
index 3eba210888..b136650dfd 100644
--- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java
+++ b/core/src/main/java/feast/core/dao/FeatureSetRepository.java
@@ -25,25 +25,16 @@ public interface FeatureSetRepository extends JpaRepository
long count();
- // Find single feature set by project, name, and version
- FeatureSet findFeatureSetByNameAndProject_NameAndVersion(
- String name, String project, Integer version);
+ // Find single feature set by project and name
+ FeatureSet findFeatureSetByNameAndProject_Name(String name, String project);
- // Find single latest version of a feature set by project and name (LIKE)
- FeatureSet findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc(
- String name, String project);
+ // find all feature sets and order by name
+ List findAllByOrderByNameAsc();
- // find all feature sets and order by name and version
- List findAllByOrderByNameAscVersionAsc();
+ // find all feature sets matching the given name pattern with a specific project.
+ List findAllByNameLikeAndProject_NameOrderByNameAsc(String name, String project_name);
- // find all feature sets within a project and order by name and version
- List findAllByProject_NameOrderByNameAscVersionAsc(String project_name);
-
- // find all versions of feature sets matching the given name pattern with a specific project.
- List findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- String name, String project_name);
-
- // find all versions of feature sets matching the given name pattern and project pattern
- List findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc(
+ // find all feature sets matching the given name pattern and project pattern
+ List findAllByNameLikeAndProject_NameLikeOrderByNameAsc(
String name, String project_name);
}
diff --git a/core/src/main/java/feast/core/dao/MetricsRepository.java b/core/src/main/java/feast/core/dao/MetricsRepository.java
deleted file mode 100644
index 7146e1e3ec..0000000000
--- a/core/src/main/java/feast/core/dao/MetricsRepository.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.dao;
-
-import feast.core.model.Metrics;
-import java.util.List;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.stereotype.Repository;
-
-@Repository
-public interface MetricsRepository extends JpaRepository {
- List findByJob_Id(String id);
-}
diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
index 42bc0ba23d..db65328b82 100644
--- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
+++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
@@ -18,37 +18,38 @@
import com.google.api.gax.rpc.InvalidArgumentException;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceGrpc.CoreServiceImplBase;
-import feast.core.CoreServiceProto.ApplyFeatureSetRequest;
-import feast.core.CoreServiceProto.ApplyFeatureSetResponse;
-import feast.core.CoreServiceProto.ArchiveProjectRequest;
-import feast.core.CoreServiceProto.ArchiveProjectResponse;
-import feast.core.CoreServiceProto.CreateProjectRequest;
-import feast.core.CoreServiceProto.CreateProjectResponse;
-import feast.core.CoreServiceProto.GetFeastCoreVersionRequest;
-import feast.core.CoreServiceProto.GetFeastCoreVersionResponse;
-import feast.core.CoreServiceProto.GetFeatureSetRequest;
-import feast.core.CoreServiceProto.GetFeatureSetResponse;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListIngestionJobsRequest;
-import feast.core.CoreServiceProto.ListIngestionJobsResponse;
-import feast.core.CoreServiceProto.ListProjectsRequest;
-import feast.core.CoreServiceProto.ListProjectsResponse;
-import feast.core.CoreServiceProto.ListStoresRequest;
-import feast.core.CoreServiceProto.ListStoresResponse;
-import feast.core.CoreServiceProto.RestartIngestionJobRequest;
-import feast.core.CoreServiceProto.RestartIngestionJobResponse;
-import feast.core.CoreServiceProto.StopIngestionJobRequest;
-import feast.core.CoreServiceProto.StopIngestionJobResponse;
-import feast.core.CoreServiceProto.UpdateStoreRequest;
-import feast.core.CoreServiceProto.UpdateStoreResponse;
+import feast.core.config.FeastProperties;
import feast.core.exception.RetrievalException;
import feast.core.grpc.interceptors.MonitoringInterceptor;
import feast.core.model.Project;
import feast.core.service.AccessManagementService;
import feast.core.service.JobService;
import feast.core.service.SpecService;
+import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetRequest;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ArchiveProjectRequest;
+import feast.proto.core.CoreServiceProto.ArchiveProjectResponse;
+import feast.proto.core.CoreServiceProto.CreateProjectRequest;
+import feast.proto.core.CoreServiceProto.CreateProjectResponse;
+import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest;
+import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse;
+import feast.proto.core.CoreServiceProto.GetFeatureSetRequest;
+import feast.proto.core.CoreServiceProto.GetFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsRequest;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsResponse;
+import feast.proto.core.CoreServiceProto.ListProjectsRequest;
+import feast.proto.core.CoreServiceProto.ListProjectsResponse;
+import feast.proto.core.CoreServiceProto.ListStoresRequest;
+import feast.proto.core.CoreServiceProto.ListStoresResponse;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobResponse;
+import feast.proto.core.CoreServiceProto.StopIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.StopIngestionJobResponse;
+import feast.proto.core.CoreServiceProto.UpdateStoreRequest;
+import feast.proto.core.CoreServiceProto.UpdateStoreResponse;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.StreamObserver;
@@ -64,6 +65,7 @@
@GRpcService(interceptors = {MonitoringInterceptor.class})
public class CoreServiceImpl extends CoreServiceImplBase {
+ private final FeastProperties feastProperties;
private SpecService specService;
private AccessManagementService accessManagementService;
private JobService jobService;
@@ -72,17 +74,28 @@ public class CoreServiceImpl extends CoreServiceImplBase {
public CoreServiceImpl(
SpecService specService,
AccessManagementService accessManagementService,
- JobService jobService) {
+ JobService jobService,
+ FeastProperties feastProperties) {
this.specService = specService;
this.accessManagementService = accessManagementService;
this.jobService = jobService;
+ this.feastProperties = feastProperties;
}
@Override
public void getFeastCoreVersion(
GetFeastCoreVersionRequest request,
StreamObserver responseObserver) {
- super.getFeastCoreVersion(request, responseObserver);
+ try {
+ GetFeastCoreVersionResponse response =
+ GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build();
+ responseObserver.onNext(response);
+ responseObserver.onCompleted();
+ } catch (RetrievalException | StatusRuntimeException e) {
+ log.error("Could not determine Feast Core version: ", e);
+ responseObserver.onError(
+ Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
+ }
}
@Override
@@ -183,6 +196,17 @@ public void archiveProject(
accessManagementService.archiveProject(request.getName());
responseObserver.onNext(ArchiveProjectResponse.getDefaultInstance());
responseObserver.onCompleted();
+ } catch (IllegalArgumentException e) {
+ log.error("Recieved an invalid request on calling archiveProject method:", e);
+ responseObserver.onError(
+ Status.INVALID_ARGUMENT
+ .withDescription(e.getMessage())
+ .withCause(e)
+ .asRuntimeException());
+ } catch (UnsupportedOperationException e) {
+ log.error("Attempted to archive an unsupported project:", e);
+ responseObserver.onError(
+ Status.UNIMPLEMENTED.withDescription(e.getMessage()).withCause(e).asRuntimeException());
} catch (Exception e) {
log.error("Exception has occurred in the createProject method: ", e);
responseObserver.onError(
diff --git a/core/src/main/java/feast/core/grpc/HealthServiceImpl.java b/core/src/main/java/feast/core/grpc/HealthServiceImpl.java
new file mode 100644
index 0000000000..3bd2f8748f
--- /dev/null
+++ b/core/src/main/java/feast/core/grpc/HealthServiceImpl.java
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.grpc;
+
+import feast.core.service.AccessManagementService;
+import io.grpc.Status;
+import io.grpc.health.v1.HealthGrpc.HealthImplBase;
+import io.grpc.health.v1.HealthProto.HealthCheckRequest;
+import io.grpc.health.v1.HealthProto.HealthCheckResponse;
+import io.grpc.health.v1.HealthProto.HealthCheckResponse.ServingStatus;
+import io.grpc.stub.StreamObserver;
+import lombok.extern.slf4j.Slf4j;
+import org.lognet.springboot.grpc.GRpcService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+@Slf4j
+@GRpcService
+public class HealthServiceImpl extends HealthImplBase {
+ private final AccessManagementService accessManagementService;
+
+ @Autowired
+ public HealthServiceImpl(AccessManagementService accessManagementService) {
+ this.accessManagementService = accessManagementService;
+ }
+
+ @Override
+ public void check(
+ HealthCheckRequest request, StreamObserver responseObserver) {
+ try {
+ accessManagementService.listProjects();
+ responseObserver.onNext(
+ HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build());
+ responseObserver.onCompleted();
+ } catch (Exception e) {
+ log.error("Health Check: unable to retrieve projects.\nError: %s", e);
+ responseObserver.onError(
+ Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
+ }
+ }
+}
diff --git a/core/src/main/java/feast/core/http/HealthController.java b/core/src/main/java/feast/core/http/HealthController.java
deleted file mode 100644
index 2451ed793e..0000000000
--- a/core/src/main/java/feast/core/http/HealthController.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.http;
-
-import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import javax.sql.DataSource;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RestController;
-
-/** Web http for pod health-check endpoints. */
-@Slf4j
-@RestController
-public class HealthController {
-
- private final DataSource db;
-
- @Autowired
- public HealthController(DataSource datasource) {
- this.db = datasource;
- }
-
- /**
- * /ping endpoint checks if the application is ready to serve traffic by checking if it is able to
- * access the metadata db.
- */
- @RequestMapping(value = "/ping", method = RequestMethod.GET)
- public ResponseEntity ping() {
- return ResponseEntity.ok("pong");
- }
-
- /**
- * /healthz endpoint checks if the application is healthy by checking if the application still has
- * access to the metadata db.
- */
- @RequestMapping(value = "/healthz", method = RequestMethod.GET)
- public ResponseEntity healthz() {
- try (Connection conn = db.getConnection()) {
- if (conn.isValid(10)) {
- return ResponseEntity.ok("healthy");
- }
- log.error("Unable to reach DB");
- return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body("Unable to establish connection with DB");
- } catch (SQLException e) {
- log.error("Unable to reach DB: {}", e);
- return ResponseEntity.status(INTERNAL_SERVER_ERROR).body(e.getMessage());
- }
- }
-}
diff --git a/core/src/main/java/feast/core/job/JobUpdateTask.java b/core/src/main/java/feast/core/job/JobUpdateTask.java
index f3afe84df7..056da34b3a 100644
--- a/core/src/main/java/feast/core/job/JobUpdateTask.java
+++ b/core/src/main/java/feast/core/job/JobUpdateTask.java
@@ -16,9 +16,7 @@
*/
package feast.core.job;
-import feast.core.FeatureSetProto;
-import feast.core.SourceProto;
-import feast.core.StoreProto;
+import com.google.common.collect.Sets;
import feast.core.log.Action;
import feast.core.log.AuditLogger;
import feast.core.log.Resource;
@@ -27,10 +25,10 @@
import feast.core.model.JobStatus;
import feast.core.model.Source;
import feast.core.model.Store;
+import feast.proto.core.FeatureSetProto.FeatureSetStatus;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -38,7 +36,6 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -52,134 +49,101 @@
@Getter
public class JobUpdateTask implements Callable {
- private final List featureSets;
- private final SourceProto.Source sourceSpec;
- private final StoreProto.Store store;
+ private final List featureSets;
+ private final Source source;
+ private final Store store;
private final Optional currentJob;
- private JobManager jobManager;
- private long jobUpdateTimeoutSeconds;
+ private final JobManager jobManager;
+ private final long jobUpdateTimeoutSeconds;
+ private final String runnerName;
public JobUpdateTask(
- List featureSets,
- SourceProto.Source sourceSpec,
- StoreProto.Store store,
+ List featureSets,
+ Source source,
+ Store store,
Optional currentJob,
JobManager jobManager,
long jobUpdateTimeoutSeconds) {
this.featureSets = featureSets;
- this.sourceSpec = sourceSpec;
+ this.source = source;
this.store = store;
this.currentJob = currentJob;
this.jobManager = jobManager;
this.jobUpdateTimeoutSeconds = jobUpdateTimeoutSeconds;
+ this.runnerName = jobManager.getRunnerType().toString();
}
@Override
public Job call() {
ExecutorService executorService = Executors.newSingleThreadExecutor();
- Source source = Source.fromProto(sourceSpec);
Future submittedJob;
- if (currentJob.isPresent()) {
- Set existingFeatureSetsPopulatedByJob =
- currentJob.get().getFeatureSets().stream()
- .map(FeatureSet::getId)
- .collect(Collectors.toSet());
- Set newFeatureSetsPopulatedByJob =
- featureSets.stream()
- .map(fs -> FeatureSet.fromProto(fs).getId())
- .collect(Collectors.toSet());
- if (existingFeatureSetsPopulatedByJob.size() == newFeatureSetsPopulatedByJob.size()
- && existingFeatureSetsPopulatedByJob.containsAll(newFeatureSetsPopulatedByJob)) {
- Job job = currentJob.get();
- JobStatus newJobStatus = jobManager.getJobStatus(job);
- if (newJobStatus != job.getStatus()) {
- AuditLogger.log(
- Resource.JOB,
- job.getId(),
- Action.STATUS_CHANGE,
- "Job status updated: changed from %s to %s",
- job.getStatus(),
- newJobStatus);
- }
- job.setStatus(newJobStatus);
- return job;
+
+ if (currentJob.isEmpty()) {
+ submittedJob = executorService.submit(this::createJob);
+ } else {
+ Job job = currentJob.get();
+
+ if (requiresUpdate(job)) {
+ submittedJob = executorService.submit(() -> updateJob(job));
} else {
- submittedJob =
- executorService.submit(() -> updateJob(currentJob.get(), featureSets, store));
+ return updateStatus(job);
}
- } else {
- String jobId = createJobId(source.getId(), store.getName());
- submittedJob = executorService.submit(() -> startJob(jobId, featureSets, sourceSpec, store));
}
- Job job = null;
try {
- job = submittedJob.get(getJobUpdateTimeoutSeconds(), TimeUnit.SECONDS);
+ return submittedJob.get(getJobUpdateTimeoutSeconds(), TimeUnit.SECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
log.warn("Unable to start job for source {} and sink {}: {}", source, store, e.getMessage());
+ return null;
+ } finally {
executorService.shutdownNow();
}
- return job;
+ }
+
+ boolean requiresUpdate(Job job) {
+ // If set of feature sets has changed
+ if (!Sets.newHashSet(featureSets).equals(Sets.newHashSet(job.getFeatureSets()))) {
+ return true;
+ }
+ // If any of the incoming feature sets were updated
+ for (FeatureSet featureSet : featureSets) {
+ if (featureSet.getStatus() == FeatureSetStatus.STATUS_PENDING) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Job createJob() {
+ String jobId = createJobId(source.getId(), store.getName());
+ return startJob(jobId);
}
/** Start or update the job to ingest data to the sink. */
- private Job startJob(
- String jobId,
- List featureSetProtos,
- SourceProto.Source source,
- StoreProto.Store sinkSpec) {
-
- List featureSets =
- featureSetProtos.stream()
- .map(
- fsp ->
- FeatureSet.fromProto(
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(fsp.getSpec())
- .setMeta(fsp.getMeta())
- .build()))
- .collect(Collectors.toList());
+ private Job startJob(String jobId) {
+
Job job =
new Job(
- jobId,
- "",
- jobManager.getRunnerType().name(),
- Source.fromProto(source),
- Store.fromProto(sinkSpec),
- featureSets,
- JobStatus.PENDING);
+ jobId, "", jobManager.getRunnerType(), source, store, featureSets, JobStatus.PENDING);
try {
- AuditLogger.log(
- Resource.JOB,
- jobId,
- Action.SUBMIT,
- "Building graph and submitting to %s",
- jobManager.getRunnerType().toString());
+ logAudit(Action.SUBMIT, job, "Building graph and submitting to %s", runnerName);
job = jobManager.startJob(job);
- if (job.getExtId().isEmpty()) {
+ var extId = job.getExtId();
+ if (extId.isEmpty()) {
throw new RuntimeException(
String.format("Could not submit job: \n%s", "unable to retrieve job external id"));
}
- AuditLogger.log(
- Resource.JOB,
- jobId,
- Action.STATUS_CHANGE,
- "Job submitted to runner %s with ext id %s.",
- jobManager.getRunnerType().toString(),
- job.getExtId());
+ var auditMessage = "Job submitted to runner %s with ext id %s.";
+ logAudit(Action.STATUS_CHANGE, job, auditMessage, runnerName, extId);
return job;
} catch (Exception e) {
log.error(e.getMessage());
- AuditLogger.log(
- Resource.JOB,
- jobId,
- Action.STATUS_CHANGE,
- "Job failed to be submitted to runner %s. Job status changed to ERROR.",
- jobManager.getRunnerType().toString());
+ var auditMessage = "Job failed to be submitted to runner %s. Job status changed to ERROR.";
+ logAudit(Action.STATUS_CHANGE, job, auditMessage, runnerName);
job.setStatus(JobStatus.ERROR);
return job;
@@ -187,33 +151,33 @@ private Job startJob(
}
/** Update the given job */
- private Job updateJob(
- Job job, List featureSets, StoreProto.Store store) {
- job.setFeatureSets(
- featureSets.stream()
- .map(
- fs ->
- FeatureSet.fromProto(
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(fs.getSpec())
- .setMeta(fs.getMeta())
- .build()))
- .collect(Collectors.toList()));
- job.setStore(feast.core.model.Store.fromProto(store));
- AuditLogger.log(
- Resource.JOB,
- job.getId(),
- Action.UPDATE,
- "Updating job %s for runner %s",
- job.getId(),
- jobManager.getRunnerType().toString());
+ private Job updateJob(Job job) {
+ job.setFeatureSets(featureSets);
+ job.setStore(store);
+ logAudit(Action.UPDATE, job, "Updating job %s for runner %s", job.getId(), runnerName);
return jobManager.updateJob(job);
}
+ private Job updateStatus(Job job) {
+ JobStatus currentStatus = job.getStatus();
+ JobStatus newStatus = jobManager.getJobStatus(job);
+ if (newStatus != currentStatus) {
+ var auditMessage = "Job status updated: changed from %s to %s";
+ logAudit(Action.STATUS_CHANGE, job, auditMessage, currentStatus, newStatus);
+ }
+
+ job.setStatus(newStatus);
+ return job;
+ }
+
String createJobId(String sourceId, String storeName) {
String dateSuffix = String.valueOf(Instant.now().toEpochMilli());
String sourceIdTrunc = sourceId.split("/")[0].toLowerCase();
String jobId = String.format("%s-to-%s", sourceIdTrunc, storeName) + dateSuffix;
return jobId.replaceAll("_", "-");
}
+
+ private void logAudit(Action action, Job job, String detail, Object... args) {
+ AuditLogger.log(Resource.JOB, job.getId(), action, detail, args);
+ }
}
diff --git a/core/src/main/java/feast/core/job/Runner.java b/core/src/main/java/feast/core/job/Runner.java
index 4e2033fed6..acccb70c8b 100644
--- a/core/src/main/java/feast/core/job/Runner.java
+++ b/core/src/main/java/feast/core/job/Runner.java
@@ -16,33 +16,37 @@
*/
package feast.core.job;
+import java.util.NoSuchElementException;
+
+/**
+ * An Apache Beam Runner, for which Feast Core supports managing ingestion jobs.
+ *
+ * @see Beam Runners
+ */
public enum Runner {
DATAFLOW("DataflowRunner"),
FLINK("FlinkRunner"),
DIRECT("DirectRunner");
- private final String name;
+ private final String humanName;
- Runner(String name) {
- this.name = name;
+ Runner(String humanName) {
+ this.humanName = humanName;
}
- /**
- * Get the human readable name of this runner. Returns a human readable name of the runner that
- * can be used for logging/config files/etc.
- */
+ /** Returns the human readable name of this runner, usable in logging, config files, etc. */
@Override
public String toString() {
- return name;
+ return humanName;
}
/** Parses a runner from its human readable name. */
- public static Runner fromString(String runner) {
+ public static Runner fromString(String humanName) {
for (Runner r : Runner.values()) {
- if (r.toString().equals(runner)) {
+ if (r.toString().equals(humanName)) {
return r;
}
}
- throw new IllegalArgumentException("Unknown value: " + runner);
+ throw new NoSuchElementException("Unknown Runner value: " + humanName);
}
}
diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobConfig.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobConfig.java
deleted file mode 100644
index a9bbf345d1..0000000000
--- a/core/src/main/java/feast/core/job/dataflow/DataflowJobConfig.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.job.dataflow;
-
-import lombok.Value;
-
-@Value
-public class DataflowJobConfig {
- private String projectId;
- private String location;
-}
diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
index c2313d75ec..2c3da255f5 100644
--- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
+++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java
@@ -18,30 +18,34 @@
import static feast.core.util.PipelineUtil.detectClassPathResourcesToStage;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.DataflowScopes;
import com.google.common.base.Strings;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
-import feast.core.FeatureSetProto;
-import feast.core.SourceProto;
-import feast.core.StoreProto;
import feast.core.config.FeastProperties.MetricsProperties;
import feast.core.exception.JobExecutionException;
import feast.core.job.JobManager;
import feast.core.job.Runner;
import feast.core.job.option.FeatureSetJsonByteConverter;
import feast.core.model.*;
-import feast.core.util.TypeConversion;
import feast.ingestion.ImportJob;
import feast.ingestion.options.BZip2Compressor;
import feast.ingestion.options.ImportOptions;
import feast.ingestion.options.OptionCompressor;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions;
+import feast.proto.core.SourceProto;
+import feast.proto.core.StoreProto;
import java.io.IOException;
+import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.beam.runners.dataflow.DataflowPipelineJob;
import org.apache.beam.runners.dataflow.DataflowRunner;
@@ -56,16 +60,48 @@ public class DataflowJobManager implements JobManager {
private final String projectId;
private final String location;
private final Dataflow dataflow;
- private final Map defaultOptions;
+ private final DataflowRunnerConfig defaultOptions;
private final MetricsProperties metrics;
public DataflowJobManager(
- Dataflow dataflow, Map defaultOptions, MetricsProperties metricsProperties) {
- this.defaultOptions = defaultOptions;
+ DataflowRunnerConfigOptions runnerConfigOptions, MetricsProperties metricsProperties) {
+ this(runnerConfigOptions, metricsProperties, getGoogleCredential());
+ }
+
+ public DataflowJobManager(
+ DataflowRunnerConfigOptions runnerConfigOptions,
+ MetricsProperties metricsProperties,
+ Credential credential) {
+
+ defaultOptions = new DataflowRunnerConfig(runnerConfigOptions);
+ Dataflow dataflow = null;
+ try {
+ dataflow =
+ new Dataflow(
+ GoogleNetHttpTransport.newTrustedTransport(),
+ JacksonFactory.getDefaultInstance(),
+ credential);
+ } catch (GeneralSecurityException e) {
+ throw new IllegalStateException("Security exception while connecting to Dataflow API", e);
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to initialize DataflowJobManager", e);
+ }
+
this.dataflow = dataflow;
this.metrics = metricsProperties;
- this.projectId = defaultOptions.get("project");
- this.location = defaultOptions.get("region");
+ this.projectId = defaultOptions.getProject();
+ this.location = defaultOptions.getRegion();
+ }
+
+ private static Credential getGoogleCredential() {
+ GoogleCredential credential = null;
+ try {
+ credential = GoogleCredential.getApplicationDefault().createScoped(DataflowScopes.all());
+ } catch (IOException e) {
+ throw new IllegalStateException(
+ "Unable to find credential required for Dataflow monitoring API", e);
+ }
+ return credential;
}
@Override
@@ -80,12 +116,15 @@ public Job startJob(Job job) {
for (FeatureSet featureSet : job.getFeatureSets()) {
featureSetProtos.add(featureSet.toProto());
}
- return submitDataflowJob(
- job.getId(),
- featureSetProtos,
- job.getSource().toProto(),
- job.getStore().toProto(),
- false);
+ String extId =
+ submitDataflowJob(
+ job.getId(),
+ featureSetProtos,
+ job.getSource().toProto(),
+ job.getStore().toProto(),
+ false);
+ job.setExtId(extId);
+ return job;
} catch (InvalidProtocolBufferException e) {
log.error(e.getMessage());
@@ -110,8 +149,18 @@ public Job updateJob(Job job) {
for (FeatureSet featureSet : job.getFeatureSets()) {
featureSetProtos.add(featureSet.toProto());
}
- return submitDataflowJob(
- job.getId(), featureSetProtos, job.getSource().toProto(), job.getStore().toProto(), true);
+
+ String extId =
+ submitDataflowJob(
+ job.getId(),
+ featureSetProtos,
+ job.getSource().toProto(),
+ job.getStore().toProto(),
+ true);
+
+ job.setExtId(extId);
+ job.setStatus(JobStatus.PENDING);
+ return job;
} catch (InvalidProtocolBufferException e) {
log.error(e.getMessage());
throw new IllegalArgumentException(
@@ -153,16 +202,15 @@ public void abortJob(String dataflowJobId) {
}
/**
- * Restart a restart dataflow job. Dataflow should ensure continuity between during the restart,
- * so no data should be lost during the restart operation.
+ * Restart a Dataflow job. Dataflow should ensure continuity such that no data should be lost
+ * during the restart operation.
*
* @param job job to restart
* @return the restarted job
*/
@Override
public Job restartJob(Job job) {
- JobStatus status = job.getStatus();
- if (JobStatus.getTerminalState().contains(status)) {
+ if (job.getStatus().isTerminal()) {
// job yet not running: just start job
return this.startJob(job);
} else {
@@ -180,7 +228,7 @@ public Job restartJob(Job job) {
*/
@Override
public JobStatus getJobStatus(Job job) {
- if (!Runner.DATAFLOW.name().equals(job.getRunner())) {
+ if (job.getRunner() != RUNNER_TYPE) {
return job.getStatus();
}
@@ -197,7 +245,7 @@ public JobStatus getJobStatus(Job job) {
return JobStatus.UNKNOWN;
}
- private Job submitDataflowJob(
+ private String submitDataflowJob(
String jobName,
List featureSetProtos,
SourceProto.Source source,
@@ -206,17 +254,8 @@ private Job submitDataflowJob(
try {
ImportOptions pipelineOptions = getPipelineOptions(jobName, featureSetProtos, sink, update);
DataflowPipelineJob pipelineResult = runPipeline(pipelineOptions);
- List featureSets =
- featureSetProtos.stream().map(FeatureSet::fromProto).collect(Collectors.toList());
String jobId = waitForJobToRun(pipelineResult);
- return new Job(
- jobName,
- jobId,
- getRunnerType().name(),
- Source.fromProto(source),
- Store.fromProto(sink),
- featureSets,
- JobStatus.PENDING);
+ return jobId;
} catch (Exception e) {
log.error("Error submitting job", e);
throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e);
@@ -228,9 +267,9 @@ private ImportOptions getPipelineOptions(
List featureSets,
StoreProto.Store sink,
boolean update)
- throws IOException {
- String[] args = TypeConversion.convertMapToArgs(defaultOptions);
- ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class);
+ throws IOException, IllegalAccessException {
+ ImportOptions pipelineOptions =
+ PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class);
OptionCompressor> featureSetJsonCompressor =
new BZip2Compressor<>(new FeatureSetJsonByteConverter());
@@ -238,6 +277,7 @@ private ImportOptions getPipelineOptions(
pipelineOptions.setFeatureSetJson(featureSetJsonCompressor.compress(featureSets));
pipelineOptions.setStoreJson(Collections.singletonList(JsonFormat.printer().print(sink)));
pipelineOptions.setProject(projectId);
+ pipelineOptions.setDefaultFeastProject(Project.DEFAULT_NAME);
pipelineOptions.setUpdate(update);
pipelineOptions.setRunner(DataflowRunner.class);
pipelineOptions.setJobName(jobName);
diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java b/core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java
new file mode 100644
index 0000000000..85628d2cd0
--- /dev/null
+++ b/core/src/main/java/feast/core/job/dataflow/DataflowRunnerConfig.java
@@ -0,0 +1,101 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.job.dataflow;
+
+import feast.core.job.option.RunnerConfig;
+import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions;
+import java.util.*;
+import javax.validation.*;
+import javax.validation.constraints.NotBlank;
+import lombok.Getter;
+import lombok.Setter;
+
+/** DataflowRunnerConfig contains configuration fields for the Dataflow job runner. */
+@Getter
+@Setter
+public class DataflowRunnerConfig extends RunnerConfig {
+
+ public DataflowRunnerConfig(DataflowRunnerConfigOptions runnerConfigOptions) {
+ this.project = runnerConfigOptions.getProject();
+ this.region = runnerConfigOptions.getRegion();
+ this.zone = runnerConfigOptions.getZone();
+ this.serviceAccount = runnerConfigOptions.getServiceAccount();
+ this.network = runnerConfigOptions.getNetwork();
+ this.subnetwork = runnerConfigOptions.getSubnetwork();
+ this.workerMachineType = runnerConfigOptions.getWorkerMachineType();
+ this.autoscalingAlgorithm = runnerConfigOptions.getAutoscalingAlgorithm();
+ this.usePublicIps = runnerConfigOptions.getUsePublicIps();
+ this.tempLocation = runnerConfigOptions.getTempLocation();
+ this.maxNumWorkers = runnerConfigOptions.getMaxNumWorkers();
+ this.deadLetterTableSpec = runnerConfigOptions.getDeadLetterTableSpec();
+ this.labels = runnerConfigOptions.getLabelsMap();
+ validate();
+ }
+
+ /* Project id to use when launching jobs. */
+ @NotBlank String project;
+
+ /* The Google Compute Engine region for creating Dataflow jobs. */
+ @NotBlank String region;
+
+ /* GCP availability zone for operations. */
+ @NotBlank String zone;
+
+ /* Run the job as a specific service account, instead of the default GCE robot. */
+ String serviceAccount;
+
+ /* GCE network for launching workers. */
+ @NotBlank String network;
+
+ /* GCE subnetwork for launching workers. */
+ @NotBlank String subnetwork;
+
+ /* Machine type to create Dataflow worker VMs as. */
+ String workerMachineType;
+
+ /* The autoscaling algorithm to use for the workerpool. */
+ String autoscalingAlgorithm;
+
+ /* Specifies whether worker pools should be started with public IP addresses. */
+ Boolean usePublicIps;
+
+ /**
+ * A pipeline level default location for storing temporary files. Support Google Cloud Storage
+ * locations, e.g. gs://bucket/object
+ */
+ @NotBlank String tempLocation;
+
+ /* The maximum number of workers to use for the workerpool. */
+ Integer maxNumWorkers;
+
+ /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */
+ String deadLetterTableSpec;
+
+ Map labels;
+
+ /** Validates Dataflow runner configuration options */
+ public void validate() {
+ ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
+ Validator validator = factory.getValidator();
+
+ Set> dataflowRunnerConfigViolation =
+ validator.validate(this);
+ if (!dataflowRunnerConfigViolation.isEmpty()) {
+ throw new ConstraintViolationException(dataflowRunnerConfigViolation);
+ }
+ }
+}
diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java b/core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java
new file mode 100644
index 0000000000..ebd327f2f7
--- /dev/null
+++ b/core/src/main/java/feast/core/job/direct/DirectRunnerConfig.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.job.direct;
+
+import feast.core.job.option.RunnerConfig;
+import feast.proto.core.RunnerProto.DirectRunnerConfigOptions;
+
+public class DirectRunnerConfig extends RunnerConfig {
+ /**
+ * Controls the amount of target parallelism the DirectRunner will use. Defaults to the greater of
+ * the number of available processors and 3. Must be a value greater than zero.
+ */
+ Integer targetParallelism;
+
+ /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */
+ String deadletterTableSpec;
+
+ public DirectRunnerConfig(DirectRunnerConfigOptions runnerConfigOptions) {
+ this.deadletterTableSpec = runnerConfigOptions.getDeadLetterTableSpec();
+ this.targetParallelism = runnerConfigOptions.getTargetParallelism();
+ }
+}
diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
index 9b3a8473e4..715adbdd43 100644
--- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
+++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java
@@ -18,8 +18,6 @@
import com.google.common.base.Strings;
import com.google.protobuf.util.JsonFormat;
-import feast.core.FeatureSetProto;
-import feast.core.StoreProto;
import feast.core.config.FeastProperties.MetricsProperties;
import feast.core.exception.JobExecutionException;
import feast.core.job.JobManager;
@@ -28,16 +26,18 @@
import feast.core.model.FeatureSet;
import feast.core.model.Job;
import feast.core.model.JobStatus;
-import feast.core.util.TypeConversion;
+import feast.core.model.Project;
import feast.ingestion.ImportJob;
import feast.ingestion.options.BZip2Compressor;
import feast.ingestion.options.ImportOptions;
import feast.ingestion.options.OptionCompressor;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.RunnerProto.DirectRunnerConfigOptions;
+import feast.proto.core.StoreProto;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.beam.runners.direct.DirectRunner;
import org.apache.beam.sdk.PipelineResult;
@@ -48,15 +48,15 @@ public class DirectRunnerJobManager implements JobManager {
private final Runner RUNNER_TYPE = Runner.DIRECT;
- protected Map defaultOptions;
+ private DirectRunnerConfig defaultOptions;
private final DirectJobRegistry jobs;
private MetricsProperties metrics;
public DirectRunnerJobManager(
- Map defaultOptions,
+ DirectRunnerConfigOptions directRunnerConfigOptions,
DirectJobRegistry jobs,
MetricsProperties metricsProperties) {
- this.defaultOptions = defaultOptions;
+ this.defaultOptions = new DirectRunnerConfig(directRunnerConfigOptions);
this.jobs = jobs;
this.metrics = metricsProperties;
}
@@ -79,7 +79,7 @@ public Job startJob(Job job) {
featureSetProtos.add(featureSet.toProto());
}
ImportOptions pipelineOptions =
- getPipelineOptions(featureSetProtos, job.getStore().toProto());
+ getPipelineOptions(job.getId(), featureSetProtos, job.getStore().toProto());
PipelineResult pipelineResult = runPipeline(pipelineOptions);
DirectJob directJob = new DirectJob(job.getId(), pipelineResult);
jobs.add(directJob);
@@ -93,16 +93,19 @@ public Job startJob(Job job) {
}
private ImportOptions getPipelineOptions(
- List featureSets, StoreProto.Store sink) throws IOException {
- String[] args = TypeConversion.convertMapToArgs(defaultOptions);
- ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class);
+ String jobName, List featureSets, StoreProto.Store sink)
+ throws IOException, IllegalAccessException {
+ ImportOptions pipelineOptions =
+ PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class);
OptionCompressor> featureSetJsonCompressor =
new BZip2Compressor<>(new FeatureSetJsonByteConverter());
pipelineOptions.setFeatureSetJson(featureSetJsonCompressor.compress(featureSets));
+ pipelineOptions.setJobName(jobName);
pipelineOptions.setStoreJson(Collections.singletonList(JsonFormat.printer().print(sink)));
pipelineOptions.setRunner(DirectRunner.class);
+ pipelineOptions.setDefaultFeastProject(Project.DEFAULT_NAME);
pipelineOptions.setProject(""); // set to default value to satisfy validation
if (metrics.isEnabled()) {
pipelineOptions.setMetricsExporterType(metrics.getType());
@@ -166,8 +169,7 @@ public PipelineResult runPipeline(ImportOptions pipelineOptions) throws IOExcept
*/
@Override
public Job restartJob(Job job) {
- JobStatus status = job.getStatus();
- if (JobStatus.getTerminalState().contains(status)) {
+ if (job.getStatus().isTerminal()) {
// job yet not running: just start job
return this.startJob(job);
} else {
diff --git a/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java b/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java
index dbd04d668f..2f6b37df1b 100644
--- a/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java
+++ b/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java
@@ -18,8 +18,8 @@
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
-import feast.core.FeatureSetProto;
import feast.ingestion.options.OptionByteConverter;
+import feast.proto.core.FeatureSetProto;
import java.util.ArrayList;
import java.util.List;
diff --git a/core/src/main/java/feast/core/job/option/RunnerConfig.java b/core/src/main/java/feast/core/job/option/RunnerConfig.java
new file mode 100644
index 0000000000..4b937074a3
--- /dev/null
+++ b/core/src/main/java/feast/core/job/option/RunnerConfig.java
@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.job.option;
+
+import feast.core.util.TypeConversion;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Value class containing the application-default configuration for a runner. When a job is started
+ * by core, all fields in the object will be converted into --key=value args to seed the beam
+ * pipeline options.
+ */
+public abstract class RunnerConfig {
+
+ /**
+ * Converts the fields in this class to a list of --key=value args to be passed to a {@link
+ * org.apache.beam.sdk.options.PipelineOptionsFactory}.
+ *
+ *
Ignores values that are proto-default (e.g. empty string, 0).
+ *
+ * @return Array of string args in the format --key=value.
+ * @throws IllegalAccessException
+ */
+ public String[] toArgs() throws IllegalAccessException {
+ List args = new ArrayList<>();
+ for (Field field : this.getClass().getFields()) {
+ if (field.get(this) == null) {
+ continue;
+ }
+ Class type = field.getType();
+ if (Map.class.equals(type)) {
+ String jsonString =
+ TypeConversion.convertMapToJsonString((Map) field.get(this));
+ args.add(String.format("--%s=%s", field.getName(), jsonString));
+ continue;
+ }
+
+ if (String.class.equals(type)) {
+ String val = (String) field.get(this);
+ if (!val.equals("")) {
+ args.add(String.format("--%s=%s", field.getName(), val));
+ }
+ continue;
+ }
+
+ if (Integer.class.equals(type)) {
+ Integer val = (Integer) field.get(this);
+ if (val != 0) {
+ args.add(String.format("--%s=%d", field.getName(), val));
+ }
+ continue;
+ }
+
+ args.add(String.format("--%s=%s", field.getName(), field.get(this)));
+ }
+ return args.toArray(String[]::new);
+ }
+}
diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java
new file mode 100644
index 0000000000..6133d492fc
--- /dev/null
+++ b/core/src/main/java/feast/core/model/Entity.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.model;
+
+import feast.proto.core.FeatureSetProto.EntitySpec;
+import feast.proto.types.ValueProto.ValueType;
+import java.util.Objects;
+import javax.persistence.*;
+import lombok.Getter;
+import lombok.Setter;
+
+/** Feast entity object. Contains name and type of the entity. */
+@Getter
+@Setter
+@javax.persistence.Entity
+@Table(
+ name = "entities",
+ uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_set_id"}))
+public class Entity {
+
+ @Id @GeneratedValue private Long id;
+
+ private String name;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ private FeatureSet featureSet;
+
+ /** Data type of the entity. String representation of {@link ValueType} * */
+ private String type;
+
+ public Entity() {}
+
+ private Entity(String name, ValueType.Enum type) {
+ this.setName(name);
+ this.setType(type.toString());
+ }
+
+ public static Entity fromProto(EntitySpec entitySpec) {
+ Entity entity = new Entity(entitySpec.getName(), entitySpec.getValueType());
+ return entity;
+ }
+
+ public EntitySpec toProto() {
+ return EntitySpec.newBuilder().setName(name).setValueType(ValueType.Enum.valueOf(type)).build();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Entity entity = (Entity) o;
+ return getName().equals(entity.getName()) && getType().equals(entity.getType());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), getName(), getType());
+ }
+}
diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java
new file mode 100644
index 0000000000..0b45749379
--- /dev/null
+++ b/core/src/main/java/feast/core/model/Feature.java
@@ -0,0 +1,280 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.model;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import feast.core.util.TypeConversion;
+import feast.proto.core.FeatureSetProto.FeatureSpec;
+import feast.proto.core.FeatureSetProto.FeatureSpec.Builder;
+import feast.proto.types.ValueProto.ValueType;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+import javax.persistence.*;
+import javax.persistence.Entity;
+import lombok.Getter;
+import lombok.Setter;
+import org.tensorflow.metadata.v0.*;
+
+/**
+ * Feature belonging to a featureset. Contains name, type as well as domain metadata about the
+ * feature.
+ */
+@Getter
+@Setter
+@Entity
+@Table(
+ name = "features",
+ uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_set_id"}))
+public class Feature {
+
+ @Id @GeneratedValue private Long id;
+
+ private String name;
+
+ @ManyToOne(fetch = FetchType.LAZY)
+ private FeatureSet featureSet;
+
+ /** Data type of the feature. String representation of {@link ValueType} * */
+ private String type;
+
+ // Labels for this feature
+ @Column(name = "labels", columnDefinition = "text")
+ private String labels;
+
+ // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec)
+ // Only one of them can be set.
+ private byte[] presence;
+ private byte[] groupPresence;
+
+ // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec)
+ // Only one of them can be set.
+ private byte[] shape;
+ private byte[] valueCount;
+
+ // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec)
+ // Only one of them can be set.
+ private String domain;
+ private byte[] intDomain;
+ private byte[] floatDomain;
+ private byte[] stringDomain;
+ private byte[] boolDomain;
+ private byte[] structDomain;
+ private byte[] naturalLanguageDomain;
+ private byte[] imageDomain;
+ private byte[] midDomain;
+ private byte[] urlDomain;
+ private byte[] timeDomain;
+ private byte[] timeOfDayDomain;
+
+ public Feature() {}
+ // Whether this feature has been archived. A archived feature cannot be
+ // retrieved from or written to.
+ private boolean archived = false;
+
+ private Feature(String name, ValueType.Enum type) {
+ this.setName(name);
+ this.setType(type.toString());
+ }
+
+ public static Feature fromProto(FeatureSpec featureSpec) {
+ Feature feature = new Feature(featureSpec.getName(), featureSpec.getValueType());
+ feature.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap());
+ feature.updateSchema(featureSpec);
+ return feature;
+ }
+
+ public FeatureSpec toProto() throws InvalidProtocolBufferException {
+ Builder featureSpecBuilder =
+ FeatureSpec.newBuilder().setName(getName()).setValueType(ValueType.Enum.valueOf(getType()));
+
+ if (getPresence() != null) {
+ featureSpecBuilder.setPresence(FeaturePresence.parseFrom(getPresence()));
+ } else if (getGroupPresence() != null) {
+ featureSpecBuilder.setGroupPresence(FeaturePresenceWithinGroup.parseFrom(getGroupPresence()));
+ }
+
+ if (getShape() != null) {
+ featureSpecBuilder.setShape(FixedShape.parseFrom(getShape()));
+ } else if (getValueCount() != null) {
+ featureSpecBuilder.setValueCount(ValueCount.parseFrom(getValueCount()));
+ }
+
+ if (getDomain() != null) {
+ featureSpecBuilder.setDomain(getDomain());
+ } else if (getIntDomain() != null) {
+ featureSpecBuilder.setIntDomain(IntDomain.parseFrom(getIntDomain()));
+ } else if (getFloatDomain() != null) {
+ featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(getFloatDomain()));
+ } else if (getStringDomain() != null) {
+ featureSpecBuilder.setStringDomain(StringDomain.parseFrom(getStringDomain()));
+ } else if (getBoolDomain() != null) {
+ featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(getBoolDomain()));
+ } else if (getStructDomain() != null) {
+ featureSpecBuilder.setStructDomain(StructDomain.parseFrom(getStructDomain()));
+ } else if (getNaturalLanguageDomain() != null) {
+ featureSpecBuilder.setNaturalLanguageDomain(
+ NaturalLanguageDomain.parseFrom(getNaturalLanguageDomain()));
+ } else if (getImageDomain() != null) {
+ featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(getImageDomain()));
+ } else if (getMidDomain() != null) {
+ featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(getMidDomain()));
+ } else if (getUrlDomain() != null) {
+ featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(getUrlDomain()));
+ } else if (getTimeDomain() != null) {
+ featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(getTimeDomain()));
+ } else if (getTimeOfDayDomain() != null) {
+ featureSpecBuilder.setTimeOfDayDomain(TimeOfDayDomain.parseFrom(getTimeOfDayDomain()));
+ }
+
+ if (getLabels() != null) {
+ featureSpecBuilder.putAllLabels(getLabels());
+ }
+ return featureSpecBuilder.build();
+ }
+
+ private void updateSchema(FeatureSpec featureSpec) {
+ switch (featureSpec.getPresenceConstraintsCase()) {
+ case PRESENCE:
+ setPresence(featureSpec.getPresence().toByteArray());
+ break;
+ case GROUP_PRESENCE:
+ setGroupPresence(featureSpec.getGroupPresence().toByteArray());
+ break;
+ case PRESENCECONSTRAINTS_NOT_SET:
+ break;
+ }
+
+ switch (featureSpec.getShapeTypeCase()) {
+ case SHAPE:
+ setShape(featureSpec.getShape().toByteArray());
+ break;
+ case VALUE_COUNT:
+ setValueCount(featureSpec.getValueCount().toByteArray());
+ break;
+ case SHAPETYPE_NOT_SET:
+ break;
+ }
+
+ switch (featureSpec.getDomainInfoCase()) {
+ case DOMAIN:
+ setDomain(featureSpec.getDomain());
+ break;
+ case INT_DOMAIN:
+ setIntDomain(featureSpec.getIntDomain().toByteArray());
+ break;
+ case FLOAT_DOMAIN:
+ setFloatDomain(featureSpec.getFloatDomain().toByteArray());
+ break;
+ case STRING_DOMAIN:
+ setStringDomain(featureSpec.getStringDomain().toByteArray());
+ break;
+ case BOOL_DOMAIN:
+ setBoolDomain(featureSpec.getBoolDomain().toByteArray());
+ break;
+ case STRUCT_DOMAIN:
+ setStructDomain(featureSpec.getStructDomain().toByteArray());
+ break;
+ case NATURAL_LANGUAGE_DOMAIN:
+ setNaturalLanguageDomain(featureSpec.getNaturalLanguageDomain().toByteArray());
+ break;
+ case IMAGE_DOMAIN:
+ setImageDomain(featureSpec.getImageDomain().toByteArray());
+ break;
+ case MID_DOMAIN:
+ setMidDomain(featureSpec.getMidDomain().toByteArray());
+ break;
+ case URL_DOMAIN:
+ setUrlDomain(featureSpec.getUrlDomain().toByteArray());
+ break;
+ case TIME_DOMAIN:
+ setTimeDomain(featureSpec.getTimeDomain().toByteArray());
+ break;
+ case TIME_OF_DAY_DOMAIN:
+ setTimeOfDayDomain(featureSpec.getTimeOfDayDomain().toByteArray());
+ break;
+ case DOMAININFO_NOT_SET:
+ break;
+ }
+ }
+
+ /** Archive this feature. */
+ public void archive() {
+ this.archived = true;
+ }
+
+ /**
+ * Update the feature object with a valid feature spec. Only schema changes are allowed.
+ *
+ * @param featureSpec {@link FeatureSpec} containing schema changes.
+ */
+ public void updateFromProto(FeatureSpec featureSpec) {
+ if (isArchived()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "You are attempting to create a feature %s that was previously archived. This isn't allowed. Please create a new feature with a different name.",
+ featureSpec.getName()));
+ }
+ if (ValueType.Enum.valueOf(type) != featureSpec.getValueType()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "You are attempting to change the type of feature %s from %s to %s. This isn't allowed. Please create a new feature.",
+ featureSpec.getName(), type, featureSpec.getValueType()));
+ }
+ updateSchema(featureSpec);
+ }
+
+ public Map getLabels() {
+ return TypeConversion.convertJsonStringToMap(this.labels);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Feature feature = (Feature) o;
+ return getName().equals(feature.getName())
+ && getType().equals(feature.getType())
+ && isArchived() == (feature.isArchived())
+ && Objects.equals(getLabels(), feature.getLabels())
+ && Arrays.equals(getPresence(), feature.getPresence())
+ && Arrays.equals(getGroupPresence(), feature.getGroupPresence())
+ && Arrays.equals(getShape(), feature.getShape())
+ && Arrays.equals(getValueCount(), feature.getValueCount())
+ && Objects.equals(getDomain(), feature.getDomain())
+ && Arrays.equals(getIntDomain(), feature.getIntDomain())
+ && Arrays.equals(getFloatDomain(), feature.getFloatDomain())
+ && Arrays.equals(getStringDomain(), feature.getStringDomain())
+ && Arrays.equals(getBoolDomain(), feature.getBoolDomain())
+ && Arrays.equals(getStructDomain(), feature.getStructDomain())
+ && Arrays.equals(getNaturalLanguageDomain(), feature.getNaturalLanguageDomain())
+ && Arrays.equals(getImageDomain(), feature.getImageDomain())
+ && Arrays.equals(getMidDomain(), feature.getMidDomain())
+ && Arrays.equals(getUrlDomain(), feature.getUrlDomain())
+ && Arrays.equals(getTimeDomain(), feature.getTimeDomain())
+ && Arrays.equals(getTimeDomain(), feature.getTimeOfDayDomain());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), getName(), getType(), getLabels());
+ }
+}
diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java
index 232a5f67d1..f7b2dc7cd4 100644
--- a/core/src/main/java/feast/core/model/FeatureSet.java
+++ b/core/src/main/java/feast/core/model/FeatureSet.java
@@ -16,73 +16,35 @@
*/
package feast.core.model;
+import com.google.common.collect.Sets;
import com.google.protobuf.Duration;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Timestamp;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.EntitySpec;
-import feast.core.FeatureSetProto.FeatureSetMeta;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.FeatureSetProto.FeatureSetStatus;
-import feast.core.FeatureSetProto.FeatureSpec;
-import feast.types.ValueProto.ValueType.Enum;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.persistence.CascadeType;
-import javax.persistence.CollectionTable;
-import javax.persistence.Column;
-import javax.persistence.ElementCollection;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-import javax.persistence.UniqueConstraint;
+import feast.core.util.TypeConversion;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.*;
+import java.util.*;
+import java.util.stream.Collectors;
+import javax.persistence.*;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.hibernate.annotations.Fetch;
-import org.hibernate.annotations.FetchMode;
-import org.tensorflow.metadata.v0.BoolDomain;
-import org.tensorflow.metadata.v0.FeaturePresence;
-import org.tensorflow.metadata.v0.FeaturePresenceWithinGroup;
-import org.tensorflow.metadata.v0.FixedShape;
-import org.tensorflow.metadata.v0.FloatDomain;
-import org.tensorflow.metadata.v0.ImageDomain;
-import org.tensorflow.metadata.v0.IntDomain;
-import org.tensorflow.metadata.v0.MIDDomain;
-import org.tensorflow.metadata.v0.NaturalLanguageDomain;
-import org.tensorflow.metadata.v0.StringDomain;
-import org.tensorflow.metadata.v0.StructDomain;
-import org.tensorflow.metadata.v0.TimeDomain;
-import org.tensorflow.metadata.v0.TimeOfDayDomain;
-import org.tensorflow.metadata.v0.URLDomain;
-import org.tensorflow.metadata.v0.ValueCount;
+import org.tensorflow.metadata.v0.*;
@Getter
@Setter
-@Entity
-@Table(name = "feature_sets")
-public class FeatureSet extends AbstractTimestampEntity implements Comparable {
+@javax.persistence.Entity
+@Table(
+ name = "feature_sets",
+ uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"}))
+public class FeatureSet extends AbstractTimestampEntity {
- // Id of the featureSet, defined as project/feature_set_name:feature_set_version
- @Id
- @Column(name = "id", nullable = false, unique = true)
- private String id;
+ @Id @GeneratedValue private long id;
// Name of the featureSet
@Column(name = "name", nullable = false)
private String name;
- // Version of the featureSet
- @Column(name = "version")
- private int version;
-
// Project that this featureSet belongs to
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "project_name")
@@ -93,19 +55,20 @@ public class FeatureSet extends AbstractTimestampEntity implements Comparable entities;
+ @OneToMany(
+ mappedBy = "featureSet",
+ cascade = CascadeType.ALL,
+ fetch = FetchType.EAGER,
+ orphanRemoval = true)
+ private Set entities;
// Feature fields inside this feature set
- @ElementCollection(fetch = FetchType.EAGER)
- @CollectionTable(
- name = "features",
- joinColumns = @JoinColumn(name = "feature_set_id"),
- uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project", "version"}))
- @Fetch(FetchMode.SUBSELECT)
- private Set features;
+ @OneToMany(
+ mappedBy = "featureSet",
+ cascade = CascadeType.ALL,
+ fetch = FetchType.EAGER,
+ orphanRemoval = true)
+ private Set features;
// Source on which feature rows can be found
@ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.EAGER)
@@ -113,8 +76,13 @@ public class FeatureSet extends AbstractTimestampEntity implements Comparable entities,
- List features,
+ List entities,
+ List features,
Source source,
+ Map labels,
FeatureSetStatus status) {
this.maxAgeSeconds = maxAgeSeconds;
this.source = source;
- this.status = status.toString();
+ this.status = status;
this.entities = new HashSet<>();
this.features = new HashSet<>();
this.name = name;
this.project = new Project(project);
- this.version = version;
- this.setId(project, name, version);
+ this.labels = TypeConversion.convertMapToJsonString(labels);
addEntities(entities);
addFeatures(features);
}
- private void setId(String project, String name, int version) {
- this.id = project + "/" + name + ":" + version;
- }
-
- public void setVersion(int version) {
- this.version = version;
- this.setId(getProjectName(), getName(), version);
- }
-
public void setName(String name) {
this.name = name;
- this.setId(getProjectName(), name, getVersion());
}
private String getProjectName() {
@@ -166,202 +123,168 @@ private String getProjectName() {
public void setProject(Project project) {
this.project = project;
- this.setId(project.getName(), getName(), getVersion());
}
public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) {
FeatureSetSpec featureSetSpec = featureSetProto.getSpec();
Source source = Source.fromProto(featureSetSpec.getSource());
- List featureSpecs = new ArrayList<>();
+ List featureSpecs = new ArrayList<>();
for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) {
- featureSpecs.add(new Field(featureSpec));
+ featureSpecs.add(Feature.fromProto(featureSpec));
}
- List entitySpecs = new ArrayList<>();
+ List entitySpecs = new ArrayList<>();
for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) {
- entitySpecs.add(new Field(entitySpec));
+ entitySpecs.add(Entity.fromProto(entitySpec));
}
return new FeatureSet(
featureSetProto.getSpec().getName(),
featureSetProto.getSpec().getProject(),
- featureSetProto.getSpec().getVersion(),
featureSetSpec.getMaxAge().getSeconds(),
entitySpecs,
featureSpecs,
source,
+ featureSetProto.getSpec().getLabelsMap(),
featureSetProto.getMeta().getStatus());
}
- public void addEntities(List fields) {
- for (Field field : fields) {
- addEntity(field);
+ // Updates the existing feature set from a proto.
+ public void updateFromProto(FeatureSetProto.FeatureSet featureSetProto)
+ throws InvalidProtocolBufferException {
+ FeatureSetSpec spec = featureSetProto.getSpec();
+ if (this.toProto().getSpec().equals(spec)) {
+ return;
+ }
+
+ // 1. validate
+ // 1a. check no change to identifiers
+ if (!name.equals(spec.getName())) {
+ throw new IllegalArgumentException(
+ String.format("Given feature set name %s does not match name %s.", spec.getName(), name));
+ }
+ if (!project.getName().equals(spec.getProject())) {
+ throw new IllegalArgumentException(
+ String.format(
+ "You are attempting to change the project of feature set %s from %s to %s. This isn't allowed. Please create a new feature set under the desired project.",
+ spec.getName(), project, spec.getProject()));
+ }
+
+ Set existingEntities =
+ entities.stream().map(Entity::toProto).collect(Collectors.toSet());
+
+ // 1b. check no change to entities
+ if (!Sets.newHashSet(spec.getEntitiesList()).equals(existingEntities)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "You are attempting to change the entities of this feature set: Given set of entities \n{%s}\n does not match existing set of entities\n {%s}. This isn't allowed. Please create a new feature set. ",
+ spec.getEntitiesList(), existingEntities));
+ }
+
+ // 4. Update max age and source.
+ maxAgeSeconds = spec.getMaxAge().getSeconds();
+ source = Source.fromProto(spec.getSource());
+
+ Map updatedFeatures =
+ spec.getFeaturesList().stream().collect(Collectors.toMap(FeatureSpec::getName, fs -> fs));
+
+ // 3. Tombstone features that are gone, update features that have changed
+ for (Feature existingFeature : features) {
+ String existingFeatureName = existingFeature.getName();
+ FeatureSpec updatedFeatureSpec = updatedFeatures.get(existingFeatureName);
+ if (updatedFeatureSpec == null) {
+ existingFeature.archive();
+ } else {
+ existingFeature.updateFromProto(updatedFeatureSpec);
+ updatedFeatures.remove(existingFeatureName);
+ }
+ }
+
+ // 4. Add new features
+ for (FeatureSpec featureSpec : updatedFeatures.values()) {
+ Feature newFeature = Feature.fromProto(featureSpec);
+ addFeature(newFeature);
}
}
- public void addEntity(Field field) {
- field.setProject(this.project.getName());
- field.setVersion(this.getVersion());
- entities.add(field);
+ public void addEntities(List entities) {
+ for (Entity entity : entities) {
+ addEntity(entity);
+ }
+ }
+
+ public void addEntity(Entity entity) {
+ entity.setFeatureSet(this);
+ entities.add(entity);
}
- public void addFeatures(List fields) {
- for (Field field : fields) {
- addFeature(field);
+ public void addFeatures(List features) {
+ for (Feature feature : features) {
+ addFeature(feature);
}
}
- public void addFeature(Field field) {
- field.setProject(this.project.getName());
- field.setVersion(this.getVersion());
- features.add(field);
+ public void addFeature(Feature feature) {
+ feature.setFeatureSet(this);
+ features.add(feature);
}
public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferException {
List entitySpecs = new ArrayList<>();
- for (Field entityField : entities) {
- EntitySpec.Builder entitySpecBuilder = EntitySpec.newBuilder();
- setEntitySpecFields(entitySpecBuilder, entityField);
- entitySpecs.add(entitySpecBuilder.build());
+ for (Entity entityField : entities) {
+ entitySpecs.add(entityField.toProto());
}
List featureSpecs = new ArrayList<>();
- for (Field featureField : features) {
- FeatureSpec.Builder featureSpecBuilder = FeatureSpec.newBuilder();
- setFeatureSpecFields(featureSpecBuilder, featureField);
- featureSpecs.add(featureSpecBuilder.build());
+ for (Feature featureField : features) {
+ if (!featureField.isArchived()) {
+ featureSpecs.add(featureField.toProto());
+ }
}
FeatureSetMeta.Builder meta =
FeatureSetMeta.newBuilder()
.setCreatedTimestamp(
Timestamp.newBuilder().setSeconds(super.getCreated().getTime() / 1000L))
- .setStatus(FeatureSetStatus.valueOf(status));
+ .setStatus(status);
FeatureSetSpec.Builder spec =
FeatureSetSpec.newBuilder()
.setName(getName())
- .setVersion(getVersion())
.setProject(project.getName())
.setMaxAge(Duration.newBuilder().setSeconds(maxAgeSeconds))
.addAllEntities(entitySpecs)
.addAllFeatures(featureSpecs)
+ .putAllLabels(TypeConversion.convertJsonStringToMap(labels))
.setSource(source.toProto());
return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build();
}
- // setEntitySpecFields and setFeatureSpecFields methods contain duplicated code because
- // Feast internally treat EntitySpec and FeatureSpec as Field class. However, the proto message
- // builder for EntitySpec and FeatureSpec are of different class.
- @SuppressWarnings("DuplicatedCode")
- private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Field entityField)
- throws InvalidProtocolBufferException {
- entitySpecBuilder
- .setName(entityField.getName())
- .setValueType(Enum.valueOf(entityField.getType()));
-
- if (entityField.getPresence() != null) {
- entitySpecBuilder.setPresence(FeaturePresence.parseFrom(entityField.getPresence()));
- } else if (entityField.getGroupPresence() != null) {
- entitySpecBuilder.setGroupPresence(
- FeaturePresenceWithinGroup.parseFrom(entityField.getGroupPresence()));
- }
-
- if (entityField.getShape() != null) {
- entitySpecBuilder.setShape(FixedShape.parseFrom(entityField.getShape()));
- } else if (entityField.getValueCount() != null) {
- entitySpecBuilder.setValueCount(ValueCount.parseFrom(entityField.getValueCount()));
- }
-
- if (entityField.getDomain() != null) {
- entitySpecBuilder.setDomain(entityField.getDomain());
- } else if (entityField.getIntDomain() != null) {
- entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain()));
- } else if (entityField.getFloatDomain() != null) {
- entitySpecBuilder.setFloatDomain(FloatDomain.parseFrom(entityField.getFloatDomain()));
- } else if (entityField.getStringDomain() != null) {
- entitySpecBuilder.setStringDomain(StringDomain.parseFrom(entityField.getStringDomain()));
- } else if (entityField.getBoolDomain() != null) {
- entitySpecBuilder.setBoolDomain(BoolDomain.parseFrom(entityField.getBoolDomain()));
- } else if (entityField.getStructDomain() != null) {
- entitySpecBuilder.setStructDomain(StructDomain.parseFrom(entityField.getStructDomain()));
- } else if (entityField.getNaturalLanguageDomain() != null) {
- entitySpecBuilder.setNaturalLanguageDomain(
- NaturalLanguageDomain.parseFrom(entityField.getNaturalLanguageDomain()));
- } else if (entityField.getImageDomain() != null) {
- entitySpecBuilder.setImageDomain(ImageDomain.parseFrom(entityField.getImageDomain()));
- } else if (entityField.getMidDomain() != null) {
- entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain()));
- } else if (entityField.getUrlDomain() != null) {
- entitySpecBuilder.setUrlDomain(URLDomain.parseFrom(entityField.getUrlDomain()));
- } else if (entityField.getTimeDomain() != null) {
- entitySpecBuilder.setTimeDomain(TimeDomain.parseFrom(entityField.getTimeDomain()));
- } else if (entityField.getTimeOfDayDomain() != null) {
- entitySpecBuilder.setTimeOfDayDomain(
- TimeOfDayDomain.parseFrom(entityField.getTimeOfDayDomain()));
- }
+ @Override
+ public int hashCode() {
+ HashCodeBuilder hcb = new HashCodeBuilder();
+ hcb.append(project.getName());
+ hcb.append(getName());
+ return hcb.toHashCode();
}
- // Refer to setEntitySpecFields method for the reason for code duplication.
- @SuppressWarnings("DuplicatedCode")
- private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Field featureField)
- throws InvalidProtocolBufferException {
- featureSpecBuilder
- .setName(featureField.getName())
- .setValueType(Enum.valueOf(featureField.getType()));
-
- if (featureField.getPresence() != null) {
- featureSpecBuilder.setPresence(FeaturePresence.parseFrom(featureField.getPresence()));
- } else if (featureField.getGroupPresence() != null) {
- featureSpecBuilder.setGroupPresence(
- FeaturePresenceWithinGroup.parseFrom(featureField.getGroupPresence()));
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
}
-
- if (featureField.getShape() != null) {
- featureSpecBuilder.setShape(FixedShape.parseFrom(featureField.getShape()));
- } else if (featureField.getValueCount() != null) {
- featureSpecBuilder.setValueCount(ValueCount.parseFrom(featureField.getValueCount()));
+ if (!(obj instanceof FeatureSet)) {
+ return false;
}
- if (featureField.getDomain() != null) {
- featureSpecBuilder.setDomain(featureField.getDomain());
- } else if (featureField.getIntDomain() != null) {
- featureSpecBuilder.setIntDomain(IntDomain.parseFrom(featureField.getIntDomain()));
- } else if (featureField.getFloatDomain() != null) {
- featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(featureField.getFloatDomain()));
- } else if (featureField.getStringDomain() != null) {
- featureSpecBuilder.setStringDomain(StringDomain.parseFrom(featureField.getStringDomain()));
- } else if (featureField.getBoolDomain() != null) {
- featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(featureField.getBoolDomain()));
- } else if (featureField.getStructDomain() != null) {
- featureSpecBuilder.setStructDomain(StructDomain.parseFrom(featureField.getStructDomain()));
- } else if (featureField.getNaturalLanguageDomain() != null) {
- featureSpecBuilder.setNaturalLanguageDomain(
- NaturalLanguageDomain.parseFrom(featureField.getNaturalLanguageDomain()));
- } else if (featureField.getImageDomain() != null) {
- featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(featureField.getImageDomain()));
- } else if (featureField.getMidDomain() != null) {
- featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(featureField.getMidDomain()));
- } else if (featureField.getUrlDomain() != null) {
- featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(featureField.getUrlDomain()));
- } else if (featureField.getTimeDomain() != null) {
- featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(featureField.getTimeDomain()));
- } else if (featureField.getTimeOfDayDomain() != null) {
- featureSpecBuilder.setTimeOfDayDomain(
- TimeOfDayDomain.parseFrom(featureField.getTimeOfDayDomain()));
+ FeatureSet other = (FeatureSet) obj;
+ if (!getName().equals(other.getName())) {
+ return false;
}
- }
- /**
- * Checks if the given featureSet's schema and source has is different from this one.
- *
- * @param other FeatureSet to compare to
- * @return boolean denoting if the source or schema have changed.
- */
- public boolean equalTo(FeatureSet other) {
- if (!getName().equals(other.getName())) {
+ if (!getLabels().equals(other.getLabels())) {
return false;
}
@@ -378,65 +301,44 @@ public boolean equalTo(FeatureSet other) {
}
// Create a map of all fields in this feature set
- Map fields = new HashMap<>();
+ Map entitiesMap = new HashMap<>();
+ Map featuresMap = new HashMap<>();
- for (Field e : entities) {
- fields.putIfAbsent(e.getName(), e);
+ for (Entity e : entities) {
+ entitiesMap.putIfAbsent(e.getName(), e);
}
- for (Field f : features) {
- fields.putIfAbsent(f.getName(), f);
+ for (Feature f : features) {
+ featuresMap.putIfAbsent(f.getName(), f);
}
// Ensure map size is consistent with existing fields
- if (fields.size() != other.getFeatures().size() + other.getEntities().size()) {
+ if (entitiesMap.size() != other.getEntities().size()) {
+ return false;
+ }
+ if (featuresMap.size() != other.getFeatures().size()) {
return false;
}
// Ensure the other entities and features exist in the field map
- for (Field e : other.getEntities()) {
- if (!fields.containsKey(e.getName())) {
+ for (Entity e : other.getEntities()) {
+ if (!entitiesMap.containsKey(e.getName())) {
return false;
}
- if (!e.equals(fields.get(e.getName()))) {
+ if (!e.equals(entitiesMap.get(e.getName()))) {
return false;
}
}
- for (Field f : other.getFeatures()) {
- if (!fields.containsKey(f.getName())) {
+ for (Feature f : other.getFeatures()) {
+ if (!featuresMap.containsKey(f.getName())) {
return false;
}
- if (!f.equals(fields.get(f.getName()))) {
+ if (!f.equals(featuresMap.get(f.getName()))) {
return false;
}
}
return true;
}
-
- @Override
- public int hashCode() {
- HashCodeBuilder hcb = new HashCodeBuilder();
- hcb.append(project.getName());
- hcb.append(getName());
- hcb.append(getVersion());
- return hcb.toHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof FeatureSet)) {
- return false;
- }
- return this.equalTo(((FeatureSet) obj));
- }
-
- @Override
- public int compareTo(FeatureSet o) {
- return Integer.compare(getVersion(), o.getVersion());
- }
}
diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java
deleted file mode 100644
index cb23e4eceb..0000000000
--- a/core/src/main/java/feast/core/model/Field.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.model;
-
-import feast.core.FeatureSetProto.EntitySpec;
-import feast.core.FeatureSetProto.FeatureSpec;
-import feast.types.ValueProto.ValueType;
-import java.util.Arrays;
-import java.util.Objects;
-import javax.persistence.Column;
-import javax.persistence.Embeddable;
-import lombok.Getter;
-import lombok.Setter;
-
-@Getter
-@Setter
-@Embeddable
-public class Field {
-
- // Name of the feature
- @Column(name = "name", nullable = false)
- private String name;
-
- // Type of the feature, should correspond with feast.types.ValueType
- @Column(name = "type", nullable = false)
- private String type;
-
- // Version of the field
- @Column(name = "version")
- private int version;
-
- // Project that this field belongs to
- @Column(name = "project")
- private String project;
-
- // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec)
- // Only one of them can be set.
- private byte[] presence;
- private byte[] groupPresence;
-
- // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec)
- // Only one of them can be set.
- private byte[] shape;
- private byte[] valueCount;
-
- // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec)
- // Only one of them can be set.
- private String domain;
- private byte[] intDomain;
- private byte[] floatDomain;
- private byte[] stringDomain;
- private byte[] boolDomain;
- private byte[] structDomain;
- private byte[] naturalLanguageDomain;
- private byte[] imageDomain;
- private byte[] midDomain;
- private byte[] urlDomain;
- private byte[] timeDomain;
- private byte[] timeOfDayDomain;
-
- public Field() {}
-
- public Field(String name, ValueType.Enum type) {
- this.name = name;
- this.type = type.toString();
- }
-
- public Field(FeatureSpec featureSpec) {
- this.name = featureSpec.getName();
- this.type = featureSpec.getValueType().toString();
-
- switch (featureSpec.getPresenceConstraintsCase()) {
- case PRESENCE:
- this.presence = featureSpec.getPresence().toByteArray();
- break;
- case GROUP_PRESENCE:
- this.groupPresence = featureSpec.getGroupPresence().toByteArray();
- break;
- case PRESENCECONSTRAINTS_NOT_SET:
- break;
- }
-
- switch (featureSpec.getShapeTypeCase()) {
- case SHAPE:
- this.shape = featureSpec.getShape().toByteArray();
- break;
- case VALUE_COUNT:
- this.valueCount = featureSpec.getValueCount().toByteArray();
- break;
- case SHAPETYPE_NOT_SET:
- break;
- }
-
- switch (featureSpec.getDomainInfoCase()) {
- case DOMAIN:
- this.domain = featureSpec.getDomain();
- break;
- case INT_DOMAIN:
- this.intDomain = featureSpec.getIntDomain().toByteArray();
- break;
- case FLOAT_DOMAIN:
- this.floatDomain = featureSpec.getFloatDomain().toByteArray();
- break;
- case STRING_DOMAIN:
- this.stringDomain = featureSpec.getStringDomain().toByteArray();
- break;
- case BOOL_DOMAIN:
- this.boolDomain = featureSpec.getBoolDomain().toByteArray();
- break;
- case STRUCT_DOMAIN:
- this.structDomain = featureSpec.getStructDomain().toByteArray();
- break;
- case NATURAL_LANGUAGE_DOMAIN:
- this.naturalLanguageDomain = featureSpec.getNaturalLanguageDomain().toByteArray();
- break;
- case IMAGE_DOMAIN:
- this.imageDomain = featureSpec.getImageDomain().toByteArray();
- break;
- case MID_DOMAIN:
- this.midDomain = featureSpec.getMidDomain().toByteArray();
- break;
- case URL_DOMAIN:
- this.urlDomain = featureSpec.getUrlDomain().toByteArray();
- break;
- case TIME_DOMAIN:
- this.timeDomain = featureSpec.getTimeDomain().toByteArray();
- break;
- case TIME_OF_DAY_DOMAIN:
- this.timeOfDayDomain = featureSpec.getTimeOfDayDomain().toByteArray();
- break;
- case DOMAININFO_NOT_SET:
- break;
- }
- }
-
- public Field(EntitySpec entitySpec) {
- this.name = entitySpec.getName();
- this.type = entitySpec.getValueType().toString();
-
- switch (entitySpec.getPresenceConstraintsCase()) {
- case PRESENCE:
- this.presence = entitySpec.getPresence().toByteArray();
- break;
- case GROUP_PRESENCE:
- this.groupPresence = entitySpec.getGroupPresence().toByteArray();
- break;
- case PRESENCECONSTRAINTS_NOT_SET:
- break;
- }
-
- switch (entitySpec.getShapeTypeCase()) {
- case SHAPE:
- this.shape = entitySpec.getShape().toByteArray();
- break;
- case VALUE_COUNT:
- this.valueCount = entitySpec.getValueCount().toByteArray();
- break;
- case SHAPETYPE_NOT_SET:
- break;
- }
-
- switch (entitySpec.getDomainInfoCase()) {
- case DOMAIN:
- this.domain = entitySpec.getDomain();
- break;
- case INT_DOMAIN:
- this.intDomain = entitySpec.getIntDomain().toByteArray();
- break;
- case FLOAT_DOMAIN:
- this.floatDomain = entitySpec.getFloatDomain().toByteArray();
- break;
- case STRING_DOMAIN:
- this.stringDomain = entitySpec.getStringDomain().toByteArray();
- break;
- case BOOL_DOMAIN:
- this.boolDomain = entitySpec.getBoolDomain().toByteArray();
- break;
- case STRUCT_DOMAIN:
- this.structDomain = entitySpec.getStructDomain().toByteArray();
- break;
- case NATURAL_LANGUAGE_DOMAIN:
- this.naturalLanguageDomain = entitySpec.getNaturalLanguageDomain().toByteArray();
- break;
- case IMAGE_DOMAIN:
- this.imageDomain = entitySpec.getImageDomain().toByteArray();
- break;
- case MID_DOMAIN:
- this.midDomain = entitySpec.getMidDomain().toByteArray();
- break;
- case URL_DOMAIN:
- this.urlDomain = entitySpec.getUrlDomain().toByteArray();
- break;
- case TIME_DOMAIN:
- this.timeDomain = entitySpec.getTimeDomain().toByteArray();
- break;
- case TIME_OF_DAY_DOMAIN:
- this.timeOfDayDomain = entitySpec.getTimeOfDayDomain().toByteArray();
- break;
- case DOMAININFO_NOT_SET:
- break;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- Field field = (Field) o;
- return Objects.equals(name, field.name)
- && Objects.equals(type, field.type)
- && Objects.equals(project, field.project)
- && Arrays.equals(presence, field.presence)
- && Arrays.equals(groupPresence, field.groupPresence)
- && Arrays.equals(shape, field.shape)
- && Arrays.equals(valueCount, field.valueCount)
- && Objects.equals(domain, field.domain)
- && Arrays.equals(intDomain, field.intDomain)
- && Arrays.equals(floatDomain, field.floatDomain)
- && Arrays.equals(stringDomain, field.stringDomain)
- && Arrays.equals(boolDomain, field.boolDomain)
- && Arrays.equals(structDomain, field.structDomain)
- && Arrays.equals(naturalLanguageDomain, field.naturalLanguageDomain)
- && Arrays.equals(imageDomain, field.imageDomain)
- && Arrays.equals(midDomain, field.midDomain)
- && Arrays.equals(urlDomain, field.urlDomain)
- && Arrays.equals(timeDomain, field.timeDomain)
- && Arrays.equals(timeOfDayDomain, field.timeOfDayDomain);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), name, type);
- }
-}
diff --git a/core/src/main/java/feast/core/model/Job.java b/core/src/main/java/feast/core/model/Job.java
index 738a16db2d..5fce3dffbe 100644
--- a/core/src/main/java/feast/core/model/Job.java
+++ b/core/src/main/java/feast/core/model/Job.java
@@ -17,23 +17,13 @@
package feast.core.model;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.FeatureSetProto;
-import feast.core.IngestionJobProto;
+import feast.core.job.Runner;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.IngestionJobProto;
import java.util.ArrayList;
import java.util.List;
-import javax.persistence.CascadeType;
-import javax.persistence.Column;
+import javax.persistence.*;
import javax.persistence.Entity;
-import javax.persistence.EnumType;
-import javax.persistence.Enumerated;
-import javax.persistence.Id;
-import javax.persistence.Index;
-import javax.persistence.JoinColumn;
-import javax.persistence.JoinTable;
-import javax.persistence.ManyToMany;
-import javax.persistence.ManyToOne;
-import javax.persistence.OneToMany;
-import javax.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
@@ -55,9 +45,9 @@ public class Job extends AbstractTimestampEntity {
private String extId;
// Runner type
- // Use Runner.name() when converting a Runner to string to assign to this property.
+ @Enumerated(EnumType.STRING)
@Column(name = "runner")
- private String runner;
+ private Runner runner;
// Source id
@ManyToOne
@@ -70,21 +60,17 @@ public class Job extends AbstractTimestampEntity {
private Store store;
// FeatureSets populated by the job
- @ManyToMany
+ @ManyToMany(cascade = CascadeType.ALL)
@JoinTable(
name = "jobs_feature_sets",
- joinColumns = @JoinColumn(name = "feature_sets_id"),
- inverseJoinColumns = @JoinColumn(name = "job_id"),
+ joinColumns = @JoinColumn(name = "job_id"),
+ inverseJoinColumns = @JoinColumn(name = "feature_sets_id"),
indexes = {
@Index(name = "idx_jobs_feature_sets_job_id", columnList = "job_id"),
@Index(name = "idx_jobs_feature_sets_feature_sets_id", columnList = "feature_sets_id")
})
private List featureSets;
- // Job Metrics
- @OneToMany(mappedBy = "job", cascade = CascadeType.ALL)
- private List metrics;
-
@Enumerated(EnumType.STRING)
@Column(name = "status", length = 16)
private JobStatus status;
@@ -93,26 +79,12 @@ public Job() {
super();
}
- public Job(
- String id,
- String extId,
- String runner,
- Source source,
- Store sink,
- List featureSets,
- JobStatus jobStatus) {
- this.id = id;
- this.extId = extId;
- this.source = source;
- this.runner = runner;
- this.store = sink;
- this.featureSets = featureSets;
- this.status = jobStatus;
+ public boolean hasTerminated() {
+ return getStatus().isTerminal();
}
- public void updateMetrics(List newMetrics) {
- metrics.clear();
- metrics.addAll(newMetrics);
+ public boolean isRunning() {
+ return getStatus() == JobStatus.RUNNING;
}
public String getSinkName() {
diff --git a/core/src/main/java/feast/core/model/JobStatus.java b/core/src/main/java/feast/core/model/JobStatus.java
index 86aa512933..6bafc06ec9 100644
--- a/core/src/main/java/feast/core/model/JobStatus.java
+++ b/core/src/main/java/feast/core/model/JobStatus.java
@@ -16,11 +16,9 @@
*/
package feast.core.model;
-import feast.core.IngestionJobProto.IngestionJobStatus;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
+import feast.proto.core.IngestionJobProto.IngestionJobStatus;
import java.util.Map;
+import java.util.Set;
public enum JobStatus {
/** Job status is not known. */
@@ -53,33 +51,41 @@ public enum JobStatus {
/** job has been suspended */
SUSPENDED;
- private static final Collection TERMINAL_STATE =
- Collections.unmodifiableList(Arrays.asList(COMPLETED, ABORTED, ERROR));
+ private static final Set TERMINAL_STATES = Set.of(COMPLETED, ABORTED, ERROR);
/**
- * Get a collection of terminal job state.
+ * Get the set of terminal job states.
*
- *
Terminal job state is final and will not change to any other state.
+ *
A terminal job state is final and will not change to any other state.
*
- * @return collection of terminal job state.
+ * @return set of terminal job states.
*/
- public static Collection getTerminalState() {
- return TERMINAL_STATE;
+ public static Set getTerminalStates() {
+ return TERMINAL_STATES;
}
- private static final Collection TRANSITIONAL_STATES =
- Collections.unmodifiableList(Arrays.asList(PENDING, ABORTING, SUSPENDING));
+ private static final Set TRANSITIONAL_STATES = Set.of(PENDING, ABORTING, SUSPENDING);
/**
- * Get Transitional Job Status states. Transitionals states are assigned to jobs that
+ * Get Transitional Job Status states. Transitional states are assigned to jobs that are
* transitioning to a more stable state (ie SUSPENDED, ABORTED etc.)
*
- * @return Collection of transitional Job Status states.
+ * @return set of transitional Job Status states.
*/
- public static final Collection getTransitionalStates() {
+ public static Set getTransitionalStates() {
return TRANSITIONAL_STATES;
}
+ /** @return true if this {@code JobStatus} is a terminal state. */
+ public boolean isTerminal() {
+ return getTerminalStates().contains(this);
+ }
+
+ /** @return true if this {@code JobStatus} is a transitional state. */
+ public boolean isTransitional() {
+ return getTransitionalStates().contains(this);
+ }
+
private static final Map INGESTION_JOB_STATUS_MAP =
Map.of(
JobStatus.UNKNOWN, IngestionJobStatus.UNKNOWN,
@@ -95,7 +101,7 @@ public static final Collection getTransitionalStates() {
/**
* Convert a Job Status to Ingestion Job Status proto
*
- * @return IngestionJobStatus proto derieved from this job status
+ * @return IngestionJobStatus proto derived from this job status
*/
public IngestionJobStatus toProto() {
// maps job models job status to ingestion job status
diff --git a/core/src/main/java/feast/core/model/Metrics.java b/core/src/main/java/feast/core/model/Metrics.java
deleted file mode 100644
index 0b7514816f..0000000000
--- a/core/src/main/java/feast/core/model/Metrics.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.model;
-
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@NoArgsConstructor
-@Getter
-@Setter
-@Entity
-@Table(name = "metrics")
-public class Metrics extends AbstractTimestampEntity {
-
- @Id
- @GeneratedValue(strategy = GenerationType.AUTO)
- private long id;
-
- @ManyToOne(fetch = FetchType.LAZY)
- @JoinColumn(name = "job_id")
- private Job job;
-
- /** Metrics name */
- private String name;
-
- /** Metrics value */
- private double value;
-
- /**
- * Create a metrics owned by a {@code job}.
- *
- * @param job owner of this metrics.
- * @param metricsName metrics name.
- * @param value metrics value.
- */
- public Metrics(Job job, String metricsName, double value) {
- this.job = job;
- this.name = metricsName;
- this.value = value;
- }
-}
diff --git a/core/src/main/java/feast/core/model/Project.java b/core/src/main/java/feast/core/model/Project.java
index d6e6149394..c55830c824 100644
--- a/core/src/main/java/feast/core/model/Project.java
+++ b/core/src/main/java/feast/core/model/Project.java
@@ -34,6 +34,7 @@
@Entity
@Table(name = "projects")
public class Project {
+ public static final String DEFAULT_NAME = "default";
// Name of the project
@Id
diff --git a/core/src/main/java/feast/core/model/Source.java b/core/src/main/java/feast/core/model/Source.java
index 28db1e9a5b..d199a1b676 100644
--- a/core/src/main/java/feast/core/model/Source.java
+++ b/core/src/main/java/feast/core/model/Source.java
@@ -18,10 +18,10 @@
import com.google.common.collect.Sets;
import com.google.protobuf.Message;
-import feast.core.SourceProto;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.Source.Builder;
-import feast.core.SourceProto.SourceType;
+import feast.proto.core.SourceProto;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.Source.Builder;
+import feast.proto.core.SourceProto.SourceType;
import io.grpc.Status;
import java.util.Objects;
import java.util.Set;
diff --git a/core/src/main/java/feast/core/model/Store.java b/core/src/main/java/feast/core/model/Store.java
index 9dc44bdc73..1f7c373bdc 100644
--- a/core/src/main/java/feast/core/model/Store.java
+++ b/core/src/main/java/feast/core/model/Store.java
@@ -17,13 +17,14 @@
package feast.core.model;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.BigQueryConfig;
-import feast.core.StoreProto.Store.Builder;
-import feast.core.StoreProto.Store.CassandraConfig;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.BigQueryConfig;
+import feast.proto.core.StoreProto.Store.Builder;
+import feast.proto.core.StoreProto.Store.CassandraConfig;
+import feast.proto.core.StoreProto.Store.RedisClusterConfig;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -82,6 +83,9 @@ public static Store fromProto(StoreProto.Store storeProto) throws IllegalArgumen
case CASSANDRA:
config = storeProto.getCassandraConfig().toByteArray();
break;
+ case REDIS_CLUSTER:
+ config = storeProto.getRedisClusterConfig().toByteArray();
+ break;
default:
throw new IllegalArgumentException("Invalid store provided");
}
@@ -106,6 +110,9 @@ public StoreProto.Store toProto() throws InvalidProtocolBufferException {
case CASSANDRA:
CassandraConfig cassConfig = CassandraConfig.parseFrom(config);
return storeProtoBuilder.setCassandraConfig(cassConfig).build();
+ case REDIS_CLUSTER:
+ RedisClusterConfig redisClusterConfig = RedisClusterConfig.parseFrom(config);
+ return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig).build();
default:
throw new InvalidProtocolBufferException("Invalid store set");
}
@@ -118,22 +125,18 @@ public List getSubscriptions() {
}
private static String convertSubscriptionToString(Subscription sub) {
- if (sub.getVersion().isEmpty() || sub.getName().isEmpty() || sub.getProject().isEmpty()) {
+ if (sub.getName().isEmpty() || sub.getProject().isEmpty()) {
throw new IllegalArgumentException(
String.format("Missing arguments in subscription string: %s", sub.toString()));
}
- return String.format("%s:%s:%s", sub.getProject(), sub.getName(), sub.getVersion());
+ return String.format("%s:%s", sub.getProject(), sub.getName());
}
private Subscription convertStringToSubscription(String sub) {
if (sub.equals("")) {
return Subscription.newBuilder().build();
}
- String[] split = sub.split(":", 3);
- return Subscription.newBuilder()
- .setProject(split[0])
- .setName(split[1])
- .setVersion(split[2])
- .build();
+ String[] split = sub.split(":", 2);
+ return Subscription.newBuilder().setProject(split[0]).setName(split[1]).build();
}
}
diff --git a/core/src/main/java/feast/core/service/AccessManagementService.java b/core/src/main/java/feast/core/service/AccessManagementService.java
index df92750e94..5b02d6f3c4 100644
--- a/core/src/main/java/feast/core/service/AccessManagementService.java
+++ b/core/src/main/java/feast/core/service/AccessManagementService.java
@@ -28,12 +28,15 @@
@Slf4j
@Service
public class AccessManagementService {
-
private ProjectRepository projectRepository;
@Autowired
public AccessManagementService(ProjectRepository projectRepository) {
this.projectRepository = projectRepository;
+ // create default project if it does not yet exist.
+ if (!projectRepository.existsById(Project.DEFAULT_NAME)) {
+ this.createProject(Project.DEFAULT_NAME);
+ }
}
/**
@@ -61,6 +64,9 @@ public void archiveProject(String name) {
if (!project.isPresent()) {
throw new IllegalArgumentException(String.format("Could not find project: \"%s\"", name));
}
+ if (name.equals(Project.DEFAULT_NAME)) {
+ throw new UnsupportedOperationException("Archiving the default project is not allowed.");
+ }
Project p = project.get();
p.setArchived(true);
projectRepository.saveAndFlush(p);
diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/core/src/main/java/feast/core/service/JobCoordinatorService.java
index b66d181022..90ee54ca16 100644
--- a/core/src/main/java/feast/core/service/JobCoordinatorService.java
+++ b/core/src/main/java/feast/core/service/JobCoordinatorService.java
@@ -17,23 +17,21 @@
package feast.core.service;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest;
-import feast.core.CoreServiceProto.ListStoresRequest.Filter;
-import feast.core.CoreServiceProto.ListStoresResponse;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.FeatureSetStatus;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.Subscription;
-import feast.core.config.FeastProperties.JobUpdatesProperties;
+import feast.core.config.FeastProperties;
+import feast.core.config.FeastProperties.JobProperties;
import feast.core.dao.FeatureSetRepository;
import feast.core.dao.JobRepository;
import feast.core.job.JobManager;
import feast.core.job.JobUpdateTask;
import feast.core.model.FeatureSet;
import feast.core.model.Job;
-import feast.core.model.JobStatus;
import feast.core.model.Source;
import feast.core.model.Store;
+import feast.proto.core.CoreServiceProto.ListStoresRequest.Filter;
+import feast.proto.core.CoreServiceProto.ListStoresResponse;
+import feast.proto.core.FeatureSetProto.FeatureSetStatus;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -44,6 +42,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
+import javax.validation.constraints.Positive;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
@@ -54,11 +53,11 @@
@Service
public class JobCoordinatorService {
- private JobRepository jobRepository;
- private FeatureSetRepository featureSetRepository;
- private SpecService specService;
- private JobManager jobManager;
- private JobUpdatesProperties jobUpdatesProperties;
+ private final JobRepository jobRepository;
+ private final FeatureSetRepository featureSetRepository;
+ private final SpecService specService;
+ private final JobManager jobManager;
+ private final JobProperties jobProperties;
@Autowired
public JobCoordinatorService(
@@ -66,12 +65,12 @@ public JobCoordinatorService(
FeatureSetRepository featureSetRepository,
SpecService specService,
JobManager jobManager,
- JobUpdatesProperties jobUpdatesProperties) {
+ FeastProperties feastProperties) {
this.jobRepository = jobRepository;
this.featureSetRepository = featureSetRepository;
this.specService = specService;
this.jobManager = jobManager;
- this.jobUpdatesProperties = jobUpdatesProperties;
+ this.jobProperties = feastProperties.getJobs();
}
/**
@@ -86,98 +85,93 @@ public JobCoordinatorService(
*
4) Updates Feature set statuses
*/
@Transactional
- @Scheduled(fixedDelayString = "${feast.jobs.updates.pollingIntervalMillis}")
+ @Scheduled(fixedDelayString = "${feast.jobs.polling_interval_milliseconds}")
public void Poll() throws InvalidProtocolBufferException {
log.info("Polling for new jobs...");
+ @Positive long updateTimeout = jobProperties.getJobUpdateTimeoutSeconds();
List jobUpdateTasks = new ArrayList<>();
ListStoresResponse listStoresResponse = specService.listStores(Filter.newBuilder().build());
- for (StoreProto.Store store : listStoresResponse.getStoreList()) {
- Set featureSets = new HashSet<>();
- for (Subscription subscription : store.getSubscriptionsList()) {
- featureSets.addAll(
- new ArrayList<>(
- specService
- .listFeatureSets(
- ListFeatureSetsRequest.Filter.newBuilder()
- .setFeatureSetName(subscription.getName())
- .setFeatureSetVersion(subscription.getVersion())
- .setProject(subscription.getProject())
- .build())
- .getFeatureSetsList()));
- }
- if (!featureSets.isEmpty()) {
- featureSets.stream()
- .collect(Collectors.groupingBy(fs -> fs.getSpec().getSource()))
- .entrySet()
- .stream()
- .forEach(
- kv -> {
- Optional originalJob =
- getJob(Source.fromProto(kv.getKey()), Store.fromProto(store));
- jobUpdateTasks.add(
- new JobUpdateTask(
- kv.getValue(),
- kv.getKey(),
- store,
- originalJob,
- jobManager,
- jobUpdatesProperties.getTimeoutSeconds()));
- });
+
+ for (StoreProto.Store storeSpec : listStoresResponse.getStoreList()) {
+ Set featureSets = new HashSet<>();
+ Store store = Store.fromProto(storeSpec);
+
+ for (Subscription subscription : store.getSubscriptions()) {
+ List featureSetsForSub =
+ featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc(
+ subscription.getName().replace('*', '%'),
+ subscription.getProject().replace('*', '%'));
+ featureSets.addAll(featureSetsForSub);
}
+
+ featureSets.stream()
+ .collect(Collectors.groupingBy(FeatureSet::getSource))
+ .forEach(
+ (source, setsForSource) -> {
+ Optional originalJob = getJob(source, store);
+ jobUpdateTasks.add(
+ new JobUpdateTask(
+ setsForSource, source, store, originalJob, jobManager, updateTimeout));
+ });
}
- if (jobUpdateTasks.size() == 0) {
+ if (jobUpdateTasks.isEmpty()) {
log.info("No jobs found.");
return;
}
log.info("Creating/Updating {} jobs...", jobUpdateTasks.size());
- ExecutorService executorService = Executors.newFixedThreadPool(jobUpdateTasks.size());
+ startOrUpdateJobs(jobUpdateTasks);
+
+ log.info("Updating feature set status");
+ updateFeatureSetStatuses(jobUpdateTasks);
+ }
+
+ void startOrUpdateJobs(List tasks) {
+ ExecutorService executorService = Executors.newFixedThreadPool(tasks.size());
ExecutorCompletionService ecs = new ExecutorCompletionService<>(executorService);
- jobUpdateTasks.forEach(ecs::submit);
+ tasks.forEach(ecs::submit);
int completedTasks = 0;
- while (completedTasks < jobUpdateTasks.size()) {
+ List startedJobs = new ArrayList<>();
+ while (completedTasks < tasks.size()) {
try {
Job job = ecs.take().get();
if (job != null) {
- jobRepository.saveAndFlush(job);
+ startedJobs.add(job);
}
} catch (ExecutionException | InterruptedException e) {
log.warn("Unable to start or update job: {}", e.getMessage());
}
completedTasks++;
}
-
- log.info("Updating feature set status");
- updateFeatureSetStatuses(jobUpdateTasks);
+ jobRepository.saveAll(startedJobs);
+ executorService.shutdown();
}
// TODO: make this more efficient
private void updateFeatureSetStatuses(List jobUpdateTasks) {
Set ready = new HashSet<>();
Set pending = new HashSet<>();
- for (JobUpdateTask jobUpdateTask : jobUpdateTasks) {
- Optional job =
- getJob(
- Source.fromProto(jobUpdateTask.getSourceSpec()),
- Store.fromProto(jobUpdateTask.getStore()));
- if (job.isPresent()) {
- if (job.get().getStatus() == JobStatus.RUNNING) {
- ready.addAll(job.get().getFeatureSets());
- } else {
- pending.addAll(job.get().getFeatureSets());
- }
- }
+ for (JobUpdateTask task : jobUpdateTasks) {
+ getJob(task.getSource(), task.getStore())
+ .ifPresent(
+ job -> {
+ if (job.isRunning()) {
+ ready.addAll(job.getFeatureSets());
+ } else {
+ pending.addAll(job.getFeatureSets());
+ }
+ });
}
ready.removeAll(pending);
ready.forEach(
fs -> {
- fs.setStatus(FeatureSetStatus.STATUS_READY.toString());
+ fs.setStatus(FeatureSetStatus.STATUS_READY);
featureSetRepository.save(fs);
});
pending.forEach(
fs -> {
- fs.setStatus(FeatureSetStatus.STATUS_PENDING.toString());
+ fs.setStatus(FeatureSetStatus.STATUS_JOB_STARTING);
featureSetRepository.save(fs);
});
featureSetRepository.flush();
@@ -188,11 +182,8 @@ public Optional getJob(Source source, Store store) {
List jobs =
jobRepository.findBySourceIdAndStoreNameOrderByLastUpdatedDesc(
source.getId(), store.getName());
- jobs =
- jobs.stream()
- .filter(job -> !JobStatus.getTerminalState().contains(job.getStatus()))
- .collect(Collectors.toList());
- if (jobs.size() == 0) {
+ jobs = jobs.stream().filter(job -> !job.hasTerminated()).collect(Collectors.toList());
+ if (jobs.isEmpty()) {
return Optional.empty();
}
// return the latest
diff --git a/core/src/main/java/feast/core/service/JobService.java b/core/src/main/java/feast/core/service/JobService.java
index bf74b90e80..cc125305ec 100644
--- a/core/src/main/java/feast/core/service/JobService.java
+++ b/core/src/main/java/feast/core/service/JobService.java
@@ -17,24 +17,25 @@
package feast.core.service;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListIngestionJobsRequest;
-import feast.core.CoreServiceProto.ListIngestionJobsResponse;
-import feast.core.CoreServiceProto.RestartIngestionJobRequest;
-import feast.core.CoreServiceProto.RestartIngestionJobResponse;
-import feast.core.CoreServiceProto.StopIngestionJobRequest;
-import feast.core.CoreServiceProto.StopIngestionJobResponse;
-import feast.core.FeatureSetReferenceProto.FeatureSetReference;
-import feast.core.IngestionJobProto;
import feast.core.dao.JobRepository;
import feast.core.job.JobManager;
+import feast.core.job.Runner;
import feast.core.log.Action;
import feast.core.log.AuditLogger;
import feast.core.log.Resource;
import feast.core.model.FeatureSet;
import feast.core.model.Job;
import feast.core.model.JobStatus;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsRequest;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsResponse;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobResponse;
+import feast.proto.core.CoreServiceProto.StopIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.StopIngestionJobResponse;
+import feast.proto.core.FeatureSetReferenceProto.FeatureSetReference;
+import feast.proto.core.IngestionJobProto;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -50,13 +51,13 @@
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
-/** Defines a Job Managemenent Service that allows users to manage feast ingestion jobs. */
+/** A Job Management Service that allows users to manage Feast ingestion jobs. */
@Slf4j
@Service
public class JobService {
- private JobRepository jobRepository;
- private SpecService specService;
- private Map jobManagers;
+ private final JobRepository jobRepository;
+ private final SpecService specService;
+ private final Map jobManagers;
@Autowired
public JobService(
@@ -66,13 +67,13 @@ public JobService(
this.jobManagers = new HashMap<>();
for (JobManager manager : jobManagerList) {
- this.jobManagers.put(manager.getRunnerType().name(), manager);
+ this.jobManagers.put(manager.getRunnerType(), manager);
}
}
/* Job Service API */
/**
- * List Ingestion Jobs in feast matching the given request. See CoreService protobuf documentation
+ * List Ingestion Jobs in Feast matching the given request. See CoreService protobuf documentation
* for more detailed documentation.
*
* @param request list ingestion jobs request specifying which jobs to include
@@ -158,6 +159,7 @@ public RestartIngestionJobResponse restartJob(RestartIngestionJobRequest request
// check job exists
Optional getJob = this.jobRepository.findById(request.getId());
if (getJob.isEmpty()) {
+ // FIXME: if getJob.isEmpty then constructing this error message will always throw an error...
throw new NoSuchElementException(
"Attempted to stop nonexistent job with id: " + getJob.get().getId());
}
@@ -165,9 +167,7 @@ public RestartIngestionJobResponse restartJob(RestartIngestionJobRequest request
// check job status is valid for restarting
Job job = getJob.get();
JobStatus status = job.getStatus();
- if (JobStatus.getTransitionalStates().contains(status)
- || JobStatus.getTerminalState().contains(status)
- || status.equals(JobStatus.UNKNOWN)) {
+ if (status.isTransitional() || status.isTerminal() || status == JobStatus.UNKNOWN) {
throw new UnsupportedOperationException(
"Restarting a job with a transitional, terminal or unknown status is unsupported");
}
@@ -208,11 +208,10 @@ public StopIngestionJobResponse stopJob(StopIngestionJobRequest request)
// check job status is valid for stopping
Job job = getJob.get();
JobStatus status = job.getStatus();
- if (JobStatus.getTerminalState().contains(status)) {
+ if (status.isTerminal()) {
// do nothing - job is already stopped
return StopIngestionJobResponse.newBuilder().build();
- } else if (JobStatus.getTransitionalStates().contains(status)
- || status.equals(JobStatus.UNKNOWN)) {
+ } else if (status.isTransitional() || status == JobStatus.UNKNOWN) {
throw new UnsupportedOperationException(
"Stopping a job with a transitional or unknown status is unsupported");
}
@@ -249,7 +248,6 @@ private ListFeatureSetsRequest.Filter toListFeatureSetFilter(FeatureSetReference
// match featuresets using contents of featureset reference
String fsName = fsReference.getName();
String fsProject = fsReference.getProject();
- Integer fsVersion = fsReference.getVersion();
// construct list featureset request filter using feature set reference
// for proto3, default value for missing values:
@@ -259,7 +257,6 @@ private ListFeatureSetsRequest.Filter toListFeatureSetFilter(FeatureSetReference
ListFeatureSetsRequest.Filter.newBuilder()
.setFeatureSetName((fsName != "") ? fsName : "*")
.setProject((fsProject != "") ? fsProject : "*")
- .setFeatureSetVersion((fsVersion != 0) ? fsVersion.toString() : "*")
.build();
return filter;
diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java
index 8fec6ac511..01cd264c76 100644
--- a/core/src/main/java/feast/core/service/SpecService.java
+++ b/core/src/main/java/feast/core/service/SpecService.java
@@ -19,23 +19,7 @@
import static feast.core.validators.Matchers.checkValidCharacters;
import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk;
-import com.google.common.collect.Ordering;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ApplyFeatureSetResponse;
-import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status;
-import feast.core.CoreServiceProto.GetFeatureSetRequest;
-import feast.core.CoreServiceProto.GetFeatureSetResponse;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListStoresRequest;
-import feast.core.CoreServiceProto.ListStoresResponse;
-import feast.core.CoreServiceProto.ListStoresResponse.Builder;
-import feast.core.CoreServiceProto.UpdateStoreRequest;
-import feast.core.CoreServiceProto.UpdateStoreResponse;
-import feast.core.FeatureSetProto;
-import feast.core.SourceProto;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.Subscription;
import feast.core.dao.FeatureSetRepository;
import feast.core.dao.ProjectRepository;
import feast.core.dao.StoreRepository;
@@ -45,10 +29,25 @@
import feast.core.model.Source;
import feast.core.model.Store;
import feast.core.validators.FeatureSetValidator;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse.Status;
+import feast.proto.core.CoreServiceProto.GetFeatureSetRequest;
+import feast.proto.core.CoreServiceProto.GetFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListStoresRequest;
+import feast.proto.core.CoreServiceProto.ListStoresResponse;
+import feast.proto.core.CoreServiceProto.ListStoresResponse.Builder;
+import feast.proto.core.CoreServiceProto.UpdateStoreRequest;
+import feast.proto.core.CoreServiceProto.UpdateStoreResponse;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.FeatureSetStatus;
+import feast.proto.core.SourceProto;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -81,7 +80,8 @@ public SpecService(
/**
* Get a feature set matching the feature name and version and project. The feature set name and
* project are required, but version can be omitted by providing 0 for its value. If the version
- * is omitted, the latest feature set will be provided.
+ * is omitted, the latest feature set will be provided. If the project is omitted, the default
+ * would be used.
*
* @param request: GetFeatureSetRequest Request containing filter parameters.
* @return Returns a GetFeatureSetResponse containing a feature set..
@@ -95,130 +95,88 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request)
if (request.getName().isEmpty()) {
throw new IllegalArgumentException("No feature set name provided");
}
+ // Autofill default project if project is not specified
if (request.getProject().isEmpty()) {
- throw new IllegalArgumentException("No project provided");
- }
- if (request.getVersion() < 0) {
- throw new IllegalArgumentException("Version number cannot be less than 0");
+ request = request.toBuilder().setProject(Project.DEFAULT_NAME).build();
}
FeatureSet featureSet;
- // Filter the list based on version
- if (request.getVersion() == 0) {
- featureSet =
- featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc(
- request.getName(), request.getProject());
-
- if (featureSet == null) {
- throw new RetrievalException(
- String.format("Feature set with name \"%s\" could not be found.", request.getName()));
- }
- } else {
- featureSet =
- featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion(
- request.getName(), request.getProject(), request.getVersion());
+ featureSet =
+ featureSetRepository.findFeatureSetByNameAndProject_Name(
+ request.getName(), request.getProject());
- if (featureSet == null) {
- throw new RetrievalException(
- String.format(
- "Feature set with name \"%s\" and version \"%s\" could " + "not be found.",
- request.getName(), request.getVersion()));
- }
+ if (featureSet == null) {
+ throw new RetrievalException(
+ String.format("Feature set with name \"%s\" could not be found.", request.getName()));
}
-
- // Only a single item in list, return successfully
return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSet.toProto()).build();
}
/**
- * Return a list of feature sets matching the feature set name, version, and project provided in
- * the filter. All fields are requried. Use '*' for all three arguments in order to return all
- * feature sets and versions in all projects.
+ * Return a list of feature sets matching the feature set name and project provided in the filter.
+ * All fields are requried. Use '*' for all arguments in order to return all feature sets in all
+ * projects.
*
*
Project name can be explicitly provided, or an asterisk can be provided to match all
- * projects. It is not possible to provide a combination of asterisks/wildcards and text.
+ * projects. It is not possible to provide a combination of asterisks/wildcards and text. If the
+ * project name is omitted, the default project would be used.
*
*
The feature set name in the filter accepts an asterisk as a wildcard. All matching feature
* sets will be returned. Regex is not supported. Explicitly defining a feature set name is not
* possible if a project name is not set explicitly
*
- *
The version field can be one of - '*' - This will match all versions - 'latest' - This will
- * match the latest feature set version - '<number>' - This will match a specific feature
- * set version. This property can only be set if both the feature set name and project name are
- * explicitly set.
- *
- * @param filter filter containing the desired featureSet name and version filter
+ * @param filter filter containing the desired featureSet name
* @return ListFeatureSetsResponse with list of featureSets found matching the filter
*/
public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter)
throws InvalidProtocolBufferException {
String name = filter.getFeatureSetName();
String project = filter.getProject();
- String version = filter.getFeatureSetVersion();
- if (project.isEmpty() || name.isEmpty() || version.isEmpty()) {
+ if (name.isEmpty()) {
throw new IllegalArgumentException(
- String.format(
- "Invalid listFeatureSetRequest, missing arguments. Must provide project, feature set name, and version.",
- filter.toString()));
+ "Invalid listFeatureSetRequest, missing arguments. Must provide feature set name:");
}
checkValidCharactersAllowAsterisk(name, "featureSetName");
checkValidCharactersAllowAsterisk(project, "projectName");
- List featureSets = new ArrayList() {};
+ // Autofill default project if project not specified
+ if (project.isEmpty()) {
+ project = Project.DEFAULT_NAME;
+ }
- if (project.equals("*")) {
- // Matching all projects
+ List featureSets = new ArrayList() {};
- if (name.equals("*") && version.equals("*")) {
+ if (project.contains("*")) {
+ // Matching a wildcard project
+ if (name.contains("*")) {
featureSets =
- featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc(
+ featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc(
name.replace('*', '%'), project.replace('*', '%'));
} else {
throw new IllegalArgumentException(
String.format(
- "Invalid listFeatureSetRequest. Version and feature set name must be set to "
+ "Invalid listFeatureSetRequest. Feature set name must be set to "
+ "\"*\" if the project name and feature set name aren't set explicitly: \n%s",
filter.toString()));
}
} else if (!project.contains("*")) {
// Matching a specific project
-
- if (name.contains("*") && version.equals("*")) {
- // Find all feature sets matching a pattern and versions in a specific project
- featureSets =
- featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- name.replace('*', '%'), project);
-
- } else if (!name.contains("*") && version.equals("*")) {
- // Find all versions of a specific feature set in a specific project
+ if (name.contains("*")) {
+ // Find all feature sets matching a pattern in a specific project
featureSets =
- featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- name, project);
-
- } else if (version.equals("latest")) {
- // Find the latest version of a feature set matching a specific pattern in a specific
- // project
- FeatureSet latestFeatureSet =
- featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc(
+ featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc(
name.replace('*', '%'), project);
- featureSets.add(latestFeatureSet);
-
- } else if (!name.contains("*") && StringUtils.isNumeric(version)) {
- // Find a specific version of a feature set matching a specific name in a specific project
- FeatureSet specificFeatureSet =
- featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion(
- name, project, Integer.parseInt(version));
- featureSets.add(specificFeatureSet);
- } else {
- throw new IllegalArgumentException(
- String.format(
- "Invalid listFeatureSetRequest. Version must be set to \"*\" if the project "
- + "name and feature set name aren't set explicitly: \n%s",
- filter.toString()));
+ } else if (!name.contains("*")) {
+ // Find a specific feature set in a specific project
+ FeatureSet featureSet =
+ featureSetRepository.findFeatureSetByNameAndProject_Name(name, project);
+ if (featureSet != null) {
+ featureSets.add(featureSet);
+ }
}
} else {
throw new IllegalArgumentException(
@@ -273,17 +231,25 @@ public ListStoresResponse listStores(ListStoresRequest.Filter filter) {
}
/**
- * Creates or updates a feature set in the repository. If there is a change in the feature set
- * schema, then the feature set version will be incremented.
+ * Creates or updates a feature set in the repository.
*
*
This function is idempotent. If no changes are detected in the incoming featureSet's schema,
* this method will update the incoming featureSet spec with the latest version stored in the
- * repository, and return that.
+ * repository, and return that. If project is not specified in the given featureSet, will assign
+ * the featureSet to the'default' project.
*
* @param newFeatureSet Feature set that will be created or updated.
*/
public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFeatureSet)
throws InvalidProtocolBufferException {
+ // Autofill default project if not specified
+ if (newFeatureSet.getSpec().getProject().isEmpty()) {
+ newFeatureSet =
+ newFeatureSet
+ .toBuilder()
+ .setSpec(newFeatureSet.getSpec().toBuilder().setProject(Project.DEFAULT_NAME).build())
+ .build();
+ }
// Validate incoming feature set
FeatureSetValidator.validateSpec(newFeatureSet);
@@ -300,53 +266,48 @@ public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFea
throw new IllegalArgumentException(String.format("Project is archived: %s", project_name));
}
- // Retrieve all existing FeatureSet objects
- List existingFeatureSets =
- featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- newFeatureSet.getSpec().getName(), project_name);
-
- if (existingFeatureSets.size() == 0) {
- // Create new feature set since it doesn't exist
+ // Set source to default if not set in proto
+ if (newFeatureSet.getSpec().getSource() == SourceProto.Source.getDefaultInstance()) {
newFeatureSet =
newFeatureSet
.toBuilder()
- .setSpec(newFeatureSet.getSpec().toBuilder().setVersion(1))
+ .setSpec(
+ newFeatureSet.getSpec().toBuilder().setSource(defaultSource.toProto()).build())
.build();
- } else {
- // Retrieve the latest feature set if the name does exist
- existingFeatureSets = Ordering.natural().reverse().sortedCopy(existingFeatureSets);
- FeatureSet latest = existingFeatureSets.get(0);
- FeatureSet featureSet = FeatureSet.fromProto(newFeatureSet);
+ }
+ // Retrieve existing FeatureSet
+ FeatureSet featureSet =
+ featureSetRepository.findFeatureSetByNameAndProject_Name(
+ newFeatureSet.getSpec().getName(), project_name);
+
+ Status status;
+ if (featureSet == null) {
+ // Create new feature set since it doesn't exist
+ newFeatureSet = newFeatureSet.toBuilder().setSpec(newFeatureSet.getSpec()).build();
+ featureSet = FeatureSet.fromProto(newFeatureSet);
+ status = Status.CREATED;
+ } else {
// If the featureSet remains unchanged, we do nothing.
- if (featureSet.equalTo(latest)) {
+ if (featureSet.toProto().getSpec().equals(newFeatureSet.getSpec())) {
return ApplyFeatureSetResponse.newBuilder()
- .setFeatureSet(latest.toProto())
+ .setFeatureSet(featureSet.toProto())
.setStatus(Status.NO_CHANGE)
.build();
}
- // TODO: There is a race condition here with incrementing the version
- newFeatureSet =
- newFeatureSet
- .toBuilder()
- .setSpec(newFeatureSet.getSpec().toBuilder().setVersion(latest.getVersion() + 1))
- .build();
- }
-
- // Build a new FeatureSet object which includes the new properties
- FeatureSet featureSet = FeatureSet.fromProto(newFeatureSet);
- if (newFeatureSet.getSpec().getSource() == SourceProto.Source.getDefaultInstance()) {
- featureSet.setSource(defaultSource);
+ featureSet.updateFromProto(newFeatureSet);
+ status = Status.UPDATED;
}
// Persist the FeatureSet object
+ featureSet.setStatus(FeatureSetStatus.STATUS_PENDING);
project.addFeatureSet(featureSet);
projectRepository.saveAndFlush(project);
// Build ApplyFeatureSetResponse
return ApplyFeatureSetResponse.newBuilder()
.setFeatureSet(featureSet.toProto())
- .setStatus(Status.CREATED)
+ .setStatus(status)
.build();
}
@@ -364,7 +325,7 @@ public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest)
List subs = newStoreProto.getSubscriptionsList();
for (Subscription sub : subs) {
// Ensure that all fields in a subscription contain values
- if ((sub.getVersion().isEmpty() || sub.getName().isEmpty()) || sub.getProject().isEmpty()) {
+ if ((sub.getName().isEmpty()) || sub.getProject().isEmpty()) {
throw new IllegalArgumentException(
String.format("Missing parameter in subscription: %s", sub));
}
diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java
index fd582929c2..8b58eaec4c 100644
--- a/core/src/main/java/feast/core/util/TypeConversion.java
+++ b/core/src/main/java/feast/core/util/TypeConversion.java
@@ -16,12 +16,10 @@
*/
package feast.core.util;
-import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Type;
import java.util.*;
-import java.util.Map.Entry;
public class TypeConversion {
private static Gson gson = new Gson();
@@ -73,23 +71,6 @@ public static Map convertJsonStringToMap(String jsonString) {
* @return json string corresponding to given map
*/
public static String convertMapToJsonString(Map map) {
- if (map.isEmpty()) {
- return "{}";
- }
return gson.toJson(map);
}
-
- /**
- * Convert a map of key value pairs to a array of java arguments in format --key=value
- *
- * @param map
- * @return array of string arguments
- */
- public static String[] convertMapToArgs(Map map) {
- List args = new ArrayList<>();
- for (Entry arg : map.entrySet()) {
- args.add(Strings.lenientFormat("--%s=%s", arg.getKey(), arg.getValue()));
- }
- return args.toArray(new String[] {});
- }
}
diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java
index 213e3898d5..9cfd136b84 100644
--- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java
+++ b/core/src/main/java/feast/core/validators/FeatureSetValidator.java
@@ -19,14 +19,15 @@
import static feast.core.validators.Matchers.checkValidCharacters;
import com.google.common.collect.Sets;
-import feast.core.FeatureSetProto.EntitySpec;
-import feast.core.FeatureSetProto.FeatureSet;
-import feast.core.FeatureSetProto.FeatureSpec;
+import feast.proto.core.FeatureSetProto.EntitySpec;
+import feast.proto.core.FeatureSetProto.FeatureSet;
+import feast.proto.core.FeatureSetProto.FeatureSpec;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
public class FeatureSetValidator {
+
public static void validateSpec(FeatureSet featureSet) {
if (featureSet.getSpec().getProject().isEmpty()) {
throw new IllegalArgumentException("Project name must be provided");
@@ -34,6 +35,9 @@ public static void validateSpec(FeatureSet featureSet) {
if (featureSet.getSpec().getName().isEmpty()) {
throw new IllegalArgumentException("Feature set name must be provided");
}
+ if (featureSet.getSpec().getLabelsMap().containsKey("")) {
+ throw new IllegalArgumentException("Feature set label keys must not be empty");
+ }
checkValidCharacters(featureSet.getSpec().getProject(), "project");
checkValidCharacters(featureSet.getSpec().getName(), "name");
@@ -44,6 +48,9 @@ public static void validateSpec(FeatureSet featureSet) {
}
for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) {
checkValidCharacters(featureSpec.getName(), "features::name");
+ if (featureSpec.getLabelsMap().containsKey("")) {
+ throw new IllegalArgumentException("Feature label keys must not be empty");
+ }
}
}
diff --git a/core/src/main/java/feast/core/validators/OneOfStringValidator.java b/core/src/main/java/feast/core/validators/OneOfStringValidator.java
new file mode 100644
index 0000000000..6b84e44b01
--- /dev/null
+++ b/core/src/main/java/feast/core/validators/OneOfStringValidator.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.validators;
+
+import java.util.Arrays;
+import javax.validation.ConstraintValidator;
+import javax.validation.ConstraintValidatorContext;
+
+/** Validates whether a string value is found within a collection. */
+public class OneOfStringValidator implements ConstraintValidator {
+
+ /** Values that are permitted for a specific instance of this validator */
+ String[] allowedValues;
+
+ /**
+ * Initialize the OneOfStringValidator with a collection of allowed String values.
+ *
+ * @param constraintAnnotation
+ */
+ @Override
+ public void initialize(OneOfStrings constraintAnnotation) {
+ allowedValues = constraintAnnotation.value();
+ }
+
+ /**
+ * Validates whether a string value is found within the collection defined in the annotation.
+ *
+ * @param value String value that should be validated
+ * @param context Provides contextual data and operation when applying a given constraint
+ * validator
+ * @return Boolean value indicating whether the string is found within the allowed values.
+ */
+ @Override
+ public boolean isValid(String value, ConstraintValidatorContext context) {
+ return Arrays.asList(allowedValues).contains(value);
+ }
+}
diff --git a/core/src/main/java/feast/core/validators/OneOfStrings.java b/core/src/main/java/feast/core/validators/OneOfStrings.java
new file mode 100644
index 0000000000..dba290438c
--- /dev/null
+++ b/core/src/main/java/feast/core/validators/OneOfStrings.java
@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.validators;
+
+import java.lang.annotation.*;
+import javax.validation.Constraint;
+import javax.validation.Payload;
+
+/**
+ * Annotation for String "one of" validation. Allows for the definition of a collection through an
+ * annotation. The collection is used to test values defined in the object.
+ */
+@Target({
+ ElementType.METHOD,
+ ElementType.FIELD,
+ ElementType.ANNOTATION_TYPE,
+ ElementType.CONSTRUCTOR,
+ ElementType.PARAMETER
+})
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+@Constraint(validatedBy = OneOfStringValidator.class)
+public @interface OneOfStrings {
+ /** @return Default error message that is returned if the incorrect value is set */
+ String message() default "Field value must be one of the following: {value}";
+
+ /** Allows for the specification of validation groups to which this constraint belongs. */
+ Class[] groups() default {};
+
+ /** An attribute payload that can be used to assign custom payload objects to a constraint. */
+ Class[] payload() default {};
+
+ /** @return Default value that is returned if no allowed values are configured */
+ String[] value() default {};
+}
diff --git a/core/src/main/resources/application.yml b/core/src/main/resources/application.yml
index ee060fffc9..51395cf644 100644
--- a/core/src/main/resources/application.yml
+++ b/core/src/main/resources/application.yml
@@ -23,18 +23,39 @@ grpc:
enable-reflection: true
feast:
-# version: @project.version@
jobs:
- # Runner type for feature population jobs. Currently supported runner types are
- # DirectRunner and DataflowRunner.
- runner: DirectRunner
- # Key-value dict of job options to be passed to the population jobs.
- options: {}
- updates:
- # Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner.
- pollingIntervalMillis: 60000
- # Timeout in seconds for each attempt to update or submit a new job to the runner.
- timeoutSeconds: 240
+ # Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner.
+ polling_interval_milliseconds: 60000
+
+ # Timeout in seconds for each attempt to update or submit a new job to the runner.
+ job_update_timeout_seconds: 240
+
+ # Name of the active runner in "runners" that should be used. Only a single runner can be active at one time.
+ active_runner: direct
+
+ # List of runner configurations. Please see protos/feast/core/Runner.proto for more details
+ # Alternatively see the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner
+ runners:
+ - name: direct
+ type: DirectRunner
+ options: {}
+
+ - name: dataflow
+ type: DataflowRunner
+ options:
+ project: my_gcp_project
+ region: asia-east1
+ zone: asia-east1-a
+ tempLocation: gs://bucket/tempLocation
+ network: default
+ subnetwork: regions/asia-east1/subnetworks/mysubnetwork
+ maxNumWorkers: 1
+ autoscalingAlgorithm: THROUGHPUT_BASED
+ usePublicIps: false
+ workerMachineType: n1-standard-1
+ deadLetterTableSpec: project_id:dataset_id.table_id
+
+ # Configuration options for metric collection for all ingestion jobs
metrics:
# Enable metrics pushing for all ingestion jobs.
enabled: false
@@ -49,9 +70,10 @@ feast:
# Feature stream type. Only kafka is supported.
type: kafka
# Feature stream options.
+ # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig
options:
topic: feast-features
- bootstrapServers: kafka:9092
+ bootstrapServers: localhost:9092
replicationFactor: 1
partitions: 1
diff --git a/core/src/test/java/feast/core/http/HealthControllerTest.java b/core/src/test/java/feast/core/http/HealthControllerTest.java
deleted file mode 100644
index 2fcd622f34..0000000000
--- a/core/src/test/java/feast/core/http/HealthControllerTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.http;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import javax.sql.DataSource;
-import org.junit.Test;
-import org.springframework.http.HttpStatus;
-import org.springframework.http.ResponseEntity;
-
-public class HealthControllerTest {
- @Test
- public void ping() {
- HealthController healthController = new HealthController(null);
- assertEquals(ResponseEntity.ok("pong"), healthController.ping());
- }
-
- @Test
- public void healthz() {
- assertEquals(ResponseEntity.ok("healthy"), mockHealthyController().healthz());
- assertEquals(
- ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body("Unable to establish connection with DB"),
- mockUnhealthyControllerBecauseInvalidConn().healthz());
- assertEquals(
- ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("mocked sqlexception"),
- mockUnhealthyControllerBecauseSQLException().healthz());
- }
-
- private HealthController mockHealthyController() {
- DataSource mockDataSource = mock(DataSource.class);
- Connection mockConnection = mock(Connection.class);
- try {
- when(mockConnection.isValid(any(int.class))).thenReturn(Boolean.TRUE);
- when(mockDataSource.getConnection()).thenReturn(mockConnection);
- } catch (Exception e) {
- e.printStackTrace();
- }
- return new HealthController(mockDataSource);
- }
-
- private HealthController mockUnhealthyControllerBecauseInvalidConn() {
- DataSource mockDataSource = mock(DataSource.class);
- Connection mockConnection = mock(Connection.class);
- try {
- when(mockConnection.isValid(any(int.class))).thenReturn(Boolean.FALSE);
- when(mockDataSource.getConnection()).thenReturn(mockConnection);
- } catch (Exception ignored) {
- }
- return new HealthController(mockDataSource);
- }
-
- private HealthController mockUnhealthyControllerBecauseSQLException() {
- DataSource mockDataSource = mock(DataSource.class);
- Connection mockConnection = mock(Connection.class);
- try {
- when(mockDataSource.getConnection()).thenThrow(new SQLException("mocked sqlexception"));
- } catch (SQLException ignored) {
- }
- return new HealthController(mockDataSource);
- }
-}
diff --git a/core/src/test/java/feast/core/job/JobUpdateTaskTest.java b/core/src/test/java/feast/core/job/JobUpdateTaskTest.java
index 2a1e80994a..d182673801 100644
--- a/core/src/test/java/feast/core/job/JobUpdateTaskTest.java
+++ b/core/src/test/java/feast/core/job/JobUpdateTaskTest.java
@@ -24,22 +24,24 @@
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.FeatureSetMeta;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.SourceProto;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
import feast.core.model.FeatureSet;
import feast.core.model.Job;
import feast.core.model.JobStatus;
import feast.core.model.Source;
import feast.core.model.Store;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.FeatureSetMeta;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.SourceProto;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.SourceType;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Optional;
import org.hamcrest.core.IsNull;
import org.junit.Before;
@@ -47,95 +49,69 @@
import org.mockito.Mock;
public class JobUpdateTaskTest {
+ private static final Runner RUNNER = Runner.DATAFLOW;
+
+ private static final FeatureSetProto.FeatureSet.Builder fsBuilder =
+ FeatureSetProto.FeatureSet.newBuilder().setMeta(FeatureSetMeta.newBuilder());
+ private static final FeatureSetSpec.Builder specBuilder = FeatureSetSpec.newBuilder();
@Mock private JobManager jobManager;
- private StoreProto.Store store;
- private SourceProto.Source source;
+ private Store store;
+ private Source source;
+ private FeatureSet featureSet1;
@Before
public void setUp() {
initMocks(this);
+ when(jobManager.getRunnerType()).thenReturn(RUNNER);
+
store =
- StoreProto.Store.newBuilder()
- .setName("test")
- .setType(StoreType.REDIS)
- .setRedisConfig(RedisConfig.newBuilder().build())
- .addSubscriptions(
- Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build())
- .build();
+ Store.fromProto(
+ StoreProto.Store.newBuilder()
+ .setName("test")
+ .setType(StoreType.REDIS)
+ .setRedisConfig(RedisConfig.newBuilder().build())
+ .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build())
+ .build());
source =
- SourceProto.Source.newBuilder()
- .setType(SourceType.KAFKA)
- .setKafkaSourceConfig(
- KafkaSourceConfig.newBuilder()
- .setTopic("topic")
- .setBootstrapServers("servers:9092")
- .build())
- .build();
+ Source.fromProto(
+ SourceProto.Source.newBuilder()
+ .setType(SourceType.KAFKA)
+ .setKafkaSourceConfig(
+ KafkaSourceConfig.newBuilder()
+ .setTopic("topic")
+ .setBootstrapServers("servers:9092")
+ .build())
+ .build());
+
+ featureSet1 =
+ FeatureSet.fromProto(fsBuilder.setSpec(specBuilder.setName("featureSet1")).build());
+ featureSet1.setSource(source);
+ }
+
+ Job makeJob(String extId, List featureSets, JobStatus status) {
+ return new Job("job", extId, RUNNER, source, store, featureSets, status);
+ }
+
+ JobUpdateTask makeTask(List featureSets, Optional currentJob) {
+ return new JobUpdateTask(featureSets, source, store, currentJob, jobManager, 100L);
}
@Test
public void shouldUpdateJobIfPresent() {
- FeatureSetProto.FeatureSet featureSet1 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet1")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
- FeatureSetProto.FeatureSet featureSet2 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet2")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
- Job originalJob =
- new Job(
- "job",
- "old_ext",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.RUNNING);
- JobUpdateTask jobUpdateTask =
- new JobUpdateTask(
- Arrays.asList(featureSet1, featureSet2),
- source,
- store,
- Optional.of(originalJob),
- jobManager,
- 100L);
- Job submittedJob =
- new Job(
- "job",
- "old_ext",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)),
- JobStatus.RUNNING);
+ FeatureSet featureSet2 =
+ FeatureSet.fromProto(fsBuilder.setSpec(specBuilder.setName("featureSet2")).build());
+ List existingFeatureSetsPopulatedByJob = Collections.singletonList(featureSet1);
+ List newFeatureSetsPopulatedByJob = Arrays.asList(featureSet1, featureSet2);
+
+ Job originalJob = makeJob("old_ext", existingFeatureSetsPopulatedByJob, JobStatus.RUNNING);
+ JobUpdateTask jobUpdateTask = makeTask(newFeatureSetsPopulatedByJob, Optional.of(originalJob));
+ Job submittedJob = makeJob("old_ext", newFeatureSetsPopulatedByJob, JobStatus.RUNNING);
- Job expected =
- new Job(
- "job",
- "new_ext",
- Runner.DATAFLOW.name(),
- Source.fromProto(source),
- Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)),
- JobStatus.PENDING);
+ Job expected = makeJob("new_ext", newFeatureSetsPopulatedByJob, JobStatus.PENDING);
when(jobManager.updateJob(submittedJob)).thenReturn(expected);
- when(jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW);
Job actual = jobUpdateTask.call();
assertThat(actual, equalTo(expected));
@@ -143,43 +119,13 @@ public void shouldUpdateJobIfPresent() {
@Test
public void shouldCreateJobIfNotPresent() {
- FeatureSetProto.FeatureSet featureSet1 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet1")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
- JobUpdateTask jobUpdateTask =
- spy(
- new JobUpdateTask(
- Arrays.asList(featureSet1), source, store, Optional.empty(), jobManager, 100L));
+ var featureSets = Collections.singletonList(featureSet1);
+ JobUpdateTask jobUpdateTask = spy(makeTask(featureSets, Optional.empty()));
doReturn("job").when(jobUpdateTask).createJobId("KAFKA/servers:9092/topic", "test");
- Job expectedInput =
- new Job(
- "job",
- "",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.PENDING);
+ Job expectedInput = makeJob("", featureSets, JobStatus.PENDING);
+ Job expected = makeJob("ext", featureSets, JobStatus.PENDING);
- Job expected =
- new Job(
- "job",
- "ext",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.RUNNING);
-
- when(jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW);
when(jobManager.startJob(expectedInput)).thenReturn(expected);
Job actual = jobUpdateTask.call();
@@ -188,83 +134,25 @@ public void shouldCreateJobIfNotPresent() {
@Test
public void shouldUpdateJobStatusIfNotCreateOrUpdate() {
- FeatureSetProto.FeatureSet featureSet1 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet1")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
- Job originalJob =
- new Job(
- "job",
- "ext",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.RUNNING);
- JobUpdateTask jobUpdateTask =
- new JobUpdateTask(
- Arrays.asList(featureSet1), source, store, Optional.of(originalJob), jobManager, 100L);
+ var featureSets = Collections.singletonList(featureSet1);
+ Job originalJob = makeJob("ext", featureSets, JobStatus.RUNNING);
+ JobUpdateTask jobUpdateTask = makeTask(featureSets, Optional.of(originalJob));
when(jobManager.getJobStatus(originalJob)).thenReturn(JobStatus.ABORTING);
- Job expected =
- new Job(
- "job",
- "ext",
- Runner.DATAFLOW.name(),
- Source.fromProto(source),
- Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.ABORTING);
- Job actual = jobUpdateTask.call();
+ Job updated = jobUpdateTask.call();
- assertThat(actual, equalTo(expected));
+ assertThat(updated.getStatus(), equalTo(JobStatus.ABORTING));
}
@Test
public void shouldReturnJobWithErrorStatusIfFailedToSubmit() {
- FeatureSetProto.FeatureSet featureSet1 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet1")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
- JobUpdateTask jobUpdateTask =
- spy(
- new JobUpdateTask(
- Arrays.asList(featureSet1), source, store, Optional.empty(), jobManager, 100L));
+ var featureSets = Collections.singletonList(featureSet1);
+ JobUpdateTask jobUpdateTask = spy(makeTask(featureSets, Optional.empty()));
doReturn("job").when(jobUpdateTask).createJobId("KAFKA/servers:9092/topic", "test");
- Job expectedInput =
- new Job(
- "job",
- "",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.PENDING);
-
- Job expected =
- new Job(
- "job",
- "",
- Runner.DATAFLOW.name(),
- feast.core.model.Source.fromProto(source),
- feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
- JobStatus.ERROR);
+ Job expectedInput = makeJob("", featureSets, JobStatus.PENDING);
+ Job expected = makeJob("", featureSets, JobStatus.ERROR);
- when(jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW);
when(jobManager.startJob(expectedInput))
.thenThrow(new RuntimeException("Something went wrong"));
@@ -274,21 +162,13 @@ public void shouldReturnJobWithErrorStatusIfFailedToSubmit() {
@Test
public void shouldTimeout() {
- FeatureSetProto.FeatureSet featureSet1 =
- FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setSource(source)
- .setProject("project1")
- .setName("featureSet1")
- .setVersion(1))
- .setMeta(FeatureSetMeta.newBuilder())
- .build();
-
+ var featureSets = Collections.singletonList(featureSet1);
+ var timeoutSeconds = 0L;
JobUpdateTask jobUpdateTask =
spy(
new JobUpdateTask(
- Arrays.asList(featureSet1), source, store, Optional.empty(), jobManager, 0L));
+ featureSets, source, store, Optional.empty(), jobManager, timeoutSeconds));
+
Job actual = jobUpdateTask.call();
assertThat(actual, is(IsNull.nullValue()));
}
diff --git a/core/src/test/java/feast/core/job/RunnerTest.java b/core/src/test/java/feast/core/job/RunnerTest.java
new file mode 100644
index 0000000000..ce1700acbe
--- /dev/null
+++ b/core/src/test/java/feast/core/job/RunnerTest.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.job;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.util.NoSuchElementException;
+import org.junit.Test;
+
+public class RunnerTest {
+
+ @Test
+ public void toStringReturnsHumanReadableName() {
+ assertThat(Runner.DATAFLOW.toString(), is("DataflowRunner"));
+ }
+
+ @Test
+ public void fromStringLoadsValueFromHumanReadableName() {
+ var humanName = Runner.DATAFLOW.toString();
+ assertThat(Runner.fromString(humanName), is(Runner.DATAFLOW));
+ }
+
+ @Test(expected = NoSuchElementException.class)
+ public void fromStringThrowsNoSuchElementExceptionForUnknownValue() {
+ Runner.fromString("this is not a valid Runner");
+ }
+}
diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
index 2d562d38df..ea9caa91ff 100644
--- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
+++ b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
@@ -22,21 +22,13 @@
import static org.mockito.Mockito.*;
import static org.mockito.MockitoAnnotations.initMocks;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential;
import com.google.api.services.dataflow.Dataflow;
import com.google.common.collect.Lists;
import com.google.protobuf.Duration;
import com.google.protobuf.util.JsonFormat;
import com.google.protobuf.util.JsonFormat.Printer;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.FeatureSetMeta;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.SourceProto;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
import feast.core.config.FeastProperties.MetricsProperties;
import feast.core.exception.JobExecutionException;
import feast.core.job.Runner;
@@ -45,11 +37,21 @@
import feast.ingestion.options.BZip2Compressor;
import feast.ingestion.options.ImportOptions;
import feast.ingestion.options.OptionCompressor;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.FeatureSetMeta;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions;
+import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions.Builder;
+import feast.proto.core.SourceProto;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.SourceType;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.beam.runners.dataflow.DataflowPipelineJob;
import org.apache.beam.runners.dataflow.DataflowRunner;
import org.apache.beam.sdk.PipelineResult.State;
@@ -68,18 +70,31 @@ public class DataflowJobManagerTest {
@Mock private Dataflow dataflow;
- private Map defaults;
+ private DataflowRunnerConfigOptions defaults;
private DataflowJobManager dfJobManager;
@Before
public void setUp() {
initMocks(this);
- defaults = new HashMap<>();
- defaults.put("project", "project");
- defaults.put("region", "region");
+ Builder optionsBuilder = DataflowRunnerConfigOptions.newBuilder();
+ optionsBuilder.setProject("project");
+ optionsBuilder.setRegion("region");
+ optionsBuilder.setZone("zone");
+ optionsBuilder.setTempLocation("tempLocation");
+ optionsBuilder.setNetwork("network");
+ optionsBuilder.setSubnetwork("subnetwork");
+ optionsBuilder.putLabels("orchestrator", "feast");
+ defaults = optionsBuilder.build();
MetricsProperties metricsProperties = new MetricsProperties();
metricsProperties.setEnabled(false);
- dfJobManager = new DataflowJobManager(dataflow, defaults, metricsProperties);
+ Credential credential = null;
+ try {
+ credential = MockGoogleCredential.getApplicationDefault();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ dfJobManager = new DataflowJobManager(defaults, metricsProperties, credential);
dfJobManager = spy(dfJobManager);
}
@@ -90,8 +105,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
.setName("SERVING")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build())
- .addSubscriptions(
- Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build())
+ .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build())
.build();
SourceProto.Source source =
@@ -111,7 +125,6 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
FeatureSetSpec.newBuilder()
.setSource(source)
.setName("featureSet")
- .setVersion(1)
.setMaxAge(Duration.newBuilder().build()))
.build();
@@ -126,6 +139,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
expectedPipelineOptions.setRegion("region");
expectedPipelineOptions.setUpdate(false);
expectedPipelineOptions.setAppName("DataflowJobManager");
+ expectedPipelineOptions.setLabels(defaults.getLabelsMap());
expectedPipelineOptions.setJobName(jobName);
expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store)));
@@ -145,7 +159,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
new Job(
jobName,
"",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
Source.fromProto(source),
Store.fromProto(store),
Lists.newArrayList(FeatureSet.fromProto(featureSet)),
@@ -159,7 +173,7 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
actualPipelineOptions.getOptionsId()); // avoid comparing this value
// We only check that we are calling getFilesToStage() manually, because the automatic approach
- // throws an error: https://github.com/gojek/feast/pull/291 i.e. do not check for the actual
+ // throws an error: https://github.com/feast-dev/feast/pull/291 i.e. do not check for the actual
// files that are staged
assertThat(
"filesToStage in pipelineOptions should not be null, job manager should set it.",
@@ -207,12 +221,7 @@ public void shouldThrowExceptionWhenJobStateTerminal() throws IOException {
FeatureSetProto.FeatureSet featureSet =
FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- FeatureSetSpec.newBuilder()
- .setName("featureSet")
- .setVersion(1)
- .setSource(source)
- .build())
+ .setSpec(FeatureSetSpec.newBuilder().setName("featureSet").setSource(source).build())
.build();
dfJobManager = Mockito.spy(dfJobManager);
@@ -226,7 +235,7 @@ public void shouldThrowExceptionWhenJobStateTerminal() throws IOException {
new Job(
"job",
"",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
Source.fromProto(source),
Store.fromProto(store),
Lists.newArrayList(FeatureSet.fromProto(featureSet)),
diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
index 76530d9f40..0128f5aa0b 100644
--- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
+++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
@@ -29,15 +29,6 @@
import com.google.protobuf.Duration;
import com.google.protobuf.util.JsonFormat;
import com.google.protobuf.util.JsonFormat.Printer;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.SourceProto;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
import feast.core.config.FeastProperties.MetricsProperties;
import feast.core.job.Runner;
import feast.core.job.option.FeatureSetJsonByteConverter;
@@ -49,11 +40,19 @@
import feast.ingestion.options.BZip2Compressor;
import feast.ingestion.options.ImportOptions;
import feast.ingestion.options.OptionCompressor;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.RunnerProto.DirectRunnerConfigOptions;
+import feast.proto.core.SourceProto;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.SourceType;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.beam.runners.direct.DirectRunner;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -71,12 +70,12 @@ public class DirectRunnerJobManagerTest {
@Mock private DirectJobRegistry directJobRegistry;
private DirectRunnerJobManager drJobManager;
- private Map defaults;
+ private DirectRunnerConfigOptions defaults;
@Before
public void setUp() {
initMocks(this);
- defaults = new HashMap<>();
+ defaults = DirectRunnerConfigOptions.newBuilder().setTargetParallelism(1).build();
MetricsProperties metricsProperties = new MetricsProperties();
metricsProperties.setEnabled(false);
@@ -91,8 +90,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
.setName("SERVING")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build())
- .addSubscriptions(
- Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build())
+ .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build())
.build();
SourceProto.Source source =
@@ -110,7 +108,6 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
.setSpec(
FeatureSetSpec.newBuilder()
.setName("featureSet")
- .setVersion(1)
.setMaxAge(Duration.newBuilder())
.setSource(source)
.build())
@@ -118,12 +115,14 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
Printer printer = JsonFormat.printer();
+ String expectedJobId = "feast-job-0";
ImportOptions expectedPipelineOptions =
PipelineOptionsFactory.fromArgs("").as(ImportOptions.class);
+ expectedPipelineOptions.setJobName(expectedJobId);
expectedPipelineOptions.setAppName("DirectRunnerJobManager");
expectedPipelineOptions.setRunner(DirectRunner.class);
expectedPipelineOptions.setBlockOnRun(false);
- expectedPipelineOptions.setProject("");
+ expectedPipelineOptions.setTargetParallelism(1);
expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store)));
expectedPipelineOptions.setProject("");
@@ -132,7 +131,6 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
expectedPipelineOptions.setFeatureSetJson(
featureSetJsonCompressor.compress(Collections.singletonList(featureSet)));
- String expectedJobId = "feast-job-0";
ArgumentCaptor pipelineOptionsCaptor =
ArgumentCaptor.forClass(ImportOptions.class);
ArgumentCaptor directJobCaptor = ArgumentCaptor.forClass(DirectJob.class);
@@ -144,7 +142,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
new Job(
expectedJobId,
"",
- Runner.DIRECT.name(),
+ Runner.DIRECT,
Source.fromProto(source),
Store.fromProto(store),
Lists.newArrayList(FeatureSet.fromProto(featureSet)),
diff --git a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java
index 2dfeef1d96..a12452b593 100644
--- a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java
+++ b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java
@@ -19,9 +19,9 @@
import static org.junit.Assert.*;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.FeatureSetProto;
-import feast.core.SourceProto;
-import feast.types.ValueProto;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.SourceProto;
+import feast.proto.types.ValueProto;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -29,7 +29,7 @@
public class FeatureSetJsonByteConverterTest {
- private FeatureSetProto.FeatureSet newFeatureSet(Integer version, Integer numberOfFeatures) {
+ private FeatureSetProto.FeatureSet newFeatureSet(Integer numberOfFeatures) {
List features =
IntStream.range(1, numberOfFeatures + 1)
.mapToObj(
@@ -51,7 +51,6 @@ private FeatureSetProto.FeatureSet newFeatureSet(Integer version, Integer number
.setBootstrapServers("somebrokers:9092")
.setTopic("sometopic")))
.addAllFeatures(features)
- .setVersion(version)
.addEntities(
FeatureSetProto.EntitySpec.newBuilder()
.setName("entity")
@@ -65,12 +64,11 @@ public void shouldConvertFeatureSetsAsJsonStringBytes() throws InvalidProtocolBu
int nrOfFeatures = 1;
List featureSets =
IntStream.range(1, nrOfFeatureSet + 1)
- .mapToObj(i -> newFeatureSet(i, nrOfFeatures))
+ .mapToObj(i -> newFeatureSet(nrOfFeatures))
.collect(Collectors.toList());
String expectedOutputString =
- "{\"version\":1,"
- + "\"entities\":[{\"name\":\"entity\",\"valueType\":2}],"
+ "{\"entities\":[{\"name\":\"entity\",\"valueType\":2}],"
+ "\"features\":[{\"name\":\"feature1\",\"valueType\":6}],"
+ "\"source\":{"
+ "\"type\":1,"
diff --git a/core/src/test/java/feast/core/model/FeatureSetTest.java b/core/src/test/java/feast/core/model/FeatureSetTest.java
new file mode 100644
index 0000000000..270dc3f3bc
--- /dev/null
+++ b/core/src/test/java/feast/core/model/FeatureSetTest.java
@@ -0,0 +1,205 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.model;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.protobuf.Duration;
+import com.google.protobuf.InvalidProtocolBufferException;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.EntitySpec;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.FeatureSetProto.FeatureSetStatus;
+import feast.proto.core.FeatureSetProto.FeatureSpec;
+import feast.proto.core.SourceProto;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.SourceType;
+import feast.proto.types.ValueProto.ValueType.Enum;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.tensorflow.metadata.v0.IntDomain;
+
+public class FeatureSetTest {
+ @Rule public final ExpectedException expectedException = ExpectedException.none();
+
+ private FeatureSetProto.FeatureSet oldFeatureSetProto;
+
+ @Before
+ public void setUp() {
+ SourceProto.Source oldSource =
+ SourceProto.Source.newBuilder()
+ .setType(SourceType.KAFKA)
+ .setKafkaSourceConfig(
+ KafkaSourceConfig.newBuilder()
+ .setBootstrapServers("kafka:9092")
+ .setTopic("mytopic"))
+ .build();
+
+ oldFeatureSetProto =
+ FeatureSetProto.FeatureSet.newBuilder()
+ .setSpec(
+ FeatureSetSpec.newBuilder()
+ .setName("featureSet")
+ .setProject("project")
+ .setMaxAge(Duration.newBuilder().setSeconds(100))
+ .setSource(oldSource)
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature1").setValueType(Enum.INT64))
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING))
+ .addEntities(
+ EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING))
+ .build())
+ .build();
+ }
+
+ @Test
+ public void shouldUpdateFromProto() throws InvalidProtocolBufferException {
+ SourceProto.Source newSource =
+ SourceProto.Source.newBuilder()
+ .setType(SourceType.KAFKA)
+ .setKafkaSourceConfig(
+ KafkaSourceConfig.newBuilder()
+ .setBootstrapServers("kafka:9092")
+ .setTopic("mytopic-changed"))
+ .build();
+
+ FeatureSetProto.FeatureSet newFeatureSetProto =
+ FeatureSetProto.FeatureSet.newBuilder()
+ .setSpec(
+ FeatureSetSpec.newBuilder()
+ .setName("featureSet")
+ .setProject("project")
+ .setMaxAge(Duration.newBuilder().setSeconds(101))
+ .setSource(newSource)
+ .addFeatures(
+ FeatureSpec.newBuilder()
+ .setName("feature1")
+ .setValueType(Enum.INT64)
+ .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0)))
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING))
+ .addEntities(
+ EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING))
+ .build())
+ .build();
+
+ FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto);
+ actual.updateFromProto(newFeatureSetProto);
+
+ FeatureSet expected = FeatureSet.fromProto(newFeatureSetProto);
+ Feature archivedFeature =
+ Feature.fromProto(
+ FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING).build());
+ archivedFeature.setArchived(true);
+ expected.addFeature(archivedFeature);
+ assertThat(actual, equalTo(expected));
+ }
+
+ @Test
+ public void shouldNotUpdateIfNoChange() throws InvalidProtocolBufferException {
+ FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto);
+ actual.setStatus(FeatureSetStatus.STATUS_READY);
+ actual.updateFromProto(oldFeatureSetProto);
+
+ FeatureSet expected = FeatureSet.fromProto(oldFeatureSetProto);
+ expected.setStatus(FeatureSetStatus.STATUS_READY);
+
+ assertThat(actual, equalTo(expected));
+ }
+
+ @Test
+ public void shouldThrowExceptionIfUpdateWithEntitiesChanged()
+ throws InvalidProtocolBufferException {
+ SourceProto.Source newSource =
+ SourceProto.Source.newBuilder()
+ .setType(SourceType.KAFKA)
+ .setKafkaSourceConfig(
+ KafkaSourceConfig.newBuilder()
+ .setBootstrapServers("kafka:9092")
+ .setTopic("mytopic-changed"))
+ .build();
+
+ FeatureSetProto.FeatureSet newFeatureSetProto =
+ FeatureSetProto.FeatureSet.newBuilder()
+ .setSpec(
+ FeatureSetSpec.newBuilder()
+ .setName("featureSet")
+ .setProject("project")
+ .setMaxAge(Duration.newBuilder().setSeconds(101))
+ .setSource(newSource)
+ .addFeatures(
+ FeatureSpec.newBuilder()
+ .setName("feature1")
+ .setValueType(Enum.INT64)
+ .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0)))
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING))
+ .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.FLOAT))
+ .build())
+ .build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(containsString("does not match existing set of entities"));
+ FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto);
+ existingFeatureSet.updateFromProto(newFeatureSetProto);
+ }
+
+ @Test
+ public void shouldThrowExceptionIfUpdateWithFeatureTypesChanged()
+ throws InvalidProtocolBufferException {
+ SourceProto.Source newSource =
+ SourceProto.Source.newBuilder()
+ .setType(SourceType.KAFKA)
+ .setKafkaSourceConfig(
+ KafkaSourceConfig.newBuilder()
+ .setBootstrapServers("kafka:9092")
+ .setTopic("mytopic-changed"))
+ .build();
+
+ FeatureSetProto.FeatureSet newFeatureSetProto =
+ FeatureSetProto.FeatureSet.newBuilder()
+ .setSpec(
+ FeatureSetSpec.newBuilder()
+ .setName("featureSet")
+ .setProject("project")
+ .setMaxAge(Duration.newBuilder().setSeconds(101))
+ .setSource(newSource)
+ .addFeatures(
+ FeatureSpec.newBuilder()
+ .setName("feature1")
+ .setValueType(Enum.INT64)
+ .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0)))
+ .addFeatures(
+ FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.FLOAT))
+ .addEntities(
+ EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING))
+ .build())
+ .build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ containsString(
+ "You are attempting to change the type of feature feature2 from STRING to FLOAT."));
+ FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto);
+ existingFeatureSet.updateFromProto(newFeatureSetProto);
+ }
+}
diff --git a/core/src/test/java/feast/core/model/JobStatusTest.java b/core/src/test/java/feast/core/model/JobStatusTest.java
new file mode 100644
index 0000000000..f5c8839386
--- /dev/null
+++ b/core/src/test/java/feast/core/model/JobStatusTest.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.model;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import org.junit.Test;
+
+public class JobStatusTest {
+
+ @Test
+ public void isTerminalReturnsTrueForJobStatusWithTerminalState() {
+ JobStatus.getTerminalStates()
+ .forEach(
+ status -> {
+ assertThat(status.isTerminal(), is(true));
+ assertThat(status.isTransitional(), is(false));
+ });
+ }
+
+ @Test
+ public void isTransitionalReturnsTrueForJobStatusWithTransitionalState() {
+ JobStatus.getTransitionalStates()
+ .forEach(
+ status -> {
+ assertThat(status.isTransitional(), is(true));
+ assertThat(status.isTerminal(), is(false));
+ });
+ }
+}
diff --git a/core/src/test/java/feast/core/service/AccessManagementServiceTest.java b/core/src/test/java/feast/core/service/AccessManagementServiceTest.java
new file mode 100644
index 0000000000..15be203709
--- /dev/null
+++ b/core/src/test/java/feast/core/service/AccessManagementServiceTest.java
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.service;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import feast.core.dao.ProjectRepository;
+import feast.core.model.Project;
+import java.util.Optional;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mock;
+
+public class AccessManagementServiceTest {
+ @Rule public ExpectedException expectedException = ExpectedException.none();
+ // mocks
+ @Mock private ProjectRepository projectRepository;
+ // dummy models
+ private Project defaultProject;
+ private Project testProject;
+
+ // test target
+ private AccessManagementService accessService;
+
+ @Before
+ public void setup() {
+ initMocks(this);
+ // setup dummy models for testing
+ this.defaultProject = new Project(Project.DEFAULT_NAME);
+ this.testProject = new Project("project");
+ // setup test target
+ when(this.projectRepository.existsById(Project.DEFAULT_NAME)).thenReturn(false);
+ this.accessService = new AccessManagementService(this.projectRepository);
+ }
+
+ @Test
+ public void testDefaultProjectCreateInConstructor() {
+ verify(this.projectRepository).saveAndFlush(this.defaultProject);
+ }
+
+ @Test
+ public void testArchiveProject() {
+ when(this.projectRepository.findById("project")).thenReturn(Optional.of(this.testProject));
+ this.accessService.archiveProject("project");
+ this.testProject.setArchived(true);
+ verify(this.projectRepository).saveAndFlush(this.testProject);
+ // reset archived flag
+ this.testProject.setArchived(false);
+ }
+
+ @Test
+ public void shouldNotArchiveDefaultProject() {
+ expectedException.expect(IllegalArgumentException.class);
+ this.accessService.archiveProject(Project.DEFAULT_NAME);
+ }
+}
diff --git a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
index aa71f201dd..8386efb28f 100644
--- a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
+++ b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
@@ -25,21 +25,10 @@
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
+import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest.Filter;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListStoresResponse;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.FeatureSetMeta;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.Source;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
-import feast.core.config.FeastProperties.JobUpdatesProperties;
+import feast.core.config.FeastProperties;
+import feast.core.config.FeastProperties.JobProperties;
import feast.core.dao.FeatureSetRepository;
import feast.core.dao.JobRepository;
import feast.core.job.JobManager;
@@ -48,7 +37,21 @@
import feast.core.model.FeatureSet;
import feast.core.model.Job;
import feast.core.model.JobStatus;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest.Filter;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListStoresResponse;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.FeatureSetMeta;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.SourceProto.KafkaSourceConfig;
+import feast.proto.core.SourceProto.Source;
+import feast.proto.core.SourceProto.SourceType;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
@@ -65,13 +68,15 @@ public class JobCoordinatorServiceTest {
@Mock SpecService specService;
@Mock FeatureSetRepository featureSetRepository;
- private JobUpdatesProperties jobUpdatesProperties;
+ private FeastProperties feastProperties;
@Before
public void setUp() {
initMocks(this);
- jobUpdatesProperties = new JobUpdatesProperties();
- jobUpdatesProperties.setTimeoutSeconds(5);
+ feastProperties = new FeastProperties();
+ JobProperties jobProperties = new JobProperties();
+ jobProperties.setJobUpdateTimeoutSeconds(5);
+ feastProperties.setJobs(jobProperties);
}
@Test
@@ -79,7 +84,7 @@ public void shouldDoNothingIfNoStoresFound() throws InvalidProtocolBufferExcepti
when(specService.listStores(any())).thenReturn(ListStoresResponse.newBuilder().build());
JobCoordinatorService jcs =
new JobCoordinatorService(
- jobRepository, featureSetRepository, specService, jobManager, jobUpdatesProperties);
+ jobRepository, featureSetRepository, specService, jobManager, feastProperties);
jcs.Poll();
verify(jobRepository, times(0)).saveAndFlush(any());
}
@@ -91,21 +96,16 @@ public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocol
.setName("test")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder().build())
- .addSubscriptions(
- Subscription.newBuilder().setProject("*").setName("*").setVersion("*").build())
+ .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build())
.build();
when(specService.listStores(any()))
.thenReturn(ListStoresResponse.newBuilder().addStore(store).build());
when(specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("*")
- .setFeatureSetName("*")
- .setFeatureSetVersion("*")
- .build()))
+ Filter.newBuilder().setProject("*").setFeatureSetName("*").build()))
.thenReturn(ListFeatureSetsResponse.newBuilder().build());
JobCoordinatorService jcs =
new JobCoordinatorService(
- jobRepository, featureSetRepository, specService, jobManager, jobUpdatesProperties);
+ jobRepository, featureSetRepository, specService, jobManager, feastProperties);
jcs.Poll();
verify(jobRepository, times(0)).saveAndFlush(any());
}
@@ -117,12 +117,7 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep
.setName("test")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder().build())
- .addSubscriptions(
- Subscription.newBuilder()
- .setProject("project1")
- .setName("features")
- .setVersion("*")
- .build())
+ .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("*").build())
.build();
Source source =
Source.newBuilder()
@@ -134,60 +129,51 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep
.build())
.build();
- FeatureSetProto.FeatureSet featureSet1 =
+ FeatureSetProto.FeatureSet featureSetProto1 =
FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetSpec.newBuilder()
.setSource(source)
.setProject("project1")
- .setName("features")
- .setVersion(1))
+ .setName("features1"))
.setMeta(FeatureSetMeta.newBuilder())
.build();
- FeatureSetProto.FeatureSet featureSet2 =
+ FeatureSet featureSet1 = FeatureSet.fromProto(featureSetProto1);
+ FeatureSetProto.FeatureSet featureSetProto2 =
FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetSpec.newBuilder()
.setSource(source)
.setProject("project1")
- .setName("features")
- .setVersion(2))
+ .setName("features2"))
.setMeta(FeatureSetMeta.newBuilder())
.build();
+ FeatureSet featureSet2 = FeatureSet.fromProto(featureSetProto2);
String extId = "ext";
- ArgumentCaptor jobArgCaptor = ArgumentCaptor.forClass(Job.class);
+ ArgumentCaptor> jobArgCaptor = ArgumentCaptor.forClass(List.class);
Job expectedInput =
new Job(
"",
"",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)),
+ Arrays.asList(featureSet1, featureSet2),
JobStatus.PENDING);
Job expected =
new Job(
"some_id",
extId,
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1), FeatureSet.fromProto(featureSet2)),
+ Arrays.asList(featureSet1, featureSet2),
JobStatus.RUNNING);
- when(specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("features")
- .setFeatureSetVersion("*")
- .build()))
- .thenReturn(
- ListFeatureSetsResponse.newBuilder()
- .addFeatureSets(featureSet1)
- .addFeatureSets(featureSet2)
- .build());
+ when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "project1"))
+ .thenReturn(Lists.newArrayList(featureSet1, featureSet2));
when(specService.listStores(any()))
.thenReturn(ListStoresResponse.newBuilder().addStore(store).build());
@@ -196,11 +182,11 @@ public void shouldGenerateAndSubmitJobsIfAny() throws InvalidProtocolBufferExcep
JobCoordinatorService jcs =
new JobCoordinatorService(
- jobRepository, featureSetRepository, specService, jobManager, jobUpdatesProperties);
+ jobRepository, featureSetRepository, specService, jobManager, feastProperties);
jcs.Poll();
- verify(jobRepository, times(1)).saveAndFlush(jobArgCaptor.capture());
- Job actual = jobArgCaptor.getValue();
- assertThat(actual, equalTo(expected));
+ verify(jobRepository, times(1)).saveAll(jobArgCaptor.capture());
+ List actual = jobArgCaptor.getValue();
+ assertThat(actual, equalTo(Collections.singletonList(expected)));
}
@Test
@@ -210,12 +196,7 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException {
.setName("test")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder().build())
- .addSubscriptions(
- Subscription.newBuilder()
- .setProject("project1")
- .setName("features")
- .setVersion("*")
- .build())
+ .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("*").build())
.build();
Source source1 =
Source.newBuilder()
@@ -236,79 +217,72 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException {
.build())
.build();
- FeatureSetProto.FeatureSet featureSet1 =
+ FeatureSetProto.FeatureSet featureSetProto1 =
FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetSpec.newBuilder()
.setSource(source1)
.setProject("project1")
- .setName("features")
- .setVersion(1))
+ .setName("features1"))
.setMeta(FeatureSetMeta.newBuilder())
.build();
- FeatureSetProto.FeatureSet featureSet2 =
+ FeatureSet featureSet1 = FeatureSet.fromProto(featureSetProto1);
+
+ FeatureSetProto.FeatureSet featureSetProto2 =
FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetSpec.newBuilder()
.setSource(source2)
.setProject("project1")
- .setName("features")
- .setVersion(2))
+ .setName("features2"))
.setMeta(FeatureSetMeta.newBuilder())
.build();
+ FeatureSet featureSet2 = FeatureSet.fromProto(featureSetProto2);
Job expectedInput1 =
new Job(
"name1",
"",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source1),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
+ Arrays.asList(featureSet1),
JobStatus.PENDING);
Job expected1 =
new Job(
"name1",
"extId1",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source1),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet1)),
+ Arrays.asList(featureSet1),
JobStatus.RUNNING);
Job expectedInput2 =
new Job(
"",
"extId2",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source2),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet2)),
+ Arrays.asList(featureSet2),
JobStatus.PENDING);
Job expected2 =
new Job(
"name2",
"extId2",
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
feast.core.model.Source.fromProto(source2),
feast.core.model.Store.fromProto(store),
- Arrays.asList(FeatureSet.fromProto(featureSet2)),
+ Arrays.asList(featureSet2),
JobStatus.RUNNING);
- ArgumentCaptor jobArgCaptor = ArgumentCaptor.forClass(Job.class);
+ ArgumentCaptor> jobArgCaptor = ArgumentCaptor.forClass(List.class);
+
+ when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "project1"))
+ .thenReturn(Lists.newArrayList(featureSet1, featureSet2));
- when(specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("features")
- .setFeatureSetVersion("*")
- .build()))
- .thenReturn(
- ListFeatureSetsResponse.newBuilder()
- .addFeatureSets(featureSet1)
- .addFeatureSets(featureSet2)
- .build());
when(specService.listStores(any()))
.thenReturn(ListStoresResponse.newBuilder().addStore(store).build());
@@ -318,11 +292,11 @@ public void shouldGroupJobsBySource() throws InvalidProtocolBufferException {
JobCoordinatorService jcs =
new JobCoordinatorService(
- jobRepository, featureSetRepository, specService, jobManager, jobUpdatesProperties);
+ jobRepository, featureSetRepository, specService, jobManager, feastProperties);
jcs.Poll();
- verify(jobRepository, times(2)).saveAndFlush(jobArgCaptor.capture());
- List actual = jobArgCaptor.getAllValues();
+ verify(jobRepository, times(1)).saveAll(jobArgCaptor.capture());
+ List actual = jobArgCaptor.getValue();
assertThat(actual.get(0), equalTo(expected1));
assertThat(actual.get(1), equalTo(expected2));
diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java
index c0e90ca43f..ff056287f9 100644
--- a/core/src/test/java/feast/core/service/JobServiceTest.java
+++ b/core/src/test/java/feast/core/service/JobServiceTest.java
@@ -26,31 +26,23 @@
import static org.mockito.MockitoAnnotations.initMocks;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListIngestionJobsRequest;
-import feast.core.CoreServiceProto.ListIngestionJobsResponse;
-import feast.core.CoreServiceProto.RestartIngestionJobRequest;
-import feast.core.CoreServiceProto.RestartIngestionJobResponse;
-import feast.core.CoreServiceProto.StopIngestionJobRequest;
-import feast.core.CoreServiceProto.StopIngestionJobResponse;
-import feast.core.FeatureSetProto.FeatureSetStatus;
-import feast.core.FeatureSetReferenceProto.FeatureSetReference;
-import feast.core.IngestionJobProto.IngestionJob;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
import feast.core.dao.JobRepository;
import feast.core.job.JobManager;
import feast.core.job.Runner;
-import feast.core.model.FeatureSet;
-import feast.core.model.Field;
-import feast.core.model.Job;
-import feast.core.model.JobStatus;
-import feast.core.model.Source;
-import feast.core.model.Store;
-import feast.types.ValueProto.ValueType.Enum;
+import feast.core.model.*;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsRequest;
+import feast.proto.core.CoreServiceProto.ListIngestionJobsResponse;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.RestartIngestionJobResponse;
+import feast.proto.core.CoreServiceProto.StopIngestionJobRequest;
+import feast.proto.core.CoreServiceProto.StopIngestionJobResponse;
+import feast.proto.core.FeatureSetReferenceProto.FeatureSetReference;
+import feast.proto.core.IngestionJobProto.IngestionJob;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.types.ValueProto.ValueType.Enum;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@@ -77,21 +69,13 @@ public class JobServiceTest {
// test target
public JobService jobService;
- /* unit test setup */
@Before
public void setup() {
initMocks(this);
// create mock objects for testing
// fake data source
- this.dataSource =
- new Source(
- SourceType.KAFKA,
- KafkaSourceConfig.newBuilder()
- .setBootstrapServers("kafka:9092")
- .setTopic("my-topic")
- .build(),
- true);
+ this.dataSource = TestObjectFactory.defaultSource;
// fake data store
this.dataStore =
new Store(
@@ -122,7 +106,6 @@ public void setup() {
new JobService(this.jobRepository, this.specService, Arrays.asList(this.jobManager));
}
- // setup fake spec service
public void setupSpecService() {
try {
ListFeatureSetsResponse response =
@@ -139,7 +122,6 @@ public void setupSpecService() {
}
}
- // setup fake job repository
public void setupJobRepository() {
when(this.jobRepository.findById(this.job.getId())).thenReturn(Optional.of(this.job));
when(this.jobRepository.findByStoreName(this.dataStore.getName()))
@@ -149,28 +131,19 @@ public void setupJobRepository() {
when(this.jobRepository.findAll()).thenReturn(Arrays.asList(this.job));
}
- // TODO: setup fake job manager
public void setupJobManager() {
when(this.jobManager.getRunnerType()).thenReturn(Runner.DATAFLOW);
when(this.jobManager.restartJob(this.job))
.thenReturn(this.newDummyJob(this.job.getId(), this.job.getExtId(), JobStatus.PENDING));
}
- // dummy model constructorss
private FeatureSet newDummyFeatureSet(String name, int version, String project) {
- Field feature = new Field(name + "_feature", Enum.INT64);
- Field entity = new Field(name + "_entity", Enum.STRING);
+ Feature feature = TestObjectFactory.CreateFeature(name + "_feature", Enum.INT64);
+ Entity entity = TestObjectFactory.CreateEntity(name + "_entity", Enum.STRING);
FeatureSet fs =
- new FeatureSet(
- name,
- project,
- version,
- 100L,
- Arrays.asList(entity),
- Arrays.asList(feature),
- this.dataSource,
- FeatureSetStatus.STATUS_READY);
+ TestObjectFactory.CreateFeatureSet(
+ name, project, Arrays.asList(entity), Arrays.asList(feature));
fs.setCreated(Date.from(Instant.ofEpochSecond(10L)));
return fs;
}
@@ -179,7 +152,7 @@ private Job newDummyJob(String id, String extId, JobStatus status) {
return new Job(
id,
extId,
- Runner.DATAFLOW.name(),
+ Runner.DATAFLOW,
this.dataSource,
this.dataStore,
Arrays.asList(this.featureSet),
@@ -190,7 +163,6 @@ private List newDummyFeatureSetReferences() {
return Arrays.asList(
// all provided: name, version and project
FeatureSetReference.newBuilder()
- .setVersion(this.featureSet.getVersion())
.setName(this.featureSet.getName())
.setProject(this.featureSet.getProject().toString())
.build(),
@@ -202,10 +174,7 @@ private List newDummyFeatureSetReferences() {
.build(),
// name and version
- FeatureSetReference.newBuilder()
- .setName(this.featureSet.getName())
- .setVersion(this.featureSet.getVersion())
- .build());
+ FeatureSetReference.newBuilder().setName(this.featureSet.getName()).build());
}
private List newDummyListRequestFilters() {
@@ -214,25 +183,21 @@ private List newDummyListRequestFilters() {
ListFeatureSetsRequest.Filter.newBuilder()
.setFeatureSetName(this.featureSet.getName())
.setProject(this.featureSet.getProject().toString())
- .setFeatureSetVersion(String.valueOf(this.featureSet.getVersion()))
.build(),
// name and project
ListFeatureSetsRequest.Filter.newBuilder()
.setFeatureSetName(this.featureSet.getName())
.setProject(this.featureSet.getProject().toString())
- .setFeatureSetVersion("*")
.build(),
// name and project
ListFeatureSetsRequest.Filter.newBuilder()
.setFeatureSetName(this.featureSet.getName())
.setProject("*")
- .setFeatureSetVersion(String.valueOf(this.featureSet.getVersion()))
.build());
}
- /* unit tests */
private ListIngestionJobsResponse tryListJobs(ListIngestionJobsRequest request) {
ListIngestionJobsResponse response = null;
try {
@@ -245,7 +210,6 @@ private ListIngestionJobsResponse tryListJobs(ListIngestionJobsRequest request)
return response;
}
- // list jobs
@Test
public void testListJobsById() {
ListIngestionJobsRequest.Filter filter =
@@ -304,7 +268,6 @@ public void testListIngestionJobByFeatureSetReference() {
assertThat(this.tryListJobs(request).getJobs(0), equalTo(this.ingestionJob));
}
- // stop jobs
private StopIngestionJobResponse tryStopJob(
StopIngestionJobRequest request, boolean expectError) {
StopIngestionJobResponse response = null;
@@ -341,10 +304,9 @@ public void testStopJobForId() {
}
@Test
- public void testStopAlreadyStop() {
+ public void testStopAlreadyStopped() {
// check that stop jobs does not trying to stop jobs that are not already stopped
- List doNothingStatuses = new ArrayList<>();
- doNothingStatuses.addAll(JobStatus.getTerminalState());
+ List doNothingStatuses = new ArrayList<>(JobStatus.getTerminalStates());
JobStatus prevStatus = this.job.getStatus();
for (JobStatus status : doNothingStatuses) {
diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java
index 43a66135dc..e584ee71e0 100644
--- a/core/src/test/java/feast/core/service/SpecServiceTest.java
+++ b/core/src/test/java/feast/core/service/SpecServiceTest.java
@@ -1,6 +1,6 @@
/*
* SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
+ * Copyright 2018-2020 The Feast Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -26,48 +26,34 @@
import com.google.api.client.util.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
-import feast.core.CoreServiceProto.ApplyFeatureSetResponse;
-import feast.core.CoreServiceProto.ApplyFeatureSetResponse.Status;
-import feast.core.CoreServiceProto.GetFeatureSetRequest;
-import feast.core.CoreServiceProto.GetFeatureSetResponse;
-import feast.core.CoreServiceProto.ListFeatureSetsRequest.Filter;
-import feast.core.CoreServiceProto.ListFeatureSetsResponse;
-import feast.core.CoreServiceProto.ListStoresRequest;
-import feast.core.CoreServiceProto.ListStoresResponse;
-import feast.core.CoreServiceProto.UpdateStoreRequest;
-import feast.core.CoreServiceProto.UpdateStoreResponse;
-import feast.core.FeatureSetProto;
-import feast.core.FeatureSetProto.EntitySpec;
-import feast.core.FeatureSetProto.FeatureSetSpec;
-import feast.core.FeatureSetProto.FeatureSetStatus;
-import feast.core.FeatureSetProto.FeatureSpec;
-import feast.core.SourceProto.KafkaSourceConfig;
-import feast.core.SourceProto.SourceType;
-import feast.core.StoreProto;
-import feast.core.StoreProto.Store.RedisConfig;
-import feast.core.StoreProto.Store.StoreType;
-import feast.core.StoreProto.Store.Subscription;
import feast.core.dao.FeatureSetRepository;
import feast.core.dao.ProjectRepository;
import feast.core.dao.StoreRepository;
import feast.core.exception.RetrievalException;
-import feast.core.model.FeatureSet;
-import feast.core.model.Field;
-import feast.core.model.Project;
-import feast.core.model.Source;
-import feast.core.model.Store;
-import feast.types.ValueProto.ValueType.Enum;
+import feast.core.model.*;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse.Status;
+import feast.proto.core.CoreServiceProto.GetFeatureSetRequest;
+import feast.proto.core.CoreServiceProto.GetFeatureSetResponse;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest.Filter;
+import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse;
+import feast.proto.core.CoreServiceProto.ListStoresRequest;
+import feast.proto.core.CoreServiceProto.ListStoresResponse;
+import feast.proto.core.CoreServiceProto.UpdateStoreRequest;
+import feast.proto.core.CoreServiceProto.UpdateStoreResponse;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.FeatureSetProto.EntitySpec;
+import feast.proto.core.FeatureSetProto.FeatureSetSpec;
+import feast.proto.core.FeatureSetProto.FeatureSpec;
+import feast.proto.core.StoreProto;
+import feast.proto.core.StoreProto.Store.RedisConfig;
+import feast.proto.core.StoreProto.Store.StoreType;
+import feast.proto.core.StoreProto.Store.Subscription;
+import feast.proto.types.ValueProto.ValueType.Enum;
import java.sql.Date;
import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Optional;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.Rule;
@@ -107,64 +93,39 @@ public class SpecServiceTest {
private List stores;
private Source defaultSource;
+ // TODO: Updates update features in place, so if tests follow the wrong order they might break.
+ // Refactor this maybe?
@Before
public void setUp() {
initMocks(this);
- defaultSource =
- new Source(
- SourceType.KAFKA,
- KafkaSourceConfig.newBuilder()
- .setBootstrapServers("kafka:9092")
- .setTopic("my-topic")
- .build(),
- true);
-
- FeatureSet featureSet1v1 = newDummyFeatureSet("f1", 1, "project1");
- FeatureSet featureSet1v2 = newDummyFeatureSet("f1", 2, "project1");
- FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1");
- FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1");
-
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
- FeatureSet featureSet3v1 =
- new FeatureSet(
- "f3",
- "project1",
- 1,
- 100L,
- Arrays.asList(f3e1),
- Arrays.asList(f3f2, f3f1),
- defaultSource,
- FeatureSetStatus.STATUS_READY);
-
- featureSets =
- Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1);
- when(featureSetRepository.findAll()).thenReturn(featureSets);
- when(featureSetRepository.findAllByOrderByNameAscVersionAsc()).thenReturn(featureSets);
+ defaultSource = TestObjectFactory.defaultSource;
+
+ FeatureSet featureSet1 = newDummyFeatureSet("f1", "project1");
+ FeatureSet featureSet2 = newDummyFeatureSet("f2", "project1");
+
+ Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64);
+ Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64);
+ Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING);
+ FeatureSet featureSet3 =
+ TestObjectFactory.CreateFeatureSet(
+ "f3", "project1", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1));
- when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 1))
+ FeatureSet featureSet4 = newDummyFeatureSet("f4", Project.DEFAULT_NAME);
+ featureSets = Arrays.asList(featureSet1, featureSet2, featureSet3, featureSet4);
+
+ when(featureSetRepository.findAll()).thenReturn(featureSets);
+ when(featureSetRepository.findAllByOrderByNameAsc()).thenReturn(featureSets);
+ when(featureSetRepository.findFeatureSetByNameAndProject_Name("f1", "project1"))
.thenReturn(featureSets.get(0));
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "f1", "project1"))
- .thenReturn(featureSets.subList(0, 3));
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "f3", "project1"))
- .thenReturn(featureSets.subList(4, 5));
- when(featureSetRepository.findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc(
- "f1", "project1"))
- .thenReturn(featureSet1v3);
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "f1", "project1"))
- .thenReturn(featureSets.subList(0, 3));
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "asd", "project1"))
+ when(featureSetRepository.findFeatureSetByNameAndProject_Name("f2", "project1"))
+ .thenReturn(featureSets.get(1));
+ when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("f1", "project1"))
+ .thenReturn(featureSets.subList(0, 1));
+ when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("asd", "project1"))
.thenReturn(Lists.newArrayList());
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "f%", "project1"))
+ when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("f%", "project1"))
.thenReturn(featureSets);
- when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc(
- "%", "%"))
+ when(featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc("%", "%"))
.thenReturn(featureSets);
when(projectRepository.findAllByArchivedIsFalse())
@@ -191,11 +152,7 @@ public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided()
throws InvalidProtocolBufferException {
ListFeatureSetsResponse actual =
specService.listFeatureSets(
- Filter.newBuilder()
- .setFeatureSetName("*")
- .setProject("*")
- .setFeatureSetVersion("*")
- .build());
+ Filter.newBuilder().setFeatureSetName("*").setProject("*").build());
List list = new ArrayList<>();
for (FeatureSet featureSet : featureSets) {
FeatureSetProto.FeatureSet toProto = featureSet.toProto();
@@ -206,48 +163,12 @@ public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided()
assertThat(actual, equalTo(expected));
}
- @Test
- public void listFeatureSetShouldFailIfFeatureSetProvidedWithoutProject()
- throws InvalidProtocolBufferException {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage(
- "Invalid listFeatureSetRequest, missing arguments. Must provide project, feature set name, and version.");
- specService.listFeatureSets(
- Filter.newBuilder().setFeatureSetName("f1").setFeatureSetVersion("1").build());
- }
-
- @Test
- public void shouldGetAllFeatureSetsMatchingNameIfWildcardVersionProvided()
- throws InvalidProtocolBufferException {
- ListFeatureSetsResponse actual =
- specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("f1")
- .setFeatureSetVersion("*")
- .build());
- List expectedFeatureSets =
- featureSets.stream().filter(fs -> fs.getName().equals("f1")).collect(Collectors.toList());
- List list = new ArrayList<>();
- for (FeatureSet expectedFeatureSet : expectedFeatureSets) {
- FeatureSetProto.FeatureSet toProto = expectedFeatureSet.toProto();
- list.add(toProto);
- }
- ListFeatureSetsResponse expected =
- ListFeatureSetsResponse.newBuilder().addAllFeatureSets(list).build();
- assertThat(actual, equalTo(expected));
- }
-
@Test
public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch()
throws InvalidProtocolBufferException {
ListFeatureSetsResponse actual =
specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("f*")
- .setFeatureSetVersion("*")
- .build());
+ Filter.newBuilder().setProject("project1").setFeatureSetName("f*").build());
List expectedFeatureSets =
featureSets.stream()
.filter(fs -> fs.getName().startsWith("f"))
@@ -263,20 +184,12 @@ public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch()
}
@Test
- public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator()
- throws InvalidProtocolBufferException {
+ public void shouldGetFeatureSetsByNameAndProject() throws InvalidProtocolBufferException {
ListFeatureSetsResponse actual =
specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("f1")
- .setFeatureSetVersion("1")
- .build());
+ Filter.newBuilder().setProject("project1").setFeatureSetName("f1").build());
List expectedFeatureSets =
- featureSets.stream()
- .filter(fs -> fs.getName().equals("f1"))
- .filter(fs -> fs.getVersion() == 1)
- .collect(Collectors.toList());
+ featureSets.stream().filter(fs -> fs.getName().equals("f1")).collect(Collectors.toList());
List list = new ArrayList<>();
for (FeatureSet expectedFeatureSet : expectedFeatureSets) {
FeatureSetProto.FeatureSet toProto = expectedFeatureSet.toProto();
@@ -287,80 +200,20 @@ public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator()
assertThat(actual, equalTo(expected));
}
- @Test
- public void shouldThrowExceptionIfGetAllFeatureSetsGivenVersionWithComparator()
- throws InvalidProtocolBufferException {
- expectedException.expect(IllegalArgumentException.class);
- specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("f1")
- .setFeatureSetVersion(">1")
- .build());
- }
-
- @Test
- public void shouldGetLatestFeatureSetGivenMissingVersionFilter()
- throws InvalidProtocolBufferException {
- GetFeatureSetResponse actual =
- specService.getFeatureSet(
- GetFeatureSetRequest.newBuilder().setName("f1").setProject("project1").build());
- FeatureSet expected = featureSets.get(2);
- assertThat(actual.getFeatureSet(), equalTo(expected.toProto()));
- }
-
- @Test
- public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter()
- throws InvalidProtocolBufferException {
- when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 2))
- .thenReturn(featureSets.get(1));
- GetFeatureSetResponse actual =
- specService.getFeatureSet(
- GetFeatureSetRequest.newBuilder()
- .setProject("project1")
- .setName("f1")
- .setVersion(2)
- .build());
- FeatureSet expected = featureSets.get(1);
- assertThat(actual.getFeatureSet(), equalTo(expected.toProto()));
- }
-
@Test
public void shouldThrowExceptionGivenMissingFeatureSetName()
throws InvalidProtocolBufferException {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("No feature set name provided");
- specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setVersion(2).build());
+ specService.getFeatureSet(GetFeatureSetRequest.newBuilder().build());
}
@Test
public void shouldThrowExceptionGivenMissingFeatureSet() throws InvalidProtocolBufferException {
expectedException.expect(RetrievalException.class);
- expectedException.expectMessage(
- "Feature set with name \"f1000\" and version \"2\" could not be found.");
+ expectedException.expectMessage("Feature set with name \"f1000\" could not be found.");
specService.getFeatureSet(
- GetFeatureSetRequest.newBuilder()
- .setName("f1000")
- .setProject("project1")
- .setVersion(2)
- .build());
- }
-
- @Test
- public void shouldThrowRetrievalExceptionGivenInvalidFeatureSetVersionComparator()
- throws InvalidProtocolBufferException {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage(
- "Invalid listFeatureSetRequest. Version must be set to \"*\" if the project name and feature set name aren't set explicitly: \n"
- + "feature_set_name: \"f1\"\n"
- + "feature_set_version: \">1\"\n"
- + "project: \"project1\"");
- specService.listFeatureSets(
- Filter.newBuilder()
- .setProject("project1")
- .setFeatureSetName("f1")
- .setFeatureSetVersion(">1")
- .build());
+ GetFeatureSetRequest.newBuilder().setName("f1000").setProject("project1").build());
}
@Test
@@ -395,10 +248,10 @@ public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() {
}
@Test
- public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHasNotChanged()
+ public void applyFeatureSetShouldReturnFeatureSetIfFeatureSetHasNotChanged()
throws InvalidProtocolBufferException {
FeatureSetSpec incomingFeatureSetSpec =
- featureSets.get(2).toProto().getSpec().toBuilder().clearVersion().build();
+ featureSets.get(0).toProto().getSpec().toBuilder().build();
ApplyFeatureSetResponse applyFeatureSetResponse =
specService.applyFeatureSet(
@@ -406,21 +259,19 @@ public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHa
verify(featureSetRepository, times(0)).save(ArgumentMatchers.any(FeatureSet.class));
assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.NO_CHANGE));
- assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(featureSets.get(2).toProto()));
+ assertThat(applyFeatureSetResponse.getFeatureSet(), equalTo(featureSets.get(0).toProto()));
}
@Test
- public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists()
+ public void applyFeatureSetShouldApplyFeatureSetIfNotExists()
throws InvalidProtocolBufferException {
- when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
- "f2", "project1"))
- .thenReturn(Lists.newArrayList());
+ when(featureSetRepository.findFeatureSetByNameAndProject_Name("f2", "project1"))
+ .thenReturn(null);
- FeatureSetProto.FeatureSet incomingFeatureSet =
- newDummyFeatureSet("f2", 1, "project1").toProto();
+ FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", "project1").toProto();
FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec =
- incomingFeatureSet.getSpec().toBuilder().clearVersion().build();
+ incomingFeatureSet.getSpec().toBuilder().build();
ApplyFeatureSetResponse applyFeatureSetResponse =
specService.applyFeatureSet(
@@ -429,24 +280,16 @@ public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists()
FeatureSetProto.FeatureSet expected =
FeatureSetProto.FeatureSet.newBuilder()
- .setSpec(
- incomingFeatureSetSpec
- .toBuilder()
- .setVersion(1)
- .setSource(defaultSource.toProto())
- .build())
+ .setSpec(incomingFeatureSetSpec.toBuilder().setSource(defaultSource.toProto()).build())
.build();
assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.CREATED));
assertThat(applyFeatureSetResponse.getFeatureSet().getSpec(), equalTo(expected.getSpec()));
- assertThat(
- applyFeatureSetResponse.getFeatureSet().getSpec().getVersion(),
- equalTo(expected.getSpec().getVersion()));
}
@Test
- public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
+ public void applyFeatureSetShouldUpdateAndSaveFeatureSetIfAlreadyExists()
throws InvalidProtocolBufferException {
- FeatureSetProto.FeatureSet incomingFeatureSet = featureSets.get(2).toProto();
+ FeatureSetProto.FeatureSet incomingFeatureSet = featureSets.get(0).toProto();
incomingFeatureSet =
incomingFeatureSet
.toBuilder()
@@ -455,7 +298,6 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
incomingFeatureSet
.getSpec()
.toBuilder()
- .clearVersion()
.addFeatures(
FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING))
.build())
@@ -466,44 +308,27 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
.toBuilder()
.setMeta(incomingFeatureSet.getMeta().toBuilder().build())
.setSpec(
- incomingFeatureSet
- .getSpec()
- .toBuilder()
- .setVersion(4)
- .setSource(defaultSource.toProto())
- .build())
+ incomingFeatureSet.getSpec().toBuilder().setSource(defaultSource.toProto()).build())
.build();
ApplyFeatureSetResponse applyFeatureSetResponse =
specService.applyFeatureSet(incomingFeatureSet);
verify(projectRepository).saveAndFlush(ArgumentMatchers.any(Project.class));
- assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.CREATED));
+ assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.UPDATED));
assertEquals(
FeatureSet.fromProto(applyFeatureSetResponse.getFeatureSet()),
FeatureSet.fromProto(expected));
- assertThat(
- applyFeatureSetResponse.getFeatureSet().getSpec().getVersion(),
- equalTo(expected.getSpec().getVersion()));
}
@Test
public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
- FeatureSetProto.FeatureSet incomingFeatureSet =
- (new FeatureSet(
- "f3",
- "project1",
- 5,
- 100L,
- Arrays.asList(f3e1),
- Arrays.asList(f3f2, f3f1),
- defaultSource,
- FeatureSetStatus.STATUS_READY))
- .toProto();
+ FeatureSet featureSet = featureSets.get(1);
+ List features = Lists.newArrayList(featureSet.getFeatures());
+ Collections.shuffle(features);
+ featureSet.setFeatures(Set.copyOf(features));
+ FeatureSetProto.FeatureSet incomingFeatureSet = featureSet.toProto();
ApplyFeatureSetResponse applyFeatureSetResponse =
specService.applyFeatureSet(incomingFeatureSet);
@@ -523,46 +348,11 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
throws InvalidProtocolBufferException {
List entitySpecs = new ArrayList<>();
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity1")
- .setValueType(Enum.INT64)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setShape(FixedShape.getDefaultInstance())
- .setDomain("mydomain")
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity2")
- .setValueType(Enum.INT64)
- .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setIntDomain(IntDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity3")
- .setValueType(Enum.FLOAT)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setFloatDomain(FloatDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity4")
- .setValueType(Enum.STRING)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setStringDomain(StringDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity5")
- .setValueType(Enum.BOOL)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setBoolDomain(BoolDomain.getDefaultInstance())
- .build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity2").setValueType(Enum.INT64).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity3").setValueType(Enum.FLOAT).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity4").setValueType(Enum.STRING).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity5").setValueType(Enum.BOOL).build());
List featureSpecs = new ArrayList<>();
featureSpecs.add(
@@ -630,27 +420,13 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
new ArrayList<>(appliedFeatureSetSpec.getFeaturesList());
appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName));
- assertEquals(appliedEntitySpecs.size(), entitySpecs.size());
- assertEquals(appliedFeatureSpecs.size(), featureSpecs.size());
-
- for (int i = 0; i < appliedEntitySpecs.size(); i++) {
- assertEquals(entitySpecs.get(i), appliedEntitySpecs.get(i));
- }
-
- for (int i = 0; i < appliedFeatureSpecs.size(); i++) {
- assertEquals(featureSpecs.get(i), appliedFeatureSpecs.get(i));
- }
+ assertEquals(appliedEntitySpecs, entitySpecs);
+ assertEquals(appliedFeatureSpecs, featureSpecs);
}
@Test
public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
throws InvalidProtocolBufferException {
- FeatureSetProto.FeatureSet existingFeatureSet = featureSets.get(2).toProto();
- assertThat(
- "Existing feature set has version 3", existingFeatureSet.getSpec().getVersion() == 3);
- assertThat(
- "Existing feature set has at least 1 feature",
- existingFeatureSet.getSpec().getFeaturesList().size() > 0);
// Map of constraint field name -> value, e.g. "shape" -> FixedShape object.
// If any of these fields are updated, SpecService should update the FeatureSet.
@@ -675,6 +451,10 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
contraintUpdates.put("time_of_day_domain", TimeOfDayDomain.getDefaultInstance());
for (Entry constraint : contraintUpdates.entrySet()) {
+ FeatureSet featureSet = newDummyFeatureSet("constraints", "project1");
+ FeatureSetProto.FeatureSet existingFeatureSet = featureSet.toProto();
+ when(featureSetRepository.findFeatureSetByNameAndProject_Name("constraints", "project1"))
+ .thenReturn(featureSet);
String name = constraint.getKey();
Object value = constraint.getValue();
FeatureSpec newFeatureSpec =
@@ -693,12 +473,8 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
assertEquals(
"Response should have CREATED status when field '" + name + "' is updated",
- Status.CREATED,
+ Status.UPDATED,
response.getStatus());
- assertEquals(
- "FeatureSet should have new version when field '" + name + "' is updated",
- existingFeatureSet.getSpec().getVersion() + 1,
- response.getFeatureSet().getSpec().getVersion());
assertEquals(
"Feature should have field '" + name + "' set correctly",
constraint.getValue(),
@@ -713,19 +489,12 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
@Test
public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64);
+ Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64);
+ Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING);
FeatureSetProto.FeatureSet incomingFeatureSet =
- (new FeatureSet(
- "f3",
- "newproject",
- 5,
- 100L,
- Arrays.asList(f3e1),
- Arrays.asList(f3f2, f3f1),
- defaultSource,
- FeatureSetStatus.STATUS_READY))
+ TestObjectFactory.CreateFeatureSet(
+ "f3", "project", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))
.toProto();
ApplyFeatureSetResponse applyFeatureSetResponse =
@@ -736,22 +505,35 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists()
equalTo(incomingFeatureSet.getSpec().getProject()));
}
+ @Test
+ public void applyFeatureSetShouldUsedDefaultProjectIfUnspecified()
+ throws InvalidProtocolBufferException {
+ Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64);
+ Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64);
+ Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING);
+
+ // In protov3, unspecified project defaults to ""
+ FeatureSetProto.FeatureSet incomingFeatureSet =
+ TestObjectFactory.CreateFeatureSet("f3", "", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))
+ .toProto();
+ ApplyFeatureSetResponse applyFeatureSetResponse =
+ specService.applyFeatureSet(incomingFeatureSet);
+ assertThat(applyFeatureSetResponse.getStatus(), equalTo(Status.CREATED));
+
+ assertThat(
+ applyFeatureSetResponse.getFeatureSet().getSpec().getProject(),
+ equalTo(Project.DEFAULT_NAME));
+ }
+
@Test
public void applyFeatureSetShouldFailWhenProjectIsArchived()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = TestObjectFactory.CreateFeature("f3f1", Enum.INT64);
+ Feature f3f2 = TestObjectFactory.CreateFeature("f3f2", Enum.INT64);
+ Entity f3e1 = TestObjectFactory.CreateEntity("f3e1", Enum.STRING);
FeatureSetProto.FeatureSet incomingFeatureSet =
- (new FeatureSet(
- "f3",
- "archivedproject",
- 5,
- 100L,
- Arrays.asList(f3e1),
- Arrays.asList(f3f2, f3f1),
- defaultSource,
- FeatureSetStatus.STATUS_READY))
+ TestObjectFactory.CreateFeatureSet(
+ "f3", "archivedproject", Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))
.toProto();
expectedException.expect(IllegalArgumentException.class);
@@ -759,6 +541,101 @@ public void applyFeatureSetShouldFailWhenProjectIsArchived()
specService.applyFeatureSet(incomingFeatureSet);
}
+ @Test
+ public void applyFeatureSetShouldAcceptFeatureLabels() throws InvalidProtocolBufferException {
+ List entitySpecs = new ArrayList<>();
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build());
+
+ Map featureLabels0 =
+ new HashMap<>() {
+ {
+ put("label1", "feast1");
+ }
+ };
+
+ Map featureLabels1 =
+ new HashMap<>() {
+ {
+ put("label1", "feast1");
+ put("label2", "feast2");
+ }
+ };
+
+ List> featureLabels = new ArrayList<>();
+ featureLabels.add(featureLabels0);
+ featureLabels.add(featureLabels1);
+
+ List featureSpecs = new ArrayList<>();
+ featureSpecs.add(
+ FeatureSpec.newBuilder()
+ .setName("feature1")
+ .setValueType(Enum.INT64)
+ .putAllLabels(featureLabels.get(0))
+ .build());
+ featureSpecs.add(
+ FeatureSpec.newBuilder()
+ .setName("feature2")
+ .setValueType(Enum.INT64)
+ .putAllLabels(featureLabels.get(1))
+ .build());
+
+ FeatureSetSpec featureSetSpec =
+ FeatureSetSpec.newBuilder()
+ .setProject("project1")
+ .setName("featureSetWithConstraints")
+ .addAllEntities(entitySpecs)
+ .addAllFeatures(featureSpecs)
+ .build();
+ FeatureSetProto.FeatureSet featureSet =
+ FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build();
+
+ ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(featureSet);
+ FeatureSetSpec appliedFeatureSetSpec = applyFeatureSetResponse.getFeatureSet().getSpec();
+
+ // appliedEntitySpecs needs to be sorted because the list returned by specService may not
+ // follow the order in the request
+ List appliedEntitySpecs = new ArrayList<>(appliedFeatureSetSpec.getEntitiesList());
+ appliedEntitySpecs.sort(Comparator.comparing(EntitySpec::getName));
+
+ // appliedFeatureSpecs needs to be sorted because the list returned by specService may not
+ // follow the order in the request
+ List appliedFeatureSpecs =
+ new ArrayList<>(appliedFeatureSetSpec.getFeaturesList());
+ appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName));
+
+ var featureSpecsLabels =
+ featureSpecs.stream().map(e -> e.getLabelsMap()).collect(Collectors.toList());
+ assertEquals(appliedEntitySpecs, entitySpecs);
+ assertEquals(appliedFeatureSpecs, featureSpecs);
+ assertEquals(featureSpecsLabels, featureLabels);
+ }
+
+ @Test
+ public void applyFeatureSetShouldAcceptFeatureSetLabels() throws InvalidProtocolBufferException {
+ Map featureSetLabels =
+ new HashMap<>() {
+ {
+ put("description", "My precious feature set");
+ }
+ };
+
+ FeatureSetSpec featureSetSpec =
+ FeatureSetSpec.newBuilder()
+ .setProject("project1")
+ .setName("preciousFeatureSet")
+ .putAllLabels(featureSetLabels)
+ .build();
+ FeatureSetProto.FeatureSet featureSet =
+ FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build();
+
+ ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(featureSet);
+ FeatureSetSpec appliedFeatureSetSpec = applyFeatureSetResponse.getFeatureSet().getSpec();
+
+ var appliedLabels = appliedFeatureSetSpec.getLabelsMap();
+
+ assertEquals(featureSetLabels, appliedLabels);
+ }
+
@Test
public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException {
when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0)));
@@ -767,8 +644,7 @@ public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferExcep
.setName("SERVING")
.setType(StoreType.REDIS)
.setRedisConfig(RedisConfig.newBuilder())
- .addSubscriptions(
- Subscription.newBuilder().setProject("project1").setName("a").setVersion(">1"))
+ .addSubscriptions(Subscription.newBuilder().setProject("project1").setName("a"))
.build();
UpdateStoreResponse actual =
specService.updateStore(UpdateStoreRequest.newBuilder().setStore(newStore).build());
@@ -799,26 +675,35 @@ public void shouldDoNothingIfNoChange() throws InvalidProtocolBufferException {
}
@Test
- public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBufferException {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("No project provided");
- specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1").build());
+ public void getOrListFeatureSetShouldUseDefaultProjectIfProjectUnspecified()
+ throws InvalidProtocolBufferException {
+ when(featureSetRepository.findFeatureSetByNameAndProject_Name("f4", Project.DEFAULT_NAME))
+ .thenReturn(featureSets.get(3));
+ FeatureSet expected = featureSets.get(3);
+ // check getFeatureSet()
+ GetFeatureSetResponse getResponse =
+ specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f4").build());
+ assertThat(getResponse.getFeatureSet(), equalTo(expected.toProto()));
+
+ // check listFeatureSets()
+ ListFeatureSetsResponse listResponse =
+ specService.listFeatureSets(Filter.newBuilder().setFeatureSetName("f4").build());
+ assertThat(listResponse.getFeatureSetsList(), equalTo(Arrays.asList(expected.toProto())));
}
- private FeatureSet newDummyFeatureSet(String name, int version, String project) {
- Field feature = new Field("feature", Enum.INT64);
- Field entity = new Field("entity", Enum.STRING);
+ private FeatureSet newDummyFeatureSet(String name, String project) {
+ FeatureSpec f1 =
+ FeatureSpec.newBuilder()
+ .setName("feature")
+ .setValueType(Enum.STRING)
+ .putLabels("key", "value")
+ .build();
+ Feature feature = Feature.fromProto(f1);
+ Entity entity = TestObjectFactory.CreateEntity("entity", Enum.STRING);
FeatureSet fs =
- new FeatureSet(
- name,
- project,
- version,
- 100L,
- Arrays.asList(entity),
- Arrays.asList(feature),
- defaultSource,
- FeatureSetStatus.STATUS_READY);
+ TestObjectFactory.CreateFeatureSet(
+ name, project, Arrays.asList(entity), Arrays.asList(feature));
fs.setCreated(Date.from(Instant.ofEpochSecond(10L)));
return fs;
}
@@ -828,7 +713,7 @@ private Store newDummyStore(String name) {
Store store = new Store();
store.setName(name);
store.setType(StoreType.REDIS.toString());
- store.setSubscriptions("*:*:*");
+ store.setSubscriptions("*:*");
store.setConfig(RedisConfig.newBuilder().setPort(6379).build().toByteArray());
return store;
}
diff --git a/core/src/test/java/feast/core/service/TestObjectFactory.java b/core/src/test/java/feast/core/service/TestObjectFactory.java
new file mode 100644
index 0000000000..40c379d3cf
--- /dev/null
+++ b/core/src/test/java/feast/core/service/TestObjectFactory.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.service;
+
+import feast.core.model.Entity;
+import feast.core.model.Feature;
+import feast.core.model.FeatureSet;
+import feast.core.model.Source;
+import feast.proto.core.FeatureSetProto;
+import feast.proto.core.SourceProto;
+import feast.proto.types.ValueProto;
+import java.util.HashMap;
+import java.util.List;
+
+public class TestObjectFactory {
+
+ public static Source defaultSource =
+ new Source(
+ SourceProto.SourceType.KAFKA,
+ SourceProto.KafkaSourceConfig.newBuilder()
+ .setBootstrapServers("kafka:9092")
+ .setTopic("my-topic")
+ .build(),
+ true);
+
+ public static FeatureSet CreateFeatureSet(
+ String name, String project, List entities, List features) {
+ return new FeatureSet(
+ name,
+ project,
+ 100L,
+ entities,
+ features,
+ defaultSource,
+ new HashMap<>(),
+ FeatureSetProto.FeatureSetStatus.STATUS_READY);
+ }
+
+ public static Feature CreateFeature(String name, ValueProto.ValueType.Enum valueType) {
+ return Feature.fromProto(
+ FeatureSetProto.FeatureSpec.newBuilder().setName(name).setValueType(valueType).build());
+ }
+
+ public static Entity CreateEntity(String name, ValueProto.ValueType.Enum valueType) {
+ return Entity.fromProto(
+ FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build());
+ }
+}
diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java
index c84a252cd0..dd858082ba 100644
--- a/core/src/test/java/feast/core/util/TypeConversionTest.java
+++ b/core/src/test/java/feast/core/util/TypeConversionTest.java
@@ -19,16 +19,10 @@
import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import com.google.protobuf.Timestamp;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.junit.Test;
public class TypeConversionTest {
@@ -84,14 +78,8 @@ public void convertMapToJsonStringShouldReturnJsonStringForGivenMap() {
}
@Test
- public void convertJsonStringToArgsShouldReturnCorrectListOfArgs() {
+ public void convertMapToJsonStringShouldReturnEmptyJsonForAnEmptyMap() {
Map input = new HashMap<>();
- input.put("key", "value");
- input.put("key2", "value2");
-
- String[] expected = new String[] {"--key=value", "--key2=value2"};
- String[] actual = TypeConversion.convertMapToArgs(input);
- assertThat(actual.length, equalTo(expected.length));
- assertTrue(Arrays.asList(actual).containsAll(Arrays.asList(expected)));
+ assertThat(TypeConversion.convertMapToJsonString(input), equalTo("{}"));
}
}
diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java
new file mode 100644
index 0000000000..155a52d100
--- /dev/null
+++ b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java
@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.validators;
+
+import feast.proto.core.FeatureSetProto;
+import feast.proto.types.ValueProto;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class FeatureSetValidatorTest {
+
+ @Rule public final ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() {
+ Map featureLabels =
+ new HashMap<>() {
+ {
+ put("", "empty_key");
+ }
+ };
+
+ List featureSpecs = new ArrayList<>();
+ featureSpecs.add(
+ FeatureSetProto.FeatureSpec.newBuilder()
+ .setName("feature1")
+ .setValueType(ValueProto.ValueType.Enum.INT64)
+ .putAllLabels(featureLabels)
+ .build());
+
+ FeatureSetProto.FeatureSetSpec featureSetSpec =
+ FeatureSetProto.FeatureSetSpec.newBuilder()
+ .setProject("project1")
+ .setName("featureSetWithConstraints")
+ .addAllFeatures(featureSpecs)
+ .build();
+ FeatureSetProto.FeatureSet featureSet =
+ FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Feature label keys must not be empty");
+ FeatureSetValidator.validateSpec(featureSet);
+ }
+
+ @Test
+ public void shouldThrowExceptionForFeatureSetLabelsWithAnEmptyKey() {
+
+ Map featureSetLabels =
+ new HashMap<>() {
+ {
+ put("", "empty_key");
+ }
+ };
+
+ FeatureSetProto.FeatureSetSpec featureSetSpec =
+ FeatureSetProto.FeatureSetSpec.newBuilder()
+ .setProject("project1")
+ .setName("featureSetWithConstraints")
+ .putAllLabels(featureSetLabels)
+ .build();
+ FeatureSetProto.FeatureSet featureSet =
+ FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Feature set label keys must not be empty");
+ FeatureSetValidator.validateSpec(featureSet);
+ }
+}
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
index 535fac73d2..f87a50f1bd 100644
--- a/datatypes/java/README.md
+++ b/datatypes/java/README.md
@@ -52,4 +52,4 @@ For developers, the publishing process is automated along with the Java SDK by
it works. Artifacts are staged to Sonatype where a maintainer needs to take a
release action for them to go live on Maven Central.
-[prow task]: https://github.com/gojek/feast/blob/17e7dca8238aae4dcbf0ff9f0db5d80ef8e035cf/.prow/config.yaml#L166-L192
+[prow task]: https://github.com/feast-dev/feast/blob/17e7dca8238aae4dcbf0ff9f0db5d80ef8e035cf/.prow/config.yaml#L166-L192
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
index a127853258..5810a6db96 100644
--- a/datatypes/java/pom.xml
+++ b/datatypes/java/pom.xml
@@ -37,15 +37,6 @@
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
-
- 8
-
-
-
org.apache.maven.pluginsmaven-dependency-plugin
diff --git a/docs/.gitbook/assets/architecture.png b/docs/.gitbook/assets/architecture.png
new file mode 100644
index 0000000000..bc655b60f3
Binary files /dev/null and b/docs/.gitbook/assets/architecture.png differ
diff --git a/docs/.gitbook/assets/basic-architecture-diagram (2).svg b/docs/.gitbook/assets/basic-architecture-diagram (2).svg
new file mode 100644
index 0000000000..b707f49046
--- /dev/null
+++ b/docs/.gitbook/assets/basic-architecture-diagram (2).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/basic-architecture-diagram (3).svg b/docs/.gitbook/assets/basic-architecture-diagram (3).svg
new file mode 100644
index 0000000000..b707f49046
--- /dev/null
+++ b/docs/.gitbook/assets/basic-architecture-diagram (3).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/blank-diagram-4 (1).svg b/docs/.gitbook/assets/blank-diagram-4 (1).svg
new file mode 100644
index 0000000000..fb5e0659e5
--- /dev/null
+++ b/docs/.gitbook/assets/blank-diagram-4 (1).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/blank-diagram-4.svg b/docs/.gitbook/assets/blank-diagram-4.svg
new file mode 100644
index 0000000000..fb5e0659e5
--- /dev/null
+++ b/docs/.gitbook/assets/blank-diagram-4.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (2).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (2).svg
new file mode 100644
index 0000000000..7f30963ec7
--- /dev/null
+++ b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (2).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3) (1).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3) (1).svg
new file mode 100644
index 0000000000..7f30963ec7
--- /dev/null
+++ b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3) (1).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3).svg
new file mode 100644
index 0000000000..7f30963ec7
--- /dev/null
+++ b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3).svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/.gitbook/assets/image (2) (1).png b/docs/.gitbook/assets/image (2) (1).png
new file mode 100644
index 0000000000..d3b359a598
Binary files /dev/null and b/docs/.gitbook/assets/image (2) (1).png differ
diff --git a/docs/.gitbook/assets/image (2).png b/docs/.gitbook/assets/image (2).png
new file mode 100644
index 0000000000..d3b359a598
Binary files /dev/null and b/docs/.gitbook/assets/image (2).png differ
diff --git a/docs/.gitbook/assets/image (3) (1).png b/docs/.gitbook/assets/image (3) (1).png
new file mode 100644
index 0000000000..2442410112
Binary files /dev/null and b/docs/.gitbook/assets/image (3) (1).png differ
diff --git a/docs/.gitbook/assets/image (3).png b/docs/.gitbook/assets/image (3).png
new file mode 100644
index 0000000000..2442410112
Binary files /dev/null and b/docs/.gitbook/assets/image (3).png differ
diff --git a/docs/README.md b/docs/README.md
index 9593e9f26e..aa0be6090d 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,4 +1,4 @@
-# What is Feast?
+# Introduction
Feast \(**Fea**ture **St**ore\) is a tool for managing and serving machine learning features.
@@ -13,7 +13,7 @@ Feast aims to:
-**TL;DR:** Feast decouples feature engineering from feature usage. Features that are added to Feast become available immediately for training and serving. Models can retrieve the same features used in training from a low latency online store in production.
+Feast decouples feature engineering from feature usage. Features that are added to Feast become available immediately for training and serving. Models can retrieve the same features used in training from a low latency online store in production.
This means that new ML projects start with a process of feature selection from a catalog instead of having to do feature engineering from scratch.
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index f74a691970..aaea553e95 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -1,28 +1,34 @@
# Table of contents
-* [What is Feast?](README.md)
-
-## Introduction
-
-* [Why Feast?](introduction/why-feast.md)
-* [Getting Help](introduction/getting-help.md)
-* [Roadmap](introduction/roadmap.md)
-* [Changelog](https://github.com/gojek/feast/blob/master/CHANGELOG.md)
-
-## Concepts
-
-* [Concepts](concepts/concepts.md)
-
-## Installation
-
-* [Overview](installation/overview.md)
-* [Docker Compose](installation/docker-compose.md)
-* [Google Kubernetes Engine \(GKE\)](installation/gke.md)
+* [Introduction](README.md)
+* [Why Feast?](why-feast.md)
+* [Getting Started](getting-started/README.md)
+ * [Deploying Feast](getting-started/deploying-feast/README.md)
+ * [Docker Compose](getting-started/deploying-feast/docker-compose.md)
+ * [Kubernetes \(GKE\)](getting-started/deploying-feast/kubernetes.md)
+ * [Connecting to Feast](getting-started/connecting-to-feast-1/README.md)
+ * [Python SDK](getting-started/connecting-to-feast-1/python-sdk.md)
+ * [Feast CLI](getting-started/connecting-to-feast-1/connecting-to-feast.md)
+* [Roadmap](roadmap.md)
+* [Changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md)
+* [Community](getting-help.md)
+
+## User Guide
+
+* [Concepts](user-guide/overview.md)
+* [Architecture](user-guide/architecture.md)
+* [Feature Sets](user-guide/feature-sets.md)
+* [Entities](user-guide/entities.md)
+* [Features](user-guide/features.md)
+* [Sources](user-guide/sources.md)
+* [Data ingestion](user-guide/data-ingestion.md)
+* [Stores](user-guide/stores.md)
+* [Feature retrieval](user-guide/feature-retrieval.md)
## Tutorials
-* [Basic](https://github.com/gojek/feast/blob/master/examples/basic/basic.ipynb)
-* [Churn Prediction \(XGBoost\)](https://github.com/gojek/feast/blob/master/examples/feast-xgboost-churn-prediction-tutorial/Telecom%20Customer%20Churn%20Prediction%20%28with%20Feast%20and%20XGBoost%29.ipynb)
+* [Basic Tutorial](https://github.com/feast-dev/feast/blob/master/examples/basic/basic.ipynb)
+* [Churn Prediction \(XGBoost\)](https://github.com/feast-dev/feast/blob/master/examples/feast-xgboost-churn-prediction-tutorial/Telecom%20Customer%20Churn%20Prediction%20%28with%20Feast%20and%20XGBoost%29.ipynb)
## Administration
@@ -30,11 +36,14 @@
## Reference
-* [Python SDK](https://api.docs.feast.dev/python/)
-* [Go SDK](https://godoc.org/github.com/gojek/feast/sdk/go)
-* [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html)
-* [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html)
-* [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html)
+* [Configuration Reference](reference/configuration-reference.md)
+* [API](reference/api/README.md)
+ * [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html)
+ * [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html)
+ * [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html)
+ * [Go SDK](https://godoc.org/github.com/feast-dev/feast/sdk/go)
+ * [Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk)
+ * [Python SDK](https://api.docs.feast.dev/python/)
## Contributing
@@ -42,4 +51,5 @@
* [Development Guide](contributing/development-guide.md)
* [Style Guide](contributing/style-guide.md)
* [Release Process](contributing/release-process.md)
+* [Adding a New Store](contributing/adding-a-new-store.md)
diff --git a/docs/administration/troubleshooting.md b/docs/administration/troubleshooting.md
index a16a74d5d9..e293945acc 100644
--- a/docs/administration/troubleshooting.md
+++ b/docs/administration/troubleshooting.md
@@ -1,6 +1,6 @@
# Troubleshooting
-If at any point in time you cannot resolve a problem, please see the [Getting Help](https://github.com/gojek/feast/tree/75f3b783e5a7c5e0217a3020422548fb0d0ce0bf/docs/getting-help.md) section for reaching out to the Feast community.
+If at any point in time you cannot resolve a problem, please see the [Community](../getting-help.md) section for reaching out to the Feast community.
## How can I verify that all services are operational?
diff --git a/docs/assets/feast_logo.png b/docs/assets/feast_logo.png
new file mode 100644
index 0000000000..cdd97770ed
Binary files /dev/null and b/docs/assets/feast_logo.png differ
diff --git a/docs/contributing/adding-a-new-store-1.md b/docs/contributing/adding-a-new-store-1.md
new file mode 100644
index 0000000000..56c06b9b4b
--- /dev/null
+++ b/docs/contributing/adding-a-new-store-1.md
@@ -0,0 +1,87 @@
+# Adding a New Store
+
+The following guide will explain the process of adding a new store through the introduction of a storage connector.
+
+## 1. Storage API
+
+Feast has an external module where storage interfaces are defined: [Storage API](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api)
+
+Feast interacts with a store at three points .
+
+1. **During initialization:** Store configuration is loaded into memory by Feast Serving and synchronized with Feast Core
+2. **During ingestion of feature data.** [writer interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer) are used by the Apache Beam ingestion jobs in order to populate stores \(historical or online\).
+3. **During retrieval of feature data:** [Retrieval interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/retriever) are used by Feast Serving in order to read data from stores in order to create training datasets or to serve online data.
+
+All three of these components should be implemented in order to have a complete storage connector.
+
+## 2. Adding a Storage Connector
+
+### 2.1 Initialization and configuration
+
+Stores are configured in Feast Serving. Feast Serving publishes its store configuration to Feast Core, after which Feast Core can start ingestion/population jobs to populate it.
+
+Store configuration is always in the form of a map<String, String>. The keys and configuration for stores are defined in [protos](https://github.com/gojek/feast/blob/master/protos/feast/core/Store.proto). This must be added in order to define a new store
+
+Then the store must be configured to be loaded through Feast Serving. The above configuration is loaded through [FeastProperties.java](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/FeastProperties.java#L175).
+
+Once configuration is loaded, the store will then be instantiated.
+
+* Feast Core: The [StoreUtil.java](https://github.com/gojek/feast/blob/master/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java#L85) instantiates new stores for the purposes of feature ingestion.
+* Feast Serving: The [ServingServiceConfig](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/ServingServiceConfig.java#L56) instantiates new stores for the purposes of retrieval
+
+{% hint style="info" %}
+In the future we plan to provide a plugin interface for adding stores.
+{% endhint %}
+
+### 2.2 Feature Ingestion \(Writer\)
+
+Feast creates and manages ingestion/population jobs that stream in data from upstream data sources. Currently Feast only supports Kafka as a data source, meaning these jobs are all long running. Batch ingestion \(from users\) results in data being pushed to Kafka topics after which they are picked up by these "population" jobs and written to stores.
+
+In order for ingestion to succeed, the destination store must be writable. This means that Feast must be able to create the appropriate tables/schemas in the store and also write data from the population job into the store.
+
+Currently Feast Core starts and manages these population jobs that ingest data into stores \(although we are planning to move this responsibility to the serving layer\). Feast Core starts an [Apache Beam](https://beam.apache.org/) job which synchronously runs migrations on the destination store and subsequently starts consuming [FeatureRows](https://github.com/gojek/feast/blob/master/protos/feast/types/FeatureRow.proto) from Kafka and writing it into stores using a [writer](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer).
+
+Below is a "happy path" of a batch ingestion process which includes a blocking step at the Python SDK.
+
+
+
+
+
+The complete ingestion flow is executed by a [FeatureSink](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java). Two methods should be implemented
+
+* [prepareWrite\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L45): Sets up storage backend for writing/ingestion. This method will be called once during pipeline initialisation. Typically this is used to apply schemas.
+* [writer\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L53): Retrieves an Apache Beam PTransform that is used to write data to this store.
+
+### 2.2 Feature Serving \(Retriever\)
+
+Feast Serving can serve both historical/batch features and online features. Depending on the store that is being added, you should implement either a historical/batch store or an online storage.
+
+#### 2.2.1 Historical Serving
+
+The historical serving interface is defined through the [HistoricalRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java) interface. Historical retrieval is an asynchronous process. The client submits a request for a dataset to be produced, and polls until it is ready.
+
+
+
+The current implementation of batch retrieval starts and ends with a file \(dataset\) in a Google Cloud Storage bucket. The user ingests an entity dataset. This dataset is loaded into a store \(BigQuery0, joined to features in a point-in-time correct way, then exported again to the bucket.
+
+Additionally, we have also implemented a [batch retrieval method ](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/sdk/python/feast/client.py#L509)in the Python SDK. Depending on the means through which this new store will export data, this client may have to change. At the very least it would change if Google Cloud Storage isn't used as the staging bucket.
+
+The means through which you implement the export/import of data into the store will depend on your store.
+
+#### 2.2.2 Online Serving
+
+In the case of online serving it is necessary to implement an [OnlineRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java). This online retriever will read rows directly and synchronously from an online database. The exact encoding strategy you use to store your data in the store would be defined in the FeatureSink. The OnlineRetriever is expected to read and decode those rows.
+
+## 3. Storage Connectors Examples
+
+Feast currently provides support for the following storage types
+
+Historical storage
+
+* [BigQuery](https://github.com/gojek/feast/tree/master/storage/connectors/bigquery)
+
+Online storage
+
+* [Redis](https://github.com/gojek/feast/tree/master/storage/connectors/redis)
+* [Redis Cluster](https://github.com/gojek/feast/tree/master/storage/connectors/rediscluster)
+
diff --git a/docs/contributing/adding-a-new-store.md b/docs/contributing/adding-a-new-store.md
new file mode 100644
index 0000000000..56c06b9b4b
--- /dev/null
+++ b/docs/contributing/adding-a-new-store.md
@@ -0,0 +1,87 @@
+# Adding a New Store
+
+The following guide will explain the process of adding a new store through the introduction of a storage connector.
+
+## 1. Storage API
+
+Feast has an external module where storage interfaces are defined: [Storage API](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api)
+
+Feast interacts with a store at three points .
+
+1. **During initialization:** Store configuration is loaded into memory by Feast Serving and synchronized with Feast Core
+2. **During ingestion of feature data.** [writer interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer) are used by the Apache Beam ingestion jobs in order to populate stores \(historical or online\).
+3. **During retrieval of feature data:** [Retrieval interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/retriever) are used by Feast Serving in order to read data from stores in order to create training datasets or to serve online data.
+
+All three of these components should be implemented in order to have a complete storage connector.
+
+## 2. Adding a Storage Connector
+
+### 2.1 Initialization and configuration
+
+Stores are configured in Feast Serving. Feast Serving publishes its store configuration to Feast Core, after which Feast Core can start ingestion/population jobs to populate it.
+
+Store configuration is always in the form of a map<String, String>. The keys and configuration for stores are defined in [protos](https://github.com/gojek/feast/blob/master/protos/feast/core/Store.proto). This must be added in order to define a new store
+
+Then the store must be configured to be loaded through Feast Serving. The above configuration is loaded through [FeastProperties.java](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/FeastProperties.java#L175).
+
+Once configuration is loaded, the store will then be instantiated.
+
+* Feast Core: The [StoreUtil.java](https://github.com/gojek/feast/blob/master/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java#L85) instantiates new stores for the purposes of feature ingestion.
+* Feast Serving: The [ServingServiceConfig](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/ServingServiceConfig.java#L56) instantiates new stores for the purposes of retrieval
+
+{% hint style="info" %}
+In the future we plan to provide a plugin interface for adding stores.
+{% endhint %}
+
+### 2.2 Feature Ingestion \(Writer\)
+
+Feast creates and manages ingestion/population jobs that stream in data from upstream data sources. Currently Feast only supports Kafka as a data source, meaning these jobs are all long running. Batch ingestion \(from users\) results in data being pushed to Kafka topics after which they are picked up by these "population" jobs and written to stores.
+
+In order for ingestion to succeed, the destination store must be writable. This means that Feast must be able to create the appropriate tables/schemas in the store and also write data from the population job into the store.
+
+Currently Feast Core starts and manages these population jobs that ingest data into stores \(although we are planning to move this responsibility to the serving layer\). Feast Core starts an [Apache Beam](https://beam.apache.org/) job which synchronously runs migrations on the destination store and subsequently starts consuming [FeatureRows](https://github.com/gojek/feast/blob/master/protos/feast/types/FeatureRow.proto) from Kafka and writing it into stores using a [writer](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer).
+
+Below is a "happy path" of a batch ingestion process which includes a blocking step at the Python SDK.
+
+
+
+
+
+The complete ingestion flow is executed by a [FeatureSink](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java). Two methods should be implemented
+
+* [prepareWrite\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L45): Sets up storage backend for writing/ingestion. This method will be called once during pipeline initialisation. Typically this is used to apply schemas.
+* [writer\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L53): Retrieves an Apache Beam PTransform that is used to write data to this store.
+
+### 2.2 Feature Serving \(Retriever\)
+
+Feast Serving can serve both historical/batch features and online features. Depending on the store that is being added, you should implement either a historical/batch store or an online storage.
+
+#### 2.2.1 Historical Serving
+
+The historical serving interface is defined through the [HistoricalRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java) interface. Historical retrieval is an asynchronous process. The client submits a request for a dataset to be produced, and polls until it is ready.
+
+
+
+The current implementation of batch retrieval starts and ends with a file \(dataset\) in a Google Cloud Storage bucket. The user ingests an entity dataset. This dataset is loaded into a store \(BigQuery0, joined to features in a point-in-time correct way, then exported again to the bucket.
+
+Additionally, we have also implemented a [batch retrieval method ](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/sdk/python/feast/client.py#L509)in the Python SDK. Depending on the means through which this new store will export data, this client may have to change. At the very least it would change if Google Cloud Storage isn't used as the staging bucket.
+
+The means through which you implement the export/import of data into the store will depend on your store.
+
+#### 2.2.2 Online Serving
+
+In the case of online serving it is necessary to implement an [OnlineRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java). This online retriever will read rows directly and synchronously from an online database. The exact encoding strategy you use to store your data in the store would be defined in the FeatureSink. The OnlineRetriever is expected to read and decode those rows.
+
+## 3. Storage Connectors Examples
+
+Feast currently provides support for the following storage types
+
+Historical storage
+
+* [BigQuery](https://github.com/gojek/feast/tree/master/storage/connectors/bigquery)
+
+Online storage
+
+* [Redis](https://github.com/gojek/feast/tree/master/storage/connectors/redis)
+* [Redis Cluster](https://github.com/gojek/feast/tree/master/storage/connectors/rediscluster)
+
diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md
index 0a32ae284a..2be097d6ff 100644
--- a/docs/contributing/contributing.md
+++ b/docs/contributing/contributing.md
@@ -1,12 +1,10 @@
# Contribution Process
-## 1. Contribution process
+We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/gojek/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues. You will need to join our [Google Group](../getting-help.md) in order to get access.
-We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/gojek/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues.
+We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. If you are unsure about which direction to take with development then please communicate your ideas through a GitHub issue or through our [Slack Channel](../getting-help.md) before starting development.
-Please communicate your ideas through a GitHub issue or through our Slack Channel before starting development.
-
-Please [submit a PR ](https://github.com/gojek/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners.
+Please [submit a PR ](https://github.com/gojek/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners.
PRs that are submitted by the general public need to be identified as `ok-to-test`. Once enabled, [Prow](https://github.com/kubernetes/test-infra/tree/master/prow) will run a range of tests to verify the submission, after which community members will help to review the pull request.
@@ -14,529 +12,3 @@ PRs that are submitted by the general public need to be identified as `ok-to-tes
Please sign the [Google CLA](https://cla.developers.google.com/) in order to have your code merged into the Feast repository.
{% endhint %}
-## 2. Development guide
-
-### 2.1 Overview
-
-The following guide will help you quickly run Feast in your local machine.
-
-The main components of Feast are:
-
-* **Feast Core:** Handles feature registration, starts and manages ingestion jobs and ensures that Feast internal metadata is consistent.
-* **Feast Ingestion Jobs:** Subscribes to streams of FeatureRows and writes these as feature
-
- values to registered databases \(online, historical\) that can be read by Feast Serving.
-
-* **Feast Serving:** Service that handles requests for features values, either online or batch.
-
-### 2.**2 Requirements**
-
-#### 2.**2.1 Development environment**
-
-The following software is required for Feast development
-
-* Java SE Development Kit 11
-* Python version 3.6 \(or above\) and pip
-* [Maven ](https://maven.apache.org/install.html)version 3.6.x
-
-Additionally, [grpc\_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) is useful for debugging and quick testing of gRPC endpoints.
-
-#### 2.**2.2 Services**
-
-The following components/services are required to develop Feast:
-
-* **Feast Core:** Requires PostgreSQL \(version 11 and above\) to store state, and requires a Kafka \(tested on version 2.x\) setup to allow for ingestion of FeatureRows.
-* **Feast Serving:** Requires Redis \(tested on version 5.x\).
-
-These services should be running before starting development. The following snippet will start the services using Docker.
-
-```bash
-# Start Postgres
-docker run --name postgres --rm -it -d --net host -e POSTGRES_DB=postgres -e POSTGRES_USER=postgres \
--e POSTGRES_PASSWORD=password postgres:12-alpine
-
-# Start Redis
-docker run --name redis --rm -it --net host -d redis:5-alpine
-
-# Start Zookeeper (needed by Kafka)
-docker run --rm \
- --net=host \
- --name=zookeeper \
- --env=ZOOKEEPER_CLIENT_PORT=2181 \
- --detach confluentinc/cp-zookeeper:5.2.1
-
-# Start Kafka
-docker run --rm \
- --net=host \
- --name=kafka \
- --env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \
- --env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
- --env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
- --detach confluentinc/cp-kafka:5.2.1
-```
-
-### 2.3 Testing and development
-
-#### 2.3.1 Running unit tests
-
-```text
-$ mvn test
-```
-
-#### 2.3.2 Running integration tests
-
-_Note: integration suite isn't yet separated from unit._
-
-```text
-$ mvn verify
-```
-
-#### 2.3.3 Running components locally
-
-The `core` and `serving` modules are Spring Boot applications. These may be run as usual for [the Spring Boot Maven plugin](https://docs.spring.io/spring-boot/docs/current/maven-plugin/index.html):
-
-```text
-$ mvn --projects core spring-boot:run
-
-# Or for short:
-$ mvn -pl core spring-boot:run
-```
-
-Note that you should execute `mvn` from the Feast repository root directory, as there are intermodule dependencies that Maven will not resolve if you `cd` to subdirectories to run.
-
-#### 2.3.4 Running from IntelliJ
-
-Compiling and running tests in IntelliJ should work as usual.
-
-Running the Spring Boot apps may work out of the box in IDEA Ultimate, which has built-in support for Spring Boot projects, but the Community Edition needs a bit of help:
-
-The Spring Boot Maven plugin automatically puts dependencies with `provided` scope on the runtime classpath when using `spring-boot:run`, such as its embedded Tomcat server. The "Play" buttons in the gutter or right-click menu of a `main()` method [do not do this](https://stackoverflow.com/questions/30237768/run-spring-boots-main-using-ide).
-
-A solution to this is:
-
-1. Open `View > Tool Windows > Maven`
-2. Drill down to e.g. `Feast Core > Plugins > spring-boot:run`, right-click and `Create 'feast-core [spring-boot'…`
-3. In the dialog that pops up, check the `Resolve Workspace artifacts` box
-4. Recommended: add `-Dspring-boot.run.fork=false` to the `Command line` field to get Debug working too
-5. Click `OK`. You should now be able to select this run configuration for the Play button in the main toolbar, keyboard shortcuts, etc.
-
-It is recommend to have IntelliJ delegate building to Maven, if this is not enabled out of the box when you import the project, for greater assurance that build behavior is consistent with CI / production builds. This is set in Preferences at `Build, Execution, Deployment > Build Tools > Maven > Runner > Delegate IDE build/run actions to Maven`.
-
-### 2.**4** Validating your setup
-
-The following section is a quick walk-through to test whether your local Feast deployment is functional for development purposes.
-
-**2.4.1 Assumptions**
-
-* PostgreSQL is running in `localhost:5432` and has a database called `postgres` which
-
- can be accessed with credentials user `postgres` and password `password`. Different database configurations can be supplied here \(`/core/src/main/resources/application.yml`\)
-
-* Redis is running locally and accessible from `localhost:6379`
-* \(optional\) The local environment has been authentication with Google Cloud Platform and has full access to BigQuery. This is only necessary for BigQuery testing/development.
-
-#### 2.4.2 Clone Feast
-
-```bash
-git clone https://github.com/gojek/feast.git && cd feast && \
-export FEAST_HOME_DIR=$(pwd)
-```
-
-#### 2.4.3 Starting Feast Core
-
-To run Feast Core locally using Maven:
-
-```bash
-# Feast Core can be configured from the following .yml file
-# $FEAST_HOME_DIR/core/src/main/resources/application.yml
-mvn --projects core spring-boot:run
-```
-
-Test whether Feast Core is running
-
-```text
-grpc_cli call localhost:6565 ListStores ''
-```
-
-The output should list **no** stores since no Feast Serving has registered its stores to Feast Core:
-
-```text
-connecting to localhost:6565
-
-Rpc succeeded with OK status
-```
-
-#### 2.4.4 Starting Feast Serving
-
-Feast Serving is configured through the `$FEAST_HOME_DIR/serving/src/main/resources/application.yml`. Each Serving deployment must be configured with a store. The default store is Redis \(used for online serving\).
-
-The configuration for this default store is located in a separate `.yml` file. The default location is `$FEAST_HOME_DIR/serving/sample_redis_config.yml`:
-
-```text
-name: serving
-type: REDIS
-redis_config:
- host: localhost
- port: 6379
-subscriptions:
- - name: "*"
- project: "*"
- version: "*"
-```
-
-Once Feast Serving is started, it will register its store with Feast Core \(by name\) and start to subscribe to a feature sets based on its subscription.
-
-Start Feast Serving GRPC server on localhost:6566 with store name `serving`
-
-```text
-mvn --projects serving spring-boot:run
-```
-
-Test connectivity to Feast Serving
-
-```text
-grpc_cli call localhost:6566 GetFeastServingInfo ''
-```
-
-```text
-connecting to localhost:6566
-version: "0.4.2-SNAPSHOT"
-type: FEAST_SERVING_TYPE_ONLINE
-
-Rpc succeeded with OK status
-```
-
-Test Feast Core to see whether it is aware of the Feast Serving deployment
-
-```text
-grpc_cli call localhost:6565 ListStores ''
-```
-
-```text
-connecting to localhost:6565
-store {
- name: "serving"
- type: REDIS
- subscriptions {
- name: "*"
- version: "*"
- project: "*"
- }
- redis_config {
- host: "localhost"
- port: 6379
- }
-}
-
-Rpc succeeded with OK status
-```
-
-In order to use BigQuery as a historical store, it is necessary to start Feast Serving with a different store type.
-
-Copy `$FEAST_HOME_DIR/serving/sample_redis_config.yml` to the following location `$FEAST_HOME_DIR/serving/my_bigquery_config.yml` and update the configuration as below:
-
-```text
-name: bigquery
-type: BIGQUERY
-bigquery_config:
- project_id: YOUR_GCP_PROJECT_ID
- dataset_id: YOUR_GCP_DATASET
-subscriptions:
- - name: "*"
- version: "*"
- project: "*"
-```
-
-Then inside `serving/src/main/resources/application.yml` modify the following key `feast.store.config-path` to point to the new store configuration.
-
-After making these changes, restart Feast Serving:
-
-```text
-mvn --projects serving spring-boot:run
-```
-
-You should see two stores registered:
-
-```text
-store {
- name: "serving"
- type: REDIS
- subscriptions {
- name: "*"
- version: "*"
- project: "*"
- }
- redis_config {
- host: "localhost"
- port: 6379
- }
-}
-store {
- name: "bigquery"
- type: BIGQUERY
- subscriptions {
- name: "*"
- version: "*"
- project: "*"
- }
- bigquery_config {
- project_id: "my_project"
- dataset_id: "my_bq_dataset"
- }
-}
-```
-
-#### 2.4.5 Registering a FeatureSet
-
-Before registering a new FeatureSet, a project is required.
-
-```text
-grpc_cli call localhost:6565 CreateProject '
- name: "your_project_name"
-'
-```
-
-When a feature set is successfully registered, Feast Core will start an **ingestion** job that listens for new features in the feature set.
-
-{% hint style="info" %}
-Note that Feast currently only supports source of type `KAFKA`, so you must have access to a running Kafka broker to register a FeatureSet successfully. It is possible to omit the `source` from a Feature Set, but Feast Core will still use Kafka behind the scenes, it is simply abstracted away from the user.
-{% endhint %}
-
-Create a new FeatureSet in Feast by sending a request to Feast Core:
-
-```text
-# Example of registering a new driver feature set
-# Note the source value, it assumes that you have access to a Kafka broker
-# running on localhost:9092
-
-grpc_cli call localhost:6565 ApplyFeatureSet '
-feature_set {
- spec {
- project: "your_project_name"
- name: "driver"
- version: 1
-
- entities {
- name: "driver_id"
- value_type: INT64
- }
-
- features {
- name: "city"
- value_type: STRING
- }
-
- source {
- type: KAFKA
- kafka_source_config {
- bootstrap_servers: "localhost:9092"
- topic: "your-kafka-topic"
- }
- }
- }
-}
-'
-```
-
-Verify that the FeatureSet has been registered correctly.
-
-```text
-# To check that the FeatureSet has been registered correctly.
-# You should also see logs from Feast Core of the ingestion job being started
-grpc_cli call localhost:6565 GetFeatureSet '
- project: "your_project_name"
- name: "driver"
-'
-```
-
-Or alternatively, list all feature sets
-
-```text
-grpc_cli call localhost:6565 ListFeatureSets '
- filter {
- project: "your_project_name"
- feature_set_name: "driver"
- feature_set_version: "1"
- }
-'
-```
-
-#### 2.4.6 Ingestion and Population of Feature Values
-
-```text
-# Produce FeatureRow messages to Kafka so it will be ingested by Feast
-# and written to the registered stores.
-# Make sure the value here is the topic assigned to the feature set
-# ... producer.send("feast-driver-features" ...)
-#
-# Install Python SDK to help writing FeatureRow messages to Kafka
-cd $FEAST_HOMEDIR/sdk/python
-pip3 install -e .
-pip3 install pendulum
-
-# Produce FeatureRow messages to Kafka so it will be ingested by Feast
-# and written to the corresponding store.
-# Make sure the value here is the topic assigned to the feature set
-# ... producer.send("feast-test_feature_set-features" ...)
-python3 - <` property, and commit.
+4. Push. For a new release branch, open a PR against master.
+5. When CI passes, merge. \(Remember _not_ to delete the new release branch\).
+6. Tag the merge commit with the release version, using a `v` prefix. Push the tag.
+7. Bump to the next working version and append `-SNAPSHOT` in `pom.xml`.
+8. Commit the POM and open a PR.
+9. Create a [GitHub release](https://github.com/feast-dev/feast/releases) which includes a summary of important changes as well as any artifacts associated with the release. Make sure to include the same change log as added in [CHANGELOG.md](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md). Use `Feast vX.Y.Z` as the title.
+10. Create one final PR to the master branch and also update its [CHANGELOG.md](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md).
+
+When a tag that matches a Semantic Version string is pushed, CI will automatically build and push the relevant artifacts to their repositories or package managers \(docker images, Python wheels, etc\). JVM artifacts are promoted from Sonatype OSSRH to Maven Central, but it sometimes takes some time for them to be available.
+
+### Creating a change log
+
+We use an [open source change log generator](https://hub.docker.com/r/ferrarimarco/github-changelog-generator/) to generate change logs. The process still requires a little bit of manual effort. 1. Create a GitHub token as [per these instructions ](https://github.com/github-changelog-generator/github-changelog-generator#github-token). The token is used as an input argument \(`-t`\) to the changelog generator. 2. The change log generator configuration below will look for unreleased changes on a specific branch. The branch will be `master` for a major/minor release, or a release branch \(`v0.4-branch`\) for a patch release. You will need to set the branch using the `--release-branch` argument. 3. You should also set the `--future-release` argument. This is the version you are releasing. The version can still be changed at a later date. 4. Update the arguments below and run the command to generate the change log to the console.
+
+```text
+docker run -it --rm ferrarimarco/github-changelog-generator \
+--user feast-dev \
+--project feast \
+--release-branch \
+--future-release \
+--unreleased-only \
+--no-issues \
+--bug-labels kind/bug \
+--enhancement-labels kind/feature \
+--breaking-labels compat/breaking \
+-t \
+--max-issues 1 \
+-o
+```
+
+1. Review each change log item.
+ * Make sure that sentences are grammatically correct and well formatted \(although we will try to enforce this at the PR review stage\).
+ * Make sure that each item is categorized correctly. You will see the following categories: `Breaking changes`, `Implemented enhancements`, `Fixed bugs`, and `Merged pull requests`. Any unlabeled PRs will be found in `Merged pull requests`. It's important to make sure that any `breaking changes`, `enhancements`, or `bug fixes` are pulled up out of `merged pull requests` into the correct category. Housekeeping, tech debt clearing, infra changes, or refactoring do not count as `enhancements`. Only enhancements a user benefits from should be listed in that category.
+ * Make sure that the "Full Changelog" link is actually comparing the correct tags \(normally your released version against the previously version\).
+ * Make sure that release notes and breaking changes are present.
diff --git a/docs/contributing/style-guide.md b/docs/contributing/style-guide.md
index 322f1e3d03..1cf1c69671 100644
--- a/docs/contributing/style-guide.md
+++ b/docs/contributing/style-guide.md
@@ -1,6 +1,8 @@
# Style Guide
-## 1. Java
+## 1. Language Specific Style Guides
+
+### 1.1 Java
We conform to the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). Maven can helpfully take care of that for you before you commit:
@@ -17,11 +19,25 @@ $ mvn verify -Dspotless.check.skip
If you're using IntelliJ, you can import [these code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) if you'd like to use the IDE's reformat function as you develop.
-## 2. Go
+### 1.2 Go
Make sure you apply `go fmt`.
-## 3. Python
+### 1.3 Python
We use [Python Black](https://github.com/psf/black) to format our Python code prior to submission.
+## 2. Formatting and Linting
+
+Code can automatically be formatted by running the following command from the project root directory
+
+```text
+make format
+```
+
+Once code that is submitted through a PR or direct push will be validated with the following command
+
+```text
+make lint
+```
+
diff --git a/docs/coverage/java/pom.xml b/docs/coverage/java/pom.xml
new file mode 100644
index 0000000000..9666b7092c
--- /dev/null
+++ b/docs/coverage/java/pom.xml
@@ -0,0 +1,105 @@
+
+
+
+ 4.0.0
+
+
+
+
+ dev.feast
+ feast-parent
+ ${revision}
+ ../../..
+
+
+ Feast Coverage Java
+ feast-coverage
+
+
+ true
+
+
+
+
+ dev.feast
+ feast-storage-api
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-bigquery
+ ${project.version}
+
+
+
+ dev.feast
+ feast-storage-connector-redis
+ ${project.version}
+
+
+
+ dev.feast
+ feast-ingestion
+ ${project.version}
+
+
+
+ dev.feast
+ feast-core
+ ${project.version}
+
+
+
+ dev.feast
+ feast-serving
+ ${project.version}
+
+
+
+ dev.feast
+ feast-sdk
+ ${project.version}
+
+
+
+
+
+
+ org.jacoco
+ jacoco-maven-plugin
+
+
+ report-aggregate
+ prepare-package
+
+ report-aggregate
+
+
+
+
+
+
+
+
diff --git a/docs/getting-help.md b/docs/getting-help.md
new file mode 100644
index 0000000000..32b21461b1
--- /dev/null
+++ b/docs/getting-help.md
@@ -0,0 +1,36 @@
+# Community
+
+## Chat
+
+* Come and say hello in [\#Feast](https://join.slack.com/t/kubeflow/shared_invite/zt-cpr020z4-PfcAue_2nw67~iIDy7maAQ) over in the Kubeflow Slack.
+
+## GitHub
+
+* Feast's GitHub repo can be [found here](https://github.com/feast-dev/feast/).
+* Found a bug or need a feature? [Create an issue on GitHub](https://github.com/feast-dev/feast/issues/new)
+
+## Community Call
+
+We have a community call every 2 weeks. Alternating between two times.
+
+* 11 am \(UTC + 8\)
+* 5 pm \(UTC + 8\)
+
+Please join the [feast-dev](getting-help.md#feast-development) mailing list to receive the the calendar invitation.
+
+## Mailing list
+
+### Feast discussion
+
+* Google Group: [https://groups.google.com/d/forum/feast-discuss](https://groups.google.com/d/forum/feast-discuss)
+* Mailing List: [feast-discuss@googlegroups.com](mailto:feast-discuss@googlegroups.com)
+
+### Feast development
+
+* Google Group: [https://groups.google.com/d/forum/feast-dev](https://groups.google.com/d/forum/feast-dev)
+* Mailing List: [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com)
+
+## Google Drive
+
+The Feast community also maintains a [Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) with documents like RFCs, meeting notes, or roadmaps. Please join one of the above mailing lists \(feast-dev or feast-discuss\) to gain access to the drive.
+
diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md
new file mode 100644
index 0000000000..9945ab197d
--- /dev/null
+++ b/docs/getting-started/README.md
@@ -0,0 +1,14 @@
+# Getting Started
+
+If you want to learn more about Feast concepts and its architecture, see the user guide
+
+If you would like to connect to an existing Feast deployment then click on `Connecting to Feast`
+
+{% page-ref page="connecting-to-feast-1/" %}
+
+If you would like to deploy a new installation of Feast, then click on `Deploying Feast`
+
+{% page-ref page="deploying-feast/" %}
+
+If you are connected to a running Feast deployment, then have a look at our [example tutorials](https://github.com/gojek/feast/tree/master/examples).
+
diff --git a/docs/getting-started/connecting-to-feast-1/README.md b/docs/getting-started/connecting-to-feast-1/README.md
new file mode 100644
index 0000000000..fd8fe8e498
--- /dev/null
+++ b/docs/getting-started/connecting-to-feast-1/README.md
@@ -0,0 +1,27 @@
+# Connecting to Feast
+
+## Python SDK
+
+* Define, register, and manage entities and features
+* Ingest data into Feast
+* Build and retrieve training datasets
+* Retrieve online features
+
+{% page-ref page="python-sdk.md" %}
+
+## Feast CLI
+
+* Define, register, and manage entities and features from the terminal
+* Ingest data into Feast
+* Manage ingestion jobs
+
+{% page-ref page="connecting-to-feast.md" %}
+
+## Online Serving Clients
+
+Feast provides three ways to retrieve data in production for online serving
+
+* [Python SDK](https://api.docs.feast.dev/python/) \(as above\)
+* [Go Client library](https://godoc.org/github.com/feast-dev/feast/sdk/go)
+* [Java Client library](https://javadoc.io/doc/dev.feast/feast-sdk)
+
diff --git a/docs/getting-started/connecting-to-feast-1/connecting-to-feast.md b/docs/getting-started/connecting-to-feast-1/connecting-to-feast.md
new file mode 100644
index 0000000000..a2cafda0db
--- /dev/null
+++ b/docs/getting-started/connecting-to-feast-1/connecting-to-feast.md
@@ -0,0 +1,38 @@
+# Feast CLI
+
+The Feast CLI is installed through pip:
+
+```bash
+pip install feast
+```
+
+Configure the CLI to connect to your Feast Core deployment
+
+```text
+feast config set core_url your.feast.deployment
+```
+
+{% hint style="info" %}
+By default, all configuration is stored in `~/.feast/config`
+{% endhint %}
+
+The CLI is a wrapper around the [Feast Python SDK](python-sdk.md)
+
+```aspnet
+$ feast
+
+Usage: feast [OPTIONS] COMMAND [ARGS]...
+
+Options:
+ --help Show this message and exit.
+
+Commands:
+ config View and edit Feast properties
+ feature-sets Create and manage feature sets
+ ingest Ingest feature data into a feature set
+ projects Create and manage projects
+ version Displays version and connectivity information
+```
+
+
+
diff --git a/docs/getting-started/connecting-to-feast-1/python-sdk.md b/docs/getting-started/connecting-to-feast-1/python-sdk.md
new file mode 100644
index 0000000000..2fea0c31af
--- /dev/null
+++ b/docs/getting-started/connecting-to-feast-1/python-sdk.md
@@ -0,0 +1,20 @@
+# Python SDK
+
+The Feast SDK can be installed directly using pip:
+
+```bash
+pip install feast
+```
+
+Users should then be able to connect to a Feast deployment as follows
+
+```python
+from feast import Client
+
+# Connect to an existing Feast Core deployment
+client = Client(core_url='feast.example.com:6565')
+
+# Ensure that your client is connected by printing out some feature sets
+client.list_feature_sets()
+```
+
diff --git a/docs/getting-started/deploying-feast/README.md b/docs/getting-started/deploying-feast/README.md
new file mode 100644
index 0000000000..4a9aac11ba
--- /dev/null
+++ b/docs/getting-started/deploying-feast/README.md
@@ -0,0 +1,16 @@
+# Deploying Feast
+
+## Docker Compose
+
+* Fastest way to get Feast up and running.
+* Provides a pre-installed Jupyter Notebook with sample code.
+
+{% page-ref page="docker-compose.md" %}
+
+## Kubernetes \(GKE\)
+
+* Recommended way to install Feast for production use.
+* The guide has dependencies on BigQuery, and Google Cloud Storage.
+
+{% page-ref page="kubernetes.md" %}
+
diff --git a/docs/getting-started/deploying-feast/docker-compose.md b/docs/getting-started/deploying-feast/docker-compose.md
new file mode 100644
index 0000000000..a8895a0252
--- /dev/null
+++ b/docs/getting-started/deploying-feast/docker-compose.md
@@ -0,0 +1,112 @@
+# Docker Compose
+
+### Overview
+
+This guide will bring Feast up using Docker Compose. This will allow you to:
+
+* Create, register, and manage feature sets
+* Ingest feature data into Feast
+* Retrieve features for online serving
+* Retrieve features for batch serving \(only if using Google Cloud Platform\)
+
+This guide is split into three parts:
+
+1. Setting up your environment
+2. Starting Feast with **online serving support only** \(does not require GCP\).
+3. Starting Feast with support for **both online and batch** serving \(requires GCP\)
+
+{% hint style="info" %}
+The docker compose setup uses Direct Runner for the Apache Beam jobs that populate data stores. Running Beam with the Direct Runner means it does not need a dedicated runner like Flink or Dataflow, but this comes at the cost of performance. We recommend the use of a dedicated runner when running Feast with very large workloads.
+{% endhint %}
+
+### 0. Requirements
+
+* [Docker compose](https://docs.docker.com/compose/install/) must be installed.
+* The following list of TCP ports must be free:
+ * 6565, 6566, 8888, and 9094.
+ * Alternatively it is possible to modify port mappings in `/docker-compose/docker-compose.yml`.
+* \(for batch serving only\) For batch serving you will also need a [GCP service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to [Google Cloud Storage](https://cloud.google.com/storage) and [BigQuery](https://cloud.google.com/bigquery).
+* \(for batch serving only\) [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use.
+
+## 1. Set up environment
+
+Clone the [Feast repository](https://github.com/feast-dev/feast/) and navigate to the `docker-compose` sub-directory:
+
+```bash
+git clone https://github.com/feast-dev/feast.git && \
+cd feast && export FEAST_HOME_DIR=$(pwd) && \
+cd infra/docker-compose
+```
+
+Make a copy of the `.env.sample` file:
+
+```bash
+cp .env.sample .env
+```
+
+## 2. Docker Compose for Online Serving Only
+
+### 2.1 Start Feast \(without batch retrieval support\)
+
+If you do not require batch serving, then its possible to simply bring up Feast:
+
+```javascript
+docker-compose up -d
+```
+
+A Jupyter Notebook environment is now available to use Feast:
+
+[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples)
+
+## 3. Docker Compose for Online and Batch Serving
+
+{% hint style="info" %}
+Batch serving requires Google Cloud Storage to function, specifically Google Cloud Storage \(GCP\) and BigQuery.
+{% endhint %}
+
+### 3.1 Set up Google Cloud Platform
+
+Create a [service account ](https://cloud.google.com/iam/docs/creating-managing-service-accounts)from the GCP console and copy it to the `infra/docker-compose/gcp-service-accounts` folder:
+
+```javascript
+cp my-service-account.json ${FEAST_HOME_DIR}/infra/docker-compose/gcp-service-accounts
+```
+
+Create a Google Cloud Storage bucket. Make sure that your service account above has read/write permissions to this bucket:
+
+```bash
+gsutil mb gs://my-feast-staging-bucket
+```
+
+### 3.2 Configure .env
+
+Configure the `.env` file based on your environment. At the very least you have to modify:
+
+| Parameter | Description |
+| :--- | :--- |
+| FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json`. |
+| FEAST\_BATCH\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
+| FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
+| FEAST\_JOB\_STAGING\_LOCATION | Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests, for example `gs://your-gcs-bucket/staging` |
+
+### 3.3 Configure .bq-store.yml
+
+We will also need to configure the `bq-store.yml` file inside `infra/docker-compose/serving/` to configure the BigQuery storage configuration as well as the feature sets that the store subscribes to. At a minimum you will need to set:
+
+| Parameter | Description |
+| :--- | :--- |
+| bigquery\_config.project\_id | This is you [GCP project Id](https://cloud.google.com/resource-manager/docs/creating-managing-projects). |
+| bigquery\_config.dataset\_id | This is the name of the BigQuery dataset that tables will be created in. Each feature set will have one table in BigQuery. |
+
+### 3.4 Start Feast \(with batch retrieval support\)
+
+Start Feast:
+
+```javascript
+docker-compose up -d
+```
+
+A Jupyter Notebook environment is now available to use Feast:
+
+[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples)
+
diff --git a/docs/getting-started/deploying-feast/kubernetes.md b/docs/getting-started/deploying-feast/kubernetes.md
new file mode 100644
index 0000000000..a7432836f3
--- /dev/null
+++ b/docs/getting-started/deploying-feast/kubernetes.md
@@ -0,0 +1,211 @@
+# Kubernetes \(GKE\)
+
+### Overview
+
+This guide will install Feast into a Kubernetes cluster on GCP. It assumes that all of your services will run within a single Kubernetes cluster. Once Feast is installed you will be able to:
+
+* Define and register features.
+* Load feature data from both batch and streaming sources.
+* Retrieve features for model training.
+* Retrieve features for online serving.
+
+{% hint style="info" %}
+This guide requires [Google Cloud Platform](https://cloud.google.com/) for installation.
+
+* [BigQuery](https://cloud.google.com/bigquery/) is used for storing historical features.
+* [Google Cloud Storage](https://cloud.google.com/storage/) is used for intermediate data storage.
+{% endhint %}
+
+## 0. Requirements
+
+1. [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use.
+2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed.
+3. [Helm](https://helm.sh/3) \(2.16.0 or greater\) installed on your local machine with Tiller installed in your cluster. Helm 3 has not been tested yet.
+
+## 1. Set up GCP
+
+First define the environmental variables that we will use throughout this installation. Please customize these to reflect your environment.
+
+```bash
+export FEAST_GCP_PROJECT_ID=my-gcp-project
+export FEAST_GCP_REGION=us-central1
+export FEAST_GCP_ZONE=us-central1-a
+export FEAST_BIGQUERY_DATASET_ID=feast
+export FEAST_GCS_BUCKET=${FEAST_GCP_PROJECT_ID}_feast_bucket
+export FEAST_GKE_CLUSTER_NAME=feast
+export FEAST_SERVICE_ACCOUNT_NAME=feast-sa
+```
+
+Create a Google Cloud Storage bucket for Feast to stage batch data exports:
+
+```bash
+gsutil mb gs://${FEAST_GCS_BUCKET}
+```
+
+Create the service account that Feast will run as:
+
+```bash
+gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME}
+
+gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \
+ --member serviceAccount:${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \
+ --role roles/editor
+
+gcloud iam service-accounts keys create key.json --iam-account \
+${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com
+```
+
+## 2. Set up a Kubernetes \(GKE\) cluster
+
+{% hint style="warning" %}
+Provisioning a GKE cluster can expose your services publicly. This guide does not cover securing access to the cluster.
+{% endhint %}
+
+Create a GKE cluster:
+
+```bash
+gcloud container clusters create ${FEAST_GKE_CLUSTER_NAME} \
+ --machine-type n1-standard-4
+```
+
+Create a secret in the GKE cluster based on your local key `key.json`:
+
+```bash
+kubectl create secret generic feast-gcp-service-account --from-file=key.json
+```
+
+For this guide we will use `NodePort` for exposing Feast services. In order to do so, we must find an External IP of at least one GKE node. This should be a public IP.
+
+```bash
+export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
+export FEAST_CORE_URL=${FEAST_IP}:32090
+export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
+export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+```
+
+Add firewall rules to open up ports on your Google Cloud Platform project:
+
+```bash
+gcloud compute firewall-rules create feast-core-port --allow tcp:32090
+gcloud compute firewall-rules create feast-online-port --allow tcp:32091
+gcloud compute firewall-rules create feast-batch-port --allow tcp:32092
+gcloud compute firewall-rules create feast-redis-port --allow tcp:32101
+gcloud compute firewall-rules create feast-kafka-ports --allow tcp:31090-31095
+```
+
+## 3. Set up Helm
+
+Run the following command to provide Tiller with authorization to install Feast:
+
+```bash
+kubectl apply -f - <
+
+
+
Option
+
Description
+
Default
+
+
+
+
+
FEAST_CONFIG
+
+
Location of Feast configuration file
+
/.feast/config
+
+
+
+
CONFIG_FEAST_ENV_VAR_PREFIX
+
+
+
Default prefix to Feast environmental variable options.
+
Does not apply to FEAST_CONFIG
+
+
+
FEAST_
+
+
+
+
PROJECT
+
+
Default Feast project to use
+
default
+
+
+
+
CORE_URL
+
+
URL used to connect to Feast Core
+
localhost:6565
+
+
+
+
CORE_ENABLE_SSL
+
+
Enables TLS/SSL on connections to Feast Core
+
False
+
+
+
+
CORE_AUTH_ENABLED
+
+
Enable user authentication when connecting to a Feast Core instance
+
False
+
+
+
+
CORE_AUTH_TOKEN
+
+
Provide a static JWT token to authenticate with Feast Core
+
Null
+
+
+
+
CORE_SERVER_SSL_CERT
+
+
Path to certificate(s) used by Feast Client to authenticate TLS connection
+ to Feast Core (not to authenticate you as a client).
+
Null
+
+
+
+
SERVING_URL
+
+
URL used to connect to Feast Serving
+
localhost:6566
+
+
+
+
SERVING_ENABLE_SSL
+
+
Enables TLS/SSL on connections to Feast Serving
+
False
+
+
+
+
SERVING_SERVER_SSL_CERT
+
+
Path to certificate(s) used by Feast Client to authenticate TLS connection
+ to Feast Serving (not to authenticate you as a client).
+
None
+
+
+
+
GRPC_CONNECTION_TIMEOUT_DEFAULT
+
+
Default gRPC connection timeout to both Feast Serving and Feast Core (in
+ seconds)
+
3
+
+
+
+
GRPC_CONNECTION_TIMEOUT_APPLY
+
+
Default gRPC connection timeout when sending an ApplyFeatureSet command
+ to Feast Core (in seconds)
+
600
+
+
+
+
BATCH_FEATURE_REQUEST_WAIT_S
+
+
Time to wait for batch feature requests before timing out.
+
600
+
+
+
+
+
+### Usage
+
+#### Configuration File
+
+Feast Configuration File \(`~/.feast/config`\)
+
+```text
+[general]
+project = default
+core_url = localhost:6565
+```
+
+#### Environmental Variables
+
+```bash
+FEAST_CORE_URL=my_feast:6565 FEAST_PROJECT=default feast projects list
+```
+
+#### Feast SDK
+
+```python
+client = Client(
+ core_url="localhost:6565",
+ project="default"
+)
+```
+
diff --git a/docs/roadmap.md b/docs/roadmap.md
new file mode 100644
index 0000000000..a047cf4125
--- /dev/null
+++ b/docs/roadmap.md
@@ -0,0 +1,31 @@
+# Roadmap
+
+## Feast 0.5 \(Technical Release\)
+
+[Discussion](https://github.com/gojek/feast/issues/527)
+
+### New functionality
+
+1. Streaming statistics and validation \(M1 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\)
+2. Support for Redis Clusters \([\#478](https://github.com/gojek/feast/issues/478), [\#502](https://github.com/gojek/feast/issues/502)\)
+3. Add feature and feature set labels, i.e. key/value registry metadata \([\#463](https://github.com/gojek/feast/issues/463)\)
+4. Job management API \([\#302](https://github.com/gojek/feast/issues/302)\)
+
+### Technical debt, refactoring, or housekeeping
+
+1. Clean up and document all configuration options \([\#525](https://github.com/gojek/feast/issues/525)\)
+2. Externalize storage interfaces \([\#402](https://github.com/gojek/feast/issues/402)\)
+3. Reduce memory usage in Redis \([\#515](https://github.com/gojek/feast/issues/515)\)
+4. Support for handling out of order ingestion \([\#273](https://github.com/gojek/feast/issues/273)\)
+5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/gojek/feast/issues/386)\) \([\#462](https://github.com/gojek/feast/issues/462)\)
+6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/gojek/feast/issues/461)\)
+7. Write Beam metrics after ingestion to store \(not prior\) \([\#489](https://github.com/gojek/feast/issues/489)\)
+
+## Feast 0.6 \(Feature Release\)
+
+### New functionality
+
+1. User authentication & authorization \([\#504](https://github.com/gojek/feast/issues/504)\)
+2. Batch statistics and validation \(M2 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\)
+3. Online feature/entity status metadata \([\#658](https://github.com/gojek/feast/pull/658)\)
+
diff --git a/docs/user-guide/architecture.md b/docs/user-guide/architecture.md
new file mode 100644
index 0000000000..5de6fe66eb
--- /dev/null
+++ b/docs/user-guide/architecture.md
@@ -0,0 +1,41 @@
+# Architecture
+
+
+
+### **Feast Core**
+
+Feast Core is the central management service of a Feast deployment. It's role is to:
+
+* Allows users to create [entities](entities.md) and [features](features.md) through the creation and management of [feature sets](feature-sets.md).
+* Starts and manages [ingestion jobs](data-ingestion.md). These jobs populate [stores](stores.md) from [sources](sources.md) based on the feature sets that are defined and the subscription\(s\) that a [store](stores.md) has.
+
+{% hint style="info" %}
+Job management may move out of Feast Core to Feast Serving in the future.
+{% endhint %}
+
+### **Feast Ingestion**
+
+Before a user ingests data into Feast, they should register one or more feature sets. These [feature sets](feature-sets.md) tell Feast where to find their data, how to ingest it, and also describe the characteristics of the data for validation purposes. Once a feature set is registered, Feast will start an Apache Beam job in order to populate a store with data from a source.
+
+In order for stores to be populated with data, users must publish the data to a [source](sources.md). Currently Feast only supports Apache Kafka as a source. Feast users \(or pipelines\) ingest batch data through the [Feast SDK](../getting-started/connecting-to-feast-1/connecting-to-feast.md) using its `ingest()` method. The SDK publishes the data straight to Kafka.
+
+Streaming systems can also ingest data into Feast. This is done by publishing to the correct Kafka topic in the expected format. Feast expects data to be in [FeatureRow.proto](https://api.docs.feast.dev/grpc/feast.types.pb.html#FeatureRow) format. The topic and brokers can be found on the feature set schema using the [Python SDK](../getting-started/connecting-to-feast-1/python-sdk.md).
+
+### **Stores**
+
+Stores are nothing more than databases used to store feature data. Feast loads data into stores through an ingestion process, after which the data can be served through the Feast Serving API. Stores are documented in the following section.
+
+{% page-ref page="stores.md" %}
+
+### **Feast Serving**
+
+ `Feast Serving` is the data access layer through which end users and production systems retrieve feature data. Each `Serving` instance is backed by a [store](stores.md).
+
+Since Feast supports multiple store types \(online, historical\) it is common to have two instances of Feast Serving deployed, one for online serving and one for historical serving. However, Feast allows for any number of `Feast Serving` deployments, meaning it is possible to deploy a `Feast Serving` deployment per production system, with its own stores and population jobs.
+
+`Serving` deployments can subscribe to a subset of feature data. Meaning they do not have to consume all features known to a `Feast Core` deployment.
+
+Feature retrieval \(and feature references\) are documented in more detail in subsequent sections
+
+{% page-ref page="feature-retrieval.md" %}
+
diff --git a/docs/user-guide/data-ingestion.md b/docs/user-guide/data-ingestion.md
new file mode 100644
index 0000000000..464a51bbfd
--- /dev/null
+++ b/docs/user-guide/data-ingestion.md
@@ -0,0 +1,126 @@
+# Data ingestion
+
+Users don't necessarily have to provide their external data sources in order to load data into Feast. Feast also allows users to exclude the source when registering a feature set. The following is a valid feature set specification.
+
+```python
+feature_set = FeatureSet(
+ name="stream_feature",
+ entities=[Entity("entity_id", ValueType.INT64)],
+ features=[Feature("feature_value1", ValueType.STRING)],
+)
+```
+
+If a user does not provide a source of data then they are required to publish data to Feast. This process is called ingestion.
+
+### Ingesting data
+
+The following example demonstrates how data ingestion works. For a full tutorial please see the [Telco Customer Churn Prediction Notebook](https://github.com/gojek/feast/blob/master/examples/feast-xgboost-churn-prediction-tutorial/Telecom%20Customer%20Churn%20Prediction%20%28with%20Feast%20and%20XGBoost%29.ipynb).
+
+1. Connect to Feast Core and load in a Pandas DataFrame.
+
+```python
+from feast import FeatureSet, Client, Entity
+import pandas as pd
+
+# Connect to Feast core
+client = Client(core_url="feast-core.example.com")
+
+# Load in customer data
+df = pd.read_csv("customer.csv")
+```
+
+1. Create, infer, and register a feature set from the DataFrame. This is a once off step that is required to initially register a feature set with Feast
+
+```python
+# Create an empty feature set
+customer_churn_fs = FeatureSet("customer_churn")
+
+# Infer the schema of the feature set from the Pandas DataFrame
+customer_churn_fs.infer_fields_from_df(
+ df,
+ entities=[Entity(name='customer_id',
+ dtype=ValueType.STRING)]
+ )
+
+# Register the feature set with Feast
+client.apply(customer_churn_fs)
+```
+
+1. We can also test that the feature set was correctly registered with Feast by retrieving it again and printing it out
+
+```text
+customer_churn_fs = client.get_feature_set('customer_churn')
+print(client.get_feature_set('customer_churn'))
+```
+
+```yaml
+{
+ "spec": {
+ "name": "customer_churn",
+ "entities": [
+ {
+ "name": "customer_id",
+ "valueType": "STRING"
+ }
+ ],
+ "features": [
+ {
+ "name": "churn",
+ "valueType": "INT64"
+ },
+ {
+ "name": "contract_month_to_month",
+ "valueType": "INT64"
+ },
+ {
+ "name": "streamingmovies",
+ "valueType": "INT64"
+ },
+ {
+ "name": "paperlessbilling",
+ "valueType": "INT64"
+ },
+ {
+ "name": "contract_two_year",
+ "valueType": "INT64"
+ },
+ {
+ "name": "partner",
+ "valueType": "INT64"
+ }
+ ],
+ "maxAge": "0s",
+ "source": {
+ "type": "KAFKA",
+ "kafkaSourceConfig": {
+ "bootstrapServers": "10.202.250.99:31190",
+ "topic": "feast"
+ }
+ },
+ "project": "default"
+ },
+ "meta": {
+ "createdTimestamp": "2020-03-15T07:47:52Z",
+ "status": "STATUS_READY"
+ }
+}
+```
+
+Once we are happy that the schema is correct, we can start to ingest the DataFrame into Feast.
+
+```text
+client.ingest(customer_churn_fs, telcom)
+```
+
+```text
+100%|██████████| 7032/7032 [00:02<00:00, 2771.19rows/s]
+Ingestion complete!
+
+Ingestion statistics:
+Success: 7032/7032A rows ingested
+```
+
+{% hint style="warning" %}
+Feast ingestion maintains the order of data that is ingested. This means that data that is written later will replace those that are written prior in stores. This is important to note when ingesting data that will end in a production system.
+{% endhint %}
+
diff --git a/docs/user-guide/entities.md b/docs/user-guide/entities.md
new file mode 100644
index 0000000000..f2383e440c
--- /dev/null
+++ b/docs/user-guide/entities.md
@@ -0,0 +1,43 @@
+# Entities
+
+An entity is any domain object that can be modeled and that information can be stored about. Entities are usually recognizable concepts, either concrete or abstract, such as persons, places, things, or events which have relevance to the modeled system.
+
+More formally, an entity is an instance of an entity type. An entity type is the class of entities where entities are the instances.
+
+* Examples of entity types in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`.
+* A specific driver, for example a driver with ID `D011234` would be an entity of the entity type `driver`
+
+An entity is the object on which features are observed. For example we could have a feature `total_trips_24h` on the driver `D01123` with a feature value of `11`.
+
+In the context of Feast, entities are important because they are used as keys when looking up feature values. Entities are also used when joining feature values between different feature sets in order to build one large data set to train a model, or to serve a model.
+
+Example entity properties
+
+{% code title="customer\_id.yaml" %}
+```yaml
+# Entity name
+name: customer_id
+
+# Entity value type
+value_type: INT64
+
+```
+{% endcode %}
+
+Entities can be created through the [Feast SDK](../getting-started/connecting-to-feast-1/connecting-to-feast.md) as follows
+
+```python
+from feast import Entity, ValueType, FeatureSet
+
+# Create a customer entity
+customer = Entity("customer_id", ValueType.INT64)
+
+# Create a feature set with only a single entity
+customer_feature_set = FeatureSet("customer_fs", entities=[customer])
+
+# Register the feature set with Feast
+client.apply(customer_feature_set)
+```
+
+Please see the [EntitySpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#EntitySpec) for the entity specification API.
+
diff --git a/docs/user-guide/feature-retrieval.md b/docs/user-guide/feature-retrieval.md
new file mode 100644
index 0000000000..0ebf79ca88
--- /dev/null
+++ b/docs/user-guide/feature-retrieval.md
@@ -0,0 +1,128 @@
+# Feature retrieval
+
+## 1. Overview
+
+Feature retrieval \(or serving\) is the process of retrieving either historical features or online features from Feast, for the purposes of training or serving a model.
+
+Feast attempts to unify the process of retrieving features in both the historical and online case. It does this through the creation of feature references. One of the major advantages of using Feast is that you have a single semantic reference to a feature. These feature references can then be stored alongside your model and loaded into a serving layer where it can be used for online feature retrieval.
+
+## 2. Feature references
+
+In Feast, each feature can be uniquely addressed through a feature reference. A feature reference is composed of the following components
+
+* Feature Set
+* Feature
+
+These components can be used to create a string based feature reference as follows
+
+`:`
+
+Feast will attempt to infer both the `feature-set` name if it is not provided, but a feature reference must provide a `feature` name.
+
+```python
+# Feature references
+features = [
+ 'partner',
+ 'daily_transactions',
+ 'customer_feature_set:dependents',
+ 'customer_feature_set:has_phone_service',
+ ]
+
+target = 'churn'
+```
+
+{% hint style="info" %}
+Where the Features from different Feature Sets use the same name, the Feature Set name `feature-set` is required to disambiguate which feature is specified.
+{% endhint %}
+
+Feature references only apply to a single `project`. Features cannot be retrieved across projects in a single request.
+
+## 3. Historical feature retrieval
+
+Historical feature retrieval can be done through either the Feast SDK or directly through the Feast Serving gRPC API. Below is an example of historical retrieval from the [Churn Prediction Notebook](https://github.com/gojek/feast/blob/master/examples/feast-xgboost-churn-prediction-tutorial/Telecom%20Customer%20Churn%20Prediction%20%28with%20Feast%20and%20XGBoost%29.ipynb).
+
+```python
+# Add the target variable to our feature list
+features = self._features + [self._target]
+
+# Retrieve training dataset from Feast. The "entity_df" is a dataframe that contains
+# timestamps and entity keys. In this case, it is a dataframe with two columns.
+# One timestamp column, and one customer id column
+dataset = client.get_batch_features(
+ feature_refs=features,
+ entity_rows=entity_df
+)
+
+# Materialize the dataset object to a Pandas DataFrame.
+# Alternatively it is possible to use a file reference if the data is too large
+df = dataset.to_dataframe()
+```
+
+{% hint style="info" %}
+When no project is specified when retrieving features with `get_batch_features()`, Feast infers that the features specified belong to the `default` project. To retrieve from another project, specify the `default` parameter when retrieving features.
+{% endhint %}
+
+In the above example, Feast does a point in time correct query from a single feature set. For each timestamp and entity key combination that is provided by `entity_df`, Feast determines the values of all the features in the `features` list at that respective point in time and then joins features values to that specific entity value and timestamp, and repeats this process for all timestamps.
+
+This is called a point in time correct join.
+
+Feast allows users to retrieve features from any feature sets and join them together in a single response dataset. The only requirement is that the user provides the correct entities in order to look up the features.
+
+### **Point-in-time-correct Join**
+
+Below is another example of how a point-in-time-correct join works. We have two dataframes. The first is the `entity dataframe` that contains timestamps, entities, and labels. The user would like to have driver features joined onto this `entity dataframe` from the `driver dataframe` to produce an `output dataframe` that contains both labels and features. They would then like to train their model on this output
+
+
+
+
+
+Typically the `input 1` DataFrame would be provided by the user, and the `input 2` DataFrame would already be ingested into Feast. To join these two, the user would call Feast as follows:
+
+```python
+# Feature references
+features = [
+ 'conv_rate',
+ 'acc_rate',
+ 'avg_daily_trips',
+ 'trip_completed'
+ ]
+
+
+dataset = client.get_batch_features(
+ feature_refs=features, # this is a list of feature references
+ entity_rows=entity_df # This is the entity dataframe above
+ )
+
+# This prints out the dataframe below
+print(dataset.to_dataframe())
+```
+
+
+
+Feast is able to intelligently join feature data with different timestamps to a single basis table in a point-in-time-correct way. This allows users to join daily batch data with high-frequency event data transparently. They simply need to know the feature names.
+
+Point-in-time-correct joins also prevents the occurrence of feature leakage by trying to accurate the state of the world at a single point in time, instead of just joining features based on the nearest timestamps.
+
+## Online feature retrieval
+
+Online feature retrieval works in much the same way as batch retrieval, with one important distinction: Online stores only maintain the current state of features. No historical data is served.
+
+```python
+features = [
+ 'conv_rate',
+ 'acc_rate',
+ 'avg_daily_trips',
+ ]
+
+data = client.get_online_features(
+ feature_refs=features, # Contains only feature references
+ entity_rows=entity_rows, # Contains only entities (driver ids)
+ )
+```
+
+{% hint style="info" %}
+When no project is specified when retrieving features with `get_online_feature()`, Feast infers that the features specified belong to the `default` project. To retrieve from another project, specify the `project` parameter when retrieving features.
+{% endhint %}
+
+Online serving with Feast is built to be very low latency. Feast Serving provides a [gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) that is backed by [Redis](https://redis.io/). We also provide support for [Python](https://api.docs.feast.dev/python/), [Go](https://godoc.org/github.com/gojek/feast/sdk/go), and Java clients.
+
diff --git a/docs/user-guide/feature-sets.md b/docs/user-guide/feature-sets.md
new file mode 100644
index 0000000000..fb03adad34
--- /dev/null
+++ b/docs/user-guide/feature-sets.md
@@ -0,0 +1,91 @@
+# Feature Sets
+
+Feature sets are both a schema and a means of identifying data sources for features.
+
+Data typically comes in the form of flat files, dataframes, tables in a database, or events on a stream. Thus the data occurs with multiple columns/fields in multiple rows/events.
+
+Feature sets are a way for defining the unique properties of these data sources, how Feast should interpret them, and how Feast should source them. Feature sets allow for groups of fields in these data sources to be [ingested](data-ingestion.md) and [stored](stores.md) together. Feature sets allow for efficient storage and logical namespacing of data within [stores](stores.md).
+
+{% hint style="info" %}
+Feature sets are a grouping of feature sets based on how they are loaded into Feast. They ensure that data is efficiently stored during ingestion. Feature sets are not a grouping of features for retrieval of features. During retrieval it is possible to retrieve feature values from any number of feature sets.
+{% endhint %}
+
+### Customer Transactions Example
+
+Below is an example specification of a basic `customer transactions` feature set that has been exported to YAML:
+
+{% tabs %}
+{% tab title="customer\_transactions\_feature\_set.yaml" %}
+```yaml
+name: customer_transactions
+entities:
+- name: customer_id
+ valueType: INT64
+features:
+- name: daily_transactions
+ valueType: FLOAT
+- name: total_transactions
+ valueType: FLOAT
+```
+{% endtab %}
+{% endtabs %}
+
+The dataframe below \(`customer_data.csv`\) contains the features and entities of the above feature set.
+
+| datetime | customer\_id | daily\_transactions | total\_tra**nsactions** |
+| :--- | :--- | :--- | :--- |
+| 2019-01-01 01:00:00 | 20001 | 5.0 | 14.0 |
+| 2019-01-01 01:00:00 | 20002 | 2.6 | 43.0 |
+| 2019-01-01 01:00:00 | 20003 | 4.1 | 154.0 |
+| 2019-01-01 01:00:00 | 20004 | 3.4 | 74.0 |
+
+In order to ingest feature data into Feast for this specific feature set:
+
+```python
+# Load dataframe
+customer_df = pd.read_csv("customer_data.csv")
+
+# Create feature set from YAML (using YAML is optional)
+cust_trans_fs = FeatureSet.from_yaml("customer_transactions_feature_set.yaml")
+
+# Apply new feature set
+client.apply(cust_trans_fs)
+
+# Load feature data into Feast for this specific feature set
+client.ingest(cust_trans_fs, customer_data)
+```
+
+{% hint style="info" %}
+When applying a Feature Set without specifying a project in its specification, Feast creates/updates the Feature Set in the `default` project. To create a Feature Set in another project, specify the project of choice in the Feature Set specification's project field.
+{% endhint %}
+
+### **Making changes to Feature Sets**
+
+In order to facilitate the need for feature set definitions to change over time, a limited set of changes can be made to existing feature sets.
+
+To apply changes to a feature set:
+
+```python
+# With existing feature set
+cust_trans_fs = FeatureSet.from_yaml("customer_transactions_feature_set.yaml")
+
+# Add new feature, avg_basket_size
+cust_trans_fs.add(Feature(name="avg_basket_size", dtype=ValueType.INT32))
+
+# Apply changed feature set
+client.apply(cust_trans_fs)
+```
+
+Permitted changes include:
+
+* Adding new features
+* Deleting existing features \(note that features are tombstoned and remain on record, rather than removed completely; as a result, new features will not be able to take the names of these deleted features\)
+* Changing features' TFX schemas
+* Changing the feature set's source and max age
+
+Note that the following are **not** allowed:
+
+* Changes to project or name of the feature set.
+* Changes to entities.
+* Changes to names and types of existing features.
+
diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md
new file mode 100644
index 0000000000..32a7b57b09
--- /dev/null
+++ b/docs/user-guide/features.md
@@ -0,0 +1,38 @@
+# Features
+
+A feature is an individual measurable property or characteristic of a phenomenon being observed. Features are the most important concepts within a feature store. Feature data is used both as input to models during training and when models are served in production.
+
+In the context of Feast, features are values that are associated with either one or more entities over time. In Feast, these values are either primitives or lists of primitives. Each feature can also have additional information attached to it.
+
+The following is a YAML representation of a feature specification. This specification would form part of a larger specification within a [feature set](feature-sets.md).
+
+{% code title="total\_trips\_feature.yaml" %}
+```yaml
+# Feature name
+name: total_trips_24h
+
+# Feature value type
+value_type: INT64
+```
+{% endcode %}
+
+ Features can be created through the[ Feast SDK](../getting-started/connecting-to-feast-1/connecting-to-feast.md) as follows
+
+```python
+from feast import Entity, Feature, ValueType, FeatureSet
+
+# Create a driver entity
+driver = Entity("driver_id", ValueType.INT64)
+
+# Create a total trips 24h feature
+total_trips_24h = Feature("total_trips_24h", ValueType.INT64)
+
+# Create a feature set with a single entity and a single feature
+driver_fs = FeatureSet("driver_fs", entities=[driver], features=[total_trips_24h])
+
+# Register the feature set with Feast
+client.apply(driver_fs)
+```
+
+Please see the [FeatureSpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#FeatureSpec) for the complete feature specification API.
+
diff --git a/docs/user-guide/overview.md b/docs/user-guide/overview.md
new file mode 100644
index 0000000000..0d45825368
--- /dev/null
+++ b/docs/user-guide/overview.md
@@ -0,0 +1,58 @@
+# Concepts
+
+## Using Feast
+
+Feast acts as the interface between ML models and data. Feast enables your team to
+
+1. Create feature specifications to manage features and load in data that should be managed
+2. Retrieve historical features for training models
+3. Retrieve online features for serving models
+
+{% hint style="info" %}
+Feast currently does not apply feature transformations to data.
+{% endhint %}
+
+### 1. Creating and managing features
+
+Feature creators model the data within their organization into Feast through the creation of [feature sets](feature-sets.md).
+
+Feature sets are specifications that contain both schema and data source information. They allow Feast to know how to interpret your data, and optionally where to find it. Feature sets allow users to define domain [entities](entities.md) along with the [features](features.md) that are available on these entities. Feature sets also allow users to define schemas that describe the properties of the data, which in turn can be used for validation purposes.
+
+Once a feature set has been registered, Feast will create the relevant schemas to store feature data within it's feature [stores](stores.md). These stores are then automatically populated by by jobs that ingest data from data [sources](sources.md), making it possible for Feast to provide access to features for training and serving. It is also possible for users to [ingest](data-ingestion.md) data into Feast instead of using an external source.
+
+Read more about [feature sets](feature-sets.md).
+
+### 2. Retrieving historical features during training
+
+Both online and historical retrieval are executed through an API call to `Feast Serving` using [feature references](feature-retrieval.md). In the case of historical serving it is necessary to provide Feast with the entities and timestamps that feature data will be joined to. Feast eagerly produces a point-in-time correct dataset based on the features that have been requested. These features can come from any number of feature sets.
+
+Stores supported: [BigQuery](https://cloud.google.com/bigquery)
+
+### 3. Retrieving online features during serving
+
+Feast also allows users to call `Feast Serving` for online feature data. Feast only stores the latest values during online serving for each feature, as opposed to historical serving where all historical values are stored. Online serving allows for very low latency requests to feature data at very high throughput.
+
+Stores supported: [Redis](https://redis.io/), [Redis Cluster](https://redis.io/topics/cluster-tutorial)
+
+## Concept Hierarchy
+
+
+
+Feast resources are arranged in the above hierarchy, with projects grouping one or more [feature sets](feature-sets.md), which in turn groups multiple [features](features.md) or [entities](entities.md).
+
+The logical grouping of these resources are important for namespacing as well as retrieval. During retrieval time it is necessary to reference individual features through feature references. These references uniquely identify a feature or entity within a Feast deployment.
+
+## Concepts
+
+[Entities](entities.md) are objects in an organization that model a specific construct. Examples of these include customers, transactions, and drivers.
+
+[Features](features.md) are measurable properties that are observed on entities. Features are used as inputs to models.
+
+[Feature Sets](feature-sets.md) are schemas that define logical groupings of entities, features, data sources, and other related metadata.
+
+[Stores](stores.md) are databases that maintain feature data that gets served to models during training or inference.
+
+[Sources](sources.md) are either internal or external data sources where feature data can be found.
+
+[Ingestion](data-ingestion.md) is the process of loading data into Feast.
+
diff --git a/docs/user-guide/sources.md b/docs/user-guide/sources.md
new file mode 100644
index 0000000000..35fcb41a11
--- /dev/null
+++ b/docs/user-guide/sources.md
@@ -0,0 +1,32 @@
+# Sources
+
+A `source` is a data source that can be used to find feature data. Users define sources as part of [feature sets](feature-sets.md). Once a feature set is registered with a source, Feast will automatically start to populate its stores with data from this source.
+
+{% hint style="info" %}
+Feast only supports [Kafka](https://kafka.apache.org/) as a source currently.
+{% endhint %}
+
+An example of a user provided source can be seen in the following code snippet
+
+```python
+feature_set = FeatureSet(
+ name="stream_feature",
+ entities=[
+ Entity("entity_id", ValueType.INT64)
+ ],
+ features=[
+ Feature("feature_value1", ValueType.STRING)
+ ],
+ source=KafkaSource(
+ brokers="mybroker:9092",
+ topic="my_feature_topic"
+ )
+)
+```
+
+Once this feature set is registered, Feast will start an ingestion job that retrieves data from this source and starts to populate all [stores](stores.md) that subscribe to it.
+
+In most cases a feature set \(and by extension its source\) will be used to populate both an online store and a historical store. This allows users to both train and serve their model with the same feature data.
+
+Feast will ensure that the source complies with the schema of the feature set. The event data has to be [Protobuf](https://developers.google.com/protocol-buffers) encoded and must contain the necessary [FeatureRow](https://api.docs.feast.dev/grpc/feast.types.pb.html#FeatureRow) structure.
+
diff --git a/docs/user-guide/stores.md b/docs/user-guide/stores.md
new file mode 100644
index 0000000000..c15cbb3db0
--- /dev/null
+++ b/docs/user-guide/stores.md
@@ -0,0 +1,37 @@
+# Stores
+
+In Feast, a store describes a database that is populated with feature data in order to be served to models.
+
+Feast supports two classes of stores
+
+* Historical stores
+* Online stores
+
+In order to populate these stores, Feast Core creates a long running ingestion job that streams in data from all feature sources to all stores that subscribe to those feature sets.
+
+
+
+## Historical Stores
+
+Historical stores maintain a complete history of feature data for the feature sets they are subscribed to.
+
+Feast currently only supports [Google BigQuery](https://cloud.google.com/bigquery) as a feature store, but we have [developed a storage API ](https://github.com/gojek/feast/issues/482)that makes adding a new store possible.
+
+Each historical store models its data differently, but in the case of a relational store \(like BigQuery\), each feature set maps directly to a table. Each feature and entity within a feature set maps directly to a column within a table.
+
+Data from historical stores can be used to train a model. In order to retrieve data from a historical store it is necessary to connect to a Feast Serving deployment and request historical features. Please see feature retrieval for more details.
+
+{% hint style="danger" %}
+Data is persisted in historical stores like BigQuery in log format. Repeated ingestions will duplicate the data is persisted in the store. Feast will automatically deduplicate data during retrieval, but it doesn't currently remove data from the stores themselves.
+{% endhint %}
+
+## Online Stores
+
+Online stores maintain only the latest values for a specific feature. Feast currently supports Redis as an online store. Online stores are meant for very high throughput writes from ingestion jobs and very low latency access to features during online serving.
+
+Please continue to the [feature retrieval](feature-retrieval.md) section for more details on retrieving data from online storage.
+
+## Subscriptions
+
+Stores are populated by ingestion jobs \(Apache Beam\) that retrieve feature data from sources based on subscriptions. These subscriptions are typically defined by the administrators of the Feast deployment. In most cases a store would simply subscribe to all features, but in some cases it may subscribe to a subset in order to improve performance or efficiency.
+
diff --git a/docs/why-feast.md b/docs/why-feast.md
new file mode 100644
index 0000000000..75048ace2c
--- /dev/null
+++ b/docs/why-feast.md
@@ -0,0 +1,32 @@
+# Why Feast?
+
+## Lack of feature reuse
+
+**Problem:** The process of engineering features is one of the most time consuming activities in building an end-to-end ML system. Despite this, many teams continue to redevelop the same features from scratch for every new project. Often these features never leaving the notebooks or pipelines they are built in.
+
+**Solution:** A centralized feature store allows organizations to build up a foundation of features that can be reused across projects. Teams are then able to utilize features developed by other teams, and as more features are added to the store it becomes easier and cheaper to build models.
+
+## Serving features is hard
+
+**Problem:** Serving up to date features at scale is hard. Raw data can come from a variety of sources, from data lakes, to even streams, to data warehouse, to simply flat files. Data scientists need the ability to produce massive datasets of features from this data in order to train their models offline. These models then need access to real-time feature data at low latency and high throughput when they are served in production.
+
+**Solution:** Feast is built to be able to ingest data from a variety of sources, supporting both streaming and batch sources. Once data is loaded into Feast as features, they become available through both a batch serving API as well as an real-time \(online serving\) API. These APIs allows data scientists and ML engineers to easily retrieve feature data for their development, training, or in production. Feast also comes with a Java, Go, and Python SDK to make this experience easy.
+
+## **Models need point-in-time correctness**
+
+**Problem:** Most data sources are not built with ML use cases in mind and by extension don't provide point-in-time correct lookups of feature data. One of the reasons why features are often re-engineered is because ML practitioners need to ensure that their models are trained on a dataset that accurately models the state of the world when the model runs in production.
+
+**Solution:** Feast allows end users to create point-in-time correct datasets across multiple entities. Feast ensures that there is no data leakage, that cross feature set joins are valid, and that models are not fed expired data.
+
+## Definitions of features vary
+
+**Problem:** Teams define features differently and there is no easy access to the documentation of a feature.
+
+**Solution:** Feast becomes the single source of truth for all feature data for all models within an organization. Teams are able to capture documentation, metadata and metrics about features. This allows teams to communicate clearly about features, test features data, and determine if a feature is useful for a particular model.
+
+## **Inconsistency between training and serving**
+
+**Problem:** Training requires access to historical data, whereas models that serve predictions need the latest values. Inconsistencies arise when data is siloed into many independent systems requiring separate tooling. Often teams are using Python for creating batch features off line, but these features are redeveloped with different libraries and languages when moving to serving or streaming systems.
+
+**Solution:** Feast provides consistency by managing and unifying the ingestion of data from batch and streaming sources into both the feature warehouse and feature serving stores. Feast becomes the bridge between your model and your data, both for training and serving. This ensures that there is a consistency in the feature data that your model receives.
+
diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb
index a6feb0ef13..a56121328c 100644
--- a/examples/basic/basic.ipynb
+++ b/examples/basic/basic.ipynb
@@ -28,23 +28,20 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"# Feast Core acts as the central feature registry\n",
- "FEAST_CORE_URL = os.getenv('FEAST_CORE_URL', 'core:6565')\n",
+ "FEAST_CORE_URL = os.getenv('FEAST_CORE_URL', 'localhost:6565')\n",
"\n",
"# Feast Online Serving allows for the retrieval of real-time feature data\n",
- "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'online-serving:6566')\n",
+ "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'localhost:6566')\n",
"\n",
"# Feast Batch Serving allows for the retrieval of historical feature data\n",
- "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'batch-serving:6567')\n",
- "\n",
- "# PYTHON_REPOSITORY_PATH is the path to the Python SDK inside the Feast Git Repo\n",
- "PYTHON_REPOSITORY_PATH = os.getenv('PYTHON_REPOSITORY_PATH', '../../')"
+ "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'localhost:6567')"
]
},
{
@@ -63,48 +60,57 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!python -m pip install --ignore-installed --upgrade feast"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "(Alternative) Install from local repository"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import sys\n",
- "os.environ['PYTHON_SDK_PATH'] = os.path.join(PYTHON_REPOSITORY_PATH, 'sdk/python')\n",
- "sys.path.append(os.environ['PYTHON_SDK_PATH'])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!echo $PYTHON_SDK_PATH"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!python -m pip install --ignore-installed --upgrade -e ${PYTHON_SDK_PATH}"
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: feast in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (0.5.0.post0)\n",
+ "Requirement already satisfied: google in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (2.0.3)\n",
+ "Requirement already satisfied: tabulate==0.8.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.8.7)\n",
+ "Requirement already satisfied: pandavro==1.5.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.5.1)\n",
+ "Requirement already satisfied: pandas==0.* in /home/zzy/.local/lib/python3.7/site-packages (from feast) (0.25.0)\n",
+ "Requirement already satisfied: google-cloud-core==1.0.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.0.3)\n",
+ "Requirement already satisfied: grpcio==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.29.0)\n",
+ "Requirement already satisfied: fastavro<0.23,>=0.22.11 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.22.13)\n",
+ "Requirement already satisfied: PyYAML==5.1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (5.1.2)\n",
+ "Requirement already satisfied: googleapis-common-protos==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.51.0)\n",
+ "Requirement already satisfied: tqdm==4.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (4.46.0)\n",
+ "Requirement already satisfied: numpy in /home/zzy/.local/lib/python3.7/site-packages (from feast) (1.17.4)\n",
+ "Requirement already satisfied: confluent-kafka in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.4.2)\n",
+ "Requirement already satisfied: google-cloud-bigquery-storage==0.7.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.7.0)\n",
+ "Requirement already satisfied: toml==0.10.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.10.1)\n",
+ "Requirement already satisfied: protobuf>=3.10 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (3.12.1)\n",
+ "Requirement already satisfied: google-cloud-bigquery==1.18.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.18.1)\n",
+ "Requirement already satisfied: pyarrow>=0.15.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (0.17.1)\n",
+ "Requirement already satisfied: google-api-core==1.14.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.14.3)\n",
+ "Requirement already satisfied: google-cloud-storage==1.20.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.20.0)\n",
+ "Requirement already satisfied: google-auth==1.6.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.6.3)\n",
+ "Requirement already satisfied: kafka-python==1.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (1.4.7)\n",
+ "Requirement already satisfied: Click==7.* in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from feast) (7.1.2)\n",
+ "Requirement already satisfied: beautifulsoup4 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google->feast) (4.9.1)\n",
+ "Requirement already satisfied: six>=1.9 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from pandavro==1.5.*->feast) (1.14.0)\n",
+ "Requirement already satisfied: python-dateutil>=2.6.1 in /home/zzy/.local/lib/python3.7/site-packages (from pandas==0.*->feast) (2.8.0)\n",
+ "Requirement already satisfied: pytz>=2017.2 in /home/zzy/.local/lib/python3.7/site-packages (from pandas==0.*->feast) (2019.1)\n",
+ "Requirement already satisfied: setuptools in /home/zzy/.local/lib/python3.7/site-packages (from protobuf>=3.10->feast) (41.6.0)\n",
+ "Requirement already satisfied: google-resumable-media<0.5.0dev,>=0.3.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-cloud-bigquery==1.18.*->feast) (0.4.1)\n",
+ "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-api-core==1.14.*->feast) (2.23.0)\n",
+ "Requirement already satisfied: rsa>=3.1.4 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (4.0)\n",
+ "Requirement already satisfied: cachetools>=2.0.0 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (4.1.0)\n",
+ "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from google-auth==1.6.*->feast) (0.2.8)\n",
+ "Requirement already satisfied: soupsieve>1.2 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from beautifulsoup4->google->feast) (2.0.1)\n",
+ "Requirement already satisfied: chardet<4,>=3.0.2 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (3.0.4)\n",
+ "Requirement already satisfied: idna<3,>=2.5 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (2.9)\n",
+ "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (1.25.8)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /home/zzy/.local/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core==1.14.*->feast) (2019.6.16)\n",
+ "Requirement already satisfied: pyasn1>=0.1.3 in /home/zzy/.conda/envs/feast-ml/lib/python3.7/site-packages (from rsa>=3.1.4->google-auth==1.6.*->feast) (0.4.8)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install feast"
]
},
{
@@ -116,7 +122,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -143,45 +149,13 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_ONLINE_SERVING_URL)"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Create a project workspace"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "client.create_project('customer_project')"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Set the active project"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "client.set_project('customer_project')"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -198,7 +172,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -210,9 +184,142 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
"
+ ],
+ "text/plain": [
+ " event_timestamp customer_id daily_transactions \\\n",
+ "0 2020-05-26 00:00:00+00:00 1001 2.416811 \n",
+ "1 2020-05-26 00:00:00+00:00 1004 6.617317 \n",
+ "2 2020-05-26 00:00:00+00:00 1003 4.409714 \n",
+ "3 2020-05-26 00:00:00+00:00 1005 1.032525 \n",
+ "4 2020-05-26 00:00:00+00:00 1002 4.817735 \n",
+ "\n",
+ " total_transactions \n",
+ "0 14 \n",
+ "1 6 \n",
+ "2 95 \n",
+ "3 86 \n",
+ "4 9 "
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"df = job.to_dataframe()\n",
- "print(df.head(10))"
+ "df.head()"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "feast-ml-py374",
"language": "python",
- "name": "python3"
+ "name": "feast-ml-py374"
},
"language_info": {
"codemirror_mode": {
@@ -501,17 +900,8 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
- },
- "pycharm": {
- "stem_cell": {
- "cell_type": "raw",
- "metadata": {
- "collapsed": false
- },
- "source": []
- }
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
}
diff --git a/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb b/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb
index e88fe970d5..c29c01efff 100644
--- a/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb
+++ b/examples/feast-xgboost-churn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb
@@ -6176,8 +6176,7 @@
"source": [
"os.environ['FEAST_CORE_URL'] = 'localhost:6565'\n",
"os.environ['FEAST_ONLINE_URL'] = 'localhost:6566'\n",
- "os.environ['FEAST_BATCH_URL'] = 'localhost:6567'\n",
- "os.environ['FEAST_PROJECT'] = 'default'"
+ "os.environ['FEAST_BATCH_URL'] = 'localhost:6567'"
]
},
{
@@ -6195,8 +6194,7 @@
"metadata": {},
"outputs": [],
"source": [
- "client = Client(core_url=os.environ['FEAST_CORE_URL'])\n",
- "client.set_project(os.environ['FEAST_PROJECT'])"
+ "client = Client(core_url=os.environ['FEAST_CORE_URL'])"
]
},
{
@@ -6490,7 +6488,6 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "\r",
" 0%| | 0/7032 [00:00 with the desired value
+kubectl create secret generic feast-postgresql \
+ --from-literal=postgresql-password=
+
+# Install Feast with Online Serving and Beam DirectRunner
+helm install --name myrelease feast-charts/feast \
+ --set feast-core.postgresql.existingSecret=feast-postgresql \
+ --set postgresql.existingSecret=feast-postgresql
```
-feast // top level feast chart
-│
-├── feast-core // feast-core subchart
-│ ├── postgresql // Postgresql dependency for feast-core (Feast database)
-│ └── kafka // Kafka dependency for feast-core (default stream source)
-│
-├── feast-serving-online // feast-serving subchart
-│ └── redis // Redis dependency for installation of store together with feast-serving
-│
-└── feast-serving-batch // feast-serving subchart
-```
+
+## Introduction
+This chart install Feast deployment on a Kubernetes cluster using the [Helm](https://v2.helm.sh/docs/using_helm/#installing-helm) package manager.
## Prerequisites
-- Kubernetes 1.13 or newer cluster
-- Helm 2.15.2 or newer
+- Kubernetes 1.12+
+- Helm 2.15+ (not tested with Helm 3)
+- Persistent Volume support on the underlying infrastructure
+
+## Chart Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| | feast-core | 0.5.0-alpha.1 |
+| | feast-serving | 0.5.0-alpha.1 |
+| | feast-serving | 0.5.0-alpha.1 |
+| | prometheus-statsd-exporter | 0.1.2 |
+| https://kubernetes-charts-incubator.storage.googleapis.com/ | kafka | 0.20.8 |
+| https://kubernetes-charts.storage.googleapis.com/ | grafana | 5.0.5 |
+| https://kubernetes-charts.storage.googleapis.com/ | postgresql | 8.6.1 |
+| https://kubernetes-charts.storage.googleapis.com/ | prometheus | 11.0.2 |
+| https://kubernetes-charts.storage.googleapis.com/ | redis | 10.5.6 |
-## Resources Required
-The chart deploys pods that consume minimum resources as specified in the resources configuration parameter.
+## Chart Values
-## Installing the Chart
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| feast-batch-serving.enabled | bool | `false` | Flag to install Feast Batch Serving |
+| feast-core.enabled | bool | `true` | Flag to install Feast Core |
+| feast-online-serving.enabled | bool | `true` | Flag to install Feast Online Serving |
+| grafana.enabled | bool | `true` | Flag to install Grafana |
+| kafka.enabled | bool | `true` | Flag to install Kafka |
+| postgresql.enabled | bool | `true` | Flag to install Postgresql |
+| prometheus-statsd-exporter.enabled | bool | `true` | Flag to install StatsD to Prometheus Exporter |
+| prometheus.enabled | bool | `true` | Flag to install Prometheus |
+| redis.enabled | bool | `true` | Flag to install Redis |
+
+## Configuration and installation details
+
+The default configuration will install Feast with Online Serving. Ingestion
+of features will use Beam [DirectRunner](https://beam.apache.org/documentation/runners/direct/)
+that runs on the same container where Feast Core is running.
-Add repository for Feast chart:
```bash
-helm repo add feast-charts https://feast-charts.storage.googleapis.com
-helm repo update
+# Create secret for Feast database, replace accordingly
+kubectl create secret generic feast-postgresql \
+ --from-literal=postgresql-password=
+
+# Install Feast with Online Serving and Beam DirectRunner
+helm install --name myrelease feast-charts/feast \
+ --set feast-core.postgresql.existingSecret=feast-postgresql \
+ --set postgresql.existingSecret=feast-postgresql
+```
+
+In order to test that the installation is successful:
+```bash
+helm test myrelease
+
+# If the installation is successful, the following should be printed
+RUNNING: myrelease-feast-online-serving-test
+PASSED: myrelease-feast-online-serving-test
+RUNNING: myrelease-grafana-test
+PASSED: myrelease-grafana-test
+RUNNING: myrelease-test-topic-create-consume-produce
+PASSED: myrelease-test-topic-create-consume-produce
+
+# Once the test completes, to check the logs
+kubectl logs myrelease-feast-online-serving-test
+```
+
+> The test pods can be safely deleted after the test finishes.
+> Check the yaml files in `templates/tests/` folder to see the processes
+> the test pods execute.
+
+### Feast metrics
+
+Feast default installation includes Grafana, StatsD exporter and Prometheus. Request
+metrics from Feast Core and Feast Serving, as well as ingestion statistic from
+Feast Ingestion are accessible from Prometheus and Grafana dashboard. The following
+show a quick example how to access the metrics.
+
+```
+# Forwards local port 9090 to the Prometheus server pod
+kubectl port-forward svc/myrelease-prometheus-server 9090:80
+```
+
+Visit http://localhost:9090 to access the Prometheus server:
+
+
+
+### Enable Batch Serving
+
+To install Feast Batch Serving for retrieval of historical features in offline
+training, access to BigQuery is required. First, create a [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) key that
+will provide the credentials to access BigQuery. Grant the service account `editor`
+role so it has write permissions to BigQuery and Cloud Storage.
+
+> In production, it is advised to give only the required [permissions](foo-feast-batch-serving-test) for the
+> the service account, versus `editor` role which is very permissive.
+
+Create a Kubernetes secret for the service account JSON file:
+```bash
+# By default Feast expects the secret to be named "feast-gcp-service-account"
+# and the JSON file to be named "credentials.json"
+kubectl create secret generic feast-gcp-service-account --from-file=credentials.json
+```
+
+Create a new Cloud Storage bucket (if not exists) and make sure the service
+account has write access to the bucket:
+```bash
+gsutil mb
```
-Install Feast release with minimal features, without batch serving and persistence:
+Use the following Helm values to enable Batch Serving:
+```yaml
+# values-batch-serving.yaml
+feast-core:
+ gcpServiceAccount:
+ enabled: true
+ postgresql:
+ existingSecret: feast-postgresql
+
+feast-batch-serving:
+ enabled: true
+ gcpServiceAccount:
+ enabled: true
+ application-override.yaml:
+ feast:
+ active_store: historical
+ stores:
+ - name: historical
+ type: BIGQUERY
+ config:
+ project_id:
+ dataset_id:
+ staging_location: gs:///feast-staging-location
+ initial_retry_delay_seconds: 3
+ total_timeout_seconds: 21600
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+postgresql:
+ existingSecret: feast-postgresql
+```
+
+> To delete the previous release, run `helm delete --purge myrelease`
+> Note this will not delete the persistent volume that has been claimed (PVC).
+> In a test cluster, run `kubectl delete pvc --all` to delete all claimed PVCs.
+
```bash
-RELEASE_NAME=demo
-helm install feast-charts/feast --name $RELEASE_NAME -f values-demo.yaml
+# Install a new release
+helm install --name myrelease -f values-batch-serving.yaml feast-charts/feast
+
+# Wait until all pods are created and running/completed (can take about 5m)
+kubectl get pods
+
+# Batch Serving is installed so `helm test` will also test for batch retrieval
+helm test myrelease
```
-Install Feast release for typical use cases, with batch and online serving:
+### Use DataflowRunner for ingestion
+
+Apache Beam [DirectRunner](https://beam.apache.org/documentation/runners/direct/)
+is not suitable for production use case because it is not easy to scale the
+number of workers and there is no convenient API to monitor and manage the
+workers. Feast supports [DataflowRunner](https://beam.apache.org/documentation/runners/dataflow/) which is a managed service on Google Cloud.
+
+> Make sure `feast-gcp-service-account` Kubernetes secret containing the
+> service account has been created and the service account has permissions
+> to manage Dataflow jobs.
+
+Since Dataflow workers run outside the Kube cluster and they will need to interact
+with Kafka brokers, Redis stores and StatsD server installed in the cluster,
+these services need to be exposed for access outside the cluster by setting
+`service.type: LoadBalancer`.
+
+In a typical use case, 5 `LoadBalancer` (internal) IP addresses are required by
+Feast when running with `DataflowRunner`. In Google Cloud, these (internal) IP
+addresses should be reserved first:
```bash
-# To install Feast Batch serving, BigQuery and Google Cloud service account
-# is required. The service account needs to have these roles:
-# - bigquery.dataEditor
-# - bigquery.jobUser
-#
-# Assuming a service account JSON file has been downloaded to /home/user/key.json,
-# run the following command to create a secret in Kubernetes
-# (make sure the file name is called key.json):
-kubectl create secret generic feast-gcp-service-account --from-file=/home/user/key.json
-
-# Set these required configuration in Feast Batch Serving
-STAGING_LOCATION=gs://bucket/path
-PROJECT_ID=google-cloud-project-id
-DATASET_ID=bigquery-dataset-id
-
-# Install the Helm release using default values.yaml
-helm install feast-charts/feast --name feast \
- --set feast-serving-batch."application\.yaml".feast.jobs.staging-location=$STAGING_LOCATION \
- --set feast-serving-batch."store\.yaml".bigquery_config.project_id=$PROJECT_ID \
- --set feast-serving-batch."store\.yaml".bigquery_config.dataset_id=$DATASET_ID
+# Check with your network configuration which IP addresses are available for use
+gcloud compute addresses create \
+ feast-kafka-1 feast-kafka-2 feast-kafka-3 feast-redis feast-statsd \
+ --region --subnet \
+ --addresses 10.128.0.11,10.128.0.12,10.128.0.13,10.128.0.14,10.128.0.15
```
-## Parameters
-
-The following table lists the configurable parameters of the Feast chart and their default values.
-
-| Parameter | Description | Default
-| --------- | ----------- | -------
-| `feast-core.enabled` | Flag to install Feast Core | `true`
-| `feast-core.postgresql.enabled` | Flag to install Postgresql as Feast database | `true`
-| `feast-core.postgresql.postgresqlDatabase` | Name of the database used by Feast Core | `feast`
-| `feast-core.postgresql.postgresqlUsername` | Username to authenticate to Feast database | `postgres`
-| `feast-core.postgresql.postgresqlPassword` | Passsword to authenticate to Feast database | `password`
-| `feast-core.kafka.enabled` | Flag to install Kafka as the default source for Feast | `true`
-| `feast-core.kafka.topics[0].name` | Default topic name in Kafka| `feast`
-| `feast-core.kafka.topics[0].replicationFactor` | No of replication factor for the topic| `1`
-| `feast-core.kafka.topics[0].partitions` | No of partitions for the topic | `1`
-| `feast-core.prometheus-statsd-exporter.enabled` | Flag to install Prometheus StatsD Exporter | `false`
-| `feast-core.prometheus-statsd-exporter.*` | Refer to this [link](charts/feast-core/charts/prometheus-statsd-exporter/values.yaml |
-| `feast-core.replicaCount` | No of pods to create | `1`
-| `feast-core.image.repository` | Repository for Feast Core Docker image | `gcr.io/kf-feast/feast-core`
-| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.4.4`
-| `feast-core.image.pullPolicy` | Image pull policy for Feast Core Docker image | `IfNotPresent`
-| `feast-core.prometheus.enabled` | Add annotations to enable Prometheus scraping | `false`
-| `feast-core.application.yaml` | Configuration for Feast Core application | Refer to this [link](charts/feast-core/values.yaml)
-| `feast-core.springConfigMountPath` | Directory to mount application.yaml | `/etc/feast/feast-core`
-| `feast-core.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
-| `feast-core.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
-| `feast-core.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
-| `feast-core.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
-| `feast-core.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
-| `feast-core.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-core.jar`
-| `feast-core.jvmOptions` | Options for the JVM | `[]`
-| `feast-core.logLevel` | Application logging level | `warn`
-| `feast-core.logType` | Application logging type (`JSON` or `Console`) | `JSON`
-| `feast-core.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
-| `feast-core.springSecretProfiles` | Map of profile name to file content for additional Spring profiles. Use this instead of springConfigProfiles if the content contains secrets. | `{}`
-| `feast-core.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
-| `feast-core.springSecretProfilesActive` | CSV of profiles to enable from `springSecretProfiles` | `""`
-| `feast-core.livenessProbe.enabled` | Flag to enable liveness probe | `true`
-| `feast-core.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
-| `feast-core.livenessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-core.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5`
-| `feast-core.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-core.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5`
-| `feast-core.readinessProbe.enabled` | Flag to enable readiness probe | `true`
-| `feast-core.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30`
-| `feast-core.readinessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-core.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10`
-| `feast-core.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-core.service.type` | Kubernetes Service Type | `ClusterIP`
-| `feast-core.http.port` | Kubernetes Service port for HTTP request| `80`
-| `feast-core.http.targetPort` | Container port for HTTP request | `8080`
-| `feast-core.grpc.port` | Kubernetes Service port for GRPC request| `6565`
-| `feast-core.grpc.targetPort` | Container port for GRPC request| `6565`
-| `feast-core.resources` | CPU and memory allocation for the pod | `{}`
-| `feast-core.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
-| `feast-serving-online.enabled` | Flag to install Feast Online Serving | `true`
-| `feast-serving-online.redis.enabled` | Flag to install Redis in Feast Serving | `false`
-| `feast-serving-online.redis.usePassword` | Flag to use password to access Redis | `false`
-| `feast-serving-online.redis.cluster.enabled` | Flag to enable Redis cluster | `false`
-| `feast-serving-online.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
-| `feast-serving-online.replicaCount` | No of pods to create | `1`
-| `feast-serving-online.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
-| `feast-serving-online.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
-| `feast-serving-online.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
-| `feast-serving-online.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml)
-| `feast-serving-online.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml)
-| `feast-serving-online.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
-| `feast-serving-online.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
-| `feast-serving-online.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
-| `feast-serving-online.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
-| `feast-serving-online.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
-| `feast-serving-online.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
-| `feast-serving-online.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
-| `feast-serving-online.jvmOptions` | Options for the JVM | `[]`
-| `feast-serving-online.logLevel` | Application logging level | `warn`
-| `feast-serving-online.logType` | Application logging type (`JSON` or `Console`) | `JSON`
-| `feast-serving-online.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
-| `feast-serving-online.springSecretProfiles` | Map of profile name to file content for additional Spring profiles. Use this instead of springConfigProfiles if the content contains secrets. | `{}`
-| `feast-serving-online.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
-| `feast-serving-online.springSecretProfilesActive` | CSV of profiles to enable from `springSecretProfiles` | `""`
-| `feast-serving-online.livenessProbe.enabled` | Flag to enable liveness probe | `true`
-| `feast-serving-online.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
-| `feast-serving-online.livenessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-serving-online.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5`
-| `feast-serving-online.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-serving-online.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5`
-| `feast-serving-online.readinessProbe.enabled` | Flag to enable readiness probe | `true`
-| `feast-serving-online.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30`
-| `feast-serving-online.readinessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-serving-online.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10`
-| `feast-serving-online.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-serving-online.service.type` | Kubernetes Service Type | `ClusterIP`
-| `feast-serving-online.http.port` | Kubernetes Service port for HTTP request| `80`
-| `feast-serving-online.http.targetPort` | Container port for HTTP request | `8080`
-| `feast-serving-online.grpc.port` | Kubernetes Service port for GRPC request| `6566`
-| `feast-serving-online.grpc.targetPort` | Container port for GRPC request| `6566`
-| `feast-serving-online.resources` | CPU and memory allocation for the pod | `{}`
-| `feast-serving-online.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
-| `feast-serving-batch.enabled` | Flag to install Feast Batch Serving | `true`
-| `feast-serving-batch.redis.enabled` | Flag to install Redis in Feast Serving | `false`
-| `feast-serving-batch.redis.usePassword` | Flag to use password to access Redis | `false`
-| `feast-serving-batch.redis.cluster.enabled` | Flag to enable Redis cluster | `false`
-| `feast-serving-batch.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
-| `feast-serving-batch.replicaCount` | No of pods to create | `1`
-| `feast-serving-batch.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
-| `feast-serving-batch.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
-| `feast-serving-batch.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
-| `feast-serving-batch.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml)
-| `feast-serving-batch.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml)
-| `feast-serving-batch.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
-| `feast-serving-batch.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
-| `feast-serving-batch.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
-| `feast-serving-batch.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
-| `feast-serving-batch.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
-| `feast-serving-batch.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
-| `feast-serving-batch.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
-| `feast-serving-batch.jvmOptions` | Options for the JVM | `[]`
-| `feast-serving-batch.logLevel` | Application logging level | `warn`
-| `feast-serving-batch.logType` | Application logging type (`JSON` or `Console`) | `JSON`
-| `feast-serving-batch.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
-| `feast-serving-batch.springSecretProfiles` | Map of profile name to file content for additional Spring profiles. Use this instead of springConfigProfiles if the content contains secrets. | `{}`
-| `feast-serving-batch.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
-| `feast-serving-batch.springSecretProfilesActive` | CSV of profiles to enable from `springSecretProfiles` | `""`
-| `feast-serving-batch.livenessProbe.enabled` | Flag to enable liveness probe | `true`
-| `feast-serving-batch.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
-| `feast-serving-batch.livenessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-serving-batch.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5`
-| `feast-serving-batch.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-serving-batch.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5`
-| `feast-serving-batch.readinessProbe.enabled` | Flag to enable readiness probe | `true`
-| `feast-serving-batch.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30`
-| `feast-serving-batch.readinessProbe.periodSeconds` | How often to perform the probe | `10`
-| `feast-serving-batch.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10`
-| `feast-serving-batch.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1`
-| `feast-serving-batch.service.type` | Kubernetes Service Type | `ClusterIP`
-| `feast-serving-batch.http.port` | Kubernetes Service port for HTTP request| `80`
-| `feast-serving-batch.http.targetPort` | Container port for HTTP request | `8080`
-| `feast-serving-batch.grpc.port` | Kubernetes Service port for GRPC request| `6566`
-| `feast-serving-batch.grpc.targetPort` | Container port for GRPC request| `6566`
-| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}`
-| `feast-serving-batch.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
-
-## Ingress Parameters
-
-The following table lists the configurable parameters of the ingress section for each Feast module.
-
-Note, there are two ingresses available for each module - `grpc` and `http`.
-
-| Parameter | Description | Default
-| ----------------------------- | ----------- | -------
-| `ingress.grcp.enabled` | Enables an ingress (endpoint) for the gRPC server | `false`
-| `ingress.grcp.*` | See below |
-| `ingress.http.enabled` | Enables an ingress (endpoint) for the HTTP server | `false`
-| `ingress.http.*` | See below |
-| `ingress.*.class` | Value for `kubernetes.io/ingress.class` | `nginx`
-| `ingress.*.hosts` | List of host-names for the ingress | `[]`
-| `ingress.*.annotations` | Additional ingress annotations | `{}`
-| `ingress.*.https.enabled` | Add a tls section to the ingress | `true`
-| `ingress.*.https.secretNames` | Map of hostname to TLS secret name | `{}` If not specified, defaults to `domain-tld-tls` e.g. `feast.example.com` uses secret `example-com-tls`
-| `ingress.*.auth.enabled` | Enable auth on the ingress (only applicable for `nginx` type | `false`
-| `ingress.*.auth.signinHost` | External hostname of the OAuth2 proxy to use | First item in `ingress.hosts`, replacing the sub-domain with 'auth' e.g. `feast.example.com` uses `auth.example.com`
-| `ingress.*.auth.authUrl` | Internal URI to internal auth endpoint | `http://auth-server.auth-ns.svc.cluster.local/auth`
-| `ingress.*.whitelist` | Subnet masks to whitelist (i.e. value for `nginx.ingress.kubernetes.io/whitelist-source-range`) | `"""`
-
-To enable all the ingresses will a config like the following (while also adding the hosts etc):
+Use the following Helm values to enable DataflowRuner (and Batch Serving),
+replacing the `<*load_balancer_ip*>` tags with the ip addresses reserved above:
```yaml
+# values-dataflow-runner.yaml
feast-core:
- ingress:
- grpc:
- enabled: true
- http:
- enabled: true
-feast-serving-online:
- ingress:
- grpc:
- enabled: true
- http:
- enabled: true
-feast-serving-batch:
- ingress:
- grpc:
- enabled: true
- http:
- enabled: true
+ gcpServiceAccount:
+ enabled: true
+ postgresql:
+ existingSecret: feast-postgresql
+ application-override.yaml:
+ feast:
+ stream:
+ options:
+ bootstrapServers:
+ jobs:
+ active_runner: dataflow
+ metrics:
+ host:
+ runners:
+ - name: dataflow
+ type: DataflowRunner
+ options:
+ project:
+ region:
+ zone:
+ tempLocation:
+ network:
+ subnetwork:
+ maxNumWorkers: 1
+ autoscalingAlgorithm: THROUGHPUT_BASED
+ usePublicIps: false
+ workerMachineType: n1-standard-1
+ deadLetterTableSpec:
+
+feast-online-serving:
+ application-override.yaml:
+ feast:
+ stores:
+ - name: online
+ type: REDIS
+ config:
+ host:
+ port: 6379
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+feast-batch-serving:
+ enabled: true
+ gcpServiceAccount:
+ enabled: true
+ application-override.yaml:
+ feast:
+ active_store: historical
+ stores:
+ - name: historical
+ type: BIGQUERY
+ config:
+ project_id:
+ dataset_id:
+ staging_location: gs:///feast-staging-location
+ initial_retry_delay_seconds: 3
+ total_timeout_seconds: 21600
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+postgresql:
+ existingSecret: feast-postgresql
+
+kafka:
+ external:
+ enabled: true
+ type: LoadBalancer
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+ firstListenerPort: 31090
+ loadBalancerIP:
+ -
+ -
+ -
+ configurationOverrides:
+ "advertised.listeners": |-
+ EXTERNAL://${LOAD_BALANCER_IP}:31090
+ "listener.security.protocol.map": |-
+ PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+ "log.retention.hours": 1
+
+redis:
+ master:
+ service:
+ type: LoadBalancer
+ loadBalancerIP:
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+
+prometheus-statsd-exporter:
+ service:
+ type: LoadBalancer
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+ loadBalancerIP:
+```
+
+```bash
+# Install a new release
+helm install --name myrelease -f values-dataflow-runner.yaml feast-charts/feast
+
+# Wait until all pods are created and running/completed (can take about 5m)
+kubectl get pods
+
+# Test the installation
+helm test myrelease
```
+If the tests are successful, Dataflow jobs should appear in Google Cloud console
+running features ingestion: https://console.cloud.google.com/dataflow
+
+
+
+### Production configuration
+
+#### Resources requests
+
+The `resources` field in the deployment spec is left empty in the examples. In
+production these should be set according to the load each services are expected
+to handle and the service level objectives (SLO). Also Feast Core and Serving
+is Java application and it is [good practice](https://stackoverflow.com/a/6916718/3949303)
+to set the minimum and maximum heap. This is an example reasonable value to set for Feast Serving:
+
+```yaml
+feast-online-serving:
+ javaOpts: "-Xms2048m -Xmx2048m"
+ resources:
+ limits:
+ memory: "2048Mi"
+ requests:
+ memory: "2048Mi"
+ cpu: "1"
+```
+
+#### High availability
+
+Default Feast installation only configures a single instance of Redis
+server. If due to network failures or out of memory error Redis is down,
+Feast serving will fail to respond to requests. Soon, Feast will support
+highly available Redis via [Redis cluster](https://redis.io/topics/cluster-tutorial),
+sentinel or additional proxies.
+
+### Documentation development
+
+This `README.md` is generated using [helm-docs](https://github.com/norwoodj/helm-docs/).
+Please run `helm-docs` to regenerate the `README.md` every time `README.md.gotmpl`
+or `values.yaml` are updated.
diff --git a/infra/charts/feast/README.md.gotmpl b/infra/charts/feast/README.md.gotmpl
new file mode 100644
index 0000000000..69d40fbb25
--- /dev/null
+++ b/infra/charts/feast/README.md.gotmpl
@@ -0,0 +1,354 @@
+{{ template "chart.header" . }}
+
+{{ template "chart.description" . }} {{ template "chart.versionLine" . }}
+
+## TL;DR;
+
+```bash
+# Add Feast Helm chart
+helm repo add feast-charts https://feast-charts.storage.googleapis.com
+helm repo update
+
+# Create secret for Feast database, replace with the desired value
+kubectl create secret generic feast-postgresql \
+ --from-literal=postgresql-password=
+
+# Install Feast with Online Serving and Beam DirectRunner
+helm install --name myrelease feast-charts/feast \
+ --set feast-core.postgresql.existingSecret=feast-postgresql \
+ --set postgresql.existingSecret=feast-postgresql
+```
+
+## Introduction
+This chart install Feast deployment on a Kubernetes cluster using the [Helm](https://v2.helm.sh/docs/using_helm/#installing-helm) package manager.
+
+## Prerequisites
+- Kubernetes 1.12+
+- Helm 2.15+ (not tested with Helm 3)
+- Persistent Volume support on the underlying infrastructure
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}
+
+## Configuration and installation details
+
+The default configuration will install Feast with Online Serving. Ingestion
+of features will use Beam [DirectRunner](https://beam.apache.org/documentation/runners/direct/)
+that runs on the same container where Feast Core is running.
+
+```bash
+# Create secret for Feast database, replace accordingly
+kubectl create secret generic feast-postgresql \
+ --from-literal=postgresql-password=
+
+# Install Feast with Online Serving and Beam DirectRunner
+helm install --name myrelease feast-charts/feast \
+ --set feast-core.postgresql.existingSecret=feast-postgresql \
+ --set postgresql.existingSecret=feast-postgresql
+```
+
+In order to test that the installation is successful:
+```bash
+helm test myrelease
+
+# If the installation is successful, the following should be printed
+RUNNING: myrelease-feast-online-serving-test
+PASSED: myrelease-feast-online-serving-test
+RUNNING: myrelease-grafana-test
+PASSED: myrelease-grafana-test
+RUNNING: myrelease-test-topic-create-consume-produce
+PASSED: myrelease-test-topic-create-consume-produce
+
+# Once the test completes, to check the logs
+kubectl logs myrelease-feast-online-serving-test
+```
+
+> The test pods can be safely deleted after the test finishes.
+> Check the yaml files in `templates/tests/` folder to see the processes
+> the test pods execute.
+
+### Feast metrics
+
+Feast default installation includes Grafana, StatsD exporter and Prometheus. Request
+metrics from Feast Core and Feast Serving, as well as ingestion statistic from
+Feast Ingestion are accessible from Prometheus and Grafana dashboard. The following
+show a quick example how to access the metrics.
+
+```
+# Forwards local port 9090 to the Prometheus server pod
+kubectl port-forward svc/myrelease-prometheus-server 9090:80
+```
+
+Visit http://localhost:9090 to access the Prometheus server:
+
+
+
+### Enable Batch Serving
+
+To install Feast Batch Serving for retrieval of historical features in offline
+training, access to BigQuery is required. First, create a [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) key that
+will provide the credentials to access BigQuery. Grant the service account `editor`
+role so it has write permissions to BigQuery and Cloud Storage.
+
+> In production, it is advised to give only the required [permissions](foo-feast-batch-serving-test) for the
+> the service account, versus `editor` role which is very permissive.
+
+Create a Kubernetes secret for the service account JSON file:
+```bash
+# By default Feast expects the secret to be named "feast-gcp-service-account"
+# and the JSON file to be named "credentials.json"
+kubectl create secret generic feast-gcp-service-account --from-file=credentials.json
+```
+
+Create a new Cloud Storage bucket (if not exists) and make sure the service
+account has write access to the bucket:
+```bash
+gsutil mb
+```
+
+Use the following Helm values to enable Batch Serving:
+```yaml
+# values-batch-serving.yaml
+feast-core:
+ gcpServiceAccount:
+ enabled: true
+ postgresql:
+ existingSecret: feast-postgresql
+
+feast-batch-serving:
+ enabled: true
+ gcpServiceAccount:
+ enabled: true
+ application-override.yaml:
+ feast:
+ active_store: historical
+ stores:
+ - name: historical
+ type: BIGQUERY
+ config:
+ project_id:
+ dataset_id:
+ staging_location: gs:///feast-staging-location
+ initial_retry_delay_seconds: 3
+ total_timeout_seconds: 21600
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+postgresql:
+ existingSecret: feast-postgresql
+```
+
+> To delete the previous release, run `helm delete --purge myrelease`
+> Note this will not delete the persistent volume that has been claimed (PVC).
+> In a test cluster, run `kubectl delete pvc --all` to delete all claimed PVCs.
+
+```bash
+# Install a new release
+helm install --name myrelease -f values-batch-serving.yaml feast-charts/feast
+
+# Wait until all pods are created and running/completed (can take about 5m)
+kubectl get pods
+
+# Batch Serving is installed so `helm test` will also test for batch retrieval
+helm test myrelease
+```
+
+### Use DataflowRunner for ingestion
+
+Apache Beam [DirectRunner](https://beam.apache.org/documentation/runners/direct/)
+is not suitable for production use case because it is not easy to scale the
+number of workers and there is no convenient API to monitor and manage the
+workers. Feast supports [DataflowRunner](https://beam.apache.org/documentation/runners/dataflow/) which is a managed service on Google Cloud.
+
+> Make sure `feast-gcp-service-account` Kubernetes secret containing the
+> service account has been created and the service account has permissions
+> to manage Dataflow jobs.
+
+Since Dataflow workers run outside the Kube cluster and they will need to interact
+with Kafka brokers, Redis stores and StatsD server installed in the cluster,
+these services need to be exposed for access outside the cluster by setting
+`service.type: LoadBalancer`.
+
+In a typical use case, 5 `LoadBalancer` (internal) IP addresses are required by
+Feast when running with `DataflowRunner`. In Google Cloud, these (internal) IP
+addresses should be reserved first:
+```bash
+# Check with your network configuration which IP addresses are available for use
+gcloud compute addresses create \
+ feast-kafka-1 feast-kafka-2 feast-kafka-3 feast-redis feast-statsd \
+ --region --subnet \
+ --addresses 10.128.0.11,10.128.0.12,10.128.0.13,10.128.0.14,10.128.0.15
+```
+
+Use the following Helm values to enable DataflowRuner (and Batch Serving),
+replacing the `<*load_balancer_ip*>` tags with the ip addresses reserved above:
+
+```yaml
+# values-dataflow-runner.yaml
+feast-core:
+ gcpServiceAccount:
+ enabled: true
+ postgresql:
+ existingSecret: feast-postgresql
+ application-override.yaml:
+ feast:
+ stream:
+ options:
+ bootstrapServers:
+ jobs:
+ active_runner: dataflow
+ metrics:
+ host:
+ runners:
+ - name: dataflow
+ type: DataflowRunner
+ options:
+ project:
+ region:
+ zone:
+ tempLocation:
+ network:
+ subnetwork:
+ maxNumWorkers: 1
+ autoscalingAlgorithm: THROUGHPUT_BASED
+ usePublicIps: false
+ workerMachineType: n1-standard-1
+ deadLetterTableSpec:
+
+feast-online-serving:
+ application-override.yaml:
+ feast:
+ stores:
+ - name: online
+ type: REDIS
+ config:
+ host:
+ port: 6379
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+feast-batch-serving:
+ enabled: true
+ gcpServiceAccount:
+ enabled: true
+ application-override.yaml:
+ feast:
+ active_store: historical
+ stores:
+ - name: historical
+ type: BIGQUERY
+ config:
+ project_id:
+ dataset_id:
+ staging_location: gs:///feast-staging-location
+ initial_retry_delay_seconds: 3
+ total_timeout_seconds: 21600
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+postgresql:
+ existingSecret: feast-postgresql
+
+kafka:
+ external:
+ enabled: true
+ type: LoadBalancer
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+ firstListenerPort: 31090
+ loadBalancerIP:
+ -
+ -
+ -
+ configurationOverrides:
+ "advertised.listeners": |-
+ EXTERNAL://${LOAD_BALANCER_IP}:31090
+ "listener.security.protocol.map": |-
+ PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+ "log.retention.hours": 1
+
+redis:
+ master:
+ service:
+ type: LoadBalancer
+ loadBalancerIP:
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+
+prometheus-statsd-exporter:
+ service:
+ type: LoadBalancer
+ annotations:
+ cloud.google.com/load-balancer-type: Internal
+ loadBalancerSourceRanges:
+ - 10.0.0.0/8
+ - 172.16.0.0/12
+ - 192.168.0.0/16
+ loadBalancerIP:
+```
+
+```bash
+# Install a new release
+helm install --name myrelease -f values-dataflow-runner.yaml feast-charts/feast
+
+# Wait until all pods are created and running/completed (can take about 5m)
+kubectl get pods
+
+# Test the installation
+helm test myrelease
+```
+
+If the tests are successful, Dataflow jobs should appear in Google Cloud console
+running features ingestion: https://console.cloud.google.com/dataflow
+
+
+
+### Production configuration
+
+#### Resources requests
+
+The `resources` field in the deployment spec is left empty in the examples. In
+production these should be set according to the load each services are expected
+to handle and the service level objectives (SLO). Also Feast Core and Serving
+is Java application and it is [good practice](https://stackoverflow.com/a/6916718/3949303)
+to set the minimum and maximum heap. This is an example reasonable value to set for Feast Serving:
+
+```yaml
+feast-online-serving:
+ javaOpts: "-Xms2048m -Xmx2048m"
+ resources:
+ limits:
+ memory: "2048Mi"
+ requests:
+ memory: "2048Mi"
+ cpu: "1"
+```
+
+#### High availability
+
+Default Feast installation only configures a single instance of Redis
+server. If due to network failures or out of memory error Redis is down,
+Feast serving will fail to respond to requests. Soon, Feast will support
+highly available Redis via [Redis cluster](https://redis.io/topics/cluster-tutorial),
+sentinel or additional proxies.
+
+### Documentation development
+
+This `README.md` is generated using [helm-docs](https://github.com/norwoodj/helm-docs/).
+Please run `helm-docs` to regenerate the `README.md` every time `README.md.gotmpl`
+or `values.yaml` are updated.
diff --git a/infra/charts/feast/charts/feast-core/.helmignore b/infra/charts/feast/charts/feast-core/.helmignore
deleted file mode 100644
index 50af031725..0000000000
--- a/infra/charts/feast/charts/feast-core/.helmignore
+++ /dev/null
@@ -1,22 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml
index 86d0699b9a..5b832943cf 100644
--- a/infra/charts/feast/charts/feast-core/Chart.yaml
+++ b/infra/charts/feast/charts/feast-core/Chart.yaml
@@ -1,4 +1,4 @@
apiVersion: v1
-description: A Helm chart for core component of Feast
+description: Feast Core registers feature specifications and manage ingestion jobs.
name: feast-core
-version: 0.4.4
+version: 0.5.0-alpha.1
diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md
new file mode 100644
index 0000000000..40d6bd9112
--- /dev/null
+++ b/infra/charts/feast/charts/feast-core/README.md
@@ -0,0 +1,70 @@
+feast-core
+==========
+Feast Core registers feature specifications and manage ingestion jobs.
+
+Current chart version is `0.5.0-alpha.1`
+
+
+
+
+
+## Chart Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration. |
+| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` |
+| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. |
+| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. |
+| envOverrides | object | `{}` | Extra environment variables to set |
+| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow |
+| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key |
+| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) |
+| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
+| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| image.repository | string | `"gcr.io/kf-feast/feast-core"` | Docker image repository |
+| image.tag | string | `"dev"` | Image tag |
+| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges |
+| ingress.http.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service |
+| ingress.http.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.http.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.http.whitelist | string | `""` | Allowed client IP source ranges |
+| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap: `-Xms2048m -Xmx2048m` |
+| livenessProbe.enabled | bool | `true` | Flag to enabled the probe |
+| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated |
+| livenessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| livenessProbe.timeoutSeconds | int | `5` | When the probe times out |
+| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` |
+| logType | string | `"Console"` | Log format, either `JSON` or `Console` |
+| nodeSelector | object | `{}` | Node labels for pod assignment |
+| postgresql.existingSecret | string | `""` | Existing secret to use for authenticating to Postgres |
+| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics |
+| readinessProbe.enabled | bool | `true` | Flag to enabled the probe |
+| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| readinessProbe.initialDelaySeconds | int | `20` | Delay before the probe is initiated |
+| readinessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| readinessProbe.timeoutSeconds | int | `10` | When the probe times out |
+| replicaCount | int | `1` | Number of pods that will be created |
+| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) |
+| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.grpc.port | int | `6565` | Service port for GRPC requests |
+| service.grpc.targetPort | int | `6565` | Container port serving GRPC requests |
+| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.http.port | int | `80` | Service port for HTTP requests |
+| service.http.targetPort | int | `8080` | Container port serving HTTP requests and Prometheus metrics |
+| service.type | string | `"ClusterIP"` | Kubernetes service type |
diff --git a/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz b/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz
deleted file mode 100644
index 76a2247577..0000000000
Binary files a/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz and /dev/null differ
diff --git a/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz b/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz
deleted file mode 100644
index f61421514d..0000000000
Binary files a/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz and /dev/null differ
diff --git a/infra/charts/feast/charts/feast-core/requirements.yaml b/infra/charts/feast/charts/feast-core/requirements.yaml
deleted file mode 100644
index ef1e39a7d0..0000000000
--- a/infra/charts/feast/charts/feast-core/requirements.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-dependencies:
-- name: postgresql
- version: 6.5.5
- repository: "@stable"
- condition: postgresql.enabled
-- name: kafka
- version: 0.20.1
- repository: "@incubator"
- condition: kafka.enabled
-- name: common
- version: 0.0.5
- repository: "@incubator"
-- name: prometheus-statsd-exporter
- version: 0.1.2
- condition: prometheus-statsd-exporter.enabled
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-core/templates/configmap.yaml b/infra/charts/feast/charts/feast-core/templates/configmap.yaml
index da45cad5bd..b48e15cc98 100644
--- a/infra/charts/feast/charts/feast-core/templates/configmap.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/configmap.yaml
@@ -10,44 +10,29 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
data:
- application.yaml: |
-{{- toYaml (index .Values "application.yaml") | nindent 4 }}
-
-{{- if .Values.postgresql.enabled }}
- application-bundled-postgresql.yaml: |
+ application-generated.yaml: |
+{{- if index .Values "application-generated.yaml" "enabled" }}
spring:
datasource:
- url: {{ printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase) }}
- driverClassName: org.postgresql.Driver
-{{- end }}
-
-{{ if .Values.kafka.enabled }}
- {{- $topic := index .Values.kafka.topics 0 }}
- application-bundled-kafka.yaml: |
+ url: jdbc:postgresql://{{ .Release.Name }}-postgresql:5432/postgres
feast:
stream:
type: kafka
- options:
- topic: {{ $topic.name | quote }}
- replicationFactor: {{ $topic.replicationFactor }}
- partitions: {{ $topic.partitions }}
- {{- if not .Values.kafka.external.enabled }}
- bootstrapServers: {{ printf "%s:9092" (printf "%s-kafka" .Release.Name) }}
- {{- end }}
-{{- end }}
-
-{{- if (index .Values "prometheus-statsd-exporter" "enabled" )}}
- application-bundled-statsd.yaml: |
- feast:
+ options:
+ bootstrapServers: {{ .Release.Name }}-kafka:9092
+ topic: feast
jobs:
metrics:
- enabled: true
+ enabled: true
type: statsd
- host: prometheus-statsd-exporter
+ host: {{ .Release.Name }}-prometheus-statsd-exporter-udp
port: 9125
-{{- end }}
-{{- range $name, $content := .Values.springConfigProfiles }}
- application-{{ $name }}.yaml: |
-{{- toYaml $content | nindent 4 }}
+ server:
+ port: {{ .Values.service.http.targetPort }}
{{- end }}
+
+ application-override.yaml: |
+{{- if index .Values "application-override.yaml" "enabled" }}
+{{- toYaml (index .Values "application-override.yaml") | nindent 4 }}
+{{- end }}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-core/templates/deployment.yaml b/infra/charts/feast/charts/feast-core/templates/deployment.yaml
index daa69f4782..6b75b5524c 100644
--- a/infra/charts/feast/charts/feast-core/templates/deployment.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/deployment.yaml
@@ -19,14 +19,11 @@ spec:
template:
metadata:
annotations:
- checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
- {{- if .Values.springSecretProfiles }}
+ checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
- {{- end }}
{{- if .Values.prometheus.enabled }}
- {{ $config := index .Values "application.yaml" }}
prometheus.io/path: /metrics
- prometheus.io/port: "{{ $config.server.port }}"
+ prometheus.io/port: "{{ .Values.service.http.targetPort }}"
prometheus.io/scrape: "true"
{{- end }}
labels:
@@ -41,31 +38,31 @@ spec:
volumes:
- name: {{ template "feast-core.fullname" . }}-config
- projected:
- sources:
- - configMap:
- name: {{ template "feast-core.fullname" . }}
- {{- if .Values.springSecretProfiles }}
- - secret:
- name: {{ template "feast-core.fullname" . }}
- {{- end }}
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
- - name: {{ template "feast-core.fullname" . }}-gcpserviceaccount
+ configMap:
+ name: {{ template "feast-core.fullname" . }}
+ - name: {{ template "feast-core.fullname" . }}-secret
+ secret:
+ secretName: {{ template "feast-core.fullname" . }}
+ {{- if .Values.gcpServiceAccount.enabled }}
+ - name: {{ template "feast-core.fullname" . }}-gcp-service-account
secret:
secretName: {{ .Values.gcpServiceAccount.existingSecret.name }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
- image: '{{ .Values.image.repository }}:{{ required "No .image.tag found. This must be provided as input." .Values.image.tag }}'
+ image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
volumeMounts:
- name: {{ template "feast-core.fullname" . }}-config
- mountPath: "{{ .Values.springConfigMountPath }}"
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
- - name: {{ template "feast-core.fullname" . }}-gcpserviceaccount
- mountPath: {{ .Values.gcpServiceAccount.mountPath }}
+ mountPath: /etc/feast
+ - name: {{ template "feast-core.fullname" . }}-secret
+ mountPath: /etc/secrets/feast
+ readOnly: true
+ {{- if .Values.gcpServiceAccount.enabled }}
+ - name: {{ template "feast-core.fullname" . }}-gcp-service-account
+ mountPath: /etc/secrets/google
readOnly: true
{{- end }}
@@ -75,40 +72,51 @@ spec:
- name: LOG_LEVEL
value: {{ .Values.logLevel | quote }}
- {{- if .Values.postgresql.enabled }}
- - name: SPRING_DATASOURCE_USERNAME
- value: {{ .Values.postgresql.postgresqlUsername | quote }}
+ {{- if .Values.postgresql.existingSecret }}
- name: SPRING_DATASOURCE_PASSWORD
- value: {{ .Values.postgresql.postgresqlPassword | quote }}
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.postgresql.existingSecret }}
+ key: postgresql-password
{{- end }}
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
+ {{- if .Values.gcpServiceAccount.enabled }}
- name: GOOGLE_APPLICATION_CREDENTIALS
- value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }}
+ value: /etc/secrets/google/{{ .Values.gcpServiceAccount.existingSecret.key }}
{{- end }}
+
{{- if .Values.gcpProjectId }}
- name: GOOGLE_CLOUD_PROJECT
value: {{ .Values.gcpProjectId | quote }}
{{- end }}
- command:
- - java
- {{- range .Values.jvmOptions }}
- - {{ . | quote }}
+ {{- if .Values.javaOpts }}
+ - name: JAVA_TOOL_OPTIONS
+ value: {{ .Values.javaOpts }}
{{- end }}
- - -jar
- - {{ .Values.jarPath | quote }}
- - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/"
- {{- $combinedProfiles := printf "%s,%s" .Values.springConfigProfilesActive .Values.springSecretProfilesActive -}}
- {{- $profilesArray := splitList "," $combinedProfiles -}}
- {{- $profilesArray = append $profilesArray (.Values.postgresql.enabled | ternary "bundled-postgresql" "") -}}
- {{- $profilesArray = append $profilesArray (.Values.kafka.enabled | ternary "bundled-kafka" "") -}}
- {{- $profilesArray = append $profilesArray (index .Values "prometheus-statsd-exporter" "enabled" | ternary "bundled-statsd" "") -}}
- {{- $profilesArray = compact $profilesArray -}}
- {{- if $profilesArray }}
- - "--spring.profiles.active={{ join "," $profilesArray }}"
+
+ {{- range $key, $value := .Values.envOverrides }}
+ - name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
+ value: {{ $value | quote }}
{{- end }}
+ command:
+ - java
+ - -jar
+ - /opt/feast/feast-core.jar
+ - --spring.config.location=
+ {{- if index .Values "application.yaml" "enabled" -}}
+ classpath:/application.yml
+ {{- end }}
+ {{- if index .Values "application-generated.yaml" "enabled" -}}
+ ,file:/etc/feast/application-generated.yaml
+ {{- end }}
+ {{- if index .Values "application-secret.yaml" "enabled" -}}
+ ,file:/etc/secrets/feast/application-secret.yaml
+ {{- end }}
+ {{- if index .Values "application-override.yaml" "enabled" -}}
+ ,file:/etc/feast/application-override.yaml
+ {{- end }}
ports:
- name: http
containerPort: {{ .Values.service.http.targetPort }}
@@ -117,9 +125,8 @@ spec:
{{- if .Values.livenessProbe.enabled }}
livenessProbe:
- httpGet:
- path: /healthz
- port: {{ .Values.service.http.targetPort }}
+ exec:
+ command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
successThreshold: {{ .Values.livenessProbe.successThreshold }}
@@ -129,9 +136,8 @@ spec:
{{- if .Values.readinessProbe.enabled }}
readinessProbe:
- httpGet:
- path: /healthz
- port: {{ .Values.service.http.targetPort }}
+ exec:
+ command: ["/usr/bin/grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
successThreshold: {{ .Values.readinessProbe.successThreshold }}
diff --git a/infra/charts/feast/charts/feast-core/templates/secret.yaml b/infra/charts/feast/charts/feast-core/templates/secret.yaml
index dc4883dd4d..dd33e2dd48 100644
--- a/infra/charts/feast/charts/feast-core/templates/secret.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/secret.yaml
@@ -1,4 +1,3 @@
-{{- if .Values.springSecretProfiles -}}
apiVersion: v1
kind: Secret
metadata:
@@ -11,9 +10,6 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
type: Opaque
-data:
-{{- range $name, $content := .Values.springSecretProfiles }}
- application-{{ $name }}.yaml: |
-{{- toYaml $content | b64enc | nindent 4 }}
-{{- end }}
-{{- end -}}
\ No newline at end of file
+stringData:
+ application-secret.yaml: |
+{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }}
diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml
index 2224a591a9..a0a7823e52 100644
--- a/infra/charts/feast/charts/feast-core/values.yaml
+++ b/infra/charts/feast/charts/feast-core/values.yaml
@@ -1,248 +1,153 @@
-# ============================================================
-# Bundled PostgreSQL
-# ============================================================
-
-# Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/stable/postgresql
-# for additional configuration.
-postgresql:
- # enabled specifies whether Postgresql should be installed as part of Feast Core.
- #
- # Feast Core requires a database to store data such as the created FeatureSets
- # and job statuses. If enabled, the database and service port specified below
- # will override "spring.datasource.url" value in application.yaml. The
- # username and password will also be set as environment variables that will
- # override "spring.datasource.username/password" in application.yaml.
- enabled: true
- # postgresqlDatabase is the name of the database used by Feast Core.
- postgresqlDatabase: feast
- # postgresqlUsername is the username to authenticate to the database.
- postgresqlUsername: postgres
- # postgresqlPassword is the password to authenticate to the database.
- postgresqlPassword: password
- service:
- # port is the TCP port that Postgresql will listen to
- port: 5432
-
-# ============================================================
-# Bundled Kafka
-# ============================================================
-
-# Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/incubator/kafka
-# for additional configuration.
-kafka:
- # enabled specifies whether Kafka should be installed as part of Feast Core.
- #
- # Feast Core requires a Kafka instance to be set as the default source for
- # FeatureRows. If enabled, "feast.stream" option in application.yaml will
- # be overridden by this installed Kafka configuration.
- enabled: true
- topics:
- # topic that will be used as default in Feast Core for the default Kafka source.
- - name: feast
- replicationFactor: 1
- partitions: 1
-
-
-# ============================================================
-# Bundled Prometheus StatsD Exporter
-# ============================================================
-
-prometheus-statsd-exporter:
- enabled: false
-
-# ============================================================
-# Feast Core
-# ============================================================
-
-# replicaCount is the number of pods that will be created.
+# replicaCount -- Number of pods that will be created
replicaCount: 1
-# image configures the Docker image for Feast Core
image:
+ # image.repository -- Docker image repository
repository: gcr.io/kf-feast/feast-core
+ # image.tag -- Image tag
+ tag: dev
+ # image.pullPolicy -- Image pull policy
pullPolicy: IfNotPresent
-# Add prometheus scraping annotations to the Pod metadata.
-# If enabled, you must also ensure server.port is specified under application.yaml
-prometheus:
- enabled: false
-
-# application.yaml is the main configuration for Feast Core application.
-#
-# Feast Core is a Spring Boot app which uses this yaml configuration file.
-# Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/core/src/main/resources/application.yml
-# for a complete list and description of the configuration.
-#
-# Note that some properties defined in application.yaml may be overriden by
-# Helm under certain conditions. For example, if postgresql and kafka dependencies
-# are enabled.
application.yaml:
- grpc:
- port: 6565
- enable-reflection: true
- feast:
- jobs:
- runner: DirectRunner
- options: {}
- updates:
- timeoutSeconds: 240
- metrics:
- enabled: false
- type: statsd
- host: localhost
- port: 9125
- stream:
- type: kafka
- options:
- topic: TOPIC
- bootstrapServers: HOST:PORT
- replicationFactor: 1
- partitions: 1
- spring:
- jpa:
- properties.hibernate.format_sql: true
- properties.hibernate.event.merge.entity_copy_observer: allow
- hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
- hibernate.ddl-auto: update
- datasource:
- driverClassName: org.postgresql.Driver
- url: jdbc:postgresql://HOST:PORT/DATABASE
- username: USERNAME
- password: PASSWORD
- management:
- metrics:
- export:
- simple:
- enabled: false
- statsd:
- enabled: false
- host: localhost
- port: 8125
-
-springConfigProfiles: {}
-springSecretProfiles: {}
-# db: |
-# spring:
-# datasource:
-# driverClassName: org.postgresql.Driver
-# url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
-springConfigProfilesActive: ""
-springSecretProfilesActive: ""
-# springConfigMountPath is the directory path where application.yaml will be
-# mounted in the container.
-springConfigMountPath: /etc/feast/feast-core
-
-# gcpServiceAccount is the service account that Feast Core will use.
+ # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration.
+ enabled: true
+
+application-generated.yaml:
+ # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast database URL, Kafka bootstrap servers and jobs metrics host. This is useful for deployment that uses default configuration for Kafka, Postgres and StatsD exporter. Please set `application-override.yaml` to override this configuration.
+ enabled: true
+
+# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management.
+application-secret.yaml:
+ enabled: true
+
+# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml`
+application-override.yaml:
+ enabled: true
+
gcpServiceAccount:
- # useExistingSecret specifies Feast to use an existing secret containing Google
- # Cloud service account JSON key file.
- useExistingSecret: false
+ # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key
+ enabled: false
existingSecret:
- # name is the secret name of the existing secret for the service account.
+ # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account
name: feast-gcp-service-account
- # key is the secret key of the existing secret for the service account.
- # key is normally derived from the file name of the JSON key file.
- key: key.json
- # mountPath is the directory path where the JSON key file will be mounted.
- # the value of "existingSecret.key" is file name of the service account file.
- mountPath: /etc/gcloud/service-accounts
-
-# Project ID picked up by the Cloud SDK (e.g. BigQuery run against this project)
+ # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account)
+ key: credentials.json
+
+postgresql:
+ # postgresql.existingSecret -- Existing secret to use for authenticating to Postgres
+ existingSecret: ""
+
+# gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow
gcpProjectId: ""
-# Path to Jar file in the Docker image.
-# If you are using gcr.io/kf-feast/feast-core this should not need to be changed
-jarPath: /opt/feast/feast-core.jar
-
-# jvmOptions are options that will be passed to the Java Virtual Machine (JVM)
-# running Feast Core.
-#
-# For example, it is good practice to set min and max heap size in JVM.
-# https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size
-#
-# Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html
-# to see other JVM options that can be set.
-#
-jvmOptions: []
-# - -Xms1024m
-# - -Xmx1024m
-
-logType: JSON
-logLevel: warn
+# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap: `-Xms2048m -Xmx2048m`
+javaOpts:
-livenessProbe:
+# logType -- Log format, either `JSON` or `Console`
+logType: Console
+# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR`
+logLevel: WARN
+
+prometheus:
+ # prometheus.enabled -- Flag to enable scraping of Feast Core metrics
enabled: true
+
+# By default we disable the liveness probe, since if the DB fails restarting core will not result
+# in application healing.
+livenessProbe:
+ # livenessProbe.enabled -- Flag to enabled the probe
+ enabled: false
+ # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated
initialDelaySeconds: 60
+ # livenessProbe.periodSeconds -- How often to perform the probe
periodSeconds: 10
+ # livenessProbe.timeoutSeconds -- When the probe times out
timeoutSeconds: 5
+ # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful
successThreshold: 1
+ # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed
failureThreshold: 5
readinessProbe:
+ # readinessProbe.enabled -- Flag to enabled the probe
enabled: true
- initialDelaySeconds: 15
+ # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated
+ initialDelaySeconds: 20
+ # readinessProbe.periodSeconds -- How often to perform the probe
periodSeconds: 10
+ # readinessProbe.timeoutSeconds -- When the probe times out
timeoutSeconds: 10
+ # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful
successThreshold: 1
+ # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed
failureThreshold: 5
service:
+ # service.type -- Kubernetes service type
type: ClusterIP
http:
+ # service.http.port -- Service port for HTTP requests
port: 80
+ # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics
targetPort: 8080
- # nodePort is the port number that each cluster node will listen to
- # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
- #
- # nodePort:
+ # service.http.nodePort -- Port number that each cluster node will listen to
+ nodePort:
grpc:
+ # service.grpc.port -- Service port for GRPC requests
port: 6565
+ # service.grpc.targetPort -- Container port serving GRPC requests
targetPort: 6565
- # nodePort is the port number that each cluster node will listen to
- # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
- #
- # nodePort:
+ # service.grpc.nodePort -- Port number that each cluster node will listen to
+ nodePort:
ingress:
grpc:
+ # ingress.grpc.enabled -- Flag to create an ingress resource for the service
enabled: false
+ # ingress.grpc.class -- Which ingress controller to use
class: nginx
+ # ingress.grpc.hosts -- List of hostnames to match when routing requests
hosts: []
+ # ingress.grpc.annotations -- Extra annotations for the ingress
annotations: {}
https:
+ # ingress.grpc.https.enabled -- Flag to enable HTTPS
enabled: true
+ # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name
secretNames: {}
+ # ingress.grpc.whitelist -- Allowed client IP source ranges
whitelist: ""
auth:
+ # ingress.grpc.auth.enabled -- Flag to enable auth
enabled: false
http:
+ # ingress.http.enabled -- Flag to create an ingress resource for the service
enabled: false
+ # ingress.http.class -- Which ingress controller to use
class: nginx
+ # ingress.http.hosts -- List of hostnames to match when routing requests
hosts: []
+ # ingress.http.annotations -- Extra annotations for the ingress
annotations: {}
https:
+ # ingress.http.https.enabled -- Flag to enable HTTPS
enabled: true
+ # ingress.http.https.secretNames -- Map of hostname to TLS secret name
secretNames: {}
+ # ingress.http.whitelist -- Allowed client IP source ranges
whitelist: ""
auth:
+ # ingress.http.auth.enabled -- Flag to enable auth
enabled: false
+ # ingress.http.auth.authUrl -- URL to an existing authentication service
authUrl: http://auth-server.auth-ns.svc.cluster.local/auth
+# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)
resources: {}
- # We usually recommend not to specify default resources and to leave this as a conscious
- # choice for the user. This also increases chances charts run on environments with little
- # resources, such as Minikube. If you do want to specify resources, uncomment the following
- # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
- #
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+# nodeSelector -- Node labels for pod assignment
nodeSelector: {}
-tolerations: []
-
-affinity: {}
+# envOverrides -- Extra environment variables to set
+envOverrides: {}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-serving/.helmignore b/infra/charts/feast/charts/feast-serving/.helmignore
deleted file mode 100644
index 50af031725..0000000000
--- a/infra/charts/feast/charts/feast-serving/.helmignore
+++ /dev/null
@@ -1,22 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml
index 2e9cf89243..7c8e6131cf 100644
--- a/infra/charts/feast/charts/feast-serving/Chart.yaml
+++ b/infra/charts/feast/charts/feast-serving/Chart.yaml
@@ -1,4 +1,4 @@
apiVersion: v1
-description: A Helm chart for serving component of Feast
+description: Feast Serving serves low-latency latest features and historical batch features.
name: feast-serving
-version: 0.4.4
+version: 0.5.0-alpha.1
diff --git a/infra/charts/feast/charts/feast-serving/README.md b/infra/charts/feast/charts/feast-serving/README.md
new file mode 100644
index 0000000000..d30da2eb3a
--- /dev/null
+++ b/infra/charts/feast/charts/feast-serving/README.md
@@ -0,0 +1,69 @@
+feast-serving
+=============
+Feast Serving serves low-latency latest features and historical batch features.
+
+Current chart version is `0.5.0-alpha.1`
+
+
+
+
+
+## Chart Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| "application-generated.yaml".enabled | bool | `true` | Flag to include Helm generated configuration for Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration. |
+| "application-override.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml` |
+| "application-secret.yaml" | object | `{"enabled":true}` | Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management. |
+| "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. |
+| envOverrides | object | `{}` | Extra environment variables to set |
+| gcpProjectId | string | `""` | Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow |
+| gcpServiceAccount.enabled | bool | `false` | Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key |
+| gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) |
+| gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account |
+| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| image.repository | string | `"gcr.io/kf-feast/feast-serving"` | Docker image repository |
+| image.tag | string | `"dev"` | Image tag |
+| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.grpc.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.grpc.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.grpc.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.grpc.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.grpc.whitelist | string | `""` | Allowed client IP source ranges |
+| ingress.http.annotations | object | `{}` | Extra annotations for the ingress |
+| ingress.http.auth.authUrl | string | `"http://auth-server.auth-ns.svc.cluster.local/auth"` | URL to an existing authentication service |
+| ingress.http.auth.enabled | bool | `false` | Flag to enable auth |
+| ingress.http.class | string | `"nginx"` | Which ingress controller to use |
+| ingress.http.enabled | bool | `false` | Flag to create an ingress resource for the service |
+| ingress.http.hosts | list | `[]` | List of hostnames to match when routing requests |
+| ingress.http.https.enabled | bool | `true` | Flag to enable HTTPS |
+| ingress.http.https.secretNames | object | `{}` | Map of hostname to TLS secret name |
+| ingress.http.whitelist | string | `""` | Allowed client IP source ranges |
+| javaOpts | string | `nil` | [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap: `-Xms2048m -Xmx2048m` |
+| livenessProbe.enabled | bool | `true` | Flag to enabled the probe |
+| livenessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| livenessProbe.initialDelaySeconds | int | `60` | Delay before the probe is initiated |
+| livenessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| livenessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| livenessProbe.timeoutSeconds | int | `5` | When the probe times out |
+| logLevel | string | `"WARN"` | Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR` |
+| logType | string | `"Console"` | Log format, either `JSON` or `Console` |
+| nodeSelector | object | `{}` | Node labels for pod assignment |
+| prometheus.enabled | bool | `true` | Flag to enable scraping of Feast Core metrics |
+| readinessProbe.enabled | bool | `true` | Flag to enabled the probe |
+| readinessProbe.failureThreshold | int | `5` | Min consecutive failures for the probe to be considered failed |
+| readinessProbe.initialDelaySeconds | int | `15` | Delay before the probe is initiated |
+| readinessProbe.periodSeconds | int | `10` | How often to perform the probe |
+| readinessProbe.successThreshold | int | `1` | Min consecutive success for the probe to be considered successful |
+| readinessProbe.timeoutSeconds | int | `10` | When the probe times out |
+| replicaCount | int | `1` | Number of pods that will be created |
+| resources | object | `{}` | CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) |
+| service.grpc.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.grpc.port | int | `6566` | Service port for GRPC requests |
+| service.grpc.targetPort | int | `6566` | Container port serving GRPC requests |
+| service.http.nodePort | string | `nil` | Port number that each cluster node will listen to |
+| service.http.port | int | `80` | Service port for HTTP requests |
+| service.http.targetPort | int | `8080` | Container port serving HTTP requests |
+| service.type | string | `"ClusterIP"` | Kubernetes service type |
diff --git a/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz b/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz
deleted file mode 100644
index 962893a825..0000000000
Binary files a/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz and /dev/null differ
diff --git a/infra/charts/feast/charts/feast-serving/requirements.yaml b/infra/charts/feast/charts/feast-serving/requirements.yaml
deleted file mode 100644
index 2cee3f8149..0000000000
--- a/infra/charts/feast/charts/feast-serving/requirements.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-dependencies:
-- name: redis
- version: 9.5.0
- repository: "@stable"
- condition: redis.enabled
-- name: common
- version: 0.0.5
- repository: "@incubator"
diff --git a/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
index ab670cc8cc..49abb6b8e5 100644
--- a/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
+++ b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
@@ -43,10 +43,3 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
-
-{{/*
-Helpers
-*/}}
-{{- define "bq_store_and_no_job_options" -}}
-{{ and (eq (index .Values "store.yaml" "type") "BIGQUERY") (empty (index .Values "application.yaml" "feast" "jobs" "store-options")) }}
-{{- end -}}
diff --git a/infra/charts/feast/charts/feast-serving/templates/configmap.yaml b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
index 3775d9f562..579fe18fd4 100644
--- a/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
@@ -10,25 +10,31 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
data:
- application.yaml: |
-{{- toYaml (index .Values "application.yaml") | nindent 4 }}
-
-{{- if .Values.core.enabled }}
- application-bundled-core.yaml: |
+ application-generated.yaml: |
+{{- if index .Values "application-generated.yaml" "enabled" }}
feast:
- core-host: {{ printf "%s-feast-core" .Release.Name }}
-{{- end }}
+ core-host: {{ .Release.Name }}-feast-core
-{{- if eq (include "bq_store_and_no_job_options" .) "true" }}
- application-bundled-redis.yaml: |
- feast:
- jobs:
- store-options:
- host: {{ printf "%s-redis-headless" .Release.Name }}
+ stores:
+ - name: online
+ type: REDIS
+ config:
+ host: {{ .Release.Name }}-redis-master
port: 6379
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+ job_store:
+ redis_host: {{ .Release.Name }}-redis-master
+ redis_port: 6379
+
+ server:
+ port: {{ .Values.service.http.targetPort }}
{{- end }}
-{{- range $name, $content := .Values.springConfigProfiles }}
- application-{{ $name }}.yaml: |
-{{- toYaml $content | nindent 4 }}
+ application-override.yaml: |
+{{- if index .Values "application-override.yaml" "enabled" }}
+{{- toYaml (index .Values "application-override.yaml") | nindent 4 }}
{{- end }}
diff --git a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
index 1a2e672758..bb8fdc55ae 100644
--- a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
@@ -19,14 +19,11 @@ spec:
template:
metadata:
annotations:
- checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
- {{- if .Values.springSecretProfiles }}
+ checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
- {{- end }}
{{- if .Values.prometheus.enabled }}
- {{ $config := index .Values "application.yaml" }}
prometheus.io/path: /metrics
- prometheus.io/port: "{{ $config.server.port }}"
+ prometheus.io/port: "{{ .Values.service.http.targetPort }}"
prometheus.io/scrape: "true"
{{- end }}
labels:
@@ -41,33 +38,31 @@ spec:
volumes:
- name: {{ template "feast-serving.fullname" . }}-config
- projected:
- sources:
- - configMap:
- name: {{ template "feast-serving.fullname" . }}
- - configMap:
- name: {{ template "feast-serving.fullname" . }}-store
- {{- if .Values.springSecretProfiles }}
- - secret:
- name: {{ template "feast-serving.fullname" . }}
- {{- end }}
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
- - name: {{ template "feast-serving.fullname" . }}-gcpserviceaccount
+ configMap:
+ name: {{ template "feast-serving.fullname" . }}
+ - name: {{ template "feast-serving.fullname" . }}-secret
+ secret:
+ secretName: {{ template "feast-serving.fullname" . }}
+ {{- if .Values.gcpServiceAccount.enabled }}
+ - name: {{ template "feast-serving.fullname" . }}-gcp-service-account
secret:
secretName: {{ .Values.gcpServiceAccount.existingSecret.name }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
- image: '{{ .Values.image.repository }}:{{ required "No .image.tag found. This must be provided as input." .Values.image.tag }}'
+ image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
volumeMounts:
- name: {{ template "feast-serving.fullname" . }}-config
- mountPath: "{{ .Values.springConfigMountPath }}"
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
- - name: {{ template "feast-serving.fullname" . }}-gcpserviceaccount
- mountPath: {{ .Values.gcpServiceAccount.mountPath }}
+ mountPath: /etc/feast
+ - name: {{ template "feast-serving.fullname" . }}-secret
+ mountPath: /etc/secrets/feast
+ readOnly: true
+ {{- if .Values.gcpServiceAccount.enabled }}
+ - name: {{ template "feast-serving.fullname" . }}-gcp-service-account
+ mountPath: /etc/secrets/google
readOnly: true
{{- end }}
@@ -77,31 +72,43 @@ spec:
- name: LOG_LEVEL
value: {{ .Values.logLevel | quote }}
- {{- if .Values.gcpServiceAccount.useExistingSecret }}
+ {{- if .Values.gcpServiceAccount.enabled }}
- name: GOOGLE_APPLICATION_CREDENTIALS
- value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }}
+ value: /etc/secrets/google/{{ .Values.gcpServiceAccount.existingSecret.key }}
{{- end }}
+
{{- if .Values.gcpProjectId }}
- name: GOOGLE_CLOUD_PROJECT
value: {{ .Values.gcpProjectId | quote }}
{{- end }}
+ {{- if .Values.javaOpts }}
+ - name: JAVA_TOOL_OPTIONS
+ value: {{ .Values.javaOpts }}
+ {{- end }}
+
+ {{- range $key, $value := .Values.envOverrides }}
+ - name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
+ value: {{ $value | quote }}
+ {{- end }}
+
command:
- java
- {{- range .Values.jvmOptions }}
- - {{ . | quote }}
- {{- end }}
- -jar
- - {{ .Values.jarPath | quote }}
- - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/"
- {{- $combinedProfiles := printf "%s,%s" .Values.springConfigProfilesActive .Values.springSecretProfilesActive -}}
- {{- $profilesArray := splitList "," $combinedProfiles -}}
- {{- $profilesArray = append $profilesArray (.Values.core.enabled | ternary "bundled-core" "") -}}
- {{- $profilesArray = append $profilesArray (eq (include "bq_store_and_no_job_options" .) "true" | ternary "bundled-redis" "") -}}
- {{- $profilesArray = compact $profilesArray -}}
- {{- if $profilesArray }}
- - "--spring.profiles.active={{ join "," $profilesArray }}"
- {{- end }}
+ - /opt/feast/feast-serving.jar
+ - --spring.config.location=
+ {{- if index .Values "application.yaml" "enabled" -}}
+ classpath:/application.yml
+ {{- end }}
+ {{- if index .Values "application-generated.yaml" "enabled" -}}
+ ,file:/etc/feast/application-generated.yaml
+ {{- end }}
+ {{- if index .Values "application-secret.yaml" "enabled" -}}
+ ,file:/etc/secrets/feast/application-secret.yaml
+ {{- end }}
+ {{- if index .Values "application-override.yaml" "enabled" -}}
+ ,file:/etc/feast/application-override.yaml
+ {{- end }}
ports:
- name: http
diff --git a/infra/charts/feast/charts/feast-serving/templates/secret.yaml b/infra/charts/feast/charts/feast-serving/templates/secret.yaml
index 941101ee4b..2ccbccfcf7 100644
--- a/infra/charts/feast/charts/feast-serving/templates/secret.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/secret.yaml
@@ -1,4 +1,3 @@
-{{- if .Values.springSecretProfiles -}}
apiVersion: v1
kind: Secret
metadata:
@@ -11,9 +10,6 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
type: Opaque
-data:
-{{- range $name, $content := .Values.springSecretProfiles }}
- application-{{ $name }}.yaml: |
-{{- toYaml $content | b64enc | nindent 4 }}
-{{- end }}
-{{- end -}}
\ No newline at end of file
+stringData:
+ application-secret.yaml: |
+{{- toYaml (index .Values "application-secret.yaml") | nindent 4 }}
diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml
index fc06c1ae57..c94c720526 100644
--- a/infra/charts/feast/charts/feast-serving/values.yaml
+++ b/infra/charts/feast/charts/feast-serving/values.yaml
@@ -1,236 +1,148 @@
-# redis configures Redis that is installed as part of Feast Serving.
-# Refer to https://github.com/helm/charts/tree/99430c4afdc88213c1ca08f40eeb03868ffcc9d7/stable/redis
-# for additional configuration
-redis:
- # enabled specifies whether Redis should be installed as part of Feast Serving.
- #
- # If enabled, "redis_config" in store.yaml will be overwritten by Helm
- # to the configuration in this Redis installation.
- enabled: false
- # usePassword specifies if password is required to access Redis. Note that
- # Feast 0.3 does not support Redis with password.
- usePassword: false
- # cluster configuration for Redis.
- cluster:
- # enabled specifies if Redis should be installed in cluster mode.
- enabled: false
-
-# core configures Feast Core in the same parent feast chart that this Feast
-# Serving connects to.
-core:
- # enabled specifies that Feast Serving will use Feast Core installed
- # in the same parent feast chart. If enabled, Helm will overwrite
- # "feast.core-host" in application.yaml with the correct value.
- enabled: true
-
-# replicaCount is the number of pods that will be created.
+# replicaCount -- Number of pods that will be created
replicaCount: 1
-# image configures the Docker image for Feast Serving
image:
+ # image.repository -- Docker image repository
repository: gcr.io/kf-feast/feast-serving
+ # image.tag -- Image tag
+ tag: dev
+ # image.pullPolicy -- Image pull policy
pullPolicy: IfNotPresent
-# application.yaml is the main configuration for Feast Serving application.
-#
-# Feast Core is a Spring Boot app which uses this yaml configuration file.
-# Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/serving/src/main/resources/application.yml
-# for a complete list and description of the configuration.
-#
-# Note that some properties defined in application.yaml may be overridden by
-# Helm under certain conditions. For example, if core is enabled, then
-# "feast.core-host" will be overridden. Also, if "type: BIGQUERY" is specified
-# in store.yaml, "feast.jobs.store-options" will be overridden as well with
-# the default option supported in Feast 0.3.
application.yaml:
- feast:
- version: 0.3
- core-host: localhost
- core-grpc-port: 6565
- tracing:
- enabled: false
- tracer-name: jaeger
- service-name: feast-serving
- store:
- config-path: /etc/feast/feast-serving/store.yaml
- redis-pool-max-size: 128
- redis-pool-max-idle: 64
- jobs:
- staging-location: ""
- store-type: ""
- store-options: {}
- grpc:
- port: 6566
- enable-reflection: true
- server:
- port: 8080
-
-# store.yaml is the configuration for Feast Store.
-#
-# Refer to this link for description:
-# https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto
-#
-# Use the correct store configuration depending on whether the installed
-# Feast Serving is "online" or "batch", by uncommenting the correct store.yaml.
-#
-# Note that if "redis.enabled: true" and "type: REDIS" in store.yaml,
-# Helm will override "redis_config" with configuration of Redis installed
-# in this chart.
-#
-# Note that if "type: BIGQUERY" in store.yaml, Helm assumes Feast Online serving
-# is also installed with Redis store. Helm will then override "feast.jobs.store-options"
-# in application.yaml with the installed Redis store configuration. This is
-# because in Feast 0.3, Redis job store is required.
-#
-# store.yaml:
-# name: online
-# type: REDIS
-# redis_config:
-# host: localhost
-# port: 6379
-# subscriptions:
-# - project: "*"
-# name: "*"
-# version: "*"
-#
-# store.yaml:
-# name: bigquery
-# type: BIGQUERY
-# bigquery_config:
-# project_id: PROJECT_ID
-# dataset_id: DATASET_ID
-# subscriptions:
-# - project: "*"
-# name: "*"
-# version: "*"
-
-springConfigProfiles: {}
-springSecretProfiles: {}
-# db: |
-# spring:
-# datasource:
-# driverClassName: org.postgresql.Driver
-# url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
-springConfigProfilesActive: ""
-springSecretProfilesActive: ""
-# springConfigMountPath is the directory path where application.yaml and
-# store.yaml will be mounted in the container.
-springConfigMountPath: /etc/feast/feast-serving
-
-# gcpServiceAccount is the service account that Feast Serving will use.
+ # "application.yaml".enabled -- Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration.
+ enabled: true
+
+application-generated.yaml:
+ # "application-generated.yaml".enabled -- Flag to include Helm generated configuration for http port, Feast Core host, Redis store and job store. This is useful for deployment that uses default configuration for Redis. Please set `application-override.yaml` to override this configuration.
+ enabled: true
+
+# "application-secret.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a Secret. `application-override.yaml` has a higher precedence than `application-secret.yaml`. It is recommended to either set `application-override.yaml` or `application-secret.yaml` only to simplify config management.
+application-secret.yaml:
+ enabled: true
+
+# "application-override.yaml" -- Configuration to override the default [application.yaml](https://github.com/feast-dev/feast/blob/master/serving/src/main/resources/application.yml). Will be created as a ConfigMap. `application-override.yaml` has a higher precedence than `application-secret.yaml`
+application-override.yaml:
+ enabled: true
+
gcpServiceAccount:
- # useExistingSecret specifies Feast to use an existing secret containing Google
+ # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key
# Cloud service account JSON key file.
- useExistingSecret: false
+ enabled: false
existingSecret:
- # name is the secret name of the existing secret for the service account.
+ # gcpServiceAccount.existingSecret.name -- Name of the existing secret containing the service account
name: feast-gcp-service-account
- # key is the secret key of the existing secret for the service account.
- # key is normally derived from the file name of the JSON key file.
- key: key.json
- # mountPath is the directory path where the JSON key file will be mounted.
- # the value of "existingSecret.key" is file name of the service account file.
- mountPath: /etc/gcloud/service-accounts
-
-# Project ID picked up by the Cloud SDK (e.g. BigQuery run against this project)
+ # gcpServiceAccount.existingSecret.key -- Key in the secret data (file name of the service account)
+ key: credentials.json
+
+# gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow
gcpProjectId: ""
-# Path to Jar file in the Docker image.
-# If using gcr.io/kf-feast/feast-serving this should not need to be changed.
-jarPath: /opt/feast/feast-serving.jar
-
-# jvmOptions are options that will be passed to the Java Virtual Machine (JVM)
-# running Feast Core.
-#
-# For example, it is good practice to set min and max heap size in JVM.
-# https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size
-#
-# Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html
-# to see other JVM options that can be set.
-#
-jvmOptions: []
-# - -Xms768m
-# - -Xmx768m
-
-logType: JSON
-logLevel: warn
+# javaOpts -- [JVM options](https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html). For better performance, it is advised to set the min and max heap: `-Xms2048m -Xmx2048m`
+javaOpts:
+
+# logType -- Log format, either `JSON` or `Console`
+logType: Console
+# logLevel -- Default log level, use either one of `DEBUG`, `INFO`, `WARN` or `ERROR`
+logLevel: WARN
+
+prometheus:
+ # prometheus.enabled -- Flag to enable scraping of Feast Core metrics
+ enabled: true
livenessProbe:
- enabled: false
+ # livenessProbe.enabled -- Flag to enabled the probe
+ enabled: true
+ # livenessProbe.initialDelaySeconds -- Delay before the probe is initiated
initialDelaySeconds: 60
+ # livenessProbe.periodSeconds -- How often to perform the probe
periodSeconds: 10
+ # livenessProbe.timeoutSeconds -- When the probe times out
timeoutSeconds: 5
+ # livenessProbe.successThreshold -- Min consecutive success for the probe to be considered successful
successThreshold: 1
+ # livenessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed
failureThreshold: 5
readinessProbe:
- enabled: false
+ # readinessProbe.enabled -- Flag to enabled the probe
+ enabled: true
+ # readinessProbe.initialDelaySeconds -- Delay before the probe is initiated
initialDelaySeconds: 15
+ # readinessProbe.periodSeconds -- How often to perform the probe
periodSeconds: 10
+ # readinessProbe.timeoutSeconds -- When the probe times out
timeoutSeconds: 10
+ # readinessProbe.successThreshold -- Min consecutive success for the probe to be considered successful
successThreshold: 1
+ # readinessProbe.failureThreshold -- Min consecutive failures for the probe to be considered failed
failureThreshold: 5
service:
+ # service.type -- Kubernetes service type
type: ClusterIP
http:
+ # service.http.port -- Service port for HTTP requests
port: 80
+ # service.http.targetPort -- Container port serving HTTP requests and Prometheus metrics
targetPort: 8080
- # nodePort is the port number that each cluster node will listen to
- # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
- #
- # nodePort:
+ # service.http.nodePort -- Port number that each cluster node will listen to
+ nodePort:
grpc:
+ # service.grpc.port -- Service port for GRPC requests
port: 6566
+ # service.grpc.targetPort -- Container port serving GRPC requests
targetPort: 6566
- # nodePort is the port number that each cluster node will listen to
- # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
- #
- # nodePort:
+ # service.grpc.nodePort -- Port number that each cluster node will listen to
+ nodePort:
ingress:
grpc:
+ # ingress.grpc.enabled -- Flag to create an ingress resource for the service
enabled: false
+ # ingress.grpc.class -- Which ingress controller to use
class: nginx
+ # ingress.grpc.hosts -- List of hostnames to match when routing requests
hosts: []
+ # ingress.grpc.annotations -- Extra annotations for the ingress
annotations: {}
https:
+ # ingress.grpc.https.enabled -- Flag to enable HTTPS
enabled: true
+ # ingress.grpc.https.secretNames -- Map of hostname to TLS secret name
secretNames: {}
+ # ingress.grpc.whitelist -- Allowed client IP source ranges
whitelist: ""
auth:
+ # ingress.grpc.auth.enabled -- Flag to enable auth
enabled: false
http:
+ # ingress.http.enabled -- Flag to create an ingress resource for the service
enabled: false
+ # ingress.http.class -- Which ingress controller to use
class: nginx
+ # ingress.http.hosts -- List of hostnames to match when routing requests
hosts: []
+ # ingress.http.annotations -- Extra annotations for the ingress
annotations: {}
https:
+ # ingress.http.https.enabled -- Flag to enable HTTPS
enabled: true
+ # ingress.http.https.secretNames -- Map of hostname to TLS secret name
secretNames: {}
+ # ingress.http.whitelist -- Allowed client IP source ranges
whitelist: ""
auth:
+ # ingress.http.auth.enabled -- Flag to enable auth
enabled: false
+ # ingress.http.auth.authUrl -- URL to an existing authentication service
authUrl: http://auth-server.auth-ns.svc.cluster.local/auth
-prometheus:
- enabled: true
-
+# resources -- CPU/memory [resource requests/limit](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)
resources: {}
- # We usually recommend not to specify default resources and to leave this as a conscious
- # choice for the user. This also increases chances charts run on environments with little
- # resources, such as Minikube. If you do want to specify resources, uncomment the following
- # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
- #
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+# nodeSelector -- Node labels for pod assignment
nodeSelector: {}
-tolerations: []
-
-affinity: {}
+# envOverrides -- Extra environment variables to set
+envOverrides: {}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/grafana-5.0.5.tgz b/infra/charts/feast/charts/grafana-5.0.5.tgz
new file mode 100644
index 0000000000..06eb83a5e5
Binary files /dev/null and b/infra/charts/feast/charts/grafana-5.0.5.tgz differ
diff --git a/infra/charts/feast/charts/kafka-0.20.8.tgz b/infra/charts/feast/charts/kafka-0.20.8.tgz
new file mode 100644
index 0000000000..f61be294ae
Binary files /dev/null and b/infra/charts/feast/charts/kafka-0.20.8.tgz differ
diff --git a/infra/charts/feast/charts/postgresql-8.6.1.tgz b/infra/charts/feast/charts/postgresql-8.6.1.tgz
new file mode 100644
index 0000000000..c1ee74e8e8
Binary files /dev/null and b/infra/charts/feast/charts/postgresql-8.6.1.tgz differ
diff --git a/infra/charts/feast/charts/prometheus-11.0.2.tgz b/infra/charts/feast/charts/prometheus-11.0.2.tgz
new file mode 100644
index 0000000000..32b3abbe4e
Binary files /dev/null and b/infra/charts/feast/charts/prometheus-11.0.2.tgz differ
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/.helmignore b/infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/.helmignore
rename to infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/Chart.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/Chart.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/README.md b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md
similarity index 91%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/README.md
rename to infra/charts/feast/charts/prometheus-statsd-exporter/README.md
index 69eb33039b..8a6739f393 100644
--- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/README.md
+++ b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md
@@ -33,7 +33,8 @@ $ helm delete my-release
## Configuration
- |Parameter | Description | Default |
+|Parameter | Description | Default |
+|----------------------------|------------------------------------------------------|----------------------------------------|
|`extraArgs` | key:value list of extra arguments to give the binary | `{}` |
|`image.pullPolicy` | Image pull policy | `IfNotPresent` |
|`image.repository` | Image repository | `prom/statsd-exporter` |
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/NOTES.txt b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/NOTES.txt
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/_helpers.tpl b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/_helpers.tpl
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/deployment.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml
similarity index 96%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/deployment.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml
index 47308ef89b..44fac913b6 100644
--- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/deployment.yaml
+++ b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "prometheus-statsd-exporter.fullname" . }}
@@ -16,7 +16,7 @@ spec:
template:
metadata:
annotations:
-{{ toYaml .Values.statsdexporter.annotations | indent 8 }}
+{{ toYaml .Values.statsdexporter.podAnnotations | indent 8 }}
labels:
app: {{ template "prometheus-statsd-exporter.name" . }}
release: {{ .Release.Name }}
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/pvc.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/pvc.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/service.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml
similarity index 69%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/service.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml
index 88d01b24a6..1690ba7fe0 100644
--- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/service.yaml
+++ b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml
@@ -15,6 +15,38 @@ metadata:
{{ toYaml .Values.service.labels | indent 4 }}
{{- end }}
name: {{ template "prometheus-statsd-exporter.fullname" . }}
+spec:
+ ports:
+ - name: metrics
+ port: {{ .Values.service.metricsPort }}
+ protocol: TCP
+ targetPort: 9102
+ - name: statsd-tcp
+ port: {{ .Values.service.statsdPort }}
+ protocol: TCP
+ targetPort: 9125
+ selector:
+ app: {{ template "prometheus-statsd-exporter.name" . }}
+ release: {{ .Release.Name }}
+ type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.service.annotations }}
+ annotations:
+{{ toYaml .Values.service.annotations | indent 4 }}
+{{- end }}
+ labels:
+ app: {{ template "prometheus-statsd-exporter.fullname" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+ component: "{{ .Chart.Name }}"
+ heritage: {{ .Release.Service }}
+ release: {{ .Release.Name }}
+{{- if .Values.service.labels }}
+{{ toYaml .Values.service.labels | indent 4 }}
+{{- end }}
+ name: {{ template "prometheus-statsd-exporter.fullname" . }}-udp
spec:
{{- if .Values.service.clusterIP }}
clusterIP: {{ .Values.service.clusterIP }}
@@ -33,14 +65,6 @@ spec:
{{- end }}
{{- end }}
ports:
- - name: metrics
- port: {{ .Values.service.metricsPort }}
- protocol: TCP
- targetPort: 9102
- - name: statsd-tcp
- port: {{ .Values.service.statsdPort }}
- protocol: TCP
- targetPort: 9125
- name: statsd-udp
port: {{ .Values.service.statsdPort }}
protocol: UDP
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
similarity index 100%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
diff --git a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml b/infra/charts/feast/charts/prometheus-statsd-exporter/values.yaml
similarity index 96%
rename from infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml
rename to infra/charts/feast/charts/prometheus-statsd-exporter/values.yaml
index f2d523771e..b3f70ee264 100644
--- a/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml
+++ b/infra/charts/feast/charts/prometheus-statsd-exporter/values.yaml
@@ -20,6 +20,9 @@ service:
statsdexporter:
podAnnotations:
+ prometheus.io/path: /metrics
+ prometheus.io/port: "9102"
+ prometheus.io/scrape: "true"
extraArgs: {}
# - --persistence.file=data-perst
diff --git a/infra/charts/feast/charts/redis-10.5.6.tgz b/infra/charts/feast/charts/redis-10.5.6.tgz
new file mode 100644
index 0000000000..f1e4ec4105
Binary files /dev/null and b/infra/charts/feast/charts/redis-10.5.6.tgz differ
diff --git a/infra/charts/feast/files/img/dataflow-jobs.png b/infra/charts/feast/files/img/dataflow-jobs.png
new file mode 100644
index 0000000000..2acf48f19e
Binary files /dev/null and b/infra/charts/feast/files/img/dataflow-jobs.png differ
diff --git a/infra/charts/feast/files/img/prometheus-server.png b/infra/charts/feast/files/img/prometheus-server.png
new file mode 100644
index 0000000000..efe31dc9e1
Binary files /dev/null and b/infra/charts/feast/files/img/prometheus-server.png differ
diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock
index d9aeab1651..c6ff995bac 100644
--- a/infra/charts/feast/requirements.lock
+++ b/infra/charts/feast/requirements.lock
@@ -1,12 +1,30 @@
dependencies:
- name: feast-core
repository: ""
- version: 0.4.4
+ version: 0.5.0-alpha.1
- name: feast-serving
repository: ""
- version: 0.4.4
+ version: 0.5.0-alpha.1
- name: feast-serving
repository: ""
- version: 0.4.4
-digest: sha256:1f812168f656d4725f80f52fcf2f919603ca4c69a9b48d62d55ea9271bf4755b
-generated: "2020-02-25T14:15:39.256627Z"
+ version: 0.5.0-alpha.1
+- name: postgresql
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ version: 8.6.1
+- name: kafka
+ repository: https://kubernetes-charts-incubator.storage.googleapis.com/
+ version: 0.20.8
+- name: redis
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ version: 10.5.6
+- name: prometheus-statsd-exporter
+ repository: ""
+ version: 0.1.2
+- name: prometheus
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ version: 11.0.2
+- name: grafana
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ version: 5.0.5
+digest: sha256:e325439384ef9b45428fbeafe8f1e230b331d4b5482c3f26f07b71cecae06c22
+generated: "2020-05-02T15:00:45.4365217+08:00"
diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml
index 1fa1826965..7de7870575 100644
--- a/infra/charts/feast/requirements.yaml
+++ b/infra/charts/feast/requirements.yaml
@@ -1,12 +1,35 @@
dependencies:
- name: feast-core
- version: 0.4.4
+ version: 0.5.0-alpha.1
condition: feast-core.enabled
- name: feast-serving
- alias: feast-serving-batch
- version: 0.4.4
- condition: feast-serving-batch.enabled
+ alias: feast-online-serving
+ version: 0.5.0-alpha.1
+ condition: feast-online-serving.enabled
- name: feast-serving
- alias: feast-serving-online
- version: 0.4.4
- condition: feast-serving-online.enabled
\ No newline at end of file
+ alias: feast-batch-serving
+ version: 0.5.0-alpha.1
+ condition: feast-batch-serving.enabled
+- name: postgresql
+ version: 8.6.1
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ condition: postgresql.enabled
+- name: kafka
+ version: 0.20.8
+ repository: https://kubernetes-charts-incubator.storage.googleapis.com/
+ condition: kafka.enabled
+- name: redis
+ version: 10.5.6
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ condition: redis.enabled
+- name: prometheus-statsd-exporter
+ version: 0.1.2
+ condition: prometheus-statsd-exporter.enabled
+- name: prometheus
+ version: 11.0.2
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ condition: prometheus.enabled
+- name: grafana
+ version: 5.0.5
+ repository: https://kubernetes-charts.storage.googleapis.com/
+ condition: grafana.enabled
diff --git a/infra/charts/feast/templates/tests/test-feast-batch-serving.yaml b/infra/charts/feast/templates/tests/test-feast-batch-serving.yaml
new file mode 100644
index 0000000000..54173021d3
--- /dev/null
+++ b/infra/charts/feast/templates/tests/test-feast-batch-serving.yaml
@@ -0,0 +1,116 @@
+{{- if and (index .Values "feast-core" "enabled") (index .Values "feast-batch-serving" "enabled") }}
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: "{{ .Release.Name }}-feast-batch-serving-test"
+ annotations:
+ "helm.sh/hook": test-success
+ namespace: {{ .Release.Namespace }}
+spec:
+ containers:
+ - name: main
+ image: python:3.7
+ command:
+ - bash
+ - -c
+ - |
+ pip install -U feast==0.4.*
+
+ cat < featureset.yaml
+ kind: feature_set
+ spec:
+ name: customer_transactions
+ entities:
+ - name: customer_id
+ valueType: INT64
+ features:
+ - name: daily_transactions
+ valueType: FLOAT
+ - name: total_transactions
+ valueType: FLOAT
+ maxAge: 3600s
+ EOF
+
+ python < featureset.yaml
+ kind: feature_set
+ spec:
+ name: customer_transactions
+ entities:
+ - name: customer_id
+ valueType: INT64
+ features:
+ - name: daily_transactions
+ valueType: FLOAT
+ - name: total_transactions
+ valueType: FLOAT
+ maxAge: 3600s
+ EOF
+
+ python <
+ dataset_id:
+ staging_location: gs:///feast-staging-location
+ initial_retry_delay_seconds: 3
+ total_timeout_seconds: 21600
+ subscriptions:
+ - name: "*"
+ project: "*"
+ version: "*"
+
+postgresql:
+ existingSecret: feast-postgresql
diff --git a/infra/charts/feast/values-dataflow-runner.yaml b/infra/charts/feast/values-dataflow-runner.yaml
new file mode 100644
index 0000000000..0469a6349e
--- /dev/null
+++ b/infra/charts/feast/values-dataflow-runner.yaml
@@ -0,0 +1,113 @@
+# values-dataflow-runner.yaml
+feast-core:
+ gcpServiceAccount:
+ enabled: true
+ postgresql:
+ existingSecret: feast-postgresql
+ application-override.yaml:
+ feast:
+ stream:
+ options:
+ bootstrapServers:
+ jobs:
+ active_runner: dataflow
+ metrics:
+ host:
+ runners:
+ - name: dataflow
+ type: DataflowRunner
+ options:
+ project:
+ region:
+ zone:
+ tempLocation:
+ network:
+ subnetwork: