Remove granularity (#162)

* Remove granularity from proto files * Remove granularity from serving * Remove granularity from proto files * Remove granularity from serving * Remove granularity from core * Remove granularity from Python SDK * Update generated protos, remove granularity * Remove granularity from ingestion * Scrub additional mentions of granularity * Update core-deploy template * Update chart distribution * Fix core service not overriding storage correctly * Shade core jar to include google nio lib * Remove granularity from UI * Update feature details page * Update Importer in Feast Python SDK to handle source options properly When the import source type is DataFrame
feast-dev · Mar 22, 2019 · cf45d0f · cf45d0f
1 parent 6646402
commit cf45d0f
Show file tree

Hide file tree

Showing 123 changed files with 1,606 additions and 2,374 deletions.
diff --git a/charts/dist/feast-0.1.0.tgz b/charts/dist/feast-0.1.0.tgz
diff --git a/charts/dist/index.yaml b/charts/dist/index.yaml
@@ -3,11 +3,11 @@ entries:
   feast:
   - apiVersion: v1
     appVersion: 0.1.0
-    created: 2019-03-15T19:35:11.848995-07:00
+    created: 2019-03-21T18:38:21.941744229+08:00
     description: A Helm chart to install Feast on kubernetes
-    digest: cbc97b4be6a84a33055900b620d2f6b6176ee81d86eb2e8bfcc9096ea040ff47
+    digest: 68584c58f742f3ed88040ae8658d549a2ee643334f01c684bad83e2937eae19e
     name: feast
     urls:
     - feast-0.1.0.tgz
     version: 0.1.0
-generated: 2019-03-15T19:35:11.844526-07:00
+generated: 2019-03-21T18:38:21.937413876+08:00
diff --git a/charts/feast/templates/core-deploy.yaml b/charts/feast/templates/core-deploy.yaml
@@ -91,6 +91,7 @@ spec:
             value: "{{ .Values.core.jobs.monitoring.period }}"
           - name: JOB_MONITOR_INITIAL_DELAY_MS
             value: "{{ .Values.core.jobs.monitoring.initialDelay }}"
+          {{- if .Values.store }}
           {{- if .Values.store.serving }}
           - name: STORE_SERVING_TYPE
             value: {{ .Values.store.serving.type }}
@@ -109,6 +110,7 @@ spec:
           - name: STORE_ERRORS_OPTIONS
             value: {{ .Values.store.errors.options | toJson}}
           {{- end }}
+          {{- end }}
           - name: STATSD_HOST
             value: {{ .Values.statsd.host }}
           - name: STATSD_PORT

diff --git a/cli/feast/pkg/parse/yaml.go b/cli/feast/pkg/parse/yaml.go
@@ -34,7 +34,6 @@ func YamlToFeatureSpec(in []byte) (*specs.FeatureSpec, error) {
 	if err != nil {
 		return nil, err
 	}
-	ymlMap["granularity"] = granularityOf(ymlMap["granularity"].(string))
 	ymlMap["valueType"] = valueTypeOf(ymlMap["valueType"].(string))
 	if err != nil {
 		return nil, err
@@ -131,10 +130,6 @@ func YamlToImportSpec(in []byte) (*specs.ImportSpec, error) {
 	return &is, err
 }
 
-func granularityOf(str string) types.Granularity_Enum {
-	return types.Granularity_Enum(types.Granularity_Enum_value[str])
-}
-
 func valueTypeOf(str string) types.ValueType_Enum {
 	return types.ValueType_Enum(types.ValueType_Enum_value[str])
 }
diff --git a/cli/feast/pkg/parse/yaml_test.go b/cli/feast/pkg/parse/yaml_test.go
@@ -34,26 +34,24 @@ func TestYamlToFeatureSpec(t *testing.T) {
 	}{
 		{
 			name: "valid yaml",
-			input: []byte(`id: test.none.test_feature_two
+			input: []byte(`id: test.test_feature_two
 name: test_feature_two
 entity: test
 owner: [email protected]
 description: testing feature
 valueType:  INT64
-granularity: NONE
 uri: https://github.com/bob/example
 dataStores:
   serving:
     id: REDIS
   warehouse:
     id: BIGQUERY`),
 			expected: &specs.FeatureSpec{
-				Id:          "test.none.test_feature_two",
+				Id:          "test.test_feature_two",
 				Owner:       "[email protected]",
 				Name:        "test_feature_two",
 				Description: "testing feature",
 				Uri:         "https://github.com/bob/example",
-				Granularity: types.Granularity_NONE,
 				ValueType:   types.ValueType_INT64,
 				Entity:      "test",
 				DataStores: &specs.DataStores{
@@ -257,7 +255,7 @@ schema:
     - name: timestamp
     - name: driver_id
     - name: last_opportunity
-      featureId: driver.none.last_opportunity`),
+      featureId: driver.last_opportunity`),
 			expected: &specs.ImportSpec{
 				Type: "file",
 				JobOptions: map[string]string{
@@ -272,7 +270,7 @@ schema:
 					Fields: []*specs.Field{
 						{Name: "timestamp"},
 						{Name: "driver_id"},
-						{Name: "last_opportunity", FeatureId: "driver.none.last_opportunity"},
+						{Name: "last_opportunity", FeatureId: "driver.last_opportunity"},
 					},
 					EntityIdColumn: "driver_id",
 					Timestamp: &specs.Schema_TimestampValue{

diff --git a/cli/feast/pkg/printer/printer_test.go b/cli/feast/pkg/printer/printer_test.go
@@ -23,12 +23,11 @@ func TestPrintFeature(t *testing.T) {
 			name: "with storage",
 			input: &core.UIServiceTypes_FeatureDetail{
 				Spec: &specs.FeatureSpec{
-					Id:          "test.none.test_feature_two",
+					Id:          "test.test_feature_two",
 					Owner:       "[email protected]",
 					Name:        "test_feature_two",
 					Description: "testing feature",
 					Uri:         "https://github.com/bob/example",
-					Granularity: types.Granularity_NONE,
 					ValueType:   types.ValueType_INT64,
 					Entity:      "test",
 					DataStores: &specs.DataStores{
@@ -45,7 +44,7 @@ func TestPrintFeature(t *testing.T) {
 				LastUpdated:  &timestamp.Timestamp{Seconds: 1},
 				Created:      &timestamp.Timestamp{Seconds: 1},
 			},
-			expected: fmt.Sprintf(`Id:	test.none.test_feature_two
+			expected: fmt.Sprintf(`Id:	test.test_feature_two
 Entity:	test
 Owner:	[email protected]
 Description:	testing feature
@@ -65,12 +64,11 @@ Related Jobs:
 			name: "no storage",
 			input: &core.UIServiceTypes_FeatureDetail{
 				Spec: &specs.FeatureSpec{
-					Id:          "test.none.test_feature_two",
+					Id:          "test.test_feature_two",
 					Owner:       "[email protected]",
 					Name:        "test_feature_two",
 					Description: "testing feature",
 					Uri:         "https://github.com/bob/example",
-					Granularity: types.Granularity_NONE,
 					ValueType:   types.ValueType_INT64,
 					Entity:      "test",
 				},
@@ -79,7 +77,7 @@ Related Jobs:
 				LastUpdated:  &timestamp.Timestamp{Seconds: 1},
 				Created:      &timestamp.Timestamp{Seconds: 1},
 			},
-			expected: fmt.Sprintf(`Id:	test.none.test_feature_two
+			expected: fmt.Sprintf(`Id:	test.test_feature_two
 Entity:	test
 Owner:	[email protected]
 Description:	testing feature

diff --git a/core/pom.xml b/core/pom.xml
@@ -16,8 +16,8 @@
   ~
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>feast</groupId>
@@ -31,8 +31,45 @@
     <build>
         <plugins>
             <plugin>
-                <groupId>org.springframework.boot</groupId>
-                <artifactId>spring-boot-maven-plugin</artifactId>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.4.3</version>
+                <configuration>
+                    <transformers>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>META-INF/spring.handlers</resource>
+                        </transformer>
+                        <transformer implementation="org.springframework.boot.maven.PropertiesMergingResourceTransformer">
+                            <resource>META-INF/spring.factories</resource>
+                        </transformer>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>META-INF/spring.schemas</resource>
+                        </transformer>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                            <resource>META-INF/spring.components</resource>
+                        </transformer>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <mainClass>feast.core.CoreApplication</mainClass>
+                        </transformer>
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ComponentsXmlResourceTransformer" />
+                        <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                    </transformers>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.springframework.boot</groupId>
+                        <artifactId>spring-boot-maven-plugin</artifactId>
+                        <version>${springBootVersion}</version>
+                    </dependency>
+                </dependencies>
             </plugin>
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>

diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
@@ -283,7 +283,7 @@ public void applyFeature(
   }
 
   public FeatureSpec applyDefaultStores(FeatureSpec featureSpec) {
-    DataStores.Builder dataStoreBuilder = DataStores.newBuilder();
+    DataStores.Builder dataStoreBuilder = featureSpec.getDataStores().toBuilder();
     if (Strings.isNullOrEmpty(featureSpec.getDataStores().getServing().getId())) {
       log.info("Feature has no serving store specified using default");
       if (storageSpecs.getServingStorageSpec() != null) {

diff --git a/core/src/main/java/feast/core/job/StatsdMetricPusher.java b/core/src/main/java/feast/core/job/StatsdMetricPusher.java
@@ -43,7 +43,7 @@ public void pushMetrics(List<Metrics> metrics) {
       // `row:result` <- all
       // `scope:scope_id:result` <- per scope, either feature or entity
       // for examples:
-      // 1. feature:driver.minute.ping_sequence_weak_dir_change_mean:error
+      // 1. feature:driver.ping_sequence_weak_dir_change_mean:error
       // 2. entity:driver:stored
       // currently there are 3 results:
       // 1. valid

diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java
@@ -23,7 +23,6 @@
 import feast.specs.FeatureSpecProto.DataStore;
 import feast.specs.FeatureSpecProto.DataStores;
 import feast.specs.FeatureSpecProto.FeatureSpec;
-import feast.types.GranularityProto.Granularity;
 import feast.types.ValueProto.ValueType;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -62,9 +61,6 @@ public class FeatureInfo extends AbstractTimestampEntity {
   @Column(name = "uri", nullable = false)
   private String uri;
 
-  @Enumerated(EnumType.STRING)
-  private Granularity.Enum granularity;
-
   @Enumerated(EnumType.STRING)
   private ValueType.Enum valueType;
 
@@ -120,7 +116,6 @@ public FeatureInfo(
     this.owner = spec.getOwner();
     this.description = spec.getDescription();
     this.uri = spec.getUri();
-    this.granularity = spec.getGranularity();
     this.valueType = spec.getValueType();
     this.entity = entityInfo;
     this.featureGroup = featureGroupInfo;
@@ -144,7 +139,6 @@ public FeatureInfo(FeatureInfo other) {
     this.owner = other.owner;
     this.description = other.description;
     this.uri = other.uri;
-    this.granularity = other.granularity;
     this.valueType = other.valueType;
     this.entity = other.entity;
     this.featureGroup = other.featureGroup;
@@ -183,7 +177,6 @@ public FeatureSpec getFeatureSpec() {
             .setOwner(owner)
             .setDescription(description)
             .setUri(uri)
-            .setGranularity(granularity)
             .setValueType(valueType)
             .setEntity(entity.getName())
             .addAllTags(convertTagStringToList(tags))
@@ -250,8 +243,8 @@ private String createBigqueryViewLink(StorageInfo warehouseStore) {
     String dataset = opts.get(BigQueryStorageManager.OPT_BIGQUERY_DATASET);
 
     return String.format(
-        "https://bigquery.cloud.google.com/table/%s:%s.%s_%s_view",
-        projectId, dataset, entity.getName(), granularity.toString().toLowerCase());
+        "https://bigquery.cloud.google.com/table/%s:%s.%s_view",
+        projectId, dataset, entity.getName());
   }
 
   /**
@@ -276,7 +269,6 @@ private boolean isLegalUpdate(FeatureSpec update) {
     FeatureSpec spec = this.getFeatureSpec();
     return spec.getName().equals(update.getName())
         && spec.getEntity().equals(update.getEntity())
-        && spec.getGranularity().equals(update.getGranularity())
         && spec.getValueType().equals(update.getValueType())
         && spec.getGroup().equals(update.getGroup())
         && spec.getOptionsMap().equals(update.getOptionsMap())

diff --git a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java
@@ -191,8 +191,7 @@ private Field createFeatureField(FeatureSpec featureSpec) {
 
   private String createTableName(FeatureSpec featureSpec) {
     String entityName = featureSpec.getEntity().toLowerCase();
-    String granularity = featureSpec.getGranularity().toString().toLowerCase();
-    return String.format("%s_%s", entityName, granularity);
+    return String.format("%s", entityName);
   }
 
   private void createOrUpdateView(String tableName, List<String> features) {

diff --git a/core/src/main/java/feast/core/storage/PostgresStorageManager.java b/core/src/main/java/feast/core/storage/PostgresStorageManager.java
@@ -91,7 +91,6 @@ private String createFieldType(FeatureSpec featureSpec) {
 
   private String createTableName(FeatureSpec featureSpec) {
     String entityName = featureSpec.getEntity().toLowerCase();
-    String granularity = featureSpec.getGranularity().toString().toLowerCase();
-    return String.format("%s_%s", entityName, granularity);
+    return String.format("%s", entityName);
   }
 }