Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor tests which utilizes feature sets #1186

Merged
merged 4 commits into from
Nov 23, 2020
Merged

Refactor tests which utilizes feature sets #1186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9d9b097
Update common module
terryyylim Nov 21, 2020
eafeb85
Update core module
terryyylim Nov 21, 2020
8dae402
Update serving module
terryyylim Nov 21, 2020
3d92bf2
Update storage module
terryyylim Nov 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 4 additions & 85 deletions common-test/src/main/java/feast/common/it/DataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,11 @@
import feast.proto.core.EntityProto;
import feast.proto.core.FeatureProto;
import feast.proto.core.FeatureProto.FeatureSpecV2;
import feast.proto.core.FeatureSetProto;
import feast.proto.core.FeatureTableProto.FeatureTableSpec;
import feast.proto.core.SourceProto;
import feast.proto.core.StoreProto;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -67,10 +64,6 @@ public static SourceProto.Source getDefaultSource() {
return defaultSource;
}

public static FeatureSetProto.FeatureSet getDefaultFeatureSet() {
return createFeatureSet(DataGenerator.getDefaultSource(), "default", "test");
}

public static SourceProto.Source createSource(String server, String topic) {
return SourceProto.Source.newBuilder()
.setType(SourceProto.SourceType.KAFKA)
Expand Down Expand Up @@ -115,20 +108,6 @@ public static StoreProto.Store createStore(
}
}

public static FeatureSetProto.FeatureSpec createFeature(
String name, ValueProto.ValueType.Enum valueType, Map<String, String> labels) {
return FeatureSetProto.FeatureSpec.newBuilder()
.setName(name)
.setValueType(valueType)
.putAllLabels(labels)
.build();
}

public static FeatureSetProto.EntitySpec createEntitySpec(
String name, ValueProto.ValueType.Enum valueType) {
return FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build();
}

public static EntityProto.EntitySpecV2 createEntitySpecV2(
String name,
String description,
Expand All @@ -151,70 +130,6 @@ public static FeatureProto.FeatureSpecV2 createFeatureSpecV2(
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
List<FeatureSetProto.EntitySpec> entities,
List<FeatureSetProto.FeatureSpec> features,
Map<String, String> labels) {
return FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetProto.FeatureSetSpec.newBuilder()
.setSource(source)
.setName(name)
.setProject(projectName)
.putAllLabels(labels)
.addAllEntities(entities)
.addAllFeatures(features)
.build())
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
Map<String, ValueProto.ValueType.Enum> entities,
Map<String, ValueProto.ValueType.Enum> features,
Map<String, String> labels) {
return FeatureSetProto.FeatureSet.newBuilder()
.setSpec(
FeatureSetProto.FeatureSetSpec.newBuilder()
.setSource(source)
.setName(name)
.setProject(projectName)
.putAllLabels(labels)
.addAllEntities(
entities.entrySet().stream()
.map(entry -> createEntitySpec(entry.getKey(), entry.getValue()))
.collect(Collectors.toList()))
.addAllFeatures(
features.entrySet().stream()
.map(
entry ->
createFeature(
entry.getKey(), entry.getValue(), Collections.emptyMap()))
.collect(Collectors.toList()))
.build())
.build();
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source,
String projectName,
String name,
Map<String, ValueProto.ValueType.Enum> entities,
Map<String, ValueProto.ValueType.Enum> features) {
return createFeatureSet(source, projectName, name, entities, features, new HashMap<>());
}

public static FeatureSetProto.FeatureSet createFeatureSet(
SourceProto.Source source, String projectName, String name) {
return createFeatureSet(
source, projectName, name, Collections.emptyMap(), Collections.emptyMap());
}

// Create a Feature Table spec without DataSources configured.
public static FeatureTableSpec createFeatureTableSpec(
String name,
Expand Down Expand Up @@ -309,6 +224,10 @@ public static ValueProto.Value createEmptyValue() {
return ValueProto.Value.newBuilder().build();
}

public static ValueProto.Value createStrValue(String val) {
return ValueProto.Value.newBuilder().setStringVal(val).build();
}

public static ValueProto.Value createDoubleValue(double value) {
return ValueProto.Value.newBuilder().setDoubleVal(value).build();
}
Expand Down
75 changes: 0 additions & 75 deletions common-test/src/main/java/feast/common/it/SimpleCoreClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,6 @@ public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) {
this.stub = stub;
}

public CoreServiceProto.ApplyFeatureSetResponse simpleApplyFeatureSet(
FeatureSetProto.FeatureSet featureSet) {
return stub.applyFeatureSet(
CoreServiceProto.ApplyFeatureSetRequest.newBuilder().setFeatureSet(featureSet).build());
}

public CoreServiceProto.ApplyEntityResponse simpleApplyEntity(
String projectName, EntityProto.EntitySpecV2 spec) {
return stub.applyEntity(
Expand Down Expand Up @@ -84,53 +78,6 @@ public List<FeatureTableProto.FeatureTable> simpleListFeatureTables(
.getTablesList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName, Map<String, String> labels) {
return stub.listFeatureSets(
CoreServiceProto.ListFeatureSetsRequest.newBuilder()
.setFilter(
CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder()
.setProject(projectName)
.setFeatureSetName(featureSetName)
.putAllLabels(labels)
.build())
.build())
.getFeatureSetsList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName, FeatureSetProto.FeatureSetStatus status) {
return stub.listFeatureSets(
CoreServiceProto.ListFeatureSetsRequest.newBuilder()
.setFilter(
CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder()
.setProject(projectName)
.setFeatureSetName(featureSetName)
.setStatus(status)
.build())
.build())
.getFeatureSetsList();
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(
String projectName, String featureSetName) {
return simpleListFeatureSets(
projectName, featureSetName, FeatureSetProto.FeatureSetStatus.STATUS_INVALID);
}

public List<FeatureSetProto.FeatureSet> simpleListFeatureSets(String featureSetName) {
return simpleListFeatureSets("default", featureSetName);
}

public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String name) {
return stub.getFeatureSet(
CoreServiceProto.GetFeatureSetRequest.newBuilder()
.setName(name)
.setProject(projectName)
.build())
.getFeatureSet();
}

public EntityProto.Entity simpleGetEntity(String projectName, String name) {
return stub.getEntity(
CoreServiceProto.GetEntityRequest.newBuilder()
Expand All @@ -149,19 +96,6 @@ public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName,
.getTable();
}

public void updateFeatureSetStatus(
String projectName, String name, FeatureSetProto.FeatureSetStatus status) {
stub.updateFeatureSetStatus(
CoreServiceProto.UpdateFeatureSetStatusRequest.newBuilder()
.setReference(
FeatureSetReferenceProto.FeatureSetReference.newBuilder()
.setProject(projectName)
.setName(name)
.build())
.setStatus(status)
.build());
}

public Map<String, FeatureProto.FeatureSpecV2> simpleListFeatures(
String projectName, Map<String, String> labels, List<String> entities) {
return stub.listFeatures(
Expand Down Expand Up @@ -200,15 +134,6 @@ public String getFeastCoreVersion() {
.getVersion();
}

public FeatureSetProto.FeatureSet getFeatureSet(String projectName, String featureSetName) {
return stub.getFeatureSet(
CoreServiceProto.GetFeatureSetRequest.newBuilder()
.setProject(projectName)
.setName(featureSetName)
.build())
.getFeatureSet();
}

public FeatureTableProto.FeatureTable applyFeatureTable(
String projectName, FeatureTableSpec spec) {
return stub.applyFeatureTable(
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/java/feast/common/logging/entry/LogResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
public abstract class LogResource {
public enum ResourceType {
JOB,
FEATURE_SET,
FEATURE_TABLE
}

public abstract ResourceType getType();
Expand Down
34 changes: 19 additions & 15 deletions common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import feast.common.logging.entry.LogResource.ResourceType;
import feast.proto.serving.ServingAPIProto.FeatureReference;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest;
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues;
import feast.proto.types.ValueProto.Value;
Expand All @@ -34,31 +34,38 @@

public class AuditLogEntryTest {
public List<AuditLogEntry> getTestAuditLogs() {
GetOnlineFeaturesRequest requestSpec =
GetOnlineFeaturesRequest.newBuilder()
.setOmitEntitiesInResponse(false)
GetOnlineFeaturesRequestV2 requestSpec =
GetOnlineFeaturesRequestV2.newBuilder()
.addAllFeatures(
Arrays.asList(
FeatureReference.newBuilder().setName("feature1").build(),
FeatureReference.newBuilder().setName("feature2").build()))
FeatureReferenceV2.newBuilder()
.setFeatureTable("featuretable_1")
.setName("feature1")
.build(),
FeatureReferenceV2.newBuilder()
.setFeatureTable("featuretable_1")
.setName("feature2")
.build()))
.build();

GetOnlineFeaturesResponse responseSpec =
GetOnlineFeaturesResponse.newBuilder()
.addAllFieldValues(
Arrays.asList(
FieldValues.newBuilder()
.putFields("feature", Value.newBuilder().setInt32Val(32).build())
.putFields(
"featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build())
.build(),
FieldValues.newBuilder()
.putFields("feature2", Value.newBuilder().setInt32Val(64).build())
.putFields(
"featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build())
.build()))
.build();

return Arrays.asList(
MessageAuditLogEntry.newBuilder()
.setComponent("feast-serving")
.setVersion("0.6")
.setVersion("0.9")
.setService("ServingService")
.setMethod("getOnlineFeatures")
.setRequest(requestSpec)
Expand All @@ -67,12 +74,9 @@ public List<AuditLogEntry> getTestAuditLogs() {
.setIdentity("[email protected]")
.build(),
ActionAuditLogEntry.of(
"core", "0.6", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
"core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
TransitionAuditLogEntry.of(
"core",
"0.6",
LogResource.of(ResourceType.FEATURE_SET, "project/feature_set"),
"READY"));
"core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY"));
}

@Test
Expand Down
Loading