diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java index f13a9cf8..4a00fd67 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java @@ -1,6 +1,8 @@ package org.hypertrace.gateway.service.common; +import static java.util.Collections.emptyMap; import static java.util.function.Predicate.not; +import static org.hypertrace.core.attribute.service.v1.AttributeSource.QS; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; @@ -58,6 +60,7 @@ public class ExpressionContext { private ImmutableMap> sourceToFilterExpressionMap; private ImmutableMap> sourceToFilterAttributeMap; private ImmutableMap> filterAttributeToSourceMap; + private Map sourceToFilterMap; // and filter private boolean isAndFilter; @@ -92,6 +95,9 @@ public ExpressionContext( buildSourceToGroupByExpressionMaps(); this.isAndFilter = gatewayServiceConfig.isEntityAndFilterEnabled() && isAndFilter(filter); + // build source to filter map only if we only have AND filter + this.sourceToFilterMap = + isAndFilter(filter) ? buildSourceToAndFilterMap(filter) : Collections.emptyMap(); } public Map> getSourceToSelectionExpressionMap() { @@ -106,6 +112,10 @@ public void setSourceToSelectionExpressionMap( .build(); } + public Map getSourceToFilterMap() { + return sourceToFilterMap; + } + public Map> getSourceToSelectionAttributeMap() { return sourceToSelectionAttributeMap; } @@ -620,6 +630,45 @@ private static Set getIntersectingSourceSets( .orElse(Collections.emptySet()); } + private Map buildSourceToAndFilterMap(Filter filter) { + Operator operator = filter.getOperator(); + if (operator == Operator.AND) { + return filter.getChildFilterList().stream() + .map(this::buildSourceToAndFilterMap) + .flatMap(map -> map.entrySet().stream()) + .collect( + Collectors.toUnmodifiableMap( + Map.Entry::getKey, + Map.Entry::getValue, + (value1, value2) -> + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(value1) + .addChildFilter(value2) + .build())); + + } else if (operator == Operator.OR) { + return Collections.emptyMap(); + } else { + List attributeSources = getAttributeSources(filter.getLhs()); + if (attributeSources.isEmpty()) { + return emptyMap(); + } + + return attributeSources.contains(QS) + ? Map.of(QS, filter) + : Map.of(attributeSources.get(0), filter); + } + } + + public List getAttributeSources(Expression expression) { + Set attributeIds = ExpressionReader.extractAttributeIds(expression); + return attributeIds.stream() + .map(attributeId -> attributeMetadataMap.get(attributeId).getSourcesList()) + .flatMap(Collection::stream) + .collect(Collectors.toUnmodifiableList()); + } + @Override public String toString() { return "ExpressionContext{" diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java index 23c9dfe5..527c789b 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java @@ -1,7 +1,6 @@ package org.hypertrace.gateway.service.entity.query; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableList; import static org.hypertrace.core.attribute.service.v1.AttributeSource.EDS; import static org.hypertrace.core.attribute.service.v1.AttributeSource.QS; @@ -9,24 +8,19 @@ import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.hypertrace.core.attribute.service.v1.AttributeMetadata; import org.hypertrace.core.attribute.service.v1.AttributeSource; import org.hypertrace.gateway.service.common.ExpressionContext; -import org.hypertrace.gateway.service.common.util.ExpressionReader; import org.hypertrace.gateway.service.common.util.TimeRangeFilterUtil; import org.hypertrace.gateway.service.entity.query.visitor.ExecutionContextBuilderVisitor; import org.hypertrace.gateway.service.entity.query.visitor.FilterOptimizingVisitor; import org.hypertrace.gateway.service.entity.query.visitor.PrintVisitor; -import org.hypertrace.gateway.service.v1.common.Expression; import org.hypertrace.gateway.service.v1.common.Filter; import org.hypertrace.gateway.service.v1.common.Operator; import org.hypertrace.gateway.service.v1.common.OrderByExpression; @@ -39,18 +33,11 @@ public class ExecutionTreeBuilder { private static final Logger LOG = LoggerFactory.getLogger(ExecutionTreeBuilder.class); - private final Map attributeMetadataMap; private final EntityExecutionContext executionContext; private final Set sourceSetsIfFilterAndOrderByAreFromSameSourceSets; public ExecutionTreeBuilder(EntityExecutionContext executionContext) { this.executionContext = executionContext; - this.attributeMetadataMap = - executionContext - .getAttributeMetadataProvider() - .getAttributesMetadata( - executionContext.getEntitiesRequestContext(), - executionContext.getEntitiesRequest().getEntityType()); this.sourceSetsIfFilterAndOrderByAreFromSameSourceSets = ExpressionContext.getSourceSetsIfFilterAndOrderByAreFromSameSourceSets( @@ -132,7 +119,7 @@ public QueryNode build() { ExecutionTreeUtils.removeDuplicateSelectionAttributes(executionContext, QS.name()); - QueryNode filterTree = buildFilterTree(executionContext, entitiesRequest.getFilter()); + QueryNode filterTree = buildFilterTreeNode(executionContext, entitiesRequest.getFilter()); if (LOG.isDebugEnabled()) { LOG.debug("Filter Tree:{}", filterTree.acceptVisitor(new PrintVisitor())); } @@ -268,8 +255,7 @@ QueryNode buildExecutionTree(EntityExecutionContext executionContext, QueryNode return rootNode; } - @VisibleForTesting - QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) { + QueryNode buildFilterTreeNode(EntityExecutionContext context, Filter filter) { EntitiesRequest entitiesRequest = executionContext.getEntitiesRequest(); // Convert the time range into a filter and set it on the request so that all downstream // components needn't treat it specially @@ -281,13 +267,12 @@ QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) { entitiesRequest.getEndTimeMillis()); boolean isAndFilter = executionContext.getExpressionContext().isAndFilter(); - return isAndFilter - ? buildAndFilterTree(entitiesRequest) - : buildFilterTree(entitiesRequest, timeRangeFilter); + return isAndFilter ? buildAndFilterTree(context) : buildFilterTree(context, timeRangeFilter); } @VisibleForTesting - QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) { + QueryNode buildFilterTree(EntityExecutionContext context, Filter filter) { + EntitiesRequest entitiesRequest = context.getEntitiesRequest(); if (filter.equals(Filter.getDefaultInstance())) { return new NoOpNode(); } @@ -295,15 +280,16 @@ QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) { if (operator == Operator.AND) { return new AndNode( filter.getChildFilterList().stream() - .map(childFilter -> buildFilterTree(entitiesRequest, childFilter)) + .map(childFilter -> buildFilterTree(context, childFilter)) .collect(Collectors.toList())); } else if (operator == Operator.OR) { return new OrNode( filter.getChildFilterList().stream() - .map(childFilter -> buildFilterTree(entitiesRequest, childFilter)) + .map(childFilter -> buildFilterTree(context, childFilter)) .collect(Collectors.toList())); } else { - List sources = getAttributeSources(filter.getLhs()); + List sources = + context.getExpressionContext().getAttributeSources(filter.getLhs()); // if the filter by and order by are from QS, pagination can be pushed down to QS // There will always be a DataFetcherNode for QS, because the results are always fetched @@ -319,7 +305,8 @@ QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) { } // filters and order by on QS, but you can still have selection on EDS - QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) { + QueryNode buildAndFilterTree(EntityExecutionContext context) { + EntitiesRequest entitiesRequest = context.getEntitiesRequest(); // If the filter by and order by are from QS (and selections are on other sources), pagination // can be pushed down to QS // Since the filter and order by are from QS, there won't be any filter on other @@ -330,7 +317,7 @@ QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) { } Map sourceToAndFilterMap = - new HashMap<>(buildSourceToAndFilterMap(entitiesRequest.getFilter())); + new HashMap<>(context.getExpressionContext().getSourceToFilterMap()); // qs node as the pivot node to fetch time range data QueryNode qsNode = @@ -387,37 +374,6 @@ QueryNode buildAndFilterTree(EntitiesRequest entitiesRequest) { } } - private Map buildSourceToAndFilterMap(Filter filter) { - Operator operator = filter.getOperator(); - if (operator == Operator.AND) { - return filter.getChildFilterList().stream() - .map(this::buildSourceToAndFilterMap) - .flatMap(map -> map.entrySet().stream()) - .collect( - Collectors.toUnmodifiableMap( - Entry::getKey, - Entry::getValue, - (value1, value2) -> - Filter.newBuilder() - .setOperator(Operator.AND) - .addChildFilter(value1) - .addChildFilter(value2) - .build())); - - } else if (operator == Operator.OR) { - return Collections.emptyMap(); - } else { - List attributeSources = getAttributeSources(filter.getLhs()); - if (attributeSources.isEmpty()) { - return emptyMap(); - } - - return attributeSources.contains(QS) - ? Map.of(QS, filter) - : Map.of(attributeSources.get(0), filter); - } - } - private QueryNode checkAndAddSortAndPaginationNode( QueryNode childNode, EntityExecutionContext executionContext) { EntitiesRequest entitiesRequest = executionContext.getEntitiesRequest(); @@ -479,12 +435,4 @@ private QueryNode createQsDataFetcherNodeWithLimitAndOffset(EntitiesRequest enti private QueryNode createPaginateOnlyNode(QueryNode queryNode, EntitiesRequest entitiesRequest) { return new PaginateOnlyNode(queryNode, entitiesRequest.getLimit(), entitiesRequest.getOffset()); } - - public List getAttributeSources(Expression expression) { - Set attributeIds = ExpressionReader.extractAttributeIds(expression); - return attributeIds.stream() - .map(attributeId -> attributeMetadataMap.get(attributeId).getSourcesList()) - .flatMap(Collection::stream) - .collect(Collectors.toUnmodifiableList()); - } } diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/visitor/ExecutionVisitor.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/visitor/ExecutionVisitor.java index f5e372ea..d9baca90 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/visitor/ExecutionVisitor.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/query/visitor/ExecutionVisitor.java @@ -10,11 +10,13 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.hypertrace.core.attribute.service.v1.AttributeSource; import org.hypertrace.gateway.service.common.datafetcher.EntityFetcherResponse; import org.hypertrace.gateway.service.common.datafetcher.EntityResponse; import org.hypertrace.gateway.service.common.datafetcher.IEntityFetcher; @@ -265,7 +267,7 @@ public EntityResponse visit(SelectionNode selectionNode) { .getExpressionContext() .getSourceToSelectionExpressionMap() .get(source)) - .setFilter(filter) + .setFilter(addSourceFilters(executionContext, source, filter)) .build(); IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source); EntitiesRequestContext context = @@ -295,7 +297,7 @@ public EntityResponse visit(SelectionNode selectionNode) { .getExpressionContext() .getSourceToMetricExpressionMap() .get(source)) - .setFilter(filter) + .setFilter(addSourceFilters(executionContext, source, filter)) .build(); IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source); EntitiesRequestContext context = @@ -325,7 +327,7 @@ public EntityResponse visit(SelectionNode selectionNode) { .getExpressionContext() .getSourceToTimeAggregationMap() .get(source)) - .setFilter(filter) + .setFilter(addSourceFilters(executionContext, source, filter)) .build(); IEntityFetcher entityFetcher = queryHandlerRegistry.getEntityFetcher(source); EntitiesRequestContext requestContext = @@ -355,6 +357,25 @@ public EntityResponse visit(SelectionNode selectionNode) { } } + private Filter addSourceFilters( + EntityExecutionContext executionContext, String source, Filter filter) { + Optional sourceFilterOptional = + Optional.ofNullable( + executionContext + .getExpressionContext() + .getSourceToFilterMap() + .get(AttributeSource.valueOf(source))); + return sourceFilterOptional + .map( + sourceFilter -> + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(filter) + .addChildFilter(sourceFilter) + .build()) + .orElse(filter); + } + Filter constructFilterFromChildNodesResult(EntityFetcherResponse result) { if (result.isEmpty()) { return Filter.getDefaultInstance(); diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/AbstractServiceTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/AbstractServiceTest.java index 34d93eeb..81d52e38 100644 --- a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/AbstractServiceTest.java +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/AbstractServiceTest.java @@ -19,7 +19,6 @@ import java.io.Reader; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -39,6 +38,7 @@ import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs; import org.hypertrace.gateway.service.common.util.QueryServiceClient; import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig; +import org.hypertrace.gateway.service.entity.config.LogConfig; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; @@ -82,14 +82,19 @@ public static void setUp() throws IOException { + " }\n" + " ]\n" + " }\n" - + "]"; + + "]\n" + + "entity.service.log.config = {\n" + + " query.threshold.millis = 1500\n" + + "}\n"; Config config = ConfigFactory.parseString(scopeFiltersConfig); scopeFilterConfigs = new ScopeFilterConfigs(config); - entityIdColumnsConfig = new EntityIdColumnsConfig(Collections.emptyMap()); + entityIdColumnsConfig = new EntityIdColumnsConfig(Map.of("BACKEND", "id")); gatewayServiceConfig = mock(GatewayServiceConfig.class); when(gatewayServiceConfig.getEntityIdColumnsConfig()).thenReturn(entityIdColumnsConfig); when(gatewayServiceConfig.getScopeFilterConfigs()).thenReturn(scopeFilterConfigs); entityTypesProvider = mock(EntityTypesProvider.class); + LogConfig logConfig = new LogConfig(config); + when(gatewayServiceConfig.getLogConfig()).thenReturn(logConfig); } private static Reader readResourceFile(String fileName) { diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/converter/EntityServiceAndGatewayServiceConverterTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/converter/EntityServiceAndGatewayServiceConverterTest.java index b990a9c6..8e0dcfff 100644 --- a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/converter/EntityServiceAndGatewayServiceConverterTest.java +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/converter/EntityServiceAndGatewayServiceConverterTest.java @@ -24,13 +24,13 @@ import org.hypertrace.gateway.service.v1.entity.EntitiesRequest; import org.junit.jupiter.api.Test; -public class EntityServiceAndGatewayServiceConverterTest extends AbstractGatewayServiceTest { +class EntityServiceAndGatewayServiceConverterTest extends AbstractGatewayServiceTest { @Test - public void testAddBetweenFilter() { + void testAddBetweenFilter() { int startTimeMillis = 1; int endTimeMillis = 2; - String timestamp = "lastActivity"; + String timestamp = "startTime"; String timestampAttributeName = BACKEND.name() + "." + timestamp; Expression.Builder expectedStartTimeConstant = diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilderTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilderTest.java index a6139e6b..0ef5f953 100644 --- a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilderTest.java +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilderTest.java @@ -194,7 +194,7 @@ public void testOptimizedFilterTreeBuilderSimpleFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); QueryNode optimizedQueryNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedQueryNode); assertTrue(optimizedQueryNode instanceof DataFetcherNode); @@ -213,7 +213,7 @@ public void testOptimizedFilterTreeBuilderAndOrFilterSingleDataSource() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); assertTrue(queryNode instanceof AndNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); @@ -232,7 +232,7 @@ public void testOptimizedFilterTreeBuilderAndOrFilterSingleDataSource() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); assertTrue(queryNode instanceof OrNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); @@ -285,7 +285,7 @@ public void testOptimizedFilterTreeBuilderAndOrFilterMultiDataSource() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); assertTrue(queryNode instanceof AndNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); @@ -311,7 +311,7 @@ public void testOptimizedFilterTreeBuilderAndOrFilterMultiDataSource() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); assertTrue(queryNode instanceof OrNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); @@ -360,7 +360,7 @@ public void testOptimizedFilterTreeBuilderNestedAndFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedNode); @@ -385,7 +385,7 @@ public void testOptimizedFilterTreeBuilderNestedAndFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedNode); @@ -412,7 +412,7 @@ public void testOptimizedFilterTreeBuilderNestedAndFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedNode); @@ -442,7 +442,7 @@ public void testOptimizedFilterTreeBuilderNestedAndOrFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedNode); @@ -467,7 +467,7 @@ public void testOptimizedFilterTreeBuilderNestedAndOrFilter() { EntitiesRequest entitiesRequest = buildEntitiesRequest(filter); EntityExecutionContext executionContext = getExecutionContext(entitiesRequest); ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext); - QueryNode queryNode = executionTreeBuilder.buildFilterTree(entitiesRequest, filter); + QueryNode queryNode = executionTreeBuilder.buildFilterTree(executionContext, filter); assertNotNull(queryNode); QueryNode optimizedNode = queryNode.acceptVisitor(new FilterOptimizingVisitor()); assertNotNull(optimizedNode); diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/explore/entity/EntityServiceTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/explore/entity/EntityServiceTest.java new file mode 100644 index 00000000..5250c0fe --- /dev/null +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/explore/entity/EntityServiceTest.java @@ -0,0 +1,60 @@ +package org.hypertrace.gateway.service.explore.entity; + +import com.google.protobuf.GeneratedMessageV3; +import java.util.concurrent.Executors; +import java.util.stream.Stream; +import org.hypertrace.entity.query.service.client.EntityQueryServiceClient; +import org.hypertrace.gateway.service.EntityTypesProvider; +import org.hypertrace.gateway.service.common.AbstractServiceTest; +import org.hypertrace.gateway.service.common.AttributeMetadataProvider; +import org.hypertrace.gateway.service.common.RequestContext; +import org.hypertrace.gateway.service.common.config.GatewayServiceConfig; +import org.hypertrace.gateway.service.common.util.QueryServiceClient; +import org.hypertrace.gateway.service.entity.EntityService; +import org.hypertrace.gateway.service.v1.entity.EntitiesRequest; +import org.hypertrace.gateway.service.v1.entity.EntitiesResponse; + +public class EntityServiceTest extends AbstractServiceTest { + private static final String SUITE_NAME = "entity"; + + public static Stream data() { + return getTestFileNames(SUITE_NAME); + } + + @Override + protected String getTestSuiteName() { + return SUITE_NAME; + } + + @Override + protected GeneratedMessageV3.Builder getGatewayServiceRequestBuilder() { + return EntitiesRequest.newBuilder(); + } + + @Override + protected GeneratedMessageV3.Builder getGatewayServiceResponseBuilder() { + return EntitiesResponse.newBuilder(); + } + + @Override + protected EntitiesResponse executeApi( + GatewayServiceConfig gatewayServiceConfig, + EntitiesRequest request, + QueryServiceClient queryServiceClient, + EntityQueryServiceClient entityQueryServiceClient, + AttributeMetadataProvider attributeMetadataProvider, + EntityTypesProvider entityTypesProvider) { + EntityService entityService = + new EntityService( + gatewayServiceConfig, + queryServiceClient, + entityQueryServiceClient, + null, + attributeMetadataProvider, + Executors.newFixedThreadPool(1)); + return entityService.getEntities( + new RequestContext( + org.hypertrace.core.grpcutils.context.RequestContext.forTenantId(TENANT_ID)), + request); + } +} diff --git a/gateway-service-impl/src/test/resources/attributes/attributes.json b/gateway-service-impl/src/test/resources/attributes/attributes.json index 48c3665c..0d1177ac 100644 --- a/gateway-service-impl/src/test/resources/attributes/attributes.json +++ b/gateway-service-impl/src/test/resources/attributes/attributes.json @@ -1312,6 +1312,39 @@ "QS" ] }, + { + "fqn": "Backend.type", + "valueKind": "TYPE_STRING", + "key": "type", + "displayName": "Type", + "scopeString": "BACKEND", + "type": "ATTRIBUTE", + "sources": [ + "QS" + ] + }, + { + "fqn": "Backend.backendApiId", + "valueKind": "TYPE_STRING", + "key": "backendApiId", + "displayName": "BackendApiId", + "scopeString": "BACKEND", + "type": "ATTRIBUTE", + "sources": [ + "QS" + ] + }, + { + "fqn": "Backend.environment", + "valueKind": "TYPE_STRING", + "key": "environment", + "displayName": "Environment", + "scopeString": "BACKEND", + "type": "ATTRIBUTE", + "sources": [ + "QS" + ] + }, { "fqn": "Backend.metrics.bytes_received", "valueKind": "TYPE_INT64", diff --git a/gateway-service-impl/src/test/resources/configs/gateway-service/application.conf b/gateway-service-impl/src/test/resources/configs/gateway-service/application.conf index 667bc910..4335956a 100644 --- a/gateway-service-impl/src/test/resources/configs/gateway-service/application.conf +++ b/gateway-service-impl/src/test/resources/configs/gateway-service/application.conf @@ -46,7 +46,7 @@ interaction.config = [ timestamp.config = [ { scope = BACKEND - timestamp = lastActivity + timestamp = startTime }, { scope = LOG_EVENT diff --git a/gateway-service-impl/src/test/resources/expected-test-responses/entity/backend-distinct-count.json b/gateway-service-impl/src/test/resources/expected-test-responses/entity/backend-distinct-count.json new file mode 100644 index 00000000..bdec9237 --- /dev/null +++ b/gateway-service-impl/src/test/resources/expected-test-responses/entity/backend-distinct-count.json @@ -0,0 +1,94 @@ +{ + "entity": [{ + "id": "backend-id-1", + "entityType": "BACKEND", + "attribute": { + "BACKEND.id": { + "valueType": "STRING", + "string": "backend-id-1" + }, + "id": { + "valueType": "STRING", + "string": "backend-id-1" + }, + "type": { + "valueType": "STRING", + "string": "HTTPS" + }, + "name": { + "valueType": "STRING", + "string": "backend-1.abc.com" + } + }, + "metric": { + "DISTINCT_COUNT_BACKEND.backendApiId_[]": { + "function": "DISTINCTCOUNT", + "value": { + "valueType": "LONG", + "long": "2" + } + } + } + }, { + "id": "backend-id-2", + "entityType": "BACKEND", + "attribute": { + "BACKEND.id": { + "valueType": "STRING", + "string": "backend-id-2" + }, + "id": { + "valueType": "STRING", + "string": "backend-id-2" + }, + "type": { + "valueType": "STRING", + "string": "HTTPS" + }, + "name": { + "valueType": "STRING", + "string": "backend-2.abc.com" + } + }, + "metric": { + "DISTINCT_COUNT_BACKEND.backendApiId_[]": { + "function": "DISTINCTCOUNT", + "value": { + "valueType": "LONG", + "long": "2" + } + } + } + }, { + "id": "backend-id-3", + "entityType": "BACKEND", + "attribute": { + "BACKEND.id": { + "valueType": "STRING", + "string": "backend-id-3" + }, + "id": { + "valueType": "STRING", + "string": "backend-id-3" + }, + "type": { + "valueType": "STRING", + "string": "HTTPS" + }, + "name": { + "valueType": "STRING", + "string": "backend-3.abc.com" + } + }, + "metric": { + "DISTINCT_COUNT_BACKEND.backendApiId_[]": { + "function": "DISTINCTCOUNT", + "value": { + "valueType": "LONG", + "long": "2" + } + } + } + }], + "total": 3 +} \ No newline at end of file diff --git a/gateway-service-impl/src/test/resources/query-service-requests-and-responses/entity/backend-distinct-count.json b/gateway-service-impl/src/test/resources/query-service-requests-and-responses/entity/backend-distinct-count.json new file mode 100644 index 00000000..f40cd04f --- /dev/null +++ b/gateway-service-impl/src/test/resources/query-service-requests-and-responses/entity/backend-distinct-count.json @@ -0,0 +1,334 @@ +[ + { + "request": { + "filter": { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }, + "operator": "NEQ", + "rhs": { + "literal": { + "value": { + "valueType": "NULL_STRING" + } + } + } + }, { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.startTime" + } + }, + "operator": "GE", + "rhs": { + "literal": { + "value": { + "valueType": "LONG", + "long": "1715779687497" + } + } + } + }, { + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.startTime" + } + }, + "operator": "LT", + "rhs": { + "literal": { + "value": { + "valueType": "LONG", + "long": "1715866087497" + } + } + } + }] + }, { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.environment", + "alias": "environment" + } + }, + "operator": "EQ", + "rhs": { + "literal": { + "value": { + "string": "testenv" + } + } + } + }, { + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.backendApiId", + "alias": "backendApiId" + } + }, + "operator": "NEQ", + "rhs": { + "literal": { + "value": { + "string": "null" + } + } + } + }] + }] + }, + "selection": [{ + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.type", + "alias": "type" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.name", + "alias": "name" + } + }, { + "function": { + "functionName": "COUNT", + "arguments": [{ + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }] + } + }], + "groupBy": [{ + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.type", + "alias": "type" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.name", + "alias": "name" + } + }], + "orderBy": [{ + "expression": { + "attributeExpression": { + "attributeId": "BACKEND.name", + "alias": "name" + } + } + }], + "limit": 10000 + }, + "response": { + "isLastChunk": true, + "resultSetMetadata": { + "columnMetadata": [{ + "columnName": "BACKEND.id" + }, { + "columnName": "type" + }, { + "columnName": "name" + }, { + "columnName": "COUNT" + }] + }, + "row": [{ + "column": [{ + "string": "backend-id-1" + }, { + "string": "HTTPS" + }, { + "string": "backend-1.abc.com" + }, { + "string": "15746" + }] + }, { + "column": [{ + "string": "backend-id-2" + }, { + "string": "HTTPS" + }, { + "string": "backend-2.abc.com" + }, { + "string": "26190" + }] + }, { + "column": [{ + "string": "backend-id-3" + }, { + "string": "HTTPS" + }, { + "string": "backend-3.abc.com" + }, { + "string": "31634" + }] + }] + } + }, + { + "request": { + "filter": { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }, + "operator": "NEQ", + "rhs": { + "literal": { + "value": { + "valueType": "NULL_STRING" + } + } + } + }, { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.startTime" + } + }, + "operator": "GE", + "rhs": { + "literal": { + "value": { + "valueType": "LONG", + "long": "1715779687497" + } + } + } + }, { + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.startTime" + } + }, + "operator": "LT", + "rhs": { + "literal": { + "value": { + "valueType": "LONG", + "long": "1715866087497" + } + } + } + }] + }, { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.id", + "alias": "entityId0" + } + }, + "operator": "IN", + "rhs": { + "literal": { + "value": { + "valueType": "STRING_ARRAY", + "stringArray": ["backend-id-3", "backend-id-1", "backend-id-2"] + } + } + } + }, { + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.environment", + "alias": "environment" + } + }, + "operator": "EQ", + "rhs": { + "literal": { + "value": { + "string": "testenv" + } + } + } + }, { + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.backendApiId", + "alias": "backendApiId" + } + }, + "operator": "NEQ", + "rhs": { + "literal": { + "value": { + "string": "null" + } + } + } + }] + }] + }] + }, + "selection": [{ + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }, { + "function": { + "functionName": "DISTINCTCOUNT", + "arguments": [{ + "attributeExpression": { + "attributeId": "BACKEND.backendApiId", + "alias": "backendApiId" + } + }], + "alias": "DISTINCT_COUNT_BACKEND.backendApiId_[]" + } + }], + "groupBy": [{ + "attributeExpression": { + "attributeId": "BACKEND.id" + } + }], + "limit": 10000 + }, + "response": { + "isLastChunk": true, + "resultSetMetadata": { + "columnMetadata": [{ + "columnName": "BACKEND.id" + }, { + "columnName": "DISTINCT_COUNT_BACKEND.backendApiId_[]" + }] + }, + "row": [{ + "column": [{ + "string": "backend-id-3" + }, { + "string": "2" + }] + }, { + "column": [{ + "string": "backend-id-2" + }, { + "string": "2" + }] + }, { + "column": [{ + "string": "backend-id-1" + }, { + "string": "2" + }] + }] + } + } +] \ No newline at end of file diff --git a/gateway-service-impl/src/test/resources/test-requests/entity/backend-distinct-count.json b/gateway-service-impl/src/test/resources/test-requests/entity/backend-distinct-count.json new file mode 100644 index 00000000..3d8acb67 --- /dev/null +++ b/gateway-service-impl/src/test/resources/test-requests/entity/backend-distinct-count.json @@ -0,0 +1,81 @@ +{ + "entityType": "BACKEND", + "startTimeMillis": "1715779687497", + "endTimeMillis": "1715866087497", + "filter": { + "operator": "AND", + "childFilter": [{ + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.environment", + "alias": "environment" + } + }, + "operator": "EQ", + "rhs": { + "literal": { + "value": { + "valueType": "STRING", + "string": "testenv" + } + } + } + }, { + "lhs": { + "attributeExpression": { + "attributeId": "BACKEND.backendApiId", + "alias": "backendApiId" + } + }, + "operator": "NEQ", + "rhs": { + "literal": { + "value": { + "valueType": "STRING", + "string": "null" + } + } + } + }] + }, + "selection": [{ + "attributeExpression": { + "attributeId": "BACKEND.type", + "alias": "type" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.id", + "alias": "id" + } + }, { + "attributeExpression": { + "attributeId": "BACKEND.name", + "alias": "name" + } + }, { + "function": { + "function": "DISTINCTCOUNT", + "arguments": [{ + "attributeExpression": { + "attributeId": "BACKEND.backendApiId", + "alias": "backendApiId" + } + }], + "alias": "DISTINCT_COUNT_BACKEND.backendApiId_[]" + } + }], + "incomingInteractions": { + }, + "outgoingInteractions": { + }, + "orderBy": [{ + "expression": { + "attributeExpression": { + "attributeId": "BACKEND.name", + "alias": "name" + } + } + }], + "limit": 50 +} \ No newline at end of file