feat(advanced-search): Complete Advanced Search: backend changes & ty…

…ing UI together (#6068)

* stashing progress

* adding remove option

* more progress

* editing

* further in

* additional rendering improvements

* stashing adv search progress

* stashing more progress

* propagating not filters back to UI

* more frontend progress

* more filters working

* getting ready for data platform selector

* add platform select functionality

* locking out switching btwn advanced and standard filters

* final polish

* remove unneeded code

* added unit and cypress tests

* resolutions after merge

* adding documentation

* cleaning up & refactoring

* removing console.log

* minor ui fix & removing unneeded code

* fixing lineage search

* fixing lints

* fix display of degree

* fixing test

* fixing lint

* responding to comments

* fixing tests

* fix smoke tests

* fixing cypress

* fixing cypress test

* responding to comments

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java

@@ -3,10 +3,13 @@
 import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableSet;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.ValidationException;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 
+import com.linkedin.datahub.graphql.generated.OrFilter;
+import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
@@ -81,20 +84,77 @@ public static Map<String, String> buildFacetFilters(@Nullable List<FacetFilterIn
             if (!validFacetFields.contains(facetFilterInput.getField())) {
                 throw new ValidationException(String.format("Unrecognized facet with name %s provided", facetFilterInput.getField()));
             }
-            facetFilters.put(facetFilterInput.getField(), facetFilterInput.getValue());
+            if (!facetFilterInput.getValues().isEmpty()) {
+                facetFilters.put(facetFilterInput.getField(), facetFilterInput.getValues().get(0));
+            }
         });
 
         return facetFilters;
     }
 
+    public static List<Criterion> criterionListFromAndFilter(List<FacetFilterInput> andFilters) {
+        return andFilters != null && !andFilters.isEmpty()
+            ? andFilters.stream()
+            .map(filter -> criterionFromFilter(filter))
+            .collect(Collectors.toList()) : Collections.emptyList();
+
+    }
+
+    // In the case that user sends filters to be or-d together, we need to build a series of conjunctive criterion
+    // arrays, rather than just one for the AND case.
+    public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr(
+        @Nonnull List<OrFilter> orFilters
+    ) {
+        return new ConjunctiveCriterionArray(orFilters.stream().map(orFilter -> {
+                CriterionArray andCriterionForOr = new CriterionArray(criterionListFromAndFilter(orFilter.getAnd()));
+                return new ConjunctiveCriterion().setAnd(
+                    andCriterionForOr
+                );
+            }
+        ).collect(Collectors.toList()));
+    }
+
     @Nullable
-    public static Filter buildFilter(@Nullable List<FacetFilterInput> facetFilterInputs) {
-        if (facetFilterInputs == null || facetFilterInputs.isEmpty()) {
+    public static Filter buildFilter(@Nullable List<FacetFilterInput> andFilters, @Nullable List<OrFilter> orFilters) {
+        if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) {
             return null;
         }
-        return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilterInputs.stream()
-            .map(filter -> new Criterion().setField(getFilterField(filter.getField())).setValue(filter.getValue()))
-            .collect(Collectors.toList())))));
+
+        // Or filters are the new default. We will check them first.
+        // If we have OR filters, we need to build a series of CriterionArrays
+        if (orFilters != null && !orFilters.isEmpty()) {
+            return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters));
+        }
+
+        // If or filters are not set, someone may be using the legacy and filters
+        final List<Criterion> andCriterions = criterionListFromAndFilter(andFilters);
+        return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions))));
+    }
+
+    // Translates a FacetFilterInput (graphql input class) into Criterion (our internal model)
+    public static Criterion criterionFromFilter(final FacetFilterInput filter) {
+        Criterion result = new Criterion();
+        result.setField(getFilterField(filter.getField()));
+        if (filter.getValues() != null) {
+            result.setValues(new StringArray(filter.getValues()));
+            if (!filter.getValues().isEmpty()) {
+                result.setValue(filter.getValues().get(0));
+            } else {
+                result.setValue("");
+            }
+        }
+
+        if (filter.getCondition() != null) {
+            result.setCondition(Condition.valueOf(filter.getCondition().toString()));
+        } else {
+            result.setCondition(Condition.EQUAL);
+        }
+
+        if (filter.getNegated() != null) {
+            result.setNegated(filter.getNegated());
+        }
+
+        return result;
     }
 
     private static String getFilterField(final String originalField) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java

@@ -9,14 +9,12 @@
 import com.linkedin.datahub.graphql.generated.AssertionRunStatus;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.generated.FilterInput;
-import com.linkedin.datahub.graphql.generated.SearchCondition;
 import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.aspect.EnvelopedAspect;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
-import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.r2.RemoteInvocationException;
@@ -102,13 +100,16 @@ private Filter buildFilter(@Nullable FilterInput filtersInput, @Nullable final S
     }
     List<FacetFilterInput> facetFilters = new ArrayList<>();
     if (status != null) {
-      facetFilters.add(new FacetFilterInput("status", status, ImmutableList.of(status), false, SearchCondition.EQUAL));
+      FacetFilterInput filter = new FacetFilterInput();
+      filter.setField("status");
+      filter.setValues(ImmutableList.of(status));
+      facetFilters.add(filter);
     }
     if (filtersInput != null) {
       facetFilters.addAll(filtersInput.getAnd());
     }
     return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilters.stream()
-        .map(filter -> new Criterion().setField(filter.getField()).setValue(filter.getValue()))
+        .map(filter -> criterionFromFilter(filter))
         .collect(Collectors.toList())))));
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.auth;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
 import com.linkedin.datahub.graphql.exception.AuthorizationException;
@@ -55,7 +56,7 @@ public CompletableFuture<ListAccessTokenResult> get(DataFetchingEnvironment envi
               new SortCriterion().setField(EXPIRES_AT_FIELD_NAME).setOrder(SortOrder.DESCENDING);
 
           final SearchResult searchResult = _entityClient.search(Constants.ACCESS_TOKEN_ENTITY_NAME, "",
-              buildFilter(filters), sortCriterion, start, count,
+              buildFilter(filters, Collections.emptyList()), sortCriterion, start, count,
               getAuthentication(environment));
 
           final List<AccessTokenMetadata> tokens = searchResult.getEntities().stream().map(entity -> {
@@ -94,6 +95,6 @@ public CompletableFuture<ListAccessTokenResult> get(DataFetchingEnvironment envi
    */
   private boolean isListingSelfTokens(final List<FacetFilterInput> filters, final QueryContext context) {
     return AuthorizationUtils.canGeneratePersonalAccessToken(context) && filters.stream()
-        .anyMatch(filter -> filter.getField().equals("ownerUrn") && filter.getValue().equals(context.getActorUrn()));
+        .anyMatch(filter -> filter.getField().equals("ownerUrn") && filter.getValues().equals(ImmutableList.of(context.getActorUrn())));
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java

@@ -12,7 +12,6 @@
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
-import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
 import com.linkedin.r2.RemoteInvocationException;
@@ -111,7 +110,7 @@ private Filter buildFilters(@Nullable FilterInput maybeFilters) {
       return null;
     }
     return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(maybeFilters.getAnd().stream()
-        .map(filter -> new Criterion().setField(filter.getField()).setValue(filter.getValue()))
+        .map(filter -> criterionFromFilter(filter))
         .collect(Collectors.toList())))));
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.recommendation;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.datahub.graphql.generated.ContentParams;
 import com.linkedin.datahub.graphql.generated.EntityProfileParams;
@@ -14,7 +15,6 @@
 import com.linkedin.datahub.graphql.generated.SearchParams;
 import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
-import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.recommendation.EntityRequestContext;
 import com.linkedin.metadata.recommendation.RecommendationsService;
@@ -31,7 +31,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
+import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
 
 
 @Slf4j
@@ -88,7 +88,7 @@ private com.linkedin.metadata.recommendation.RecommendationRequestContext mapReq
         searchRequestContext.setFilters(new CriterionArray(requestContext.getSearchRequestContext()
             .getFilters()
             .stream()
-            .map(facetField -> new Criterion().setField(facetField.getField()).setValue(facetField.getValue()))
+            .map(facetField -> criterionFromFilter(facetField))
             .collect(Collectors.toList())));
       }
       mappedRequestContext.setSearchRequestContext(searchRequestContext);
@@ -148,7 +148,8 @@ private RecommendationParams mapRecommendationParams(
         searchParams.setFilters(params.getSearchParams()
             .getFilters()
             .stream()
-            .map(criterion -> Filter.builder().setField(criterion.getField()).setValue(criterion.getValue()).build())
+            .map(criterion -> Filter.builder().setField(criterion.getField()).setValues(
+                ImmutableList.of(criterion.getValue())).build())
             .collect(Collectors.toList()));
       }
       mappedParams.setSearchParams(searchParams);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java

@@ -52,7 +52,7 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
             "Executing search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",
             input.getTypes(), input.getQuery(), input.getFilters(), start, count);
         return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities(entityNames, sanitizedQuery,
-            ResolverUtils.buildFilter(input.getFilters()), start, count, ResolverUtils.getAuthentication(environment)));
+            ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), start, count, ResolverUtils.getAuthentication(environment)));
       } catch (Exception e) {
         log.error(
             "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}",

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java

@@ -72,7 +72,7 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
             urn, resolvedDirection, input.getTypes(), input.getQuery(), filters, start, count);
         return UrnSearchAcrossLineageResultsMapper.map(
             _entityClient.searchAcrossLineage(urn, resolvedDirection, entityNames, sanitizedQuery,
-                maxHops, ResolverUtils.buildFilter(filters), null, start, count,
+                maxHops, ResolverUtils.buildFilter(filters, input.getOrFilters()), null, start, count,
                 ResolverUtils.getAuthentication(environment)));
       } catch (RemoteInvocationException e) {
         log.error(
@@ -89,7 +89,7 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
   private Integer getMaxHops(List<FacetFilterInput> filters) {
     Set<String> degreeFilterValues = filters.stream()
         .filter(filter -> filter.getField().equals("degree"))
-        .map(FacetFilterInput::getValue)
+        .flatMap(filter -> filter.getValues().stream())
         .collect(Collectors.toSet());
     Integer maxHops = null;
     if (!degreeFilterValues.contains("3+")) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java

@@ -41,17 +41,17 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
 
     return CompletableFuture.supplyAsync(() -> {
       try {
-        log.debug("Executing search. entity type {}, query {}, filters: {}, start: {}, count: {}", input.getType(),
-            input.getQuery(), input.getFilters(), start, count);
+        log.debug("Executing search. entity type {}, query {}, filters: {}, orFilters: {}, start: {}, count: {}", input.getType(),
+            input.getQuery(), input.getFilters(), input.getOrFilters(), start, count);
         return UrnSearchResultsMapper.map(
-            _entityClient.search(entityName, sanitizedQuery, ResolverUtils.buildFilter(input.getFilters()), null, start,
+            _entityClient.search(entityName, sanitizedQuery, ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), null, start,
                 count, ResolverUtils.getAuthentication(environment)));
       } catch (Exception e) {
-        log.error("Failed to execute search: entity type {}, query {}, filters: {}, start: {}, count: {}",
-            input.getType(), input.getQuery(), input.getFilters(), start, count);
+        log.error("Failed to execute search: entity type {}, query {}, filters: {}, orFilters: {}, start: {}, count: {}",
+            input.getType(), input.getQuery(), input.getFilters(), input.getOrFilters(), start, count);
         throw new RuntimeException(
-            "Failed to execute search: " + String.format("entity type %s, query %s, filters: %s, start: %s, count: %s",
-                input.getType(), input.getQuery(), input.getFilters(), start, count), e);
+            "Failed to execute search: " + String.format("entity type %s, query %s, filters: %s, orFilters: %s, start: %s, count: %s",
+                input.getType(), input.getQuery(), input.getFilters(), input.getOrFilters(), start, count), e);
       }
     });
   }

datahub-graphql-core/src/main/resources/recommendation.graphql

@@ -217,7 +217,7 @@ type SearchParams {
   """
   Entity types to be searched. If this is not provided, all entities will be searched.
   """
-	types: [EntityType!]
+  types: [EntityType!]
 
   """
   Search query
@@ -237,12 +237,22 @@ type Filter {
   """
   Name of field to filter by
   """
-	field: String!
+  field: String!
 
-	"""
-	Value of the field to filter by
-	"""
-	value: String!
+  """
+  Values, one of which the intended field should match.
+  """
+  values: [String!]!
+
+  """
+  If the filter should or should not be matched
+  """
+  negated: Boolean
+
+  """
+  Condition for the values. How to If unset, assumed to be equality
+  """
+  condition: FilterOperator
 }
 
 """
@@ -269,4 +279,4 @@ type ContentParams {
   Number of entities corresponding to the recommended content
   """
   count: Long
-}
+}