From f95ef894ad2995aa4cec0c393e00ac72d8f9f914 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 28 Sep 2022 09:40:19 -0700 Subject: [PATCH] fix(): Attempting to test paypal optimizations (#6073) --- .../metadata/client/JavaEntityClient.java | 2 +- .../client/CachingEntitySearchService.java | 79 +++++++++++++++++++ .../elasticsearch/query/ESBrowseDAO.java | 8 +- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index 8af042320c02af..ae7740021a1504 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -184,7 +184,7 @@ public BrowseResult browse( int limit, @Nonnull final Authentication authentication) throws RemoteInvocationException { return ValidationUtils.validateBrowseResult( - _entitySearchService.browse(entityType, path, newFilter(requestFilters), start, limit), _entityService); + _cachingEntitySearchService.browse(entityType, path, newFilter(requestFilters), start, limit, null), _entityService); } @SneakyThrows diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java index a2d2b8857ae489..90eef80e24bde4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/client/CachingEntitySearchService.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.search.client; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; @@ -19,6 +20,7 @@ public class CachingEntitySearchService { private static final String ENTITY_SEARCH_SERVICE_SEARCH_CACHE_NAME = "entitySearchServiceSearch"; private static final String ENTITY_SEARCH_SERVICE_AUTOCOMPLETE_CACHE_NAME = "entitySearchServiceAutoComplete"; + private static final String ENTITY_SEARCH_SERVICE_BROWSE_CACHE_NAME = "entitySearchServiceBrowse"; private final CacheManager cacheManager; private final EntitySearchService entitySearchService; // This is a shared component, also used in search aggregation @@ -71,6 +73,29 @@ public AutoCompleteResult autoComplete( return getCachedAutoCompleteResults(entityName, input, field, filters, limit, flags); } + /** + * Retrieves cached auto complete results + * + * @param entityName type of entity to query + * @param path the path to be browsed + * @param filters the request map with fields and values as filters + * @param from index of the first entity located in path + * @param size the max number of entities contained in the response + * + * @return a {@link SearchResult} containing the requested batch of search results + */ + public BrowseResult browse( + @Nonnull String entityName, + @Nonnull String path, + @Nullable Filter filters, + int from, + int size, + @Nullable SearchFlags flags) { + return getCachedBrowseResults(entityName, path, filters, from, size, flags); + } + + + /** * Get search results corresponding to the input "from" and "size" * It goes through batches, starting from the beginning, until we get enough results to return @@ -129,6 +154,43 @@ public AutoCompleteResult getCachedAutoCompleteResults( return result; } + /** + * Returns cached browse results. + */ + public BrowseResult getCachedBrowseResults( + @Nonnull String entityName, + @Nonnull String path, + @Nullable Filter filters, + int from, + int size, + @Nullable SearchFlags flags) { + Cache cache = cacheManager.getCache(ENTITY_SEARCH_SERVICE_BROWSE_CACHE_NAME); + BrowseResult result; + if (enableCache(flags)) { + Object cacheKey = Quintet.with(entityName, path, filters, from, size); + result = cache.get(cacheKey, BrowseResult.class); + if (result == null) { + result = getRawBrowseResults( + entityName, + path, + filters, + from, + size + ); + cache.put(cacheKey, result); + } + } else { + result = getRawBrowseResults( + entityName, + path, + filters, + from, + size + ); + } + return result; + } + /** * Executes the expensive search query using the {@link EntitySearchService} */ @@ -165,6 +227,23 @@ private AutoCompleteResult getRawAutoCompleteResults( limit); } + /** + * Executes the expensive autocomplete query using the {@link EntitySearchService} + */ + private BrowseResult getRawBrowseResults( + final String entityName, + final String input, + final Filter filters, + final int start, + final int count) { + return entitySearchService.browse( + entityName, + input, + filters, + start, + count); + } + /** * Returns true if the cache should be used or skipped when fetching search results */ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index fcbe62a801c712..122b7450389f9d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -59,7 +59,6 @@ public class ESBrowseDAO { private static final String REMOVED = "removed"; private static final String GROUP_AGG = "groups"; - private static final String ALL_PATHS = "allPaths"; // Set explicit max size for grouping private static final int AGGREGATION_MAX_SIZE = 2000; @@ -137,16 +136,11 @@ public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nu private AggregationBuilder buildAggregations(@Nonnull String path) { final String currentLevel = ESUtils.escapeReservedCharacters(path) + "/.*"; final String nextLevel = ESUtils.escapeReservedCharacters(path) + "/.*/.*"; - final String nextNextLevel = ESUtils.escapeReservedCharacters(path) + "/.*/.*/.*"; return AggregationBuilders.terms(GROUP_AGG) .field(BROWSE_PATH) .size(AGGREGATION_MAX_SIZE) - .includeExclude(new IncludeExclude(currentLevel, nextLevel)) - .subAggregation(AggregationBuilders.terms(ALL_PATHS) - .field(BROWSE_PATH) - .size(AGGREGATION_MAX_SIZE) - .includeExclude(new IncludeExclude(nextLevel, nextNextLevel))); + .includeExclude(new IncludeExclude(currentLevel, nextLevel)); } /**