diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java index b2cb1dee5362f..427af12c6c45e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java @@ -69,7 +69,7 @@ public class TimelineServerBasedWriteMarkers extends WriteMarkers { private final int timelineServerPort; private final int timeoutSecs; private static final TypeReference BOOLEAN_TYPE_REFERENCE = new TypeReference() {}; - private static final TypeReference> STRING_TYPE_REFERENCE = new TypeReference>() {}; + private static final TypeReference> SET_TYPE_REFERENCE = new TypeReference>() {}; public TimelineServerBasedWriteMarkers(HoodieTable table, String instantTime) { this(table.getMetaClient().getBasePath(), @@ -115,7 +115,7 @@ public Set createdAndMergedDataPaths(HoodieEngineContext context, int pa Map paramsMap = Collections.singletonMap(MARKER_DIR_PATH_PARAM, markerDirPath.toString()); try { Set markerPaths = executeRequestToTimelineServer( - CREATE_AND_MERGE_MARKERS_URL, paramsMap, STRING_TYPE_REFERENCE, RequestMethod.GET); + CREATE_AND_MERGE_MARKERS_URL, paramsMap, SET_TYPE_REFERENCE, RequestMethod.GET); return markerPaths.stream().map(WriteMarkers::stripMarkerSuffix).collect(Collectors.toSet()); } catch (IOException e) { throw new HoodieRemoteException("Failed to get CREATE and MERGE data file paths in " @@ -128,7 +128,7 @@ public Set allMarkerFilePaths() { Map paramsMap = Collections.singletonMap(MARKER_DIR_PATH_PARAM, markerDirPath.toString()); try { return executeRequestToTimelineServer( - ALL_MARKERS_URL, paramsMap, STRING_TYPE_REFERENCE, RequestMethod.GET); + ALL_MARKERS_URL, paramsMap, SET_TYPE_REFERENCE, RequestMethod.GET); } catch (IOException e) { throw new HoodieRemoteException("Failed to get all markers in " + markerDirPath.toString(), e); } diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 0936e1c6386e4..591b0aa46cf2c 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -132,6 +132,12 @@ jackson-module-afterburner + + + com.fasterxml.jackson.module + jackson-module-afterburner + + org.apache.avro diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java index b225e1b85b0b9..a6318608bcf75 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java @@ -64,6 +64,8 @@ */ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, Serializable { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new AfterburnerModule()); + private static final String BASE_URL = "/v1/hoodie/view"; public static final String LATEST_PARTITION_SLICES_URL = String.format("%s/%s", BASE_URL, "slices/partition/latest/"); public static final String LATEST_PARTITION_SLICE_URL = String.format("%s/%s", BASE_URL, "slices/file/latest/"); @@ -113,7 +115,6 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, public static final String PENDING_CLUSTERING_FILEGROUPS = String.format("%s/%s", BASE_URL, "clustering/pending/"); - public static final String LAST_INSTANT = String.format("%s/%s", BASE_URL, "timeline/instant/last"); public static final String LAST_INSTANTS = String.format("%s/%s", BASE_URL, "timeline/instants/last"); @@ -147,7 +148,6 @@ public class RemoteHoodieTableFileSystemView implements SyncableFileSystemView, private static final TypeReference> BASE_FILE_DTOS_REFERENCE = new TypeReference>() {}; private static final TypeReference>> BASE_FILE_MAP_REFERENCE = new TypeReference>>() {}; private static final TypeReference>> FILE_SLICE_MAP_REFERENCE = new TypeReference>>() {}; - private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new AfterburnerModule()); private final String serverHost; private final int serverPort; @@ -202,7 +202,7 @@ private T executeRequest(String requestPath, Map queryParame LOG.info("Sending request : (" + url + ")"); Response response = retryHelper != null ? retryHelper.start(() -> get(timeoutMs, url, method)) : get(timeoutMs, url, method); String content = response.returnContent().asString(Consts.UTF_8); - return MAPPER.readValue(content, reference); + return (T) OBJECT_MAPPER.readValue(content, reference); } private Map getParamsWithPartitionPath(String partitionPath) { @@ -363,7 +363,8 @@ public Stream getLatestFileSlicesBeforeOrOn(String partitionPath, Str new String[] {MAX_INSTANT_PARAM, INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM}, new String[] {maxCommitTime, String.valueOf(includeFileSlicesInPendingCompaction)}); try { - List dataFiles = executeRequest(LATEST_SLICES_BEFORE_ON_INSTANT_URL, paramsMap, FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET); + List dataFiles = executeRequest(LATEST_SLICES_BEFORE_ON_INSTANT_URL, paramsMap, + FILE_SLICE_DTOS_REFERENCE, RequestMethod.GET); return dataFiles.stream().map(FileSliceDTO::toFileSlice); } catch (IOException e) { throw new HoodieRemoteException(e); diff --git a/pom.xml b/pom.xml index 7ab571678c7c3..02bb38c05487f 100644 --- a/pom.xml +++ b/pom.xml @@ -566,6 +566,11 @@ org.apache.hudi.org.apache.hadoop.metrics2.util.MetricSampleQuantiles + + com.fasterxml.jackson.module + org.apache.hudi.com.fasterxml.jackson.module + + @@ -879,6 +884,12 @@ ${fasterxml.version} + + com.fasterxml.jackson.module + jackson-module-afterburner + ${fasterxml.jackson.databind.version} + + org.glassfish.jersey.core @@ -2180,7 +2191,7 @@ 1.8.2 4.7 2.6.7 - 2.6.7.3 + ${fasterxml.version} 2.6.7.1 2.7.4 true @@ -2212,7 +2223,7 @@ 1.8.2 4.7 2.6.7 - 2.6.7.3 + ${fasterxml.version} 2.6.7.1 2.7.4 true