[core] Populate more metadata to object table (#4987)

apache · Jan 23, 2025 · cd32438 · cd32438
1 parent bfd0a0a
commit cd32438
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 7 deletions.
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java
@@ -22,6 +22,8 @@
 
 import javax.annotation.Nullable;
 
+import java.util.Map;
+
 /**
  * Interface that represents the client side information for a file independent of the file system.
  *
@@ -78,4 +80,65 @@ default long getAccessTime() {
     default String getOwner() {
         return null;
     }
+
+    /**
+     * Returns the generation of this file.
+     *
+     * @return the generation of this file
+     */
+    @Nullable
+    default Integer getGeneration() {
+        return null;
+    }
+
+    /**
+     * Returns the content type of this file.
+     *
+     * @return the content type of this file
+     */
+    @Nullable
+    default String getContentType() {
+        return null;
+    }
+
+    /**
+     * Returns the storage class of this file.
+     *
+     * @return the storage class of this file
+     */
+    @Nullable
+    default String getStorageClass() {
+        return null;
+    }
+
+    /**
+     * Returns the MD5 hash of this file.
+     *
+     * @return the MD5 hash of this file
+     */
+    @Nullable
+    default String getMd5Hash() {
+        return null;
+    }
+
+    /**
+     * Returns the last modification time of the file's metadata.
+     *
+     * @return A long value representing the time of the file's metadata was last modified, measured
+     *     in milliseconds since the epoch (UTC January 1, 1970).
+     */
+    @Nullable
+    default Long getMetadataModificationTime() {
+        return null;
+    }
+
+    /**
+     * Returns the metadata key-value pairs of the file.
+     *
+     * @return the metadata key-value pairs of the file
+     */
+    @Nullable
+    default Map<String, String> getMetadata() {
+        return null;
+    }
 }
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectRefresh.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectRefresh.java
@@ -30,7 +30,7 @@
 import org.apache.paimon.table.sink.BatchTableWrite;
 import org.apache.paimon.table.sink.BatchWriteBuilder;
 
-import java.util.Collections;
+import java.util.Optional;
 
 /** Util class for refreshing object table. */
 public class ObjectRefresh {
@@ -68,12 +68,14 @@ private static InternalRow toRow(FileStatus file) {
                 Timestamp.fromEpochMillis(file.getModificationTime()),
                 Timestamp.fromEpochMillis(file.getAccessTime()),
                 file.getOwner(),
-                null,
-                null,
-                null,
-                null,
-                null,
-                new GenericMap(Collections.emptyMap()));
+                file.getGeneration(),
+                file.getContentType(),
+                file.getStorageClass(),
+                file.getMd5Hash(),
+                Optional.ofNullable(file.getMetadataModificationTime())
+                        .map(Timestamp::fromEpochMillis)
+                        .orElse(null),
+                Optional.ofNullable(file.getMetadata()).map(GenericMap::new).orElse(null));
     }
 
     public static GenericRow toRow(Object... values) {