[CELEBORN-1482] Add partition meta handler and new partition type

apache · Jan 8, 2025 · bba23b7 · bba23b7
1 parent 4714e91
commit bba23b7
Show file tree

Hide file tree

Showing 5 changed files with 438 additions and 4 deletions.
diff --git a/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java b/common/src/main/java/org/apache/celeborn/common/protocol/PartitionType.java
@@ -20,7 +20,8 @@
 public enum PartitionType {
   REDUCE(0),
   MAP(1),
-  MAPGROUP(2);
+  MAPGROUP(2),
+  MAP_SEGMENT(3);
 
   private final int value;
 

diff --git a/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala b/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
@@ -112,7 +112,7 @@ object PbSerDeUtils {
         // writer always closed as this is committed file info.
         fileMeta.setIsWriterClosed(true)
         fileMeta
-      case PartitionType.MAPGROUP =>
+      case PartitionType.MAPGROUP | PartitionType.MAP_SEGMENT =>
         throw new NotImplementedError("Map group is not implemented")
     }
     new DiskFileInfo(

diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -1188,6 +1188,7 @@ object Utils extends Logging {
       case 0 => PartitionType.REDUCE
       case 1 => PartitionType.MAP
       case 2 => PartitionType.MAPGROUP
+      case 3 => PartitionType.MAP_SEGMENT
       case _ =>
         logWarning(s"invalid partitionType $value, fallback to ReducePartition")
         PartitionType.REDUCE