Skip to content

Commit

Permalink
Check if we know the mapping of the internal row to a partition path
Browse files Browse the repository at this point in the history
  • Loading branch information
Steve Vaughan Jr committed Mar 15, 2024
1 parent 161c982 commit bfd1d14
Showing 1 changed file with 10 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -262,13 +262,16 @@ class BasicWriteJobStatsTracker(
totalNumOutput += summary.numRows

summary.partitionsStats.foreach(s => {
val path = partitionsMap.getOrElse(s._1, "")
val current = partitionMetrics(path)
driverSidePartitionMetrics(path) = BasicWritePartitionTaskStats(
current.numFiles + s._2.numFiles,
current.numBytes + s._2.numBytes,
current.numRows + s._2.numRows
)
// Check if we know the mapping of the internal row to a partition path
if (partitionsMap.contains(s._1)) {
val path = partitionsMap(s._1)
val current = partitionMetrics(path)
driverSidePartitionMetrics(path) = BasicWritePartitionTaskStats(
current.numFiles + s._2.numFiles,
current.numBytes + s._2.numBytes,
current.numRows + s._2.numRows
)
}
})
}

Expand Down

0 comments on commit bfd1d14

Please sign in to comment.