diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index 0c29edea364f..4fcb110db87c 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -189,7 +189,7 @@ public String partitionToPath(StructLike data) { if (i > 0) { sb.append("/"); } - sb.append(field.name()).append("=").append(escape(valueString)); + sb.append(escape(field.name())).append("=").append(escape(valueString)); } return sb.toString(); } diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java index 2fda247a33c8..5455415da015 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java @@ -32,7 +32,8 @@ public class TestPartitionPaths { new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), Types.NestedField.optional(2, "data", Types.StringType.get()), - Types.NestedField.optional(3, "ts", Types.TimestampType.withoutZone())); + Types.NestedField.optional(3, "ts", Types.TimestampType.withoutZone()), + Types.NestedField.optional(4, "\"esc\"#1", Types.StringType.get())); @Test public void testPartitionPath() { @@ -62,4 +63,13 @@ public void testEscapedStrings() { .as("Should escape / as %2F") .isEqualTo("data=a%2Fb%2Fc%2Fd/data_trunc=a%2Fb%2Fc%2Fd"); } + + @Test + public void testEscapedFieldNames() { + PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("\"esc\"#1").build(); + + assertThat(spec.partitionToPath(Row.of("a/b/c/d"))) + .as("Should escape \" as %22 and # as %23") + .isEqualTo("%22esc%22%231=a%2Fb%2Fc%2Fd"); + } } diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java index b763c9acebd7..89145b2465e5 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import java.io.IOException; @@ -57,7 +57,7 @@ static ObjectMapper mapper() { // even though using new PropertyNamingStrategy.KebabCaseStrategy() is deprecated // and PropertyNamingStrategies.KebabCaseStrategy.INSTANCE (introduced in jackson 2.14) is // recommended, we can't use it because Spark still relies on jackson 2.13.x stuff - MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategy.KebabCaseStrategy()); + MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); MAPPER.registerModule(initModule()); isInitialized = true; } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java index f9572b227d10..36ed10916ab1 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTObjectMapper.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; class RESTObjectMapper { private static final JsonFactory FACTORY = new JsonFactory(); @@ -38,7 +38,7 @@ static ObjectMapper mapper() { if (!isInitialized) { MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategy.KebabCaseStrategy()); + MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); RESTSerializers.registerAll(MAPPER); isInitialized = true; } diff --git a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java index 1b9c6581200a..93dd1a8555ce 100644 --- a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java +++ b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.base.Splitter; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -285,4 +286,22 @@ public void testObjectStorageWithinTableLocation() { assertThat(parts).element(2).asString().isNotEmpty(); assertThat(parts).element(3).asString().isEqualTo("test.parquet"); } + + @TestTemplate + public void testEncodedFieldNameInPartitionPath() { + // Update the table to use a string field for partitioning with special characters in the name + table.updateProperties().set(TableProperties.OBJECT_STORE_ENABLED, "true").commit(); + table.updateSchema().addColumn("data#1", Types.StringType.get()).commit(); + table.updateSpec().addField("data#1").commit(); + + // Use a partition value that has a special character + StructLike partitionData = TestHelpers.CustomRow.of(0, "val#1"); + + String fileLocation = + table.locationProvider().newDataLocation(table.spec(), partitionData, "test.parquet"); + List parts = Splitter.on("/").splitToList(fileLocation); + String partitionString = parts.get(parts.size() - 2); + + assertThat(partitionString).isEqualTo("data%231=val%231"); + } } diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 9037cf174564..10479067770f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -27,6 +27,7 @@ nav: - configuration.md - evolution.md - maintenance.md + - metrics-reporting.md - partitioning.md - performance.md - reliability.md diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 68d94aaf3b42..aaac1258ea22 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,12 +23,12 @@ activation = "1.1.1" aliyun-sdk-oss = "3.10.2" antlr = "4.9.3" -aircompressor = "0.26" +aircompressor = "0.27" arrow = "15.0.2" avro = "1.11.3" assertj-core = "3.25.3" awaitility = "4.2.1" -awssdk-bom = "2.25.57" +awssdk-bom = "2.25.60" azuresdk-bom = "1.2.23" awssdk-s3accessgrants = "2.0.0" caffeine = "2.9.3" @@ -65,9 +65,9 @@ kryo-shaded = "4.0.3" microprofile-openapi-api = "3.1.1" mockito = "4.11.0" mockserver = "5.15.0" -nessie = "0.82.0" -netty-buffer = "4.1.109.Final" -netty-buffer-compat = "4.1.109.Final" +nessie = "0.83.2" +netty-buffer = "4.1.110.Final" +netty-buffer-compat = "4.1.110.Final" object-client-bundle = "3.3.2" orc = "1.9.3" parquet = "1.13.1" @@ -81,7 +81,7 @@ spark-hive33 = "3.3.4" spark-hive34 = "3.4.3" spark-hive35 = "3.5.1" spring-boot = "2.7.18" -spring-web = "5.3.35" +spring-web = "5.3.36" sqlite-jdbc = "3.45.3.0" testcontainers = "1.19.8" tez010 = "0.10.3" diff --git a/site/link-checker-config.json b/site/link-checker-config.json index 54ff94c0145c..8746089a91c3 100644 --- a/site/link-checker-config.json +++ b/site/link-checker-config.json @@ -8,9 +8,6 @@ }, { "pattern": "^../../javadoc" - }, - { - "pattern": "^https://search.maven.org/" } ], "replacementPatterns": [ @@ -22,5 +19,6 @@ "pattern": "^../../", "replacement": "{{BASEURL}}/site/docs/" } - ] -} \ No newline at end of file + ], + "aliveStatusCodes": [0, 200] +}