Skip to content

Commit

Permalink
Remove unspecified bloom filter when setting properties in Iceberg
Browse files Browse the repository at this point in the history
  • Loading branch information
ebyhr committed Dec 25, 2024
1 parent 4f2e41d commit abde2be
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2285,14 +2285,14 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta
List<String> parquetBloomFilterColumns = (List<String>) properties.get(PARQUET_BLOOM_FILTER_COLUMNS_PROPERTY)
.orElseThrow(() -> new IllegalArgumentException("The parquet_bloom_filter_columns property cannot be empty"));
validateParquetBloomFilterColumns(getColumnMetadatas(SchemaParser.fromJson(table.getTableSchemaJson()), typeManager), parquetBloomFilterColumns);
if (parquetBloomFilterColumns.isEmpty()) {
icebergTable.properties().keySet().stream()
.filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX))
.forEach(updateProperties::remove);
}
else {
parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true"));
}

Set<String> existingParquetBloomFilterColumns = icebergTable.properties().keySet().stream()
.filter(key -> key.startsWith(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX))
.map(key -> key.substring(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX.length()))
.collect(toImmutableSet());
Set<String> removeParquetBloomFilterColumns = Sets.difference(existingParquetBloomFilterColumns, Set.copyOf(parquetBloomFilterColumns));
removeParquetBloomFilterColumns.forEach(column -> updateProperties.remove(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column));
parquetBloomFilterColumns.forEach(column -> updateProperties.set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + column, "true"));
}

if (properties.containsKey(FILE_FORMAT_PROPERTY)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ void testBloomFilterPropertiesArePersistedDuringSetProperties()
assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['a','B']");
verifyTableProperties(tableName);

assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY['a']");
assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName))
.contains("parquet_bloom_filter_columns = ARRAY['a']");

assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES parquet_bloom_filter_columns = ARRAY[]");
assertThat((String) computeScalar("SHOW CREATE TABLE " + tableName))
.doesNotContain("parquet_bloom_filter_columns");
Expand Down

0 comments on commit abde2be

Please sign in to comment.