From d04a2b90494d74e169e0950a8e32a84f56eba56a Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 15 Sep 2026 09:59:33 +0200 Subject: [PATCH 1/2] HIVE-30035: Upgrade Parquet to 1.18.0 iceberg-data brings parquet-avro, and iceberg-shading bundles org.apache.parquet into the shaded Iceberg jar without relocating it. Left alone that puts Parquet 1.17.1 classes on the classpath beside Hive's 1.18.0. Pin parquet-avro, the root of that subtree, in the Iceberg pom where the bundle is built, rather than managing Parquet artifacts for the whole build. The version comes from iceberg.parquet.version, alongside the existing iceberg.avro.version. It defaults to parquet.version and has to stay equal to it, since the embedded classes keep their original package names. Update TestHiveSchemaConverter for Parquet 1.18: ConversionPatterns.mapType no longer stamps the deprecated MAP_KEY_VALUE annotation on the inner key_value group of a MAP (the Parquet MAP standard treats it as redundant), so drop it from the expected schema strings and expect a null annotation in testMapOriginalType. --- iceberg/pom.xml | 8 ++++++++ pom.xml | 2 +- .../ql/io/parquet/TestHiveSchemaConverter.java | 17 ++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/iceberg/pom.xml b/iceberg/pom.xml index f337e0d5dcc6..c098727e04e1 100644 --- a/iceberg/pom.xml +++ b/iceberg/pom.xml @@ -34,6 +34,8 @@ 4.0.3 5.2.0 1.12.0 + + ${parquet.version} 2.44.4 2.29.2 3.27.3 @@ -94,6 +96,12 @@ iceberg-parquet ${iceberg.version} + + + org.apache.parquet + parquet-avro + ${iceberg.parquet.version} + org.apache.iceberg iceberg-orc diff --git a/pom.xml b/pom.xml index 2313c30e5e27..4f7bdf16d031 100644 --- a/pom.xml +++ b/pom.xml @@ -203,7 +203,7 @@ 4.1.137.Final 4.5.8 2.8 - 1.16.0 + 1.18.0 0.16.0 1.5.6 3.25.5 diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java index 29395c0bc7a9..1f2e34b17166 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestHiveSchemaConverter.java @@ -334,7 +334,7 @@ public void testMap() throws Exception { "map", "message hive_schema {\n" + " optional group mapCol (MAP) {\n" - + " repeated group key_value (MAP_KEY_VALUE) {\n" + + " repeated group key_value {\n" + " required binary key (UTF8);\n" + " optional binary value (UTF8);\n" + " }\n" @@ -348,7 +348,7 @@ public void testMapDecimal() throws Exception { "map", "message hive_schema {\n" + " optional group mapCol (MAP) {\n" - + " repeated group key_value (MAP_KEY_VALUE) {\n" + + " repeated group key_value {\n" + " required binary key (UTF8);\n" + " optional fixed_len_byte_array(3) value (DECIMAL(5,2));\n" + " }\n" @@ -362,7 +362,7 @@ public void testMapInts() throws Exception { "map", "message hive_schema {\n" + " optional group mapCol (MAP) {\n" - + " repeated group key_value (MAP_KEY_VALUE) {\n" + + " repeated group key_value {\n" + " required int32 key (INT_16);\n" + " optional int32 value (INT_8);\n" + " }\n" @@ -376,7 +376,7 @@ public void testMapStruct() throws Exception { "map>", "message hive_schema {\n" + " optional group mapCol (MAP) {\n" - + " repeated group key_value (MAP_KEY_VALUE) {\n" + + " repeated group key_value {\n" + " required binary key (UTF8);\n" + " optional group value {\n" + " optional int32 a (INT_16);\n" @@ -393,7 +393,7 @@ public void testMapList() throws Exception { "map>", "message hive_schema {\n" + " optional group mapCol (MAP) {\n" - + " repeated group key_value (MAP_KEY_VALUE) {\n" + + " repeated group key_value {\n" + " required binary key (UTF8);\n" + " optional group value (LIST) {\n" + " repeated group bag {\n" @@ -434,10 +434,9 @@ public void testMapOriginalType() throws Exception { assertEquals(1, topLevel.asGroupType().getFieldCount()); Type secondLevel = topLevel.asGroupType().getFields().get(0); - // there is one repeated field for mapCol, the field name is "map" and its original Type is - // MAP_KEY_VALUE; - checkField(secondLevel, "key_value", Repetition.REPEATED, - LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance()); + // the repeated key_value group carries no annotation: Parquet's MAP standard treats + // MAP_KEY_VALUE as redundant, and ConversionPatterns.mapType no longer stamps it. + checkField(secondLevel, "key_value", Repetition.REPEATED, null); } @Test From 829b6c1bbda39844f5c979b4504225050b65526b Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 16 Sep 2026 08:50:48 +0200 Subject: [PATCH 2/2] pin iceberg.avro.version to avro version --- iceberg/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iceberg/pom.xml b/iceberg/pom.xml index c098727e04e1..7fcb6387e4f8 100644 --- a/iceberg/pom.xml +++ b/iceberg/pom.xml @@ -33,7 +33,7 @@ . 4.0.3 5.2.0 - 1.12.0 + ${avro.version} ${parquet.version} 2.44.4