diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java index 3ed5a56e52f..c84410e45a3 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java @@ -23,7 +23,6 @@ import org.apache.spark.sql.types.FloatType; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.types.LongType; -import org.apache.spark.sql.types.MapType; import org.apache.spark.sql.types.ShortType; import org.apache.spark.sql.types.StringType; import org.apache.spark.sql.types.StructField; @@ -125,9 +124,6 @@ private static ArrowType convertType(DataType sparkType) { return new ArrowType.List(); } else if (sparkType instanceof StructType) { return new ArrowType.Struct(); - } else if (sparkType instanceof MapType) { - // Map is represented as List> in Arrow - return new ArrowType.List(); } else { throw new UnsupportedOperationException("Unsupported Spark type for Arrow conversion: " + sparkType.getClass().getName()); diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java index cd9b3ca184c..2df9c5c4a50 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java @@ -173,4 +173,14 @@ void unsupportedTypeIsRejected() { assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema)); assertTrue(e.getMessage().contains("CalendarIntervalType")); } + + @Test + @DisplayName("MapType is rejected rather than converted to a childless List") + void mapTypeIsRejected() { + StructType schema = + new StructType().add("m", DataTypes.createMapType(DataTypes.StringType, DataTypes.IntegerType)); + UnsupportedOperationException e = + assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema)); + assertTrue(e.getMessage().contains("MapType")); + } }