From 8d8001ef542c4263c475e67223697d69a4308422 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 19:50:18 +0800 Subject: [PATCH 1/6] [core] Support geospatial data types --- docs/docs/iceberg/index.md | 7 + docs/docs/pypaimon/python-api.mdx | 7 + .../apache/paimon/types/DataTypeCasts.java | 10 + .../paimon/types/DataTypeDefaultVisitor.java | 10 + .../paimon/types/DataTypeJsonParser.java | 41 ++++ .../org/apache/paimon/types/DataTypeRoot.java | 4 + .../apache/paimon/types/DataTypeVisitor.java | 8 + .../org/apache/paimon/types/DataTypes.java | 20 ++ .../apache/paimon/types/EdgeAlgorithm.java | 51 ++++ .../apache/paimon/types/GeographyType.java | 104 +++++++++ .../org/apache/paimon/types/GeometryType.java | 121 ++++++++++ .../paimon/types/GeospatialTypeTest.java | 78 +++++++ .../arrow/ArrowFieldTypeConversion.java | 24 ++ .../org/apache/paimon/arrow/ArrowUtils.java | 51 ++-- .../Arrow2PaimonVectorConverter.java | 12 + .../ArrowFieldWriterFactoryVisitor.java | 12 + .../apache/paimon/arrow/ArrowUtilsTest.java | 18 ++ .../org/apache/paimon/data/BinaryArray.java | 2 + .../apache/paimon/data/BinaryArrayWriter.java | 2 + .../org/apache/paimon/data/BinaryWriter.java | 4 + .../org/apache/paimon/data/InternalArray.java | 2 + .../org/apache/paimon/data/InternalRow.java | 4 + .../data/columnar/ColumnVectorUtils.java | 2 + .../data/columnar/RowToColumnConverter.java | 12 + .../data/serializer/InternalSerializers.java | 2 + .../serializer/RowCompactedSerializer.java | 4 + .../paimon/format/SimpleStatsCollector.java | 13 ++ .../types/InternalRowToSizeVisitor.java | 14 ++ .../apache/paimon/utils/InternalRowUtils.java | 2 + .../apache/paimon/utils/TypeCheckUtils.java | 14 +- .../paimon/utils/VectorMappingUtils.java | 12 + .../format/GeospatialStatsCollectorTest.java | 47 ++++ .../iceberg/manifest/IcebergDataFileMeta.java | 4 +- .../iceberg/metadata/IcebergDataField.java | 50 ++++ .../paimon/schema/SchemaValidation.java | 70 +++++- .../manifest/IcebergDataFileMetaTest.java | 47 ++++ .../metadata/IcebergDataFieldTest.java | 37 +++ .../paimon/schema/DataTypeJsonParserTest.java | 15 +- .../paimon/schema/SchemaValidationTest.java | 134 +++++++++++ .../paimon/table/GeospatialTypeTableTest.java | 114 +++++++++ .../paimon/flink/DataTypeToLogicalType.java | 20 ++ .../flink/GeospatialTypeTableITCase.java | 90 ++++++++ .../paimon/format/ArrowSchemaMetadata.java | 19 +- .../parquet/ParquetSchemaConverter.java | 34 +++ .../parquet/ParquetSimpleStatsExtractor.java | 4 + .../reader/ParquetVectorUpdaterFactory.java | 12 + .../parquet/writer/ParquetRowDataWriter.java | 2 + .../format/FormatMetadataUtilsTest.java | 20 ++ .../parquet/ParquetFormatReadWriteTest.java | 37 +++ .../parquet/ParquetSchemaConverterTest.java | 36 +++ .../pypaimon/casting/data_type_casts.py | 14 +- paimon-python/pypaimon/schema/data_types.py | 217 +++++++++++++++--- .../pypaimon/schema/schema_manager.py | 94 +++++++- .../pypaimon/tests/geospatial_type_test.py | 126 ++++++++++ .../pypaimon/write/writer/data_writer.py | 31 ++- .../spark/sql/paimon/shims/Spark4Shim.scala | 40 ++++ .../paimon/spark/GeospatialTypeTest.java | 102 ++++++++ .../spark/sql/GeospatialTypeSQLTest.scala | 75 ++++++ .../spark/AbstractSparkInternalRow.java | 16 ++ .../apache/paimon/spark/DataConverter.java | 13 ++ .../paimon/spark/SparkInternalRowWrapper.java | 13 ++ .../org/apache/paimon/spark/SparkRow.java | 14 ++ .../apache/paimon/spark/SparkTypeUtils.java | 21 ++ .../spark/sql/paimon/shims/SparkShim.scala | 31 +++ .../spark/sql/paimon/shims/Spark3Shim.scala | 40 ++++ .../paimon/spark/data/Spark4ArrayData.scala | 16 +- .../paimon/spark/data/Spark4InternalRow.scala | 21 +- .../spark/sql/paimon/shims/Spark4Shim.scala | 72 +++++- 68 files changed, 2339 insertions(+), 76 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/types/EdgeAlgorithm.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/types/GeographyType.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/types/GeometryType.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/types/GeospatialTypeTest.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/format/GeospatialStatsCollectorTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/table/GeospatialTypeTableTest.java create mode 100644 paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java create mode 100644 paimon-python/pypaimon/tests/geospatial_type_test.py create mode 100644 paimon-spark/paimon-spark-4.1/src/test/java/org/apache/paimon/spark/GeospatialTypeTest.java create mode 100644 paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala diff --git a/docs/docs/iceberg/index.md b/docs/docs/iceberg/index.md index 6c4b30751773..c494833adc31 100644 --- a/docs/docs/iceberg/index.md +++ b/docs/docs/iceberg/index.md @@ -98,6 +98,8 @@ Paimon Iceberg compatibility currently supports the following data types. | `TIMESTAMP_LTZ` (precision 3-6) | `timestamptz` | | `TIMESTAMP` (precision 7-9) | `timestamp_ns` | | `TIMESTAMP_LTZ` (precision 7-9) | `timestamptz_ns` | +| `GEOMETRY(crs)` | `geometry(crs)` | +| `GEOGRAPHY(crs, algorithm)` | `geography(crs, algorithm)` | | `ARRAY` | `list` | | `MAP` | `map` | | `ROW` | `struct` | @@ -108,4 +110,9 @@ Paimon Iceberg compatibility currently supports the following data types. - `TIMESTAMP` and `TIMESTAMP_LTZ` types with precision from 3 to 6 are mapped to standard Iceberg timestamp types - `TIMESTAMP` and `TIMESTAMP_LTZ` types with precision from 7 to 9 use nanosecond precision and require Iceberg v3 format +**Note on Geospatial Types:** +- `GEOMETRY` and `GEOGRAPHY` values use OGC Well-Known Binary (WKB). The default CRS is `OGC:CRS84`, and the default geography edge algorithm is `spherical`. +- Geospatial columns require Parquet for data, per-level, and changelog files. When Iceberg metadata is enabled, set `metadata.iceberg.format-version` to `3`. +- Geospatial columns cannot be primary, partition, bucket, or sequence keys. Paimon records null counts but does not publish byte-wise lower or upper bounds for WKB values. + ::: diff --git a/docs/docs/pypaimon/python-api.mdx b/docs/docs/pypaimon/python-api.mdx index 23a27a1e130a..77333280d786 100644 --- a/docs/docs/pypaimon/python-api.mdx +++ b/docs/docs/pypaimon/python-api.mdx @@ -960,6 +960,13 @@ Row kind values: | `datetime.datetime` | `pyarrow.timestamp(unit, tz='UTC')` | `TIMESTAMP_LTZ(p)` — same unit/p mapping as above | | `datetime.date` | `pyarrow.date32()` | `DATE` | | `datetime.time` | `pyarrow.time32('ms')` | `TIME(p)` | +| WKB `bytes` | `pyarrow.binary()` | `GEOMETRY(crs)`, `GEOGRAPHY(crs, algorithm)` | + +For geospatial fields, PyPaimon stores OGC Well-Known Binary (WKB) in Arrow binary arrays and preserves the logical +type in the field's `paimon.type` metadata. This metadata is also retained for fields nested in arrays, maps, and rows. +Use `GeometryType` or `GeographyType` in the Paimon table schema; an unannotated Arrow binary field is inferred as +`BYTES`. Geospatial tables require Parquet, and non-spherical geography algorithms may not be supported by every +query engine. ### Complex Types diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java index a49e7e4ff6f6..e92c35ec4585 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java @@ -211,6 +211,11 @@ public static boolean supportsCompatibleCast(DataType sourceType, DataType targe return true; } + if (sourceType.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY) + || targetType.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY)) { + return false; + } + return compatibleCastingRules .get(targetType.getTypeRoot()) .contains(sourceType.getTypeRoot()); @@ -230,6 +235,11 @@ private static boolean supportsCasting( return true; } + if (sourceType.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY) + || targetType.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY)) { + return false; + } + final DataTypeRoot sourceRoot = sourceType.getTypeRoot(); final DataTypeRoot targetRoot = targetType.getTypeRoot(); diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java index af680ede62e2..bdd31d194752 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeDefaultVisitor.java @@ -119,6 +119,16 @@ public R visit(BlobType blobType) { return defaultMethod(blobType); } + @Override + public R visit(GeometryType geometryType) { + return defaultMethod(geometryType); + } + + @Override + public R visit(GeographyType geographyType) { + return defaultMethod(geographyType); + } + @Override public R visit(ArrayType arrayType) { return defaultMethod(arrayType); diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java index 4079dd8c47c0..f808114666d5 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java @@ -331,6 +331,8 @@ private enum Keyword { LEGACY, VARIANT, BLOB, + GEOMETRY, + GEOGRAPHY, NOT } @@ -549,6 +551,10 @@ private DataType parseTypeByKeyword() { return new VariantType(); case BLOB: return new BlobType(); + case GEOMETRY: + return parseGeometryType(); + case GEOGRAPHY: + return parseGeographyType(); case VECTOR: return parseVectorType(); default: @@ -683,5 +689,40 @@ private DataType parseVectorType() { nextToken(TokenType.END_SUBTYPE); return DataTypes.VECTOR(length, elementType); } + + private DataType parseGeometryType() { + if (!hasNextToken(TokenType.BEGIN_PARAMETER)) { + return DataTypes.GEOMETRY(); + } + nextToken(TokenType.BEGIN_PARAMETER); + String crs = parseGeospatialParameter(); + nextToken(TokenType.END_PARAMETER); + return DataTypes.GEOMETRY(crs); + } + + private DataType parseGeographyType() { + if (!hasNextToken(TokenType.BEGIN_PARAMETER)) { + return DataTypes.GEOGRAPHY(); + } + nextToken(TokenType.BEGIN_PARAMETER); + String crs = parseGeospatialParameter(); + EdgeAlgorithm algorithm = GeographyType.DEFAULT_ALGORITHM; + if (hasNextToken(TokenType.LIST_SEPARATOR)) { + nextToken(TokenType.LIST_SEPARATOR); + algorithm = EdgeAlgorithm.fromName(parseGeospatialParameter()); + } + nextToken(TokenType.END_PARAMETER); + return DataTypes.GEOGRAPHY(crs, algorithm); + } + + private String parseGeospatialParameter() { + nextToken(); + if (token().type != TokenType.IDENTIFIER + && token().type != TokenType.LITERAL_STRING + && token().type != TokenType.KEYWORD) { + throw parsingError("Geospatial type parameter expected."); + } + return token().value; + } } } diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java index f55da9c4706f..27deb50d93d6 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeRoot.java @@ -104,6 +104,10 @@ public enum DataTypeRoot { BLOB(DataTypeFamily.PREDEFINED), + GEOMETRY(DataTypeFamily.PREDEFINED), + + GEOGRAPHY(DataTypeFamily.PREDEFINED), + ARRAY(DataTypeFamily.CONSTRUCTED, DataTypeFamily.COLLECTION), VECTOR(DataTypeFamily.CONSTRUCTED, DataTypeFamily.COLLECTION), diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java index 6e377309f237..0b546e8a914f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeVisitor.java @@ -66,6 +66,14 @@ public interface DataTypeVisitor { R visit(BlobType blobType); + default R visit(GeometryType geometryType) { + throw new UnsupportedOperationException("Unsupported type: " + geometryType); + } + + default R visit(GeographyType geographyType) { + throw new UnsupportedOperationException("Unsupported type: " + geographyType); + } + R visit(ArrayType arrayType); R visit(VectorType vectorType); diff --git a/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java b/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java index 39b180651ef5..b953a14d9bd0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java +++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypes.java @@ -163,6 +163,26 @@ public static BlobType BLOB() { return new BlobType(); } + public static GeometryType GEOMETRY() { + return new GeometryType(); + } + + public static GeometryType GEOMETRY(String crs) { + return new GeometryType(crs); + } + + public static GeographyType GEOGRAPHY() { + return new GeographyType(); + } + + public static GeographyType GEOGRAPHY(String crs) { + return new GeographyType(crs); + } + + public static GeographyType GEOGRAPHY(String crs, EdgeAlgorithm algorithm) { + return new GeographyType(crs, algorithm); + } + public static OptionalInt getPrecision(DataType dataType) { return dataType.accept(PRECISION_EXTRACTOR); } diff --git a/paimon-api/src/main/java/org/apache/paimon/types/EdgeAlgorithm.java b/paimon-api/src/main/java/org/apache/paimon/types/EdgeAlgorithm.java new file mode 100644 index 000000000000..b16675d8b076 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/types/EdgeAlgorithm.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import org.apache.paimon.annotation.Public; + +import java.util.Locale; + +/** Algorithm used to interpolate geography edges. */ +@Public +public enum EdgeAlgorithm { + SPHERICAL, + VINCENTY, + THOMAS, + ANDOYER, + KARNEY; + + public static EdgeAlgorithm fromName(String algorithmName) { + if (algorithmName == null) { + throw new IllegalArgumentException("Invalid edge interpolation algorithm: null"); + } + + try { + return valueOf(algorithmName.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid edge interpolation algorithm: " + algorithmName, e); + } + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/types/GeographyType.java b/paimon-api/src/main/java/org/apache/paimon/types/GeographyType.java new file mode 100644 index 000000000000..d4f115c51249 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/types/GeographyType.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import org.apache.paimon.annotation.Public; + +import java.util.Locale; +import java.util.Objects; + +/** Geography encoded as OGC Well-Known Binary. */ +@Public +public class GeographyType extends DataType { + + private static final long serialVersionUID = 1L; + + public static final String DEFAULT_CRS = "OGC:CRS84"; + + public static final EdgeAlgorithm DEFAULT_ALGORITHM = EdgeAlgorithm.SPHERICAL; + + private static final String FORMAT = "GEOGRAPHY(%s, %s)"; + + private final String crs; + + private final EdgeAlgorithm algorithm; + + public GeographyType(boolean isNullable, String crs, EdgeAlgorithm algorithm) { + super(isNullable, DataTypeRoot.GEOGRAPHY); + this.crs = GeometryType.validateCrs(crs == null ? DEFAULT_CRS : crs); + this.algorithm = algorithm == null ? DEFAULT_ALGORITHM : algorithm; + } + + public GeographyType(String crs, EdgeAlgorithm algorithm) { + this(true, crs, algorithm); + } + + public GeographyType(String crs) { + this(crs, DEFAULT_ALGORITHM); + } + + public GeographyType() { + this(DEFAULT_CRS); + } + + public String getCrs() { + return crs; + } + + public EdgeAlgorithm getAlgorithm() { + return algorithm; + } + + @Override + public int defaultSize() { + return 20; + } + + @Override + public DataType copy(boolean isNullable) { + return new GeographyType(isNullable, crs, algorithm); + } + + @Override + public String asSQLString() { + return withNullability(FORMAT, GeometryType.formatCrs(crs), algorithm); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + GeographyType that = (GeographyType) o; + return crs.equalsIgnoreCase(that.crs) && algorithm == that.algorithm; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), crs.toUpperCase(Locale.ROOT), algorithm); + } + + @Override + public R accept(DataTypeVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/types/GeometryType.java b/paimon-api/src/main/java/org/apache/paimon/types/GeometryType.java new file mode 100644 index 000000000000..2a98b4a2ac39 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/types/GeometryType.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import org.apache.paimon.annotation.Public; + +import java.util.Locale; +import java.util.Objects; + +import static org.apache.paimon.utils.EncodingUtils.escapeSingleQuotes; + +/** Planar geometry encoded as OGC Well-Known Binary. */ +@Public +public class GeometryType extends DataType { + + private static final long serialVersionUID = 1L; + + public static final String DEFAULT_CRS = "OGC:CRS84"; + + private static final String FORMAT = "GEOMETRY(%s)"; + + private final String crs; + + public GeometryType(boolean isNullable, String crs) { + super(isNullable, DataTypeRoot.GEOMETRY); + this.crs = validateCrs(crs == null ? DEFAULT_CRS : crs); + } + + public GeometryType(String crs) { + this(true, crs); + } + + public GeometryType() { + this(DEFAULT_CRS); + } + + public String getCrs() { + return crs; + } + + @Override + public int defaultSize() { + return 20; + } + + @Override + public DataType copy(boolean isNullable) { + return new GeometryType(isNullable, crs); + } + + @Override + public String asSQLString() { + return withNullability(FORMAT, formatCrs(crs)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass() || !super.equals(o)) { + return false; + } + GeometryType that = (GeometryType) o; + return crs.equalsIgnoreCase(that.crs); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), crs.toUpperCase(Locale.ROOT)); + } + + @Override + public R accept(DataTypeVisitor visitor) { + return visitor.visit(this); + } + + static String validateCrs(String crs) { + if (crs.isEmpty()) { + throw new IllegalArgumentException("Invalid CRS: " + crs); + } + return crs; + } + + static String formatCrs(String crs) { + if (Character.isDigit(crs.charAt(0))) { + return "'" + escapeSingleQuotes(crs) + "'"; + } + for (int i = 0; i < crs.length(); i++) { + char character = crs.charAt(i); + if (Character.isWhitespace(character) + || character == '<' + || character == '>' + || character == '(' + || character == ')' + || character == ',' + || character == '.' + || character == '\'' + || character == '`') { + return "'" + escapeSingleQuotes(crs) + "'"; + } + } + return crs; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/types/GeospatialTypeTest.java b/paimon-api/src/test/java/org/apache/paimon/types/GeospatialTypeTest.java new file mode 100644 index 000000000000..d467b7b91ae1 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/types/GeospatialTypeTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class GeospatialTypeTest { + + @Test + void testIcebergCompatibleDefaultsAndFormatting() { + assertThat(new GeometryType().asSQLString()).isEqualTo("GEOMETRY(OGC:CRS84)"); + assertThat(new GeographyType().asSQLString()).isEqualTo("GEOGRAPHY(OGC:CRS84, spherical)"); + assertThat(new GeographyType().notNull().asSQLString()) + .isEqualTo("GEOGRAPHY(OGC:CRS84, spherical) NOT NULL"); + } + + @Test + void testCrsEqualityIsCaseInsensitive() { + assertThat(new GeometryType("OGC:CRS84")).isEqualTo(new GeometryType("ogc:crs84")); + assertThat(new GeometryType("OGC:CRS84").hashCode()) + .isEqualTo(new GeometryType("ogc:crs84").hashCode()); + assertThat(new GeographyType("OGC:CRS84", EdgeAlgorithm.KARNEY)) + .isEqualTo(new GeographyType("ogc:crs84", EdgeAlgorithm.KARNEY)); + assertThat(new GeographyType("OGC:CRS84", EdgeAlgorithm.KARNEY)) + .isNotEqualTo(new GeographyType("OGC:CRS84", EdgeAlgorithm.SPHERICAL)); + GeometryType custom = new GeometryType("custom, crs's definition"); + assertThat(DataTypeJsonParser.parseAtomicTypeSQLString(custom.asSQLString())) + .isEqualTo(custom); + } + + @Test + void testInvalidParameters() { + assertThatThrownBy(() -> new GeometryType("")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid CRS"); + assertThatThrownBy(() -> EdgeAlgorithm.fromName("rhumb")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid edge interpolation algorithm"); + } + + @Test + void testOnlyIdenticalGeospatialTypesCanBeCast() { + assertThat( + DataTypeCasts.supportsCast( + new GeometryType("OGC:CRS84"), + new GeometryType("ogc:crs84").notNull(), + true)) + .isTrue(); + assertThat( + DataTypeCasts.supportsCast( + new GeometryType("OGC:CRS84"), new GeometryType("EPSG:3857"), true)) + .isFalse(); + assertThat( + DataTypeCasts.supportsCompatibleCast( + new GeographyType("OGC:CRS84", EdgeAlgorithm.SPHERICAL), + new GeographyType("OGC:CRS84", EdgeAlgorithm.KARNEY))) + .isFalse(); + } +} diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java index 80d9208053b8..4ee158cedc98 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java @@ -30,6 +30,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -49,9 +51,13 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; +import java.util.Collections; + /** Utils for conversion between Paimon {@link DataType} and Arrow {@link FieldType}. */ public class ArrowFieldTypeConversion { + public static final String PAIMON_TYPE = "paimon.type"; + public static final ArrowFieldTypeVisitor ARROW_FIELD_TYPE_VISITOR = new ArrowFieldTypeVisitor(); @@ -85,6 +91,24 @@ public FieldType visit(VarBinaryType varBinaryType) { varBinaryType.isNullable(), Types.MinorType.VARBINARY.getType(), null); } + @Override + public FieldType visit(GeometryType geometryType) { + return geospatialFieldType(geometryType); + } + + @Override + public FieldType visit(GeographyType geographyType) { + return geospatialFieldType(geographyType); + } + + private FieldType geospatialFieldType(DataType dataType) { + return new FieldType( + dataType.isNullable(), + Types.MinorType.VARBINARY.getType(), + null, + Collections.singletonMap(PAIMON_TYPE, dataType.asSQLString())); + } + @Override public FieldType visit(DecimalType decimalType) { return new FieldType( diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java index 5ea4ac8aaac9..92812bbeab67 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java @@ -56,7 +56,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.apache.paimon.utils.StringUtils.toLowerCaseIfNeed; @@ -132,12 +134,7 @@ public static Field toArrowField( int depth, ArrowFieldTypeConversion.ArrowFieldTypeVisitor visitor) { FieldType fieldType = dataType.accept(visitor); - fieldType = - new FieldType( - fieldType.isNullable(), - fieldType.getType(), - fieldType.getDictionary(), - Collections.singletonMap(PARQUET_FIELD_ID, String.valueOf(fieldId))); + fieldType = withFieldId(fieldType, fieldId); List children = null; if (dataType instanceof ArrayType || dataType instanceof VectorType) { final DataType elementType; @@ -157,11 +154,10 @@ public static Field toArrowField( typeInner.isNullable(), typeInner.getType(), typeInner.getDictionary(), - Collections.singletonMap( - PARQUET_FIELD_ID, - String.valueOf( - SpecialFields.getArrayElementFieldId( - fieldId, depth + 1)))), + withFieldIdMetadata( + typeInner, + SpecialFields.getArrayElementFieldId( + fieldId, depth + 1))), field.getChildren()); children = Collections.singletonList(field); } else if (dataType instanceof MapType) { @@ -182,11 +178,9 @@ public static Field toArrowField( keyType.isNullable(), keyType.getType(), keyType.getDictionary(), - Collections.singletonMap( - PARQUET_FIELD_ID, - String.valueOf( - SpecialFields.getMapKeyFieldId( - fieldId, depth + 1)))), + withFieldIdMetadata( + keyType, + SpecialFields.getMapKeyFieldId(fieldId, depth + 1))), keyField.getChildren()); Field valueField = @@ -204,11 +198,9 @@ public static Field toArrowField( valueType.isNullable(), valueType.getType(), valueType.getDictionary(), - Collections.singletonMap( - PARQUET_FIELD_ID, - String.valueOf( - SpecialFields.getMapValueFieldId( - fieldId, depth + 1)))), + withFieldIdMetadata( + valueType, + SpecialFields.getMapValueFieldId(fieldId, depth + 1))), valueField.getChildren()); FieldType structType = @@ -246,6 +238,23 @@ public static Field toArrowField( return new Field(fieldName, fieldType, children); } + private static FieldType withFieldId(FieldType fieldType, int fieldId) { + return new FieldType( + fieldType.isNullable(), + fieldType.getType(), + fieldType.getDictionary(), + withFieldIdMetadata(fieldType, fieldId)); + } + + private static Map withFieldIdMetadata(FieldType fieldType, int fieldId) { + Map metadata = new LinkedHashMap<>(); + if (fieldType.getMetadata() != null) { + metadata.putAll(fieldType.getMetadata()); + } + metadata.put(PARQUET_FIELD_ID, String.valueOf(fieldId)); + return metadata; + } + public static ArrowFieldWriter[] createArrowFieldWriters( VectorSchemaRoot vectorSchemaRoot, RowType rowType) { ArrowFieldWriter[] fieldWriters = new ArrowFieldWriter[rowType.getFieldCount()]; diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java index ce0d067e6392..e3a9a53d51bc 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/converter/Arrow2PaimonVectorConverter.java @@ -57,6 +57,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -237,6 +239,16 @@ public byte[] getBytes() { }; } + @Override + public Arrow2PaimonVectorConverter visit(GeometryType geometryType) { + return visit(new VarBinaryType(geometryType.isNullable(), VarBinaryType.MAX_LENGTH)); + } + + @Override + public Arrow2PaimonVectorConverter visit(GeographyType geographyType) { + return visit(new VarBinaryType(geographyType.isNullable(), VarBinaryType.MAX_LENGTH)); + } + @Override public Arrow2PaimonVectorConverter visit(DecimalType decimalType) { return vector -> diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java index 419da16afdd3..972955955cc4 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java @@ -29,6 +29,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -81,6 +83,16 @@ public ArrowFieldWriterFactory visit(VarBinaryType varBinaryType) { return ArrowFieldWriters.BinaryWriter::new; } + @Override + public ArrowFieldWriterFactory visit(GeometryType geometryType) { + return ArrowFieldWriters.BinaryWriter::new; + } + + @Override + public ArrowFieldWriterFactory visit(GeographyType geographyType) { + return ArrowFieldWriters.BinaryWriter::new; + } + @Override public ArrowFieldWriterFactory visit(DecimalType decimalType) { return (fieldVector, isNullable) -> diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java index 75d70c849b9c..35cb10b172c8 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java @@ -125,6 +125,24 @@ public void testVectorType() { Assertions.assertThat(field.getChildren()).hasSize(1); } + @Test + public void testGeospatialTypeMetadata() { + Field geometry = ArrowUtils.toArrowField("geom", 7, DataTypes.GEOMETRY(), 0); + Assertions.assertThat(geometry.getType()).isEqualTo(ArrowType.Binary.INSTANCE); + Assertions.assertThat(geometry.getMetadata()) + .containsEntry(ArrowUtils.PARQUET_FIELD_ID, "7") + .containsEntry(ArrowFieldTypeConversion.PAIMON_TYPE, "GEOMETRY(OGC:CRS84)"); + + Field geography = + ArrowUtils.toArrowField("geographies", 8, DataTypes.ARRAY(DataTypes.GEOGRAPHY()), 0) + .getChildren() + .get(0); + Assertions.assertThat(geography.getType()).isEqualTo(ArrowType.Binary.INSTANCE); + Assertions.assertThat(geography.getMetadata()) + .containsEntry( + ArrowFieldTypeConversion.PAIMON_TYPE, "GEOGRAPHY(OGC:CRS84, spherical)"); + } + @Test public void testSameRootAllocatorIncludesNestedVectors() { try (RootAllocator allocator = new RootAllocator(); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java index 5ce7b779f025..7a62523711e1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java @@ -75,6 +75,8 @@ public static int calculateFixLengthPartSize(DataType type) { case VARCHAR: case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: case DECIMAL: case BIGINT: case DOUBLE: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArrayWriter.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArrayWriter.java index 58f98dc18933..9b6accdfea23 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArrayWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArrayWriter.java @@ -229,6 +229,8 @@ public static NullSetter createNullSetter(DataType elementType) { case VARCHAR: case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: case DECIMAL: case BIGINT: case TIMESTAMP_WITHOUT_TIME_ZONE: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java index 2e0cd5701b71..47960aa3691b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryWriter.java @@ -148,6 +148,8 @@ static void write( break; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: byte[] bytes = (byte[]) o; writer.writeBinary(pos, bytes, 0, bytes.length); break; @@ -181,6 +183,8 @@ static ValueSetter createValueSetter(DataType elementType, Serializer seriali return (writer, pos, value) -> writer.writeBoolean(pos, (boolean) value); case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: return (writer, pos, value) -> { byte[] bytes = (byte[]) value; writer.writeBinary(pos, bytes, 0, bytes.length); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java index 41640f3771af..78382e8bbd23 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java @@ -86,6 +86,8 @@ static ElementGetter createElementGetter(DataType elementType) { break; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: elementGetter = InternalArray::getBinary; break; case DECIMAL: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java index e4aa7e335cd6..445d8854a19a 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java @@ -126,6 +126,8 @@ static Class getDataClass(DataType type) { return Boolean.class; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: return byte[].class; case DECIMAL: return Decimal.class; @@ -180,6 +182,8 @@ static FieldGetter createFieldGetter(DataType fieldType, int fieldPos) { break; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: fieldGetter = row -> row.getBinary(fieldPos); break; case DECIMAL: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnVectorUtils.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnVectorUtils.java index 716d9a582e0e..ee21025b9672 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnVectorUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnVectorUtils.java @@ -97,6 +97,8 @@ private static WritableColumnVector createWritableColumnVector( case VARCHAR: case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: case BLOB: return new HeapBytesVector(capacity); case DECIMAL: diff --git a/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java index e35c7d77b1f6..3e354c4821da 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/RowToColumnConverter.java @@ -55,6 +55,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -367,6 +369,16 @@ public TypeConverter visit(VarBinaryType varBinaryType) { return binaryConverter(varBinaryType.isNullable()); } + @Override + public TypeConverter visit(GeometryType geometryType) { + return binaryConverter(geometryType.isNullable()); + } + + @Override + public TypeConverter visit(GeographyType geographyType) { + return binaryConverter(geographyType.isNullable()); + } + @Override public TypeConverter visit(DecimalType decimalType) { return createConverter( diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java index 6669f347ff27..fb40d944854e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalSerializers.java @@ -54,6 +54,8 @@ private static Serializer createInternal(DataType type) { return BooleanSerializer.INSTANCE; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: return BinarySerializer.INSTANCE; case DECIMAL: return new DecimalSerializer(getPrecision(type), getScale(type)); diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java index c7fb58417afa..baa4d0432564 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java @@ -185,6 +185,8 @@ private static FieldWriter createFieldWriter(DataType fieldType) { break; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: fieldWriter = (writer, pos, value) -> writer.writeBinary((byte[]) value); break; case DECIMAL: @@ -301,6 +303,8 @@ private static FieldReader createFieldReader(DataType fieldType) { break; case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: fieldReader = (reader, pos) -> reader.readBinary(); break; case DECIMAL: diff --git a/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsCollector.java b/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsCollector.java index b446e4fb2228..96c10d78a028 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsCollector.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsCollector.java @@ -21,6 +21,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalSerializers; import org.apache.paimon.data.serializer.Serializer; +import org.apache.paimon.statistics.CountsSimpleColStatsCollector; import org.apache.paimon.statistics.NoneSimpleColStatsCollector; import org.apache.paimon.statistics.SimpleColStatsCollector; import org.apache.paimon.types.RowType; @@ -52,6 +53,18 @@ public SimpleStatsCollector( numFields, collectorFactory.length); this.statsCollectors = SimpleColStatsCollector.create(collectorFactory); + for (int i = 0; i < numFields; i++) { + switch (rowType.getTypeAt(i).getTypeRoot()) { + case GEOMETRY: + case GEOGRAPHY: + if (!(statsCollectors[i] instanceof NoneSimpleColStatsCollector)) { + statsCollectors[i] = new CountsSimpleColStatsCollector(); + } + break; + default: + // Keep the configured collector for non-geospatial fields. + } + } this.converter = new RowDataToObjectArrayConverter(rowType); this.fieldSerializers = new Serializer[numFields]; for (int i = 0; i < numFields; i++) { diff --git a/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java b/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java index dbac55a07dde..8487100c8f4c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/InternalRowToSizeVisitor.java @@ -88,6 +88,20 @@ public BiFunction visit(VarBinaryType varBinaryTy }; } + @Override + public BiFunction visit(GeometryType geometryType) { + return binarySize(); + } + + @Override + public BiFunction visit(GeographyType geographyType) { + return binarySize(); + } + + private BiFunction binarySize() { + return (row, index) -> row.isNullAt(index) ? NULL_SIZE : row.getBinary(index).length; + } + @Override public BiFunction visit(DecimalType decimalType) { return (row, index) -> { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java index 8fb0b8f8050d..f5d0e87934d5 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java @@ -381,6 +381,8 @@ public static Object get(DataGetters dataGetters, int pos, DataType fieldType) { return dataGetters.getRow(pos, ((RowType) fieldType).getFieldCount()); case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: return dataGetters.getBinary(pos); case VARIANT: return dataGetters.getVariant(pos); diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeCheckUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TypeCheckUtils.java index c1520be34107..138c2ca9125b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeCheckUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeCheckUtils.java @@ -26,6 +26,8 @@ import static org.apache.paimon.types.DataTypeRoot.BLOB; import static org.apache.paimon.types.DataTypeRoot.BOOLEAN; import static org.apache.paimon.types.DataTypeRoot.DECIMAL; +import static org.apache.paimon.types.DataTypeRoot.GEOGRAPHY; +import static org.apache.paimon.types.DataTypeRoot.GEOMETRY; import static org.apache.paimon.types.DataTypeRoot.INTEGER; import static org.apache.paimon.types.DataTypeRoot.MAP; import static org.apache.paimon.types.DataTypeRoot.MULTISET; @@ -110,6 +112,14 @@ public static boolean isBlob(DataType type) { return type.getTypeRoot() == BLOB; } + public static boolean isGeometry(DataType type) { + return type.getTypeRoot() == GEOMETRY; + } + + public static boolean isGeography(DataType type) { + return type.getTypeRoot() == GEOGRAPHY; + } + public static boolean isComparable(DataType type) { return !isMap(type) && !isMultiset(type) @@ -117,7 +127,9 @@ public static boolean isComparable(DataType type) { && !isArray(type) && !isVector(type) && !isVariant(type) - && !isBlob(type); + && !isBlob(type) + && !isGeometry(type) + && !isGeography(type); } public static boolean isMutable(DataType type) { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java index 54007caaf828..b12fda4f7d13 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java @@ -55,6 +55,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -162,6 +164,16 @@ public ColumnVector visit(VarBinaryType varBinaryType) { return bytesColumnVector(); } + @Override + public ColumnVector visit(GeometryType geometryType) { + return bytesColumnVector(); + } + + @Override + public ColumnVector visit(GeographyType geographyType) { + return bytesColumnVector(); + } + @Override public ColumnVector visit(DecimalType decimalType) { return new DecimalColumnVector() { diff --git a/paimon-common/src/test/java/org/apache/paimon/format/GeospatialStatsCollectorTest.java b/paimon-common/src/test/java/org/apache/paimon/format/GeospatialStatsCollectorTest.java new file mode 100644 index 000000000000..53aa2f8e01f2 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/format/GeospatialStatsCollectorTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class GeospatialStatsCollectorTest { + + @Test + void testWkbDoesNotProduceLexicographicBounds() { + SimpleStatsCollector collector = + new SimpleStatsCollector(RowType.of(DataTypes.GEOMETRY(), DataTypes.GEOGRAPHY())); + + collector.collect(GenericRow.of(new byte[] {2}, new byte[] {9})); + collector.collect(GenericRow.of(new byte[] {1}, null)); + + SimpleColStats[] stats = collector.extract(); + assertThat(stats[0].min()).isNull(); + assertThat(stats[0].max()).isNull(); + assertThat(stats[0].nullCount()).isZero(); + assertThat(stats[1].min()).isNull(); + assertThat(stats[1].max()).isNull(); + assertThat(stats[1].nullCount()).isOne(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java index 7334f7528af7..950da63b9d11 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java @@ -187,7 +187,9 @@ public static IcebergDataFileMeta create( || typeRoot == DataTypeRoot.MULTISET || typeRoot == DataTypeRoot.VARIANT || typeRoot == DataTypeRoot.VECTOR - || typeRoot == DataTypeRoot.BLOB) { + || typeRoot == DataTypeRoot.BLOB + || typeRoot == DataTypeRoot.GEOMETRY + || typeRoot == DataTypeRoot.GEOGRAPHY) { continue; } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java index 9862ff7f90c4..af3bfaec9854 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java @@ -28,7 +28,10 @@ import org.apache.paimon.types.DateType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.EdgeAlgorithm; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -196,6 +199,12 @@ private static Object toTypeObject(DataType dataType, int fieldId, int depth) { return timestampLtzPrecision >= 7 ? "timestamptz_ns" : "timestamptz"; case VARIANT: return "variant"; + case GEOMETRY: + return String.format("geometry(%s)", ((GeometryType) dataType).getCrs()); + case GEOGRAPHY: + GeographyType geographyType = (GeographyType) dataType; + return String.format( + "geography(%s, %s)", geographyType.getCrs(), geographyType.getAlgorithm()); case ARRAY: ArrayType arrayType = (ArrayType) dataType; return new IcebergListType( @@ -290,6 +299,16 @@ private DataType getDataTypeFromType(Object icebergType, boolean isRequired) { return new LocalZonedTimestampType(!isRequired, 9); case "variant": // iceberg v3 format return new VariantType(!isRequired); + case "geometry": // iceberg v3 format + return new GeometryType(!isRequired, geometryParameter(simpleType)); + case "geography": // iceberg v3 format + String[] parameters = geographyParameters(simpleType); + Preconditions.checkArgument( + parameters.length == 2, + "Invalid Iceberg geography type: %s", + simpleType); + return new GeographyType( + !isRequired, parameters[0], EdgeAlgorithm.fromName(parameters[1])); default: throw new UnsupportedOperationException( "Unsupported primitive data type: " + icebergType); @@ -321,6 +340,37 @@ private DataType getDataTypeFromType(Object icebergType, boolean isRequired) { } } + private static String geometryParameter(String simpleType) { + if ("geometry".equals(simpleType)) { + return GeometryType.DEFAULT_CRS; + } + int start = simpleType.indexOf('('); + Preconditions.checkArgument( + start >= 0 && simpleType.endsWith(")"), + "Invalid Iceberg geometry type: %s", + simpleType); + return simpleType.substring(start + 1, simpleType.length() - 1).trim(); + } + + private static String[] geographyParameters(String simpleType) { + int start = simpleType.indexOf('('); + if (start < 0) { + if ("geography".equals(simpleType)) { + return new String[] {GeometryType.DEFAULT_CRS, EdgeAlgorithm.SPHERICAL.toString()}; + } + } + Preconditions.checkArgument( + start >= 0 && simpleType.endsWith(")"), + "Invalid Iceberg geospatial type: %s", + simpleType); + String parameters = simpleType.substring(start + 1, simpleType.length() - 1); + String[] result = parameters.split(",", -1); + for (int i = 0; i < result.length; i++) { + result[i] = result[i].trim(); + } + return result; + } + public DataField toDatafield() { return new DataField(id, name, dataType(), doc); } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 607f5d20fa15..d6e089fba2b3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -32,6 +32,7 @@ import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.bitmap.BitmapGlobalIndexerFactory; import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory; @@ -43,6 +44,8 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -125,7 +128,9 @@ public class SchemaValidation { RowType.class, MultisetType.class, VectorType.class, - VariantType.class); + VariantType.class, + GeometryType.class, + GeographyType.class); /** * Validate the {@link TableSchema} and {@link CoreOptions}. @@ -226,6 +231,7 @@ public static void validateTableSchema(TableSchema schema, Set dynamicOp FileFormat fileFormat = FileFormat.fromIdentifier(options.formatType(), new Options(schema.options())); RowType tableRowType = new RowType(schema.fields()); + validateGeospatialTypes(schema, options, tableRowType); validateBlobFields(tableRowType, options); Set blobDescriptorFields = validateBlobDescriptorFields(tableRowType, options); Set blobViewFields = @@ -449,6 +455,68 @@ private static void validateOnlyContainPrimitiveType( } } + private static void validateGeospatialTypes( + TableSchema schema, CoreOptions options, RowType rowType) { + boolean hasGeospatial = + containsType( + rowType, + type -> type.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY)); + if (!hasGeospatial) { + return; + } + + checkArgument( + CoreOptions.FILE_FORMAT_PARQUET.equals(options.formatType()), + "Geometry and geography columns require '%s'='parquet', but was '%s'.", + CoreOptions.FILE_FORMAT.key(), + options.formatType()); + options.fileFormatPerLevel() + .forEach( + (level, format) -> + checkArgument( + CoreOptions.FILE_FORMAT_PARQUET.equals(format), + "Geometry and geography columns require parquet at every level, but '%s' contains '%s:%s'.", + CoreOptions.FILE_FORMAT_PER_LEVEL.key(), + level, + format)); + checkArgument( + options.changelogFileFormat() == null + || CoreOptions.FILE_FORMAT_PARQUET.equals(options.changelogFileFormat()), + "Geometry and geography columns require '%s' to be parquet, but was '%s'.", + CoreOptions.CHANGELOG_FILE_FORMAT.key(), + options.changelogFileFormat()); + if (options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE) + != IcebergOptions.StorageType.DISABLED) { + checkArgument( + options.toConfiguration().get(IcebergOptions.FORMAT_VERSION) == 3, + "Geometry and geography columns require '%s'='3' when Iceberg metadata is enabled.", + IcebergOptions.FORMAT_VERSION.key()); + } + + Set geospatialFields = + schema.fields().stream() + .filter( + field -> + field.type() + .isAnyOf( + DataTypeRoot.GEOMETRY, + DataTypeRoot.GEOGRAPHY)) + .map(DataField::name) + .collect(Collectors.toSet()); + Set geospatialBucketKeys = new HashSet<>(schema.bucketKeys()); + geospatialBucketKeys.retainAll(geospatialFields); + checkArgument( + geospatialBucketKeys.isEmpty(), + "Geometry and geography columns cannot be bucket keys: %s.", + geospatialBucketKeys); + Set geospatialSequenceFields = new HashSet<>(options.sequenceField()); + geospatialSequenceFields.retainAll(geospatialFields); + checkArgument( + geospatialSequenceFields.isEmpty(), + "Geometry and geography columns cannot be sequence fields: %s.", + geospatialSequenceFields); + } + private static void validateStartupMode(CoreOptions options) { if (options.startupMode() == CoreOptions.StartupMode.FROM_TIMESTAMP) { checkExactOneOptionExistInMode( diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java index c4888312b053..b0b7154b18d2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java @@ -228,4 +228,51 @@ void testRequiredNestedFieldSkipped() { assertThat((byte[]) ((GenericMap) meta.upperBounds()).get(1)) .isEqualTo(new byte[] {5, 0, 0, 0}); } + + @Test + @DisplayName("Test geospatial fields publish null counts but not WKB bounds") + void testGeospatialBoundsSkipped() { + IcebergSchema icebergSchema = + new IcebergSchema( + 0, + Arrays.asList( + new IcebergDataField( + new DataField(1, "geom", DataTypes.GEOMETRY())), + new IcebergDataField( + new DataField(2, "geog", DataTypes.GEOGRAPHY())))); + + byte[] wkbPoint = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0x40 + }; + BinaryRow values = new BinaryRow(2); + BinaryRowWriter rowWriter = new BinaryRowWriter(values); + rowWriter.writeBinary(0, wkbPoint, 0, wkbPoint.length); + rowWriter.writeBinary(1, wkbPoint, 0, wkbPoint.length); + rowWriter.complete(); + + BinaryArray nullCounts = new BinaryArray(); + BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8); + arrayWriter.writeLong(0, 1L); + arrayWriter.writeLong(1, 2L); + arrayWriter.complete(); + + IcebergDataFileMeta meta = + IcebergDataFileMeta.create( + IcebergDataFileMeta.Content.DATA, + "path", + "parquet", + BinaryRow.EMPTY_ROW, + 10, + 100, + icebergSchema, + new SimpleStats(values, values, nullCounts), + null); + + assertThat(meta.nullValueCounts().size()).isEqualTo(2); + assertThat(((GenericMap) meta.nullValueCounts()).get(1)).isEqualTo(1L); + assertThat(((GenericMap) meta.nullValueCounts()).get(2)).isEqualTo(2L); + assertThat(meta.lowerBounds().size()).isZero(); + assertThat(meta.upperBounds().size()).isZero(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java index 0cfd5f0fe5fc..11f5e12f0744 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java @@ -28,7 +28,10 @@ import org.apache.paimon.types.DateType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.EdgeAlgorithm; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -153,6 +156,40 @@ void testPrimitiveTypeConversions() { assertThat(icebergVarBinary.type()).isEqualTo("binary"); } + @Test + @DisplayName("Test Iceberg v3 geospatial type conversions") + void testGeospatialTypeConversions() { + IcebergDataField geometry = + new IcebergDataField( + new DataField(1, "geom", new GeometryType(false, "EPSG:3857"))); + assertThat(geometry.type()).isEqualTo("geometry(EPSG:3857)"); + + IcebergDataField geography = + new IcebergDataField( + new DataField( + 2, + "geog", + new GeographyType(true, "OGC:CRS84", EdgeAlgorithm.KARNEY))); + assertThat(geography.type()).isEqualTo("geography(OGC:CRS84, karney)"); + + assertThat(new IcebergDataField(3, "geom", true, "geometry(EPSG:3857)", null).dataType()) + .isEqualTo(new GeometryType(false, "EPSG:3857")); + assertThat( + new IcebergDataField( + 4, "geog", false, "geography(OGC:CRS84, vincenty)", null) + .dataType()) + .isEqualTo(new GeographyType(true, "OGC:CRS84", EdgeAlgorithm.VINCENTY)); + + assertThat(new IcebergDataField(5, "geom", false, "geometry", null).dataType()) + .isEqualTo(new GeometryType()); + assertThat(new IcebergDataField(6, "geog", false, "geography", null).dataType()) + .isEqualTo(new GeographyType()); + assertThat( + new IcebergDataField(7, "geom", false, "geometry(custom, definition)", null) + .dataType()) + .isEqualTo(new GeometryType("custom, definition")); + } + @Test @DisplayName("Test decimal type conversion") void testDecimalTypeConversion() { diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java index fc6b6abd682e..369010a0f2ba 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java @@ -31,7 +31,10 @@ import org.apache.paimon.types.DateType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.EdgeAlgorithm; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -114,6 +117,14 @@ private static Stream testData() { TestSpec.forString("TIMESTAMP_LTZ(3)").expectType(new LocalZonedTimestampType(3)), TestSpec.forString("VARIANT").expectType(new VariantType()), TestSpec.forString("BLOB").expectType(new BlobType()), + TestSpec.forString("GEOMETRY") + .expectType(new GeometryType(GeometryType.DEFAULT_CRS)), + TestSpec.forString("geometry(ogc:crs84) NOT NULL") + .expectType(new GeometryType(false, "OGC:CRS84")), + TestSpec.forString("GEOGRAPHY") + .expectType(new GeographyType(GeographyType.DEFAULT_CRS)), + TestSpec.forString("GEOGRAPHY(EPSG:4326, karney)") + .expectType(new GeographyType("EPSG:4326", EdgeAlgorithm.KARNEY)), TestSpec.forString("VECTOR") .expectType(DataTypes.VECTOR(3, DataTypes.FLOAT())), TestSpec.forString("VECTOR NOT NULL") @@ -198,7 +209,9 @@ private static Stream testData() { TestSpec.forString("VARCHAR(test)").expectErrorMessage(" expected"), TestSpec.forString("VARCHAR(33333333333)") - .expectErrorMessage("Invalid integer value")); + .expectErrorMessage("Invalid integer value"), + TestSpec.forString("GEOGRAPHY(OGC:CRS84, rhumb)") + .expectErrorMessage("Invalid edge interpolation algorithm")); } @ParameterizedTest(name = "{index}: [From: {0}, To: {1}]") diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 3ba69852f33a..187b710cd84d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.schema; import org.apache.paimon.CoreOptions; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.table.BucketMode; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -1621,6 +1622,139 @@ public void testFileFormatPerLevelAcceptsCompatibleSchema() { new TableSchema(1, fields, 10, emptyList(), singletonList("k"), options, "")); } + @Test + public void testGeospatialTypeValidation() { + List fields = + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField(1, "geom", DataTypes.GEOMETRY()), + new DataField(2, "geog", DataTypes.GEOGRAPHY())); + + assertThatNoException() + .isThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + new HashMap<>()))); + + Map avroOptions = new HashMap<>(); + avroOptions.put(CoreOptions.FILE_FORMAT.key(), "avro"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, emptyList(), emptyList(), avroOptions))) + .hasMessageContaining("require 'file.format'='parquet'"); + + Map perLevelOptions = new HashMap<>(); + perLevelOptions.put(CoreOptions.FILE_FORMAT_PER_LEVEL.key(), "0:orc"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, emptyList(), emptyList(), perLevelOptions))) + .hasMessageContaining("require parquet at every level"); + + Map changelogOptions = new HashMap<>(); + changelogOptions.put(CoreOptions.CHANGELOG_FILE_FORMAT.key(), "orc"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + changelogOptions))) + .hasMessageContaining("require 'changelog-file.format' to be parquet"); + + Map icebergV2Options = new HashMap<>(); + icebergV2Options.put(IcebergOptions.METADATA_ICEBERG_STORAGE.key(), "table-location"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + icebergV2Options))) + .hasMessageContaining("require 'metadata.iceberg.format-version'='3'"); + + icebergV2Options.put(IcebergOptions.FORMAT_VERSION.key(), "3"); + assertThatNoException() + .isThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + icebergV2Options))); + } + + @Test + public void testGeospatialTypeRejectsKeyAndOrderingSemantics() { + List fields = + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField(1, "geom", DataTypes.GEOMETRY()), + new DataField(2, "geog", DataTypes.GEOGRAPHY())); + + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + singletonList("geom"), + emptyList(), + new HashMap<>()))) + .hasMessage("The type GeometryType in partition field geom is unsupported"); + + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + singletonList("geog"), + new HashMap<>()))) + .hasMessage("The type GeographyType in primary key field geog is unsupported"); + + Map bucketOptions = new HashMap<>(); + bucketOptions.put(CoreOptions.BUCKET_KEY.key(), "geom"); + bucketOptions.put(BUCKET.key(), "1"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, emptyList(), emptyList(), bucketOptions))) + .hasMessage("Geometry and geography columns cannot be bucket keys: [geom]."); + + Map sequenceOptions = new HashMap<>(); + sequenceOptions.put(CoreOptions.SEQUENCE_FIELD.key(), "geog"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + singletonList("id"), + sequenceOptions))) + .hasMessage("Geometry and geography columns cannot be sequence fields: [geog]."); + } + + private TableSchema geospatialSchema( + List fields, + List partitionKeys, + List primaryKeys, + Map options) { + options.putIfAbsent(BUCKET.key(), "-1"); + return new TableSchema( + 1, fields, 10, partitionKeys, primaryKeys, options, "geospatial test"); + } + @Test void testManifestSortValidation() { List fields = diff --git a/paimon-core/src/test/java/org/apache/paimon/table/GeospatialTypeTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/GeospatialTypeTableTest.java new file mode 100644 index 000000000000..309b20b902cd --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/GeospatialTypeTableTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; + +import org.junit.jupiter.api.Test; + +import java.util.Comparator; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests table read and write with Iceberg-compatible geospatial types. */ +public class GeospatialTypeTableTest extends TableTestBase { + + private static final byte[] POINT_1_2_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] POINT_3_4_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x40, 0, 0, 0, 0, 0, 0, 0x10, 0x40}; + + @Test + public void testReadWriteAndStats() throws Exception { + createTableDefault(); + FileStoreTable table = getTableDefault(); + + assertThat(table.schema().fields().get(1).type()) + .isEqualTo(DataTypes.GEOMETRY("EPSG:3857")); + assertThat(table.schema().fields().get(2).type()) + .isEqualTo(DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.KARNEY)); + + write( + table, + GenericRow.of( + 1, + POINT_1_2_WKB, + POINT_3_4_WKB, + new GenericArray(new Object[] {POINT_1_2_WKB, null, POINT_3_4_WKB})), + GenericRow.of(2, null, POINT_1_2_WKB, new GenericArray(new Object[0])), + GenericRow.of(3, POINT_3_4_WKB, null, null)); + + List rows = read(table); + rows.sort(Comparator.comparingInt(row -> row.getInt(0))); + + assertThat(rows).hasSize(3); + assertThat(rows.get(0).getBinary(1)).isEqualTo(POINT_1_2_WKB); + assertThat(rows.get(0).getBinary(2)).isEqualTo(POINT_3_4_WKB); + InternalArray geometries = rows.get(0).getArray(3); + assertThat(geometries.size()).isEqualTo(3); + assertThat(geometries.getBinary(0)).isEqualTo(POINT_1_2_WKB); + assertThat(geometries.isNullAt(1)).isTrue(); + assertThat(geometries.getBinary(2)).isEqualTo(POINT_3_4_WKB); + + assertThat(rows.get(1).isNullAt(1)).isTrue(); + assertThat(rows.get(1).getBinary(2)).isEqualTo(POINT_1_2_WKB); + assertThat(rows.get(1).getArray(3).size()).isZero(); + assertThat(rows.get(2).getBinary(1)).isEqualTo(POINT_3_4_WKB); + assertThat(rows.get(2).isNullAt(2)).isTrue(); + assertThat(rows.get(2).isNullAt(3)).isTrue(); + + DataSplit split = (DataSplit) table.newScan().plan().splits().get(0); + assertThat(split.dataFiles()).hasSize(1); + DataFileMeta file = split.dataFiles().get(0); + assertThat(file.fileFormat()).isEqualTo(CoreOptions.FILE_FORMAT_PARQUET); + + SimpleStats stats = file.valueStats(); + assertThat(stats.minValues().isNullAt(1)).isTrue(); + assertThat(stats.maxValues().isNullAt(1)).isTrue(); + assertThat(stats.nullCounts().getLong(1)).isEqualTo(1L); + assertThat(stats.minValues().isNullAt(2)).isTrue(); + assertThat(stats.maxValues().isNullAt(2)).isTrue(); + assertThat(stats.nullCounts().getLong(2)).isEqualTo(1L); + } + + @Override + protected Schema schemaDefault() { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("geom", DataTypes.GEOMETRY("EPSG:3857")) + .column("geog", DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.KARNEY)) + .column("geometries", DataTypes.ARRAY(DataTypes.GEOMETRY())) + .option(CoreOptions.FILE_FORMAT.key(), CoreOptions.FILE_FORMAT_PARQUET) + .build(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java index 92ae714ca577..f15684933599 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java @@ -31,6 +31,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -155,6 +157,24 @@ public LogicalType visit(BlobType blobType) { org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); } + @Override + public LogicalType visit(GeometryType geometryType) { + // Flink has no native geospatial logical type. Expose WKB through SQL while preserving the + // geospatial type in the Paimon schema. + return new org.apache.flink.table.types.logical.VarBinaryType( + geometryType.isNullable(), + org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); + } + + @Override + public LogicalType visit(GeographyType geographyType) { + // Flink has no native geospatial logical type. Expose WKB through SQL while preserving the + // geospatial type in the Paimon schema. + return new org.apache.flink.table.types.logical.VarBinaryType( + geographyType.isNullable(), + org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); + } + @Override public LogicalType visit(ArrayType arrayType) { return new org.apache.flink.table.types.logical.ArrayType( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java new file mode 100644 index 000000000000..25fb4e8d110a --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; + +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests Flink SQL interoperability with Paimon geospatial columns. */ +public class GeospatialTypeTableITCase extends CatalogITCaseBase { + + private static final String TABLE_NAME = "geospatial_table"; + + private static final byte[] POINT_1_2_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] POINT_3_4_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x40, 0, 0, 0, 0, 0, 0, 0x10, 0x40}; + + @Test + public void testReadWriteGeospatialColumnsAsWkb() throws Exception { + flinkCatalog() + .catalog() + .createTable( + Identifier.create(tEnv.getCurrentDatabase(), TABLE_NAME), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("geom", DataTypes.GEOMETRY("EPSG:3857")) + .column( + "geog", + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.SPHERICAL)) + .option( + CoreOptions.FILE_FORMAT.key(), + CoreOptions.FILE_FORMAT_PARQUET) + .build(), + false); + + List columnTypes = + tEnv.from(TABLE_NAME).getResolvedSchema().getColumnDataTypes(); + assertThat(columnTypes.get(0).getLogicalType().is(LogicalTypeRoot.INTEGER)).isTrue(); + assertThat(columnTypes.get(1).getLogicalType().is(LogicalTypeRoot.VARBINARY)).isTrue(); + assertThat(columnTypes.get(2).getLogicalType().is(LogicalTypeRoot.VARBINARY)).isTrue(); + + batchSql( + "INSERT INTO %s VALUES " + + "(1, X'0101000000000000000000F03F0000000000000040', " + + "X'010100000000000000000008400000000000001040'), " + + "(2, CAST(NULL AS BYTES), " + + "X'0101000000000000000000F03F0000000000000040')", + TABLE_NAME); + + List rows = batchSql("SELECT * FROM %s ORDER BY id", TABLE_NAME); + assertThat(rows) + .containsExactly( + Row.of(1, POINT_1_2_WKB, POINT_3_4_WKB), Row.of(2, null, POINT_1_2_WKB)); + + assertThat(paimonTable(TABLE_NAME).schema().fields().get(1).type()) + .isEqualTo(DataTypes.GEOMETRY("EPSG:3857")); + assertThat(paimonTable(TABLE_NAME).schema().fields().get(2).type()) + .isEqualTo(DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.SPHERICAL)); + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java index e8abaae336a8..3f225dc62a44 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java @@ -33,6 +33,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -71,6 +73,8 @@ */ class ArrowSchemaMetadata { + private static final String PAIMON_TYPE = "paimon.type"; + private static final String LIST_DATA_VECTOR_NAME = "$data$"; private static final String MAP_DATA_VECTOR_NAME = "entries"; private static final String MAP_KEY_NAME = "key"; @@ -396,7 +400,10 @@ private static ArrowField withMetadata(ArrowField field, Map met private static ArrowField toArrowField( String fieldName, int fieldId, DataType dataType, int depth, String fieldIdKey) { ArrowTypeInfo type = dataType.accept(ArrowFieldTypeVisitor.INSTANCE); - Map metadata = fieldIdMetadata(fieldId, fieldIdKey); + Map metadata = new LinkedHashMap<>(fieldIdMetadata(fieldId, fieldIdKey)); + if (dataType instanceof GeometryType || dataType instanceof GeographyType) { + metadata.put(PAIMON_TYPE, dataType.asSQLString()); + } List children = Collections.emptyList(); if (dataType instanceof ArrayType || dataType instanceof VectorType) { DataType elementType = @@ -565,6 +572,16 @@ public ArrowTypeInfo visit(VarBinaryType varBinaryType) { return ArrowTypeInfo.simple(TYPE_BINARY); } + @Override + public ArrowTypeInfo visit(GeometryType geometryType) { + return ArrowTypeInfo.simple(TYPE_BINARY); + } + + @Override + public ArrowTypeInfo visit(GeographyType geographyType) { + return ArrowTypeInfo.simple(TYPE_BINARY); + } + @Override public ArrowTypeInfo visit(DecimalType decimalType) { ArrowTypeInfo type = ArrowTypeInfo.simple(TYPE_DECIMAL); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 309ed5c5ffd4..e8d760d137e9 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -25,6 +25,9 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.EdgeAlgorithm; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -34,6 +37,7 @@ import org.apache.paimon.types.VectorType; import org.apache.paimon.utils.Pair; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.ConversionPatterns; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; @@ -128,6 +132,21 @@ public static Type convertToParquetType(String name, DataType type, int fieldId, return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .named(name) .withId(fieldId); + case GEOMETRY: + return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) + .as(LogicalTypeAnnotation.geometryType(((GeometryType) type).getCrs())) + .named(name) + .withId(fieldId); + case GEOGRAPHY: + GeographyType geographyType = (GeographyType) type; + return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) + .as( + LogicalTypeAnnotation.geographyType( + geographyType.getCrs(), + EdgeInterpolationAlgorithm.valueOf( + geographyType.getAlgorithm().name()))) + .named(name) + .withId(fieldId); case DECIMAL: int precision = ((DecimalType) type).getPrecision(); int scale = ((DecimalType) type).getScale(); @@ -336,6 +355,21 @@ public static DataField convertToPaimonField(Type parquetType) { case BINARY: if (logicalType instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { paimonDataType = DataTypes.STRING(); + } else if (logicalType + instanceof LogicalTypeAnnotation.GeometryLogicalTypeAnnotation) { + paimonDataType = + DataTypes.GEOMETRY( + ((LogicalTypeAnnotation.GeometryLogicalTypeAnnotation) + logicalType) + .getCrs()); + } else if (logicalType + instanceof LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) { + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geography = + (LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) logicalType; + paimonDataType = + DataTypes.GEOGRAPHY( + geography.getCrs(), + EdgeAlgorithm.valueOf(geography.getAlgorithm().name())); } else { paimonDataType = DataTypes.BYTES(); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java index fa91dbf28927..39492e8bdf2a 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java @@ -152,6 +152,10 @@ private SimpleColStats toFieldStats( binaryStats.genericGetMax().getBytes(), nullCount); break; + case GEOMETRY: + case GEOGRAPHY: + fieldStats = new SimpleColStats(null, null, nullCount); + break; case BOOLEAN: assertStatsClass(field, stats, BooleanStatistics.class); BooleanStatistics boolStats = (BooleanStatistics) stats; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java index 6239c9fbb51a..c0445b3f136d 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetVectorUpdaterFactory.java @@ -45,6 +45,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -129,6 +131,16 @@ public UpdaterFactory visit(VarBinaryType varBinaryType) { }; } + @Override + public UpdaterFactory visit(GeometryType geometryType) { + return c -> new BinaryUpdater(); + } + + @Override + public UpdaterFactory visit(GeographyType geographyType) { + return c -> new BinaryUpdater(); + } + @Override public UpdaterFactory visit(DecimalType decimalType) { return c -> { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java index 89a45c38c610..3594f6fcd7ae 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetRowDataWriter.java @@ -93,6 +93,8 @@ private FieldWriter createWriter(DataType t, Type type) { return new BooleanWriter(); case BINARY: case VARBINARY: + case GEOMETRY: + case GEOGRAPHY: return new BinaryWriter(); case BLOB: return new BlobDescriptorWriter(); diff --git a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java index f01854f9e314..4a2d3d2cb374 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java @@ -145,4 +145,24 @@ public void testBuildArrowSchemaWithoutFieldIdMetadata() { assertThat(metadata.get("name")) .doesNotContainKey(FormatMetadataUtils.PARQUET_FIELD_ID_KEY); } + + @Test + public void testBuildArrowSchemaWithGeospatialMetadata() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY()), + DataTypes.FIELD(1, "geog", DataTypes.GEOGRAPHY())); + + byte[] schemaBytes = + FormatMetadataUtils.buildArrowSchemaMetadata( + rowType, + java.util.Collections.emptyMap(), + FormatMetadataUtils.PARQUET_FIELD_ID_KEY); + + Map> metadata = + FormatMetadataUtils.readFieldMetadata(schemaBytes); + assertThat(metadata.get("geom")).containsEntry("paimon.type", "GEOMETRY(OGC:CRS84)"); + assertThat(metadata.get("geog")) + .containsEntry("paimon.type", "GEOGRAPHY(OGC:CRS84, spherical)"); + } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index 40e714291c67..cebe1acb2684 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -19,7 +19,11 @@ package org.apache.paimon.format.parquet; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FileFormatFactory; import org.apache.paimon.format.FormatMetadataUtils; @@ -30,6 +34,7 @@ import org.apache.paimon.format.SupportsWriterMetadata; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -68,6 +73,38 @@ public void testArrayBlobDescriptors() throws Exception { testArrayBlobDescriptorRoundTrip(); } + @Test + public void testGeospatialWkbRoundTrip() throws Exception { + byte[] pointWkb = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY()), + DataTypes.FIELD(1, "geog", DataTypes.GEOGRAPHY()), + DataTypes.FIELD(2, "geometries", DataTypes.ARRAY(DataTypes.GEOMETRY()))); + + write( + fileFormat().createWriterFactory(rowType), + file, + GenericRow.of(pointWkb, pointWkb, new GenericArray(new Object[] {pointWkb, null}))); + + try (RecordReader reader = + fileFormat() + .createReaderFactory(rowType, rowType, java.util.Collections.emptyList()) + .createReader( + new FormatReaderContext( + fileIO, file, fileIO.getFileSize(file), null, null))) { + InternalRow row = new InternalRowSerializer(rowType).copy(reader.readBatch().next()); + Assertions.assertThat(row.getBinary(0)).isEqualTo(pointWkb); + Assertions.assertThat(row.getBinary(1)).isEqualTo(pointWkb); + InternalArray geometries = row.getArray(2); + Assertions.assertThat(geometries.getBinary(0)).isEqualTo(pointWkb); + Assertions.assertThat(geometries.isNullAt(1)).isTrue(); + } + } + @Test public void testWriteMetadata() throws Exception { ParquetFileFormat format = diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index 2808cc535abb..5888f5739c07 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; @@ -28,6 +29,7 @@ import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Types; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -35,6 +37,7 @@ import static org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToPaimonRowType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToParquetMessageType; import static org.apache.paimon.types.DataTypesTest.assertThat; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; /** Test for {@link ParquetSchemaConverter}. */ @@ -141,4 +144,37 @@ public void testPaimonParquetSchemaConvert() { RowType rowType = convertToPaimonRowType(messageType); assertThat(ALL_TYPES).isEqualTo(rowType); } + + @Test + public void testGeospatialLogicalTypesRoundTrip() { + RowType expected = + new RowType( + Arrays.asList( + new DataField(0, "geom", DataTypes.GEOMETRY("EPSG:3857")), + new DataField( + 1, + "geog", + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.KARNEY) + .notNull()))); + + MessageType messageType = convertToParquetMessageType(expected); + Type geometry = messageType.getType("geom"); + Type geography = messageType.getType("geog"); + + Assertions.assertThat(geometry.asPrimitiveType().getPrimitiveTypeName()).isEqualTo(BINARY); + Assertions.assertThat(geometry.getLogicalTypeAnnotation()) + .isInstanceOf(LogicalTypeAnnotation.GeometryLogicalTypeAnnotation.class); + Assertions.assertThat( + ((LogicalTypeAnnotation.GeometryLogicalTypeAnnotation) + geometry.getLogicalTypeAnnotation()) + .getCrs()) + .isEqualTo("EPSG:3857"); + Assertions.assertThat( + ((LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) + geography.getLogicalTypeAnnotation()) + .getAlgorithm() + .name()) + .isEqualTo("KARNEY"); + assertThat(expected).isEqualTo(convertToPaimonRowType(messageType)); + } } diff --git a/paimon-python/pypaimon/casting/data_type_casts.py b/paimon-python/pypaimon/casting/data_type_casts.py index 819b9c115c00..fb812b9ab066 100644 --- a/paimon-python/pypaimon/casting/data_type_casts.py +++ b/paimon-python/pypaimon/casting/data_type_casts.py @@ -29,7 +29,7 @@ import pyarrow as pa from pypaimon.schema.data_types import (ArrayType, AtomicType, DataTypeParser, - MapType, MultisetType, + GeographyType, GeometryType, MapType, MultisetType, PyarrowFieldParser, RowType, VectorType) @@ -58,6 +58,8 @@ VECTOR = "VECTOR" VARIANT = "VARIANT" BLOB = "BLOB" +GEOMETRY = "GEOMETRY" +GEOGRAPHY = "GEOGRAPHY" # ---- Families ---------------------------------------------------------------- @@ -92,6 +94,10 @@ def _root(data_type) -> str: return MULTISET if isinstance(data_type, VectorType): return VECTOR + if isinstance(data_type, GeometryType): + return GEOMETRY + if isinstance(data_type, GeographyType): + return GEOGRAPHY if isinstance(data_type, AtomicType): t = data_type.type.upper() if t.startswith("DECIMAL") or t.startswith("NUMERIC") or t.startswith("DEC"): @@ -125,7 +131,7 @@ def _build_rules(): implicit = {} explicit = {} # Identity cast for every root. - for root in (PREDEFINED | CONSTRUCTED | {VARIANT, BLOB}): + for root in (PREDEFINED | CONSTRUCTED | {VARIANT, BLOB, GEOMETRY, GEOGRAPHY}): implicit[root] = {root} explicit[root] = set() @@ -168,6 +174,8 @@ def supports_cast(source_type, target_type, allow_explicit: bool = True) -> bool if source_type.nullable and not target_type.nullable and not allow_explicit: return False if source_root == target_root: + if source_root in {GEOMETRY, GEOGRAPHY}: + return _equals_ignore_nullable(source_type, target_type) if source_root in CONSTRUCTED: # A constructed type is only castable to an (ignoring outer # nullability) identical constructed type. Reshaping is done @@ -216,6 +224,8 @@ def can_execute_cast(source_type, target_type) -> bool: # Same root: identity, or a same-shape constructed type whose value is # rebuilt by the read path's field-id alignment rather than a value cast. if source_root == target_root: + if source_root in {GEOMETRY, GEOGRAPHY}: + return _equals_ignore_nullable(source_type, target_type) return True # Constructed -> character string is rendered by the read path's custom # ``_constructed_to_string_array`` (see DataFileBatchReader), not a cast. diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py index f01ee089aca1..f9cb92d8f132 100755 --- a/paimon-python/pypaimon/schema/data_types.py +++ b/paimon-python/pypaimon/schema/data_types.py @@ -106,6 +106,105 @@ def __str__(self) -> str: return "{}{}".format(self.type, null_suffix) +class EdgeAlgorithm(Enum): + SPHERICAL = "spherical" + VINCENTY = "vincenty" + THOMAS = "thomas" + ANDOYER = "andoyer" + KARNEY = "karney" + + @classmethod + def from_name(cls, name: str) -> "EdgeAlgorithm": + if name is None: + raise ValueError("Invalid edge interpolation algorithm: null") + try: + return cls(name.lower()) + except ValueError as exc: + raise ValueError( + "Invalid edge interpolation algorithm: {}".format(name)) from exc + + def __str__(self) -> str: + return self.value + + +@dataclass +class GeometryType(DataType): + crs: str + + DEFAULT_CRS = "OGC:CRS84" + + def __init__(self, crs: str = DEFAULT_CRS, nullable: bool = True): + super().__init__(nullable) + self.crs = self._validate_crs(crs) + + @staticmethod + def _validate_crs(crs: str) -> str: + if not crs: + raise ValueError("Invalid CRS: {}".format(crs)) + return crs + + @staticmethod + def _format_crs(crs: str) -> str: + if (crs[0].isdigit() + or any(character.isspace() or character in "<>().,'`" + for character in crs)): + return "'{}'".format(crs.replace("'", "''")) + return crs + + def __eq__(self, other): + return (isinstance(other, GeometryType) + and self.nullable == other.nullable + and self.crs.lower() == other.crs.lower()) + + def __hash__(self): + return hash((self.crs.upper(), self.nullable)) + + def to_dict(self) -> str: + return str(self) + + def __str__(self) -> str: + null_suffix = "" if self.nullable else " NOT NULL" + return "GEOMETRY({}){}".format( + self._format_crs(self.crs), null_suffix) + + +@dataclass +class GeographyType(DataType): + crs: str + algorithm: EdgeAlgorithm + + DEFAULT_CRS = "OGC:CRS84" + DEFAULT_ALGORITHM = EdgeAlgorithm.SPHERICAL + + def __init__(self, crs: str = DEFAULT_CRS, + algorithm: EdgeAlgorithm = DEFAULT_ALGORITHM, + nullable: bool = True): + super().__init__(nullable) + self.crs = GeometryType._validate_crs(crs) + if algorithm is None: + algorithm = self.DEFAULT_ALGORITHM + if not isinstance(algorithm, EdgeAlgorithm): + algorithm = EdgeAlgorithm.from_name(algorithm) + self.algorithm = algorithm + + def __eq__(self, other): + return (isinstance(other, GeographyType) + and self.nullable == other.nullable + and self.crs.lower() == other.crs.lower() + and self.algorithm == other.algorithm) + + def __hash__(self): + return hash((self.crs.upper(), self.algorithm, self.nullable)) + + def to_dict(self) -> str: + return str(self) + + def __str__(self) -> str: + null_suffix = "" if self.nullable else " NOT NULL" + return "GEOGRAPHY({}, {}){}".format( + GeometryType._format_crs(self.crs), self.algorithm, null_suffix) + + @dataclass class ArrayType(DataType): element: DataType @@ -500,6 +599,8 @@ class Keyword(Enum): TIMESTAMP = "TIMESTAMP" TIMESTAMP_LTZ = "TIMESTAMP_LTZ" VARIANT = "VARIANT" + GEOMETRY = "GEOMETRY" + GEOGRAPHY = "GEOGRAPHY" class DataTypeParser: @@ -515,7 +616,7 @@ def parse_nullability(type_string: str) -> bool: @staticmethod def parse_atomic_type_sql_string(type_string: str) -> DataType: nullable = DataTypeParser.parse_nullability(type_string) - type_upper = type_string.upper().strip() + type_text = type_string.strip() # Strip the trailing nullability suffix so it is stored only in # ``nullable``, not baked into the atomic type string. The space-split # branch below drops it for plain types ("BIGINT NOT NULL"), but a @@ -523,10 +624,35 @@ def parse_atomic_type_sql_string(type_string: str) -> DataType: # takes the paren branch and would otherwise keep the suffix in # ``AtomicType.type`` -- doubling it on the next ``to_dict()``. for suffix in (" NOT NULL", " NULL"): - if type_upper.endswith(suffix): - type_upper = type_upper[: -len(suffix)].rstrip() + if type_text.upper().endswith(suffix): + type_text = type_text[: -len(suffix)].rstrip() break + geometry_match = re.fullmatch( + r"GEOMETRY(?:\(\s*(?:'((?:''|[^'])*)'|([^)]*?))\s*\))?", + type_text, re.IGNORECASE) + if geometry_match: + quoted_crs, raw_crs = geometry_match.groups() + crs = quoted_crs.replace("''", "'") if quoted_crs is not None else raw_crs + return GeometryType(crs or GeometryType.DEFAULT_CRS, nullable) + if type_text.upper().startswith("GEOMETRY"): + raise ValueError("Invalid geometry type: {}".format(type_text)) + + geography_match = re.fullmatch( + r"GEOGRAPHY(?:\(\s*(?:'((?:''|[^'])*)'|([^,]*?))\s*" + r"(?:,\s*([^(),]+?)\s*)?\))?", type_text, re.IGNORECASE) + if geography_match: + quoted_crs, raw_crs, raw_algorithm = geography_match.groups() + crs = quoted_crs.replace("''", "'") if quoted_crs is not None else raw_crs + crs = crs or GeographyType.DEFAULT_CRS + algorithm = EdgeAlgorithm.from_name( + raw_algorithm or GeographyType.DEFAULT_ALGORITHM.value) + return GeographyType(crs, algorithm, nullable) + if type_text.upper().startswith("GEOGRAPHY"): + raise ValueError("Invalid geography type: {}".format(type_text)) + + type_upper = type_text.upper() + if "(" in type_upper: base_type = type_upper.split("(")[0] elif " " in type_upper: @@ -657,9 +783,30 @@ def is_variant_struct(pa_type: pyarrow.StructType) -> bool: class PyarrowFieldParser: + @staticmethod + def _field_metadata(data_type: DataType, + description: Optional[str] = None) -> Dict[bytes, bytes]: + metadata = {} + if isinstance(data_type, (GeometryType, GeographyType)): + metadata[b'paimon.type'] = str(data_type).encode('utf-8') + if description: + metadata[b'description'] = description.encode('utf-8') + return metadata + + @staticmethod + def _from_paimon_named_type(name: str, data_type: DataType, + description: Optional[str] = None) -> pyarrow.Field: + return pyarrow.field( + name, + PyarrowFieldParser.from_paimon_type(data_type), + nullable=data_type.nullable, + metadata=PyarrowFieldParser._field_metadata(data_type, description)) + @staticmethod def from_paimon_type(data_type: DataType) -> pyarrow.DataType: # Based on Paimon DataTypes Doc: https://paimon.apache.org/docs/master/concepts/data-types/ + if isinstance(data_type, (GeometryType, GeographyType)): + return pyarrow.binary() if isinstance(data_type, AtomicType): type_name = data_type.type.upper() if type_name == 'TINYINT': @@ -720,42 +867,32 @@ def from_paimon_type(data_type: DataType) -> pyarrow.DataType: if type_name.startswith('TIME'): return pyarrow.time32('ms') elif isinstance(data_type, ArrayType): - element_type = PyarrowFieldParser.from_paimon_type(data_type.element) return pyarrow.list_( - pyarrow.field( - "item", - element_type, - nullable=data_type.element.nullable, - ) + PyarrowFieldParser._from_paimon_named_type("item", data_type.element) ) elif isinstance(data_type, VectorType): return pyarrow.list_(PyarrowFieldParser.from_paimon_type(data_type.element), data_type.length) elif isinstance(data_type, MapType): - key_type = PyarrowFieldParser.from_paimon_type(data_type.key) - value_type = PyarrowFieldParser.from_paimon_type(data_type.value) return pyarrow.map_( - pyarrow.field("key", key_type, nullable=False), pyarrow.field( - "value", - value_type, - nullable=data_type.value.nullable, - ), + "key", + PyarrowFieldParser.from_paimon_type(data_type.key), + nullable=False, + metadata=PyarrowFieldParser._field_metadata(data_type.key)), + PyarrowFieldParser._from_paimon_named_type( + "value", data_type.value), ) elif isinstance(data_type, RowType): pa_fields = [] for field in data_type.fields: - pa_field_type = PyarrowFieldParser.from_paimon_type(field.type) - pa_fields.append(pyarrow.field(field.name, pa_field_type, nullable=field.type.nullable)) + pa_fields.append(PyarrowFieldParser.from_paimon_field(field)) return pyarrow.struct(pa_fields) raise ValueError("Unsupported data type: {}".format(data_type)) @staticmethod def from_paimon_field(data_field: DataField) -> pyarrow.Field: - pa_field_type = PyarrowFieldParser.from_paimon_type(data_field.type) - metadata = {} - if data_field.description: - metadata[b'description'] = data_field.description.encode('utf-8') - return pyarrow.field(data_field.name, pa_field_type, nullable=data_field.type.nullable, metadata=metadata) + return PyarrowFieldParser._from_paimon_named_type( + data_field.name, data_field.type, data_field.description) @staticmethod def from_paimon_schema(data_fields: List[DataField]): @@ -806,19 +943,16 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: type_name = 'TIME(0)' elif types.is_fixed_size_list(pa_type): pa_type: pyarrow.FixedSizeListType - element_type = PyarrowFieldParser.to_paimon_type(pa_type.value_type, pa_type.value_field.nullable) + element_type = PyarrowFieldParser._to_paimon_field_type(pa_type.value_field) return VectorType(nullable, element_type, pa_type.list_size) elif types.is_list(pa_type) or types.is_large_list(pa_type): pa_type: pyarrow.ListType - element_type = PyarrowFieldParser.to_paimon_type( - pa_type.value_type, pa_type.value_field.nullable) + element_type = PyarrowFieldParser._to_paimon_field_type(pa_type.value_field) return ArrayType(nullable, element_type) elif types.is_map(pa_type): pa_type: pyarrow.MapType - key_type = PyarrowFieldParser.to_paimon_type( - pa_type.key_type, pa_type.key_field.nullable) - value_type = PyarrowFieldParser.to_paimon_type( - pa_type.item_type, pa_type.item_field.nullable) + key_type = PyarrowFieldParser._to_paimon_field_type(pa_type.key_field) + value_type = PyarrowFieldParser._to_paimon_field_type(pa_type.item_field) return MapType(nullable, key_type, value_type) elif types.is_struct(pa_type) and is_variant_struct(pa_type): return AtomicType('VARIANT', nullable) @@ -826,7 +960,7 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: pa_type: pyarrow.StructType fields = [] for i, pa_field in enumerate(pa_type): - field_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) + field_type = PyarrowFieldParser._to_paimon_field_type(pa_field) fields.append(DataField( id=i, name=pa_field.name, @@ -839,7 +973,7 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: @staticmethod def to_paimon_field(field_idx: int, pa_field: pyarrow.Field) -> DataField: - data_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) + data_type = PyarrowFieldParser._to_paimon_field_type(pa_field) description = pa_field.metadata.get(b'description', b'').decode('utf-8') \ if pa_field.metadata and b'description' in pa_field.metadata else None return DataField( @@ -849,6 +983,23 @@ def to_paimon_field(field_idx: int, pa_field: pyarrow.Field) -> DataField: description=description ) + @staticmethod + def _to_paimon_field_type(pa_field: pyarrow.Field) -> DataType: + if pa_field.metadata and b'paimon.type' in pa_field.metadata: + data_type = DataTypeParser.parse_atomic_type_sql_string( + pa_field.metadata[b'paimon.type'].decode('utf-8')) + if (isinstance(data_type, (GeometryType, GeographyType)) + and not (types.is_binary(pa_field.type) + or types.is_fixed_size_binary(pa_field.type))): + raise ValueError( + "Geospatial field metadata requires a binary Arrow type: {}" + .format(pa_field)) + data_type.nullable = pa_field.nullable + return data_type + else: + return PyarrowFieldParser.to_paimon_type( + pa_field.type, pa_field.nullable) + @staticmethod def to_paimon_schema(pa_schema: pyarrow.Schema) -> List[DataField]: # Convert PyArrow schema to Paimon fields, assigning globally-unique ids: @@ -861,7 +1012,7 @@ def to_paimon_schema(pa_schema: pyarrow.Schema) -> List[DataField]: for pa_field in pa_schema: pa_field: pyarrow.Field top_id = field_id.increment_and_get() - data_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) + data_type = PyarrowFieldParser._to_paimon_field_type(pa_field) data_type = reassign_field_id(data_type, field_id) description = pa_field.metadata.get(b'description', b'').decode('utf-8') \ if pa_field.metadata and b'description' in pa_field.metadata else None diff --git a/paimon-python/pypaimon/schema/schema_manager.py b/paimon-python/pypaimon/schema/schema_manager.py index 928e177a8b8b..77b8fbc898a7 100644 --- a/paimon-python/pypaimon/schema/schema_manager.py +++ b/paimon-python/pypaimon/schema/schema_manager.py @@ -28,7 +28,8 @@ remove_dropped_directive_options) from pypaimon.casting.data_type_casts import can_execute_cast, supports_cast from pypaimon.schema.data_types import (ArrayType, AtomicInteger, DataField, - DataType, MapType, MultisetType, RowType, + DataType, GeographyType, GeometryType, + MapType, MultisetType, RowType, is_array_blob_type, is_blob_file_field, is_blob_file_type, is_blob_type, is_map_blob_type, reassign_field_id) @@ -467,6 +468,91 @@ def _validate_options(options: dict): ) +def _contains_geospatial_type(data_type: DataType) -> bool: + if isinstance(data_type, (GeometryType, GeographyType)): + return True + if isinstance(data_type, (ArrayType, MultisetType)): + return _contains_geospatial_type(data_type.element) + if isinstance(data_type, MapType): + return (_contains_geospatial_type(data_type.key) + or _contains_geospatial_type(data_type.value)) + if isinstance(data_type, RowType): + return any(_contains_geospatial_type(field.type) + for field in data_type.fields) + return False + + +def _validate_geospatial_fields( + fields: List[DataField], + options: dict, + primary_keys: List[str], + partition_keys: List[str], +): + if not any(_contains_geospatial_type(field.type) for field in fields): + return + + options = options or {} + core_options = CoreOptions(Options(options)) + file_format = (core_options.file_format('parquet') or '').lower() + if file_format != 'parquet': + raise ValueError( + "Geometry and geography columns require 'file.format'='parquet', " + "but was '{}'.".format(file_format)) + + per_level = core_options.file_format_per_level({}) or {} + if isinstance(per_level, str): + per_level = dict( + part.split(':', 1) for part in per_level.split(',') if part) + for level, level_format in per_level.items(): + if str(level_format).lower() != 'parquet': + raise ValueError( + "Geometry and geography columns require parquet at every level, " + "but 'file.format.per.level' contains '{}:{}'." + .format(level, level_format)) + + changelog_format = core_options.changelog_file_format() + if changelog_format and changelog_format.lower() != 'parquet': + raise ValueError( + "Geometry and geography columns require 'changelog-file.format' " + "to be parquet, but was '{}'.".format(changelog_format)) + + iceberg_storage = options.get('metadata.iceberg.storage', 'disabled').lower() + if (iceberg_storage != 'disabled' + and str(options.get('metadata.iceberg.format-version', '2')) != '3'): + raise ValueError( + "Geometry and geography columns require " + "'metadata.iceberg.format-version'='3' when Iceberg metadata is enabled.") + + geospatial_fields = { + field.name for field in fields + if isinstance(field.type, (GeometryType, GeographyType)) + } + primary_geo = geospatial_fields.intersection(primary_keys) + if primary_geo: + raise ValueError( + "Geometry and geography columns cannot be primary keys: {}." + .format(sorted(primary_geo))) + partition_geo = geospatial_fields.intersection(partition_keys) + if partition_geo: + raise ValueError( + "Geometry and geography columns cannot be partition keys: {}." + .format(sorted(partition_geo))) + + bucket_value = options.get(CoreOptions.BUCKET_KEY.key(), '') + bucket_keys = {key.strip() for key in bucket_value.split(',') if key.strip()} + bucket_geo = geospatial_fields.intersection(bucket_keys) + if bucket_geo: + raise ValueError( + "Geometry and geography columns cannot be bucket keys: {}." + .format(sorted(bucket_geo))) + + sequence_geo = geospatial_fields.intersection(core_options.sequence_field()) + if sequence_geo: + raise ValueError( + "Geometry and geography columns cannot be sequence fields: {}." + .format(sorted(sequence_geo))) + + def _contains_blob_type(data_type: DataType) -> bool: if is_blob_type(data_type): return True @@ -658,6 +744,12 @@ def create_table(self, schema: Schema) -> TableSchema: def commit(self, new_schema: TableSchema) -> bool: _validate_options(new_schema.options) + _validate_geospatial_fields( + new_schema.fields, + new_schema.options, + new_schema.primary_keys, + new_schema.partition_keys, + ) _validate_blob_fields( new_schema.fields, new_schema.options, diff --git a/paimon-python/pypaimon/tests/geospatial_type_test.py b/paimon-python/pypaimon/tests/geospatial_type_test.py new file mode 100644 index 000000000000..d342697f1892 --- /dev/null +++ b/paimon-python/pypaimon/tests/geospatial_type_test.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pyarrow +import pyarrow.parquet as parquet +import pytest + +from pypaimon.casting.data_type_casts import supports_cast +from pypaimon.schema.data_types import DataField +from pypaimon.schema.data_types import DataTypeParser +from pypaimon.schema.data_types import EdgeAlgorithm +from pypaimon.schema.data_types import GeographyType +from pypaimon.schema.data_types import GeometryType +from pypaimon.schema.data_types import ArrayType +from pypaimon.schema.data_types import PyarrowFieldParser +from pypaimon.schema.data_types import RowType +from pypaimon.schema.schema_manager import _validate_geospatial_fields +from pypaimon.write.writer.data_writer import DataWriter + + +def test_iceberg_compatible_type_round_trip(): + geometry = DataTypeParser.parse_data_type("GEOMETRY(ogc:crs84) NOT NULL") + geography = DataTypeParser.parse_data_type("GEOGRAPHY(EPSG:4326, karney)") + + assert geometry == GeometryType("OGC:CRS84", nullable=False) + assert geography == GeographyType("EPSG:4326", EdgeAlgorithm.KARNEY) + assert DataTypeParser.parse_data_type(geometry.to_dict()) == geometry + assert DataTypeParser.parse_data_type(geography.to_dict()) == geography + custom = GeometryType("custom, crs's definition") + assert DataTypeParser.parse_data_type(custom.to_dict()) == custom + + +def test_defaults_and_invalid_parameters(): + assert str(DataTypeParser.parse_data_type("GEOMETRY")) == "GEOMETRY(OGC:CRS84)" + assert str(DataTypeParser.parse_data_type("GEOGRAPHY")) == \ + "GEOGRAPHY(OGC:CRS84, spherical)" + + with pytest.raises(ValueError, match="Invalid edge interpolation algorithm"): + DataTypeParser.parse_data_type("GEOGRAPHY(OGC:CRS84, rhumb)") + with pytest.raises(ValueError, match="Invalid geometry type"): + DataTypeParser.parse_data_type("GEOMETRY(EPSG:4326) trailing") + + +def test_pyarrow_uses_wkb_binary_and_preserves_type_metadata(): + fields = [ + DataField(0, "geom", GeometryType()), + DataField(1, "geog", GeographyType("EPSG:4326", EdgeAlgorithm.VINCENTY, + nullable=False)), + ] + + arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) + assert arrow_schema.field("geom").type == pyarrow.binary() + assert arrow_schema.field("geog").type == pyarrow.binary() + assert arrow_schema.field("geom").metadata[b'paimon.type'] == \ + b'GEOMETRY(OGC:CRS84)' + assert PyarrowFieldParser.to_paimon_schema(arrow_schema) == fields + + +def test_nested_arrow_and_parquet_wkb_round_trip(tmp_path): + fields = [ + DataField(0, "nested", RowType(True, [ + DataField(1, "geom", GeometryType(nullable=False)), + ])), + DataField(2, "geographies", ArrayType(True, GeographyType())), + ] + schema = PyarrowFieldParser.from_paimon_schema(fields) + assert schema.field("nested").type.field("geom").metadata[b'paimon.type'] == \ + b'GEOMETRY(OGC:CRS84) NOT NULL' + assert schema.field("geographies").type.value_field.metadata[b'paimon.type'] == \ + b'GEOGRAPHY(OGC:CRS84, spherical)' + assert PyarrowFieldParser.to_paimon_schema(schema) == fields + + point_wkb = bytes.fromhex( + "0101000000000000000000f03f0000000000000040") + table = pyarrow.Table.from_arrays([ + pyarrow.array([{"geom": point_wkb}], type=schema.field("nested").type), + pyarrow.array([[point_wkb]], type=schema.field("geographies").type), + ], schema=schema) + path = tmp_path / "geo.parquet" + parquet.write_table(table, path) + restored = parquet.read_table(path) + assert restored.to_pylist() == table.to_pylist() + assert PyarrowFieldParser.to_paimon_schema(restored.schema) == fields + + +def test_geospatial_cast_stats_and_schema_validation(): + assert supports_cast(GeometryType("OGC:CRS84"), + GeometryType("ogc:crs84", nullable=False)) + assert not supports_cast(GeometryType(), GeometryType("EPSG:3857")) + assert not supports_cast( + GeographyType(algorithm=EdgeAlgorithm.SPHERICAL), + GeographyType(algorithm=EdgeAlgorithm.KARNEY)) + + fields = [DataField(0, "geom", GeometryType())] + values = pyarrow.table({"geom": [b'\x02', None, b'\x01']}) + stats_fields = DataWriter._resolve_stats_fields(values.schema, fields) + assert stats_fields == fields + stats = DataWriter._get_column_stats(values, "geom", GeometryType()) + assert stats == {"min_values": None, "max_values": None, "null_counts": 1} + + _validate_geospatial_fields(fields, {}, [], []) + with pytest.raises(ValueError, match="file.format"): + _validate_geospatial_fields(fields, {"file.format": "orc"}, [], []) + with pytest.raises(ValueError, match="primary keys"): + _validate_geospatial_fields(fields, {}, ["geom"], []) + with pytest.raises(ValueError, match="format-version"): + _validate_geospatial_fields( + fields, {"metadata.iceberg.storage": "table-location"}, [], []) + _validate_geospatial_fields( + fields, + {"metadata.iceberg.storage": "table-location", + "metadata.iceberg.format-version": "3"}, + [], []) diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py index e34127ddaf00..49aa2e8de057 100644 --- a/paimon-python/pypaimon/write/writer/data_writer.py +++ b/paimon-python/pypaimon/write/writer/data_writer.py @@ -26,7 +26,7 @@ from pypaimon.data.timestamp import Timestamp from pypaimon.manifest.schema.data_file_meta import DataFileMeta from pypaimon.manifest.schema.simple_stats import SimpleStats -from pypaimon.schema.data_types import PyarrowFieldParser +from pypaimon.schema.data_types import GeographyType, GeometryType, PyarrowFieldParser from pypaimon.table.bucket_mode import BucketMode from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.writer.mosaic_writer_options import create_mosaic_writer_options @@ -273,12 +273,15 @@ def _write_data_to_file(self, data: pa.Table): # key stats & value stats value_stats_enabled = self.options.metadata_stats_enabled() if value_stats_enabled: - stats_fields = self.table.fields if self.table.is_primary_key_table \ - else PyarrowFieldParser.to_paimon_schema(data.schema) + if self.table.is_primary_key_table: + stats_fields = self.table.fields + else: + stats_fields = self._resolve_stats_fields( + data.schema, self.table.fields) else: stats_fields = self.table.trimmed_primary_keys_fields column_stats = { - field.name: self._get_column_stats(data, field.name) + field.name: self._get_column_stats(data, field.name, field.type) for field in stats_fields } key_fields = self.trimmed_primary_keys_fields @@ -451,7 +454,7 @@ def _collect_value_stats(self, data: pa.Table, fields: List, if column_stats is None or not column_stats: column_stats = { - field.name: self._get_column_stats(data, field.name) + field.name: self._get_column_stats(data, field.name, field.type) for field in fields } @@ -466,8 +469,24 @@ def _collect_value_stats(self, data: pa.Table, fields: List, ) @staticmethod - def _get_column_stats(record_batch: pa.RecordBatch, column_name: str) -> Dict: + def _resolve_stats_fields(arrow_schema, table_fields: List) -> List: + inferred_fields = PyarrowFieldParser.to_paimon_schema(arrow_schema) + table_fields_by_name = {field.name: field for field in table_fields} + return [ + table_fields_by_name.get(field.name, field) + for field in inferred_fields + ] + + @staticmethod + def _get_column_stats(record_batch: pa.RecordBatch, column_name: str, + data_type=None) -> Dict: column_array = record_batch.column(column_name) + if isinstance(data_type, (GeometryType, GeographyType)): + return { + "min_values": None, + "max_values": None, + "null_counts": column_array.null_count, + } if column_array.null_count == len(column_array): return { "min_values": None, diff --git a/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala index f9e7ba1c0b8d..12dddf4db571 100644 --- a/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala +++ b/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -381,6 +381,46 @@ class Spark4Shim extends SparkShim { override def SparkVariantType(): org.apache.spark.sql.types.DataType = DataTypes.VariantType + override def toPaimonGeometry(o: Object): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeometry(row: InternalRow, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeometry(array: ArrayData, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(o: Object): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(row: InternalRow, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(array: ArrayData, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toSparkGeometry(wkb: Array[Byte], crs: String): Object = unsupportedGeospatial() + + override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = + unsupportedGeospatial() + + override def isSparkGeometryType(dataType: org.apache.spark.sql.types.DataType): Boolean = false + + override def isSparkGeographyType(dataType: org.apache.spark.sql.types.DataType): Boolean = false + + override def SparkGeometryType(crs: String): org.apache.spark.sql.types.DataType = + unsupportedGeospatial() + + override def SparkGeographyType( + crs: String, + algorithm: String): org.apache.spark.sql.types.DataType = unsupportedGeospatial() + + override def sparkGeometryCrs(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + override def sparkGeographyCrs(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + override def sparkGeographyAlgorithm(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + private def unsupportedGeospatial[T](): T = + throw new UnsupportedOperationException("Geometry and geography require Spark 4.1 or later") + // SQL UDFs (CREATE FUNCTION ... RETURN ...). override def rewritePaimonSQLFunctionCommands(spark: SparkSession): Rule[LogicalPlan] = org.apache.spark.sql.catalyst.parser.extensions.RewritePaimonSQLFunctionCommands(spark) diff --git a/paimon-spark/paimon-spark-4.1/src/test/java/org/apache/paimon/spark/GeospatialTypeTest.java b/paimon-spark/paimon-spark-4.1/src/test/java/org/apache/paimon/spark/GeospatialTypeTest.java new file mode 100644 index 000000000000..5d46832e5503 --- /dev/null +++ b/paimon-spark/paimon-spark-4.1/src/test/java/org/apache/paimon/spark/GeospatialTypeTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.spark.data.SparkInternalRow; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; +import org.apache.paimon.types.RowType; + +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.catalyst.util.STUtils; +import org.apache.spark.sql.types.Geography; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.Geometry; +import org.apache.spark.sql.types.GeometryType; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests Spark 4.1 geometry and geography interoperability. */ +class GeospatialTypeTest { + + private static final byte[] POINT_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0x40}; + + @Test + void testTypeRoundTrip() { + RowType paimonType = + DataTypes.ROW( + DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY()), + DataTypes.FIELD( + 1, + "geog", + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.SPHERICAL))); + + StructType sparkType = SparkTypeUtils.fromPaimonRowType(paimonType); + assertThat(sparkType.apply("geom").dataType()).isInstanceOf(GeometryType.class); + assertThat(((GeometryType) sparkType.apply("geom").dataType()).crs()) + .isEqualTo("OGC:CRS84"); + assertThat(sparkType.apply("geog").dataType()).isInstanceOf(GeographyType.class); + assertThat(((GeographyType) sparkType.apply("geog").dataType()).crs()) + .isEqualTo("OGC:CRS84"); + assertThat(((GeographyType) sparkType.apply("geog").dataType()).algorithm().toString()) + .isEqualTo("SPHERICAL"); + assertThat(SparkTypeUtils.toPaimonType(sparkType)).isEqualTo(paimonType); + + assertThatThrownBy( + () -> + SparkTypeUtils.fromPaimonType( + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.KARNEY))) + .hasMessageContaining("karney"); + } + + @Test + void testWkbReadWriteRoundTrip() { + RowType paimonType = + DataTypes.ROW( + DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY()), + DataTypes.FIELD(1, "geog", DataTypes.GEOGRAPHY())); + StructType sparkType = SparkTypeUtils.fromPaimonRowType(paimonType); + + SparkInternalRow sparkRow = + SparkInternalRow.create(paimonType).replace(GenericRow.of(POINT_WKB, POINT_WKB)); + assertThat(STUtils.stAsBinary(sparkRow.getGeometry(0))).isEqualTo(POINT_WKB); + assertThat(STUtils.stSrid(sparkRow.getGeometry(0))).isEqualTo(4326); + assertThat(STUtils.stAsBinary(sparkRow.getGeography(1))).isEqualTo(POINT_WKB); + assertThat(STUtils.stSrid(sparkRow.getGeography(1))).isEqualTo(4326); + + SparkInternalRowWrapper internalWrapper = + new SparkInternalRowWrapper(sparkType, 2).replace(sparkRow); + assertThat(internalWrapper.getBinary(0)).isEqualTo(POINT_WKB); + assertThat(internalWrapper.getBinary(1)).isEqualTo(POINT_WKB); + + SparkRow externalWrapper = + new SparkRow( + paimonType, + RowFactory.create( + Geometry.fromWKB(POINT_WKB, 4326), + Geography.fromWKB(POINT_WKB, 4326))); + assertThat(externalWrapper.getBinary(0)).isEqualTo(POINT_WKB); + assertThat(externalWrapper.getBinary(1)).isEqualTo(POINT_WKB); + } +} diff --git a/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala b/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala new file mode 100644 index 000000000000..94e4c70c42db --- /dev/null +++ b/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.types.{DataTypes, EdgeAlgorithm} + +import org.apache.spark.sql.Row + +/** Tests Spark 4.1 SQL interoperability with Paimon geospatial columns. */ +class GeospatialTypeSQLTest extends PaimonSparkTestBase { + + test("Spark SQL reads and writes native geospatial values") { + withTable("t") { + sql(""" + |CREATE TABLE t ( + | id INT, + | geom GEOMETRY(4326), + | geog GEOGRAPHY(4326) + |) TBLPROPERTIES ('file.format' = 'parquet') + |""".stripMargin) + + sql(""" + |INSERT INTO t VALUES + | (1, + | ST_SetSrid( + | ST_GeomFromWKB(unhex('0101000000000000000000F03F0000000000000040')), + | 4326), + | ST_GeogFromWKB(unhex('010100000000000000000008400000000000001040'))), + | (2, NULL, + | ST_GeogFromWKB(unhex('0101000000000000000000F03F0000000000000040'))) + |""".stripMargin) + + checkAnswer( + sql(""" + |SELECT id, + | hex(ST_AsBinary(geom)), ST_Srid(geom), + | hex(ST_AsBinary(geog)), ST_Srid(geog) + |FROM t ORDER BY id + |""".stripMargin), + Seq( + Row( + 1, + "0101000000000000000000F03F0000000000000040", + 4326, + "010100000000000000000008400000000000001040", + 4326), + Row(2, null, null, "0101000000000000000000F03F0000000000000040", 4326) + ) + ) + + val fields = loadTable("t").schema().fields() + assert(fields.get(1).`type`() == DataTypes.GEOMETRY("OGC:CRS84")) + assert( + fields.get(2).`type`() == + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.SPHERICAL)) + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java index f522994937cf..46c48833d0bc 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.util.ArrayData; import org.apache.spark.sql.catalyst.util.MapData; +import org.apache.spark.sql.paimon.shims.SparkShimLoader; import org.apache.spark.sql.types.BinaryType; import org.apache.spark.sql.types.BooleanType; import org.apache.spark.sql.types.ByteType; @@ -256,6 +257,21 @@ public Object get(int ordinal, org.apache.spark.sql.types.DataType dataType) { if (dataType instanceof UserDefinedType) { return get(ordinal, ((UserDefinedType) dataType).sqlType()); } + if (SparkShimLoader.shim().isSparkGeometryType(dataType)) { + org.apache.paimon.types.GeometryType geometryType = + (org.apache.paimon.types.GeometryType) rowType.getTypeAt(ordinal); + return SparkShimLoader.shim() + .toSparkGeometry(row.getBinary(ordinal), geometryType.getCrs()); + } + if (SparkShimLoader.shim().isSparkGeographyType(dataType)) { + org.apache.paimon.types.GeographyType geographyType = + (org.apache.paimon.types.GeographyType) rowType.getTypeAt(ordinal); + return SparkShimLoader.shim() + .toSparkGeography( + row.getBinary(ordinal), + geographyType.getCrs(), + geographyType.getAlgorithm().toString()); + } throw new UnsupportedOperationException("Unsupported data type " + dataType.simpleString()); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java index 505df4e91207..d34a9c7da7d5 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java @@ -30,6 +30,8 @@ import org.apache.paimon.spark.util.shim.TypeUtils; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.MultisetType; @@ -40,6 +42,7 @@ import org.apache.spark.sql.catalyst.util.ArrayData; import org.apache.spark.sql.catalyst.util.DateTimeUtils; import org.apache.spark.sql.catalyst.util.MapData; +import org.apache.spark.sql.paimon.shims.SparkShimLoader; import org.apache.spark.sql.types.Decimal; import org.apache.spark.unsafe.types.UTF8String; @@ -70,6 +73,16 @@ public static Object fromPaimon(Object o, DataType type) { return fromPaimon((InternalRow) o, (RowType) type); case BLOB: return ((Blob) o).toData(); + case GEOMETRY: + return SparkShimLoader.shim() + .toSparkGeometry((byte[]) o, ((GeometryType) type).getCrs()); + case GEOGRAPHY: + GeographyType geographyType = (GeographyType) type; + return SparkShimLoader.shim() + .toSparkGeography( + (byte[]) o, + geographyType.getCrs(), + geographyType.getAlgorithm().toString()); default: return o; } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java index 782383981694..8e24dcf712cc 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRowWrapper.java @@ -241,6 +241,13 @@ public byte[] getBinary(int pos) { if (actualPos == -1 || internalRow.isNullAt(actualPos)) { return null; } + DataType dataType = tableSchema.fields()[pos].dataType(); + if (SparkShimLoader.shim().isSparkGeometryType(dataType)) { + return SparkShimLoader.shim().toPaimonGeometry(internalRow, actualPos); + } + if (SparkShimLoader.shim().isSparkGeographyType(dataType)) { + return SparkShimLoader.shim().toPaimonGeography(internalRow, actualPos); + } return internalRow.getBinary(actualPos); } @@ -462,6 +469,12 @@ public Timestamp getTimestamp(int pos, int precision) { @Override public byte[] getBinary(int pos) { + if (SparkShimLoader.shim().isSparkGeometryType(elementType)) { + return SparkShimLoader.shim().toPaimonGeometry(arrayData, pos); + } + if (SparkShimLoader.shim().isSparkGeographyType(elementType)) { + return SparkShimLoader.shim().toPaimonGeography(arrayData, pos); + } return arrayData.getBinary(pos); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java index 643fc016d940..a8e6ae143baa 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java @@ -33,6 +33,8 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DateType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; @@ -160,6 +162,12 @@ public Timestamp getTimestamp(int i, int precision) { @Override public byte[] getBinary(int i) { + if (type.getTypeAt(i) instanceof GeometryType) { + return SparkShimLoader.shim().toPaimonGeometry(row.getAs(i)); + } + if (type.getTypeAt(i) instanceof GeographyType) { + return SparkShimLoader.shim().toPaimonGeography(row.getAs(i)); + } return row.getAs(i); } @@ -345,6 +353,12 @@ public Timestamp getTimestamp(int i, int precision) { @Override public byte[] getBinary(int i) { + if (elementType instanceof GeometryType) { + return SparkShimLoader.shim().toPaimonGeometry(getAs(i)); + } + if (elementType instanceof GeographyType) { + return SparkShimLoader.shim().toPaimonGeography(getAs(i)); + } return getAs(i); } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java index cd7de8f53539..7ced9f29ffee 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTypeUtils.java @@ -32,6 +32,8 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.GeographyType; +import org.apache.paimon.types.GeometryType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; @@ -261,6 +263,18 @@ public DataType visit(VariantType variantType) { return SparkShimLoader.shim().SparkVariantType(); } + @Override + public DataType visit(GeometryType geometryType) { + return SparkShimLoader.shim().SparkGeometryType(geometryType.getCrs()); + } + + @Override + public DataType visit(GeographyType geographyType) { + return SparkShimLoader.shim() + .SparkGeographyType( + geographyType.getCrs(), geographyType.getAlgorithm().toString()); + } + @Override public DataType visit(ArrayType arrayType) { org.apache.paimon.types.DataType elementType = arrayType.getElementType(); @@ -449,6 +463,13 @@ public org.apache.paimon.types.DataType atomic(DataType atomic) { return new TimestampType(); } else if (SparkShimLoader.shim().isSparkVariantType(atomic)) { return new VariantType(); + } else if (SparkShimLoader.shim().isSparkGeometryType(atomic)) { + return new GeometryType(SparkShimLoader.shim().sparkGeometryCrs(atomic)); + } else if (SparkShimLoader.shim().isSparkGeographyType(atomic)) { + return new GeographyType( + SparkShimLoader.shim().sparkGeographyCrs(atomic), + org.apache.paimon.types.EdgeAlgorithm.fromName( + SparkShimLoader.shim().sparkGeographyAlgorithm(atomic))); } throw new UnsupportedOperationException( diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala index df21168bd343..690c20b85b74 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala @@ -270,6 +270,37 @@ trait SparkShim { def SparkVariantType(): org.apache.spark.sql.types.DataType + // Geometry and geography are available in Spark 4.1 and later. + def toPaimonGeometry(o: Object): Array[Byte] + + def toPaimonGeometry(row: InternalRow, pos: Int): Array[Byte] + + def toPaimonGeometry(array: ArrayData, pos: Int): Array[Byte] + + def toPaimonGeography(o: Object): Array[Byte] + + def toPaimonGeography(row: InternalRow, pos: Int): Array[Byte] + + def toPaimonGeography(array: ArrayData, pos: Int): Array[Byte] + + def toSparkGeometry(wkb: Array[Byte], crs: String): Object + + def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object + + def isSparkGeometryType(dataType: org.apache.spark.sql.types.DataType): Boolean + + def isSparkGeographyType(dataType: org.apache.spark.sql.types.DataType): Boolean + + def SparkGeometryType(crs: String): org.apache.spark.sql.types.DataType + + def SparkGeographyType(crs: String, algorithm: String): org.apache.spark.sql.types.DataType + + def sparkGeometryCrs(dataType: org.apache.spark.sql.types.DataType): String + + def sparkGeographyCrs(dataType: org.apache.spark.sql.types.DataType): String + + def sparkGeographyAlgorithm(dataType: org.apache.spark.sql.types.DataType): String + // SQL UDFs (`CREATE FUNCTION ... RETURN ...`) are Spark 4.0+; the spark3 shim no-ops these. /** Parser-stage rule rewriting a Paimon-catalog `CreateUserDefinedFunction` into a create command. */ diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala index 5ffbf6a14530..b568ce16c3c4 100644 --- a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala @@ -355,6 +355,46 @@ class Spark3Shim extends SparkShim { override def SparkVariantType(): org.apache.spark.sql.types.DataType = throw new UnsupportedOperationException() + override def toPaimonGeometry(o: Object): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeometry(row: InternalRow, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeometry(array: ArrayData, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(o: Object): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(row: InternalRow, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toPaimonGeography(array: ArrayData, pos: Int): Array[Byte] = unsupportedGeospatial() + + override def toSparkGeometry(wkb: Array[Byte], crs: String): Object = unsupportedGeospatial() + + override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = + unsupportedGeospatial() + + override def isSparkGeometryType(dataType: org.apache.spark.sql.types.DataType): Boolean = false + + override def isSparkGeographyType(dataType: org.apache.spark.sql.types.DataType): Boolean = false + + override def SparkGeometryType(crs: String): org.apache.spark.sql.types.DataType = + unsupportedGeospatial() + + override def SparkGeographyType( + crs: String, + algorithm: String): org.apache.spark.sql.types.DataType = unsupportedGeospatial() + + override def sparkGeometryCrs(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + override def sparkGeographyCrs(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + override def sparkGeographyAlgorithm(dataType: org.apache.spark.sql.types.DataType): String = + unsupportedGeospatial() + + private def unsupportedGeospatial[T](): T = + throw new UnsupportedOperationException("Geometry and geography require Spark 4.1 or later") + override def toPaimonVariant(row: InternalRow, pos: Int): Variant = throw new UnsupportedOperationException() diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala index b6904d86cf39..80e0456568e5 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala @@ -18,8 +18,9 @@ package org.apache.paimon.spark.data -import org.apache.paimon.types.DataType +import org.apache.paimon.types.{DataType, GeographyType, GeometryType} +import org.apache.spark.sql.paimon.shims.SparkShimLoader import org.apache.spark.unsafe.types.{GeographyVal, GeometryVal, VariantVal} class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkArrayData { @@ -30,8 +31,17 @@ class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkA } override def getGeography(ordinal: Int): GeographyVal = - throw new UnsupportedOperationException("Paimon does not support Geography type") + SparkShimLoader.shim + .toSparkGeography( + paimonArray.getBinary(ordinal), + elementType.asInstanceOf[GeographyType].getCrs, + elementType.asInstanceOf[GeographyType].getAlgorithm.toString) + .asInstanceOf[GeographyVal] override def getGeometry(ordinal: Int): GeometryVal = - throw new UnsupportedOperationException("Paimon does not support Geometry type") + SparkShimLoader.shim + .toSparkGeometry( + paimonArray.getBinary(ordinal), + elementType.asInstanceOf[GeometryType].getCrs) + .asInstanceOf[GeometryVal] } diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala index ea73692c689c..dc54eb4c6094 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala @@ -19,8 +19,9 @@ package org.apache.paimon.spark.data import org.apache.paimon.spark.AbstractSparkInternalRow -import org.apache.paimon.types.RowType +import org.apache.paimon.types.{GeographyType, GeometryType, RowType} +import org.apache.spark.sql.paimon.shims.SparkShimLoader import org.apache.spark.unsafe.types.{GeographyVal, GeometryVal, VariantVal} class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) { @@ -31,8 +32,22 @@ class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowTy } override def getGeography(ordinal: Int): GeographyVal = - throw new UnsupportedOperationException("Paimon does not support Geography type") + SparkShimLoader.shim + .toSparkGeography( + row.getBinary(ordinal), + rowType.getTypeAt(ordinal).asInstanceOf[GeographyType].getCrs, + rowType + .getTypeAt(ordinal) + .asInstanceOf[GeographyType] + .getAlgorithm + .toString + ) + .asInstanceOf[GeographyVal] override def getGeometry(ordinal: Int): GeometryVal = - throw new UnsupportedOperationException("Paimon does not support Geometry type") + SparkShimLoader.shim + .toSparkGeometry( + row.getBinary(ordinal), + rowType.getTypeAt(ordinal).asInstanceOf[GeometryType].getCrs) + .asInstanceOf[GeometryVal] } diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala index 00e0b1ae4ff0..eea503d7d5ce 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -41,7 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Assignment, Colum import org.apache.spark.sql.catalyst.plans.logical.MergeRows.{Copy, Insert, Keep, Update} import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns} +import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns, STUtils} import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Column, Identifier, StagingTableCatalog, Table, TableCatalog} import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.read.Scan @@ -53,7 +53,7 @@ import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, Data import org.apache.spark.sql.execution.streaming.runtime.MetadataLogFileIndex import org.apache.spark.sql.execution.streaming.sinks.FileStreamSink import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DataTypes, StructType, VariantType} +import org.apache.spark.sql.types.{DataTypes, Geography, GeographyType, Geometry, GeometryType, StructType, VariantType} import org.apache.spark.unsafe.types.VariantVal import java.util.{Map => JMap} @@ -359,6 +359,74 @@ class Spark4Shim extends SparkShim { override def SparkVariantType(): org.apache.spark.sql.types.DataType = DataTypes.VariantType + override def toPaimonGeometry(o: Object): Array[Byte] = + o.asInstanceOf[Geometry].getBytes + + override def toPaimonGeometry(row: InternalRow, pos: Int): Array[Byte] = + STUtils.stAsBinary(row.getGeometry(pos)) + + override def toPaimonGeometry(array: ArrayData, pos: Int): Array[Byte] = + STUtils.stAsBinary(array.getGeometry(pos)) + + override def toPaimonGeography(o: Object): Array[Byte] = + o.asInstanceOf[Geography].getBytes + + override def toPaimonGeography(row: InternalRow, pos: Int): Array[Byte] = + STUtils.stAsBinary(row.getGeography(pos)) + + override def toPaimonGeography(array: ArrayData, pos: Int): Array[Byte] = + STUtils.stAsBinary(array.getGeography(pos)) + + override def toSparkGeometry(wkb: Array[Byte], crs: String): Object = { + val geometryType = sparkGeometryType(crs) + STUtils.stGeomFromWKB(wkb, geometryType.srid) + } + + override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = { + val geographyType = sparkGeographyType(crs, algorithm) + STUtils.stSetSrid(STUtils.stGeogFromWKB(wkb), geographyType.srid) + } + + override def isSparkGeometryType(dataType: org.apache.spark.sql.types.DataType): Boolean = + dataType.isInstanceOf[GeometryType] + + override def isSparkGeographyType(dataType: org.apache.spark.sql.types.DataType): Boolean = + dataType.isInstanceOf[GeographyType] + + override def SparkGeometryType(crs: String): org.apache.spark.sql.types.DataType = + sparkGeometryType(crs) + + override def SparkGeographyType( + crs: String, + algorithm: String): org.apache.spark.sql.types.DataType = sparkGeographyType(crs, algorithm) + + override def sparkGeometryCrs(dataType: org.apache.spark.sql.types.DataType): String = { + val geometryType = dataType.asInstanceOf[GeometryType] + require(!geometryType.isMixedSrid, "Paimon does not support mixed-SRID geometry values") + geometryType.crs + } + + override def sparkGeographyCrs(dataType: org.apache.spark.sql.types.DataType): String = { + val geographyType = dataType.asInstanceOf[GeographyType] + require(!geographyType.isMixedSrid, "Paimon does not support mixed-SRID geography values") + geographyType.crs + } + + override def sparkGeographyAlgorithm(dataType: org.apache.spark.sql.types.DataType): String = + dataType.asInstanceOf[GeographyType].algorithm.toString + + private def sparkGeometryType(crs: String): GeometryType = { + val geometryType = GeometryType(crs) + require(!geometryType.isMixedSrid, "Paimon does not support mixed-SRID geometry values") + geometryType + } + + private def sparkGeographyType(crs: String, algorithm: String): GeographyType = { + val geographyType = GeographyType(crs, algorithm) + require(!geographyType.isMixedSrid, "Paimon does not support mixed-SRID geography values") + geographyType + } + // SQL UDFs (CREATE FUNCTION ... RETURN ...). override def rewritePaimonSQLFunctionCommands(spark: SparkSession): Rule[LogicalPlan] = org.apache.spark.sql.catalyst.parser.extensions.RewritePaimonSQLFunctionCommands(spark) From 720bacc782411a212445a4dfb237fcc7336f3be5 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 20:51:42 +0800 Subject: [PATCH 2/6] [core] Fix geospatial type interoperability --- docs/docs/iceberg/index.md | 2 + docs/docs/pypaimon/python-api.mdx | 5 +- .../apache/paimon/codegen/GenerateUtils.scala | 6 +- .../paimon/codegen/ScalarOperatorGens.scala | 8 +++ .../codegen/EqualiserCodeGeneratorTest.java | 8 +++ .../iceberg/metadata/IcebergDataField.java | 8 ++- .../paimon/schema/SchemaValidation.java | 10 +++- .../metadata/IcebergDataFieldTest.java | 4 +- .../paimon/schema/SchemaValidationTest.java | 14 +++++ .../paimon/flink/DataTypeToLogicalType.java | 19 +++---- .../flink/GeospatialTypeTableITCase.java | 55 +++++++------------ paimon-python/pypaimon/schema/data_types.py | 37 ++++++++++--- .../pypaimon/schema/schema_manager.py | 15 +---- .../pypaimon/tests/geospatial_type_test.py | 25 ++++++++- .../pypaimon/tests/write/table_write_test.py | 14 +++++ paimon-python/pypaimon/write/table_write.py | 14 ++++- .../spark/sql/GeospatialUnsupportedTest.scala | 21 +++++++ .../spark/sql/GeospatialUnsupportedTest.scala | 21 +++++++ .../spark/sql/GeospatialUnsupportedTest.scala | 21 +++++++ .../spark/sql/GeospatialUnsupportedTest.scala | 21 +++++++ .../spark/sql/GeospatialUnsupportedTest.scala | 21 +++++++ .../sql/GeospatialUnsupportedTestBase.scala | 49 +++++++++++++++++ 22 files changed, 317 insertions(+), 81 deletions(-) create mode 100644 paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala create mode 100644 paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala create mode 100644 paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala create mode 100644 paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala create mode 100644 paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTestBase.scala diff --git a/docs/docs/iceberg/index.md b/docs/docs/iceberg/index.md index c494833adc31..65161647f816 100644 --- a/docs/docs/iceberg/index.md +++ b/docs/docs/iceberg/index.md @@ -113,6 +113,8 @@ Paimon Iceberg compatibility currently supports the following data types. **Note on Geospatial Types:** - `GEOMETRY` and `GEOGRAPHY` values use OGC Well-Known Binary (WKB). The default CRS is `OGC:CRS84`, and the default geography edge algorithm is `spherical`. - Geospatial columns require Parquet for data, per-level, and changelog files. When Iceberg metadata is enabled, set `metadata.iceberg.format-version` to `3`. +- Spark SQL supports geospatial columns with Spark 4.1 or later. Spark 3.x, Spark 4.0, and Flink SQL reject these columns instead of exposing them as binary and losing the CRS or edge algorithm. +- Iceberg REST catalog publication does not yet support geospatial columns. Use `table-location`, `hadoop-catalog`, or `hive-catalog` metadata storage instead. - Geospatial columns cannot be primary, partition, bucket, or sequence keys. Paimon records null counts but does not publish byte-wise lower or upper bounds for WKB values. ::: diff --git a/docs/docs/pypaimon/python-api.mdx b/docs/docs/pypaimon/python-api.mdx index 77333280d786..8323143f8db7 100644 --- a/docs/docs/pypaimon/python-api.mdx +++ b/docs/docs/pypaimon/python-api.mdx @@ -965,8 +965,9 @@ Row kind values: For geospatial fields, PyPaimon stores OGC Well-Known Binary (WKB) in Arrow binary arrays and preserves the logical type in the field's `paimon.type` metadata. This metadata is also retained for fields nested in arrays, maps, and rows. Use `GeometryType` or `GeographyType` in the Paimon table schema; an unannotated Arrow binary field is inferred as -`BYTES`. Geospatial tables require Parquet, and non-spherical geography algorithms may not be supported by every -query engine. +`BYTES`. PyArrow does not currently expose the native Parquet `GEOMETRY` and `GEOGRAPHY` logical annotations required +by Paimon and Iceberg v3, so PyPaimon can represent and read these fields but rejects table writes containing them. +Geospatial tables require Parquet, and non-spherical geography algorithms may not be supported by every query engine. ### Complex Types diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/GenerateUtils.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/GenerateUtils.scala index 87e1bcca2897..f1043858ce6f 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/GenerateUtils.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/GenerateUtils.scala @@ -375,7 +375,7 @@ object GenerateUtils { // ordered by type root definition case CHAR | VARCHAR => BINARY_STRING case BOOLEAN => className[JBoolean] - case BINARY | VARBINARY => "byte[]" + case BINARY | VARBINARY | GEOMETRY | GEOGRAPHY => "byte[]" case DECIMAL => className[Decimal] case TINYINT => className[JByte] case SMALLINT => className[JShort] @@ -404,7 +404,7 @@ object GenerateUtils { s"(($BINARY_STRING) $rowTerm.getString($indexTerm))" case BOOLEAN => s"$rowTerm.getBoolean($indexTerm)" - case BINARY | VARBINARY => + case BINARY | VARBINARY | GEOMETRY | GEOGRAPHY => s"$rowTerm.getBinary($indexTerm)" case DECIMAL => s"$rowTerm.getDecimal($indexTerm, ${getPrecision(t)}, ${getScale(t)})" @@ -594,7 +594,7 @@ object GenerateUtils { s"$writerTerm.writeString($indexTerm, $fieldValTerm)" case BOOLEAN => s"$writerTerm.writeBoolean($indexTerm, $fieldValTerm)" - case BINARY | VARBINARY => + case BINARY | VARBINARY | GEOMETRY | GEOGRAPHY => s"$writerTerm.writeBinary($indexTerm, $fieldValTerm, 0, $fieldValTerm.length)" case DECIMAL => s"$writerTerm.writeDecimal($indexTerm, $fieldValTerm, ${getPrecision(t)})" diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala index fb8a04404968..d2cf7362ba3f 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala @@ -61,6 +61,14 @@ object ScalarOperatorGens { generateOperatorIfNotNull(ctx, resultType, left, right)( (leftTerm, rightTerm) => s"$leftTerm.equals($rightTerm)") } + // geospatial values use WKB byte arrays internally + else if ( + (isGeometry(left.resultType) && isGeometry(right.resultType)) || + (isGeography(left.resultType) && isGeography(right.resultType)) + ) { + generateOperatorIfNotNull(ctx, resultType, left, right)( + (leftTerm, rightTerm) => s"java.util.Arrays.equals($leftTerm, $rightTerm)") + } // numeric types else if (isNumeric(left.resultType) && isNumeric(right.resultType)) { generateComparison(ctx, "==", left, right, resultType) diff --git a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java index 6bbb4f765135..3d75cdcf41a6 100644 --- a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java +++ b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java @@ -82,6 +82,14 @@ public class EqualiserCodeGeneratorTest { TEST_DATA.put( DataTypeRoot.VARBINARY, new GeneratedData(DataTypes.VARBINARY(1), Pair.of("7".getBytes(), "8".getBytes()))); + TEST_DATA.put( + DataTypeRoot.GEOMETRY, + new GeneratedData( + DataTypes.GEOMETRY(), Pair.of("geom-1".getBytes(), "geom-2".getBytes()))); + TEST_DATA.put( + DataTypeRoot.GEOGRAPHY, + new GeneratedData( + DataTypes.GEOGRAPHY(), Pair.of("geog-1".getBytes(), "geog-2".getBytes()))); TEST_DATA.put( DataTypeRoot.DECIMAL, new GeneratedData( diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java index af3bfaec9854..510e884f9c12 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java @@ -304,11 +304,15 @@ private DataType getDataTypeFromType(Object icebergType, boolean isRequired) { case "geography": // iceberg v3 format String[] parameters = geographyParameters(simpleType); Preconditions.checkArgument( - parameters.length == 2, + parameters.length == 1 || parameters.length == 2, "Invalid Iceberg geography type: %s", simpleType); return new GeographyType( - !isRequired, parameters[0], EdgeAlgorithm.fromName(parameters[1])); + !isRequired, + parameters[0], + parameters.length == 1 + ? GeographyType.DEFAULT_ALGORITHM + : EdgeAlgorithm.fromName(parameters[1])); default: throw new UnsupportedOperationException( "Unsupported primitive data type: " + icebergType); diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index d6e089fba2b3..092e9d03f39a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -485,12 +485,18 @@ private static void validateGeospatialTypes( "Geometry and geography columns require '%s' to be parquet, but was '%s'.", CoreOptions.CHANGELOG_FILE_FORMAT.key(), options.changelogFileFormat()); - if (options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE) - != IcebergOptions.StorageType.DISABLED) { + IcebergOptions.StorageType icebergStorage = + options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE); + if (icebergStorage != IcebergOptions.StorageType.DISABLED) { checkArgument( options.toConfiguration().get(IcebergOptions.FORMAT_VERSION) == 3, "Geometry and geography columns require '%s'='3' when Iceberg metadata is enabled.", IcebergOptions.FORMAT_VERSION.key()); + checkArgument( + icebergStorage != IcebergOptions.StorageType.REST_CATALOG, + "Geometry and geography columns do not support '%s'='%s' because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial types.", + IcebergOptions.METADATA_ICEBERG_STORAGE.key(), + IcebergOptions.StorageType.REST_CATALOG); } Set geospatialFields = diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java index 11f5e12f0744..e728fd191e35 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java @@ -184,8 +184,10 @@ void testGeospatialTypeConversions() { .isEqualTo(new GeometryType()); assertThat(new IcebergDataField(6, "geog", false, "geography", null).dataType()) .isEqualTo(new GeographyType()); + assertThat(new IcebergDataField(7, "geog", false, "geography(EPSG:4326)", null).dataType()) + .isEqualTo(new GeographyType(true, "EPSG:4326", GeographyType.DEFAULT_ALGORITHM)); assertThat( - new IcebergDataField(7, "geom", false, "geometry(custom, definition)", null) + new IcebergDataField(8, "geom", false, "geometry(custom, definition)", null) .dataType()) .isEqualTo(new GeometryType("custom, definition")); } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 187b710cd84d..5317fff4c390 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -1692,6 +1692,20 @@ fields, emptyList(), emptyList(), perLevelOptions))) emptyList(), emptyList(), icebergV2Options))); + + Map icebergRestOptions = new HashMap<>(); + icebergRestOptions.put(IcebergOptions.METADATA_ICEBERG_STORAGE.key(), "rest-catalog"); + icebergRestOptions.put(IcebergOptions.FORMAT_VERSION.key(), "3"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + icebergRestOptions))) + .hasMessageContaining("do not support 'metadata.iceberg.storage'='rest-catalog'") + .hasMessageContaining("REST client"); } @Test diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java index f15684933599..a1fe89efb295 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataTypeToLogicalType.java @@ -159,20 +159,19 @@ public LogicalType visit(BlobType blobType) { @Override public LogicalType visit(GeometryType geometryType) { - // Flink has no native geospatial logical type. Expose WKB through SQL while preserving the - // geospatial type in the Paimon schema. - return new org.apache.flink.table.types.logical.VarBinaryType( - geometryType.isNullable(), - org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); + throw unsupportedGeospatialType(geometryType); } @Override public LogicalType visit(GeographyType geographyType) { - // Flink has no native geospatial logical type. Expose WKB through SQL while preserving the - // geospatial type in the Paimon schema. - return new org.apache.flink.table.types.logical.VarBinaryType( - geographyType.isNullable(), - org.apache.flink.table.types.logical.VarBinaryType.MAX_LENGTH); + throw unsupportedGeospatialType(geographyType); + } + + private UnsupportedOperationException unsupportedGeospatialType(DataType dataType) { + return new UnsupportedOperationException( + "Flink SQL does not support Paimon geospatial type " + + dataType.asSQLString() + + ". Exposing it as VARBINARY would lose its CRS and edge algorithm."); } @Override diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java index 25fb4e8d110a..4a319a645986 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/GeospatialTypeTableITCase.java @@ -24,29 +24,30 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.EdgeAlgorithm; -import org.apache.flink.table.types.logical.LogicalTypeRoot; -import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests Flink SQL interoperability with Paimon geospatial columns. */ public class GeospatialTypeTableITCase extends CatalogITCaseBase { private static final String TABLE_NAME = "geospatial_table"; - private static final byte[] POINT_1_2_WKB = - new byte[] { - 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xf0, 0x3f, 0, 0, 0, 0, 0, 0, 0, 0x40 - }; + @Test + public void testRejectGeospatialColumnsInFlinkSql() throws Exception { + createGeospatialTable(); - private static final byte[] POINT_3_4_WKB = - new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x40, 0, 0, 0, 0, 0, 0, 0x10, 0x40}; + assertUnsupported(() -> batchSql("SELECT * FROM %s", TABLE_NAME)); + assertUnsupported( + () -> + batchSql( + "CREATE TABLE geospatial_like LIKE %s (EXCLUDING OPTIONS)", + TABLE_NAME)); + assertUnsupported( + () -> batchSql("CREATE TABLE geospatial_ctas AS SELECT * FROM %s", TABLE_NAME)); + } - @Test - public void testReadWriteGeospatialColumnsAsWkb() throws Exception { + private void createGeospatialTable() throws Exception { flinkCatalog() .catalog() .createTable( @@ -62,29 +63,11 @@ public void testReadWriteGeospatialColumnsAsWkb() throws Exception { CoreOptions.FILE_FORMAT_PARQUET) .build(), false); + } - List columnTypes = - tEnv.from(TABLE_NAME).getResolvedSchema().getColumnDataTypes(); - assertThat(columnTypes.get(0).getLogicalType().is(LogicalTypeRoot.INTEGER)).isTrue(); - assertThat(columnTypes.get(1).getLogicalType().is(LogicalTypeRoot.VARBINARY)).isTrue(); - assertThat(columnTypes.get(2).getLogicalType().is(LogicalTypeRoot.VARBINARY)).isTrue(); - - batchSql( - "INSERT INTO %s VALUES " - + "(1, X'0101000000000000000000F03F0000000000000040', " - + "X'010100000000000000000008400000000000001040'), " - + "(2, CAST(NULL AS BYTES), " - + "X'0101000000000000000000F03F0000000000000040')", - TABLE_NAME); - - List rows = batchSql("SELECT * FROM %s ORDER BY id", TABLE_NAME); - assertThat(rows) - .containsExactly( - Row.of(1, POINT_1_2_WKB, POINT_3_4_WKB), Row.of(2, null, POINT_1_2_WKB)); - - assertThat(paimonTable(TABLE_NAME).schema().fields().get(1).type()) - .isEqualTo(DataTypes.GEOMETRY("EPSG:3857")); - assertThat(paimonTable(TABLE_NAME).schema().fields().get(2).type()) - .isEqualTo(DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.SPHERICAL)); + private void assertUnsupported(org.assertj.core.api.ThrowableAssert.ThrowingCallable callable) { + assertThatThrownBy(callable) + .hasStackTraceContaining("Flink SQL does not support Paimon geospatial type") + .hasStackTraceContaining("Exposing it as VARBINARY would lose its CRS"); } } diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py index f9cb92d8f132..b3fd11603d31 100755 --- a/paimon-python/pypaimon/schema/data_types.py +++ b/paimon-python/pypaimon/schema/data_types.py @@ -508,6 +508,20 @@ def get_field_index(self, field_name: str) -> int: raise ValueError("Field {} not found in {}".format(field_name, self)) +def _contains_geospatial_type(data_type: DataType) -> bool: + if isinstance(data_type, (GeometryType, GeographyType)): + return True + if isinstance(data_type, (ArrayType, MultisetType)): + return _contains_geospatial_type(data_type.element) + if isinstance(data_type, MapType): + return (_contains_geospatial_type(data_type.key) + or _contains_geospatial_type(data_type.value)) + if isinstance(data_type, RowType): + return any(_contains_geospatial_type(field.type) + for field in data_type.fields) + return False + + def reassign_field_id(data_type: DataType, field_id: "AtomicInteger") -> DataType: """Return a copy of *data_type* with every nested field id reassigned from *field_id*, depth-first with children allocated before their parent field. @@ -633,8 +647,12 @@ def parse_atomic_type_sql_string(type_string: str) -> DataType: type_text, re.IGNORECASE) if geometry_match: quoted_crs, raw_crs = geometry_match.groups() - crs = quoted_crs.replace("''", "'") if quoted_crs is not None else raw_crs - return GeometryType(crs or GeometryType.DEFAULT_CRS, nullable) + if quoted_crs is None and raw_crs is None: + crs = GeometryType.DEFAULT_CRS + else: + crs = quoted_crs.replace("''", "'") \ + if quoted_crs is not None else raw_crs + return GeometryType(crs, nullable) if type_text.upper().startswith("GEOMETRY"): raise ValueError("Invalid geometry type: {}".format(type_text)) @@ -643,8 +661,11 @@ def parse_atomic_type_sql_string(type_string: str) -> DataType: r"(?:,\s*([^(),]+?)\s*)?\))?", type_text, re.IGNORECASE) if geography_match: quoted_crs, raw_crs, raw_algorithm = geography_match.groups() - crs = quoted_crs.replace("''", "'") if quoted_crs is not None else raw_crs - crs = crs or GeographyType.DEFAULT_CRS + if quoted_crs is None and raw_crs is None: + crs = GeographyType.DEFAULT_CRS + else: + crs = quoted_crs.replace("''", "'") \ + if quoted_crs is not None else raw_crs algorithm = EdgeAlgorithm.from_name( raw_algorithm or GeographyType.DEFAULT_ALGORITHM.value) return GeographyType(crs, algorithm, nullable) @@ -988,9 +1009,11 @@ def _to_paimon_field_type(pa_field: pyarrow.Field) -> DataType: if pa_field.metadata and b'paimon.type' in pa_field.metadata: data_type = DataTypeParser.parse_atomic_type_sql_string( pa_field.metadata[b'paimon.type'].decode('utf-8')) - if (isinstance(data_type, (GeometryType, GeographyType)) - and not (types.is_binary(pa_field.type) - or types.is_fixed_size_binary(pa_field.type))): + if not isinstance(data_type, (GeometryType, GeographyType)): + raise ValueError( + "Arrow field metadata 'paimon.type' is reserved for " + "geospatial types: {}".format(pa_field)) + if not types.is_binary(pa_field.type): raise ValueError( "Geospatial field metadata requires a binary Arrow type: {}" .format(pa_field)) diff --git a/paimon-python/pypaimon/schema/schema_manager.py b/paimon-python/pypaimon/schema/schema_manager.py index 77b8fbc898a7..1ef052ef57c7 100644 --- a/paimon-python/pypaimon/schema/schema_manager.py +++ b/paimon-python/pypaimon/schema/schema_manager.py @@ -30,6 +30,7 @@ from pypaimon.schema.data_types import (ArrayType, AtomicInteger, DataField, DataType, GeographyType, GeometryType, MapType, MultisetType, RowType, + _contains_geospatial_type, is_array_blob_type, is_blob_file_field, is_blob_file_type, is_blob_type, is_map_blob_type, reassign_field_id) @@ -468,20 +469,6 @@ def _validate_options(options: dict): ) -def _contains_geospatial_type(data_type: DataType) -> bool: - if isinstance(data_type, (GeometryType, GeographyType)): - return True - if isinstance(data_type, (ArrayType, MultisetType)): - return _contains_geospatial_type(data_type.element) - if isinstance(data_type, MapType): - return (_contains_geospatial_type(data_type.key) - or _contains_geospatial_type(data_type.value)) - if isinstance(data_type, RowType): - return any(_contains_geospatial_type(field.type) - for field in data_type.fields) - return False - - def _validate_geospatial_fields( fields: List[DataField], options: dict, diff --git a/paimon-python/pypaimon/tests/geospatial_type_test.py b/paimon-python/pypaimon/tests/geospatial_type_test.py index d342697f1892..bcea1063c8fb 100644 --- a/paimon-python/pypaimon/tests/geospatial_type_test.py +++ b/paimon-python/pypaimon/tests/geospatial_type_test.py @@ -22,9 +22,9 @@ from pypaimon.schema.data_types import DataField from pypaimon.schema.data_types import DataTypeParser from pypaimon.schema.data_types import EdgeAlgorithm +from pypaimon.schema.data_types import ArrayType from pypaimon.schema.data_types import GeographyType from pypaimon.schema.data_types import GeometryType -from pypaimon.schema.data_types import ArrayType from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.schema.data_types import RowType from pypaimon.schema.schema_manager import _validate_geospatial_fields @@ -48,6 +48,10 @@ def test_defaults_and_invalid_parameters(): assert str(DataTypeParser.parse_data_type("GEOGRAPHY")) == \ "GEOGRAPHY(OGC:CRS84, spherical)" + for invalid_type in ("GEOMETRY()", "GEOGRAPHY()", + "GEOGRAPHY(, spherical)"): + with pytest.raises(ValueError, match="Invalid CRS"): + DataTypeParser.parse_data_type(invalid_type) with pytest.raises(ValueError, match="Invalid edge interpolation algorithm"): DataTypeParser.parse_data_type("GEOGRAPHY(OGC:CRS84, rhumb)") with pytest.raises(ValueError, match="Invalid geometry type"): @@ -57,8 +61,11 @@ def test_defaults_and_invalid_parameters(): def test_pyarrow_uses_wkb_binary_and_preserves_type_metadata(): fields = [ DataField(0, "geom", GeometryType()), - DataField(1, "geog", GeographyType("EPSG:4326", EdgeAlgorithm.VINCENTY, - nullable=False)), + DataField( + 1, + "geog", + GeographyType( + "EPSG:4326", EdgeAlgorithm.VINCENTY, nullable=False)), ] arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) @@ -68,6 +75,18 @@ def test_pyarrow_uses_wkb_binary_and_preserves_type_metadata(): b'GEOMETRY(OGC:CRS84)' assert PyarrowFieldParser.to_paimon_schema(arrow_schema) == fields + fixed_geometry = pyarrow.field( + "geom", + pyarrow.binary(21), + metadata={b'paimon.type': b'GEOMETRY(OGC:CRS84)'}) + with pytest.raises(ValueError, match="requires a binary Arrow type"): + PyarrowFieldParser.to_paimon_schema(pyarrow.schema([fixed_geometry])) + + non_geospatial = pyarrow.field( + "value", pyarrow.binary(), metadata={b'paimon.type': b'INT'}) + with pytest.raises(ValueError, match="reserved for geospatial types"): + PyarrowFieldParser.to_paimon_schema(pyarrow.schema([non_geospatial])) + def test_nested_arrow_and_parquet_wkb_round_trip(tmp_path): fields = [ diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py index a640b8d31f26..fd8e5f7727c3 100644 --- a/paimon-python/pypaimon/tests/write/table_write_test.py +++ b/paimon-python/pypaimon/tests/write/table_write_test.py @@ -31,6 +31,7 @@ from pypaimon.common.json_util import JSON from pypaimon.common.options.core_options import CoreOptions from pypaimon.manifest.manifest_list_manager import ManifestListManager +from pypaimon.schema.data_types import DataField, GeometryType from pypaimon.write.table_write import TableWrite from pypaimon.write.writer.append_only_data_writer import AppendOnlyDataWriter @@ -1900,6 +1901,19 @@ def test_validate_schema_allows_binary_family_for_write_cols(self): ('payload', pa.binary(4)), ])) + def test_write_rejects_geospatial_type_without_native_parquet_annotation(self): + schema = Schema([DataField(0, 'geom', GeometryType())]) + self.catalog.create_table( + 'default.test_reject_geospatial_write', schema, False) + table = self.catalog.get_table( + 'default.test_reject_geospatial_write') + data = pa.table({'geom': [bytes.fromhex( + '0101000000000000000000f03f0000000000000040')]}) + + with self.assertRaisesRegex( + NotImplementedError, 'native Parquet GEOMETRY or GEOGRAPHY'): + self._commit_arrow(table, data) + @parameterized.expand([('parquet',), ('orc',), ('avro',)]) def test_write_time_type(self, file_format): time_schema = pa.schema([ diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py index f0a68bcf6fc1..ea264276f092 100644 --- a/paimon-python/pypaimon/write/table_write.py +++ b/paimon-python/pypaimon/write/table_write.py @@ -19,7 +19,10 @@ import pyarrow as pa -from pypaimon.schema.data_types import PyarrowFieldParser +from pypaimon.schema.data_types import ( + PyarrowFieldParser, + _contains_geospatial_type, +) from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.row.blob import BlobConsumer from pypaimon.write.row_utils import ( @@ -329,6 +332,15 @@ def _release_prepared_indexes(self) -> None: release() def _validate_pyarrow_schema(self, data_schema: pa.Schema): + if any(_contains_geospatial_type(field.type) + for field in self.table.table_schema.fields): + raise NotImplementedError( + "PyPaimon does not support writing geospatial columns because " + "PyArrow cannot produce the native Parquet GEOMETRY or " + "GEOGRAPHY logical annotation required by Paimon and " + "Iceberg v3." + ) + if self._is_compatible_pyarrow_schema(data_schema, self.table_pyarrow_schema): return diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala new file mode 100644 index 000000000000..dde6fabf6a09 --- /dev/null +++ b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class GeospatialUnsupportedTest extends GeospatialUnsupportedTestBase {} diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala new file mode 100644 index 000000000000..dde6fabf6a09 --- /dev/null +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class GeospatialUnsupportedTest extends GeospatialUnsupportedTestBase {} diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala new file mode 100644 index 000000000000..dde6fabf6a09 --- /dev/null +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class GeospatialUnsupportedTest extends GeospatialUnsupportedTestBase {} diff --git a/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala new file mode 100644 index 000000000000..dde6fabf6a09 --- /dev/null +++ b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class GeospatialUnsupportedTest extends GeospatialUnsupportedTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala new file mode 100644 index 000000000000..dde6fabf6a09 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class GeospatialUnsupportedTest extends GeospatialUnsupportedTestBase {} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTestBase.scala new file mode 100644 index 000000000000..cc45c9b1f04c --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/GeospatialUnsupportedTestBase.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.catalog.Identifier +import org.apache.paimon.schema.Schema +import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.types.DataTypes + +abstract class GeospatialUnsupportedTestBase extends PaimonSparkTestBase { + + test("Spark SQL rejects geospatial columns before Spark 4.1") { + val identifier = Identifier.create(dbName0, "geospatial_table") + paimonCatalog.createTable( + identifier, + Schema.newBuilder + .column("id", DataTypes.INT()) + .column("geom", DataTypes.GEOMETRY()) + .column("geog", DataTypes.GEOGRAPHY()) + .build, + false + ) + + try { + val error = intercept[UnsupportedOperationException] { + sql("SELECT * FROM geospatial_table") + } + assert(error.getMessage.contains("Geometry and geography require Spark 4.1 or later")) + } finally { + paimonCatalog.dropTable(identifier, true) + } + } +} From 9aab7dc37a6cf5a66d26c73654a60b06feda1b5a Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 21:13:28 +0800 Subject: [PATCH 3/6] [core] Align geospatial Arrow metadata with Iceberg --- docs/docs/pypaimon/python-api.mdx | 16 +- .../arrow/ArrowFieldTypeConversion.java | 16 +- .../ArrowSchemaMetadataCompatibilityTest.java | 37 ++++ .../apache/paimon/arrow/ArrowUtilsTest.java | 30 ++-- .../paimon/format/ArrowSchemaMetadata.java | 27 ++- .../format/FormatMetadataUtilsTest.java | 13 +- paimon-python/pypaimon/schema/data_types.py | 83 ++++++--- .../pypaimon/tests/geospatial_type_test.py | 161 +++++++++++++----- paimon-python/setup.py | 3 + 9 files changed, 287 insertions(+), 99 deletions(-) diff --git a/docs/docs/pypaimon/python-api.mdx b/docs/docs/pypaimon/python-api.mdx index 8323143f8db7..026b55e305cd 100644 --- a/docs/docs/pypaimon/python-api.mdx +++ b/docs/docs/pypaimon/python-api.mdx @@ -960,13 +960,15 @@ Row kind values: | `datetime.datetime` | `pyarrow.timestamp(unit, tz='UTC')` | `TIMESTAMP_LTZ(p)` — same unit/p mapping as above | | `datetime.date` | `pyarrow.date32()` | `DATE` | | `datetime.time` | `pyarrow.time32('ms')` | `TIME(p)` | -| WKB `bytes` | `pyarrow.binary()` | `GEOMETRY(crs)`, `GEOGRAPHY(crs, algorithm)` | - -For geospatial fields, PyPaimon stores OGC Well-Known Binary (WKB) in Arrow binary arrays and preserves the logical -type in the field's `paimon.type` metadata. This metadata is also retained for fields nested in arrays, maps, and rows. -Use `GeometryType` or `GeographyType` in the Paimon table schema; an unannotated Arrow binary field is inferred as -`BYTES`. PyArrow does not currently expose the native Parquet `GEOMETRY` and `GEOGRAPHY` logical annotations required -by Paimon and Iceberg v3, so PyPaimon can represent and read these fields but rejects table writes containing them. +| WKB `bytes` | GeoArrow WKB extension or `pyarrow.large_binary()` | `GEOMETRY(crs)`, `GEOGRAPHY(crs, algorithm)` | + +For geospatial fields, PyPaimon follows PyIceberg and uses the standard GeoArrow WKB extension type when the optional +`geoarrow-pyarrow` package is installed (`pip install pypaimon[geoarrow]`). The extension metadata preserves the CRS +and edge-interpolation algorithm, including for fields nested in arrays, maps, and rows. Without GeoArrow, PyPaimon +falls back to `pyarrow.large_binary()` and relies on the Paimon table schema for the logical type; an unannotated Arrow +binary field is inferred as `BYTES` or `BLOB`. PyArrow does not currently expose the native Parquet `GEOMETRY` and +`GEOGRAPHY` logical annotations required by Paimon and Iceberg v3, so PyPaimon can represent and read these fields but +rejects table writes containing them. Geospatial tables require Parquet, and non-spherical geography algorithms may not be supported by every query engine. ### Complex Types diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java index 4ee158cedc98..a6a1328a3163 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowFieldTypeConversion.java @@ -51,13 +51,9 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; -import java.util.Collections; - /** Utils for conversion between Paimon {@link DataType} and Arrow {@link FieldType}. */ public class ArrowFieldTypeConversion { - public static final String PAIMON_TYPE = "paimon.type"; - public static final ArrowFieldTypeVisitor ARROW_FIELD_TYPE_VISITOR = new ArrowFieldTypeVisitor(); @@ -93,20 +89,12 @@ public FieldType visit(VarBinaryType varBinaryType) { @Override public FieldType visit(GeometryType geometryType) { - return geospatialFieldType(geometryType); + throw new UnsupportedOperationException("Unsupported primitive type: " + geometryType); } @Override public FieldType visit(GeographyType geographyType) { - return geospatialFieldType(geographyType); - } - - private FieldType geospatialFieldType(DataType dataType) { - return new FieldType( - dataType.isNullable(), - Types.MinorType.VARBINARY.getType(), - null, - Collections.singletonMap(PAIMON_TYPE, dataType.asSQLString())); + throw new UnsupportedOperationException("Unsupported primitive type: " + geographyType); } @Override diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java index 73b8a2b28a4d..72ce097cfd89 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java @@ -21,8 +21,10 @@ import org.apache.paimon.format.FormatMetadataUtils; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; import org.apache.paimon.types.RowType; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; @@ -30,6 +32,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -106,6 +109,40 @@ public void testArrowJavaSchemaCanBeReadByFormatMetadata() { .containsAllEntriesOf(tagsMetadata); } + @Test + public void testGeoArrowMetadataCanBeReadByArrowJava() { + RowType rowType = + DataTypes.ROW( + DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY("EPSG:3857")), + DataTypes.FIELD( + 1, + "nested", + DataTypes.ROW( + DataTypes.FIELD( + 2, + "geog", + DataTypes.GEOGRAPHY( + "EPSG:4326", EdgeAlgorithm.KARNEY))))); + + byte[] schemaBytes = + FormatMetadataUtils.buildArrowSchemaMetadata( + rowType, Collections.emptyMap(), FormatMetadataUtils.PARQUET_FIELD_ID_KEY); + Schema schema = Schema.deserializeMessage(ByteBuffer.wrap(schemaBytes)); + + Field geometry = schema.findField("geom"); + assertThat(geometry.getType()).isEqualTo(ArrowType.Binary.INSTANCE); + assertThat(geometry.getMetadata()) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry("ARROW:extension:metadata", "{\"crs\":\"EPSG:3857\"}"); + + Field geography = schema.findField("nested").getChildren().get(0); + assertThat(geography.getType()).isEqualTo(ArrowType.Binary.INSTANCE); + assertThat(geography.getMetadata()) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry( + "ARROW:extension:metadata", "{\"edges\":\"karney\",\"crs\":\"EPSG:4326\"}"); + } + private static RowType rowType() { return DataTypes.ROW( DataTypes.FIELD(0, "id", DataTypes.INT()), diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java index 35cb10b172c8..fd9c6eb0b916 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java @@ -126,21 +126,21 @@ public void testVectorType() { } @Test - public void testGeospatialTypeMetadata() { - Field geometry = ArrowUtils.toArrowField("geom", 7, DataTypes.GEOMETRY(), 0); - Assertions.assertThat(geometry.getType()).isEqualTo(ArrowType.Binary.INSTANCE); - Assertions.assertThat(geometry.getMetadata()) - .containsEntry(ArrowUtils.PARQUET_FIELD_ID, "7") - .containsEntry(ArrowFieldTypeConversion.PAIMON_TYPE, "GEOMETRY(OGC:CRS84)"); - - Field geography = - ArrowUtils.toArrowField("geographies", 8, DataTypes.ARRAY(DataTypes.GEOGRAPHY()), 0) - .getChildren() - .get(0); - Assertions.assertThat(geography.getType()).isEqualTo(ArrowType.Binary.INSTANCE); - Assertions.assertThat(geography.getMetadata()) - .containsEntry( - ArrowFieldTypeConversion.PAIMON_TYPE, "GEOGRAPHY(OGC:CRS84, spherical)"); + public void testGeospatialTypesUnsupported() { + Assertions.assertThatThrownBy( + () -> ArrowUtils.toArrowField("geom", 7, DataTypes.GEOMETRY(), 0)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Unsupported primitive type: GEOMETRY(OGC:CRS84)"); + + Assertions.assertThatThrownBy( + () -> + ArrowUtils.toArrowField( + "geographies", + 8, + DataTypes.ARRAY(DataTypes.GEOGRAPHY()), + 0)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Unsupported primitive type: GEOGRAPHY(OGC:CRS84, spherical)"); } @Test diff --git a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java index 3f225dc62a44..a045a6442089 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java @@ -48,6 +48,7 @@ import org.apache.paimon.types.VarCharType; import org.apache.paimon.types.VariantType; import org.apache.paimon.types.VectorType; +import org.apache.paimon.utils.JsonSerdeUtil; import com.google.flatbuffers.FlatBufferBuilder; @@ -69,11 +70,15 @@ * FlatBuffers layout, enum values, and defaults are adapted from Apache Arrow Java / Arrow format * generated classes. This class implements only the subset needed by Paimon field metadata so that * {@code paimon-format} can stay compatible with {@code ARROW:schema} without depending on the - * Arrow runtime. + * Arrow runtime. Geospatial types are the exception to the direct {@code ArrowUtils} mapping: this + * metadata-only encoder writes the standard GeoArrow WKB extension metadata, while the Java Arrow + * API rejects geospatial conversion until it can expose the extension type itself. */ class ArrowSchemaMetadata { - private static final String PAIMON_TYPE = "paimon.type"; + private static final String ARROW_EXTENSION_NAME = "ARROW:extension:name"; + private static final String ARROW_EXTENSION_METADATA = "ARROW:extension:metadata"; + private static final String GEOARROW_WKB_EXTENSION_NAME = "geoarrow.wkb"; private static final String LIST_DATA_VECTOR_NAME = "$data$"; private static final String MAP_DATA_VECTOR_NAME = "entries"; @@ -402,7 +407,7 @@ private static ArrowField toArrowField( ArrowTypeInfo type = dataType.accept(ArrowFieldTypeVisitor.INSTANCE); Map metadata = new LinkedHashMap<>(fieldIdMetadata(fieldId, fieldIdKey)); if (dataType instanceof GeometryType || dataType instanceof GeographyType) { - metadata.put(PAIMON_TYPE, dataType.asSQLString()); + metadata.putAll(geospatialMetadata(dataType)); } List children = Collections.emptyList(); if (dataType instanceof ArrayType || dataType instanceof VectorType) { @@ -454,6 +459,22 @@ private static ArrowField toArrowField( return new ArrowField(fieldName, dataType.isNullable(), type, children, metadata); } + private static Map geospatialMetadata(DataType dataType) { + Map extensionMetadata = new LinkedHashMap<>(); + if (dataType instanceof GeographyType) { + GeographyType geographyType = (GeographyType) dataType; + extensionMetadata.put("edges", geographyType.getAlgorithm().toString()); + extensionMetadata.put("crs", geographyType.getCrs()); + } else { + extensionMetadata.put("crs", ((GeometryType) dataType).getCrs()); + } + + Map metadata = new LinkedHashMap<>(); + metadata.put(ARROW_EXTENSION_NAME, GEOARROW_WKB_EXTENSION_NAME); + metadata.put(ARROW_EXTENSION_METADATA, JsonSerdeUtil.toFlatJson(extensionMetadata)); + return metadata; + } + private static ArrowField toArrowMapEntryField( int fieldId, MapType mapType, int depth, String fieldIdKey) { ArrowField keyField = diff --git a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java index 4a2d3d2cb374..30213231e37d 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/FormatMetadataUtilsTest.java @@ -161,8 +161,17 @@ public void testBuildArrowSchemaWithGeospatialMetadata() { Map> metadata = FormatMetadataUtils.readFieldMetadata(schemaBytes); - assertThat(metadata.get("geom")).containsEntry("paimon.type", "GEOMETRY(OGC:CRS84)"); + assertThat(metadata.get("geom")) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry("ARROW:extension:metadata", "{\"crs\":\"OGC:CRS84\"}") + .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "0") + .doesNotContainKey("paimon.type"); assertThat(metadata.get("geog")) - .containsEntry("paimon.type", "GEOGRAPHY(OGC:CRS84, spherical)"); + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry( + "ARROW:extension:metadata", + "{\"edges\":\"spherical\",\"crs\":\"OGC:CRS84\"}") + .containsEntry(FormatMetadataUtils.PARQUET_FIELD_ID_KEY, "1") + .doesNotContainKey("paimon.type"); } } diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py index b3fd11603d31..875a11e139c3 100755 --- a/paimon-python/pypaimon/schema/data_types.py +++ b/paimon-python/pypaimon/schema/data_types.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import re import threading from abc import ABC, abstractmethod @@ -808,12 +809,66 @@ class PyarrowFieldParser: def _field_metadata(data_type: DataType, description: Optional[str] = None) -> Dict[bytes, bytes]: metadata = {} - if isinstance(data_type, (GeometryType, GeographyType)): - metadata[b'paimon.type'] = str(data_type).encode('utf-8') if description: metadata[b'description'] = description.encode('utf-8') return metadata + @staticmethod + def _geospatial_to_pyarrow(data_type: DataType) -> pyarrow.DataType: + try: + import geoarrow.pyarrow as geoarrow + except ImportError: + return pyarrow.large_binary() + + wkb_type = geoarrow.wkb().with_crs(data_type.crs) + if isinstance(data_type, GeographyType): + try: + edge_type = getattr(geoarrow.EdgeType, data_type.algorithm.name) + except AttributeError as exc: + raise ValueError( + "GeoArrow does not support edge interpolation algorithm: {}" + .format(data_type.algorithm)) from exc + wkb_type = wkb_type.with_edge_type(edge_type) + return wkb_type + + @staticmethod + def _geoarrow_to_geospatial(pa_type: pyarrow.ExtensionType, + nullable: bool) -> DataType: + storage_type = pa_type.storage_type + if not (types.is_binary(storage_type) + or types.is_large_binary(storage_type)): + raise ValueError( + "GeoArrow WKB requires a binary Arrow storage type: {}" + .format(pa_type)) + + try: + metadata = json.loads( + pa_type.__arrow_ext_serialize__().decode('utf-8')) + except (AttributeError, UnicodeDecodeError, ValueError) as exc: + raise ValueError( + "Invalid GeoArrow WKB extension metadata: {}".format(pa_type) + ) from exc + if not isinstance(metadata, dict): + raise ValueError( + "Invalid GeoArrow WKB extension metadata: {}".format(metadata)) + + crs = metadata.get('crs') + if isinstance(crs, dict): + crs_id = crs.get('id') + if (isinstance(crs_id, dict) + and crs_id.get('authority') is not None + and crs_id.get('code') is not None): + crs = '{}:{}'.format(crs_id['authority'], crs_id['code']) + if not isinstance(crs, str) or not crs: + raise ValueError( + "GeoArrow WKB extension metadata must contain an identifiable CRS: {}" + .format(metadata)) + + edges = metadata.get('edges', 'planar') + if str(edges).lower() == 'planar': + return GeometryType(crs, nullable) + return GeographyType(crs, EdgeAlgorithm.from_name(str(edges)), nullable) + @staticmethod def _from_paimon_named_type(name: str, data_type: DataType, description: Optional[str] = None) -> pyarrow.Field: @@ -827,7 +882,7 @@ def _from_paimon_named_type(name: str, data_type: DataType, def from_paimon_type(data_type: DataType) -> pyarrow.DataType: # Based on Paimon DataTypes Doc: https://paimon.apache.org/docs/master/concepts/data-types/ if isinstance(data_type, (GeometryType, GeographyType)): - return pyarrow.binary() + return PyarrowFieldParser._geospatial_to_pyarrow(data_type) if isinstance(data_type, AtomicType): type_name = data_type.type.upper() if type_name == 'TINYINT': @@ -927,6 +982,10 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: # Based on Arrow DataTypes Doc: https://arrow.apache.org/docs/python/api/datatypes.html # All safe mappings are already implemented, adding new mappings requires rigorous evaluation # to avoid potential data loss + if (isinstance(pa_type, pyarrow.ExtensionType) + and pa_type.extension_name == 'geoarrow.wkb'): + return PyarrowFieldParser._geoarrow_to_geospatial(pa_type, nullable) + type_name = None if types.is_int8(pa_type): type_name = 'TINYINT' @@ -1006,22 +1065,8 @@ def to_paimon_field(field_idx: int, pa_field: pyarrow.Field) -> DataField: @staticmethod def _to_paimon_field_type(pa_field: pyarrow.Field) -> DataType: - if pa_field.metadata and b'paimon.type' in pa_field.metadata: - data_type = DataTypeParser.parse_atomic_type_sql_string( - pa_field.metadata[b'paimon.type'].decode('utf-8')) - if not isinstance(data_type, (GeometryType, GeographyType)): - raise ValueError( - "Arrow field metadata 'paimon.type' is reserved for " - "geospatial types: {}".format(pa_field)) - if not types.is_binary(pa_field.type): - raise ValueError( - "Geospatial field metadata requires a binary Arrow type: {}" - .format(pa_field)) - data_type.nullable = pa_field.nullable - return data_type - else: - return PyarrowFieldParser.to_paimon_type( - pa_field.type, pa_field.nullable) + return PyarrowFieldParser.to_paimon_type( + pa_field.type, pa_field.nullable) @staticmethod def to_paimon_schema(pa_schema: pyarrow.Schema) -> List[DataField]: diff --git a/paimon-python/pypaimon/tests/geospatial_type_test.py b/paimon-python/pypaimon/tests/geospatial_type_test.py index bcea1063c8fb..f23c9fb3d43d 100644 --- a/paimon-python/pypaimon/tests/geospatial_type_test.py +++ b/paimon-python/pypaimon/tests/geospatial_type_test.py @@ -14,11 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import builtins +import json +import sys +from enum import Enum +from types import ModuleType + import pyarrow -import pyarrow.parquet as parquet import pytest from pypaimon.casting.data_type_casts import supports_cast +from pypaimon.schema.data_types import AtomicType from pypaimon.schema.data_types import DataField from pypaimon.schema.data_types import DataTypeParser from pypaimon.schema.data_types import EdgeAlgorithm @@ -58,7 +64,55 @@ def test_defaults_and_invalid_parameters(): DataTypeParser.parse_data_type("GEOMETRY(EPSG:4326) trailing") -def test_pyarrow_uses_wkb_binary_and_preserves_type_metadata(): +class _TestEdgeType(Enum): + PLANAR = 'planar' + SPHERICAL = 'spherical' + VINCENTY = 'vincenty' + THOMAS = 'thomas' + ANDOYER = 'andoyer' + KARNEY = 'karney' + + +class _TestGeoArrowWkbType(pyarrow.ExtensionType): + + def __init__(self, crs=None, edges=None): + self._crs = crs + self._edges = edges + super().__init__(pyarrow.binary(), 'geoarrow.wkb') + + def __arrow_ext_serialize__(self): + metadata = {} + if self._crs is not None: + metadata['crs'] = self._crs + if self._edges is not None: + metadata['edges'] = self._edges + return json.dumps(metadata).encode('utf-8') + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + metadata = json.loads(serialized.decode('utf-8')) + return cls(metadata.get('crs'), metadata.get('edges')) + + def with_crs(self, crs): + return _TestGeoArrowWkbType(crs, self._edges) + + def with_edge_type(self, edge_type): + return _TestGeoArrowWkbType(self._crs, edge_type.value) + + +def _install_test_geoarrow(monkeypatch): + geoarrow_package = ModuleType('geoarrow') + geoarrow_package.__path__ = [] + geoarrow_module = ModuleType('geoarrow.pyarrow') + geoarrow_module.EdgeType = _TestEdgeType + geoarrow_module.wkb = _TestGeoArrowWkbType + geoarrow_package.pyarrow = geoarrow_module + monkeypatch.setitem(sys.modules, 'geoarrow', geoarrow_package) + monkeypatch.setitem(sys.modules, 'geoarrow.pyarrow', geoarrow_module) + + +def test_pyarrow_uses_geoarrow_wkb_extension(monkeypatch): + _install_test_geoarrow(monkeypatch) fields = [ DataField(0, "geom", GeometryType()), DataField( @@ -69,50 +123,79 @@ def test_pyarrow_uses_wkb_binary_and_preserves_type_metadata(): ] arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) - assert arrow_schema.field("geom").type == pyarrow.binary() - assert arrow_schema.field("geog").type == pyarrow.binary() - assert arrow_schema.field("geom").metadata[b'paimon.type'] == \ - b'GEOMETRY(OGC:CRS84)' + geometry_type = arrow_schema.field("geom").type + geography_type = arrow_schema.field("geog").type + assert geometry_type.extension_name == 'geoarrow.wkb' + assert json.loads(geometry_type.__arrow_ext_serialize__()) == { + 'crs': 'OGC:CRS84' + } + assert json.loads(geography_type.__arrow_ext_serialize__()) == { + 'crs': 'EPSG:4326', + 'edges': 'vincenty', + } + assert b'paimon.type' not in (arrow_schema.field("geom").metadata or {}) assert PyarrowFieldParser.to_paimon_schema(arrow_schema) == fields - fixed_geometry = pyarrow.field( - "geom", - pyarrow.binary(21), - metadata={b'paimon.type': b'GEOMETRY(OGC:CRS84)'}) - with pytest.raises(ValueError, match="requires a binary Arrow type"): - PyarrowFieldParser.to_paimon_schema(pyarrow.schema([fixed_geometry])) + nested_fields = [ + DataField(0, "nested", RowType(True, [ + DataField(1, "geom", GeometryType(nullable=False)), + ])), + DataField( + 2, + "geographies", + ArrayType( + True, + GeographyType(algorithm=EdgeAlgorithm.KARNEY))), + ] + nested_schema = PyarrowFieldParser.from_paimon_schema(nested_fields) + assert PyarrowFieldParser.to_paimon_schema(nested_schema) == nested_fields + - non_geospatial = pyarrow.field( - "value", pyarrow.binary(), metadata={b'paimon.type': b'INT'}) - with pytest.raises(ValueError, match="reserved for geospatial types"): - PyarrowFieldParser.to_paimon_schema(pyarrow.schema([non_geospatial])) +def test_pyarrow_geospatial_fallback_without_geoarrow(monkeypatch): + original_import = builtins.__import__ + def block_geoarrow(name, *args, **kwargs): + if name.startswith('geoarrow'): + raise ImportError("No module named '{}'".format(name)) + return original_import(name, *args, **kwargs) -def test_nested_arrow_and_parquet_wkb_round_trip(tmp_path): + monkeypatch.setattr(builtins, '__import__', block_geoarrow) fields = [ - DataField(0, "nested", RowType(True, [ - DataField(1, "geom", GeometryType(nullable=False)), - ])), - DataField(2, "geographies", ArrayType(True, GeographyType())), + DataField(0, "geom", GeometryType()), + DataField(1, "geog", GeographyType()), + ] + arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) + assert arrow_schema.field("geom").type == pyarrow.large_binary() + assert arrow_schema.field("geog").type == pyarrow.large_binary() + assert b'paimon.type' not in (arrow_schema.field("geom").metadata or {}) + + inferred = PyarrowFieldParser.to_paimon_schema(arrow_schema) + assert inferred == [ + DataField(0, "geom", AtomicType('BLOB')), + DataField(1, "geog", AtomicType('BLOB')), + ] + + legacy_field = pyarrow.field( + "value", + pyarrow.binary(), + metadata={b'paimon.type': b'GEOMETRY(OGC:CRS84)'}) + assert PyarrowFieldParser.to_paimon_schema( + pyarrow.schema([legacy_field])) == [ + DataField(0, "value", AtomicType('BYTES')), + ] + + +def test_real_geoarrow_type_round_trip(): + pytest.importorskip('geoarrow.pyarrow') + fields = [ + DataField(0, "geom", GeometryType("EPSG:3857")), + DataField( + 1, + "geog", + GeographyType("EPSG:4326", EdgeAlgorithm.THOMAS)), ] - schema = PyarrowFieldParser.from_paimon_schema(fields) - assert schema.field("nested").type.field("geom").metadata[b'paimon.type'] == \ - b'GEOMETRY(OGC:CRS84) NOT NULL' - assert schema.field("geographies").type.value_field.metadata[b'paimon.type'] == \ - b'GEOGRAPHY(OGC:CRS84, spherical)' - assert PyarrowFieldParser.to_paimon_schema(schema) == fields - - point_wkb = bytes.fromhex( - "0101000000000000000000f03f0000000000000040") - table = pyarrow.Table.from_arrays([ - pyarrow.array([{"geom": point_wkb}], type=schema.field("nested").type), - pyarrow.array([[point_wkb]], type=schema.field("geographies").type), - ], schema=schema) - path = tmp_path / "geo.parquet" - parquet.write_table(table, path) - restored = parquet.read_table(path) - assert restored.to_pylist() == table.to_pylist() - assert PyarrowFieldParser.to_paimon_schema(restored.schema) == fields + assert PyarrowFieldParser.to_paimon_schema( + PyarrowFieldParser.from_paimon_schema(fields)) == fields def test_geospatial_cast_stats_and_schema_validation(): diff --git a/paimon-python/setup.py b/paimon-python/setup.py index a5b1b6209e07..3e3489cdb631 100644 --- a/paimon-python/setup.py +++ b/paimon-python/setup.py @@ -200,6 +200,9 @@ def read_requirements(): 'hdfs': [ 'hdfs-native>=0.13,<1; python_version >= "3.10" and platform_system != "Windows"', ], + 'geoarrow': [ + 'geoarrow-pyarrow>=0.2.0; python_version >= "3.9"', + ], }, description="Apache Paimon Python API", long_description=long_description, From 2ea13aefeeb3ab8cc8a065c6ae9d9ea7a85a224c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 22:03:47 +0800 Subject: [PATCH 4/6] [core] Restrict geospatial support to JVM --- docs/docs/pypaimon/python-api.mdx | 10 - .../ArrowSchemaMetadataCompatibilityTest.java | 29 +- .../iceberg/metadata/IcebergDataField.java | 4 + .../paimon/schema/SchemaValidation.java | 25 ++ .../metadata/IcebergDataFieldTest.java | 11 + .../aggregate/FieldAggregatorTest.java | 6 +- .../paimon/schema/SchemaValidationTest.java | 19 ++ .../paimon/format/ArrowSchemaMetadata.java | 6 +- .../parquet/ParquetSchemaConverter.java | 3 + .../parquet/ParquetFormatReadWriteTest.java | 10 + .../parquet/ParquetSchemaConverterTest.java | 5 + .../pypaimon/casting/data_type_casts.py | 14 +- paimon-python/pypaimon/schema/data_types.py | 285 ++---------------- .../pypaimon/schema/schema_manager.py | 81 +---- .../pypaimon/tests/geospatial_type_test.py | 228 -------------- .../pypaimon/tests/write/table_write_test.py | 14 - paimon-python/pypaimon/write/table_write.py | 14 +- .../pypaimon/write/writer/data_writer.py | 31 +- paimon-python/setup.py | 3 - 19 files changed, 157 insertions(+), 641 deletions(-) delete mode 100644 paimon-python/pypaimon/tests/geospatial_type_test.py diff --git a/docs/docs/pypaimon/python-api.mdx b/docs/docs/pypaimon/python-api.mdx index 026b55e305cd..23a27a1e130a 100644 --- a/docs/docs/pypaimon/python-api.mdx +++ b/docs/docs/pypaimon/python-api.mdx @@ -960,16 +960,6 @@ Row kind values: | `datetime.datetime` | `pyarrow.timestamp(unit, tz='UTC')` | `TIMESTAMP_LTZ(p)` — same unit/p mapping as above | | `datetime.date` | `pyarrow.date32()` | `DATE` | | `datetime.time` | `pyarrow.time32('ms')` | `TIME(p)` | -| WKB `bytes` | GeoArrow WKB extension or `pyarrow.large_binary()` | `GEOMETRY(crs)`, `GEOGRAPHY(crs, algorithm)` | - -For geospatial fields, PyPaimon follows PyIceberg and uses the standard GeoArrow WKB extension type when the optional -`geoarrow-pyarrow` package is installed (`pip install pypaimon[geoarrow]`). The extension metadata preserves the CRS -and edge-interpolation algorithm, including for fields nested in arrays, maps, and rows. Without GeoArrow, PyPaimon -falls back to `pyarrow.large_binary()` and relies on the Paimon table schema for the logical type; an unannotated Arrow -binary field is inferred as `BYTES` or `BLOB`. PyArrow does not currently expose the native Parquet `GEOMETRY` and -`GEOGRAPHY` logical annotations required by Paimon and Iceberg v3, so PyPaimon can represent and read these fields but -rejects table writes containing them. -Geospatial tables require Parquet, and non-spherical geography algorithms may not be supported by every query engine. ### Complex Types diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java index 72ce097cfd89..53c6e4ee55be 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowSchemaMetadataCompatibilityTest.java @@ -122,7 +122,15 @@ public void testGeoArrowMetadataCanBeReadByArrowJava() { 2, "geog", DataTypes.GEOGRAPHY( - "EPSG:4326", EdgeAlgorithm.KARNEY))))); + "EPSG:4326", EdgeAlgorithm.KARNEY)))), + DataTypes.FIELD( + 3, "geometries", DataTypes.ARRAY(DataTypes.GEOMETRY("EPSG:32632"))), + DataTypes.FIELD( + 4, + "geo_map", + DataTypes.MAP( + DataTypes.GEOGRAPHY("EPSG:4326", EdgeAlgorithm.THOMAS), + DataTypes.GEOMETRY("EPSG:3857")))); byte[] schemaBytes = FormatMetadataUtils.buildArrowSchemaMetadata( @@ -141,6 +149,25 @@ public void testGeoArrowMetadataCanBeReadByArrowJava() { .containsEntry("ARROW:extension:name", "geoarrow.wkb") .containsEntry( "ARROW:extension:metadata", "{\"edges\":\"karney\",\"crs\":\"EPSG:4326\"}"); + + Field arrayElement = schema.findField("geometries").getChildren().get(0); + assertThat(arrayElement.getMetadata()) + .containsKey(FormatMetadataUtils.PARQUET_FIELD_ID_KEY) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry("ARROW:extension:metadata", "{\"crs\":\"EPSG:32632\"}"); + + Field mapEntry = schema.findField("geo_map").getChildren().get(0); + Field mapKey = mapEntry.getChildren().get(0); + assertThat(mapKey.getMetadata()) + .containsKey(FormatMetadataUtils.PARQUET_FIELD_ID_KEY) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry( + "ARROW:extension:metadata", "{\"edges\":\"thomas\",\"crs\":\"EPSG:4326\"}"); + Field mapValue = mapEntry.getChildren().get(1); + assertThat(mapValue.getMetadata()) + .containsKey(FormatMetadataUtils.PARQUET_FIELD_ID_KEY) + .containsEntry("ARROW:extension:name", "geoarrow.wkb") + .containsEntry("ARROW:extension:metadata", "{\"crs\":\"EPSG:3857\"}"); } private static RowType rowType() { diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java index 510e884f9c12..d69fef3fb3ef 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java @@ -203,6 +203,10 @@ private static Object toTypeObject(DataType dataType, int fieldId, int depth) { return String.format("geometry(%s)", ((GeometryType) dataType).getCrs()); case GEOGRAPHY: GeographyType geographyType = (GeographyType) dataType; + Preconditions.checkArgument( + !geographyType.getCrs().contains(","), + "Geography CRS '%s' cannot contain ',' in Iceberg metadata.", + geographyType.getCrs()); return String.format( "geography(%s, %s)", geographyType.getCrs(), geographyType.getAlgorithm()); case ARRAY: diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 092e9d03f39a..8663128bf8b5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -497,6 +497,7 @@ private static void validateGeospatialTypes( "Geometry and geography columns do not support '%s'='%s' because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial types.", IcebergOptions.METADATA_ICEBERG_STORAGE.key(), IcebergOptions.StorageType.REST_CATALOG); + validateIcebergGeographyCrs(rowType); } Set geospatialFields = @@ -886,6 +887,30 @@ private static boolean containsType(DataType dataType, Predicate predi return false; } + private static void validateIcebergGeographyCrs(DataType dataType) { + if (dataType.is(DataTypeRoot.GEOGRAPHY)) { + String crs = ((GeographyType) dataType).getCrs(); + checkArgument( + !crs.contains(","), + "Geography CRS '%s' cannot contain ',' when Iceberg metadata is enabled.", + crs); + } else if (dataType instanceof RowType) { + for (DataField field : ((RowType) dataType).getFields()) { + validateIcebergGeographyCrs(field.type()); + } + } else if (dataType instanceof ArrayType) { + validateIcebergGeographyCrs(((ArrayType) dataType).getElementType()); + } else if (dataType instanceof MultisetType) { + validateIcebergGeographyCrs(((MultisetType) dataType).getElementType()); + } else if (dataType instanceof MapType) { + MapType mapType = (MapType) dataType; + validateIcebergGeographyCrs(mapType.getKeyType()); + validateIcebergGeographyCrs(mapType.getValueType()); + } else if (dataType instanceof VectorType) { + validateIcebergGeographyCrs(((VectorType) dataType).getElementType()); + } + } + private static void validateMapSharedShreddingFileFormats(CoreOptions options) { validateMapSharedShreddingFileFormat( CoreOptions.FILE_FORMAT.key(), options.fileFormatString()); diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java index e728fd191e35..f1daf206f421 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/metadata/IcebergDataFieldTest.java @@ -190,6 +190,17 @@ void testGeospatialTypeConversions() { new IcebergDataField(8, "geom", false, "geometry(custom, definition)", null) .dataType()) .isEqualTo(new GeometryType("custom, definition")); + + assertThatThrownBy( + () -> + new IcebergDataField( + new DataField( + 9, + "geog", + new GeographyType("custom, definition")))) + .hasMessageContaining("Geography CRS") + .hasMessageContaining("custom, definition") + .hasMessageContaining("Iceberg metadata"); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java index 90ade26f7129..ef3603a7a08c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java @@ -586,7 +586,9 @@ public void testFieldMaxMinAggComparableTypesAreAllSupported() { DataTypeRoot.ARRAY, DataTypeRoot.VECTOR, DataTypeRoot.VARIANT, - DataTypeRoot.BLOB)); + DataTypeRoot.BLOB, + DataTypeRoot.GEOMETRY, + DataTypeRoot.GEOGRAPHY)); assertThat(sampledRoots) .as("a comparable type root must be covered here and in InternalRowUtils.compare") .isEqualTo(expectedRoots); @@ -604,6 +606,8 @@ public void testFieldMaxMinAggWithIncomparableTypeShouldFail() { DataTypes.ROW(DataTypes.FIELD(0, "f0", DataTypes.INT())), DataTypes.VARIANT(), DataTypes.BLOB(), + DataTypes.GEOMETRY(), + DataTypes.GEOGRAPHY(), DataTypes.VECTOR(3, DataTypes.FLOAT()))) { assertThatThrownBy(() -> new FieldMaxAggFactory().create(incomparable, null, "label")) .isInstanceOf(IllegalArgumentException.class) diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 5317fff4c390..8b57bd7bb9da 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -1706,6 +1706,25 @@ fields, emptyList(), emptyList(), perLevelOptions))) icebergRestOptions))) .hasMessageContaining("do not support 'metadata.iceberg.storage'='rest-catalog'") .hasMessageContaining("REST client"); + + List customCrsFields = + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField( + 1, + "geographies", + DataTypes.ARRAY(DataTypes.GEOGRAPHY("custom, definition")))); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + customCrsFields, + emptyList(), + emptyList(), + icebergV2Options))) + .hasMessageContaining("Geography CRS") + .hasMessageContaining("custom, definition") + .hasMessageContaining("Iceberg metadata"); } @Test diff --git a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java index a045a6442089..acf1e3dd3dcd 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/ArrowSchemaMetadata.java @@ -539,8 +539,10 @@ private ArrowField( this.metadata = metadata; } - private ArrowField withMetadata(Map metadata) { - return new ArrowField(name, nullable, type, children, metadata); + private ArrowField withMetadata(Map additionalMetadata) { + Map mergedMetadata = new LinkedHashMap<>(metadata); + mergedMetadata.putAll(additionalMetadata); + return new ArrowField(name, nullable, type, children, mergedMetadata); } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index e8d760d137e9..053123c08511 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -38,6 +38,7 @@ import org.apache.paimon.utils.Pair; import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; +import org.apache.parquet.schema.ColumnOrder; import org.apache.parquet.schema.ConversionPatterns; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; @@ -135,6 +136,7 @@ public static Type convertToParquetType(String name, DataType type, int fieldId, case GEOMETRY: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .as(LogicalTypeAnnotation.geometryType(((GeometryType) type).getCrs())) + .columnOrder(ColumnOrder.undefined()) .named(name) .withId(fieldId); case GEOGRAPHY: @@ -145,6 +147,7 @@ public static Type convertToParquetType(String name, DataType type, int fieldId, geographyType.getCrs(), EdgeInterpolationAlgorithm.valueOf( geographyType.getAlgorithm().name()))) + .columnOrder(ColumnOrder.undefined()) .named(name) .withId(fieldId); case DECIMAL: diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index cebe1acb2684..fa812286e02c 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -103,6 +103,16 @@ public void testGeospatialWkbRoundTrip() throws Exception { Assertions.assertThat(geometries.getBinary(0)).isEqualTo(pointWkb); Assertions.assertThat(geometries.isNullAt(1)).isTrue(); } + + try (ParquetFileReader reader = + ParquetUtil.getParquetReader( + fileIO, file, fileIO.getFileSize(file), new Options())) { + Map columns = new HashMap<>(); + for (ColumnChunkMetaData column : reader.getFooter().getBlocks().get(0).getColumns()) { + columns.put(column.getPath().toDotString(), column); + } + Assertions.assertThat(columns.get("geom").getGeospatialStatistics()).isNotNull(); + } } @Test diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index 5888f5739c07..91f78ff47f4d 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.parquet.schema.ColumnOrder; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; @@ -162,6 +163,10 @@ public void testGeospatialLogicalTypesRoundTrip() { Type geography = messageType.getType("geog"); Assertions.assertThat(geometry.asPrimitiveType().getPrimitiveTypeName()).isEqualTo(BINARY); + Assertions.assertThat(geometry.asPrimitiveType().columnOrder().getColumnOrderName()) + .isEqualTo(ColumnOrder.ColumnOrderName.UNDEFINED); + Assertions.assertThat(geography.asPrimitiveType().columnOrder().getColumnOrderName()) + .isEqualTo(ColumnOrder.ColumnOrderName.UNDEFINED); Assertions.assertThat(geometry.getLogicalTypeAnnotation()) .isInstanceOf(LogicalTypeAnnotation.GeometryLogicalTypeAnnotation.class); Assertions.assertThat( diff --git a/paimon-python/pypaimon/casting/data_type_casts.py b/paimon-python/pypaimon/casting/data_type_casts.py index fb812b9ab066..819b9c115c00 100644 --- a/paimon-python/pypaimon/casting/data_type_casts.py +++ b/paimon-python/pypaimon/casting/data_type_casts.py @@ -29,7 +29,7 @@ import pyarrow as pa from pypaimon.schema.data_types import (ArrayType, AtomicType, DataTypeParser, - GeographyType, GeometryType, MapType, MultisetType, + MapType, MultisetType, PyarrowFieldParser, RowType, VectorType) @@ -58,8 +58,6 @@ VECTOR = "VECTOR" VARIANT = "VARIANT" BLOB = "BLOB" -GEOMETRY = "GEOMETRY" -GEOGRAPHY = "GEOGRAPHY" # ---- Families ---------------------------------------------------------------- @@ -94,10 +92,6 @@ def _root(data_type) -> str: return MULTISET if isinstance(data_type, VectorType): return VECTOR - if isinstance(data_type, GeometryType): - return GEOMETRY - if isinstance(data_type, GeographyType): - return GEOGRAPHY if isinstance(data_type, AtomicType): t = data_type.type.upper() if t.startswith("DECIMAL") or t.startswith("NUMERIC") or t.startswith("DEC"): @@ -131,7 +125,7 @@ def _build_rules(): implicit = {} explicit = {} # Identity cast for every root. - for root in (PREDEFINED | CONSTRUCTED | {VARIANT, BLOB, GEOMETRY, GEOGRAPHY}): + for root in (PREDEFINED | CONSTRUCTED | {VARIANT, BLOB}): implicit[root] = {root} explicit[root] = set() @@ -174,8 +168,6 @@ def supports_cast(source_type, target_type, allow_explicit: bool = True) -> bool if source_type.nullable and not target_type.nullable and not allow_explicit: return False if source_root == target_root: - if source_root in {GEOMETRY, GEOGRAPHY}: - return _equals_ignore_nullable(source_type, target_type) if source_root in CONSTRUCTED: # A constructed type is only castable to an (ignoring outer # nullability) identical constructed type. Reshaping is done @@ -224,8 +216,6 @@ def can_execute_cast(source_type, target_type) -> bool: # Same root: identity, or a same-shape constructed type whose value is # rebuilt by the read path's field-id alignment rather than a value cast. if source_root == target_root: - if source_root in {GEOMETRY, GEOGRAPHY}: - return _equals_ignore_nullable(source_type, target_type) return True # Constructed -> character string is rendered by the read path's custom # ``_constructed_to_string_array`` (see DataFileBatchReader), not a cast. diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py index 875a11e139c3..f01ee089aca1 100755 --- a/paimon-python/pypaimon/schema/data_types.py +++ b/paimon-python/pypaimon/schema/data_types.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -import json import re import threading from abc import ABC, abstractmethod @@ -107,105 +106,6 @@ def __str__(self) -> str: return "{}{}".format(self.type, null_suffix) -class EdgeAlgorithm(Enum): - SPHERICAL = "spherical" - VINCENTY = "vincenty" - THOMAS = "thomas" - ANDOYER = "andoyer" - KARNEY = "karney" - - @classmethod - def from_name(cls, name: str) -> "EdgeAlgorithm": - if name is None: - raise ValueError("Invalid edge interpolation algorithm: null") - try: - return cls(name.lower()) - except ValueError as exc: - raise ValueError( - "Invalid edge interpolation algorithm: {}".format(name)) from exc - - def __str__(self) -> str: - return self.value - - -@dataclass -class GeometryType(DataType): - crs: str - - DEFAULT_CRS = "OGC:CRS84" - - def __init__(self, crs: str = DEFAULT_CRS, nullable: bool = True): - super().__init__(nullable) - self.crs = self._validate_crs(crs) - - @staticmethod - def _validate_crs(crs: str) -> str: - if not crs: - raise ValueError("Invalid CRS: {}".format(crs)) - return crs - - @staticmethod - def _format_crs(crs: str) -> str: - if (crs[0].isdigit() - or any(character.isspace() or character in "<>().,'`" - for character in crs)): - return "'{}'".format(crs.replace("'", "''")) - return crs - - def __eq__(self, other): - return (isinstance(other, GeometryType) - and self.nullable == other.nullable - and self.crs.lower() == other.crs.lower()) - - def __hash__(self): - return hash((self.crs.upper(), self.nullable)) - - def to_dict(self) -> str: - return str(self) - - def __str__(self) -> str: - null_suffix = "" if self.nullable else " NOT NULL" - return "GEOMETRY({}){}".format( - self._format_crs(self.crs), null_suffix) - - -@dataclass -class GeographyType(DataType): - crs: str - algorithm: EdgeAlgorithm - - DEFAULT_CRS = "OGC:CRS84" - DEFAULT_ALGORITHM = EdgeAlgorithm.SPHERICAL - - def __init__(self, crs: str = DEFAULT_CRS, - algorithm: EdgeAlgorithm = DEFAULT_ALGORITHM, - nullable: bool = True): - super().__init__(nullable) - self.crs = GeometryType._validate_crs(crs) - if algorithm is None: - algorithm = self.DEFAULT_ALGORITHM - if not isinstance(algorithm, EdgeAlgorithm): - algorithm = EdgeAlgorithm.from_name(algorithm) - self.algorithm = algorithm - - def __eq__(self, other): - return (isinstance(other, GeographyType) - and self.nullable == other.nullable - and self.crs.lower() == other.crs.lower() - and self.algorithm == other.algorithm) - - def __hash__(self): - return hash((self.crs.upper(), self.algorithm, self.nullable)) - - def to_dict(self) -> str: - return str(self) - - def __str__(self) -> str: - null_suffix = "" if self.nullable else " NOT NULL" - return "GEOGRAPHY({}, {}){}".format( - GeometryType._format_crs(self.crs), self.algorithm, null_suffix) - - @dataclass class ArrayType(DataType): element: DataType @@ -509,20 +409,6 @@ def get_field_index(self, field_name: str) -> int: raise ValueError("Field {} not found in {}".format(field_name, self)) -def _contains_geospatial_type(data_type: DataType) -> bool: - if isinstance(data_type, (GeometryType, GeographyType)): - return True - if isinstance(data_type, (ArrayType, MultisetType)): - return _contains_geospatial_type(data_type.element) - if isinstance(data_type, MapType): - return (_contains_geospatial_type(data_type.key) - or _contains_geospatial_type(data_type.value)) - if isinstance(data_type, RowType): - return any(_contains_geospatial_type(field.type) - for field in data_type.fields) - return False - - def reassign_field_id(data_type: DataType, field_id: "AtomicInteger") -> DataType: """Return a copy of *data_type* with every nested field id reassigned from *field_id*, depth-first with children allocated before their parent field. @@ -614,8 +500,6 @@ class Keyword(Enum): TIMESTAMP = "TIMESTAMP" TIMESTAMP_LTZ = "TIMESTAMP_LTZ" VARIANT = "VARIANT" - GEOMETRY = "GEOMETRY" - GEOGRAPHY = "GEOGRAPHY" class DataTypeParser: @@ -631,7 +515,7 @@ def parse_nullability(type_string: str) -> bool: @staticmethod def parse_atomic_type_sql_string(type_string: str) -> DataType: nullable = DataTypeParser.parse_nullability(type_string) - type_text = type_string.strip() + type_upper = type_string.upper().strip() # Strip the trailing nullability suffix so it is stored only in # ``nullable``, not baked into the atomic type string. The space-split # branch below drops it for plain types ("BIGINT NOT NULL"), but a @@ -639,42 +523,10 @@ def parse_atomic_type_sql_string(type_string: str) -> DataType: # takes the paren branch and would otherwise keep the suffix in # ``AtomicType.type`` -- doubling it on the next ``to_dict()``. for suffix in (" NOT NULL", " NULL"): - if type_text.upper().endswith(suffix): - type_text = type_text[: -len(suffix)].rstrip() + if type_upper.endswith(suffix): + type_upper = type_upper[: -len(suffix)].rstrip() break - geometry_match = re.fullmatch( - r"GEOMETRY(?:\(\s*(?:'((?:''|[^'])*)'|([^)]*?))\s*\))?", - type_text, re.IGNORECASE) - if geometry_match: - quoted_crs, raw_crs = geometry_match.groups() - if quoted_crs is None and raw_crs is None: - crs = GeometryType.DEFAULT_CRS - else: - crs = quoted_crs.replace("''", "'") \ - if quoted_crs is not None else raw_crs - return GeometryType(crs, nullable) - if type_text.upper().startswith("GEOMETRY"): - raise ValueError("Invalid geometry type: {}".format(type_text)) - - geography_match = re.fullmatch( - r"GEOGRAPHY(?:\(\s*(?:'((?:''|[^'])*)'|([^,]*?))\s*" - r"(?:,\s*([^(),]+?)\s*)?\))?", type_text, re.IGNORECASE) - if geography_match: - quoted_crs, raw_crs, raw_algorithm = geography_match.groups() - if quoted_crs is None and raw_crs is None: - crs = GeographyType.DEFAULT_CRS - else: - crs = quoted_crs.replace("''", "'") \ - if quoted_crs is not None else raw_crs - algorithm = EdgeAlgorithm.from_name( - raw_algorithm or GeographyType.DEFAULT_ALGORITHM.value) - return GeographyType(crs, algorithm, nullable) - if type_text.upper().startswith("GEOGRAPHY"): - raise ValueError("Invalid geography type: {}".format(type_text)) - - type_upper = type_text.upper() - if "(" in type_upper: base_type = type_upper.split("(")[0] elif " " in type_upper: @@ -805,84 +657,9 @@ def is_variant_struct(pa_type: pyarrow.StructType) -> bool: class PyarrowFieldParser: - @staticmethod - def _field_metadata(data_type: DataType, - description: Optional[str] = None) -> Dict[bytes, bytes]: - metadata = {} - if description: - metadata[b'description'] = description.encode('utf-8') - return metadata - - @staticmethod - def _geospatial_to_pyarrow(data_type: DataType) -> pyarrow.DataType: - try: - import geoarrow.pyarrow as geoarrow - except ImportError: - return pyarrow.large_binary() - - wkb_type = geoarrow.wkb().with_crs(data_type.crs) - if isinstance(data_type, GeographyType): - try: - edge_type = getattr(geoarrow.EdgeType, data_type.algorithm.name) - except AttributeError as exc: - raise ValueError( - "GeoArrow does not support edge interpolation algorithm: {}" - .format(data_type.algorithm)) from exc - wkb_type = wkb_type.with_edge_type(edge_type) - return wkb_type - - @staticmethod - def _geoarrow_to_geospatial(pa_type: pyarrow.ExtensionType, - nullable: bool) -> DataType: - storage_type = pa_type.storage_type - if not (types.is_binary(storage_type) - or types.is_large_binary(storage_type)): - raise ValueError( - "GeoArrow WKB requires a binary Arrow storage type: {}" - .format(pa_type)) - - try: - metadata = json.loads( - pa_type.__arrow_ext_serialize__().decode('utf-8')) - except (AttributeError, UnicodeDecodeError, ValueError) as exc: - raise ValueError( - "Invalid GeoArrow WKB extension metadata: {}".format(pa_type) - ) from exc - if not isinstance(metadata, dict): - raise ValueError( - "Invalid GeoArrow WKB extension metadata: {}".format(metadata)) - - crs = metadata.get('crs') - if isinstance(crs, dict): - crs_id = crs.get('id') - if (isinstance(crs_id, dict) - and crs_id.get('authority') is not None - and crs_id.get('code') is not None): - crs = '{}:{}'.format(crs_id['authority'], crs_id['code']) - if not isinstance(crs, str) or not crs: - raise ValueError( - "GeoArrow WKB extension metadata must contain an identifiable CRS: {}" - .format(metadata)) - - edges = metadata.get('edges', 'planar') - if str(edges).lower() == 'planar': - return GeometryType(crs, nullable) - return GeographyType(crs, EdgeAlgorithm.from_name(str(edges)), nullable) - - @staticmethod - def _from_paimon_named_type(name: str, data_type: DataType, - description: Optional[str] = None) -> pyarrow.Field: - return pyarrow.field( - name, - PyarrowFieldParser.from_paimon_type(data_type), - nullable=data_type.nullable, - metadata=PyarrowFieldParser._field_metadata(data_type, description)) - @staticmethod def from_paimon_type(data_type: DataType) -> pyarrow.DataType: # Based on Paimon DataTypes Doc: https://paimon.apache.org/docs/master/concepts/data-types/ - if isinstance(data_type, (GeometryType, GeographyType)): - return PyarrowFieldParser._geospatial_to_pyarrow(data_type) if isinstance(data_type, AtomicType): type_name = data_type.type.upper() if type_name == 'TINYINT': @@ -943,32 +720,42 @@ def from_paimon_type(data_type: DataType) -> pyarrow.DataType: if type_name.startswith('TIME'): return pyarrow.time32('ms') elif isinstance(data_type, ArrayType): + element_type = PyarrowFieldParser.from_paimon_type(data_type.element) return pyarrow.list_( - PyarrowFieldParser._from_paimon_named_type("item", data_type.element) + pyarrow.field( + "item", + element_type, + nullable=data_type.element.nullable, + ) ) elif isinstance(data_type, VectorType): return pyarrow.list_(PyarrowFieldParser.from_paimon_type(data_type.element), data_type.length) elif isinstance(data_type, MapType): + key_type = PyarrowFieldParser.from_paimon_type(data_type.key) + value_type = PyarrowFieldParser.from_paimon_type(data_type.value) return pyarrow.map_( + pyarrow.field("key", key_type, nullable=False), pyarrow.field( - "key", - PyarrowFieldParser.from_paimon_type(data_type.key), - nullable=False, - metadata=PyarrowFieldParser._field_metadata(data_type.key)), - PyarrowFieldParser._from_paimon_named_type( - "value", data_type.value), + "value", + value_type, + nullable=data_type.value.nullable, + ), ) elif isinstance(data_type, RowType): pa_fields = [] for field in data_type.fields: - pa_fields.append(PyarrowFieldParser.from_paimon_field(field)) + pa_field_type = PyarrowFieldParser.from_paimon_type(field.type) + pa_fields.append(pyarrow.field(field.name, pa_field_type, nullable=field.type.nullable)) return pyarrow.struct(pa_fields) raise ValueError("Unsupported data type: {}".format(data_type)) @staticmethod def from_paimon_field(data_field: DataField) -> pyarrow.Field: - return PyarrowFieldParser._from_paimon_named_type( - data_field.name, data_field.type, data_field.description) + pa_field_type = PyarrowFieldParser.from_paimon_type(data_field.type) + metadata = {} + if data_field.description: + metadata[b'description'] = data_field.description.encode('utf-8') + return pyarrow.field(data_field.name, pa_field_type, nullable=data_field.type.nullable, metadata=metadata) @staticmethod def from_paimon_schema(data_fields: List[DataField]): @@ -982,10 +769,6 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: # Based on Arrow DataTypes Doc: https://arrow.apache.org/docs/python/api/datatypes.html # All safe mappings are already implemented, adding new mappings requires rigorous evaluation # to avoid potential data loss - if (isinstance(pa_type, pyarrow.ExtensionType) - and pa_type.extension_name == 'geoarrow.wkb'): - return PyarrowFieldParser._geoarrow_to_geospatial(pa_type, nullable) - type_name = None if types.is_int8(pa_type): type_name = 'TINYINT' @@ -1023,16 +806,19 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: type_name = 'TIME(0)' elif types.is_fixed_size_list(pa_type): pa_type: pyarrow.FixedSizeListType - element_type = PyarrowFieldParser._to_paimon_field_type(pa_type.value_field) + element_type = PyarrowFieldParser.to_paimon_type(pa_type.value_type, pa_type.value_field.nullable) return VectorType(nullable, element_type, pa_type.list_size) elif types.is_list(pa_type) or types.is_large_list(pa_type): pa_type: pyarrow.ListType - element_type = PyarrowFieldParser._to_paimon_field_type(pa_type.value_field) + element_type = PyarrowFieldParser.to_paimon_type( + pa_type.value_type, pa_type.value_field.nullable) return ArrayType(nullable, element_type) elif types.is_map(pa_type): pa_type: pyarrow.MapType - key_type = PyarrowFieldParser._to_paimon_field_type(pa_type.key_field) - value_type = PyarrowFieldParser._to_paimon_field_type(pa_type.item_field) + key_type = PyarrowFieldParser.to_paimon_type( + pa_type.key_type, pa_type.key_field.nullable) + value_type = PyarrowFieldParser.to_paimon_type( + pa_type.item_type, pa_type.item_field.nullable) return MapType(nullable, key_type, value_type) elif types.is_struct(pa_type) and is_variant_struct(pa_type): return AtomicType('VARIANT', nullable) @@ -1040,7 +826,7 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: pa_type: pyarrow.StructType fields = [] for i, pa_field in enumerate(pa_type): - field_type = PyarrowFieldParser._to_paimon_field_type(pa_field) + field_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) fields.append(DataField( id=i, name=pa_field.name, @@ -1053,7 +839,7 @@ def to_paimon_type(pa_type: pyarrow.DataType, nullable: bool) -> DataType: @staticmethod def to_paimon_field(field_idx: int, pa_field: pyarrow.Field) -> DataField: - data_type = PyarrowFieldParser._to_paimon_field_type(pa_field) + data_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) description = pa_field.metadata.get(b'description', b'').decode('utf-8') \ if pa_field.metadata and b'description' in pa_field.metadata else None return DataField( @@ -1063,11 +849,6 @@ def to_paimon_field(field_idx: int, pa_field: pyarrow.Field) -> DataField: description=description ) - @staticmethod - def _to_paimon_field_type(pa_field: pyarrow.Field) -> DataType: - return PyarrowFieldParser.to_paimon_type( - pa_field.type, pa_field.nullable) - @staticmethod def to_paimon_schema(pa_schema: pyarrow.Schema) -> List[DataField]: # Convert PyArrow schema to Paimon fields, assigning globally-unique ids: @@ -1080,7 +861,7 @@ def to_paimon_schema(pa_schema: pyarrow.Schema) -> List[DataField]: for pa_field in pa_schema: pa_field: pyarrow.Field top_id = field_id.increment_and_get() - data_type = PyarrowFieldParser._to_paimon_field_type(pa_field) + data_type = PyarrowFieldParser.to_paimon_type(pa_field.type, pa_field.nullable) data_type = reassign_field_id(data_type, field_id) description = pa_field.metadata.get(b'description', b'').decode('utf-8') \ if pa_field.metadata and b'description' in pa_field.metadata else None diff --git a/paimon-python/pypaimon/schema/schema_manager.py b/paimon-python/pypaimon/schema/schema_manager.py index 1ef052ef57c7..928e177a8b8b 100644 --- a/paimon-python/pypaimon/schema/schema_manager.py +++ b/paimon-python/pypaimon/schema/schema_manager.py @@ -28,9 +28,7 @@ remove_dropped_directive_options) from pypaimon.casting.data_type_casts import can_execute_cast, supports_cast from pypaimon.schema.data_types import (ArrayType, AtomicInteger, DataField, - DataType, GeographyType, GeometryType, - MapType, MultisetType, RowType, - _contains_geospatial_type, + DataType, MapType, MultisetType, RowType, is_array_blob_type, is_blob_file_field, is_blob_file_type, is_blob_type, is_map_blob_type, reassign_field_id) @@ -469,77 +467,6 @@ def _validate_options(options: dict): ) -def _validate_geospatial_fields( - fields: List[DataField], - options: dict, - primary_keys: List[str], - partition_keys: List[str], -): - if not any(_contains_geospatial_type(field.type) for field in fields): - return - - options = options or {} - core_options = CoreOptions(Options(options)) - file_format = (core_options.file_format('parquet') or '').lower() - if file_format != 'parquet': - raise ValueError( - "Geometry and geography columns require 'file.format'='parquet', " - "but was '{}'.".format(file_format)) - - per_level = core_options.file_format_per_level({}) or {} - if isinstance(per_level, str): - per_level = dict( - part.split(':', 1) for part in per_level.split(',') if part) - for level, level_format in per_level.items(): - if str(level_format).lower() != 'parquet': - raise ValueError( - "Geometry and geography columns require parquet at every level, " - "but 'file.format.per.level' contains '{}:{}'." - .format(level, level_format)) - - changelog_format = core_options.changelog_file_format() - if changelog_format and changelog_format.lower() != 'parquet': - raise ValueError( - "Geometry and geography columns require 'changelog-file.format' " - "to be parquet, but was '{}'.".format(changelog_format)) - - iceberg_storage = options.get('metadata.iceberg.storage', 'disabled').lower() - if (iceberg_storage != 'disabled' - and str(options.get('metadata.iceberg.format-version', '2')) != '3'): - raise ValueError( - "Geometry and geography columns require " - "'metadata.iceberg.format-version'='3' when Iceberg metadata is enabled.") - - geospatial_fields = { - field.name for field in fields - if isinstance(field.type, (GeometryType, GeographyType)) - } - primary_geo = geospatial_fields.intersection(primary_keys) - if primary_geo: - raise ValueError( - "Geometry and geography columns cannot be primary keys: {}." - .format(sorted(primary_geo))) - partition_geo = geospatial_fields.intersection(partition_keys) - if partition_geo: - raise ValueError( - "Geometry and geography columns cannot be partition keys: {}." - .format(sorted(partition_geo))) - - bucket_value = options.get(CoreOptions.BUCKET_KEY.key(), '') - bucket_keys = {key.strip() for key in bucket_value.split(',') if key.strip()} - bucket_geo = geospatial_fields.intersection(bucket_keys) - if bucket_geo: - raise ValueError( - "Geometry and geography columns cannot be bucket keys: {}." - .format(sorted(bucket_geo))) - - sequence_geo = geospatial_fields.intersection(core_options.sequence_field()) - if sequence_geo: - raise ValueError( - "Geometry and geography columns cannot be sequence fields: {}." - .format(sorted(sequence_geo))) - - def _contains_blob_type(data_type: DataType) -> bool: if is_blob_type(data_type): return True @@ -731,12 +658,6 @@ def create_table(self, schema: Schema) -> TableSchema: def commit(self, new_schema: TableSchema) -> bool: _validate_options(new_schema.options) - _validate_geospatial_fields( - new_schema.fields, - new_schema.options, - new_schema.primary_keys, - new_schema.partition_keys, - ) _validate_blob_fields( new_schema.fields, new_schema.options, diff --git a/paimon-python/pypaimon/tests/geospatial_type_test.py b/paimon-python/pypaimon/tests/geospatial_type_test.py deleted file mode 100644 index f23c9fb3d43d..000000000000 --- a/paimon-python/pypaimon/tests/geospatial_type_test.py +++ /dev/null @@ -1,228 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import builtins -import json -import sys -from enum import Enum -from types import ModuleType - -import pyarrow -import pytest - -from pypaimon.casting.data_type_casts import supports_cast -from pypaimon.schema.data_types import AtomicType -from pypaimon.schema.data_types import DataField -from pypaimon.schema.data_types import DataTypeParser -from pypaimon.schema.data_types import EdgeAlgorithm -from pypaimon.schema.data_types import ArrayType -from pypaimon.schema.data_types import GeographyType -from pypaimon.schema.data_types import GeometryType -from pypaimon.schema.data_types import PyarrowFieldParser -from pypaimon.schema.data_types import RowType -from pypaimon.schema.schema_manager import _validate_geospatial_fields -from pypaimon.write.writer.data_writer import DataWriter - - -def test_iceberg_compatible_type_round_trip(): - geometry = DataTypeParser.parse_data_type("GEOMETRY(ogc:crs84) NOT NULL") - geography = DataTypeParser.parse_data_type("GEOGRAPHY(EPSG:4326, karney)") - - assert geometry == GeometryType("OGC:CRS84", nullable=False) - assert geography == GeographyType("EPSG:4326", EdgeAlgorithm.KARNEY) - assert DataTypeParser.parse_data_type(geometry.to_dict()) == geometry - assert DataTypeParser.parse_data_type(geography.to_dict()) == geography - custom = GeometryType("custom, crs's definition") - assert DataTypeParser.parse_data_type(custom.to_dict()) == custom - - -def test_defaults_and_invalid_parameters(): - assert str(DataTypeParser.parse_data_type("GEOMETRY")) == "GEOMETRY(OGC:CRS84)" - assert str(DataTypeParser.parse_data_type("GEOGRAPHY")) == \ - "GEOGRAPHY(OGC:CRS84, spherical)" - - for invalid_type in ("GEOMETRY()", "GEOGRAPHY()", - "GEOGRAPHY(, spherical)"): - with pytest.raises(ValueError, match="Invalid CRS"): - DataTypeParser.parse_data_type(invalid_type) - with pytest.raises(ValueError, match="Invalid edge interpolation algorithm"): - DataTypeParser.parse_data_type("GEOGRAPHY(OGC:CRS84, rhumb)") - with pytest.raises(ValueError, match="Invalid geometry type"): - DataTypeParser.parse_data_type("GEOMETRY(EPSG:4326) trailing") - - -class _TestEdgeType(Enum): - PLANAR = 'planar' - SPHERICAL = 'spherical' - VINCENTY = 'vincenty' - THOMAS = 'thomas' - ANDOYER = 'andoyer' - KARNEY = 'karney' - - -class _TestGeoArrowWkbType(pyarrow.ExtensionType): - - def __init__(self, crs=None, edges=None): - self._crs = crs - self._edges = edges - super().__init__(pyarrow.binary(), 'geoarrow.wkb') - - def __arrow_ext_serialize__(self): - metadata = {} - if self._crs is not None: - metadata['crs'] = self._crs - if self._edges is not None: - metadata['edges'] = self._edges - return json.dumps(metadata).encode('utf-8') - - @classmethod - def __arrow_ext_deserialize__(cls, storage_type, serialized): - metadata = json.loads(serialized.decode('utf-8')) - return cls(metadata.get('crs'), metadata.get('edges')) - - def with_crs(self, crs): - return _TestGeoArrowWkbType(crs, self._edges) - - def with_edge_type(self, edge_type): - return _TestGeoArrowWkbType(self._crs, edge_type.value) - - -def _install_test_geoarrow(monkeypatch): - geoarrow_package = ModuleType('geoarrow') - geoarrow_package.__path__ = [] - geoarrow_module = ModuleType('geoarrow.pyarrow') - geoarrow_module.EdgeType = _TestEdgeType - geoarrow_module.wkb = _TestGeoArrowWkbType - geoarrow_package.pyarrow = geoarrow_module - monkeypatch.setitem(sys.modules, 'geoarrow', geoarrow_package) - monkeypatch.setitem(sys.modules, 'geoarrow.pyarrow', geoarrow_module) - - -def test_pyarrow_uses_geoarrow_wkb_extension(monkeypatch): - _install_test_geoarrow(monkeypatch) - fields = [ - DataField(0, "geom", GeometryType()), - DataField( - 1, - "geog", - GeographyType( - "EPSG:4326", EdgeAlgorithm.VINCENTY, nullable=False)), - ] - - arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) - geometry_type = arrow_schema.field("geom").type - geography_type = arrow_schema.field("geog").type - assert geometry_type.extension_name == 'geoarrow.wkb' - assert json.loads(geometry_type.__arrow_ext_serialize__()) == { - 'crs': 'OGC:CRS84' - } - assert json.loads(geography_type.__arrow_ext_serialize__()) == { - 'crs': 'EPSG:4326', - 'edges': 'vincenty', - } - assert b'paimon.type' not in (arrow_schema.field("geom").metadata or {}) - assert PyarrowFieldParser.to_paimon_schema(arrow_schema) == fields - - nested_fields = [ - DataField(0, "nested", RowType(True, [ - DataField(1, "geom", GeometryType(nullable=False)), - ])), - DataField( - 2, - "geographies", - ArrayType( - True, - GeographyType(algorithm=EdgeAlgorithm.KARNEY))), - ] - nested_schema = PyarrowFieldParser.from_paimon_schema(nested_fields) - assert PyarrowFieldParser.to_paimon_schema(nested_schema) == nested_fields - - -def test_pyarrow_geospatial_fallback_without_geoarrow(monkeypatch): - original_import = builtins.__import__ - - def block_geoarrow(name, *args, **kwargs): - if name.startswith('geoarrow'): - raise ImportError("No module named '{}'".format(name)) - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, '__import__', block_geoarrow) - fields = [ - DataField(0, "geom", GeometryType()), - DataField(1, "geog", GeographyType()), - ] - arrow_schema = PyarrowFieldParser.from_paimon_schema(fields) - assert arrow_schema.field("geom").type == pyarrow.large_binary() - assert arrow_schema.field("geog").type == pyarrow.large_binary() - assert b'paimon.type' not in (arrow_schema.field("geom").metadata or {}) - - inferred = PyarrowFieldParser.to_paimon_schema(arrow_schema) - assert inferred == [ - DataField(0, "geom", AtomicType('BLOB')), - DataField(1, "geog", AtomicType('BLOB')), - ] - - legacy_field = pyarrow.field( - "value", - pyarrow.binary(), - metadata={b'paimon.type': b'GEOMETRY(OGC:CRS84)'}) - assert PyarrowFieldParser.to_paimon_schema( - pyarrow.schema([legacy_field])) == [ - DataField(0, "value", AtomicType('BYTES')), - ] - - -def test_real_geoarrow_type_round_trip(): - pytest.importorskip('geoarrow.pyarrow') - fields = [ - DataField(0, "geom", GeometryType("EPSG:3857")), - DataField( - 1, - "geog", - GeographyType("EPSG:4326", EdgeAlgorithm.THOMAS)), - ] - assert PyarrowFieldParser.to_paimon_schema( - PyarrowFieldParser.from_paimon_schema(fields)) == fields - - -def test_geospatial_cast_stats_and_schema_validation(): - assert supports_cast(GeometryType("OGC:CRS84"), - GeometryType("ogc:crs84", nullable=False)) - assert not supports_cast(GeometryType(), GeometryType("EPSG:3857")) - assert not supports_cast( - GeographyType(algorithm=EdgeAlgorithm.SPHERICAL), - GeographyType(algorithm=EdgeAlgorithm.KARNEY)) - - fields = [DataField(0, "geom", GeometryType())] - values = pyarrow.table({"geom": [b'\x02', None, b'\x01']}) - stats_fields = DataWriter._resolve_stats_fields(values.schema, fields) - assert stats_fields == fields - stats = DataWriter._get_column_stats(values, "geom", GeometryType()) - assert stats == {"min_values": None, "max_values": None, "null_counts": 1} - - _validate_geospatial_fields(fields, {}, [], []) - with pytest.raises(ValueError, match="file.format"): - _validate_geospatial_fields(fields, {"file.format": "orc"}, [], []) - with pytest.raises(ValueError, match="primary keys"): - _validate_geospatial_fields(fields, {}, ["geom"], []) - with pytest.raises(ValueError, match="format-version"): - _validate_geospatial_fields( - fields, {"metadata.iceberg.storage": "table-location"}, [], []) - _validate_geospatial_fields( - fields, - {"metadata.iceberg.storage": "table-location", - "metadata.iceberg.format-version": "3"}, - [], []) diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py index fd8e5f7727c3..a640b8d31f26 100644 --- a/paimon-python/pypaimon/tests/write/table_write_test.py +++ b/paimon-python/pypaimon/tests/write/table_write_test.py @@ -31,7 +31,6 @@ from pypaimon.common.json_util import JSON from pypaimon.common.options.core_options import CoreOptions from pypaimon.manifest.manifest_list_manager import ManifestListManager -from pypaimon.schema.data_types import DataField, GeometryType from pypaimon.write.table_write import TableWrite from pypaimon.write.writer.append_only_data_writer import AppendOnlyDataWriter @@ -1901,19 +1900,6 @@ def test_validate_schema_allows_binary_family_for_write_cols(self): ('payload', pa.binary(4)), ])) - def test_write_rejects_geospatial_type_without_native_parquet_annotation(self): - schema = Schema([DataField(0, 'geom', GeometryType())]) - self.catalog.create_table( - 'default.test_reject_geospatial_write', schema, False) - table = self.catalog.get_table( - 'default.test_reject_geospatial_write') - data = pa.table({'geom': [bytes.fromhex( - '0101000000000000000000f03f0000000000000040')]}) - - with self.assertRaisesRegex( - NotImplementedError, 'native Parquet GEOMETRY or GEOGRAPHY'): - self._commit_arrow(table, data) - @parameterized.expand([('parquet',), ('orc',), ('avro',)]) def test_write_time_type(self, file_format): time_schema = pa.schema([ diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py index ea264276f092..f0a68bcf6fc1 100644 --- a/paimon-python/pypaimon/write/table_write.py +++ b/paimon-python/pypaimon/write/table_write.py @@ -19,10 +19,7 @@ import pyarrow as pa -from pypaimon.schema.data_types import ( - PyarrowFieldParser, - _contains_geospatial_type, -) +from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.row.blob import BlobConsumer from pypaimon.write.row_utils import ( @@ -332,15 +329,6 @@ def _release_prepared_indexes(self) -> None: release() def _validate_pyarrow_schema(self, data_schema: pa.Schema): - if any(_contains_geospatial_type(field.type) - for field in self.table.table_schema.fields): - raise NotImplementedError( - "PyPaimon does not support writing geospatial columns because " - "PyArrow cannot produce the native Parquet GEOMETRY or " - "GEOGRAPHY logical annotation required by Paimon and " - "Iceberg v3." - ) - if self._is_compatible_pyarrow_schema(data_schema, self.table_pyarrow_schema): return diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py index 49aa2e8de057..e34127ddaf00 100644 --- a/paimon-python/pypaimon/write/writer/data_writer.py +++ b/paimon-python/pypaimon/write/writer/data_writer.py @@ -26,7 +26,7 @@ from pypaimon.data.timestamp import Timestamp from pypaimon.manifest.schema.data_file_meta import DataFileMeta from pypaimon.manifest.schema.simple_stats import SimpleStats -from pypaimon.schema.data_types import GeographyType, GeometryType, PyarrowFieldParser +from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.table.bucket_mode import BucketMode from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.writer.mosaic_writer_options import create_mosaic_writer_options @@ -273,15 +273,12 @@ def _write_data_to_file(self, data: pa.Table): # key stats & value stats value_stats_enabled = self.options.metadata_stats_enabled() if value_stats_enabled: - if self.table.is_primary_key_table: - stats_fields = self.table.fields - else: - stats_fields = self._resolve_stats_fields( - data.schema, self.table.fields) + stats_fields = self.table.fields if self.table.is_primary_key_table \ + else PyarrowFieldParser.to_paimon_schema(data.schema) else: stats_fields = self.table.trimmed_primary_keys_fields column_stats = { - field.name: self._get_column_stats(data, field.name, field.type) + field.name: self._get_column_stats(data, field.name) for field in stats_fields } key_fields = self.trimmed_primary_keys_fields @@ -454,7 +451,7 @@ def _collect_value_stats(self, data: pa.Table, fields: List, if column_stats is None or not column_stats: column_stats = { - field.name: self._get_column_stats(data, field.name, field.type) + field.name: self._get_column_stats(data, field.name) for field in fields } @@ -469,24 +466,8 @@ def _collect_value_stats(self, data: pa.Table, fields: List, ) @staticmethod - def _resolve_stats_fields(arrow_schema, table_fields: List) -> List: - inferred_fields = PyarrowFieldParser.to_paimon_schema(arrow_schema) - table_fields_by_name = {field.name: field for field in table_fields} - return [ - table_fields_by_name.get(field.name, field) - for field in inferred_fields - ] - - @staticmethod - def _get_column_stats(record_batch: pa.RecordBatch, column_name: str, - data_type=None) -> Dict: + def _get_column_stats(record_batch: pa.RecordBatch, column_name: str) -> Dict: column_array = record_batch.column(column_name) - if isinstance(data_type, (GeometryType, GeographyType)): - return { - "min_values": None, - "max_values": None, - "null_counts": column_array.null_count, - } if column_array.null_count == len(column_array): return { "min_values": None, diff --git a/paimon-python/setup.py b/paimon-python/setup.py index 3e3489cdb631..a5b1b6209e07 100644 --- a/paimon-python/setup.py +++ b/paimon-python/setup.py @@ -200,9 +200,6 @@ def read_requirements(): 'hdfs': [ 'hdfs-native>=0.13,<1; python_version >= "3.10" and platform_system != "Windows"', ], - 'geoarrow': [ - 'geoarrow-pyarrow>=0.2.0; python_version >= "3.9"', - ], }, description="Apache Paimon Python API", long_description=long_description, From aa276e99416f5f8b8708888344f1df86ddcafa06 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 22:16:46 +0800 Subject: [PATCH 5/6] [docs] Document geospatial type support --- docs/docs/concepts/data-types.md | 25 +++++++++++++++++++ docs/docs/concepts/spec/fileformat.md | 12 +++++++++ docs/docs/flink/quick-start.mdx | 6 +++++ docs/docs/iceberg/index.md | 3 ++- docs/docs/iceberg/rest-catalog.mdx | 7 +++++- .../primary-key-table/sequence-rowkind.mdx | 2 +- docs/docs/program-api/java-api.mdx | 18 +++++++++++++ docs/docs/spark/quick-start.mdx | 18 ++++++++++++- 8 files changed, 87 insertions(+), 4 deletions(-) diff --git a/docs/docs/concepts/data-types.md b/docs/docs/concepts/data-types.md index d8149e0c078b..6d30407b59f3 100644 --- a/docs/docs/concepts/data-types.md +++ b/docs/docs/concepts/data-types.md @@ -76,6 +76,23 @@ All data types supported by Paimon are as follows: BYTES is a synonym for VARBINARY(2147483647). + + GEOMETRY
+ GEOMETRY(crs) + + Data type of a planar geometry encoded as OGC Well-Known Binary (WKB).

+ The optional crs identifies the coordinate reference system. The default is OGC:CRS84. + + + + GEOGRAPHY
+ GEOGRAPHY(crs)
+ GEOGRAPHY(crs, algorithm) + + Data type of a geography whose edges are interpolated on the surface of the coordinate reference system, encoded as OGC Well-Known Binary (WKB).

+ The default crs is OGC:CRS84 and the default edge interpolation algorithm is spherical. Supported algorithms are spherical, vincenty, thomas, andoyer, and karney. + + DECIMAL
DECIMAL(p)
@@ -189,3 +206,11 @@ All data types supported by Paimon are as follows: + +:::note Geospatial type availability + +`GEOMETRY` and `GEOGRAPHY` columns require Parquet for data, per-level, and changelog files. They cannot be used as primary, partition, bucket, or sequence keys. + +The Paimon Java API supports geospatial columns. Spark 4.1 supports CRSs recognized by Spark and supports only the `spherical` geography edge algorithm. Flink SQL, Spark 3.x, and Spark 4.0 reject geospatial columns instead of exposing them as binary and losing the CRS or edge algorithm. + +::: diff --git a/docs/docs/concepts/spec/fileformat.md b/docs/docs/concepts/spec/fileformat.md index 252be7f8de85..81ffdbf3b9c7 100644 --- a/docs/docs/concepts/spec/fileformat.md +++ b/docs/docs/concepts/spec/fileformat.md @@ -63,6 +63,16 @@ The following table lists the type mapping from Paimon type to Parquet type. BINARY + + GEOMETRY(crs) + BINARY + GEOMETRY(crs) + + + GEOGRAPHY(crs, algorithm) + BINARY + GEOGRAPHY(crs, algorithm) + DECIMAL(P, S) P <= 9: INT32, P <= 18: INT64, P > 18: FIXED_LEN_BYTE_ARRAY @@ -142,8 +152,10 @@ The following table lists the type mapping from Paimon type to Parquet type. Limitations: + 1. [Parquet does not support nullable map keys](https://github.com/apache/parquet-format/blob/master/LogicalTypes#maps). 2. Parquet TIMESTAMP type with precision 9 will use INT96, but this int96 is a time zone converted value and requires additional adjustments. +3. Tables containing `GEOMETRY` or `GEOGRAPHY` columns must use Parquet for `file.format`, every entry in `file.format-per-level`, and `changelog-file.format` when configured. ## AVRO diff --git a/docs/docs/flink/quick-start.mdx b/docs/docs/flink/quick-start.mdx index a9054a06738e..e39426c11811 100644 --- a/docs/docs/flink/quick-start.mdx +++ b/docs/docs/flink/quick-start.mdx @@ -31,6 +31,12 @@ under the License. This documentation is a guide for using Paimon in Flink. +:::warning + +Flink SQL does not currently support Paimon `GEOMETRY` or `GEOGRAPHY` columns. Reading, writing, or copying a table whose schema contains either type fails explicitly instead of exposing the column as `VARBINARY` and losing its CRS or edge algorithm. Use the Paimon Java API or Spark 4.1 for geospatial columns. + +::: + ## Jars Paimon currently supports Flink 2.2, 2.1, 2.0, 1.20, 1.19, 1.18, 1.17, 1.16. We recommend the latest Flink version for a better experience. diff --git a/docs/docs/iceberg/index.md b/docs/docs/iceberg/index.md index 65161647f816..5961506e397d 100644 --- a/docs/docs/iceberg/index.md +++ b/docs/docs/iceberg/index.md @@ -113,7 +113,8 @@ Paimon Iceberg compatibility currently supports the following data types. **Note on Geospatial Types:** - `GEOMETRY` and `GEOGRAPHY` values use OGC Well-Known Binary (WKB). The default CRS is `OGC:CRS84`, and the default geography edge algorithm is `spherical`. - Geospatial columns require Parquet for data, per-level, and changelog files. When Iceberg metadata is enabled, set `metadata.iceberg.format-version` to `3`. -- Spark SQL supports geospatial columns with Spark 4.1 or later. Spark 3.x, Spark 4.0, and Flink SQL reject these columns instead of exposing them as binary and losing the CRS or edge algorithm. +- Spark SQL supports geospatial columns in Spark 4.1 for CRSs recognized by Spark, with the `spherical` geography edge algorithm. Spark 3.x, Spark 4.0, and Flink SQL reject these columns instead of exposing them as binary and losing the CRS or edge algorithm. +- When Iceberg metadata is enabled, a `GEOGRAPHY` CRS cannot contain a comma, including in nested columns, because Iceberg's geospatial type grammar uses commas to separate parameters. - Iceberg REST catalog publication does not yet support geospatial columns. Use `table-location`, `hadoop-catalog`, or `hive-catalog` metadata storage instead. - Geospatial columns cannot be primary, partition, bucket, or sequence keys. Paimon records null counts but does not publish byte-wise lower or upper bounds for WKB values. diff --git a/docs/docs/iceberg/rest-catalog.mdx b/docs/docs/iceberg/rest-catalog.mdx index 457b138385e6..21d6fbc8fc7b 100644 --- a/docs/docs/iceberg/rest-catalog.mdx +++ b/docs/docs/iceberg/rest-catalog.mdx @@ -35,6 +35,12 @@ You need to provide information about Rest Catalog by setting options prefixed w `'metadata.iceberg.rest.uri' = 'https://localhost/'`. Paimon will try to use these options to initialize an iceberg rest catalog, and use this rest catalog to commit metadata. +:::warning + +Tables containing `GEOMETRY` or `GEOGRAPHY` columns cannot use `'metadata.iceberg.storage' = 'rest-catalog'` because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial types. Use `table-location`, `hadoop-catalog`, or `hive-catalog` metadata storage instead. + +::: + **Dependency:** This feature needs dependency: @@ -128,4 +134,3 @@ There are some cases when committing to iceberg rest catalog: 1. table not exists in iceberg rest-catalog. It'll create the table in rest catalog first, and commit metadata. 2. table exists in iceberg rest-catalog and is compatible with the base metadata stored in the separate directory. It'll directly get the table and commit metadata. 3. table exists, and isn't compatible with the base metadata stored in the separate directory. It'll **drop the table and recreate the table**, then commit metadata. - diff --git a/docs/docs/primary-key-table/sequence-rowkind.mdx b/docs/docs/primary-key-table/sequence-rowkind.mdx index 8a0f49fb7307..bc18e4fcb3e3 100644 --- a/docs/docs/primary-key-table/sequence-rowkind.mdx +++ b/docs/docs/primary-key-table/sequence-rowkind.mdx @@ -55,7 +55,7 @@ CREATE TABLE my_table ( The record with the largest `sequence.field` value will be the last to merge, if the values are the same, the input -order will be used to determine which one is the last one. `sequence.field` supports fields of all data types. +order will be used to determine which one is the last one. `sequence.field` does not support `GEOMETRY`, `GEOGRAPHY`, or managed `BLOB` fields. You can define multiple fields for `sequence.field`, for example `'update_time,flag'`, multiple fields will be compared in order. diff --git a/docs/docs/program-api/java-api.mdx b/docs/docs/program-api/java-api.mdx index 887f36c1bbc1..60695034015d 100644 --- a/docs/docs/program-api/java-api.mdx +++ b/docs/docs/program-api/java-api.mdx @@ -459,6 +459,24 @@ public class StreamWriteTable { | map | org.apache.paimon.data.InternalMap | | InternalRow | org.apache.paimon.data.InternalRow | +### Geospatial Types + +Use the public type factories to declare geospatial columns: + +```java +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.EdgeAlgorithm; + +DataType defaultGeometry = DataTypes.GEOMETRY(); +DataType projectedGeometry = DataTypes.GEOMETRY("EPSG:3857"); +DataType defaultGeography = DataTypes.GEOGRAPHY(); +DataType karneyGeography = + DataTypes.GEOGRAPHY("OGC:CRS84", EdgeAlgorithm.KARNEY); +``` + +`GEOMETRY` and `GEOGRAPHY` values are represented as OGC Well-Known Binary (WKB) byte arrays in Paimon's internal row API. The default CRS is `OGC:CRS84`, and the default geography edge interpolation algorithm is `EdgeAlgorithm.SPHERICAL`. + ## Predicate Types | SQL Predicate | Paimon Predicate | diff --git a/docs/docs/spark/quick-start.mdx b/docs/docs/spark/quick-start.mdx index d62d3a2b4ee5..9da878022c4e 100644 --- a/docs/docs/spark/quick-start.mdx +++ b/docs/docs/spark/quick-start.mdx @@ -374,6 +374,16 @@ All Spark's data types are available in package `org.apache.spark.sql.types`. VarBinaryType, BinaryType true + + GeometryType (Spark 4.1) + GeometryType + true + + + GeographyType (Spark 4.1) + GeographyType + true + VariantType(Spark4.0+) VariantType @@ -384,10 +394,16 @@ All Spark's data types are available in package `org.apache.spark.sql.types`. :::warning +Native `GeometryType` and `GeographyType` conversion is supported only in Spark 4.1 and only for CRSs recognized by Spark. Spark 4.1 supports only the `spherical` geography edge algorithm, so Paimon geography types using `vincenty`, `thomas`, `andoyer`, or `karney` cannot be converted. Spark 3.x and Spark 4.0 reject Paimon geospatial columns instead of exposing them as `BinaryType`, which would lose the CRS or edge algorithm. Paimon does not support Spark geospatial types with mixed SRIDs. + +::: + +:::warning + Due to the previous design, in Spark3.3 and below, Paimon will map both Paimon's TimestampType and LocalZonedTimestamp to Spark's TimestampType, and only correctly handle with TimestampType. Therefore, when using Spark3.3 and below, reads Paimon table with LocalZonedTimestamp type written by other engines, such as Flink, the query result of LocalZonedTimestamp type will have time zone offset, which needs to be adjusted manually. When using Spark3.4 and above, all timestamp types can be parsed correctly. -::: \ No newline at end of file +::: From a4b00f2fad81311e41ac0ac8c291cb009f76627f Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 16 Aug 2026 23:23:08 +0800 Subject: [PATCH 6/6] [core] Harden geospatial interoperability --- docs/docs/concepts/data-types.md | 4 +- docs/docs/concepts/spec/fileformat.md | 2 +- docs/docs/iceberg/index.md | 4 +- docs/docs/spark/quick-start.mdx | 2 +- .../paimon/iceberg/IcebergCommitCallback.java | 3 + .../apache/paimon/schema/SchemaManager.java | 15 ++++ .../paimon/schema/SchemaValidation.java | 59 ++++++++++++---- .../paimon/schema/SchemaManagerTest.java | 69 +++++++++++++++++++ .../paimon/schema/SchemaValidationTest.java | 32 +++++++++ .../parquet/ParquetSchemaConverter.java | 9 +-- .../ColumnCompressionPageWriteStore.java | 13 ++++ .../parquet/ParquetFormatReadWriteTest.java | 53 +++++++++++++- .../parquet/ParquetSchemaConverterTest.java | 24 +++++++ .../spark/sql/GeospatialTypeSQLTest.scala | 16 ++++- 14 files changed, 278 insertions(+), 27 deletions(-) diff --git a/docs/docs/concepts/data-types.md b/docs/docs/concepts/data-types.md index 6d30407b59f3..a72ae3fe01fa 100644 --- a/docs/docs/concepts/data-types.md +++ b/docs/docs/concepts/data-types.md @@ -209,8 +209,8 @@ All data types supported by Paimon are as follows: :::note Geospatial type availability -`GEOMETRY` and `GEOGRAPHY` columns require Parquet for data, per-level, and changelog files. They cannot be used as primary, partition, bucket, or sequence keys. +`GEOMETRY` and `GEOGRAPHY` columns require Parquet for data, per-level, and changelog files. They cannot be used as primary, partition, bucket, sequence, or clustering keys. -The Paimon Java API supports geospatial columns. Spark 4.1 supports CRSs recognized by Spark and supports only the `spherical` geography edge algorithm. Flink SQL, Spark 3.x, and Spark 4.0 reject geospatial columns instead of exposing them as binary and losing the CRS or edge algorithm. +The Paimon Java API supports geospatial columns. Spark 4.1 requires `spark.sql.geospatial.enabled=true`, supports CRSs recognized by Spark, and supports only the `spherical` geography edge algorithm. Flink SQL, Spark 3.x, and Spark 4.0 reject geospatial columns instead of exposing them as binary and losing the CRS or edge algorithm. ::: diff --git a/docs/docs/concepts/spec/fileformat.md b/docs/docs/concepts/spec/fileformat.md index 81ffdbf3b9c7..65a4d5af0c44 100644 --- a/docs/docs/concepts/spec/fileformat.md +++ b/docs/docs/concepts/spec/fileformat.md @@ -155,7 +155,7 @@ Limitations: 1. [Parquet does not support nullable map keys](https://github.com/apache/parquet-format/blob/master/LogicalTypes#maps). 2. Parquet TIMESTAMP type with precision 9 will use INT96, but this int96 is a time zone converted value and requires additional adjustments. -3. Tables containing `GEOMETRY` or `GEOGRAPHY` columns must use Parquet for `file.format`, every entry in `file.format-per-level`, and `changelog-file.format` when configured. +3. Tables containing `GEOMETRY` or `GEOGRAPHY` columns must use Parquet for `file.format`, every entry in `file.format.per.level`, and `changelog-file.format` when configured. ## AVRO diff --git a/docs/docs/iceberg/index.md b/docs/docs/iceberg/index.md index 5961506e397d..a3bc42631213 100644 --- a/docs/docs/iceberg/index.md +++ b/docs/docs/iceberg/index.md @@ -113,9 +113,9 @@ Paimon Iceberg compatibility currently supports the following data types. **Note on Geospatial Types:** - `GEOMETRY` and `GEOGRAPHY` values use OGC Well-Known Binary (WKB). The default CRS is `OGC:CRS84`, and the default geography edge algorithm is `spherical`. - Geospatial columns require Parquet for data, per-level, and changelog files. When Iceberg metadata is enabled, set `metadata.iceberg.format-version` to `3`. -- Spark SQL supports geospatial columns in Spark 4.1 for CRSs recognized by Spark, with the `spherical` geography edge algorithm. Spark 3.x, Spark 4.0, and Flink SQL reject these columns instead of exposing them as binary and losing the CRS or edge algorithm. +- Spark SQL supports geospatial columns in Spark 4.1 when `spark.sql.geospatial.enabled=true`, for CRSs recognized by Spark, with the `spherical` geography edge algorithm. Spark 3.x, Spark 4.0, and Flink SQL reject these columns instead of exposing them as binary and losing the CRS or edge algorithm. - When Iceberg metadata is enabled, a `GEOGRAPHY` CRS cannot contain a comma, including in nested columns, because Iceberg's geospatial type grammar uses commas to separate parameters. - Iceberg REST catalog publication does not yet support geospatial columns. Use `table-location`, `hadoop-catalog`, or `hive-catalog` metadata storage instead. -- Geospatial columns cannot be primary, partition, bucket, or sequence keys. Paimon records null counts but does not publish byte-wise lower or upper bounds for WKB values. +- Geospatial columns cannot be primary, partition, bucket, sequence, or clustering keys. Paimon records null counts but does not publish byte-wise lower or upper bounds for WKB values. ::: diff --git a/docs/docs/spark/quick-start.mdx b/docs/docs/spark/quick-start.mdx index 9da878022c4e..b552aa3b0817 100644 --- a/docs/docs/spark/quick-start.mdx +++ b/docs/docs/spark/quick-start.mdx @@ -394,7 +394,7 @@ All Spark's data types are available in package `org.apache.spark.sql.types`. :::warning -Native `GeometryType` and `GeographyType` conversion is supported only in Spark 4.1 and only for CRSs recognized by Spark. Spark 4.1 supports only the `spherical` geography edge algorithm, so Paimon geography types using `vincenty`, `thomas`, `andoyer`, or `karney` cannot be converted. Spark 3.x and Spark 4.0 reject Paimon geospatial columns instead of exposing them as `BinaryType`, which would lose the CRS or edge algorithm. Paimon does not support Spark geospatial types with mixed SRIDs. +Native `GeometryType` and `GeographyType` conversion is supported only in Spark 4.1 and only for CRSs recognized by Spark. Enable it explicitly in production with `--conf spark.sql.geospatial.enabled=true`; Spark enables it automatically only in its test environment. Spark 4.1 supports only the `spherical` geography edge algorithm, so Paimon geography types using `vincenty`, `thomas`, `andoyer`, or `karney` cannot be converted. Spark 3.x and Spark 4.0 reject Paimon geospatial columns instead of exposing them as `BinaryType`, which would lose the CRS or edge algorithm. Paimon does not support Spark geospatial types with mixed SRIDs. ::: diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java index cfeb6acb1c7d..af6c86a9ab0b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java @@ -53,6 +53,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.SchemaValidation; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.CommitCallback; @@ -1967,6 +1968,8 @@ private IcebergSchema get(long schemaId) { TableSchema schema = schemaManager.schema(id); // backstop: reject variant on each schema as it is emitted checkVariantNotPublishable(schema.logicalRowType()); + SchemaValidation.validateIcebergGeospatialTypes( + schema.logicalRowType(), table.coreOptions()); return IcebergSchema.create(schema); }); } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index bcb4a59b4f00..28175245af11 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -25,6 +25,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.schema.ColumnDirectiveUtils.ConvertedColumn; import org.apache.paimon.schema.SchemaChange.AddColumn; import org.apache.paimon.schema.SchemaChange.DropColumn; @@ -1202,11 +1203,25 @@ protected void updateLastColumn(int depth, List newFields, String fie @VisibleForTesting public boolean commit(TableSchema newSchema) throws Exception { SchemaValidation.validateTableSchema(newSchema); + validateHistoricalIcebergGeospatialTypes(newSchema); SchemaValidation.validateFallbackBranch(this, newSchema); Path schemaPath = toSchemaPath(newSchema.id()); return fileIO.tryToWriteAtomic(schemaPath, newSchema.toString()); } + private void validateHistoricalIcebergGeospatialTypes(TableSchema newSchema) { + CoreOptions options = new CoreOptions(newSchema.options()); + IcebergOptions.StorageType storage = + options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE); + if (storage == IcebergOptions.StorageType.DISABLED) { + return; + } + + for (TableSchema schema : listAll()) { + SchemaValidation.validateIcebergGeospatialTypes(schema.logicalRowType(), options); + } + } + /** Read schema for schema id. */ public TableSchema schema(long id) { return fromPath(fileIO, toSchemaPath(id)); diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index 8663128bf8b5..1fe54ea47905 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -485,20 +485,25 @@ private static void validateGeospatialTypes( "Geometry and geography columns require '%s' to be parquet, but was '%s'.", CoreOptions.CHANGELOG_FILE_FORMAT.key(), options.changelogFileFormat()); - IcebergOptions.StorageType icebergStorage = - options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE); - if (icebergStorage != IcebergOptions.StorageType.DISABLED) { - checkArgument( - options.toConfiguration().get(IcebergOptions.FORMAT_VERSION) == 3, - "Geometry and geography columns require '%s'='3' when Iceberg metadata is enabled.", - IcebergOptions.FORMAT_VERSION.key()); - checkArgument( - icebergStorage != IcebergOptions.StorageType.REST_CATALOG, - "Geometry and geography columns do not support '%s'='%s' because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial types.", - IcebergOptions.METADATA_ICEBERG_STORAGE.key(), - IcebergOptions.StorageType.REST_CATALOG); - validateIcebergGeographyCrs(rowType); - } + validateIcebergGeospatialTypes(rowType, options); + + List geospatialClusteringColumns = + schema.fields().stream() + .filter(field -> options.clusteringColumns().contains(field.name())) + .filter( + field -> + containsType( + field.type(), + type -> + type.isAnyOf( + DataTypeRoot.GEOMETRY, + DataTypeRoot.GEOGRAPHY))) + .map(DataField::name) + .collect(Collectors.toList()); + checkArgument( + geospatialClusteringColumns.isEmpty(), + "Geometry and geography columns cannot be clustering columns: %s.", + geospatialClusteringColumns); Set geospatialFields = schema.fields().stream() @@ -524,6 +529,32 @@ private static void validateGeospatialTypes( geospatialSequenceFields); } + /** Validate geospatial types in a schema that will be published as Iceberg metadata. */ + public static void validateIcebergGeospatialTypes(DataType dataType, CoreOptions options) { + boolean hasGeospatial = + containsType( + dataType, + type -> type.isAnyOf(DataTypeRoot.GEOMETRY, DataTypeRoot.GEOGRAPHY)); + if (!hasGeospatial) { + return; + } + + IcebergOptions.StorageType icebergStorage = + options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE); + if (icebergStorage != IcebergOptions.StorageType.DISABLED) { + checkArgument( + options.toConfiguration().get(IcebergOptions.FORMAT_VERSION) == 3, + "Geometry and geography columns require '%s'='3' when Iceberg metadata is enabled.", + IcebergOptions.FORMAT_VERSION.key()); + checkArgument( + icebergStorage != IcebergOptions.StorageType.REST_CATALOG, + "Geometry and geography columns do not support '%s'='%s' because the bundled Iceberg REST client cannot parse Iceberg v3 geospatial types.", + IcebergOptions.METADATA_ICEBERG_STORAGE.key(), + IcebergOptions.StorageType.REST_CATALOG); + validateIcebergGeographyCrs(dataType); + } + } + private static void validateStartupMode(CoreOptions options) { if (options.startupMode() == CoreOptions.StartupMode.FROM_TIMESTAMP) { checkExactOneOptionExistInMode( diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index c080d0eef1e3..044513251c2c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.reader.RecordReaderIterator; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -172,6 +173,74 @@ public void testUpdateOptions() throws Exception { assertThat(latest.get().options()).containsEntry("new_k", "new_v"); } + @Test + public void testEnableIcebergMetadataValidatesHistoricalGeospatialSchemas() throws Exception { + Map geospatialOptions = new HashMap<>(); + geospatialOptions.put(CoreOptions.BUCKET.key(), "-1"); + Schema geospatialSchema = + new Schema( + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField(1, "geom", DataTypes.GEOMETRY())), + Collections.emptyList(), + Collections.emptyList(), + geospatialOptions, + ""); + + retryArtificialException(() -> manager.createTable(geospatialSchema)); + retryArtificialException(() -> manager.commitChanges(SchemaChange.dropColumn("geom"))); + + assertThatThrownBy( + () -> + retryArtificialException( + () -> + manager.commitChanges( + SchemaChange.setOption( + IcebergOptions + .METADATA_ICEBERG_STORAGE + .key(), + "table-location")))) + .hasStackTraceContaining( + "Geometry and geography columns require 'metadata.iceberg.format-version'='3'"); + + assertThatThrownBy( + () -> + retryArtificialException( + () -> + manager.commitChanges( + Arrays.asList( + SchemaChange.setOption( + IcebergOptions + .METADATA_ICEBERG_STORAGE + .key(), + "rest-catalog"), + SchemaChange.setOption( + IcebergOptions + .FORMAT_VERSION + .key(), + "3"))))) + .hasStackTraceContaining( + "Geometry and geography columns do not support 'metadata.iceberg.storage'='rest-catalog'"); + + assertThatCode( + () -> + retryArtificialException( + () -> + manager.commitChanges( + Arrays.asList( + SchemaChange.setOption( + IcebergOptions + .METADATA_ICEBERG_STORAGE + .key(), + "table-location"), + SchemaChange.setOption( + IcebergOptions + .FORMAT_VERSION + .key(), + "3"))))) + .doesNotThrowAnyException(); + } + @Test public void testChangeMapStorageLayoutForExistingField() throws Exception { retryArtificialException(() -> manager.createTable(mapStorageLayoutSchema("default"))); diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java index 8b57bd7bb9da..5f5aa2fce0eb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java @@ -1776,6 +1776,38 @@ fields, emptyList(), emptyList(), bucketOptions))) singletonList("id"), sequenceOptions))) .hasMessage("Geometry and geography columns cannot be sequence fields: [geog]."); + + Map clusteringOptions = new HashMap<>(); + clusteringOptions.put(CoreOptions.CLUSTERING_COLUMNS.key(), "geom"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + fields, + emptyList(), + emptyList(), + clusteringOptions))) + .hasMessage("Geometry and geography columns cannot be clustering columns: [geom]."); + + List nestedFields = + Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField( + 1, + "nested", + DataTypes.ROW(DataTypes.FIELD(2, "geog", DataTypes.GEOGRAPHY())))); + Map nestedClusteringOptions = new HashMap<>(); + nestedClusteringOptions.put(CoreOptions.CLUSTERING_COLUMNS.key(), "nested"); + assertThatThrownBy( + () -> + validateTableSchema( + geospatialSchema( + nestedFields, + emptyList(), + emptyList(), + nestedClusteringOptions))) + .hasMessage( + "Geometry and geography columns cannot be clustering columns: [nested]."); } private TableSchema geospatialSchema( diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 053123c08511..912baec0721c 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -369,10 +369,11 @@ public static DataField convertToPaimonField(Type parquetType) { instanceof LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) { LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geography = (LogicalTypeAnnotation.GeographyLogicalTypeAnnotation) logicalType; - paimonDataType = - DataTypes.GEOGRAPHY( - geography.getCrs(), - EdgeAlgorithm.valueOf(geography.getAlgorithm().name())); + EdgeAlgorithm algorithm = + geography.getAlgorithm() == null + ? null + : EdgeAlgorithm.valueOf(geography.getAlgorithm().name()); + paimonDataType = DataTypes.GEOGRAPHY(geography.getCrs(), algorithm); } else { paimonDataType = DataTypes.BYTES(); } diff --git a/paimon-format/src/main/java/org/apache/parquet/hadoop/ColumnCompressionPageWriteStore.java b/paimon-format/src/main/java/org/apache/parquet/hadoop/ColumnCompressionPageWriteStore.java index 12250aa3f8df..a5c267b92c8a 100644 --- a/paimon-format/src/main/java/org/apache/parquet/hadoop/ColumnCompressionPageWriteStore.java +++ b/paimon-format/src/main/java/org/apache/parquet/hadoop/ColumnCompressionPageWriteStore.java @@ -44,6 +44,7 @@ import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder; import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder; import org.apache.parquet.io.ParquetEncodingException; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.util.AutoCloseables; import org.slf4j.Logger; @@ -76,6 +77,7 @@ public class ColumnCompressionPageWriteStore implements PageWriteStore, BloomFil private static final class ColumnChunkPageWriter implements PageWriter, BloomFilterWriter { private final ColumnDescriptor path; + private final boolean geospatial; private final CompressionCodecFactory.BytesInputCompressor compressor; private final ByteArrayOutputStream tempOutputStream = new ByteArrayOutputStream(); @@ -123,6 +125,11 @@ private ColumnChunkPageWriter( int rowGroupOrdinal, int columnOrdinal) { this.path = path; + LogicalTypeAnnotation logicalType = path.getPrimitiveType().getLogicalTypeAnnotation(); + this.geospatial = + logicalType instanceof LogicalTypeAnnotation.GeometryLogicalTypeAnnotation + || logicalType + instanceof LogicalTypeAnnotation.GeographyLogicalTypeAnnotation; this.compressor = compressor; this.releaser = new ByteBufferReleaser(allocator); this.buf = new ConcatenatingByteBufferCollector(allocator); @@ -426,6 +433,12 @@ private void mergeColumnStatistics( Statistics statistics, SizeStatistics sizeStatistics, GeospatialStatistics geospatialStatistics) { + if (geospatial && statistics != null && statistics.isNumNullsSet()) { + statistics = + Statistics.getBuilderForReading(path.getPrimitiveType()) + .withNumNulls(statistics.getNumNulls()) + .build(); + } totalSizeStatistics.mergeStatistics(sizeStatistics); if (!totalSizeStatistics.isValid()) { sizeStatistics = null; diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index fa812286e02c..2784f157a113 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -20,8 +20,10 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.format.FileFormat; @@ -51,6 +53,7 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; @@ -83,12 +86,28 @@ public void testGeospatialWkbRoundTrip() throws Exception { DataTypes.ROW( DataTypes.FIELD(0, "geom", DataTypes.GEOMETRY()), DataTypes.FIELD(1, "geog", DataTypes.GEOGRAPHY()), - DataTypes.FIELD(2, "geometries", DataTypes.ARRAY(DataTypes.GEOMETRY()))); + DataTypes.FIELD(2, "geometries", DataTypes.ARRAY(DataTypes.GEOMETRY())), + DataTypes.FIELD( + 3, + "geospatial_map", + DataTypes.MAP(DataTypes.GEOMETRY(), DataTypes.GEOGRAPHY())), + DataTypes.FIELD( + 4, + "nested", + DataTypes.ROW( + DataTypes.FIELD(5, "nested_geom", DataTypes.GEOMETRY())))); + Map map = new LinkedHashMap<>(); + map.put(pointWkb, pointWkb); write( fileFormat().createWriterFactory(rowType), file, - GenericRow.of(pointWkb, pointWkb, new GenericArray(new Object[] {pointWkb, null}))); + GenericRow.of( + pointWkb, + pointWkb, + new GenericArray(new Object[] {pointWkb, null}), + GenericMap.fromBinaryKeyMap(map), + GenericRow.of(pointWkb))); try (RecordReader reader = fileFormat() @@ -102,6 +121,10 @@ public void testGeospatialWkbRoundTrip() throws Exception { InternalArray geometries = row.getArray(2); Assertions.assertThat(geometries.getBinary(0)).isEqualTo(pointWkb); Assertions.assertThat(geometries.isNullAt(1)).isTrue(); + InternalMap geospatialMap = row.getMap(3); + Assertions.assertThat(geospatialMap.keyArray().getBinary(0)).isEqualTo(pointWkb); + Assertions.assertThat(geospatialMap.valueArray().getBinary(0)).isEqualTo(pointWkb); + Assertions.assertThat(row.getRow(4, 1).getBinary(0)).isEqualTo(pointWkb); } try (ParquetFileReader reader = @@ -111,7 +134,33 @@ public void testGeospatialWkbRoundTrip() throws Exception { for (ColumnChunkMetaData column : reader.getFooter().getBlocks().get(0).getColumns()) { columns.put(column.getPath().toDotString(), column); } + Assertions.assertThat(columns) + .containsKeys( + "geom", + "geog", + "geometries.list.element", + "geospatial_map.key_value.key", + "geospatial_map.key_value.value", + "nested.nested_geom"); + for (ColumnChunkMetaData column : columns.values()) { + Assertions.assertThat(column.getStatistics().hasNonNullValue()) + .as(column.getPath().toDotString()) + .isFalse(); + Assertions.assertThat(column.getStatistics().isNumNullsSet()) + .as(column.getPath().toDotString()) + .isTrue(); + } + Assertions.assertThat( + columns.get("geometries.list.element").getStatistics().getNumNulls()) + .isEqualTo(1); Assertions.assertThat(columns.get("geom").getGeospatialStatistics()).isNotNull(); + Assertions.assertThat(columns.get("geometries.list.element").getGeospatialStatistics()) + .isNotNull(); + Assertions.assertThat( + columns.get("geospatial_map.key_value.key").getGeospatialStatistics()) + .isNotNull(); + Assertions.assertThat(columns.get("nested.nested_geom").getGeospatialStatistics()) + .isNotNull(); } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index 91f78ff47f4d..f40550831dff 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -182,4 +182,28 @@ public void testGeospatialLogicalTypesRoundTrip() { .isEqualTo("KARNEY"); assertThat(expected).isEqualTo(convertToPaimonRowType(messageType)); } + + @Test + public void testGeographyLogicalTypeDefaults() { + MessageType messageType = + new MessageType( + "geography-defaults", + Types.primitive(BINARY, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.geographyType()) + .named("default_geography") + .withId(0), + Types.primitive(BINARY, Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.geographyType("EPSG:4326", null)) + .named("default_algorithm") + .withId(1)); + + RowType expected = + new RowType( + Arrays.asList( + new DataField(0, "default_geography", DataTypes.GEOGRAPHY()), + new DataField( + 1, "default_algorithm", DataTypes.GEOGRAPHY("EPSG:4326")))); + + assertThat(expected).isEqualTo(convertToPaimonRowType(messageType)); + } } diff --git a/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala b/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala index 94e4c70c42db..204c7d43d483 100644 --- a/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala +++ b/paimon-spark/paimon-spark-4.1/src/test/scala/org/apache/paimon/spark/sql/GeospatialTypeSQLTest.scala @@ -21,11 +21,25 @@ package org.apache.paimon.spark.sql import org.apache.paimon.spark.PaimonSparkTestBase import org.apache.paimon.types.{DataTypes, EdgeAlgorithm} -import org.apache.spark.sql.Row +import org.apache.spark.SparkConf +import org.apache.spark.sql.{AnalysisException, Row} /** Tests Spark 4.1 SQL interoperability with Paimon geospatial columns. */ class GeospatialTypeSQLTest extends PaimonSparkTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf.set("spark.sql.geospatial.enabled", "true") + } + + test("Spark SQL requires geospatial support to be enabled") { + withSparkSQLConf("spark.sql.geospatial.enabled" -> "false") { + val error = intercept[AnalysisException] { + sql("CREATE TABLE geospatial_disabled (geom GEOMETRY(4326)) USING paimon") + } + assert(error.getMessage.contains("GEOSPATIAL_DISABLED")) + } + } + test("Spark SQL reads and writes native geospatial values") { withTable("t") { sql("""