diff --git a/.github/actions/setup-spark-builder/action.yaml b/.github/actions/setup-spark-builder/action.yaml index 40f3a1e4a8e..6a21ebedfe5 100644 --- a/.github/actions/setup-spark-builder/action.yaml +++ b/.github/actions/setup-spark-builder/action.yaml @@ -88,13 +88,18 @@ runs: # then resolves Spark's deps, Coursier sees the POM in mavenLocal, # declares the artifact "found locally", and fails on the missing JAR # without falling back to Maven Central. Delete those partial entries - # so sbt re-fetches the full artifact remotely. + # so sbt re-fetches the full artifact remotely. A POM with no + # element is a jar, Maven's default. Arrow's POMs leave it + # out, and Spark 3.5's sql/api does not compile without the + # arrow-memory-core and arrow-memory-netty JARs. for repo in "$HOME/.m2/repository" /root/.m2/repository; do [ -d "$repo" ] || continue find "$repo" -name '*.pom' | while read -r pom; do jar="${pom%.pom}.jar" [ -f "$jar" ] && continue - grep -q 'jar\|bundle' "$pom" 2>/dev/null || continue + if grep -q '' "$pom" 2>/dev/null; then + grep -q 'jar\|bundle' "$pom" 2>/dev/null || continue + fi rm -f "$pom" "${pom}.sha1" "${pom%.pom}.pom.lastUpdated" \ "$(dirname "$pom")/_remote.repositories" done diff --git a/contrib/delta/native/Cargo.lock b/contrib/delta/native/Cargo.lock index 213875ceb33..d816421a350 100644 --- a/contrib/delta/native/Cargo.lock +++ b/contrib/delta/native/Cargo.lock @@ -488,9 +488,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" dependencies = [ "arrow", "arrow-schema", @@ -534,9 +534,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" dependencies = [ "arrow", "async-trait", @@ -559,9 +559,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" dependencies = [ "arrow", "async-trait", @@ -591,9 +591,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" dependencies = [ "arrow", "arrow-ipc", @@ -616,9 +616,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" dependencies = [ "futures", "log", @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" dependencies = [ "arrow", "async-trait", @@ -657,9 +657,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" dependencies = [ "arrow", "arrow-ipc", @@ -681,9 +681,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" dependencies = [ "arrow", "async-trait", @@ -704,9 +704,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" dependencies = [ "arrow", "async-trait", @@ -727,9 +727,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" dependencies = [ "arrow", "arrow-schema", @@ -759,15 +759,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" [[package]] name = "datafusion-execution" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" dependencies = [ "arrow", "arrow-buffer", @@ -791,9 +791,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" dependencies = [ "arrow", "arrow-schema", @@ -807,14 +807,15 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap", "itertools 0.15.0", + "serde", "serde_json", ] [[package]] name = "datafusion-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" dependencies = [ "arrow", "datafusion-common", @@ -824,9 +825,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" dependencies = [ "arrow", "arrow-buffer", @@ -852,9 +853,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" dependencies = [ "arrow", "datafusion-common", @@ -873,9 +874,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" dependencies = [ "arrow", "datafusion-common", @@ -885,9 +886,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" dependencies = [ "arrow", "async-trait", @@ -901,9 +902,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" dependencies = [ "arrow", "datafusion-common", @@ -918,9 +919,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -928,9 +929,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", @@ -939,9 +940,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" dependencies = [ "arrow", "chrono", @@ -958,9 +959,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" dependencies = [ "arrow", "arrow-schema", @@ -980,9 +981,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" dependencies = [ "arrow", "datafusion-common", @@ -995,9 +996,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" dependencies = [ "arrow", "chrono", @@ -1012,9 +1013,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" dependencies = [ "arrow", "datafusion-common", @@ -1031,9 +1032,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" dependencies = [ "arrow", "arrow-data", @@ -1060,15 +1061,16 @@ dependencies = [ "num-traits", "parking_lot", "pin-project-lite", + "serde", "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" dependencies = [ "arrow", "datafusion-common", @@ -1082,9 +1084,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" dependencies = [ "arrow-schema", "async-trait", @@ -2119,6 +2121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", + "serde_derive", ] [[package]] @@ -2147,7 +2150,6 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap", "itoa", "memchr", "serde", diff --git a/contrib/delta/native/Cargo.toml b/contrib/delta/native/Cargo.toml index 77723df3893..5aadf800802 100644 --- a/contrib/delta/native/Cargo.toml +++ b/contrib/delta/native/Cargo.toml @@ -47,4 +47,4 @@ crate-type = ["rlib"] datafusion-comet-proto = { path = "../../../native/proto" } # For `ExecutionPlan` (return type), `SchemaRef` (the shim hands these in), and `DataFusionError` # (the not-implemented error). Versions/features mirror core so the shared lockfile resolves cleanly. -datafusion = { version = "55.1.0", default-features = false, features = ["parquet"] } +datafusion = { version = "55.2.0", default-features = false, features = ["parquet"] } diff --git a/native/Cargo.lock b/native/Cargo.lock index 00252bb43f6..ca8c6b6f64e 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1866,9 +1866,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" dependencies = [ "arrow", "arrow-schema", @@ -1915,9 +1915,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" dependencies = [ "arrow", "async-trait", @@ -1940,9 +1940,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" dependencies = [ "arrow", "async-trait", @@ -2123,9 +2123,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" dependencies = [ "arrow", "arrow-ipc", @@ -2150,9 +2150,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" dependencies = [ "futures", "log", @@ -2161,9 +2161,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" dependencies = [ "arrow", "async-compression", @@ -2197,9 +2197,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" dependencies = [ "arrow", "arrow-ipc", @@ -2221,9 +2221,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" dependencies = [ "arrow", "async-trait", @@ -2244,9 +2244,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" dependencies = [ "arrow", "async-trait", @@ -2267,9 +2267,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" dependencies = [ "arrow", "arrow-schema", @@ -2299,15 +2299,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" [[package]] name = "datafusion-execution" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" dependencies = [ "arrow", "arrow-buffer", @@ -2332,9 +2332,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" dependencies = [ "arrow", "arrow-schema", @@ -2348,15 +2348,16 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.2", "itertools 0.15.0", + "serde", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" dependencies = [ "arrow", "datafusion-common", @@ -2366,9 +2367,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" dependencies = [ "arrow", "arrow-buffer", @@ -2398,9 +2399,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" dependencies = [ "arrow", "datafusion-common", @@ -2419,9 +2420,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" dependencies = [ "arrow", "datafusion-common", @@ -2431,9 +2432,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08079b0604cdcc43c7d6cf006cfd5d3ed9c235ade0c1ebe91085a62f875603df" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" dependencies = [ "arrow", "arrow-ord", @@ -2456,9 +2457,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" dependencies = [ "arrow", "async-trait", @@ -2472,9 +2473,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" dependencies = [ "arrow", "datafusion-common", @@ -2489,9 +2490,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2499,9 +2500,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" dependencies = [ "datafusion-doc", "quote", @@ -2510,9 +2511,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" dependencies = [ "arrow", "chrono", @@ -2529,9 +2530,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" dependencies = [ "arrow", "arrow-schema", @@ -2551,9 +2552,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" dependencies = [ "arrow", "datafusion-common", @@ -2566,9 +2567,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" dependencies = [ "arrow", "chrono", @@ -2583,9 +2584,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" dependencies = [ "arrow", "datafusion-common", @@ -2602,9 +2603,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" dependencies = [ "arrow", "arrow-data", @@ -2631,15 +2632,16 @@ dependencies = [ "num-traits", "parking_lot", "pin-project-lite", + "serde", "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" dependencies = [ "arrow", "datafusion-common", @@ -2653,9 +2655,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" dependencies = [ "arrow-schema", "async-trait", @@ -2668,23 +2670,22 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7809ecefb16fcb950c6b22ee3df99cb412dca6fae72c329d64c938ae53254f9" +checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", @@ -2698,9 +2699,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "55.1.0" +version = "55.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4848be66dd008d55adbe1dbc8cbd6a2502588ad0c0b8483bab0f43fd4495d0" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" dependencies = [ "arrow", "bigdecimal", @@ -2868,12 +2869,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "dissimilar" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" - [[package]] name = "dlv-list" version = "0.5.2" @@ -2974,16 +2969,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "expect-test" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" -dependencies = [ - "dissimilar", - "once_cell", -] - [[package]] name = "fastnum" version = "0.7.5" @@ -3550,7 +3535,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=bb1e4a4861f02377489eff818b75138f414c4cb0#bb1e4a4861f02377489eff818b75138f414c4cb0" +source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" dependencies = [ "aes-gcm", "anyhow", @@ -3573,7 +3558,6 @@ dependencies = [ "chrono", "crc32fast", "derive_builder", - "expect-test", "fastnum", "flate2", "fnv", @@ -3609,7 +3593,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=bb1e4a4861f02377489eff818b75138f414c4cb0#bb1e4a4861f02377489eff818b75138f414c4cb0" +source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" dependencies = [ "proc-macro2", "quote", @@ -3619,7 +3603,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=bb1e4a4861f02377489eff818b75138f414c4cb0#bb1e4a4861f02377489eff818b75138f414c4cb0" +source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" dependencies = [ "anyhow", "async-trait", @@ -3627,6 +3611,7 @@ dependencies = [ "cfg-if", "futures", "iceberg", + "iceberg-property-macro", "opendal", "reqsign-aws-v4", "reqsign-core", diff --git a/native/Cargo.toml b/native/Cargo.toml index ff3c7f6d2b7..350bdd8864d 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -43,10 +43,10 @@ arrow-select = { version = "59.2.0" } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "59.2.0", default-features = false, features = ["experimental"] } -datafusion = { version = "55.1.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { version = "55.1.0" } -datafusion-physical-expr-adapter = { version = "55.1.0" } -datafusion-spark = { version = "55.1.0", features = ["core"] } +datafusion = { version = "55.2.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { version = "55.2.0" } +datafusion-physical-expr-adapter = { version = "55.2.0" } +datafusion-spark = { version = "55.2.0", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } @@ -70,8 +70,8 @@ aws-sdk-sts = { version = "1.114.0", default-features = false } # Pinned to a revision rather than a release because no published iceberg-rust # version is on our arrow: 0.10.1 requires arrow/parquet ^58. Move to a plain # version requirement once a release ships on arrow 59. Bump policy: #5645. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "bb1e4a4861f02377489eff818b75138f414c4cb0" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "bb1e4a4861f02377489eff818b75138f414c4cb0", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] } +iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" } +iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc86178c38c1c6db0578bdd9fb7021e3", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] } reqsign-core = "3" [profile.release] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 5cbefc66a10..e2f9f264e76 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -106,7 +106,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "10" hex = "0.4.3" -datafusion-functions-nested = { version = "55.1.0" } +datafusion-functions-nested = { version = "55.2.0" } # Test-only helpers for a hand-rolled in-memory HTTP client that exercises the web-identity STS # retry path without a socket, plus reqsign's SigV4 signer for the signing round-trip test. Already # in the dependency graph (transitive via aws-config and iceberg-storage-opendal). diff --git a/native/core/src/execution/operators/iceberg_location_scoped.rs b/native/core/src/execution/operators/iceberg_location_scoped.rs index eb7fd937991..d1ad4744859 100644 --- a/native/core/src/execution/operators/iceberg_location_scoped.rs +++ b/native/core/src/execution/operators/iceberg_location_scoped.rs @@ -654,12 +654,12 @@ impl LocationScopedFileWrite { /// `result` of a write call made at the snapshot of `generation`, after fetching `bucket`'s /// locations again if it failed in a way that may mean they changed. A free function so the /// writer, which is not `Sync`, is not borrowed across the refresh. -async fn refresh_on_failure( +async fn refresh_on_failure( inner: &Inner, bucket: &BucketLocations, generation: u64, - result: Result<()>, -) -> Result<()> { + result: Result, +) -> Result { match result { Err(e) if may_mean_stale_locations(&e) => { match inner.shared.refresh(bucket, generation).await { @@ -679,7 +679,7 @@ impl FileWrite for LocationScopedFileWrite { refresh_on_failure(&self.inner, &self.route.bucket, generation, result).await } - async fn close(&mut self) -> Result<()> { + async fn close(&mut self) -> Result { let generation = self.generation(); let result = self.writer.close().await; refresh_on_failure(&self.inner, &self.route.bucket, generation, result).await @@ -797,8 +797,10 @@ mod tests { Ok(()) } - async fn close(&mut self) -> Result<()> { - self.0.check("close", &self.1) + async fn close(&mut self) -> Result { + self.0 + .check("close", &self.1) + .map(|_| FileMetadata { size: 0 }) } } @@ -1512,7 +1514,7 @@ mod tests { let output = storage.new_output("s3://b/t/new/f.parquet").unwrap(); let mut writer = output.writer().await.unwrap(); writer.write(Bytes::from("data")).await.unwrap(); - let err = writer.close().await.unwrap_err(); + let err = writer.close().await.err().expect("close should fail"); assert!(may_mean_stale_locations(&err), "{err}"); assert_eq!(fixture.fetches(), 1); let mut writer = output.writer().await.unwrap(); diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index c2cd01f50e9..9d0122981a7 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -7453,15 +7453,20 @@ mod tests { .expect("schema"); let schema_arc = Arc::new(iceberg_schema); + let identity_field = |source_id: i32, field_id: i32, name: &str| { + iceberg::spec::UnboundPartitionField::builder() + .source_ids(vec![source_id]) + .field_id(field_id) + .name(name) + .transform(iceberg::spec::Transform::Identity) + .build() + .expect("unbound field") + }; + // Spec 0 (older): field 1000 named "region_old". let spec0 = PartitionSpec::builder(Arc::clone(&schema_arc)) .with_spec_id(0) - .add_unbound_field(iceberg::spec::UnboundPartitionField { - source_id: 2, - field_id: Some(1000), - name: "region_old".to_string(), - transform: iceberg::spec::Transform::Identity, - }) + .add_unbound_field(identity_field(2, 1000, "region_old")) .expect("add field") .build() .expect("build spec0"); @@ -7469,19 +7474,9 @@ mod tests { // Spec 1 (newer): same field id 1000 renamed to "region_new", plus a new field 2000. let spec1 = PartitionSpec::builder(Arc::clone(&schema_arc)) .with_spec_id(1) - .add_unbound_field(iceberg::spec::UnboundPartitionField { - source_id: 2, - field_id: Some(1000), - name: "region_new".to_string(), - transform: iceberg::spec::Transform::Identity, - }) + .add_unbound_field(identity_field(2, 1000, "region_new")) .expect("add field") - .add_unbound_field(iceberg::spec::UnboundPartitionField { - source_id: 3, - field_id: Some(2000), - name: "category".to_string(), - transform: iceberg::spec::Transform::Identity, - }) + .add_unbound_field(identity_field(3, 2000, "category")) .expect("add field") .build() .expect("build spec1"); diff --git a/native/core/src/parquet/eager_page_index_reader_factory.rs b/native/core/src/parquet/eager_page_index_reader_factory.rs index dbc7d0f3b80..b5f1c58d3eb 100644 --- a/native/core/src/parquet/eager_page_index_reader_factory.rs +++ b/native/core/src/parquet/eager_page_index_reader_factory.rs @@ -52,9 +52,8 @@ //! `ignoreMissing` is set, and it walks the raw `MessageType` to decide. That walk has to run //! over the Parquet schema rather than the Arrow schema the schema adapter is handed later, //! because an id on a repeated `list` or `key_value` group, or on the message root, never -//! reaches an Arrow field. Until apache/datafusion#24790, which is not in DataFusion 55.1.0, -//! the INT96 coercion also rebuilt container fields without their metadata, so a struct id could -//! vanish on the way to Arrow as well. +//! reaches an Arrow field. DataFusion 55.2.0 preserves container metadata through INT96 coercion +//! (apache/datafusion#24790), so struct ids no longer vanish on the way to Arrow. //! //! The second is the Variant footer rewrite, `with_spark_arrow_schema`, which replaces the //! Arrow schema hint in the footer for scans that project Variant. diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 49a3022de45..8c0cd4a37a6 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -42,8 +42,10 @@ import org.apache.spark.sql.{CometTestBase, DataFrame, Row} import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, OptimizeIn} import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.comet.{CometNativeScanExec, CometScanExec} +import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils +import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -2598,6 +2600,114 @@ abstract class ParquetReadSuite extends CometTestBase { } } + // Spark writes timestamps as INT96 by default, and the native scan's INT96 coercion rebuilds + // struct, list and map fields, which must keep their metadata or an id on the container is + // lost and the column reads as nulls. The read renames the column so only its id matches it. + private val int96Ts = Timestamp.valueOf("2020-01-01 00:00:00") + private val int96Inner = new StructType().add("a", IntegerType).add("ts", TimestampType) + + Seq[(String, DataType, Any)]( + ("struct", int96Inner, Row(1, int96Ts)), + ("list", ArrayType(int96Inner), Seq(Row(1, int96Ts), Row(2, int96Ts))), + ("map", MapType(StringType, int96Inner), Map("k" -> Row(1, int96Ts)))).foreach { + case (label, dataType, value) => + test(s"an id on a $label column holding an INT96 timestamp matches it natively") { + withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") { + withTempPath { dir => + val rows = Seq(Row(value), Row(null)) + withSQLConf( + CometConf.COMET_ENABLED.key -> "false", + SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") { + spark + .createDataFrame( + spark.sparkContext.parallelize(rows), + new StructType().add("c", dataType, true, withId(1))) + .write + .parquet(dir.getCanonicalPath) + } + val readSchema = new StructType().add("renamed", dataType, true, withId(1)) + val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath) + checkSparkAnswerAndOperator(df) + checkAnswer(df, rows) + } + } + } + } + + // Ids on a struct below a struct or a list, and on its INT96 leaf, match renamed fields. + // Spark's vectorized reader before 4.0 rejects the renamed struct below the list, so the + // comparison with Spark runs from 4.0 on and the pinned rows hold everywhere. + test("an id on a struct nested in a struct or list with an INT96 leaf matches it natively") { + def schema(innerName: String, tsName: String): StructType = { + def inner(innerId: Int, tsId: Int): StructType = new StructType().add( + innerName, + new StructType().add("a", IntegerType).add(tsName, TimestampType, true, withId(tsId)), + true, + withId(innerId)) + new StructType().add("s", inner(2, 3)).add("l", ArrayType(inner(4, 5))) + } + val rows = Seq( + Row(Row(Row(1, int96Ts)), Seq(Row(Row(2, int96Ts)), Row(Row(3, int96Ts)))), + Row(Row(null), Seq(Row(Row(4, null)), null))) + + withSQLConf(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true") { + withTempPath { dir => + withSQLConf( + CometConf.COMET_ENABLED.key -> "false", + SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") { + spark + .createDataFrame(spark.sparkContext.parallelize(rows), schema("inner", "ts")) + .write + .parquet(dir.getCanonicalPath) + } + val df = spark.read + .schema(schema("renamed_inner", "renamed_ts")) + .parquet(dir.getCanonicalPath) + if (isSpark40Plus) { + checkSparkAnswerAndOperator(df) + } + val plan = stripAQEPlan(df.queryExecution.executedPlan) + assert( + collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty, + s"expected CometNativeScanExec in the plan:\n$plan") + checkAnswer(df, rows) + } + } + } + + // A Variant column is a struct of value and metadata in the Arrow schema, so the same coercion + // rebuilds it when the file holds an INT96 column anywhere, and its id has to survive. + test("an id on a Variant column next to an INT96 timestamp matches it natively") { + assume(Utils.variantType.isDefined, "VariantType requires Spark 4.0+") + withSQLConf( + SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key -> "true", + "spark.sql.variant.allowReadingShredded" -> "true", + "spark.sql.variant.pushVariantIntoScan" -> "false") { + withTempPath { dir => + withSQLConf( + CometConf.COMET_ENABLED.key -> "false", + SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") { + spark + .sql("""SELECT parse_json(j) AS v, t AS ts FROM VALUES + | ('{"a":1}', TIMESTAMP'2020-01-01 00:00:00'), (NULL, NULL) AS input(j, t) + |""".stripMargin) + .select(col("v").as("v", withId(1)), col("ts")) + .coalesce(1) + .write + .parquet(dir.getCanonicalPath) + } + val readSchema = new StructType() + .add("renamed", Utils.variantType.get, true, withId(1)) + .add("ts", TimestampType) + val df = spark.read.schema(readSchema).parquet(dir.getCanonicalPath) + checkSparkAnswerAndOperator(df) + checkAnswer( + df.selectExpr("to_json(renamed)", "ts"), + Row("""{"a":1}""", int96Ts) :: Row(null, null) :: Nil) + } + } + } + // Spark checks each file on its own. A directory holding one file with ids and one without // raises on the second, and with `ignoreMissing` the file without ids reads as nulls because // no root field of it carries the requested id. Each side is written as one file. Spread over