Prepare for DF52 release (#1337)

* Prepare for DF52 release

* Update datafusion to point to release version

* round now respects decimal

* Update pathlib

* Update documentation

* Commit copilot suggestions

* Pass codec capsule to table providers

* Simplify codec passing around

* Update signatures for FFI passing of logical codec around

* Update upgrade guide for new method signature

* Add more unit tests to cover FFI providers

* Missing run command

Author: timsaucer

Cargo.lock

@@ -57,10 +57,10 @@ pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
 pyo3-log = "0.13.2"
 arrow = { version = "57", features = ["pyarrow"] }
 arrow-select = { version = "57" }
-datafusion = { version = "51", features = ["avro", "unicode_expressions"] }
-datafusion-substrait = { version = "51", optional = true }
-datafusion-proto = { version = "51" }
-datafusion-ffi = { version = "51" }
+datafusion = { version = "52", features = ["avro", "unicode_expressions"] }
+datafusion-substrait = { version = "52", optional = true }
+datafusion-proto = { version = "52" }
+datafusion-ffi = { version = "52" }
 prost = "0.14.1" # keep in line with `datafusion-substrait`
 uuid = { version = "1.18", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = [

Cargo.toml

@@ -275,7 +275,7 @@ needing to activate the virtual environment:
 
 ```bash
 uv run --no-project maturin develop --uv
-uv --no-project pytest .
+uv run --no-project pytest .
 ```
 
 ### Running & Installing pre-commit hooks

README.md

@@ -23,7 +23,7 @@
 
 
 def bench(data_path, query_path) -> None:
-    with Path.open("results.csv", "w") as results:
+    with Path("results.csv").open("w") as results:
         # register tables
         start = time.time()
         total_time_millis = 0
@@ -46,7 +46,7 @@ def bench(data_path, query_path) -> None:
         print("Configuration:\n", ctx)
 
         # register tables
-        with Path.open("create_tables.sql") as f:
+        with Path("create_tables.sql").open() as f:
             sql = ""
             for line in f.readlines():
                 if line.startswith("--"):
@@ -66,7 +66,7 @@ def bench(data_path, query_path) -> None:
 
         # run queries
         for query in range(1, 23):
-            with Path.open(f"{query_path}/q{query}.sql") as f:
+            with Path(f"{query_path}/q{query}.sql").open() as f:
                 text = f.read()
                 tmp = text.split(";")
                 queries = [s.strip() for s in tmp if len(s.strip()) > 0]

benchmarks/tpch/tpch.py

@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. _ffi:
+
 Python Extensions
 =================
 

docs/source/contributor-guide/ffi.rst

@@ -77,6 +77,7 @@ Example
    user-guide/io/index
    user-guide/configuration
    user-guide/sql
+   user-guide/upgrade-guides
 
 
 .. _toc.contributor_guide:

docs/source/index.rst

@@ -0,0 +1,96 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Upgrade Guides
+==============
+
+DataFusion 52.0.0
+-----------------
+
+This version includes a major update to the :ref:`ffi` due to upgrades
+to the `Foreign Function Interface <https://doc.rust-lang.org/nomicon/ffi.html>`_.
+Users who contribute their own ``CatalogProvider``, ``SchemaProvider``,
+``TableProvider`` or ``TableFunction`` via FFI must now provide access to a
+``LogicalExtensionCodec`` and a ``TaskContextProvider``. The function signatures
+for the methods to get these ``PyCapsule`` objects now requires an additional
+parameter, which is a Python object that can be used to extract the
+``FFI_LogicalExtensionCodec`` that is necessary.
+
+A complete example can be found in the `FFI example <https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example>`_.
+Your methods need to be updated to take an additional parameter like in this
+example.
+
+.. code-block:: rust
+
+    #[pymethods]
+    impl MyCatalogProvider {
+        pub fn __datafusion_catalog_provider__<'py>(
+            &self,
+            py: Python<'py>,
+            session: Bound<PyAny>,
+        ) -> PyResult<Bound<'py, PyCapsule>> {
+            let name = cr"datafusion_catalog_provider".into();
+
+            let provider = Arc::clone(&self.inner) as Arc<dyn CatalogProvider + Send>;
+
+            let codec = ffi_logical_codec_from_pycapsule(session)?;
+            let provider = FFI_CatalogProvider::new_with_ffi_codec(provider, None, codec);
+
+            PyCapsule::new(py, provider, Some(name))
+        }
+    }
+
+To extract the logical extension codec FFI object from the provided object you
+can implement a helper method such as:
+
+.. code-block:: rust
+
+    pub(crate) fn ffi_logical_codec_from_pycapsule(
+        obj: Bound<PyAny>,
+    ) -> PyResult<FFI_LogicalExtensionCodec> {
+        let attr_name = "__datafusion_logical_extension_codec__";
+        let capsule = if obj.hasattr(attr_name)? {
+            obj.getattr(attr_name)?.call0()?
+        } else {
+            obj
+        };
+
+        let capsule = capsule.downcast::<PyCapsule>()?;
+        validate_pycapsule(capsule, "datafusion_logical_extension_codec")?;
+
+        let codec = unsafe { capsule.reference::<FFI_LogicalExtensionCodec>() };
+
+        Ok(codec.clone())
+    }
+
+
+The DataFusion FFI interface updates no longer depend directly on the
+``datafusion`` core crate. You can improve your build times and potentially
+reduce your library binary size by removing this dependency and instead
+using the specific datafusion project crates.
+
+For example, instead of including expressions like:
+
+.. code-block:: rust
+
+    use datafusion::catalog::MemTable;
+
+Instead you can now write:
+
+.. code-block:: rust
+
+    use datafusion_catalog::MemTable;