feat: declare Requires-Python >=3.10, warn on PySpark newer than the runtime, and test newer pyspark-client

.github/workflows/tests.yaml

@@ -28,7 +28,7 @@ concurrency:
 
 jobs:
   test:
-    name: Run unit tests (${{ matrix.os }}, [${{ matrix.spark-extra }}])
+    name: Run unit tests (${{ matrix.os }}, Python ${{ matrix.python-version }}, [${{ matrix.spark-extra }}]${{ matrix.smoke && ', smoke' || '' }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
     defaults:
@@ -38,9 +38,33 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        # The Python version of the default Managed Spark runtime, which
+        # matters most since Python UDFs need client and server to match.
+        python-version: ["3.12"]
         # Mutually exclusive: [client] installs pyspark-client, [spark] full
         # pyspark[connect].
         spark-extra: ["client", "spark"]
+        # The rest of the versions setup.py declares get smoke tests only, on
+        # one OS and extra: they are there to catch syntax, dependency
+        # resolution, import and standard library problems, which are neither
+        # OS- nor extra-specific.
+        include:
+          - os: "ubuntu-latest"
+            python-version: "3.10"
+            spark-extra: "client"
+            smoke: true
+          - os: "ubuntu-latest"
+            python-version: "3.11"
+            spark-extra: "client"
+            smoke: true
+          - os: "ubuntu-latest"
+            python-version: "3.13"
+            spark-extra: "client"
+            smoke: true
+          - os: "ubuntu-latest"
+            python-version: "3.14"
+            spark-extra: "client"
+            smoke: true
 
     steps:
     - name: Checkout code
@@ -51,7 +75,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
       with:
-        python-version: "3.12"
+        python-version: ${{ matrix.python-version }}
 
     - name: Set up uv
       uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1
@@ -63,10 +87,10 @@ jobs:
           ~/.cache/uv
           ~/Library/Caches/uv
           ~/AppData/Local/uv/cache
-        key: ${{ runner.os }}-uv-${{ matrix.spark-extra }}-${{ hashFiles('setup.py', 'requirements-test.txt') }}
+        key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ matrix.spark-extra }}-${{ hashFiles('setup.py', 'requirements-test.txt') }}
         restore-keys: |
-          ${{ runner.os }}-uv-${{ matrix.spark-extra }}-
-          ${{ runner.os }}-uv-
+          ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ matrix.spark-extra }}-
+          ${{ runner.os }}-uv-py${{ matrix.python-version }}-
 
     # Installs through the package's own extras, rather than
     # requirements-dev.txt (which pins pyspark-client), so that both extras are
@@ -76,18 +100,67 @@ jobs:
       run: uv pip install --system -e '.[${{ matrix.spark-extra }}]' -r requirements-test.txt
 
     - name: Run unit tests without [interactive]
-      run: python -m pytest tests/unit/test_init.py tests/unit/test_pypi_artifacts.py -k "TestWithoutInteractiveExtra or TestPythonVersionCheck or TestRuntimeVersionCompatibility or PyPiArtifactsTest" -v --tb=short
+      run: python -m pytest tests/unit/test_init.py tests/unit/test_pypi_artifacts.py -k "TestWithoutInteractiveExtra or TestPythonVersionCheck or TestPySparkVersionCheck or TestRuntimeVersionCompatibility or PyPiArtifactsTest" -v --tb=short
 
     - name: Install [interactive] extra
       run: uv pip install --system -e '.[${{ matrix.spark-extra }},interactive]'
 
     - name: Run unit tests
+      if: ${{ !matrix.smoke }}
       run: python -m pytest tests/unit/ -v --tb=short -n auto
 
+    # All but the slow unit tests. Those are mostly waiting, and cover next to
+    # nothing the others do not, so this covers nearly every line the full suite
+    # does in a fraction of the time, catching standard library APIs this Python
+    # lacks as well as syntax. (The tests import every module; vermin.yaml
+    # checks syntax statically everywhere else.)
+    - name: Run smoke tests
+      if: ${{ matrix.smoke }}
+      run: python -m pytest tests/unit/ -m "not slow" -v --tb=short -n auto
+
     # Starts Spark in-process, so it runs serially, outside of xdist. Needs
     # the JVM that only full pyspark ships.
     - name: Run local Spark tests
       if: matrix.spark-extra == 'spark'
       run: |
         export JAVA_HOME="${JAVA_HOME_17_X64:-${JAVA_HOME_17_ARM64:-${JAVA_HOME_21_X64:-${JAVA_HOME_21_ARM64:-$JAVA_HOME}}}}"
         python -m pytest tests/local/ -v --tb=short
+
+  # The [client] and [spark] extras pin the pyspark minor version of the
+  # default Managed Spark runtime. Users of other runtimes, and environments
+  # that preinstall pyspark, can have a newer one, so also run the full suite
+  # against the newest pyspark-client. On Python 3.13 rather than 3.12, which
+  # the main matrix already covers with the pinned pyspark.
+  test-newest-pyspark:
+    name: Run unit tests (ubuntu-latest, Python 3.13, newest pyspark-client)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: Setup Python
+      uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+      with:
+        python-version: "3.13"
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1
+
+    # No extra: it would pin pyspark-client back to the default runtime's.
+    - name: Install dependencies
+      run: uv pip install --system -e '.[interactive]' pyspark-client -r requirements-test.txt
+
+    # Serially, without xdist: pyspark 4.2 exits on import when __main__ has
+    # no __file__, as in xdist workers, mistaking that for running its own
+    # doctests.
+    - name: Run unit tests
+      run: |
+        python -c 'import pyspark; print("pyspark", pyspark.__version__)'
+        python -m pytest tests/unit/ -v --tb=short

.github/workflows/vermin.yaml

@@ -0,0 +1,62 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Statically checks that the code only uses language features and standard
+# library modules, functions and arguments that are available in the oldest
+# Python version setup.py declares. Complements the smoke tests in tests.yaml,
+# which only catch the APIs they run, and are run on CI's Python versions only.
+# vermin only knows the types of names it can resolve statically, so it can
+# miss methods on objects whose types it cannot infer. Silence false positives
+# with a "# novm" comment.
+name: Vermin
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  vermin:
+    name: Check minimum Python version
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: Setup Python
+      uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+      with:
+        python-version: "3.12"
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1
+
+    - name: Install vermin
+      run: uv pip install --system 'vermin~=1.9'
+
+    # The target must match python_requires in setup.py. The trailing "-" fails
+    # only if the code needs a newer version, not if it would also run on an
+    # older one.
+    - name: Run vermin
+      run: vermin --no-tips --violations --eval-annotations --target=3.10- google tests setup.py

README.md

@@ -35,6 +35,13 @@ Other installation options:
 > later uninstalled. In particular, never combine the `client` and `spark`
 > extras (`'[client,spark]'`).
 
+### Python versions
+
+This package requires Python 3.10 or later. To use Python UDFs (including
+`applyInPandas`, `mapInPandas`, and similar APIs), the client's Python minor
+version must match the Python of the Managed Spark runtime the session runs on,
+for example Python 3.12 for runtime 3.0. The client warns when they differ.
+
 ## Uninstall
 
 ```sh

google/cloud/managed_spark_connect/_runtimes.py

@@ -0,0 +1,73 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""What the Managed Spark runtime versions ship, for client compatibility checks.
+
+Spark Connect needs the client's PySpark minor version to match the server's
+Spark minor version, and Python UDFs need the client's Python minor version to
+match the server's, since PySpark refuses to run a UDF pickled by a different
+Python minor version.
+"""
+
+import dataclasses
+from typing import Optional, Tuple
+
+from packaging.version import InvalidVersion, Version
+
+MajorMinor = Tuple[int, int]
+
+
+@dataclasses.dataclass(frozen=True)
+class RuntimeInfo:
+    """The versions a Managed Spark runtime version ships.
+
+    Attributes:
+        spark: Apache Spark (major, minor) version.
+        python: Python (major, minor) version.
+    """
+
+    spark: MajorMinor
+    python: MajorMinor
+
+
+# Keyed by runtime (major, minor) version. Runtime versions missing from here
+# are not checked, so a runtime newer than this release still works, just
+# without the compatibility warnings.
+_RUNTIMES = {
+    (3, 0): RuntimeInfo(spark=(4, 0), python=(3, 12)),
+}
+
+
+def major_minor(version_str: str) -> Optional[MajorMinor]:
+    """The (major, minor) of a version string, or None if it has no minor.
+
+    Accepts both runtime versions ("3.0", or subminor versions like "3.0.12")
+    and PySpark versions ("4.0.1", "4.1.0.dev1").
+    """
+    try:
+        release = Version(version_str).release
+    except InvalidVersion:
+        return None
+    if len(release) < 2:
+        return None
+    return release[0], release[1]
+
+
+def get_runtime_info(runtime_version: str) -> Optional[RuntimeInfo]:
+    """What the given runtime version ships, or None if unknown."""
+    key = major_minor(runtime_version)
+    return _RUNTIMES.get(key) if key else None
+
+
+def format_version(version: MajorMinor) -> str:
+    return f"{version[0]}.{version[1]}"

google/cloud/managed_spark_connect/session.py

@@ -85,6 +85,7 @@
     TerminateSessionRequest,
 )
 from google.cloud.dataproc_v1.types import sessions
+from google.cloud.managed_spark_connect import _runtimes
 from google.cloud.managed_spark_connect import environment
 from google.cloud.managed_spark_connect import ide
 from pyspark.sql.connect.session import SparkSession
@@ -1185,6 +1186,9 @@ def _get_session_config(self):
             self._check_python_version_compatibility(
                 session_config.runtime_config.version
             )
+            self._check_pyspark_version_compatibility(
+                session_config.runtime_config.version
+            )
 
             # Use local variable to improve readability of deeply nested attribute access
             exec_config = session_config.environment_config.execution_config
@@ -1365,24 +1369,57 @@ def _check_python_version_compatibility(self, runtime_version):
             import sys
             import warnings
 
-            # Runtime version to server Python version mapping
-            RUNTIME_PYTHON_MAP = {
-                "3.0": (3, 12),
-            }
+            runtime_info = _runtimes.get_runtime_info(runtime_version)
+            if runtime_info is None:
+                return
 
             client_python = sys.version_info[:2]  # (major, minor)
+            server_python = runtime_info.python
+
+            if client_python != server_python:
+                warnings.warn(
+                    f"Python version mismatch detected: Client is using Python {client_python[0]}.{client_python[1]}, "
+                    f"but Managed Spark runtime {runtime_version} uses Python {server_python[0]}.{server_python[1]}. "
+                    f"This mismatch may cause issues with Python UDF (User Defined Function) compatibility. "
+                    f"Consider using Python {server_python[0]}.{server_python[1]} for optimal UDF execution.",
+                    stacklevel=3,
+                )
 
-            if runtime_version in RUNTIME_PYTHON_MAP:
-                server_python = RUNTIME_PYTHON_MAP[runtime_version]
+        def _check_pyspark_version_compatibility(self, runtime_version):
+            """Check that the installed PySpark is not newer than the runtime's Spark.
 
-                if client_python != server_python:
-                    warnings.warn(
-                        f"Python version mismatch detected: Client is using Python {client_python[0]}.{client_python[1]}, "
-                        f"but Managed Spark runtime {runtime_version} uses Python {server_python[0]}.{server_python[1]}. "
-                        f"This mismatch may cause issues with Python UDF (User Defined Function) compatibility. "
-                        f"Consider using Python {server_python[0]}.{server_python[1]} for optimal UDF execution.",
-                        stacklevel=3,
-                    )
+            Spark Connect servers support clients with older Spark minor
+            versions, but not newer ones: for example, PySpark 4.2's
+            createDataFrame() reads SQL configs Spark 4.0 does not have.
+            """
+            import warnings
+
+            from pyspark.version import __version__ as pyspark_version
+
+            runtime_info = _runtimes.get_runtime_info(runtime_version)
+            client_spark = _runtimes.major_minor(pyspark_version)
+            if runtime_info is None or client_spark is None:
+                return
+
+            server_spark = runtime_info.spark
+            if client_spark > server_spark:
+                from pyspark.util import is_remote_only
+
+                server_spark_str = _runtimes.format_version(server_spark)
+                # Downgrade whichever distribution is installed, so as not to
+                # end up with both writing to the pyspark module.
+                package = (
+                    "pyspark-client" if is_remote_only() else "pyspark[connect]"
+                )
+                warnings.warn(
+                    f"PySpark {pyspark_version} is installed, but Managed Spark "
+                    f"runtime {runtime_version} runs Spark {server_spark_str}. "
+                    "Spark Connect does not support clients newer than the "
+                    "server, so some operations will fail. Install a matching "
+                    f"PySpark, e.g. pip install '{package}~={server_spark_str}.0', "
+                    "or choose a newer runtime version with runtimeVersion().",
+                    stacklevel=3,
+                )
 
         def _check_runtime_compatibility(self, session_config):
             """Check if runtime version 3.0 client is compatible with older runtime versions.

pyproject.toml

@@ -10,4 +10,5 @@ pyink-indentation = 4 #  (default is 4)
 markers = [
     "integration: marks tests as integration tests",
     "ci_safe: marks tests that work in CI environment",
+    "slow: marks unit tests that take a second or more, mostly waiting; the smoke tests skip them",
 ]