ci: shard the Iceberg extensions test task

.github/workflows/iceberg_spark_test_reusable.yml

@@ -193,8 +193,11 @@ jobs:
 
   iceberg-spark-extensions:
     needs: build-native
-    name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}
+    name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}/shard-${{ matrix.shard }}
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.build-native.outputs.shard-matrix) }}
     container:
       image: amd64/rust
     env:
@@ -224,7 +227,40 @@ jobs:
           rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
           ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ inputs.spark-short }} -DscalaVersion=${{ inputs.scala }} -DflinkVersions= -DkafkaVersions= \
             :iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test \
+            --init-script ../dev/ci/iceberg-test-shards.gradle \
+            -PcometShardTask=:iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test \
+            -PcometShardIndex=${{ matrix.shard }} -PcometShardCount=${{ needs.build-native.outputs.shard-count }} \
             -Pquick=true -x javadoc
+      - name: Upload Iceberg shard inventory and test reports
+        if: ${{ !cancelled() }}
+        # iceberg-spark-extensions-shard-coverage downloads the inventory, so a
+        # flaky upload here fails that job too. Retry rather than fail on 403.
+        uses: ./.github/actions/upload-artifact-retry
+        with:
+          name: iceberg-spark-extensions-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-scala-${{ inputs.scala }}-jdk${{ inputs.java }}-shard-${{ matrix.shard }}-attempt-${{ github.run_attempt }}
+          path: |
+            apache-iceberg/**/build/comet-shards/*.json
+            apache-iceberg/**/build/test-results/test/*.xml
+          retention-days: 7
+
+  iceberg-spark-extensions-shard-coverage:
+    needs: iceberg-spark-extensions
+    if: ${{ !cancelled() }}
+    name: iceberg-spark-extensions-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@v7
+      - name: Download Iceberg extensions shard inventories
+        uses: ./.github/actions/download-artifact-retry
+        with:
+          # Download all attempts: a failed-job rerun retains earlier successful
+          # shards. The checker selects the latest inventory for each index.
+          pattern: iceberg-spark-extensions-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-scala-${{ inputs.scala }}-jdk${{ inputs.java }}-shard-*-attempt-*
+          path: iceberg-shard-reports
+      - name: Verify complete, disjoint Iceberg extensions candidate coverage
+        run: |
+          python3 dev/ci/check-iceberg-shards.py --manifests iceberg-shard-reports \
+            --task :iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test
 
   iceberg-spark-runtime:
     needs: build-native

dev/ci/check-ci-config.py

@@ -1200,6 +1200,39 @@ def check_cache_save_scope():
     return not failures
 
 
+def check_iceberg_extensions_shards():
+    """The Iceberg extensions task must be sharded like the core task.
+
+    Its job is the longest unsharded one in the Iceberg workflow. It has to use
+    the shared matrix and the same Gradle init script, and a coverage job must
+    verify the extensions task's candidate inventories, not the core task's.
+    """
+    path = WORKFLOWS / "iceberg_spark_test_reusable.yml"
+    text = path.read_text(encoding="utf-8")
+    failures = []
+
+    job = re.search(r"^  iceberg-spark-extensions:\n(.*?)(?=^  \S)", text, re.M | re.S)
+    block = job.group(1) if job else ""
+    if "fromJSON(needs.build-native.outputs.shard-matrix)" not in block:
+        failures.append("iceberg-spark-extensions does not use the shared shard matrix")
+    if "cometShardTask=:iceberg-spark:iceberg-spark-extensions-" not in block:
+        failures.append("iceberg-spark-extensions does not pass -PcometShardTask for its own task")
+    if "cometShardIndex=" not in block or "cometShardCount=" not in block:
+        failures.append("iceberg-spark-extensions does not pass the shard index/count")
+
+    coverage = re.search(
+        r"^  iceberg-spark-extensions-shard-coverage:\n(.*?)(?=^  \S)", text, re.M | re.S)
+    cblock = coverage.group(1) if coverage else ""
+    if (not coverage or "check-iceberg-shards.py --manifests" not in cblock
+            or "--task :iceberg-spark:iceberg-spark-extensions-" not in cblock):
+        failures.append(
+            "no iceberg-spark-extensions-shard-coverage job verifying the extensions task")
+
+    for failure in failures:
+        print(f"iceberg extensions shards: {failure}")
+    return not failures
+
+
 if __name__ == "__main__":
     ok = check_change_filters()
     ok = check_event_policy() and ok
@@ -1213,6 +1246,7 @@ def check_cache_save_scope():
     ok = check_nightly_base_fallback() and ok
     ok = check_cache_save_scope() and ok
     ok = check_local_ci_config() and ok
+    ok = check_iceberg_extensions_shards() and ok
     if not ok:
         sys.exit(1)
     print("CI config checks passed")