diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml index ae20b6d7683..617559bda27 100644 --- a/.github/workflows/iceberg_spark_test_reusable.yml +++ b/.github/workflows/iceberg_spark_test_reusable.yml @@ -193,8 +193,11 @@ jobs: iceberg-spark-extensions: needs: build-native - name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} + name: iceberg-spark-extensions/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}/shard-${{ matrix.shard }} runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.build-native.outputs.shard-matrix) }} container: image: amd64/rust env: @@ -224,7 +227,40 @@ jobs: rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ inputs.spark-short }} -DscalaVersion=${{ inputs.scala }} -DflinkVersions= -DkafkaVersions= \ :iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test \ + --init-script ../dev/ci/iceberg-test-shards.gradle \ + -PcometShardTask=:iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test \ + -PcometShardIndex=${{ matrix.shard }} -PcometShardCount=${{ needs.build-native.outputs.shard-count }} \ -Pquick=true -x javadoc + - name: Upload Iceberg shard inventory and test reports + if: ${{ !cancelled() }} + # iceberg-spark-extensions-shard-coverage downloads the inventory, so a + # flaky upload here fails that job too. Retry rather than fail on 403. + uses: ./.github/actions/upload-artifact-retry + with: + name: iceberg-spark-extensions-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-scala-${{ inputs.scala }}-jdk${{ inputs.java }}-shard-${{ matrix.shard }}-attempt-${{ github.run_attempt }} + path: | + apache-iceberg/**/build/comet-shards/*.json + apache-iceberg/**/build/test-results/test/*.xml + retention-days: 7 + + iceberg-spark-extensions-shard-coverage: + needs: iceberg-spark-extensions + if: ${{ !cancelled() }} + name: iceberg-spark-extensions-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }} + runs-on: ubuntu-slim + steps: + - uses: actions/checkout@v7 + - name: Download Iceberg extensions shard inventories + uses: ./.github/actions/download-artifact-retry + with: + # Download all attempts: a failed-job rerun retains earlier successful + # shards. The checker selects the latest inventory for each index. + pattern: iceberg-spark-extensions-${{ inputs.iceberg-full }}-spark-${{ inputs.spark-full }}-scala-${{ inputs.scala }}-jdk${{ inputs.java }}-shard-*-attempt-* + path: iceberg-shard-reports + - name: Verify complete, disjoint Iceberg extensions candidate coverage + run: | + python3 dev/ci/check-iceberg-shards.py --manifests iceberg-shard-reports \ + --task :iceberg-spark:iceberg-spark-extensions-${{ inputs.spark-short }}_${{ inputs.scala }}:test iceberg-spark-runtime: needs: build-native diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 55edd5f5e27..5b5fb70ea57 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -1200,6 +1200,39 @@ def check_cache_save_scope(): return not failures +def check_iceberg_extensions_shards(): + """The Iceberg extensions task must be sharded like the core task. + + Its job is the longest unsharded one in the Iceberg workflow. It has to use + the shared matrix and the same Gradle init script, and a coverage job must + verify the extensions task's candidate inventories, not the core task's. + """ + path = WORKFLOWS / "iceberg_spark_test_reusable.yml" + text = path.read_text(encoding="utf-8") + failures = [] + + job = re.search(r"^ iceberg-spark-extensions:\n(.*?)(?=^ \S)", text, re.M | re.S) + block = job.group(1) if job else "" + if "fromJSON(needs.build-native.outputs.shard-matrix)" not in block: + failures.append("iceberg-spark-extensions does not use the shared shard matrix") + if "cometShardTask=:iceberg-spark:iceberg-spark-extensions-" not in block: + failures.append("iceberg-spark-extensions does not pass -PcometShardTask for its own task") + if "cometShardIndex=" not in block or "cometShardCount=" not in block: + failures.append("iceberg-spark-extensions does not pass the shard index/count") + + coverage = re.search( + r"^ iceberg-spark-extensions-shard-coverage:\n(.*?)(?=^ \S)", text, re.M | re.S) + cblock = coverage.group(1) if coverage else "" + if (not coverage or "check-iceberg-shards.py --manifests" not in cblock + or "--task :iceberg-spark:iceberg-spark-extensions-" not in cblock): + failures.append( + "no iceberg-spark-extensions-shard-coverage job verifying the extensions task") + + for failure in failures: + print(f"iceberg extensions shards: {failure}") + return not failures + + if __name__ == "__main__": ok = check_change_filters() ok = check_event_policy() and ok @@ -1213,6 +1246,7 @@ def check_cache_save_scope(): ok = check_nightly_base_fallback() and ok ok = check_cache_save_scope() and ok ok = check_local_ci_config() and ok + ok = check_iceberg_extensions_shards() and ok if not ok: sys.exit(1) print("CI config checks passed")