Merge pull request #890 from dhellmann/consolidate-e2e-jobs

feat(ci): consolidate e2e tests

Author: LalatenduMohanty

.github/workflows/test.yaml

@@ -75,41 +75,12 @@ jobs:
           # RHEL 9.4 has rustc 1.75
           - "1.75"
         test-script:
-          - bootstrap
-          - bootstrap_extras
-          - bootstrap_build_tags
-          - bootstrap_prerelease
-          - bootstrap_conflicting_requirements
-          - bootstrap_constraints
-          - bootstrap_cache
-          - bootstrap_sdist_only
-          - bootstrap_git_url
-          - bootstrap_git_url_tag
-          - bootstrap_parallel
-          - bootstrap_parallel_git_url
-          - bootstrap_parallel_git_url_tag
-          - bootstrap_skip_constraints
-          - build
-          - build_order
-          - build_parallel
-          - build_sequence_git_url
-          - build_steps
-          - build_settings
-          - graph_to_constraints
-          - meson
-          - migrate_graph
-          - override
-          - pep517_build_sdist
-          - post_bootstrap_hook
-          - prebuilt_wheels_alt_server
-          - report_missing_dependency
-          - rust_vendor
-          - download_sequence
-          - optimize_build
-          - extra_metadata
-          - elfdeps
-          - prebuilt_wheel_hook
-          - lint_requirements
+          - ci_bootstrap_suite
+          - ci_bootstrap_parallel_suite
+          - ci_build_suite
+          - ci_config_suite
+          - ci_specialized_suite
+          - ci_workflow_suite
         os:
           - ubuntu-latest
           - macos-latest
@@ -147,7 +118,7 @@ jobs:
         run: python -m pip install hatch 'click!=8.3.0'
 
       - name: Run tests
-        run: ./e2e/test_${{ matrix.test-script }}.sh
+        run: HATCH_PYTHON=${{ matrix.python-version }} ./e2e/${{ matrix.test-script }}.sh
 
       - name: Upload logs for debugging
         if: ${{ failure() }}

.gitignore

@@ -21,6 +21,7 @@ __pycache__/
 /package.json
 /e2e-output/
 /dist/
+/e2e/flit_core_override/build
 /e2e/fromager_hooks/build
 /e2e/fromager_hooks/dist
 /e2e/pyo3_test/.cargo

.mergify.yml

@@ -37,111 +37,25 @@ pull_request_rules:
           # MyPy type checking
           - check-success=mypy
 
-          - check-success=e2e (3.11, 1.75, bootstrap, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_conflicting_requirements, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_parallel, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_sdist_only, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, bootstrap_skip_constraints, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build_parallel, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build_sequence_git_url, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, build_steps, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, download_sequence, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, elfdeps, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, extra_metadata, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, graph_to_constraints, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, meson, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, migrate_graph, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, optimize_build, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, override, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, pep517_build_sdist, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, post_bootstrap_hook, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, prebuilt_wheel_hook, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest)
-          - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_build_tags, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_conflicting_requirements, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_conflicting_requirements, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_git_url, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_git_url, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_sdist_only, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_sdist_only, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_skip_constraints, macos-latest)
-          - check-success=e2e (3.12, 1.75, bootstrap_skip_constraints, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build, macos-latest)
-          - check-success=e2e (3.12, 1.75, build, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build_order, macos-latest)
-          - check-success=e2e (3.12, 1.75, build_order, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build_parallel, macos-latest)
-          - check-success=e2e (3.12, 1.75, build_parallel, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build_sequence_git_url, macos-latest)
-          - check-success=e2e (3.12, 1.75, build_sequence_git_url, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build_settings, macos-latest)
-          - check-success=e2e (3.12, 1.75, build_settings, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, build_steps, macos-latest)
-          - check-success=e2e (3.12, 1.75, build_steps, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, download_sequence, macos-latest)
-          - check-success=e2e (3.12, 1.75, download_sequence, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, elfdeps, macos-latest)
-          - check-success=e2e (3.12, 1.75, elfdeps, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, extra_metadata, macos-latest)
-          - check-success=e2e (3.12, 1.75, extra_metadata, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, graph_to_constraints, macos-latest)
-          - check-success=e2e (3.12, 1.75, graph_to_constraints, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
-          - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, meson, macos-latest)
-          - check-success=e2e (3.12, 1.75, meson, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, migrate_graph, macos-latest)
-          - check-success=e2e (3.12, 1.75, migrate_graph, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, optimize_build, macos-latest)
-          - check-success=e2e (3.12, 1.75, optimize_build, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, override, macos-latest)
-          - check-success=e2e (3.12, 1.75, override, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, pep517_build_sdist, macos-latest)
-          - check-success=e2e (3.12, 1.75, pep517_build_sdist, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, post_bootstrap_hook, macos-latest)
-          - check-success=e2e (3.12, 1.75, post_bootstrap_hook, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, macos-latest)
-          - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, macos-latest)
-          - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, report_missing_dependency, macos-latest)
-          - check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest)
-          - check-success=e2e (3.12, 1.75, rust_vendor, macos-latest)
-          - check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest)
+          # E2E test suites (consolidated from individual tests)
+          - check-success=e2e (3.11, 1.75, ci_bootstrap_parallel_suite, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, ci_bootstrap_suite, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, ci_build_suite, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, ci_config_suite, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, ci_specialized_suite, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, ci_workflow_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_bootstrap_parallel_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_bootstrap_parallel_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_bootstrap_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_bootstrap_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_build_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_build_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_config_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_config_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_specialized_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_specialized_suite, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, ci_workflow_suite, macos-latest)
+          - check-success=e2e (3.12, 1.75, ci_workflow_suite, ubuntu-latest)
 
           # At least 1 reviewer from maintainers
           - and:

AGENTS.md

@@ -121,10 +121,10 @@ These run automatically on commit if installed with `hatch run lint:install-hook
 
 ## Code Patterns
 
-**Import Guidelines:**
+**Import Guidelines (ALWAYS FOLLOW):**
 
-- **PEP 8: imports should be at the top**: All import statements must be placed at the top of the file, after module docstrings and before other code
-- **No local imports**: Do not place import statements inside functions, methods, or conditional blocks
+- **PEP 8: imports should be at the top**: All import statements MUST be placed at the top of the file, after module docstrings and before other code. This applies to ALL Python files in the project.
+- **No local imports**: Do not place import statements inside functions, methods, or conditional blocks under any circumstances
 
 ### Testing Pattern
 

e2e/ci_bootstrap_parallel_suite.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# CI wrapper script for all bootstrap-parallel-related e2e tests
+# This script runs multiple bootstrap parallel tests sequentially to reduce CI job count
+# while preserving the ability to run individual tests during development.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/ci_suite_framework.sh"
+
+init_suite "Bootstrap Parallel Test Suite" "bootstrap-parallel tests"
+
+# Bootstrap parallel tests
+test_section "bootstrap parallel tests"
+run_test "bootstrap_parallel"
+run_test "bootstrap_parallel_git_url"
+run_test "bootstrap_parallel_git_url_tag"
+
+finish_suite

e2e/ci_bootstrap_suite.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# CI wrapper script for all bootstrap-related e2e tests
+# This script runs multiple bootstrap tests sequentially to reduce CI job count
+# while preserving the ability to run individual tests during development.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/ci_suite_framework.sh"
+
+init_suite "Bootstrap Test Suite" "standard bootstrap tests (parallel bootstrap tests are in separate suite)"
+
+# Bootstrap tests in logical order - basic first, then variations
+test_section "basic bootstrap tests"
+run_test "bootstrap"
+run_test "bootstrap_extras"
+run_test "bootstrap_build_tags"
+
+test_section "bootstrap constraint tests"
+run_test "bootstrap_constraints"
+run_test "bootstrap_skip_constraints"
+run_test "bootstrap_conflicting_requirements"
+
+test_section "bootstrap configuration tests"
+run_test "bootstrap_prerelease"
+run_test "bootstrap_cache"
+run_test "bootstrap_sdist_only"
+
+test_section "bootstrap git URL tests"
+run_test "bootstrap_git_url"
+run_test "bootstrap_git_url_tag"
+
+finish_suite

e2e/ci_build_suite.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# CI wrapper script for all build-related e2e tests
+# This script runs multiple build tests sequentially to reduce CI job count
+# while preserving the ability to run individual tests during development.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/ci_suite_framework.sh"
+
+init_suite "Build Test Suite" "build-related tests"
+
+# Build tests in logical order
+test_section "core build tests"
+run_test "build"
+run_test "build_order"
+run_test "build_steps"
+
+test_section "advanced build tests"
+run_test "build_parallel"
+run_test "build_sequence_git_url"
+
+finish_suite

e2e/ci_config_suite.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# CI wrapper script for all configuration/settings-related e2e tests
+# This script runs multiple configuration tests sequentially to reduce CI job count
+# while preserving the ability to run individual tests during development.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/ci_suite_framework.sh"
+
+init_suite "Configuration Test Suite" "configuration/settings tests"
+
+# Configuration and settings tests
+test_section "configuration tests"
+run_test "build_settings"
+run_test "override"
+run_test "extra_metadata"
+run_test "lint_requirements"
+
+finish_suite

e2e/ci_specialized_suite.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# CI wrapper script for all specialized build-related e2e tests
+# This script runs multiple specialized build tests sequentially to reduce CI job count
+# while preserving the ability to run individual tests during development.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/ci_suite_framework.sh"
+
+init_suite "Specialized Build Test Suite" "specialized build tests"
+
+# Specialized build tests
+test_section "specialized build system tests"
+run_test "meson"
+run_test "rust_vendor"
+
+test_section "build standard tests"
+run_test "pep517_build_sdist"
+
+test_section "platform-specific tests"
+run_test "elfdeps"
+
+test_section "hook tests"
+run_test "prebuilt_wheel_hook"
+
+finish_suite