diff --git a/.github/scripts/extract_matrix.py b/.github/scripts/extract_matrix.py index 2995274940..7347eaae72 100644 --- a/.github/scripts/extract_matrix.py +++ b/.github/scripts/extract_matrix.py @@ -360,7 +360,7 @@ def __gt__(self, other): return NotImplemented -def get_tagged_jobs(buildspec, target, filter=None): +def get_tagged_jobs(buildspec, target, filter=None, artifact_mode="all"): jobs = [Job({"name": target}).to_dict()] for job in sorted([Job(build) for build in buildspec.get("builds", [])]): if not any(t for t in job.targets if t in [target]): @@ -371,15 +371,19 @@ def get_tagged_jobs(buildspec, target, filter=None): continue if [x for x in JOB_EXCLUSION_TERMS if x in str(job)]: continue + if artifact_mode == "providers" and not job.upload_artifact: + continue + if artifact_mode == "non-providers" and job.upload_artifact: + continue jobs.append(job.to_dict()) return jobs -def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False): +def main(jsonnet_bin, ci_jsonnet, target, filter=None, artifact_mode="all", indent=False): result = subprocess.check_output([jsonnet_bin, ci_jsonnet], text=True) buildspec = json.loads(result) - tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter) + tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter, artifact_mode=artifact_mode) matrix = tagged_jobs print(json.dumps(matrix, indent=2 if indent else None)) @@ -390,6 +394,12 @@ def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False): parser.add_argument("ci_jsonnet", help="Path to ci.jsonnet spec") parser.add_argument("target", help="Target name (e.g., tier1)") parser.add_argument("filter", nargs="?", default=None, help="Regex filter for job names (optional)") + parser.add_argument( + "--artifact-mode", + choices=["all", "providers", "non-providers"], + default="all", + help="Select all jobs, only artifact providers, or only jobs that do not publish artifacts.", + ) parser.add_argument('--indent', action='store_true', help='Indent output JSON') args = parser.parse_args() main( @@ -397,5 +407,6 @@ def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False): ci_jsonnet=args.ci_jsonnet, target=args.target, filter=args.filter, + artifact_mode=args.artifact_mode, indent=args.indent or sys.stdout.isatty(), ) diff --git a/.github/scripts/unpack-artifact b/.github/scripts/unpack-artifact index 501f1d6079..a0650dd4b5 100755 --- a/.github/scripts/unpack-artifact +++ b/.github/scripts/unpack-artifact @@ -1,17 +1,24 @@ #!/bin/bash +set -euo pipefail -to_extract="$ARTIFACT_PATHS" +to_extract="${ARTIFACT_PATHS:-}" if [ -n "$1" ]; then to_extract="${ARTIFACT_PATH_PREFIX}/${1}.tar" fi +if [ -z "$to_extract" ]; then + echo "::error::No artifact archive path was provided." + exit 1 +fi + for i in $to_extract; do if [ -f "$i" ]; then tar -x -f "$i" -C "${ARTIFACT_PATH_PREFIX}" -v echo "Unpacked $i in $(pwd)/${ARTIFACT_PATH_PREFIX}" else - echo "file not found in ${ARTIFACT_PATH_PREFIX}" + echo "::error::Required artifact archive not found: $i" ls "${ARTIFACT_PATH_PREFIX}" + exit 1 fi -done \ No newline at end of file +done diff --git a/.github/workflows/ci-matrix-gen.yml b/.github/workflows/ci-matrix-gen.yml index 398e221dd0..af9672b602 100644 --- a/.github/workflows/ci-matrix-gen.yml +++ b/.github/workflows/ci-matrix-gen.yml @@ -29,6 +29,7 @@ jobs: generate-tier1: runs-on: ubuntu-latest outputs: + artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }} matrix: ${{ steps.set-matrix.outputs.matrix }} env: TARGET: tier1 @@ -42,13 +43,17 @@ jobs: - name: Extract job matrix id: set-matrix run: | - python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} > matrix.json + python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} --artifact-mode providers > artifact-matrix.json + python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} --artifact-mode non-providers > matrix.json + cat artifact-matrix.json cat matrix.json + echo "artifact_matrix=$(cat artifact-matrix.json)" >> $GITHUB_OUTPUT echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT generate-tier2: runs-on: ubuntu-latest outputs: + artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }} matrix: ${{ steps.set-matrix.outputs.matrix }} env: TARGET: tier2 @@ -58,19 +63,20 @@ jobs: generate-tier3: runs-on: ubuntu-latest outputs: + artifact_matrix: ${{ steps.set-matrix.outputs.artifact_matrix }} matrix: ${{ steps.set-matrix.outputs.matrix }} env: TARGET: tier3 JOBS: ${{ inputs.jobs_to_run }} steps: *generate_matrix - tier1: + tier1-artifacts: needs: generate-tier1 runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }} + include: ${{ fromJson(needs.generate-tier1.outputs.artifact_matrix) }} steps: &buildsteps - name: Process matrix downloads if: ${{ matrix.downloads_steps }} @@ -168,9 +174,9 @@ jobs: git config --global http.timeout 600 git clone https://github.com/graalvm/mx if [[ "${RUNNER_OS}" == "Windows" ]]; then - ./mx/mx.cmd fetch-jdk -A --jdk-id labsjdk-ce-latest + ./mx/mx.cmd fetch-jdk -A --configuration main/ci/graal/common.json --jdk-id labsjdk-ce-latest else - ./mx/mx fetch-jdk -A --jdk-id labsjdk-ce-latest + ./mx/mx fetch-jdk -A --configuration main/ci/graal/common.json --jdk-id labsjdk-ce-latest fi git -C mx checkout ${{ matrix.mx_version }} @@ -200,7 +206,6 @@ jobs: with: pattern: ${{ matrix.require_artifact[0] }} merge-multiple: true - continue-on-error: true - name: Export artifact paths Linux if: ${{ matrix.require_artifact }} @@ -288,20 +293,50 @@ jobs: path: /tmp/test-report-*.json retention-days: 1 - tier2: - if: ${{ success() || inputs.jobs_to_run }} + tier1: + if: ${{ always() && needs.generate-tier1.result == 'success' && needs.tier1-artifacts.result == 'success' }} + needs: [generate-tier1, tier1-artifacts] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }} + steps: *buildsteps + + tier2-artifacts: + if: ${{ always() && needs.generate-tier2.result == 'success' && (needs.tier1.result == 'success' || inputs.jobs_to_run) }} needs: [generate-tier2, tier1] runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-tier2.outputs.artifact_matrix) }} + steps: *buildsteps + + tier2: + if: ${{ always() && needs.generate-tier2.result == 'success' && needs.tier2-artifacts.result == 'success' }} + needs: [generate-tier2, tier2-artifacts] + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: ${{ fromJson(needs.generate-tier2.outputs.matrix) }} steps: *buildsteps - tier3: - if: ${{ success() || inputs.jobs_to_run }} + tier3-artifacts: + if: ${{ always() && needs.generate-tier3.result == 'success' && (needs.tier2.result == 'success' || inputs.jobs_to_run) }} needs: [generate-tier3, tier2] runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-tier3.outputs.artifact_matrix) }} + steps: *buildsteps + + tier3: + if: ${{ always() && needs.generate-tier3.result == 'success' && needs.tier3-artifacts.result == 'success' }} + needs: [generate-tier3, tier3-artifacts] + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py index cdc39e2fa7..9687d19dcb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py @@ -230,47 +230,52 @@ def test_graal_python_args(self): if sys.implementation.name == "graalpy": import subprocess - env = {"GRAAL_PYTHON_ARGS": "-c 12"} + def env_with_graal_python_args(args): + env = os.environ.copy() + env["GRAAL_PYTHON_ARGS"] = args + return env + + env = env_with_graal_python_args("-c 12") result = subprocess.run([sys.executable], env=env) self.assertEqual(0, result.returncode) - env = {"GRAAL_PYTHON_ARGS": "-c 'print(12)'"} + env = env_with_graal_python_args("-c 'print(12)'") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('12\n', result) - env = {"GRAAL_PYTHON_ARGS": """-c 'print("Hello world")'"""} + env = env_with_graal_python_args("""-c 'print("Hello world")'""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('Hello world\n', result) - env = {"GRAAL_PYTHON_ARGS": """-c ""'print("Hello world")'"""""} + env = env_with_graal_python_args("""-c ""'print("Hello world")'""""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('Hello world\n', result) - env = {"GRAAL_PYTHON_ARGS": r"""-c 'print(\'"Hello world"\')'"""""} + env = env_with_graal_python_args(r"""-c 'print(\'"Hello world"\')'""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('"Hello world"\n', result) - env = {"GRAAL_PYTHON_ARGS": """\v-c\vprint('"Hello world"')"""} + env = env_with_graal_python_args("""\v-c\vprint('"Hello world"')""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('"Hello world"\n', result) - env = {"GRAAL_PYTHON_ARGS": """\v-c\vprint('Hello', "world")"""} + env = env_with_graal_python_args("""\v-c\vprint('Hello', "world")""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('Hello world\n', result) # check that the subprocess receives the args and thus it should fail because it recurses args = """\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))""" - env = {"GRAAL_PYTHON_ARGS": args} + env = env_with_graal_python_args(args) result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual(f"{args}\n", result) # check that the subprocess does not receive the args when we end with \v - env = {"GRAAL_PYTHON_ARGS": """\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))\v"""} + env = env_with_graal_python_args("""\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))\v""") result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual('None\n', result) # check that the subprocess receives an empty arg args = """\v-c\vimport sys\nprint(repr(sys.argv))\va1\v\va3""" - env = {"GRAAL_PYTHON_ARGS": args} + env = env_with_graal_python_args(args) result = subprocess.check_output([sys.executable], env=env, text=True) self.assertEqual("['-c', 'a1', '', 'a3']\n", result) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_venv.py b/graalpython/com.oracle.graal.python.test/src/tests/test_venv.py index dcde552f60..539d5d8d53 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_venv.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_venv.py @@ -74,6 +74,8 @@ def test_venv_launcher(self): with open(tmpfile, "ab") as f: sz = f.write(launcher_command.encode("utf-16le")) assert f.write(struct.pack("@I", sz)) == 4 + env = os.environ.copy() + env["PYLAUNCHER_DEBUG"] = "1" try: out = subprocess.check_output([tmpfile, "-c", """if True: import sys, os @@ -81,9 +83,12 @@ def test_venv_launcher(self): print("Hello", sys.executable) print("Base", sys._base_executable) print("Original", __graalpython__.venvlauncher_command) - """], env={"PYLAUNCHER_DEBUG": "1"}, text=True) + """], env=env, stderr=subprocess.STDOUT, text=True) except subprocess.CalledProcessError as err: - out = err.output.decode(errors="replace") if err.output else "" + if isinstance(err.output, str): + out = err.output + else: + out = err.output.decode(errors="replace") if err.output else "" print("out=", out, sep="\n") assert f"Hello {tmpfile}" in out, out assert f"Base {os.path.realpath(sys.executable)}" in out, out diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 8392bc9d86..865a37a3b0 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -3116,6 +3116,10 @@ def apply(self, config): def processDeps(self, deps): if PythonMxUnittestConfig.useResources: deps.add(mx.distribution('GRAALPYTHON_RESOURCES', fatalIfMissing=True)) + if mx.suite('compiler', fatalIfMissing=False): + # punittest enables compiler-failure checks when /compiler is imported; + # the corresponding engine options are registered by the optimized Truffle runtime. + deps.add(mx.distribution('truffle:TRUFFLE_RUNTIME', fatalIfMissing=True)) mx_unittest.register_unittest_config(PythonMxUnittestConfig('python-internal'))