E2e pipeline overhaul

.github/actions/archive-test-results/action.yaml

@@ -30,11 +30,46 @@ runs:
         name: ${{ inputs.platform }}-screenshots-${{ inputs.test-type }}
         path: ${{ inputs.workspace-path }}/maestro/images/actual/${{ inputs.platform }}/**/*.png
         if-no-files-found: ignore
-
+
+    - name: Collect screenshot diffs
+      # On a failed assertScreenshot, Maestro writes the visual diff (the most useful artifact
+      # for triaging a mismatch) next to the baseline — but under a *doubled* path, e.g.
+      # maestro/images/expected/ios/maestro/images/expected/ios/<name>_diff.png, because it
+      # resolves the diff name against the baseline's own relative path. That nested location
+      # was never matched by the screenshots glob above, so diffs were silently lost. Gather
+      # any *_diff.png from wherever it landed into one flat dir for upload.
+      if: always()
+      shell: bash
+      run: |
+        dest="${{ inputs.workspace-path }}/maestro/images/diff/${{ inputs.platform }}"
+        mkdir -p "$dest"
+        find "${{ inputs.workspace-path }}/maestro/images" -name '*_diff.png' \
+          -not -path "$dest/*" -exec cp -f {} "$dest/" \; 2>/dev/null || true
+        ls -1 "$dest" 2>/dev/null || true
+
+    - name: Archive screenshot diffs
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7
+      if: always()
+      with:
+        name: ${{ inputs.platform }}-diffs-${{ inputs.test-type }}
+        path: ${{ inputs.workspace-path }}/maestro/images/diff/${{ inputs.platform }}/*.png
+        if-no-files-found: ignore
+
     - name: Archive artifacts
       uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7
       if: always()
       with:
         name: ${{ inputs.platform }}-artifacts-${{ inputs.test-type }}
         path: packages/pluggableWidgets/**/artifacts/
+        if-no-files-found: ignore
+
+    # Per-flow videos are recorded for every flow but kept only for FAILED flows (passing
+    # clips are deleted in helpers.sh to save storage), so on a fully-green shard this dir is
+    # empty and the upload is a no-op (if-no-files-found: ignore).
+    - name: Archive failure videos
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7
+      if: always()
+      with:
+        name: ${{ inputs.platform }}-videos-${{ inputs.test-type }}
+        path: maestro/videos/*.mp4
         if-no-files-found: ignore

.github/actions/setup-maestro/action.yaml

@@ -1,21 +1,30 @@
 name: "Setup Maestro"
-description: "Install and cache Maestro"
+description: "Install and cache a pinned version of Maestro"
+inputs:
+  version:
+    description: "Maestro CLI version to install (release tag is cli-<version>)"
+    required: false
+    default: "2.6.1"
 runs:
   using: "composite"
   steps:
     - name: "Cache Maestro"
       uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
       with:
-        path: $HOME/.local/bin/maestro
-        key: maestro-${{ runner.os }}-v1
-
+        # Use ~ rather than $HOME: actions/cache does not expand environment variables.
+        path: ~/.local/bin/maestro
+        key: maestro-${{ runner.os }}-${{ inputs.version }}
+
     - name: "Install Maestro"
       shell: bash
+      env:
+        MAESTRO_VERSION: ${{ inputs.version }}
       run: |
+        set -euo pipefail
         if [ ! -f "$HOME/.local/bin/maestro/bin/maestro" ]; then
-          mkdir -p $HOME/.local/bin
-          curl -L "https://github.com/mobile-dev-inc/maestro/releases/latest/download/maestro.zip" -o maestro.zip
-          unzip maestro.zip -d $HOME/.local/bin
-          chmod +x $HOME/.local/bin/maestro/bin/maestro
+          mkdir -p "$HOME/.local/bin"
+          curl -fL "https://github.com/mobile-dev-inc/maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip" -o maestro.zip
+          unzip -q maestro.zip -d "$HOME/.local/bin"
+          chmod +x "$HOME/.local/bin/maestro/bin/maestro"
         fi
-        echo "$HOME/.local/bin" >> $GITHUB_PATH
+        echo "$HOME/.local/bin" >> "$GITHUB_PATH"

.github/actions/setup-tools/action.yaml

@@ -1,41 +1,25 @@
 name: "Setup Tools"
 description: "Common setup for widget and js test jobs"
-inputs:
-  mendix_version:
-    description: "Mendix version"
-    required: true
 outputs:
   java-path:
     description: "Path to the installed Java"
     value: ${{ steps.setup-java.outputs.path }}
 runs:
   using: "composite"
   steps:
-    - name: "Setup Python and dependencies"
-      uses: ./.github/actions/setup-python
+    # Test jobs need only Java + Maestro:
+    #  - the portable app package bundles the runtime, so there's no CDN runtime download and no
+    #    m2ee tooling to set up (and nothing here needs Python).
+    #  - Maestro is a standalone binary, so there's no Node/pnpm install — which would also run the
+    #    root `prepare` lifecycle the test jobs don't need (a known ENOTEMPTY flake source).
 
-    - name: "Setup Node and dependencies"
-      uses: ./.github/actions/setup-node-with-cache
-
     - name: "Setup Java 21"
       id: setup-java
       uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 #v5.2.0
       with:
         distribution: "temurin"
         java-version: "21"
 
-    - name: "Cache Mendix runtime"
-      uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 #v5
-      with:
-        path: tmp/runtime.tar.gz
-        key: mendix-runtime-${{ inputs.mendix_version }}
-
-    - name: "Cache m2ee-tools"
-      uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 #v5
-      with:
-        path: tmp/m2ee
-        key: m2ee-tools-v1
-
     - name: "Install Maestro"
       uses: ./.github/actions/setup-maestro
 

.github/actions/start-mendix-runtime/action.yaml

@@ -0,0 +1,81 @@
+name: "Start Mendix runtime (portable app)"
+description: "Extract the portable app package, start it in the background, and wait until it actually serves requests"
+inputs:
+  app-package:
+    description: "Path to the portable app package zip produced by 'mxbuild --target=portable-app-package'"
+    required: false
+    default: "app.zip"
+  java-home:
+    description: "JAVA_HOME to run the runtime with (defaults to the JAVA_HOME already in the environment)"
+    required: false
+    default: ""
+  admin-pass:
+    description: "Value for M2EE_ADMIN_PASS (secures the runtime admin port)"
+    required: false
+    default: "Password1!"
+  runtime-url:
+    description: "URL polled until the runtime responds, confirming readiness"
+    required: false
+    default: "http://localhost:8080/"
+  timeout-seconds:
+    description: "How long to wait for the runtime to become ready before failing"
+    required: false
+    default: "300"
+  session-timeout-ms:
+    description: >
+      Server-side session idle timeout (ms). The runtime uses a trial license capped at
+      6 concurrent named users, and every Maestro flow does `launchApp clearState: true`,
+      which starts a FRESH server session while the previous one lingers for the full
+      timeout. With Mendix's 10-min default, a widget with >5 flows piles past 6 sessions
+      and the next login fails with "Maximum number of sessions exceeded". A short idle
+      timeout lets each flow's session expire before the next ones accumulate. Active
+      sessions aren't affected — client requests keep resetting the idle timer mid-flow.
+    required: false
+    default: "60000"
+runs:
+  using: "composite"
+  steps:
+    - name: "Start runtime and wait until ready"
+      shell: bash
+      env:
+        APP_PACKAGE: ${{ inputs.app-package }}
+        INPUT_JAVA_HOME: ${{ inputs.java-home }}
+        M2EE_ADMIN_PASS: ${{ inputs.admin-pass }}
+        RUNTIME_URL: ${{ inputs.runtime-url }}
+        TIMEOUT_SECONDS: ${{ inputs.timeout-seconds }}
+        # Read by the portable app's HOCON config (etc/variables.conf: ${?RUNTIME_PARAMS_*}),
+        # inherited by the JVM started below. Caps session pile-up under the 6-user trial
+        # license — see the session-timeout-ms input for the full rationale.
+        RUNTIME_PARAMS_SESSIONTIMEOUT: ${{ inputs.session-timeout-ms }}
+        # Reap expired sessions promptly (default cluster cleanup runs only every 5 min, which
+        # would hold license slots long after the sessions go idle).
+        RUNTIME_PARAMS_CLUSTERMANAGERACTIONINTERVAL: "30000"
+      run: |
+        set -euo pipefail
+
+        # Allow an explicit JAVA_HOME override; otherwise rely on the one set up earlier.
+        if [ -n "$INPUT_JAVA_HOME" ]; then
+          export JAVA_HOME="$INPUT_JAVA_HOME"
+        fi
+        echo "Using JAVA_HOME=${JAVA_HOME:-<from PATH>}"
+
+        rm -rf runtime-app
+        mkdir -p runtime-app log
+        unzip -qq "$APP_PACKAGE" -d runtime-app
+        chmod +x runtime-app/bin/start
+
+        # Start the bundled runtime in the background; it must outlive this step.
+        (cd runtime-app && nohup ./bin/start etc/Default > "$GITHUB_WORKSPACE/log/runtime.log" 2>&1 &)
+
+        echo "Waiting up to ${TIMEOUT_SECONDS}s for the runtime at ${RUNTIME_URL} ..."
+        deadline=$(( SECONDS + TIMEOUT_SECONDS ))
+        until curl -fsS -o /dev/null "$RUNTIME_URL"; do
+          if [ "$SECONDS" -ge "$deadline" ]; then
+            echo "::error::Mendix runtime did not become ready within ${TIMEOUT_SECONDS}s"
+            echo "----- last 100 lines of runtime.log -----"
+            tail -n 100 "$GITHUB_WORKSPACE/log/runtime.log" 2>/dev/null || echo "(no log)"
+            exit 1
+          fi
+          sleep 3
+        done
+        echo "Mendix runtime is ready and serving at ${RUNTIME_URL}"

.github/scripts/README.md

@@ -4,40 +4,74 @@ This directory contains helper scripts used by the CI/CD pipeline.
 
 ## determine-widget-scope.sh
 
-Determines which widgets and JS actions should be built and tested based on changed files or manual input.
+Determines which widgets and JS actions should be built and tested, based on the
+files changed in a PR or on manual `workflow_dispatch` input.
 
 ### Usage
 
 ```bash
-./determine-widget-scope.sh <event_name> <input_workspace> <before_commit> <current_commit>
+./determine-widget-scope.sh <event_name> <input_workspace> <base_commit> <current_commit>
 ```
 
+For a `pull_request`, `<base_commit>` is the PR base SHA
+(`github.event.pull_request.base.sha`). The script diffs against the **merge-base** of
+that and `<current_commit>`, so every commit in the PR is considered (not just the
+latest). This needs full git history — the `scope` job checks out with `fetch-depth: 0`.
+On `workflow_dispatch`, `<input_workspace>` drives the scope and the commit args are
+ignored.
+
 ### Outputs
 
-| Output               | Description                                          |
-| -------------------- | ---------------------------------------------------- |
-| `scope`              | pnpm filter scope for building                       |
-| `widgets`            | JSON array of widgets to BUILD                       |
-| `widgets_to_test`    | JSON array of widgets to TEST (used for test matrix) |
-| `js_actions_changed` | Boolean flag indicating if JS actions changed        |
+| Output               | Description                                                                      |
+| -------------------- | -------------------------------------------------------------------------------- |
+| `scope`              | pnpm filter scope (used by the unit-test step)                                   |
+| `widgets`            | JSON array of widgets to BUILD                                                   |
+| `widgets_to_test`    | JSON array of widgets to TEST (the test matrix)                                  |
+| `js_actions_changed` | Boolean — whether JS actions changed                                             |
+| `full_build`         | Boolean — true when every widget is (re)built; gates the dist cache restore/save |
+
+### Build vs. test scope
+
+`widgets_to_test` and `widgets` are deliberately **separate**:
+
+-   A change anywhere under a widget folder marks it for **testing**.
+-   Only a **build-affecting** change (anything outside the widget's `e2e/` folder, e.g.
+    `src/**` or `package.json`) marks it for **building**. A change confined to `e2e/`
+    (Maestro flows / screenshots) changes only the test, so the widget is tested against
+    the test project's baseline `.mpk` instead of being rebuilt.
+
+So `widgets` can be a strict subset of `widgets_to_test` (or empty while
+`widgets_to_test` is non-empty, for an e2e-only PR).
 
 ### Pipeline Behavior
 
-| Scenario               | Widgets Built   | Widgets Tested  | JS Actions Built | JS Actions Tested |
-| ---------------------- | --------------- | --------------- | ---------------- | ----------------- |
-| Only JS actions change | All             | None            | Yes              | Yes               |
-| Only widgets change    | Changed only    | Changed only    | No               | No                |
-| Both change            | Changed widgets | Changed widgets | Yes              | Yes               |
-| Full run (`*-native`)  | All             | All             | Yes              | Yes               |
-| Manual: `js-actions`   | All             | None            | Yes              | Yes               |
+| Scenario                        | Widgets Built          | Widgets Tested  | JS Actions Built | JS Actions Tested |
+| ------------------------------- | ---------------------- | --------------- | ---------------- | ----------------- |
+| Only JS actions change          | All                    | None            | Yes              | Yes               |
+| Only widget `src`/deps change   | Changed only           | Changed only    | No               | No                |
+| Only widget `e2e/` changes      | None (use baseline)    | Changed only    | No               | No                |
+| Both widgets and JS actions     | Build-affected widgets | Changed widgets | Yes              | Yes               |
+| Full run (`*-native` / default) | All                    | All             | Yes              | Yes               |
+| Manual: `js-actions`            | All                    | None            | Yes              | Yes               |
 
 ### Why build all widgets when only JS actions change?
 
-The JS action tests run against a full test project that requires all widgets to be present. Without building all widgets, the test project would be incomplete and tests would fail.
+The JS action tests run against a full test project that requires all widgets to be
+present. Without building all widgets, the test project would be incomplete and tests
+would fail.
 
 ## Other Scripts
 
--   **determine-nt-version.py** - Determines the Native Template version based on Mendix version
--   **mxbuild.Dockerfile** - Docker image for mxbuild
--   **setup-runtime.sh** - Sets up the Mendix runtime
--   **start-runtime-with-verification.sh** - Starts the runtime with health verification
+-   **determine-nt-version.mjs** — resolves the Native Template release/branch from the
+    Mendix version. Node ESM, no external deps (uses global `fetch` + a small version
+    comparator); run with the runner's preinstalled `node`. Fails loudly rather than
+    silently pinning `master`.
+-   **mxbuild.Dockerfile** — Docker image for mxbuild.
+
+> The Mendix runtime for the test jobs is produced as a **portable app package**
+> (`mxbuild --target=portable-app-package`) in the `project` job and started by the
+> `start-mendix-runtime` composite action, which waits for a real readiness probe.
+> That runtime runs under a 6-user trial license, so the action shortens the server
+> session idle timeout (`RUNTIME_PARAMS_SESSIONTIMEOUT`) — each Maestro flow opens a
+> fresh session via `launchApp clearState`, and a short timeout lets them expire
+> between flows instead of piling up past the cap.