Skip to content

Add snapshot golden tests #1446

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ludfjig wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add snapshot golden tests #1446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e9fe886
fix(guest): prefault exception handler counter
ludfjig Jun 24, 2026
0658445
Golden tests
ludfjig Jun 24, 2026
2219634
Pr feedback
ludfjig Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 248 additions & 0 deletions .github/workflows/RegenSnapshotGoldens.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json

# Publish snapshot goldens to
# ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens.
#
# Runs automatically when a merge to main changes GOLDENS_VERSION (the
# version string lives in
# src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs). The check-published
# job reads that version and checks GHCR for its `{version}-complete`
# marker. If the marker is absent, the matrix walks every (hv, cpu,
# config) combination, dumps the canonical snapshot, and uploads it as a
# workflow artifact. A single publish job then downloads every artifact,
# pushes each as a tag named `{version}-{hv}-{cpu}-{profile}`, and
# pushes the marker last. Publishing the whole set from one job means a
# partial run leaves no marker and is republished on the next run.
#
# A version whose marker exists is left untouched, so a merge that does
# not bump the version, or a re-run of the same version, is a no-op.
# Manual dispatch with `force: true` overwrites an existing version and
# exists for recovery only.
#
# See docs/snapshot-versioning.md

name: Regenerate Snapshot Goldens

on:
push:
branches: [main]
paths:
- src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs
workflow_dispatch:
inputs:
version:
description: Goldens version string. Must match GOLDENS_VERSION in source (e.g. "v1.0").
required: true
type: string
force:
description: Overwrite tags even if the version is already published (recovery only).
type: boolean
default: false

env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: full
GHCR_IMAGE: ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens

permissions:
contents: read
packages: write

concurrency:
group: regen-snapshot-goldens-${{ github.ref }}
cancel-in-progress: false

defaults:
run:
shell: bash

jobs:
check-published:
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
outputs:
version: ${{ steps.decide.outputs.version }}
needs_publish: ${{ steps.decide.outputs.needs_publish }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- name: Install oras
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
with:
version: 1.3.2

- name: Decide version and whether to publish
id: decide
env:
EVENT_NAME: ${{ github.event_name }}
INPUT_VERSION: ${{ inputs.version }}
FORCE: ${{ inputs.force }}
GHCR_USER: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
if ! [[ "${SRC}" =~ ^v[0-9]+\.[0-9]+$ ]]; then
echo "::error::GOLDENS_VERSION in source must match ^v[0-9]+\.[0-9]+$ (e.g. v1.0), found '${SRC}'"
exit 1
fi

# On manual dispatch the input must name the version that the
# dispatched ref actually carries. This catches a stale input.
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${INPUT_VERSION}" != "${SRC}" ]; then
echo "::error::version input '${INPUT_VERSION}' does not match GOLDENS_VERSION in source '${SRC}'"
exit 1
fi

echo "version=${SRC}" >> "$GITHUB_OUTPUT"

if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${FORCE}" = "true" ]; then
echo "force requested: will publish ${SRC} even if it already exists"
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
exit 0
fi

# A version is frozen once its completion marker exists on
# GHCR. The marker is pushed only after every matrix job has
# uploaded its tag, so a partial push (some jobs failed)
# leaves no marker and the next run republishes the missing
# combinations. Publishing only when the marker is absent makes the
# workflow idempotent and never clobbers a complete baseline.
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
if oras repo tags "${GHCR_IMAGE}" 2>/dev/null | grep -qxF "${SRC}-complete"; then
echo "${SRC} already published (marker ${SRC}-complete present). Nothing to do."
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
else
echo "${SRC} not fully published yet. Will publish."
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
fi

build-guests:
needs: check-published
if: needs.check-published.outputs.needs_publish == 'true'
strategy:
matrix:
config: [debug, release]
uses: ./.github/workflows/dep_build_guests.yml
with:
config: ${{ matrix.config }}
secrets: inherit

generate-snapshots:
needs: [check-published, build-guests]
if: needs.check-published.outputs.needs_publish == 'true'
strategy:
fail-fast: false
matrix:
hypervisor: [kvm, mshv3, hyperv-ws2025]
cpu: [amd, intel]
config: [debug, release]
runs-on: ${{ fromJson(
format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=regen-goldens-{3}-{4}-{5}-{6}"]',
matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
matrix.cpu,
matrix.config,
github.run_id,
github.run_number,
github.run_attempt)) }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
with:
rust-toolchain: "1.94"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Fix cargo home permissions
if: runner.os == 'Linux'
run: sudo chown -R $(id -u):$(id -g) /opt/cargo || true

- name: Download Rust guests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rust-guests-${{ matrix.config }}
path: src/tests/rust_guests/bin/${{ matrix.config }}/

- name: Confirm source matches resolved version
env:
RESOLVED_VERSION: ${{ needs.check-published.outputs.version }}
run: |
set -euo pipefail
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
if [ "${SRC}" != "${RESOLVED_VERSION}" ]; then
echo "::error::source GOLDENS_VERSION '${SRC}' does not match resolved '${RESOLVED_VERSION}'"
exit 1
fi

- name: Generate snapshots
run: just snapshot-goldens-generate ${{ matrix.config }} "$RUNNER_TEMP/snapshot-goldens"

- name: Resolve produced tag
id: tag
env:
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
run: |
set -euo pipefail
shopt -s nullglob
layouts=("$RUNNER_TEMP/snapshot-goldens/${GOLDENS_VERSION}-"*/)
if [ "${#layouts[@]}" -ne 1 ]; then
echo "::error::expected exactly one golden layout under $RUNNER_TEMP/snapshot-goldens, found ${#layouts[@]}: ${layouts[*]:-none}"
exit 1
fi
layout="${layouts[0]%/}"
echo "tag=$(basename "${layout}")" >> "$GITHUB_OUTPUT"
echo "dir=${layout}" >> "$GITHUB_OUTPUT"

- name: Upload golden layout
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: golden-${{ steps.tag.outputs.tag }}
path: ${{ steps.tag.outputs.dir }}/
if-no-files-found: error
retention-days: 1

# Push every matrix job's snapshot from this single job, so the published set is
# whole or absent. `generate-snapshots` runs `fail-fast: false` and uploads each
# snapshot as an artifact, so this job's `needs` succeeds only when
# all matrix jobs did. It downloads every artifact, pushes each tag, then
# pushes the `{version}-complete` marker that `check-published` gates on. A
# push that dies partway leaves no marker, so the next run republishes.
publish:
needs: [check-published, generate-snapshots]
if: needs.check-published.outputs.needs_publish == 'true'
runs-on: ubuntu-latest
steps:
- name: Install oras
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
with:
version: 1.3.2

- name: Download all golden layouts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: golden-*
path: layouts

- name: Push goldens and completion marker
env:
GHCR_USER: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
run: |
set -euo pipefail
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
for layout in layouts/golden-*/; do
tag=$(basename "${layout%/}")
tag=${tag#golden-}
echo "::group::push ${tag}"
oras cp --from-oci-layout "${layout%/}:${tag}" "${GHCR_IMAGE}:${tag}"
echo "::endgroup::"
done
printf '%s' "${GOLDENS_VERSION}" > complete.txt
oras push "${GHCR_IMAGE}:${GOLDENS_VERSION}-complete" \
--artifact-type application/vnd.hyperlight.goldens.complete.v1 \
complete.txt:text/plain
47 changes: 47 additions & 0 deletions .github/workflows/ValidatePullRequest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,31 @@ jobs:
with:
docs_only: ${{ needs.docs-pr.outputs.docs-only }}

# Pick the goldens mode. The `regen-goldens` label means regenerate. No label means pull.
goldens-mode:
runs-on: ubuntu-latest
outputs:
regen: ${{ steps.check.outputs.regen || 'false' }}
steps:
- id: check
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
run: |
labels="$(gh pr view ${{ github.event.pull_request.number }} \
--repo ${{ github.repository }} --json labels -q '.labels[].name')"
if grep -qx regen-goldens <<<"$labels"; then
echo "regen=true" >> "$GITHUB_OUTPUT"
else
echo "regen=false" >> "$GITHUB_OUTPUT"
fi

# Build and test - needs guest artifacts
build-test:
needs:
- docs-pr
- build-guests
- goldens-mode
# Required because update-guest-locks is skipped on non-dependabot PRs,
# and a skipped dependency transitively skips all downstream jobs.
# See: https://github.com/actions/runner/issues/2205
Expand All @@ -101,6 +121,31 @@ jobs:
hypervisor: ${{ matrix.hypervisor }}
cpu: ${{ matrix.cpu }}
config: ${{ matrix.config }}
regen_goldens: ${{ needs.goldens-mode.outputs.regen }}

# Cross-CPU snapshot check for regenerated baselines. Only the regen
# path needs it: the pull path already runs self+cross per cell.
# Each build-test cell uploads its generated layout, then this job
# loads the peer CPU's layout and verifies the opposite vendor.
snapshot-cross-verify:
needs:
- docs-pr
- build-test
- goldens-mode
if: ${{ !cancelled() && !failure() && needs.goldens-mode.outputs.regen == 'true' }}
strategy:
fail-fast: false
matrix:
hypervisor: ['hyperv-ws2025', mshv3, kvm]
cpu: [amd, intel]
config: [debug, release]
uses: ./.github/workflows/dep_snapshot_cross_verify.yml
secrets: inherit
with:
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
hypervisor: ${{ matrix.hypervisor }}
cpu: ${{ matrix.cpu }}
config: ${{ matrix.config }}

# Run examples - needs guest artifacts, runs in parallel with build-test
run-examples:
Expand Down Expand Up @@ -164,7 +209,9 @@ jobs:
- update-guest-locks
- build-guests
- code-checks
- goldens-mode
- build-test
- snapshot-cross-verify
- run-examples
- fuzzing
- spelling
Expand Down
Loading