acceptance: schema-driven invariant fuzzing for bundle configs

.github/workflows/push.yml

@@ -408,6 +408,72 @@ jobs:
         run: |
           go tool -modfile=tools/task/go.mod task test-sandbox
 
+  test-fuzz:
+    needs:
+      - cleanups
+
+    # Wide rotating seed window with drift on: too slow for every PR, so nightly only and
+    # not part of test-result. The committed acceptance test still checks no-panic per PR.
+    if: ${{ github.event_name == 'schedule' }}
+    name: "task test-fuzz"
+    runs-on:
+      group: databricks-protected-runner-group-large
+      labels: linux-ubuntu-latest-large
+
+    defaults:
+      run:
+        shell: bash
+
+    permissions:
+      id-token: write
+      contents: read
+      # For the failure-reporting step's PR comment.
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-fuzz
+
+      - name: Run tests
+        env:
+          FUZZ_SEED_COUNT: "25"
+        run: |
+          # Non-overlapping windows, so CI explores new configs every run.
+          export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT ))
+          go tool -modfile=tools/task/go.mod task test-fuzz
+
+      # Not in test-result, so surface failures by commenting on the PR under test.
+      - name: Report failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          body=$(cat <<EOF
+          The nightly schema fuzz invariant job (\`task test-fuzz\`) failed while running against
+          commit \`$COMMIT\`. Each night covers a new seed window, so the failing config is most
+          likely pre-existing rather than introduced by this commit.
+
+          Run: $RUN_URL
+
+          The job log prints a ready-to-run repro as \`LOG.repro: fuzz: seed <n> ...\`.
+          EOF
+          )
+
+          # The commit's pulls endpoint returns the PR that introduced it.
+          pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty')
+          if [ -n "$pr" ]; then
+            gh pr comment "$pr" --body "$body"
+          else
+            echo "No PR found for commit $COMMIT; skipping failure comment" >&2
+          fi
+
   # This job groups the result of all the above test jobs.
   # It is a required check, so it blocks auto-merge and the merge queue.
   #

.gitignore

@@ -19,6 +19,7 @@
 *.log
 coverage.txt
 coverage-acceptance.txt
+coverage-fuzz.txt
 .coverage
 
 __pycache__

Taskfile.yml

@@ -730,6 +730,53 @@ tasks:
           --packages ./acceptance/... \
           -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox"
 
+  test-fuzz:
+    desc: Run schema fuzz invariant tests (random configs, direct engine)
+    # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't see.
+    cmds:
+      - |
+        # Wider window than the committed run, with drift on; a repro narrows it via
+        # FUZZ_SEED_START/COUNT.
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}"
+        export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}"
+        # -count=1: only the script reads FUZZ_*, so the test cache would serve another window's
+        # result as this one's.
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -count=1 -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz"
+
+  test-fuzz-cover:
+    desc: Run the schema fuzzer under coverage and report which CLI packages it exercises
+    # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars.
+    cmds:
+      - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/
+      - mkdir -p ./acceptance/build/cover-fuzz-merged/
+      - |
+        # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, so every
+        # fuzzed `bundle` invocation drops aggregatable counter files. Drift off so the run
+        # exercises the full deploy/plan path instead of stopping on the first drift.
+        export CLI_GOCOVERDIR=build/cover-fuzz
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}"
+        unset FUZZ_CHECK_DRIFT
+        {{.GO_TOOL}} gotestsum \
+          --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \
+          --no-summary=skipped \
+          --packages ./acceptance/... \
+          -- -count=1 -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true
+      - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/"
+      - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt
+      - |
+        echo "== total CLI coverage exercised by the fuzz run =="
+        go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}'
+        echo
+        echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) =="
+        go tool covdata percent -i=acceptance/build/cover-fuzz-merged \
+          | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%%  %s\n", p, $1}' \
+          | grep -E '  (bundle|cmd/bundle)/' \
+          | sort -n
+
   # --- Integration tests ---
 
   integration:

acceptance/bin/emit_fuzz_config.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""
+Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from FUZZ_MODE so
+the invariant scripts don't each duplicate the branch:
+
+  generate (default) - build from scratch by walking `bundle schema` (gen_fuzz_config.py).
+  mutate             - perturb a curated invariant config (mutate_fuzz_config.py).
+
+Reads its inputs from the environment the invariant scripts export: FUZZ_SEED, FUZZ_SCHEMA,
+UNIQUE_NAME, TESTDIR.
+"""
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from envsubst import substitute_variables
+from gen_fuzz_config import gen_config, to_yaml
+from mutate_fuzz_config import load_yaml, mutate
+
+# Curated single-resource configs that deploy standalone (only $UNIQUE_NAME, no init script). All
+# are in the invariant INPUT_CONFIG matrix, so they stay deploy-verified.
+MUTATE_BASES = [
+    "catalog",
+    "external_location",
+    "job",
+    "model",
+    "model_serving_endpoint",
+    "pipeline",
+    "registered_model",
+    "schema",
+    "secret_scope",
+    "sql_warehouse",
+    "volume",
+]
+
+
+def generate(seed):
+    with open(os.environ["FUZZ_SCHEMA"]) as f:
+        schema = json.load(f)
+    return to_yaml(gen_config(schema, seed, os.environ["UNIQUE_NAME"]))
+
+
+def mutate_base(seed):
+    name = MUTATE_BASES[seed % len(MUTATE_BASES)]
+    path = os.path.join(os.environ["TESTDIR"], "..", "configs", name + ".yml.tmpl")
+    unique = os.environ["UNIQUE_NAME"]
+    with open(path) as f:
+        rendered = substitute_variables(f.read())
+    config = load_yaml(rendered)
+    # The schema lets mutate inject valid optional fields, not just perturb existing ones.
+    with open(os.environ["FUZZ_SCHEMA"]) as f:
+        schema = json.load(f)
+    return to_yaml(mutate(config, seed, schema=schema, unique=unique))
+
+
+def main():
+    seed = int(os.environ["FUZZ_SEED"])
+    mode = os.environ.get("FUZZ_MODE", "generate")
+    if mode == "generate":
+        sys.stdout.write(generate(seed))
+    elif mode == "mutate":
+        sys.stdout.write(mutate_base(seed))
+    else:
+        sys.exit(f"emit_fuzz_config: unknown FUZZ_MODE {mode!r}")
+
+
+if __name__ == "__main__":
+    main()