feat(coverage): add parity runtime and publish tasks, update README and Taskfile

Author: aledbf

.github/workflows/go-cli.yml

@@ -250,8 +250,8 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - run: task reference
-      - run: task parity:runtime
-      - run: task parity:publish
+      - run: task coverage:parity-runtime
+      - run: task coverage:parity-publish
       - if: always()
         run: |
           mkdir -p artifacts

README.md

@@ -146,6 +146,8 @@ single percentage:
 task coverage                 # hermetic unit profile + ratchet gate
 task coverage:e2e             # real Docker E2E profile (requires Docker)
 task coverage:parity-contract # instrumented CLI subprocess vs the TS oracle
+task coverage:parity-runtime  # full Docker parity, collected nightly in CI
+task coverage:parity-publish  # real publication to an ephemeral OCI registry
 ```
 
 The subprocess profile uses Go's application coverage (`go build -cover` plus

Taskfile.yml

@@ -11,6 +11,13 @@ vars:
   # Minimum hermetic-unit statement coverage the `coverage` gate enforces. Set as a
   # ratchet just below the current level; raise it as coverage grows (never lower).
   COVERAGE_MIN: "48"
+  # Package ratchets prevent well-covered parsing packages from masking a
+  # regression in the orchestration layers. Keep these just below the measured
+  # hermetic baseline and only move them upward.
+  COVERAGE_CLI_MIN: "28"
+  COVERAGE_DOCKER_MIN: "77"
+  COVERAGE_LIFECYCLE_MIN: "73"
+  COVERAGE_OCI_MIN: "85"
 
 tasks:
   default:
@@ -81,6 +88,27 @@ tasks:
             if (t+0 < m+0) { printf "FAIL: coverage %.1f%% is below the %s%% floor\n", t, m; exit 1 }
             printf "OK: coverage %.1f%% >= %s%% floor\n", t, m
           }'
+      # Weighted statement coverage per package. This catches the failure mode
+      # where gains in parsers hide lost coverage in CLI/OCI orchestration.
+      - cmd: |
+          check_pkg() {
+            pkg="$1"; min="$2"
+            result=$(awk -v p="github.com/devcontainers/cli/internal/${pkg}/" '
+              NR > 1 && index($1,p)==1 { total += $2; if ($3 > 0) covered += $2 }
+              END { if (total == 0) print "missing"; else printf "%.1f", covered*100/total }
+            ' coverage.out)
+            if [ "$result" = missing ]; then
+              echo "FAIL: no coverage data for internal/$pkg"; return 1
+            fi
+            awk -v p="$pkg" -v t="$result" -v m="$min" 'BEGIN {
+              if (t+0 < m+0) { printf "FAIL: internal/%s coverage %.1f%% is below %s%%\n", p,t,m; exit 1 }
+              printf "OK: internal/%s coverage %.1f%% >= %s%%\n", p,t,m
+            }'
+          }
+          check_pkg cli {{.COVERAGE_CLI_MIN}}
+          check_pkg docker {{.COVERAGE_DOCKER_MIN}}
+          check_pkg lifecycle {{.COVERAGE_LIFECYCLE_MIN}}
+          check_pkg oci {{.COVERAGE_OCI_MIN}}
 
   coverage:e2e:
     desc: Measure the real in-process E2E paths (requires Docker)
@@ -108,6 +136,37 @@ tasks:
       - go tool covdata textfmt -i=artifacts/coverage/data/contract -o=artifacts/coverage/parity-contract.out
       - go tool cover -func=artifacts/coverage/parity-contract.out
 
+  coverage:parity-runtime:
+    desc: Measure the instrumented Go binary through the full Docker parity lane
+    deps: [build:coverage, reference:compile]
+    preconditions:
+      - sh: docker info >/dev/null 2>&1
+        msg: Docker is required for runtime parity coverage
+      - sh: |
+          d=$(mktemp -d) && printf 'FROM busybox\n' > "$d/Dockerfile" && \
+          docker buildx build --cache-to=type=local,dest="$d/c" -t parity-coverage-builder-probe "$d" >/dev/null 2>&1; \
+          rc=$?; rm -rf "$d" >/dev/null 2>&1; exit $rc
+        msg: The Docker builder must support cache export for runtime parity coverage
+    cmds:
+      - rm -rf artifacts/coverage/data/runtime
+      - mkdir -p artifacts/coverage/data/runtime
+      - GOCOVERDIR={{.ROOT_DIR}}/artifacts/coverage/data/runtime CLI_GO={{.ROOT_DIR}}/artifacts/coverage/bin/{{.BINARY}} PARITY_REPORT_FILE={{.ROOT_DIR}}/artifacts/parity-runtime.json PARITY_STRICT=true PARITY_LANE=all go test ./internal/cli -run '^TestParityMatrix$' -count=1 -parallel {{.PARITY_PARALLEL | default "2"}} -timeout 60m -v {{.CLI_ARGS}}
+      - go tool covdata textfmt -i=artifacts/coverage/data/runtime -o=artifacts/coverage/parity-runtime.out
+      - go tool cover -func=artifacts/coverage/parity-runtime.out
+
+  coverage:parity-publish:
+    desc: Measure real feature/template publication against an ephemeral OCI registry
+    deps: [build:coverage, reference:compile]
+    preconditions:
+      - sh: docker info >/dev/null 2>&1
+        msg: Docker is required for publish parity coverage
+    cmds:
+      - rm -rf artifacts/coverage/data/publish
+      - mkdir -p artifacts/coverage/data/publish
+      - GOCOVERDIR={{.ROOT_DIR}}/artifacts/coverage/data/publish CLI_GO={{.ROOT_DIR}}/artifacts/coverage/bin/{{.BINARY}} go test ./internal/cli -run '^TestPublishParity$' -count=1 -timeout 10m -v {{.CLI_ARGS}}
+      - go tool covdata textfmt -i=artifacts/coverage/data/publish -o=artifacts/coverage/parity-publish.out
+      - go tool cover -func=artifacts/coverage/parity-publish.out
+
   test:e2e:
     desc: Run Go E2E tests with Docker
     deps: [build]

internal/cli/publish_parity_test.go

@@ -26,7 +26,7 @@ func TestPublishParity(t *testing.T) {
 	if _, err := os.Stat(filepath.Join(repoRoot, "reference", "dist", "spec-node", "devContainersSpecCLI.js")); err != nil {
 		t.Skip("TS reference not compiled")
 	}
-	goCLI := filepath.Join(repoRoot, "devcontainer")
+	goCLI := envOr("CLI_GO", filepath.Join(repoRoot, "devcontainer"))
 	if _, err := os.Stat(goCLI); err != nil {
 		t.Skip("Go CLI not built")
 	}

internal/docker/runargs_test.go

@@ -152,3 +152,27 @@ func TestCreateContainerArgs_MountConsistency(t *testing.T) {
 		t.Errorf("args = %q", joined)
 	}
 }
+
+// FuzzParseMountSpec targets user-controlled --mount strings. Beyond not
+// panicking, every accepted result must satisfy the invariants relied on by
+// container creation: a destination and an explicit/defaulted mount type.
+func FuzzParseMountSpec(f *testing.F) {
+	f.Add("type=bind,source=/workspace,target=/workspaces/project,readonly")
+	f.Add("type=volume,source=data,destination=/data")
+	f.Add("src=/tmp,dst=/tmp/container,ro=true")
+	f.Add("target=/default-bind")
+	f.Add("type=volume,source=missing-target")
+
+	f.Fuzz(func(t *testing.T, spec string) {
+		m, err := ParseMountSpec(spec)
+		if err != nil {
+			return
+		}
+		if m.Target == "" {
+			t.Fatal("accepted mount has an empty target")
+		}
+		if m.Type == "" {
+			t.Fatal("accepted mount has an empty type")
+		}
+	})
+}

internal/exec/exec.go

@@ -39,6 +39,12 @@ func (r OSRunner) Run(ctx context.Context, name string, args ...string) ([]byte,
 
 	err := cmd.Run()
 	if err != nil {
+		// CommandContext normally reports the signal used to kill the child as an
+		// *exec.ExitError. Preserve the more useful cancellation/deadline cause at
+		// this process boundary so callers can reliably use errors.Is.
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return stdout.Bytes(), stderr.Bytes(), -1, ctxErr
+		}
 		if exitErr, ok := err.(*osexec.ExitError); ok {
 			// Ran and exited non-zero: report the code, not an error.
 			return stdout.Bytes(), stderr.Bytes(), exitErr.ExitCode(), nil

internal/exec/exec_test.go

@@ -0,0 +1,105 @@
+package exec
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestHelperProcess is a real child-process endpoint used by OSRunner tests.
+// It deliberately exercises the os/exec boundary (argv, environment, both
+// streams, exit status and cancellation) instead of mocking CommandContext.
+func TestHelperProcess(t *testing.T) {
+	if os.Getenv("GO_WANT_EXEC_HELPER") != "1" {
+		return
+	}
+
+	args := helperArgs(os.Args)
+	if len(args) == 0 {
+		os.Exit(2)
+	}
+	switch args[0] {
+	case "echo":
+		_, _ = os.Stdout.WriteString(strings.Join(args[1:], "|"))
+		_, _ = os.Stderr.WriteString(os.Getenv("EXEC_HELPER_STDERR"))
+	case "exit":
+		os.Exit(17)
+	case "wait":
+		time.Sleep(time.Hour)
+	default:
+		os.Exit(2)
+	}
+	os.Exit(0)
+}
+
+func helperArgs(args []string) []string {
+	for i, arg := range args {
+		if arg == "--" {
+			return args[i+1:]
+		}
+	}
+	return nil
+}
+
+func helperRunner(env ...string) OSRunner {
+	return OSRunner{Env: append([]string{
+		"GO_WANT_EXEC_HELPER=1",
+	}, env...)}
+}
+
+func TestOSRunnerProcessContract(t *testing.T) {
+	stdout, stderr, code, err := helperRunner("EXEC_HELPER_STDERR=warning").Run(
+		t.Context(), os.Args[0], "-test.run=^TestHelperProcess$", "--", "echo", "one", "two",
+	)
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if got := string(stdout); got != "one|two" {
+		t.Errorf("stdout = %q, want one|two", got)
+	}
+	if got := string(stderr); got != "warning" {
+		t.Errorf("stderr = %q, want warning", got)
+	}
+	if code != 0 {
+		t.Errorf("code = %d, want 0", code)
+	}
+}
+
+func TestOSRunnerNonZeroIsExitCodeNotTransportError(t *testing.T) {
+	_, _, code, err := helperRunner().Run(
+		t.Context(), os.Args[0], "-test.run=^TestHelperProcess$", "--", "exit",
+	)
+	if err != nil {
+		t.Fatalf("non-zero child returned transport error: %v", err)
+	}
+	if code != 17 {
+		t.Fatalf("code = %d, want 17", code)
+	}
+}
+
+func TestOSRunnerHonorsCancellation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, _, code, err := helperRunner().Run(
+		ctx, os.Args[0], "-test.run=^TestHelperProcess$", "--", "wait",
+	)
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("error = %v, want context deadline exceeded", err)
+	}
+	if code != -1 {
+		t.Fatalf("code = %d, want -1 for a process killed by its context", code)
+	}
+}
+
+func TestOSRunnerMissingBinaryIsTransportError(t *testing.T) {
+	_, _, code, err := (OSRunner{}).Run(t.Context(), "/definitely/not/a/devcontainer-test-binary")
+	if err == nil {
+		t.Fatal("missing binary returned nil error")
+	}
+	if code != -1 {
+		t.Fatalf("code = %d, want -1", code)
+	}
+}

internal/jsonc/jsonc_test.go

@@ -1,8 +1,10 @@
 package jsonc
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 )
 
@@ -132,3 +134,34 @@ func TestUnmarshal_AllFixtures(t *testing.T) {
 	}
 	t.Logf("parsed %d fixtures successfully", count)
 }
+
+// FuzzStandardizePreservesMeaning checks the contract that accepted JSONC is
+// converted into valid JSON without changing the decoded value. The seed corpus
+// includes the syntax extensions and BOM behavior accepted by devcontainer.json.
+func FuzzStandardizePreservesMeaning(f *testing.F) {
+	f.Add([]byte(`{"image":"ubuntu"}`))
+	f.Add([]byte("\xef\xbb\xbf{\"image\":\"ubuntu\",}"))
+	f.Add([]byte("{/* comment */\"features\":{},}"))
+	f.Add([]byte(`{"nested":[1,true,null,{"x":"// not a comment"}]}`))
+
+	f.Fuzz(func(t *testing.T, input []byte) {
+		standard, err := StripComments(input)
+		if err != nil {
+			return
+		}
+		if !json.Valid(standard) {
+			t.Fatalf("StripComments succeeded with invalid JSON: %q", standard)
+		}
+
+		var fromJSONC, fromStandard any
+		if err := Unmarshal(input, &fromJSONC); err != nil {
+			t.Fatalf("StripComments accepted input that Unmarshal rejected: %v", err)
+		}
+		if err := json.Unmarshal(standard, &fromStandard); err != nil {
+			t.Fatalf("standard output cannot be decoded: %v", err)
+		}
+		if !reflect.DeepEqual(fromJSONC, fromStandard) {
+			t.Fatalf("standardization changed value: JSONC=%#v JSON=%#v", fromJSONC, fromStandard)
+		}
+	})
+}