test: add CNCF format tests for artifact validation and media type checks

Author: ilopezluna

pkg/distribution/builder/builder_test.go

@@ -2,6 +2,7 @@ package builder_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -11,6 +12,8 @@ import (
 
 	"github.com/docker/model-runner/pkg/distribution/builder"
 	"github.com/docker/model-runner/pkg/distribution/internal/testutil"
+	"github.com/docker/model-runner/pkg/distribution/modelpack"
+	"github.com/docker/model-runner/pkg/distribution/oci"
 	"github.com/docker/model-runner/pkg/distribution/types"
 )
 
@@ -422,6 +425,336 @@ func TestFromModelErrorHandling(t *testing.T) {
 	}
 }
 
+// TestFromPathCNCFFormat verifies that FromPath with WithFormat(BuildFormatCNCF) produces
+// a valid CNCF ModelPack artifact with correct media types, artifact type, and config.
+func TestFromPathCNCFFormat(t *testing.T) {
+	ggufPath := filepath.Join("..", "assets", "dummy.gguf")
+	fixedTime := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
+
+	b, err := builder.FromPath(ggufPath,
+		builder.WithFormat(builder.BuildFormatCNCF),
+		builder.WithCreated(fixedTime),
+	)
+	if err != nil {
+		t.Fatalf("FromPath with CNCF format failed: %v", err)
+	}
+
+	target := &fakeTarget{}
+	if err := b.Build(t.Context(), target, nil); err != nil {
+		t.Fatalf("Build failed: %v", err)
+	}
+
+	// 1. Verify manifest has CNCF artifact type.
+	manifest, err := target.artifact.Manifest()
+	if err != nil {
+		t.Fatalf("Failed to get manifest: %v", err)
+	}
+	if manifest.ArtifactType != modelpack.ArtifactTypeModelManifest {
+		t.Errorf("Expected artifactType %q, got %q",
+			modelpack.ArtifactTypeModelManifest, manifest.ArtifactType)
+	}
+
+	// 2. Verify config media type is CNCF model config.
+	if manifest.Config.MediaType != oci.MediaType(modelpack.MediaTypeModelConfigV1) {
+		t.Errorf("Expected config media type %q, got %q",
+			modelpack.MediaTypeModelConfigV1, manifest.Config.MediaType)
+	}
+
+	// 3. Verify all layers have CNCF media types (not Docker media types).
+	for i, layer := range manifest.Layers {
+		mt := string(layer.MediaType)
+		if !strings.HasPrefix(mt, modelpack.MediaTypePrefix) {
+			t.Errorf("Layer %d has non-CNCF media type %q (expected prefix %q)",
+				i, mt, modelpack.MediaTypePrefix)
+		}
+	}
+
+	// 4. Verify the weight layer specifically uses the CNCF weight media type.
+	if len(manifest.Layers) == 0 {
+		t.Fatal("Expected at least one layer")
+	}
+	weightMT := manifest.Layers[0].MediaType
+	if weightMT != oci.MediaType(modelpack.MediaTypeWeightRaw) {
+		t.Errorf("Expected weight layer media type %q, got %q",
+			modelpack.MediaTypeWeightRaw, weightMT)
+	}
+
+	// 5. Verify the raw config is valid ModelPack JSON with correct fields.
+	rawCfg, err := target.artifact.RawConfigFile()
+	if err != nil {
+		t.Fatalf("Failed to get raw config: %v", err)
+	}
+	var mp modelpack.Model
+	if err := json.Unmarshal(rawCfg, &mp); err != nil {
+		t.Fatalf("Failed to unmarshal CNCF config: %v", err)
+	}
+	if mp.Config.Format != "gguf" {
+		t.Errorf("Expected config.format %q, got %q", "gguf", mp.Config.Format)
+	}
+	if mp.ModelFS.Type != "layers" {
+		t.Errorf("Expected modelfs.type %q, got %q", "layers", mp.ModelFS.Type)
+	}
+	if len(mp.ModelFS.DiffIDs) == 0 {
+		t.Error("Expected at least one diffId in modelfs")
+	}
+	if mp.Descriptor.CreatedAt == nil {
+		t.Error("Expected descriptor.createdAt to be set")
+	} else if !mp.Descriptor.CreatedAt.Equal(fixedTime) {
+		t.Errorf("Expected descriptor.createdAt %v, got %v", fixedTime, *mp.Descriptor.CreatedAt)
+	}
+
+	// 6. Verify the JSON tags are camelCase (spec-compliant).
+	var rawMap map[string]json.RawMessage
+	if err := json.Unmarshal(rawCfg, &rawMap); err != nil {
+		t.Fatalf("Failed to unmarshal config to map: %v", err)
+	}
+	// Must have "modelfs" (not "model_fs").
+	if _, ok := rawMap["modelfs"]; !ok {
+		t.Error("Config JSON missing 'modelfs' key")
+	}
+	// Verify modelfs contains "diffIds" (camelCase, not "diff_ids").
+	if modelfsRaw, ok := rawMap["modelfs"]; ok {
+		var modelfsMap map[string]json.RawMessage
+		if err := json.Unmarshal(modelfsRaw, &modelfsMap); err != nil {
+			t.Fatalf("Failed to unmarshal modelfs: %v", err)
+		}
+		if _, ok := modelfsMap["diffIds"]; !ok {
+			t.Error("modelfs JSON missing 'diffIds' key (expected camelCase)")
+		}
+		if _, ok := modelfsMap["diff_ids"]; ok {
+			t.Error("modelfs JSON has 'diff_ids' (snake_case) — should be 'diffIds' (camelCase)")
+		}
+	}
+	// Verify config contains "paramSize" (not "param_size").
+	if configRaw, ok := rawMap["config"]; ok {
+		var configMap map[string]json.RawMessage
+		if err := json.Unmarshal(configRaw, &configMap); err != nil {
+			t.Fatalf("Failed to unmarshal config section: %v", err)
+		}
+		if _, ok := configMap["param_size"]; ok {
+			t.Error("config JSON has 'param_size' (snake_case) — should be 'paramSize' (camelCase)")
+		}
+	}
+}
+
+// TestFromPathCNCFWithAdditionalLayers verifies that additional layers added
+// to a CNCF builder get CNCF media types, not Docker media types.
+func TestFromPathCNCFWithAdditionalLayers(t *testing.T) {
+	ggufPath := filepath.Join("..", "assets", "dummy.gguf")
+
+	b, err := builder.FromPath(ggufPath, builder.WithFormat(builder.BuildFormatCNCF))
+	if err != nil {
+		t.Fatalf("FromPath failed: %v", err)
+	}
+
+	// Add license
+	b, err = b.WithLicense(filepath.Join("..", "assets", "license.txt"))
+	if err != nil {
+		t.Fatalf("Failed to add license: %v", err)
+	}
+
+	// Add multimodal projector
+	b, err = b.WithMultimodalProjector(filepath.Join("..", "assets", "dummy.mmproj"))
+	if err != nil {
+		t.Fatalf("Failed to add multimodal projector: %v", err)
+	}
+
+	// Add chat template
+	b, err = b.WithChatTemplateFile(filepath.Join("..", "assets", "template.jinja"))
+	if err != nil {
+		t.Fatalf("Failed to add chat template: %v", err)
+	}
+
+	target := &fakeTarget{}
+	if err := b.Build(t.Context(), target, nil); err != nil {
+		t.Fatalf("Build failed: %v", err)
+	}
+
+	manifest, err := target.artifact.Manifest()
+	if err != nil {
+		t.Fatalf("Failed to get manifest: %v", err)
+	}
+
+	// Should have 4 layers: weight + license + mmproj + chat template
+	if len(manifest.Layers) != 4 {
+		t.Fatalf("Expected 4 layers, got %d", len(manifest.Layers))
+	}
+
+	// ALL layers must have CNCF media type prefix.
+	for i, layer := range manifest.Layers {
+		mt := string(layer.MediaType)
+		if !strings.HasPrefix(mt, modelpack.MediaTypePrefix) {
+			t.Errorf("Layer %d has non-CNCF media type %q", i, mt)
+		}
+	}
+
+	// No Docker media types should appear.
+	dockerMTs := []oci.MediaType{
+		types.MediaTypeGGUF,
+		types.MediaTypeLicense,
+		types.MediaTypeMultimodalProjector,
+		types.MediaTypeChatTemplate,
+	}
+	for _, layer := range manifest.Layers {
+		for _, dmt := range dockerMTs {
+			if layer.MediaType == dmt {
+				t.Errorf("Found Docker media type %q in CNCF artifact", dmt)
+			}
+		}
+	}
+}
+
+// TestFromPathCNCFContextSizeError verifies that WithContextSize returns an error
+// when the output format is CNCF (context size is not in the CNCF spec).
+func TestFromPathCNCFContextSizeError(t *testing.T) {
+	ggufPath := filepath.Join("..", "assets", "dummy.gguf")
+
+	b, err := builder.FromPath(ggufPath, builder.WithFormat(builder.BuildFormatCNCF))
+	if err != nil {
+		t.Fatalf("FromPath failed: %v", err)
+	}
+
+	_, err = b.WithContextSize(4096)
+	if err == nil {
+		t.Fatal("Expected error when setting context size with CNCF format, got nil")
+	}
+	if !strings.Contains(err.Error(), "--context-size is not supported") {
+		t.Errorf("Expected error about context-size not supported, got: %v", err)
+	}
+}
+
+// TestFromModelToCNCF verifies that FromModel with WithFormat(BuildFormatCNCF) correctly
+// converts a Docker-format model to CNCF ModelPack format.
+func TestFromModelToCNCF(t *testing.T) {
+	// Step 1: Create a Docker-format model with a license layer.
+	dockerBuilder, err := builder.FromPath(filepath.Join("..", "assets", "dummy.gguf"))
+	if err != nil {
+		t.Fatalf("FromPath failed: %v", err)
+	}
+	dockerBuilder, err = dockerBuilder.WithLicense(filepath.Join("..", "assets", "license.txt"))
+	if err != nil {
+		t.Fatalf("WithLicense failed: %v", err)
+	}
+
+	dockerTarget := &fakeTarget{}
+	if err := dockerBuilder.Build(t.Context(), dockerTarget, nil); err != nil {
+		t.Fatalf("Build Docker model failed: %v", err)
+	}
+
+	// Verify the Docker model has Docker media types.
+	dockerManifest, err := dockerTarget.artifact.Manifest()
+	if err != nil {
+		t.Fatalf("Failed to get Docker manifest: %v", err)
+	}
+	for _, layer := range dockerManifest.Layers {
+		if strings.HasPrefix(string(layer.MediaType), modelpack.MediaTypePrefix) {
+			t.Fatalf("Docker model should not have CNCF media types, found %q", layer.MediaType)
+		}
+	}
+
+	// Step 2: Convert Docker model to CNCF format.
+	cncfBuilder, err := builder.FromModel(dockerTarget.artifact, builder.WithFormat(builder.BuildFormatCNCF))
+	if err != nil {
+		t.Fatalf("FromModel with CNCF format failed: %v", err)
+	}
+
+	cncfTarget := &fakeTarget{}
+	if err := cncfBuilder.Build(t.Context(), cncfTarget, nil); err != nil {
+		t.Fatalf("Build CNCF model failed: %v", err)
+	}
+
+	// Step 3: Verify the CNCF model.
+	cncfManifest, err := cncfTarget.artifact.Manifest()
+	if err != nil {
+		t.Fatalf("Failed to get CNCF manifest: %v", err)
+	}
+
+	// Artifact type must be set.
+	if cncfManifest.ArtifactType != modelpack.ArtifactTypeModelManifest {
+		t.Errorf("Expected artifactType %q, got %q",
+			modelpack.ArtifactTypeModelManifest, cncfManifest.ArtifactType)
+	}
+
+	// Config media type must be CNCF.
+	if cncfManifest.Config.MediaType != oci.MediaType(modelpack.MediaTypeModelConfigV1) {
+		t.Errorf("Expected config media type %q, got %q",
+			modelpack.MediaTypeModelConfigV1, cncfManifest.Config.MediaType)
+	}
+
+	// Same number of layers must be preserved.
+	if len(cncfManifest.Layers) != len(dockerManifest.Layers) {
+		t.Fatalf("Expected %d layers, got %d", len(dockerManifest.Layers), len(cncfManifest.Layers))
+	}
+
+	// All layers must have CNCF media types.
+	for i, layer := range cncfManifest.Layers {
+		mt := string(layer.MediaType)
+		if !strings.HasPrefix(mt, modelpack.MediaTypePrefix) {
+			t.Errorf("Layer %d has non-CNCF media type %q after conversion", i, mt)
+		}
+	}
+
+	// Layer digests should be preserved (same content, different media type).
+	for i := range dockerManifest.Layers {
+		if dockerManifest.Layers[i].Digest != cncfManifest.Layers[i].Digest {
+			t.Errorf("Layer %d digest changed after conversion: %v → %v",
+				i, dockerManifest.Layers[i].Digest, cncfManifest.Layers[i].Digest)
+		}
+	}
+
+	// Config should have the model architecture and format.
+	cfg, err := cncfTarget.artifact.Config()
+	if err != nil {
+		t.Fatalf("Failed to get config: %v", err)
+	}
+	if cfg.GetFormat() != types.FormatGGUF {
+		t.Errorf("Expected format %q, got %q", types.FormatGGUF, cfg.GetFormat())
+	}
+}
+
+// TestFromPathCNCFDeterministicDigest verifies that CNCF format builds
+// with the same inputs produce the same digests.
+func TestFromPathCNCFDeterministicDigest(t *testing.T) {
+	ggufPath := filepath.Join("..", "assets", "dummy.gguf")
+	fixedTime := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
+
+	b1, err := builder.FromPath(ggufPath,
+		builder.WithFormat(builder.BuildFormatCNCF),
+		builder.WithCreated(fixedTime),
+	)
+	if err != nil {
+		t.Fatalf("FromPath (first) failed: %v", err)
+	}
+	b2, err := builder.FromPath(ggufPath,
+		builder.WithFormat(builder.BuildFormatCNCF),
+		builder.WithCreated(fixedTime),
+	)
+	if err != nil {
+		t.Fatalf("FromPath (second) failed: %v", err)
+	}
+
+	target1 := &fakeTarget{}
+	target2 := &fakeTarget{}
+	if err := b1.Build(t.Context(), target1, nil); err != nil {
+		t.Fatalf("Build (first) failed: %v", err)
+	}
+	if err := b2.Build(t.Context(), target2, nil); err != nil {
+		t.Fatalf("Build (second) failed: %v", err)
+	}
+
+	digest1, err := target1.artifact.Digest()
+	if err != nil {
+		t.Fatalf("Digest (first) failed: %v", err)
+	}
+	digest2, err := target2.artifact.Digest()
+	if err != nil {
+		t.Fatalf("Digest (second) failed: %v", err)
+	}
+	if digest1 != digest2 {
+		t.Errorf("Expected identical digests for CNCF format with same inputs, got %v and %v", digest1, digest2)
+	}
+}
+
 var _ builder.Target = &fakeTarget{}
 
 type fakeTarget struct {