diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 33e984f4..a08d7dd1 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -16,6 +16,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | +| Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (see [Tests and Test Groups](tests.md)) | ### Bare Components @@ -300,6 +301,24 @@ rpm-channel = "rpm-devel" rpm-channel = "none" ``` +## Component Tests + +The `[components..tests]` subtable lists test or test-group +references that apply to the component. Each entry is a [TestRef](tests.md#test-reference) +with exactly one of `name` or `group`. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries | + +```toml +[components.kernel.tests] +tests = [ + { group = "kernel-bvt" }, + { name = "kdump-smoke" }, +] +``` + ## Source File References The `[[components..source-files]]` array defines additional source files that azldev should download before building. These are files not available in the dist-git repository or lookaside cache — typically binaries, pre-built artifacts, or files from custom hosting. @@ -461,5 +480,6 @@ lines = ["cp -vf %{shimdirx64}/$(basename %{shimefix64}) %{shimefix64} ||:"] - [Distros](distros.md) — distro definitions and `default-component-config` inheritance - [Component Groups](component-groups.md) — grouping components with shared defaults - [Package Groups](package-groups.md) — project-level package groups and full resolution order +- [Tests and Test Groups](tests.md) — definitions referenced by `[components..tests]` - [Configuration System](../../explanation/config-system.md) — inheritance and merge behavior - [JSON Schema](../../../../schemas/azldev.schema.json) — machine-readable schema diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index e1ae30bb..19abd972 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -15,6 +15,8 @@ All config files share the same schema — there is no distinction between a "ro | `component-groups` | map of objects | Named groups of components with shared defaults | [Component Groups](component-groups.md) | | `images` | map of objects | Image definitions (VMs, containers) | [Images](images.md) | | `test-suites` | map of objects | Named test suite definitions referenced by images | [Test Suites](test-suites.md) | +| `tests` | map of objects | Named test definitions (new-shape, parse-only) | [Tests and Test Groups](tests.md) | +| `test-groups` | map of objects | Named bundles of test references (new-shape, parse-only) | [Tests and Test Groups](tests.md) | | `tools` | object | Configuration for external tools used by azldev | [Tools](tools.md) | | `default-package-config` | object | Project-wide default applied to all binary packages | [Package Groups — Resolution Order](package-groups.md#resolution-order) | | `package-groups` | map of objects | Named groups of binary packages with shared config | [Package Groups](package-groups.md) | diff --git a/docs/user/reference/config/images.md b/docs/user/reference/config/images.md index ebbd5e53..34dd9cb9 100644 --- a/docs/user/reference/config/images.md +++ b/docs/user/reference/config/images.md @@ -35,11 +35,12 @@ The `capabilities` subtable describes what the image supports. All fields are op ## Image Tests -The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section. +The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section, and/or to entries from the new-shape [`[tests]` / `[test-groups]`](tests.md) sections. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Test Suites | `test-suites` | array of inline tables | No | List of test suite references. Each entry must have a `name` field matching a key in `[test-suites]`. | +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries (parse-only; see [Tests and Test Groups](tests.md)). | ## Image Publish @@ -118,4 +119,5 @@ channels = ["registry-prod", "registry-staging"] - [Config File Structure](config-file.md) — top-level config file layout - [Test Suites](test-suites.md) — test suite definitions +- [Tests and Test Groups](tests.md) — new-shape test/group definitions referenced by `[images..tests]` - [Tools](tools.md) — Image Customizer tool configuration diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md new file mode 100644 index 00000000..a6442412 --- /dev/null +++ b/docs/user/reference/config/tests.md @@ -0,0 +1,89 @@ +# Tests and Test Groups + +The `[tests]` and `[test-groups]` sections declare framework-agnostic test +metadata that components and images can target by name. Each test entry +binds a single test (a pytest run, a LISA case, or a TMT plan) +to a named identifier; each group entry bundles tests (and +named references) under one name so callers can reference a curated set +without enumerating every member. + +## Test Definition + +Each entry under `[tests.]` describes one configuration of one +runner. Framework-specific options live in a typed subtable +(`pytest`, `lisa`, `tmt`) whose contents are passed through +to the runner; their internal schemas are intentionally not validated +by azldev so frameworks can evolve independently. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Type | `type` | string | Yes | Test framework: `pytest`, `lisa`, or `tmt` | +| Description | `description` | string | No | Human-readable description | +| Kind | `kind` | string | No | Test kind hint: `functional` or `performance` | +| Long running | `long-running` | boolean | No | Hints that this test may run for hours | +| Required capabilities | `required-capabilities` | string array | No | Capability tokens the image must declare for this test to be applicable | +| Lisa | `lisa` | table | No | LISA-specific configuration (opaque to azldev) | +| Tmt | `tmt` | table | No | TMT-specific configuration (opaque to azldev) | +| Pytest | `pytest` | table | No | pytest-specific configuration (opaque to azldev) | + +## Test Group + +Each entry under `[test-groups.]` names an ordered list of test +references that callers can target as a single unit. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Description | `description` | string | No | Human-readable description | +| Tests | `tests` | array of [TestRef](#test-reference) | No | Ordered members of the group (name refs only) | + +## Test Reference + +`TestRef` is an inline table with exactly one of `name` or `group`: + +| Field | TOML Key | Type | Description | +|-------|----------|------|-------------| +| Name | `name` | string | References a `[tests.]` entry | +| Group | `group` | string | References a `[test-groups.]` entry | + +## Referencing from Components and Images + +Components and images both expose a `tests` subtable that holds a list +of `TestRef`s: + +```toml +[components.kernel.tests] +tests = [{ group = "kernel-bvt" }, { name = "kdump-smoke" }] + +[images.vm-base.tests] +tests = [{ group = "bvt" }] +``` + +## Example + +```toml +[tests.bvt-ssh] +type = "pytest" +description = "Basic SSH boot verification" +kind = "functional" +required-capabilities = ["ssh"] +pytest = { working-dir = "tests/bvt", test-paths = ["test_ssh.py"] } + +[tests.kdump-smoke] +type = "lisa" +description = "Smoke test for kdump" +lisa = { case = "kdump.smoke" } + +[test-groups.bvt] +description = "Build verification tests" +tests = [ + { name = "bvt-ssh" }, + { name = "kdump-smoke" }, +] +``` + +## Related Resources + +- [Test Suites](test-suites.md) - legacy test suite definitions +- [Components](components.md#component-tests) — per-component `tests` field +- [Images](images.md#image-tests) — per-image `tests` field +- [Config File Structure](config-file.md) — top-level config layout diff --git a/go.mod b/go.mod index b3eaa94d..a3c97d6b 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/muesli/termenv v0.16.0 github.com/nxadm/tail v1.4.11 github.com/opencontainers/selinux v1.15.1 + github.com/pb33f/ordered-map/v2 v2.3.1 github.com/pelletier/go-toml/v2 v2.4.2 github.com/pmezard/go-difflib v1.0.0 github.com/samber/lo v1.53.0 @@ -131,7 +132,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/internal/app/azldev/cmds/image/list.go b/internal/app/azldev/cmds/image/list.go index d0d6bc78..0c42af0f 100644 --- a/internal/app/azldev/cmds/image/list.go +++ b/internal/app/azldev/cmds/image/list.go @@ -38,7 +38,7 @@ type ImageListResult struct { // Tests holds the test configuration for this image, matching the original config // structure. - Tests projectconfig.ImageTestsConfig `json:"tests" table:"-"` + Tests *projectconfig.ImageTestsConfig `json:"tests,omitempty" table:"-"` // TestsSummary is a comma-separated summary of test suite names for table display. TestsSummary string `json:"-" table:"Tests"` diff --git a/internal/app/azldev/cmds/image/list_test.go b/internal/app/azldev/cmds/image/list_test.go index 6c572ac3..6289c08f 100644 --- a/internal/app/azldev/cmds/image/list_test.go +++ b/internal/app/azldev/cmds/image/list_test.go @@ -89,7 +89,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { MachineBootable: lo.ToPtr(true), Systemd: lo.ToPtr(true), }, - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, {Name: "integration"}, @@ -105,7 +105,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Capabilities: projectconfig.ImageCapabilities{ Container: lo.ToPtr(true), }, - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, }, @@ -131,9 +131,10 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, lo.ToPtr(true), results[0].Capabilities.Container) assert.Nil(t, results[0].Capabilities.MachineBootable) assert.Equal(t, "container", results[0].CapabilitiesSummary) + require.NotNil(t, results[0].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}, - }, results[0].Tests) + }, *results[0].Tests) assert.Equal(t, "smoke", results[0].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ Channels: []string{"registry-prod"}, @@ -144,7 +145,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Nil(t, results[1].Capabilities.MachineBootable) assert.Nil(t, results[1].Capabilities.Container) assert.Empty(t, results[1].CapabilitiesSummary) - assert.Empty(t, results[1].Tests.TestSuites) + assert.Nil(t, results[1].Tests) assert.Empty(t, results[1].TestsSummary) assert.Empty(t, results[1].Publish.Channels) assert.Empty(t, results[1].PublishSummary) @@ -154,9 +155,10 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, lo.ToPtr(true), results[2].Capabilities.Systemd) assert.Nil(t, results[2].Capabilities.Container) assert.Equal(t, "machine-bootable, systemd", results[2].CapabilitiesSummary) + require.NotNil(t, results[2].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, - }, results[2].Tests) + }, *results[2].Tests) assert.Equal(t, "smoke, integration", results[2].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ Channels: []string{"registry-prod", "registry-staging"}, diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index dce2f264..d9dcd925 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -305,6 +305,14 @@ type ComponentConfig struct { // all packages produced by this component. Overridden by package-group and per-package settings // for binary and debuginfo channels. Publish ComponentPublishConfig `toml:"publish,omitempty" json:"publish,omitempty" table:"-" jsonschema:"title=Publish settings,description=Component-level publish channel settings" fingerprint:"-"` + + // Tests holds the new-shape per-component tests block: + // + // tests.tests = [{ name = "..." }, { group = "..." }] + // + // References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests *ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Per-component test or test-group references" fingerprint:"-"` } // AllowedSourceFilesHashTypes defines the set of hash types that are supported @@ -408,6 +416,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi // here so inherited patterns can be interpreted relative to the concrete component // config file. OverlayFiles: slices.Clone(c.OverlayFiles), + Tests: deep.MustCopy(c.Tests), } // Fix up paths. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e16b79b7..cb102e5d 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -66,6 +66,12 @@ type ConfigFile struct { // Definitions of test suites. TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Test Suites,description=Definitions of test suites for this project"` + // Definitions of individual tests (new schema, [tests.X]). + Tests map[string]TestDefinition `toml:"tests,omitempty" validate:"dive" jsonschema:"title=Tests,description=Definitions of individual tests"` + + // Definitions of test groups (new schema, [test-groups.X]). + TestGroups map[string]TestGroup `toml:"test-groups,omitempty" validate:"dive" jsonschema:"title=Test Groups,description=Definitions of named bundles of tests"` + // Internal fields used to track the origin of the config file; `dir` is the directory // that the config file's relative paths are based from. sourcePath string `toml:"-"` @@ -131,8 +137,23 @@ func (f ConfigFile) Validate() error { } } - // Validate test suite configurations. - for suiteName, suite := range f.TestSuites { + if err := validateTestSuites(f.TestSuites); err != nil { + return err + } + + if err := validateTestDefinitions(f.Tests); err != nil { + return err + } + + if err := validateNewTestReferences(f); err != nil { + return err + } + + return nil +} + +func validateTestSuites(testSuites map[string]TestSuiteConfig) error { + for suiteName, suite := range testSuites { // Suite names are used as path components (e.g., for the per-suite venv directory), // so reject anything that could escape the intended directory or otherwise be unsafe // across platforms. @@ -150,6 +171,186 @@ func (f ConfigFile) Validate() error { return nil } +func validateTestDefinitions(tests map[string]TestDefinition) error { + for testName, testDef := range tests { + if err := testDef.Validate(testName); err != nil { + return fmt.Errorf("invalid test %#q:\n%w", testName, err) + } + } + + return nil +} + +func validateNewTestReferences(cfgFile ConfigFile) error { + for groupName, group := range cfgFile.TestGroups { + scope := fmt.Sprintf("test-group %#q tests", groupName) + if err := validateTestGroupMembers(scope, group.Tests, cfgFile.Tests); err != nil { + return err + } + } + + for componentName, component := range cfgFile.Components { + if component.Tests == nil { + continue + } + + scope := fmt.Sprintf("component %#q tests.tests", componentName) + if err := validateTestRefList(scope, component.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + return err + } + } + + for imageName, image := range cfgFile.Images { + if image.Tests == nil { + continue + } + + scope := fmt.Sprintf("image %#q tests.tests", imageName) + if err := validateTestRefList(scope, image.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + return err + } + } + + return nil +} + +func validateTestGroupMembers( + scope string, + refs []TestRef, + tests map[string]TestDefinition, +) error { + seenRefs := make(map[string]int, len(refs)) + + for idx, ref := range refs { + hasName := ref.Name != "" + hasGroup := ref.Group != "" + + if hasName == hasGroup { + return fmt.Errorf( + "%w: %s[%d] must set exactly one of 'name' or 'group'", + ErrInvalidTestRef, + scope, + idx, + ) + } + + if hasGroup { + return fmt.Errorf( + "%w: %s[%d].group is not allowed in [test-groups]; use .name to reference a [tests] entry", + ErrNestedTestGroupReference, + scope, + idx, + ) + } + + if _, ok := tests[ref.Name]; !ok { + return fmt.Errorf( + "%w: %s[%d].name references undefined test %#q", + ErrUndefinedTest, + scope, + idx, + ref.Name, + ) + } + + refKey := "name:" + ref.Name + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Name, + ) + } + + seenRefs[refKey] = idx + } + + return nil +} + +func validateTestRefList( + scope string, + refs []TestRef, + tests map[string]TestDefinition, + groups map[string]TestGroup, +) error { + seenRefs := make(map[string]int, len(refs)) + + for idx, ref := range refs { + hasName := ref.Name != "" + hasGroup := ref.Group != "" + + if hasName == hasGroup { + return fmt.Errorf( + "%w: %s[%d] must set exactly one of 'name' or 'group'", + ErrInvalidTestRef, + scope, + idx, + ) + } + + if hasName { + if _, ok := tests[ref.Name]; !ok { + return fmt.Errorf( + "%w: %s[%d].name references undefined test %#q", + ErrUndefinedTest, + scope, + idx, + ref.Name, + ) + } + + refKey := "name:" + ref.Name + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Name, + ) + } + + seenRefs[refKey] = idx + + continue + } + + if _, ok := groups[ref.Group]; !ok { + return fmt.Errorf( + "%w: %s[%d].group references undefined test-group %#q", + ErrUndefinedTestGroup, + scope, + idx, + ref.Group, + ) + } + + refKey := "group:" + ref.Group + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Group, + ) + } + + seenRefs[refKey] = idx + } + + return nil +} + // validateSourceFiles checks 'source-files' configuration for a component: // - All filenames must be unique. // - Hash type must be a supported algorithm when specified. diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index d61f2866..f09e787f 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -17,6 +17,281 @@ func TestProjectConfigFileValidation_EmptyFile(t *testing.T) { assert.NoError(t, file.Validate()) } +func TestProjectConfigFileValidation_TestDefinitionMismatchedSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + Lisa: map[string]any{"suite": "vm"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "invalid test") + assert.Contains(t, err.Error(), "smoke") + assert.Contains(t, err.Error(), "lisa") +} + +func TestProjectConfigFileValidation_TestDefinitionMatchingSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_TestDefinitionRequiredSubtablePresentButEmpty(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{}, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_TestDefinitionDisallowedSubtablePresentButEmpty(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + Lisa: map[string]any{}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "smoke") + assert.Contains(t, err.Error(), "lisa") +} + +func TestProjectConfigFileValidation_TestDefinitionInvalidKind(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Kind: projectconfig.TestKind("unknown-kind"), + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUnknownTestKind) + assert.Contains(t, err.Error(), "unknown-kind") +} + +func TestProjectConfigFileValidation_LisaSelectionMissing(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "source": map[string]any{"git-url": "https://example.com/lisa.git", "ref": "main"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidLisaSelection) + assert.Contains(t, err.Error(), "must set at least one LISA selector") +} + +func TestProjectConfigFileValidation_LisaSelectionCriteriaValid(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": map[string]any{"priority": []any{1, 2}, "tags": []any{"vm", "smoke"}}, + }, + }, + "perf": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": []any{ + map[string]any{"area": "network", "category": "performance"}, + map[string]any{"testcaseNames": []any{"case_a", "case_b"}}, + }, + }, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_LisaSelectionUnsupportedCriteriaKey(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": map[string]any{"suite": "smoke"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidLisaSelection) + assert.Contains(t, err.Error(), "unsupported selector") +} + +func TestProjectConfigFileValidation_UndefinedTestReferenceInGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{{Name: "does-not-exist"}}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUndefinedTest) + assert.Contains(t, err.Error(), "does-not-exist") +} + +func TestProjectConfigFileValidation_UndefinedTestGroupReferenceInComponent(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "openssl": { + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Group: "missing-group"}}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUndefinedTestGroup) + assert.Contains(t, err.Error(), "missing-group") +} + +func TestProjectConfigFileValidation_InvalidTestReferenceShapeInImage(t *testing.T) { + file := projectconfig.ConfigFile{ + Images: map[string]projectconfig.ImageConfig{ + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, + }, + }, + }, + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidTestRef) + assert.Contains(t, err.Error(), "exactly one") +} + +func TestProjectConfigFileValidation_DuplicateTestReferenceInGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{ + {Name: "smoke"}, + {Name: "smoke"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) + assert.Contains(t, err.Error(), "duplicates") + assert.Contains(t, err.Error(), "smoke") +} + +func TestProjectConfigFileValidation_DuplicateTestGroupReferenceInImage(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{{Name: "smoke"}}, + }, + }, + Images: map[string]projectconfig.ImageConfig{ + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{ + {Group: "bvt"}, + {Group: "bvt"}, + }, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) + assert.Contains(t, err.Error(), "duplicates") + assert.Contains(t, err.Error(), "bvt") +} + +func TestProjectConfigFileValidation_NestedTestGroupReferenceNotAllowed(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "a": {Tests: []projectconfig.TestRef{{Group: "b"}}}, + "b": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrNestedTestGroupReference) + assert.Contains(t, err.Error(), "is not allowed in [test-groups]") +} + func TestProjectConfigFileValidation_DefaultProjectInfo(t *testing.T) { file := projectconfig.ConfigFile{ Project: &projectconfig.ProjectInfo{}, diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 9fe56870..d8b73315 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -65,6 +65,9 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // ComponentConfig.Publish — post-build routing (where to publish), not a build input. "ComponentConfig.Publish": true, + // ComponentConfig.Tests — test selection metadata (new schema), not a build input. + "ComponentConfig.Tests": true, + // ComponentOverlay.Description — human-readable documentation for the overlay. "ComponentOverlay.Description": true, // ComponentOverlay.Source — absolute path that varies by checkout location. diff --git a/internal/projectconfig/image.go b/internal/projectconfig/image.go index f2000851..b8033dc6 100644 --- a/internal/projectconfig/image.go +++ b/internal/projectconfig/image.go @@ -30,7 +30,7 @@ type ImageConfig struct { // Tests holds the test configuration for this image, including which test suites // apply to it. - Tests ImageTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Test configuration for this image"` + Tests *ImageTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Test configuration for this image"` // Publish holds the publish settings for this image. Publish ImagePublishConfig `toml:"publish,omitempty" json:"publish,omitempty" jsonschema:"title=Publish settings,description=Publishing settings for this image"` @@ -115,6 +115,11 @@ type ImageTestsConfig struct { // reference identifies a test suite defined in the top-level [test-suites] section // and may carry per-test metadata in the future (e.g., required vs optional). TestSuites []TestSuiteRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=List of test suite references that apply to this image"` + + // Tests is the new-shape list of test or test-group references that apply to this + // image. References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this image"` } // TestSuiteRef is a reference to a named test suite. Using a structured type (rather than @@ -126,6 +131,10 @@ type TestSuiteRef struct { // TestNames returns the test suite names referenced by this image. func (i *ImageConfig) TestNames() []string { + if i.Tests == nil { + return nil + } + names := make([]string, len(i.Tests.TestSuites)) for idx, ref := range i.Tests.TestSuites { names[idx] = ref.Name diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index f825d55b..afdc3cca 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -985,6 +985,7 @@ test-suites = [{ name = "smoke" }] require.NoError(t, err) if assert.Contains(t, config.Images, "myimage") { + require.NotNil(t, config.Images["myimage"].Tests) assert.Equal(t, []TestSuiteRef{{Name: "smoke"}}, config.Images["myimage"].Tests.TestSuites) } } diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go new file mode 100644 index 00000000..7e06b0d5 --- /dev/null +++ b/internal/projectconfig/tests.go @@ -0,0 +1,514 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/invopop/jsonschema" + orderedmap "github.com/pb33f/ordered-map/v2" +) + +// TestKind indicates what kind of behavior a test exercises. +type TestKind string + +const ( + TestKindFunctional TestKind = "functional" + TestKindPerformance TestKind = "performance" +) + +func (k TestKind) IsValid() bool { + switch k { + case "": + return true + case TestKindFunctional, + TestKindPerformance: + return true + default: + return false + } +} + +func orderedMapWithConst(key string, value any) *orderedmap.OrderedMap[string, *jsonschema.Schema] { + m := orderedmap.New[string, *jsonschema.Schema]() + m.Set(key, &jsonschema.Schema{Const: value}) + + return m +} + +// TestDefinition is the new-shape [tests.X] declaration: one configuration of one +// runner/harness with framework-specific options. Framework subtables are kept as +// loosely-typed maps so the resolver can evolve their schemas without requiring +// matching struct changes here. +type TestDefinition struct { + // Type identifies the framework/runner. Required, and constrained to the + // closed enum in the schema tag at the schema layer. The loader still + // accepts unknown values permissively; the resolver is the source of truth. + Type string `toml:"type" json:"type" jsonschema:"required,title=Type,description=Test framework type,enum=pytest,enum=lisa,enum=tmt"` + + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test"` + + // Kind hints at what the test exercises. + Kind TestKind `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Kind hint for the test,enum=functional,enum=performance"` + + // LongRunning hints to schedulers/policy that this test may take a long time. + LongRunning bool `toml:"long-running,omitempty" json:"longRunning,omitempty" jsonschema:"title=Long running,description=Hints that this test may run for hours"` + + // RequiredCapabilities lists capability tokens an image must declare to be a + // valid target for this test. Tokens are matched against [ImageCapabilities]. + RequiredCapabilities []string `toml:"required-capabilities,omitempty" json:"requiredCapabilities,omitempty" jsonschema:"title=Required capabilities,description=Capability tokens the image must declare"` + + // Framework-specific subtables. Kept untyped so framework schema can evolve + // independently of the dev-tools type definitions. + Lisa map[string]any `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration"` + Tmt map[string]any `toml:"tmt,omitempty" json:"tmt,omitempty" jsonschema:"title=TMT config,description=TMT-specific configuration"` + Pytest map[string]any `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=pytest-specific configuration"` +} + +// TestGroup is a [test-groups.X] declaration: a named bundle of test references that +// images or components can target via a single name. +type TestGroup struct { + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test group"` + + // Tests is the ordered list of test references that make up the group's + // membership. Group refs are validated as invalid at load time. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Ordered test references for this group (name only)"` +} + +// TestRef is a reference to either a test (by name) or another group (by name). +// Exactly one of Name or Group should be set; semantic validation is the resolver's +// responsibility. +type TestRef struct { + // Name references a [tests.X] entry. + Name string `toml:"name,omitempty" json:"name,omitempty" jsonschema:"title=Name,description=Name of a test (mutually exclusive with group)"` + + // Group references a [test-groups.X] entry. + Group string `toml:"group,omitempty" json:"group,omitempty" jsonschema:"title=Group,description=Name of a test group (mutually exclusive with name)"` +} + +// ComponentTestsConfig holds the new-shape per-component tests block: +// +// tests.tests = [{ name = "..." }, { group = "..." }] +type ComponentTestsConfig struct { + // Tests is the list of test or test-group references that apply to the component. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Per-component test or test-group references"` +} + +// Validate checks that exactly one framework subtable is set and it matches Type. +func (t TestDefinition) Validate(testName string) error { + if t.Type == "" { + return fmt.Errorf("%w: test %#q is missing required field 'type'", ErrMissingTestField, testName) + } + + if !t.Kind.IsValid() { + return fmt.Errorf("%w: test %#q has invalid kind %#q", ErrUnknownTestKind, testName, t.Kind) + } + + type testTypeRule struct { + required string + disallowed []string + } + + typeRules := map[string]testTypeRule{ + "pytest": {required: "pytest", disallowed: []string{"lisa", "tmt"}}, + "lisa": {required: "lisa", disallowed: []string{"pytest", "tmt"}}, + "tmt": {required: "tmt", disallowed: []string{"pytest", "lisa"}}, + } + + rule, ok := typeRules[t.Type] + if !ok { + return fmt.Errorf("%w: %#q (test: %#q)", ErrUnknownTestType, t.Type, testName) + } + + subtablePresence := map[string]bool{ + "pytest": t.Pytest != nil, + "lisa": t.Lisa != nil, + "tmt": t.Tmt != nil, + } + + if !subtablePresence[rule.required] { + return fmt.Errorf( + "%w: test %#q of type %#q requires a [%s] subtable", + ErrMissingTestField, + testName, + t.Type, + rule.required, + ) + } + + for _, subtable := range rule.disallowed { + if subtablePresence[subtable] { + return fmt.Errorf( + "%w: test %#q of type %#q cannot include subtable '%s'", + ErrMismatchedTestSubtable, + testName, + t.Type, + subtable, + ) + } + } + + if t.Type == "lisa" { + if err := validateLisaSelection(t.Lisa, testName); err != nil { + return err + } + } + + return nil +} + +// JSONSchemaExtend narrows [TestGroup.Tests] to name-only refs so editor-time +// validation matches runtime behavior (group refs in [test-groups] are rejected). +func (TestGroup) JSONSchemaExtend(schema *jsonschema.Schema) { + if schema == nil || schema.Properties == nil { + return + } + + testsProp, ok := schema.Properties.Get("tests") + if !ok || testsProp == nil { + return + } + + minLen := uint64(1) + itemProps := orderedmap.New[string, *jsonschema.Schema]() + itemProps.Set("name", &jsonschema.Schema{ + Type: "string", + MinLength: &minLen, + Pattern: "^\\S", + Description: "Name of a test", + }) + + testsProp.Items = &jsonschema.Schema{ + Type: "object", + Properties: itemProps, + Required: []string{"name"}, + Not: &jsonschema.Schema{ + Required: []string{"group"}, + }, + } +} + +func validateLisaSelection(lisa map[string]any, testName string) error { + hasSelector := false + + if rawCriteria, ok := lisa["criteria"]; ok { + hasSelector = true + + if err := validateLisaCriteria(rawCriteria, testName); err != nil { + return err + } + } + + if rawName, ok := lisa["testcaseName"]; ok { + hasSelector = true + + if !isNonEmptyString(rawName) { + return fmt.Errorf( + "%w: test %#q lisa.testcaseName must be a non-empty string", + ErrInvalidLisaSelection, + testName, + ) + } + } + + if rawName, ok := lisa["name"]; ok { + hasSelector = true + + if !isNonEmptyString(rawName) { + return fmt.Errorf( + "%w: test %#q lisa.name must be a non-empty string", + ErrInvalidLisaSelection, + testName, + ) + } + } + + if rawNames, ok := lisa["testcaseNames"]; ok { + hasSelector = true + + if err := validateStringList(rawNames, "lisa.testcaseNames", testName); err != nil { + return err + } + } + + if !hasSelector { + return fmt.Errorf( + "%w: test %#q of type %#q must set at least one LISA selector: criteria, testcaseName, testcaseNames, or name", + ErrInvalidLisaSelection, + testName, + "lisa", + ) + } + + return nil +} + +func validateLisaCriteria(rawCriteria any, testName string) error { + criteriaList, err := normalizeCriteriaList(rawCriteria) + if err != nil { + return fmt.Errorf("%w: test %#q lisa.criteria %w", ErrInvalidLisaSelection, testName, err) + } + + for idx, criteria := range criteriaList { + if err := validateSingleLisaCriteria(criteria, testName, idx); err != nil { + return err + } + } + + return nil +} + +func normalizeCriteriaList(rawCriteria any) ([]map[string]any, error) { + switch criteriaValue := rawCriteria.(type) { + case map[string]any: + if len(criteriaValue) == 0 { + return nil, errors.New("must not be empty") + } + + return []map[string]any{criteriaValue}, nil + case []any: + if len(criteriaValue) == 0 { + return nil, errors.New("must not be an empty list") + } + + result := make([]map[string]any, 0, len(criteriaValue)) + + for entryIndex, item := range criteriaValue { + criteriaMap, ok := item.(map[string]any) + if !ok { + return nil, fmt.Errorf("entry %d must be a table/object", entryIndex) + } + + if len(criteriaMap) == 0 { + return nil, fmt.Errorf("entry %d must not be empty", entryIndex) + } + + result = append(result, criteriaMap) + } + + return result, nil + default: + return nil, errors.New("must be a table or list of tables") + } +} + +func validateSingleLisaCriteria(criteria map[string]any, testName string, idx int) error { + allowedKeys := map[string]bool{ + "name": true, + "area": true, + "category": true, + "priority": true, + "tags": true, + "testcaseName": true, + "testcaseNames": true, + } + + hasSelector := false + + for key, value := range criteria { + if !allowedKeys[key] { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d] contains unsupported selector %#q", + ErrInvalidLisaSelection, + testName, + idx, + key, + ) + } + + switch key { + case "name", "area", "category", "testcaseName": + if !isNonEmptyString(value) { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d].%s must be a non-empty string", + ErrInvalidLisaSelection, + testName, + idx, + key, + ) + } + + hasSelector = true + case "priority": + if err := validateLisaPriority(value, testName, idx); err != nil { + return err + } + + hasSelector = true + case "tags", "testcaseNames": + fieldName := "lisa.criteria[" + strconv.Itoa(idx) + "]." + key + + if err := validateStringList(value, fieldName, testName); err != nil { + return err + } + + hasSelector = true + } + } + + if !hasSelector { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d] must include at least one selector", + ErrInvalidLisaSelection, + testName, + idx, + ) + } + + return nil +} + +func validateLisaPriority(value any, testName string, idx int) error { + if isLisaPriorityValue(value) { + return nil + } + + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d].priority must be an integer 0..4 or a non-empty list of integers 0..4", + ErrInvalidLisaSelection, + testName, + idx, + ) +} + +func isLisaPriorityValue(value any) bool { + if parsed, ok := parseLisaPriority(value); ok { + return parsed >= 0 && parsed <= 4 + } + + priorityList, ok := value.([]any) + if !ok || len(priorityList) == 0 { + return false + } + + for _, item := range priorityList { + parsed, ok := parseLisaPriority(item) + if !ok || parsed < 0 || parsed > 4 { + return false + } + } + + return true +} + +func parseLisaPriority(value any) (int, bool) { + switch typed := value.(type) { + case int: + return typed, true + case int64: + return int(typed), true + case float64: + if typed != float64(int(typed)) { + return 0, false + } + + return int(typed), true + default: + return 0, false + } +} + +func validateStringList(value any, fieldName string, testName string) error { + items, ok := value.([]any) + if !ok || len(items) == 0 { + return fmt.Errorf( + "%w: test %#q %s must be a non-empty list of non-empty strings", + ErrInvalidLisaSelection, + testName, + fieldName, + ) + } + + for _, item := range items { + if !isNonEmptyString(item) { + return fmt.Errorf( + "%w: test %#q %s must be a non-empty list of non-empty strings", + ErrInvalidLisaSelection, + testName, + fieldName, + ) + } + } + + return nil +} + +func isNonEmptyString(value any) bool { + s, ok := value.(string) + if !ok { + return false + } + + return strings.TrimSpace(s) != "" +} + +// JSONSchemaExtend tightens [TestDefinition] so the framework-specific subtable +// must match the declared type. +func (TestDefinition) JSONSchemaExtend(schema *jsonschema.Schema) { + if schema == nil { + return + } + + onlyPytest := &jsonschema.Schema{ + Required: []string{"pytest"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"lisa"}}, + {Required: []string{"tmt"}}, + }}, + } + + onlyLisa := &jsonschema.Schema{ + Required: []string{"lisa"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"pytest"}}, + {Required: []string{"tmt"}}, + }}, + } + + onlyTmt := &jsonschema.Schema{ + Required: []string{"tmt"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"pytest"}}, + {Required: []string{"lisa"}}, + }}, + } + + schema.AllOf = append(schema.AllOf, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "pytest")}, Then: onlyPytest}, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "lisa")}, Then: onlyLisa}, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "tmt")}, Then: onlyTmt}, + ) +} + +// JSONSchemaExtend tightens the generated schema for [TestRef] so editors can +// flag refs that set neither or both of name/group, or empty/whitespace-only values. +// The runtime resolver ([ErrInvalidTestRef]) is the source of truth; this keeps the schema in sync. +func (TestRef) JSONSchemaExtend(schema *jsonschema.Schema) { + // Exactly one of name|group: encoded as oneOf with `required` on each and + // `not` excluding the other, which forbids both `{}` and `{name, group}`. + schema.OneOf = []*jsonschema.Schema{ + {Required: []string{"name"}, Not: &jsonschema.Schema{Required: []string{"group"}}}, + {Required: []string{"group"}, Not: &jsonschema.Schema{Required: []string{"name"}}}, + } + + // Prevent empty or leading-whitespace identifiers in both name and group fields. + minLen := uint64(1) + + if schema.Properties != nil { + if nameProp, ok := schema.Properties.Get("name"); ok && nameProp != nil { + nameProp.MinLength = &minLen + nameProp.Pattern = "^\\S" + } + + if groupProp, ok := schema.Properties.Get("group"); ok && groupProp != nil { + groupProp.MinLength = &minLen + groupProp.Pattern = "^\\S" + } + } +} diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index d8424592..0c19d329 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -29,9 +29,23 @@ var ( ErrMissingTestField = errors.New("missing required test field") // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. ErrUndefinedTestSuite = errors.New("undefined test suite reference") + // ErrUndefinedTest is returned when a test reference points to a missing [tests] entry. + ErrUndefinedTest = errors.New("undefined test reference") + // ErrUndefinedTestGroup is returned when a test reference points to a missing [test-groups] entry. + ErrUndefinedTestGroup = errors.New("undefined test group reference") + // ErrInvalidTestRef is returned when a TestRef has neither or both of name/group set. + ErrInvalidTestRef = errors.New("invalid test reference") + // ErrDuplicateTestRef is returned when a list contains the same test ref more than once. + ErrDuplicateTestRef = errors.New("duplicate test reference") + // ErrNestedTestGroupReference is returned when a [test-groups] member uses a group ref. + ErrNestedTestGroupReference = errors.New("nested test group reference") // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not // match its declared type. ErrMismatchedTestSubtable = errors.New("mismatched test subtable") + // ErrUnknownTestKind is returned for unrecognized test kinds. + ErrUnknownTestKind = errors.New("unknown test kind") + // ErrInvalidLisaSelection is returned when a lisa test has invalid or missing selectors. + ErrInvalidLisaSelection = errors.New("invalid lisa selection") // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. ErrInvalidInstallMode = errors.New("invalid install mode") ) diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index e24f758f..88b51a6e 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -50,7 +50,7 @@ func TestImageCapabilities_EnabledNames(t *testing.T) { func TestImageConfig_TestNames(t *testing.T) { t.Run("with tests", func(t *testing.T) { img := projectconfig.ImageConfig{ - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, {Name: "integration"}, @@ -294,7 +294,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, + Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, }, }, TestSuites: map[string]projectconfig.TestSuiteConfig{ @@ -320,7 +320,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, + Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, }, }, TestSuites: make(map[string]projectconfig.TestSuiteConfig), diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index d665b45f..e9ed6d37 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -432,6 +451,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -710,6 +745,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,6 +1331,230 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "type": "string", + "enum": [ + "functional", + "performance" + ], + "title": "Kind", + "description": "Kind hint for the test" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "array", + "title": "Tests", + "description": "Ordered test references for this group (name only)" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index d665b45f..e9ed6d37 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -432,6 +451,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -710,6 +745,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,6 +1331,230 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "type": "string", + "enum": [ + "functional", + "performance" + ], + "title": "Kind", + "description": "Kind hint for the test" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "array", + "title": "Tests", + "description": "Ordered test references for this group (name only)" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index d665b45f..e9ed6d37 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -173,6 +173,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -347,6 +352,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -432,6 +451,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -710,6 +745,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1288,6 +1331,230 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "type": "string", + "enum": [ + "functional", + "performance" + ], + "title": "Kind", + "description": "Kind hint for the test" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "array", + "title": "Tests", + "description": "Ordered test references for this group (name only)" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": {