From 7037c115435010589dc17a0ffb452c706eb073f8 Mon Sep 17 00:00:00 2001 From: riddhibhagwat-db Date: Mon, 27 Jul 2026 19:06:36 +0000 Subject: [PATCH 1/5] AIR CLI: send dependencies inline, stop uploading requirements.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `air run` now carries the user's declared dependencies — an inline list, or the dependencies read from a requirements.yaml file — on the submission's environments[].spec.dependencies, and no longer uploads a requirements.yaml artifact at all. The AI Runtime backend installs the inline deps via --deps-config and treats a missing co-located requirements.yaml as "no requirements" (databricks-eng/universe#2297011), so the upload is unnecessary. This removes the vestigial empty requirements.yaml that a no-dependency run used to upload just to satisfy the launcher's derived path. When no dependencies are declared, spec.dependencies is omitted and the payload is unchanged. A -r/--requirement include in a requirements file is rejected, since the referenced file is never uploaded with the run. Ports databricks-eng/universe#2178617 and aligns with the backend skip in #2297011. Co-authored-by: Isaac --- experimental/air/cmd/runsubmit.go | 40 +++++++++++-- experimental/air/cmd/runsubmit_test.go | 79 +++++++++++++++++++++++++- experimental/air/cmd/runupload.go | 61 +++++++++++--------- experimental/air/cmd/runupload_test.go | 26 ++++----- 4 files changed, 155 insertions(+), 51 deletions(-) diff --git a/experimental/air/cmd/runsubmit.go b/experimental/air/cmd/runsubmit.go index 8c0be55260d..1a7ce59d76b 100644 --- a/experimental/air/cmd/runsubmit.go +++ b/experimental/air/cmd/runsubmit.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "path/filepath" "strconv" "strings" @@ -38,8 +39,26 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string { return strings.TrimPrefix(img, "CLIENT-GPU-") } +// environmentDependencies resolves the user's declared dependencies as a flat +// list to carry inline on the serverless environment's spec.dependencies: the +// inline list directly, or the dependencies read from a requirements file +// (resolved against the config's directory). Returns nil when none are declared. +func environmentDependencies(cfg *runConfig, configPath string) ([]string, error) { + if deps, ok := cfg.inlineDependencies(); ok { + return deps, nil + } + if reqPath, ok := cfg.requirementsFile(); ok { + if !filepath.IsAbs(reqPath) { + reqPath = filepath.Join(filepath.Dir(configPath), reqPath) + } + return readRequirementsDependencies(reqPath) + } + return nil, nil +} + // buildSubmitPayload assembles the runs/submit payload. commandPath is the -// workspace path of the uploaded command.sh; dlImage is the runtime channel. +// workspace path of the uploaded command.sh; dlImage is the runtime channel; +// deps is the user's declared dependencies (nil when none are declared). // // max_retries is always sent (including 0) so the user's YAML value is honored: // setting it to 0 explicitly disables retries rather than falling back to the @@ -47,7 +66,7 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string { // omitempty so the wire form matches the Python CLI (which never emits a bare // "false"). Jobs performs the retries — each attempt is a fresh AI Runtime // workload. -func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapshotResult) jobs.SubmitRun { +func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapshotResult, deps []string) jobs.SubmitRun { task := jobs.AiRuntimeTask{ Experiment: cfg.ExperimentName, Deployments: []jobs.DeploymentSpec{{ @@ -89,13 +108,22 @@ func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapsh ForceSendFields: []string{"MaxRetries"}, } + // Carry the user's declared deps inline on spec.dependencies; the AI Runtime + // backend installs them via --deps-config. The SDK marshaler drops nil and empty + // slices, so a no-deps run omits the key. Deps are no longer uploaded as a + // requirements.yaml (see buildArtifacts). + envSpec := &compute.Environment{EnvironmentVersion: dlImage} + if len(deps) > 0 { + envSpec.Dependencies = deps + } + return jobs.SubmitRun{ RunName: cfg.ExperimentName, TimeoutSeconds: cfg.timeoutSeconds(), Tasks: []jobs.SubmitTask{st}, Environments: []jobs.JobEnvironment{{ EnvironmentKey: aiRuntimeEnvironmentKey, - Spec: &compute.Environment{EnvironmentVersion: dlImage}, + Spec: envSpec, }}, } } @@ -175,8 +203,12 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run } } + deps, err := environmentDependencies(cfg, configPath) + if err != nil { + return 0, "", err + } runtimeVersion, _ := cfg.runtimeVersion() - payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap) + payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap, deps) payload.IdempotencyToken = token // Submit returns as soon as the run is created; we don't wait for it to finish. diff --git a/experimental/air/cmd/runsubmit_test.go b/experimental/air/cmd/runsubmit_test.go index fd5103599df..fb3e58f96de 100644 --- a/experimental/air/cmd/runsubmit_test.go +++ b/experimental/air/cmd/runsubmit_test.go @@ -2,6 +2,7 @@ package aircmd import ( "encoding/json" + "os" "path/filepath" "strings" "testing" @@ -37,7 +38,7 @@ func TestBuildSubmitPayload(t *testing.T) { MLflowExperimentDirectory: new("/Workspace/Users/me/exp"), } - p := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}) + p := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, nil) assert.Equal(t, "exp", p.RunName) assert.Equal(t, 1800, p.TimeoutSeconds) @@ -72,7 +73,7 @@ func TestBuildSubmitPayloadDefaultRetries(t *testing.T) { Command: new("x"), Compute: &computeConfig{AcceleratorType: "GPU_1xH100", NumAccelerators: 1}, } - task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}).Tasks[0] + task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}, nil).Tasks[0] assert.Equal(t, defaultMaxRetries, task.MaxRetries) assert.True(t, task.RetryOnTimeout) } @@ -87,7 +88,7 @@ func TestBuildSubmitPayloadNoRetries(t *testing.T) { Compute: &computeConfig{AcceleratorType: "GPU_1xH100", NumAccelerators: 1}, MaxRetries: new(0), } - task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}).Tasks[0] + task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}, nil).Tasks[0] assert.Equal(t, 0, task.MaxRetries) assert.False(t, task.RetryOnTimeout) @@ -97,6 +98,78 @@ func TestBuildSubmitPayloadNoRetries(t *testing.T) { assert.NotContains(t, string(b), "retry_on_timeout") } +func TestBuildSubmitPayloadInlineDependencies(t *testing.T) { + cfg := &runConfig{ + ExperimentName: "exp", + Command: new("x"), + Compute: &computeConfig{AcceleratorType: "GPU_8xH100", NumAccelerators: 8}, + } + + // Declared deps ride inline on spec.dependencies; the runtime channel is preserved. + deps := []string{"torch==2.3.0", "--extra-index-url https://internal/pypi", "numpy"} + spec := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, deps).Environments[0].Spec + assert.Equal(t, deps, spec.Dependencies) + assert.Equal(t, "5", spec.EnvironmentVersion) + + // An empty or absent deps list omits the key (the SDK marshaler drops empty + // slices), leaving the payload unchanged from before. + for _, empty := range [][]string{{}, nil} { + spec = buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, empty).Environments[0].Spec + assert.Empty(t, spec.Dependencies) + b, err := json.Marshal(spec) + require.NoError(t, err) + assert.NotContains(t, string(b), "dependencies") + } +} + +func TestEnvironmentDependencies(t *testing.T) { + // Inline list is returned directly. + cfg := &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: true, list: []string{"torch", "numpy"}}, + }} + deps, err := environmentDependencies(cfg, "run.yaml") + require.NoError(t, err) + assert.Equal(t, []string{"torch", "numpy"}, deps) + + // A requirements file's deps are read and resolved against the config's directory. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: \"5\"\ndependencies:\n - pandas\n"), 0o600)) + cfg = &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}, + }} + deps, err = environmentDependencies(cfg, filepath.Join(dir, "run.yaml")) + require.NoError(t, err) + assert.Equal(t, []string{"pandas"}, deps) + + // No dependencies declared returns nil. + deps, err = environmentDependencies(&runConfig{}, "run.yaml") + require.NoError(t, err) + assert.Nil(t, deps) +} + +func TestReadRequirementsDependencies(t *testing.T) { + dir := t.TempDir() + + reqPath := filepath.Join(dir, "requirements.yaml") + require.NoError(t, os.WriteFile(reqPath, []byte("version: \"5\"\ndependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) + deps, err := readRequirementsDependencies(reqPath) + require.NoError(t, err) + assert.Equal(t, []string{"torch==2.3.0", "numpy"}, deps) + + // No dependencies key yields an empty list. + emptyPath := filepath.Join(dir, "empty.yaml") + require.NoError(t, os.WriteFile(emptyPath, []byte("version: \"5\"\n"), 0o600)) + deps, err = readRequirementsDependencies(emptyPath) + require.NoError(t, err) + assert.Empty(t, deps) + + // A -r/--requirement include is rejected. + includePath := filepath.Join(dir, "include.yaml") + require.NoError(t, os.WriteFile(includePath, []byte("dependencies:\n - -r other.txt\n"), 0o600)) + _, err = readRequirementsDependencies(includePath) + require.ErrorContains(t, err, "requirements-file include") +} + func TestSubmitToken(t *testing.T) { cfg := &runConfig{IdempotencyToken: new("from-config")} diff --git a/experimental/air/cmd/runupload.go b/experimental/air/cmd/runupload.go index fb9ca00b987..3184273a3aa 100644 --- a/experimental/air/cmd/runupload.go +++ b/experimental/air/cmd/runupload.go @@ -8,7 +8,6 @@ import ( "io" "maps" "os" - "path/filepath" "slices" "strings" @@ -22,7 +21,6 @@ import ( const ( trainingConfigName = "training_config.yaml" commandScriptName = "command.sh" - requirementsName = "requirements.yaml" hyperparametersName = "hyperparameters.yaml" envVarsName = "env_vars.json" secretEnvVarsName = "secret_env_vars.json" @@ -45,16 +43,45 @@ type fileWriter interface { Write(ctx context.Context, name string, reader io.Reader, mode ...filer.WriteMode) error } -// requirementsDoc mirrors the on-disk requirements.yaml format so the worker -// parses synthesized inline dependencies identically to a user-provided file. +// requirementsDoc mirrors the on-disk requirements.yaml format, used to read a +// user-provided requirements file's dependencies for the inline spec.dependencies. type requirementsDoc struct { Version string `yaml:"version,omitempty"` Dependencies []string `yaml:"dependencies"` } +// readRequirementsDependencies reads the dependencies list out of a +// requirements.yaml file so file-form deps can be carried on the serverless +// environment's inline spec.dependencies. Returns an empty list when the file +// declares no dependencies. +func readRequirementsDependencies(reqPath string) ([]string, error) { + data, err := os.ReadFile(reqPath) + if err != nil { + return nil, fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) + } + var doc requirementsDoc + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("failed to parse requirements file %s: %w", reqPath, err) + } + for _, dep := range doc.Dependencies { + // A -r/--requirement include points at a second file that is never uploaded + // with the run, so it never installs on the node. Reject it rather than emit + // a dependency that silently fails at install time. + if fields := strings.Fields(dep); len(fields) > 0 && (fields[0] == "-r" || fields[0] == "--requirement") { + return nil, fmt.Errorf("requirements file dependency %q uses a requirements-file include (-r/--requirement), which is not supported; list the dependencies directly instead", dep) + } + } + return doc.Dependencies, nil +} + // buildArtifacts assembles the files to upload for a run: the merged config, the -// inline command as a script, requirements (from a file or synthesized from -// inline dependencies), and hyperparameters. configPath is the local YAML path. +// inline command as a script, and hyperparameters. configPath is the local YAML +// path. +// +// Dependencies are no longer uploaded as a requirements.yaml; they ride inline on +// the serverless environment's spec.dependencies instead (see buildSubmitPayload). +// The AI Runtime launcher treats a missing co-located requirements.yaml as "no +// requirements" (databricks-eng/universe#2297011), so uploading one is unnecessary. func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { // TODO(DABs): with no _bases_/overrides ported yet, the merged config is the // file as-is; once those land, upload the re-serialized merged YAML instead. @@ -72,28 +99,6 @@ func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { {commandScriptName, []byte(*cfg.Command)}, } - switch reqPath, ok := cfg.requirementsFile(); { - case ok: - // Resolve a relative requirements path against the config's directory. - if !filepath.IsAbs(reqPath) { - reqPath = filepath.Join(filepath.Dir(configPath), reqPath) - } - data, err := os.ReadFile(reqPath) - if err != nil { - return nil, fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) - } - items = append(items, uploadItem{requirementsName, data}) - default: - if deps, ok := cfg.inlineDependencies(); ok { - version, _ := cfg.runtimeVersion() - data, err := yaml.Marshal(requirementsDoc{Version: version, Dependencies: deps}) - if err != nil { - return nil, fmt.Errorf("failed to synthesize requirements.yaml: %w", err) - } - items = append(items, uploadItem{requirementsName, data}) - } - } - if len(cfg.Parameters) > 0 { data, err := yaml.Marshal(cfg.Parameters) if err != nil { diff --git a/experimental/air/cmd/runupload_test.go b/experimental/air/cmd/runupload_test.go index 0c87524735d..e07b81f3c23 100644 --- a/experimental/air/cmd/runupload_test.go +++ b/experimental/air/cmd/runupload_test.go @@ -57,7 +57,7 @@ func TestBuildArtifacts_CommandAndConfig(t *testing.T) { assert.Equal(t, "python train.py", string(items[1].data)) } -func TestBuildArtifacts_InlineRequirementsAndParameters(t *testing.T) { +func TestBuildArtifacts_ParametersButNoRequirements(t *testing.T) { path := writeConfigFile(t, "run.yaml", "x: y\n") cfg := &runConfig{ Command: new("echo hi"), @@ -68,19 +68,11 @@ func TestBuildArtifacts_InlineRequirementsAndParameters(t *testing.T) { Parameters: map[string]any{"lr": 0.1}, } + // Dependencies ride on spec.dependencies, not a requirements.yaml upload, so the + // artifacts are just the config, command, and hyperparameters. items, err := buildArtifacts(cfg, path) require.NoError(t, err) - assert.Equal(t, []string{trainingConfigName, commandScriptName, requirementsName, hyperparametersName}, itemNames(items)) - - var reqIdx int - for i, it := range items { - if it.name == requirementsName { - reqIdx = i - } - } - req := string(items[reqIdx].data) - assert.Contains(t, req, "version: \"5\"") - assert.Contains(t, req, "- torch") + assert.Equal(t, []string{trainingConfigName, commandScriptName, hyperparametersName}, itemNames(items)) } func TestBuildArtifacts_EnvVarsAndSecrets(t *testing.T) { @@ -103,7 +95,7 @@ func TestBuildArtifacts_EnvVarsAndSecrets(t *testing.T) { assert.JSONEq(t, `[{"name":"HF_TOKEN","secret_scope":"myscope","secret_key":"hf"}]`, string(byName[secretEnvVarsName])) } -func TestBuildArtifacts_RequirementsFile(t *testing.T) { +func TestBuildArtifacts_RequirementsFileNotUploaded(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "run.yaml"), []byte("x: y\n"), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: 4\n"), 0o600)) @@ -112,9 +104,11 @@ func TestBuildArtifacts_RequirementsFile(t *testing.T) { Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}}, } + // A declared requirements file's deps travel on spec.dependencies, not as an + // uploaded artifact. items, err := buildArtifacts(cfg, filepath.Join(dir, "run.yaml")) require.NoError(t, err) - assert.Contains(t, itemNames(items), requirementsName) + assert.Equal(t, []string{trainingConfigName, commandScriptName}, itemNames(items)) } func TestBuildArtifacts_OversizeConfigRejected(t *testing.T) { @@ -144,12 +138,12 @@ func TestUploadArtifacts_WriteError(t *testing.T) { require.ErrorContains(t, err, "failed to upload "+trainingConfigName) } -func TestBuildArtifacts_MissingRequirementsFile(t *testing.T) { +func TestEnvironmentDependencies_MissingRequirementsFile(t *testing.T) { cfgPath := writeConfigFile(t, "run.yaml", "x: y\n") cfg := &runConfig{ Command: new("echo hi"), Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}}, } - _, err := buildArtifacts(cfg, cfgPath) + _, err := environmentDependencies(cfg, cfgPath) require.ErrorContains(t, err, "failed to read requirements file") } From 96cca4a002964d391a91db60fdc23bad6258c0cb Mon Sep 17 00:00:00 2001 From: riddhibhagwat-db Date: Mon, 27 Jul 2026 21:03:23 +0000 Subject: [PATCH 2/5] AIR CLI: acceptance coverage for inline deps + tighten unit tests Add a run-submit-deps acceptance test that submits with inline dependencies and asserts the golden payload carries them on environments[].spec.dependencies and that only command.sh and training_config.yaml are uploaded (no requirements.yaml). Consolidate the dependency unit tests: fold the missing-file case into TestEnvironmentDependencies (it exercises runsubmit.go, not runupload.go) and trim the comments to state each test's intent. Co-authored-by: Isaac --- .../air/run-submit-deps/databricks.yml | 2 + .../air/run-submit-deps/out.test.toml | 3 ++ .../air/run-submit-deps/output.txt | 54 +++++++++++++++++++ .../experimental/air/run-submit-deps/run.yaml | 10 ++++ .../experimental/air/run-submit-deps/script | 8 +++ .../air/run-submit-deps/test.toml | 19 +++++++ experimental/air/cmd/runsubmit_test.go | 36 +++++++------ experimental/air/cmd/runupload_test.go | 16 +----- 8 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 acceptance/experimental/air/run-submit-deps/databricks.yml create mode 100644 acceptance/experimental/air/run-submit-deps/out.test.toml create mode 100644 acceptance/experimental/air/run-submit-deps/output.txt create mode 100644 acceptance/experimental/air/run-submit-deps/run.yaml create mode 100644 acceptance/experimental/air/run-submit-deps/script create mode 100644 acceptance/experimental/air/run-submit-deps/test.toml diff --git a/acceptance/experimental/air/run-submit-deps/databricks.yml b/acceptance/experimental/air/run-submit-deps/databricks.yml new file mode 100644 index 00000000000..a073ec04b4f --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-submit-deps diff --git a/acceptance/experimental/air/run-submit-deps/out.test.toml b/acceptance/experimental/air/run-submit-deps/out.test.toml new file mode 100644 index 00000000000..d6187dcb046 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/air/run-submit-deps/output.txt b/acceptance/experimental/air/run-submit-deps/output.txt new file mode 100644 index 00000000000..6c490162116 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/output.txt @@ -0,0 +1,54 @@ + +=== submit with inline dependencies +>>> [CLI] experimental air run -f run.yaml +Submitted run 555 +View at: [DATABRICKS_URL]/jobs/runs/555 + +=== only config + command are uploaded; no requirements.yaml +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\n"} + +=== declared deps ride on environments[].spec.dependencies +>>> print_requests.py //api/2.2/jobs/runs/submit +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/submit", + "body": { + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "numpy", + "torch==2.3.0" + ], + "environment_version": "5" + } + } + ], + "idempotency_token": "[UUID]", + "run_name": "deps-smoke", + "tasks": [ + { + "ai_runtime_task": { + "deployments": [ + { + "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", + "compute": { + "accelerator_count": 1, + "accelerator_type": "GPU_1xH100" + } + } + ], + "experiment": "deps-smoke" + }, + "environment_key": "default", + "max_retries": 3, + "retry_on_timeout": true, + "run_if": "ALL_SUCCESS", + "task_key": "deps-smoke" + } + ] + } +} diff --git a/acceptance/experimental/air/run-submit-deps/run.yaml b/acceptance/experimental/air/run-submit-deps/run.yaml new file mode 100644 index 00000000000..fbb7e8681b5 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/run.yaml @@ -0,0 +1,10 @@ +experiment_name: deps-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +environment: + version: 5 + dependencies: + - numpy + - torch==2.3.0 diff --git a/acceptance/experimental/air/run-submit-deps/script b/acceptance/experimental/air/run-submit-deps/script new file mode 100644 index 00000000000..77e7825cc34 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/script @@ -0,0 +1,8 @@ +title "submit with inline dependencies" +trace $CLI experimental air run -f run.yaml + +title "only config + command are uploaded; no requirements.yaml" +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep + +title "declared deps ride on environments[].spec.dependencies" +trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/experimental/air/run-submit-deps/test.toml b/acceptance/experimental/air/run-submit-deps/test.toml new file mode 100644 index 00000000000..7ebac7a298a --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/test.toml @@ -0,0 +1,19 @@ +# A non-dry-run submit with inline dependencies (no code_source): asserts the deps +# land on spec.dependencies and that requirements.yaml is not uploaded. +RecordRequests = true + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.2/jobs/runs/submit" +Response.Body = ''' +{"run_id": 555} +''' + +# The per-run launch directory ends in _<16 hex>; the random suffix varies. +[[Repls]] +Old = 'deps-smoke_[0-9a-f]{16}' +New = 'deps-smoke_[RUN_ID]' diff --git a/experimental/air/cmd/runsubmit_test.go b/experimental/air/cmd/runsubmit_test.go index fb3e58f96de..4185820deda 100644 --- a/experimental/air/cmd/runsubmit_test.go +++ b/experimental/air/cmd/runsubmit_test.go @@ -98,6 +98,9 @@ func TestBuildSubmitPayloadNoRetries(t *testing.T) { assert.NotContains(t, string(b), "retry_on_timeout") } +// TestBuildSubmitPayloadInlineDependencies covers how deps land on the environment +// spec: a non-empty list is set alongside the runtime channel; empty and nil omit +// the key so the payload is unchanged. func TestBuildSubmitPayloadInlineDependencies(t *testing.T) { cfg := &runConfig{ ExperimentName: "exp", @@ -105,65 +108,68 @@ func TestBuildSubmitPayloadInlineDependencies(t *testing.T) { Compute: &computeConfig{AcceleratorType: "GPU_8xH100", NumAccelerators: 8}, } - // Declared deps ride inline on spec.dependencies; the runtime channel is preserved. deps := []string{"torch==2.3.0", "--extra-index-url https://internal/pypi", "numpy"} spec := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, deps).Environments[0].Spec assert.Equal(t, deps, spec.Dependencies) assert.Equal(t, "5", spec.EnvironmentVersion) - // An empty or absent deps list omits the key (the SDK marshaler drops empty - // slices), leaving the payload unchanged from before. + // The SDK marshaler drops empty/nil slices, so no "dependencies" key is emitted. for _, empty := range [][]string{{}, nil} { spec = buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, empty).Environments[0].Spec - assert.Empty(t, spec.Dependencies) b, err := json.Marshal(spec) require.NoError(t, err) assert.NotContains(t, string(b), "dependencies") } } +// TestEnvironmentDependencies covers how declared deps are resolved to a flat list: +// an inline list, a requirements file (path resolved against the config dir), none, +// and a missing file. func TestEnvironmentDependencies(t *testing.T) { - // Inline list is returned directly. - cfg := &runConfig{Environment: &environmentConfig{ + inline := &runConfig{Environment: &environmentConfig{ Dependencies: dependencies{set: true, isList: true, list: []string{"torch", "numpy"}}, }} - deps, err := environmentDependencies(cfg, "run.yaml") + deps, err := environmentDependencies(inline, "run.yaml") require.NoError(t, err) assert.Equal(t, []string{"torch", "numpy"}, deps) - // A requirements file's deps are read and resolved against the config's directory. dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: \"5\"\ndependencies:\n - pandas\n"), 0o600)) - cfg = &runConfig{Environment: &environmentConfig{ + require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("dependencies:\n - pandas\n"), 0o600)) + fromFile := &runConfig{Environment: &environmentConfig{ Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}, }} - deps, err = environmentDependencies(cfg, filepath.Join(dir, "run.yaml")) + deps, err = environmentDependencies(fromFile, filepath.Join(dir, "run.yaml")) require.NoError(t, err) assert.Equal(t, []string{"pandas"}, deps) - // No dependencies declared returns nil. deps, err = environmentDependencies(&runConfig{}, "run.yaml") require.NoError(t, err) assert.Nil(t, deps) + + missing := &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}, + }} + _, err = environmentDependencies(missing, filepath.Join(dir, "run.yaml")) + require.ErrorContains(t, err, "failed to read requirements file") } +// TestReadRequirementsDependencies covers reading a requirements file's dependency +// list, with a missing key yielding an empty list and a -r include rejected. func TestReadRequirementsDependencies(t *testing.T) { dir := t.TempDir() reqPath := filepath.Join(dir, "requirements.yaml") - require.NoError(t, os.WriteFile(reqPath, []byte("version: \"5\"\ndependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) + require.NoError(t, os.WriteFile(reqPath, []byte("dependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) deps, err := readRequirementsDependencies(reqPath) require.NoError(t, err) assert.Equal(t, []string{"torch==2.3.0", "numpy"}, deps) - // No dependencies key yields an empty list. emptyPath := filepath.Join(dir, "empty.yaml") require.NoError(t, os.WriteFile(emptyPath, []byte("version: \"5\"\n"), 0o600)) deps, err = readRequirementsDependencies(emptyPath) require.NoError(t, err) assert.Empty(t, deps) - // A -r/--requirement include is rejected. includePath := filepath.Join(dir, "include.yaml") require.NoError(t, os.WriteFile(includePath, []byte("dependencies:\n - -r other.txt\n"), 0o600)) _, err = readRequirementsDependencies(includePath) diff --git a/experimental/air/cmd/runupload_test.go b/experimental/air/cmd/runupload_test.go index e07b81f3c23..e57a8aa2c41 100644 --- a/experimental/air/cmd/runupload_test.go +++ b/experimental/air/cmd/runupload_test.go @@ -68,8 +68,7 @@ func TestBuildArtifacts_ParametersButNoRequirements(t *testing.T) { Parameters: map[string]any{"lr": 0.1}, } - // Dependencies ride on spec.dependencies, not a requirements.yaml upload, so the - // artifacts are just the config, command, and hyperparameters. + // Inline deps are not uploaded, so the artifacts are config, command, and params. items, err := buildArtifacts(cfg, path) require.NoError(t, err) assert.Equal(t, []string{trainingConfigName, commandScriptName, hyperparametersName}, itemNames(items)) @@ -104,8 +103,7 @@ func TestBuildArtifacts_RequirementsFileNotUploaded(t *testing.T) { Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}}, } - // A declared requirements file's deps travel on spec.dependencies, not as an - // uploaded artifact. + // A declared requirements file is not uploaded; its deps travel on spec.dependencies. items, err := buildArtifacts(cfg, filepath.Join(dir, "run.yaml")) require.NoError(t, err) assert.Equal(t, []string{trainingConfigName, commandScriptName}, itemNames(items)) @@ -137,13 +135,3 @@ func TestUploadArtifacts_WriteError(t *testing.T) { err := uploadArtifacts(t.Context(), errWriter{}, []uploadItem{{trainingConfigName, []byte("x")}}) require.ErrorContains(t, err, "failed to upload "+trainingConfigName) } - -func TestEnvironmentDependencies_MissingRequirementsFile(t *testing.T) { - cfgPath := writeConfigFile(t, "run.yaml", "x: y\n") - cfg := &runConfig{ - Command: new("echo hi"), - Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}}, - } - _, err := environmentDependencies(cfg, cfgPath) - require.ErrorContains(t, err, "failed to read requirements file") -} From 0a789ebd56e37e45ce5f003739be91e084c268a5 Mon Sep 17 00:00:00 2001 From: riddhibhagwat-db Date: Mon, 27 Jul 2026 21:48:19 +0000 Subject: [PATCH 3/5] AIR CLI: force LF on run-submit-deps run.yaml; drop verbose comments The run-submit-deps acceptance test uploads run.yaml's contents verbatim as training_config.yaml, so a Windows CRLF checkout changed the recorded payload and failed the golden. Pin run.yaml to eol=lf via a scoped .gitattributes. Also drop two over-explanatory comments in runupload.go. Co-authored-by: Isaac --- acceptance/experimental/air/run-submit-deps/.gitattributes | 3 +++ experimental/air/cmd/runupload.go | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 acceptance/experimental/air/run-submit-deps/.gitattributes diff --git a/acceptance/experimental/air/run-submit-deps/.gitattributes b/acceptance/experimental/air/run-submit-deps/.gitattributes new file mode 100644 index 00000000000..19cc7fc1706 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/.gitattributes @@ -0,0 +1,3 @@ +# run.yaml's contents are uploaded verbatim as training_config.yaml, so its line +# endings must stay \n on every OS — a Windows \r would change the recorded payload. +run.yaml text eol=lf diff --git a/experimental/air/cmd/runupload.go b/experimental/air/cmd/runupload.go index 3184273a3aa..f86acdb386b 100644 --- a/experimental/air/cmd/runupload.go +++ b/experimental/air/cmd/runupload.go @@ -64,9 +64,6 @@ func readRequirementsDependencies(reqPath string) ([]string, error) { return nil, fmt.Errorf("failed to parse requirements file %s: %w", reqPath, err) } for _, dep := range doc.Dependencies { - // A -r/--requirement include points at a second file that is never uploaded - // with the run, so it never installs on the node. Reject it rather than emit - // a dependency that silently fails at install time. if fields := strings.Fields(dep); len(fields) > 0 && (fields[0] == "-r" || fields[0] == "--requirement") { return nil, fmt.Errorf("requirements file dependency %q uses a requirements-file include (-r/--requirement), which is not supported; list the dependencies directly instead", dep) } @@ -80,8 +77,6 @@ func readRequirementsDependencies(reqPath string) ([]string, error) { // // Dependencies are no longer uploaded as a requirements.yaml; they ride inline on // the serverless environment's spec.dependencies instead (see buildSubmitPayload). -// The AI Runtime launcher treats a missing co-located requirements.yaml as "no -// requirements" (databricks-eng/universe#2297011), so uploading one is unnecessary. func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { // TODO(DABs): with no _bases_/overrides ported yet, the merged config is the // file as-is; once those land, upload the re-serialized merged YAML instead. From e87da092f1e3beade0e73fbdbc39903aa0af645c Mon Sep 17 00:00:00 2001 From: riddhibhagwat-db Date: Tue, 28 Jul 2026 18:18:53 +0000 Subject: [PATCH 4/5] AIR CLI: resolve dependencies before uploading artifacts Move environmentDependencies resolution ahead of the workspace uploads so a bad requirements file (e.g. a -r/--requirement include) fails fast without leaving orphaned artifacts in the workspace. Covered by a new guard subtest asserting no import-file request is made when dependency resolution fails. Co-authored-by: Isaac --- experimental/air/cmd/runsubmit.go | 11 +++++++---- experimental/air/cmd/runsubmit_test.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/experimental/air/cmd/runsubmit.go b/experimental/air/cmd/runsubmit.go index 1a7ce59d76b..ebeae220484 100644 --- a/experimental/air/cmd/runsubmit.go +++ b/experimental/air/cmd/runsubmit.go @@ -161,6 +161,13 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } + // Resolve dependencies before any upload too, so a bad requirements file fails + // fast without leaving orphaned artifacts in the workspace. + deps, err := environmentDependencies(cfg, configPath) + if err != nil { + return 0, "", err + } + experimentDir := "" if cfg.MLflowExperimentDirectory != nil { experimentDir = *cfg.MLflowExperimentDirectory @@ -203,10 +210,6 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run } } - deps, err := environmentDependencies(cfg, configPath) - if err != nil { - return 0, "", err - } runtimeVersion, _ := cfg.runtimeVersion() payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap, deps) payload.IdempotencyToken = token diff --git a/experimental/air/cmd/runsubmit_test.go b/experimental/air/cmd/runsubmit_test.go index 4185820deda..65d4685a31e 100644 --- a/experimental/air/cmd/runsubmit_test.go +++ b/experimental/air/cmd/runsubmit_test.go @@ -292,4 +292,23 @@ func TestSubmitWorkloadGuards(t *testing.T) { _, _, err := submitWorkload(t.Context(), w, &cfg, cfgPath, "") require.ErrorContains(t, err, "usage_policy_name is not yet supported") }) + + t.Run("bad requirements file fails before any upload", func(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var uploaded bool + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(testserver.Request) any { + uploaded = true + return nil + }) + testserver.AddDefaultHandlers(server) + tw, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + cfg := *base + cfg.Environment = &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "missing.yaml"}} + _, _, err = submitWorkload(t.Context(), tw, &cfg, cfgPath, "") + require.ErrorContains(t, err, "failed to read requirements file") + assert.False(t, uploaded, "no artifacts should be uploaded when dependency resolution fails") + }) } From a3897878f3029784749be7c7acaf5ea02a389b09 Mon Sep 17 00:00:00 2001 From: riddhibhagwat-db Date: Wed, 29 Jul 2026 01:19:20 +0000 Subject: [PATCH 5/5] AIR CLI: resolve requirements-file version; reword comments; extend deps test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For file-form dependencies (environment.dependencies pointing at a requirements file), read the version declared inside that file and use it to select the runtime image when top-level environment.version is unset — which validation disallows for file-form. Previously the file version was decoded but dropped, so these runs silently got the default image. readRequirementsDependencies and environmentDependencies now return the file version alongside the deps. Extend the run-submit-deps acceptance test with a file-form case asserting environment_version flows from the file and that neither the config nor the requirements file is uploaded. Reword two comments to describe current behavior rather than the removed requirements.yaml upload. Co-authored-by: Isaac --- .../air/run-submit-deps/.gitattributes | 7 ++- .../air/run-submit-deps/output.txt | 54 +++++++++++++++++++ .../air/run-submit-deps/reqs.yaml | 4 ++ .../air/run-submit-deps/run-file.yaml | 7 +++ .../experimental/air/run-submit-deps/script | 9 ++++ .../air/run-submit-deps/test.toml | 4 ++ experimental/air/cmd/runsubmit.go | 23 +++++--- experimental/air/cmd/runsubmit_test.go | 28 +++++----- experimental/air/cmd/runupload.go | 20 +++---- 9 files changed, 124 insertions(+), 32 deletions(-) create mode 100644 acceptance/experimental/air/run-submit-deps/reqs.yaml create mode 100644 acceptance/experimental/air/run-submit-deps/run-file.yaml diff --git a/acceptance/experimental/air/run-submit-deps/.gitattributes b/acceptance/experimental/air/run-submit-deps/.gitattributes index 19cc7fc1706..0d9562f4bfe 100644 --- a/acceptance/experimental/air/run-submit-deps/.gitattributes +++ b/acceptance/experimental/air/run-submit-deps/.gitattributes @@ -1,3 +1,6 @@ -# run.yaml's contents are uploaded verbatim as training_config.yaml, so its line -# endings must stay \n on every OS — a Windows \r would change the recorded payload. +# These YAML files' contents are uploaded verbatim (as training_config.yaml and +# requirements.yaml), so their line endings must stay \n on every OS — a Windows +# \r would change the recorded payload. run.yaml text eol=lf +run-file.yaml text eol=lf +reqs.yaml text eol=lf diff --git a/acceptance/experimental/air/run-submit-deps/output.txt b/acceptance/experimental/air/run-submit-deps/output.txt index 6c490162116..1178a2e5d1c 100644 --- a/acceptance/experimental/air/run-submit-deps/output.txt +++ b/acceptance/experimental/air/run-submit-deps/output.txt @@ -52,3 +52,57 @@ View at: [DATABRICKS_URL]/jobs/runs/555 ] } } + +=== file-form deps: version comes from the requirements file +>>> [CLI] experimental air run -f run-file.yaml +Submitted run 555 +View at: [DATABRICKS_URL]/jobs/runs/555 + +=== file-form deps: the requirements file is not uploaded either +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-file-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n dependencies: ./reqs.yaml\n"} + +=== file-form deps ride on environments[].spec.dependencies +>>> print_requests.py //api/2.2/jobs/runs/submit +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/submit", + "body": { + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "numpy", + "torch==2.3.0" + ], + "environment_version": "5" + } + } + ], + "idempotency_token": "[UUID]", + "run_name": "deps-file-smoke", + "tasks": [ + { + "ai_runtime_task": { + "deployments": [ + { + "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh", + "compute": { + "accelerator_count": 1, + "accelerator_type": "GPU_1xH100" + } + } + ], + "experiment": "deps-file-smoke" + }, + "environment_key": "default", + "max_retries": 3, + "retry_on_timeout": true, + "run_if": "ALL_SUCCESS", + "task_key": "deps-file-smoke" + } + ] + } +} diff --git a/acceptance/experimental/air/run-submit-deps/reqs.yaml b/acceptance/experimental/air/run-submit-deps/reqs.yaml new file mode 100644 index 00000000000..da4b89534c2 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/reqs.yaml @@ -0,0 +1,4 @@ +version: 5 +dependencies: + - numpy + - torch==2.3.0 diff --git a/acceptance/experimental/air/run-submit-deps/run-file.yaml b/acceptance/experimental/air/run-submit-deps/run-file.yaml new file mode 100644 index 00000000000..3e5fe54b7b0 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/run-file.yaml @@ -0,0 +1,7 @@ +experiment_name: deps-file-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +environment: + dependencies: ./reqs.yaml diff --git a/acceptance/experimental/air/run-submit-deps/script b/acceptance/experimental/air/run-submit-deps/script index 77e7825cc34..52035bbae65 100644 --- a/acceptance/experimental/air/run-submit-deps/script +++ b/acceptance/experimental/air/run-submit-deps/script @@ -6,3 +6,12 @@ trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort - title "declared deps ride on environments[].spec.dependencies" trace print_requests.py //api/2.2/jobs/runs/submit + +title "file-form deps: version comes from the requirements file" +trace $CLI experimental air run -f run-file.yaml + +title "file-form deps: the requirements file is not uploaded either" +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep + +title "file-form deps ride on environments[].spec.dependencies" +trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/experimental/air/run-submit-deps/test.toml b/acceptance/experimental/air/run-submit-deps/test.toml index 7ebac7a298a..590c2918a25 100644 --- a/acceptance/experimental/air/run-submit-deps/test.toml +++ b/acceptance/experimental/air/run-submit-deps/test.toml @@ -17,3 +17,7 @@ Response.Body = ''' [[Repls]] Old = 'deps-smoke_[0-9a-f]{16}' New = 'deps-smoke_[RUN_ID]' + +[[Repls]] +Old = 'deps-file-smoke_[0-9a-f]{16}' +New = 'deps-file-smoke_[RUN_ID]' diff --git a/experimental/air/cmd/runsubmit.go b/experimental/air/cmd/runsubmit.go index ebeae220484..1c0b8bdef6e 100644 --- a/experimental/air/cmd/runsubmit.go +++ b/experimental/air/cmd/runsubmit.go @@ -42,10 +42,13 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string { // environmentDependencies resolves the user's declared dependencies as a flat // list to carry inline on the serverless environment's spec.dependencies: the // inline list directly, or the dependencies read from a requirements file -// (resolved against the config's directory). Returns nil when none are declared. -func environmentDependencies(cfg *runConfig, configPath string) ([]string, error) { +// (resolved against the config's directory). For file-form deps it also returns +// the version declared inside that file, which selects the runtime image since +// top-level environment.version is not allowed there. Returns nil when none are +// declared. +func environmentDependencies(cfg *runConfig, configPath string) (deps []string, fileVersion string, err error) { if deps, ok := cfg.inlineDependencies(); ok { - return deps, nil + return deps, "", nil } if reqPath, ok := cfg.requirementsFile(); ok { if !filepath.IsAbs(reqPath) { @@ -53,7 +56,7 @@ func environmentDependencies(cfg *runConfig, configPath string) ([]string, error } return readRequirementsDependencies(reqPath) } - return nil, nil + return nil, "", nil } // buildSubmitPayload assembles the runs/submit payload. commandPath is the @@ -110,8 +113,7 @@ func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapsh // Carry the user's declared deps inline on spec.dependencies; the AI Runtime // backend installs them via --deps-config. The SDK marshaler drops nil and empty - // slices, so a no-deps run omits the key. Deps are no longer uploaded as a - // requirements.yaml (see buildArtifacts). + // slices, so a no-deps run omits the key. envSpec := &compute.Environment{EnvironmentVersion: dlImage} if len(deps) > 0 { envSpec.Dependencies = deps @@ -163,7 +165,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run // Resolve dependencies before any upload too, so a bad requirements file fails // fast without leaving orphaned artifacts in the workspace. - deps, err := environmentDependencies(cfg, configPath) + deps, fileVersion, err := environmentDependencies(cfg, configPath) if err != nil { return 0, "", err } @@ -210,7 +212,12 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run } } - runtimeVersion, _ := cfg.runtimeVersion() + // Top-level environment.version wins; for file-form deps it is disallowed, so + // fall back to the version declared inside the requirements file. + runtimeVersion, ok := cfg.runtimeVersion() + if !ok { + runtimeVersion = fileVersion + } payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap, deps) payload.IdempotencyToken = token diff --git a/experimental/air/cmd/runsubmit_test.go b/experimental/air/cmd/runsubmit_test.go index 65d4685a31e..113bb8afa9a 100644 --- a/experimental/air/cmd/runsubmit_test.go +++ b/experimental/air/cmd/runsubmit_test.go @@ -123,56 +123,60 @@ func TestBuildSubmitPayloadInlineDependencies(t *testing.T) { } // TestEnvironmentDependencies covers how declared deps are resolved to a flat list: -// an inline list, a requirements file (path resolved against the config dir), none, -// and a missing file. +// an inline list (no file version), a requirements file (path resolved against the +// config dir, version read from the file), none, and a missing file. func TestEnvironmentDependencies(t *testing.T) { inline := &runConfig{Environment: &environmentConfig{ Dependencies: dependencies{set: true, isList: true, list: []string{"torch", "numpy"}}, }} - deps, err := environmentDependencies(inline, "run.yaml") + deps, version, err := environmentDependencies(inline, "run.yaml") require.NoError(t, err) assert.Equal(t, []string{"torch", "numpy"}, deps) + assert.Empty(t, version) dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("dependencies:\n - pandas\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: \"5\"\ndependencies:\n - pandas\n"), 0o600)) fromFile := &runConfig{Environment: &environmentConfig{ Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}, }} - deps, err = environmentDependencies(fromFile, filepath.Join(dir, "run.yaml")) + deps, version, err = environmentDependencies(fromFile, filepath.Join(dir, "run.yaml")) require.NoError(t, err) assert.Equal(t, []string{"pandas"}, deps) + assert.Equal(t, "5", version) - deps, err = environmentDependencies(&runConfig{}, "run.yaml") + deps, _, err = environmentDependencies(&runConfig{}, "run.yaml") require.NoError(t, err) assert.Nil(t, deps) missing := &runConfig{Environment: &environmentConfig{ Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}, }} - _, err = environmentDependencies(missing, filepath.Join(dir, "run.yaml")) + _, _, err = environmentDependencies(missing, filepath.Join(dir, "run.yaml")) require.ErrorContains(t, err, "failed to read requirements file") } // TestReadRequirementsDependencies covers reading a requirements file's dependency -// list, with a missing key yielding an empty list and a -r include rejected. +// list and version, with a missing key yielding an empty list and a -r include +// rejected. func TestReadRequirementsDependencies(t *testing.T) { dir := t.TempDir() reqPath := filepath.Join(dir, "requirements.yaml") - require.NoError(t, os.WriteFile(reqPath, []byte("dependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) - deps, err := readRequirementsDependencies(reqPath) + require.NoError(t, os.WriteFile(reqPath, []byte("version: \"5\"\ndependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) + deps, version, err := readRequirementsDependencies(reqPath) require.NoError(t, err) assert.Equal(t, []string{"torch==2.3.0", "numpy"}, deps) + assert.Equal(t, "5", version) emptyPath := filepath.Join(dir, "empty.yaml") require.NoError(t, os.WriteFile(emptyPath, []byte("version: \"5\"\n"), 0o600)) - deps, err = readRequirementsDependencies(emptyPath) + deps, _, err = readRequirementsDependencies(emptyPath) require.NoError(t, err) assert.Empty(t, deps) includePath := filepath.Join(dir, "include.yaml") require.NoError(t, os.WriteFile(includePath, []byte("dependencies:\n - -r other.txt\n"), 0o600)) - _, err = readRequirementsDependencies(includePath) + _, _, err = readRequirementsDependencies(includePath) require.ErrorContains(t, err, "requirements-file include") } diff --git a/experimental/air/cmd/runupload.go b/experimental/air/cmd/runupload.go index f86acdb386b..a6e68655d5d 100644 --- a/experimental/air/cmd/runupload.go +++ b/experimental/air/cmd/runupload.go @@ -50,33 +50,33 @@ type requirementsDoc struct { Dependencies []string `yaml:"dependencies"` } -// readRequirementsDependencies reads the dependencies list out of a +// readRequirementsDependencies reads the dependencies and version out of a // requirements.yaml file so file-form deps can be carried on the serverless -// environment's inline spec.dependencies. Returns an empty list when the file -// declares no dependencies. -func readRequirementsDependencies(reqPath string) ([]string, error) { +// environment's inline spec.dependencies and its version can select the runtime +// image. Returns an empty list when the file declares no dependencies. +func readRequirementsDependencies(reqPath string) ([]string, string, error) { data, err := os.ReadFile(reqPath) if err != nil { - return nil, fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) + return nil, "", fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) } var doc requirementsDoc if err := yaml.Unmarshal(data, &doc); err != nil { - return nil, fmt.Errorf("failed to parse requirements file %s: %w", reqPath, err) + return nil, "", fmt.Errorf("failed to parse requirements file %s: %w", reqPath, err) } for _, dep := range doc.Dependencies { if fields := strings.Fields(dep); len(fields) > 0 && (fields[0] == "-r" || fields[0] == "--requirement") { - return nil, fmt.Errorf("requirements file dependency %q uses a requirements-file include (-r/--requirement), which is not supported; list the dependencies directly instead", dep) + return nil, "", fmt.Errorf("requirements file dependency %q uses a requirements-file include (-r/--requirement), which is not supported; list the dependencies directly instead", dep) } } - return doc.Dependencies, nil + return doc.Dependencies, doc.Version, nil } // buildArtifacts assembles the files to upload for a run: the merged config, the // inline command as a script, and hyperparameters. configPath is the local YAML // path. // -// Dependencies are no longer uploaded as a requirements.yaml; they ride inline on -// the serverless environment's spec.dependencies instead (see buildSubmitPayload). +// Dependencies are not uploaded here; they ride inline on the serverless +// environment's spec.dependencies (see buildSubmitPayload). func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { // TODO(DABs): with no _bases_/overrides ported yet, the merged config is the // file as-is; once those land, upload the re-serialized merged YAML instead.