diff --git a/acceptance/experimental/air/run-submit-deps/.gitattributes b/acceptance/experimental/air/run-submit-deps/.gitattributes new file mode 100644 index 00000000000..0d9562f4bfe --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/.gitattributes @@ -0,0 +1,6 @@ +# These YAML files' contents are uploaded verbatim (as training_config.yaml and +# requirements.yaml), so their line endings must stay \n on every OS — a Windows +# \r would change the recorded payload. +run.yaml text eol=lf +run-file.yaml text eol=lf +reqs.yaml text eol=lf diff --git a/acceptance/experimental/air/run-submit-deps/databricks.yml b/acceptance/experimental/air/run-submit-deps/databricks.yml new file mode 100644 index 00000000000..a073ec04b4f --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: air-run-submit-deps diff --git a/acceptance/experimental/air/run-submit-deps/out.test.toml b/acceptance/experimental/air/run-submit-deps/out.test.toml new file mode 100644 index 00000000000..d6187dcb046 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/experimental/air/run-submit-deps/output.txt b/acceptance/experimental/air/run-submit-deps/output.txt new file mode 100644 index 00000000000..1178a2e5d1c --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/output.txt @@ -0,0 +1,108 @@ + +=== submit with inline dependencies +>>> [CLI] experimental air run -f run.yaml +Submitted run 555 +View at: [DATABRICKS_URL]/jobs/runs/555 + +=== only config + command are uploaded; no requirements.yaml +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\n"} + +=== declared deps ride on environments[].spec.dependencies +>>> print_requests.py //api/2.2/jobs/runs/submit +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/submit", + "body": { + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "numpy", + "torch==2.3.0" + ], + "environment_version": "5" + } + } + ], + "idempotency_token": "[UUID]", + "run_name": "deps-smoke", + "tasks": [ + { + "ai_runtime_task": { + "deployments": [ + { + "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", + "compute": { + "accelerator_count": 1, + "accelerator_type": "GPU_1xH100" + } + } + ], + "experiment": "deps-smoke" + }, + "environment_key": "default", + "max_retries": 3, + "retry_on_timeout": true, + "run_if": "ALL_SUCCESS", + "task_key": "deps-smoke" + } + ] + } +} + +=== file-form deps: version comes from the requirements file +>>> [CLI] experimental air run -f run-file.yaml +Submitted run 555 +View at: [DATABRICKS_URL]/jobs/runs/555 + +=== file-form deps: the requirements file is not uploaded either +>>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} +{"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-file-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\nenvironment:\n dependencies: ./reqs.yaml\n"} + +=== file-form deps ride on environments[].spec.dependencies +>>> print_requests.py //api/2.2/jobs/runs/submit +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/submit", + "body": { + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "numpy", + "torch==2.3.0" + ], + "environment_version": "5" + } + } + ], + "idempotency_token": "[UUID]", + "run_name": "deps-file-smoke", + "tasks": [ + { + "ai_runtime_task": { + "deployments": [ + { + "command_path": "/Workspace/Users/[USERNAME]/.air/cli_launch/deps-file-smoke/deps-file-smoke_[RUN_ID]/command.sh", + "compute": { + "accelerator_count": 1, + "accelerator_type": "GPU_1xH100" + } + } + ], + "experiment": "deps-file-smoke" + }, + "environment_key": "default", + "max_retries": 3, + "retry_on_timeout": true, + "run_if": "ALL_SUCCESS", + "task_key": "deps-file-smoke" + } + ] + } +} diff --git a/acceptance/experimental/air/run-submit-deps/reqs.yaml b/acceptance/experimental/air/run-submit-deps/reqs.yaml new file mode 100644 index 00000000000..da4b89534c2 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/reqs.yaml @@ -0,0 +1,4 @@ +version: 5 +dependencies: + - numpy + - torch==2.3.0 diff --git a/acceptance/experimental/air/run-submit-deps/run-file.yaml b/acceptance/experimental/air/run-submit-deps/run-file.yaml new file mode 100644 index 00000000000..3e5fe54b7b0 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/run-file.yaml @@ -0,0 +1,7 @@ +experiment_name: deps-file-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +environment: + dependencies: ./reqs.yaml diff --git a/acceptance/experimental/air/run-submit-deps/run.yaml b/acceptance/experimental/air/run-submit-deps/run.yaml new file mode 100644 index 00000000000..fbb7e8681b5 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/run.yaml @@ -0,0 +1,10 @@ +experiment_name: deps-smoke +command: python train.py +compute: + accelerator_type: GPU_1xH100 + num_accelerators: 1 +environment: + version: 5 + dependencies: + - numpy + - torch==2.3.0 diff --git a/acceptance/experimental/air/run-submit-deps/script b/acceptance/experimental/air/run-submit-deps/script new file mode 100644 index 00000000000..52035bbae65 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/script @@ -0,0 +1,17 @@ +title "submit with inline dependencies" +trace $CLI experimental air run -f run.yaml + +title "only config + command are uploaded; no requirements.yaml" +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep + +title "declared deps ride on environments[].spec.dependencies" +trace print_requests.py //api/2.2/jobs/runs/submit + +title "file-form deps: version comes from the requirements file" +trace $CLI experimental air run -f run-file.yaml + +title "file-form deps: the requirements file is not uploaded either" +trace print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep + +title "file-form deps ride on environments[].spec.dependencies" +trace print_requests.py //api/2.2/jobs/runs/submit diff --git a/acceptance/experimental/air/run-submit-deps/test.toml b/acceptance/experimental/air/run-submit-deps/test.toml new file mode 100644 index 00000000000..590c2918a25 --- /dev/null +++ b/acceptance/experimental/air/run-submit-deps/test.toml @@ -0,0 +1,23 @@ +# A non-dry-run submit with inline dependencies (no code_source): asserts the deps +# land on spec.dependencies and that requirements.yaml is not uploaded. +RecordRequests = true + +# The SDK probes host reachability with a HEAD request; stub it for determinism. +[[Server]] +Pattern = "HEAD /" +Response.Body = '' + +[[Server]] +Pattern = "POST /api/2.2/jobs/runs/submit" +Response.Body = ''' +{"run_id": 555} +''' + +# The per-run launch directory ends in _<16 hex>; the random suffix varies. +[[Repls]] +Old = 'deps-smoke_[0-9a-f]{16}' +New = 'deps-smoke_[RUN_ID]' + +[[Repls]] +Old = 'deps-file-smoke_[0-9a-f]{16}' +New = 'deps-file-smoke_[RUN_ID]' diff --git a/experimental/air/cmd/runsubmit.go b/experimental/air/cmd/runsubmit.go index 8c0be55260d..1c0b8bdef6e 100644 --- a/experimental/air/cmd/runsubmit.go +++ b/experimental/air/cmd/runsubmit.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "path/filepath" "strconv" "strings" @@ -38,8 +39,29 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string { return strings.TrimPrefix(img, "CLIENT-GPU-") } +// environmentDependencies resolves the user's declared dependencies as a flat +// list to carry inline on the serverless environment's spec.dependencies: the +// inline list directly, or the dependencies read from a requirements file +// (resolved against the config's directory). For file-form deps it also returns +// the version declared inside that file, which selects the runtime image since +// top-level environment.version is not allowed there. Returns nil when none are +// declared. +func environmentDependencies(cfg *runConfig, configPath string) (deps []string, fileVersion string, err error) { + if deps, ok := cfg.inlineDependencies(); ok { + return deps, "", nil + } + if reqPath, ok := cfg.requirementsFile(); ok { + if !filepath.IsAbs(reqPath) { + reqPath = filepath.Join(filepath.Dir(configPath), reqPath) + } + return readRequirementsDependencies(reqPath) + } + return nil, "", nil +} + // buildSubmitPayload assembles the runs/submit payload. commandPath is the -// workspace path of the uploaded command.sh; dlImage is the runtime channel. +// workspace path of the uploaded command.sh; dlImage is the runtime channel; +// deps is the user's declared dependencies (nil when none are declared). // // max_retries is always sent (including 0) so the user's YAML value is honored: // setting it to 0 explicitly disables retries rather than falling back to the @@ -47,7 +69,7 @@ func dlRuntimeImage(ctx context.Context, runtimeVersion string) string { // omitempty so the wire form matches the Python CLI (which never emits a bare // "false"). Jobs performs the retries — each attempt is a fresh AI Runtime // workload. -func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapshotResult) jobs.SubmitRun { +func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapshotResult, deps []string) jobs.SubmitRun { task := jobs.AiRuntimeTask{ Experiment: cfg.ExperimentName, Deployments: []jobs.DeploymentSpec{{ @@ -89,13 +111,21 @@ func buildSubmitPayload(cfg *runConfig, commandPath, dlImage string, snap snapsh ForceSendFields: []string{"MaxRetries"}, } + // Carry the user's declared deps inline on spec.dependencies; the AI Runtime + // backend installs them via --deps-config. The SDK marshaler drops nil and empty + // slices, so a no-deps run omits the key. + envSpec := &compute.Environment{EnvironmentVersion: dlImage} + if len(deps) > 0 { + envSpec.Dependencies = deps + } + return jobs.SubmitRun{ RunName: cfg.ExperimentName, TimeoutSeconds: cfg.timeoutSeconds(), Tasks: []jobs.SubmitTask{st}, Environments: []jobs.JobEnvironment{{ EnvironmentKey: aiRuntimeEnvironmentKey, - Spec: &compute.Environment{EnvironmentVersion: dlImage}, + Spec: envSpec, }}, } } @@ -133,6 +163,13 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } + // Resolve dependencies before any upload too, so a bad requirements file fails + // fast without leaving orphaned artifacts in the workspace. + deps, fileVersion, err := environmentDependencies(cfg, configPath) + if err != nil { + return 0, "", err + } + experimentDir := "" if cfg.MLflowExperimentDirectory != nil { experimentDir = *cfg.MLflowExperimentDirectory @@ -175,8 +212,13 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run } } - runtimeVersion, _ := cfg.runtimeVersion() - payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap) + // Top-level environment.version wins; for file-form deps it is disallowed, so + // fall back to the version declared inside the requirements file. + runtimeVersion, ok := cfg.runtimeVersion() + if !ok { + runtimeVersion = fileVersion + } + payload := buildSubmitPayload(cfg, path.Join(funcDir, commandScriptName), dlRuntimeImage(ctx, runtimeVersion), snap, deps) payload.IdempotencyToken = token // Submit returns as soon as the run is created; we don't wait for it to finish. diff --git a/experimental/air/cmd/runsubmit_test.go b/experimental/air/cmd/runsubmit_test.go index fd5103599df..113bb8afa9a 100644 --- a/experimental/air/cmd/runsubmit_test.go +++ b/experimental/air/cmd/runsubmit_test.go @@ -2,6 +2,7 @@ package aircmd import ( "encoding/json" + "os" "path/filepath" "strings" "testing" @@ -37,7 +38,7 @@ func TestBuildSubmitPayload(t *testing.T) { MLflowExperimentDirectory: new("/Workspace/Users/me/exp"), } - p := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}) + p := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, nil) assert.Equal(t, "exp", p.RunName) assert.Equal(t, 1800, p.TimeoutSeconds) @@ -72,7 +73,7 @@ func TestBuildSubmitPayloadDefaultRetries(t *testing.T) { Command: new("x"), Compute: &computeConfig{AcceleratorType: "GPU_1xH100", NumAccelerators: 1}, } - task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}).Tasks[0] + task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}, nil).Tasks[0] assert.Equal(t, defaultMaxRetries, task.MaxRetries) assert.True(t, task.RetryOnTimeout) } @@ -87,7 +88,7 @@ func TestBuildSubmitPayloadNoRetries(t *testing.T) { Compute: &computeConfig{AcceleratorType: "GPU_1xH100", NumAccelerators: 1}, MaxRetries: new(0), } - task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}).Tasks[0] + task := buildSubmitPayload(cfg, "/d/command.sh", "4", snapshotResult{}, nil).Tasks[0] assert.Equal(t, 0, task.MaxRetries) assert.False(t, task.RetryOnTimeout) @@ -97,6 +98,88 @@ func TestBuildSubmitPayloadNoRetries(t *testing.T) { assert.NotContains(t, string(b), "retry_on_timeout") } +// TestBuildSubmitPayloadInlineDependencies covers how deps land on the environment +// spec: a non-empty list is set alongside the runtime channel; empty and nil omit +// the key so the payload is unchanged. +func TestBuildSubmitPayloadInlineDependencies(t *testing.T) { + cfg := &runConfig{ + ExperimentName: "exp", + Command: new("x"), + Compute: &computeConfig{AcceleratorType: "GPU_8xH100", NumAccelerators: 8}, + } + + deps := []string{"torch==2.3.0", "--extra-index-url https://internal/pypi", "numpy"} + spec := buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, deps).Environments[0].Spec + assert.Equal(t, deps, spec.Dependencies) + assert.Equal(t, "5", spec.EnvironmentVersion) + + // The SDK marshaler drops empty/nil slices, so no "dependencies" key is emitted. + for _, empty := range [][]string{{}, nil} { + spec = buildSubmitPayload(cfg, "/d/command.sh", "5", snapshotResult{}, empty).Environments[0].Spec + b, err := json.Marshal(spec) + require.NoError(t, err) + assert.NotContains(t, string(b), "dependencies") + } +} + +// TestEnvironmentDependencies covers how declared deps are resolved to a flat list: +// an inline list (no file version), a requirements file (path resolved against the +// config dir, version read from the file), none, and a missing file. +func TestEnvironmentDependencies(t *testing.T) { + inline := &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: true, list: []string{"torch", "numpy"}}, + }} + deps, version, err := environmentDependencies(inline, "run.yaml") + require.NoError(t, err) + assert.Equal(t, []string{"torch", "numpy"}, deps) + assert.Empty(t, version) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: \"5\"\ndependencies:\n - pandas\n"), 0o600)) + fromFile := &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}, + }} + deps, version, err = environmentDependencies(fromFile, filepath.Join(dir, "run.yaml")) + require.NoError(t, err) + assert.Equal(t, []string{"pandas"}, deps) + assert.Equal(t, "5", version) + + deps, _, err = environmentDependencies(&runConfig{}, "run.yaml") + require.NoError(t, err) + assert.Nil(t, deps) + + missing := &runConfig{Environment: &environmentConfig{ + Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}, + }} + _, _, err = environmentDependencies(missing, filepath.Join(dir, "run.yaml")) + require.ErrorContains(t, err, "failed to read requirements file") +} + +// TestReadRequirementsDependencies covers reading a requirements file's dependency +// list and version, with a missing key yielding an empty list and a -r include +// rejected. +func TestReadRequirementsDependencies(t *testing.T) { + dir := t.TempDir() + + reqPath := filepath.Join(dir, "requirements.yaml") + require.NoError(t, os.WriteFile(reqPath, []byte("version: \"5\"\ndependencies:\n - torch==2.3.0\n - numpy\n"), 0o600)) + deps, version, err := readRequirementsDependencies(reqPath) + require.NoError(t, err) + assert.Equal(t, []string{"torch==2.3.0", "numpy"}, deps) + assert.Equal(t, "5", version) + + emptyPath := filepath.Join(dir, "empty.yaml") + require.NoError(t, os.WriteFile(emptyPath, []byte("version: \"5\"\n"), 0o600)) + deps, _, err = readRequirementsDependencies(emptyPath) + require.NoError(t, err) + assert.Empty(t, deps) + + includePath := filepath.Join(dir, "include.yaml") + require.NoError(t, os.WriteFile(includePath, []byte("dependencies:\n - -r other.txt\n"), 0o600)) + _, _, err = readRequirementsDependencies(includePath) + require.ErrorContains(t, err, "requirements-file include") +} + func TestSubmitToken(t *testing.T) { cfg := &runConfig{IdempotencyToken: new("from-config")} @@ -213,4 +296,23 @@ func TestSubmitWorkloadGuards(t *testing.T) { _, _, err := submitWorkload(t.Context(), w, &cfg, cfgPath, "") require.ErrorContains(t, err, "usage_policy_name is not yet supported") }) + + t.Run("bad requirements file fails before any upload", func(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var uploaded bool + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path...}", func(testserver.Request) any { + uploaded = true + return nil + }) + testserver.AddDefaultHandlers(server) + tw, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + cfg := *base + cfg.Environment = &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "missing.yaml"}} + _, _, err = submitWorkload(t.Context(), tw, &cfg, cfgPath, "") + require.ErrorContains(t, err, "failed to read requirements file") + assert.False(t, uploaded, "no artifacts should be uploaded when dependency resolution fails") + }) } diff --git a/experimental/air/cmd/runupload.go b/experimental/air/cmd/runupload.go index fb9ca00b987..a6e68655d5d 100644 --- a/experimental/air/cmd/runupload.go +++ b/experimental/air/cmd/runupload.go @@ -8,7 +8,6 @@ import ( "io" "maps" "os" - "path/filepath" "slices" "strings" @@ -22,7 +21,6 @@ import ( const ( trainingConfigName = "training_config.yaml" commandScriptName = "command.sh" - requirementsName = "requirements.yaml" hyperparametersName = "hyperparameters.yaml" envVarsName = "env_vars.json" secretEnvVarsName = "secret_env_vars.json" @@ -45,16 +43,40 @@ type fileWriter interface { Write(ctx context.Context, name string, reader io.Reader, mode ...filer.WriteMode) error } -// requirementsDoc mirrors the on-disk requirements.yaml format so the worker -// parses synthesized inline dependencies identically to a user-provided file. +// requirementsDoc mirrors the on-disk requirements.yaml format, used to read a +// user-provided requirements file's dependencies for the inline spec.dependencies. type requirementsDoc struct { Version string `yaml:"version,omitempty"` Dependencies []string `yaml:"dependencies"` } +// readRequirementsDependencies reads the dependencies and version out of a +// requirements.yaml file so file-form deps can be carried on the serverless +// environment's inline spec.dependencies and its version can select the runtime +// image. Returns an empty list when the file declares no dependencies. +func readRequirementsDependencies(reqPath string) ([]string, string, error) { + data, err := os.ReadFile(reqPath) + if err != nil { + return nil, "", fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) + } + var doc requirementsDoc + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, "", fmt.Errorf("failed to parse requirements file %s: %w", reqPath, err) + } + for _, dep := range doc.Dependencies { + if fields := strings.Fields(dep); len(fields) > 0 && (fields[0] == "-r" || fields[0] == "--requirement") { + return nil, "", fmt.Errorf("requirements file dependency %q uses a requirements-file include (-r/--requirement), which is not supported; list the dependencies directly instead", dep) + } + } + return doc.Dependencies, doc.Version, nil +} + // buildArtifacts assembles the files to upload for a run: the merged config, the -// inline command as a script, requirements (from a file or synthesized from -// inline dependencies), and hyperparameters. configPath is the local YAML path. +// inline command as a script, and hyperparameters. configPath is the local YAML +// path. +// +// Dependencies are not uploaded here; they ride inline on the serverless +// environment's spec.dependencies (see buildSubmitPayload). func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { // TODO(DABs): with no _bases_/overrides ported yet, the merged config is the // file as-is; once those land, upload the re-serialized merged YAML instead. @@ -72,28 +94,6 @@ func buildArtifacts(cfg *runConfig, configPath string) ([]uploadItem, error) { {commandScriptName, []byte(*cfg.Command)}, } - switch reqPath, ok := cfg.requirementsFile(); { - case ok: - // Resolve a relative requirements path against the config's directory. - if !filepath.IsAbs(reqPath) { - reqPath = filepath.Join(filepath.Dir(configPath), reqPath) - } - data, err := os.ReadFile(reqPath) - if err != nil { - return nil, fmt.Errorf("failed to read requirements file %s: %w", reqPath, err) - } - items = append(items, uploadItem{requirementsName, data}) - default: - if deps, ok := cfg.inlineDependencies(); ok { - version, _ := cfg.runtimeVersion() - data, err := yaml.Marshal(requirementsDoc{Version: version, Dependencies: deps}) - if err != nil { - return nil, fmt.Errorf("failed to synthesize requirements.yaml: %w", err) - } - items = append(items, uploadItem{requirementsName, data}) - } - } - if len(cfg.Parameters) > 0 { data, err := yaml.Marshal(cfg.Parameters) if err != nil { diff --git a/experimental/air/cmd/runupload_test.go b/experimental/air/cmd/runupload_test.go index 0c87524735d..e57a8aa2c41 100644 --- a/experimental/air/cmd/runupload_test.go +++ b/experimental/air/cmd/runupload_test.go @@ -57,7 +57,7 @@ func TestBuildArtifacts_CommandAndConfig(t *testing.T) { assert.Equal(t, "python train.py", string(items[1].data)) } -func TestBuildArtifacts_InlineRequirementsAndParameters(t *testing.T) { +func TestBuildArtifacts_ParametersButNoRequirements(t *testing.T) { path := writeConfigFile(t, "run.yaml", "x: y\n") cfg := &runConfig{ Command: new("echo hi"), @@ -68,19 +68,10 @@ func TestBuildArtifacts_InlineRequirementsAndParameters(t *testing.T) { Parameters: map[string]any{"lr": 0.1}, } + // Inline deps are not uploaded, so the artifacts are config, command, and params. items, err := buildArtifacts(cfg, path) require.NoError(t, err) - assert.Equal(t, []string{trainingConfigName, commandScriptName, requirementsName, hyperparametersName}, itemNames(items)) - - var reqIdx int - for i, it := range items { - if it.name == requirementsName { - reqIdx = i - } - } - req := string(items[reqIdx].data) - assert.Contains(t, req, "version: \"5\"") - assert.Contains(t, req, "- torch") + assert.Equal(t, []string{trainingConfigName, commandScriptName, hyperparametersName}, itemNames(items)) } func TestBuildArtifacts_EnvVarsAndSecrets(t *testing.T) { @@ -103,7 +94,7 @@ func TestBuildArtifacts_EnvVarsAndSecrets(t *testing.T) { assert.JSONEq(t, `[{"name":"HF_TOKEN","secret_scope":"myscope","secret_key":"hf"}]`, string(byName[secretEnvVarsName])) } -func TestBuildArtifacts_RequirementsFile(t *testing.T) { +func TestBuildArtifacts_RequirementsFileNotUploaded(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "run.yaml"), []byte("x: y\n"), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(dir, "reqs.yaml"), []byte("version: 4\n"), 0o600)) @@ -112,9 +103,10 @@ func TestBuildArtifacts_RequirementsFile(t *testing.T) { Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "reqs.yaml"}}, } + // A declared requirements file is not uploaded; its deps travel on spec.dependencies. items, err := buildArtifacts(cfg, filepath.Join(dir, "run.yaml")) require.NoError(t, err) - assert.Contains(t, itemNames(items), requirementsName) + assert.Equal(t, []string{trainingConfigName, commandScriptName}, itemNames(items)) } func TestBuildArtifacts_OversizeConfigRejected(t *testing.T) { @@ -143,13 +135,3 @@ func TestUploadArtifacts_WriteError(t *testing.T) { err := uploadArtifacts(t.Context(), errWriter{}, []uploadItem{{trainingConfigName, []byte("x")}}) require.ErrorContains(t, err, "failed to upload "+trainingConfigName) } - -func TestBuildArtifacts_MissingRequirementsFile(t *testing.T) { - cfgPath := writeConfigFile(t, "run.yaml", "x: y\n") - cfg := &runConfig{ - Command: new("echo hi"), - Environment: &environmentConfig{Dependencies: dependencies{set: true, isList: false, path: "nope.yaml"}}, - } - _, err := buildArtifacts(cfg, cfgPath) - require.ErrorContains(t, err, "failed to read requirements file") -}