diff --git a/cmd/ob/job.go b/cmd/ob/job.go index 4196f141..4a7afef8 100644 --- a/cmd/ob/job.go +++ b/cmd/ob/job.go @@ -34,18 +34,20 @@ func addJobCommand(root *cobra.Command, g *globalFlags) { plan.Flags().StringVar(&backupReportOut, "backup-report-out", "", "write a plan-bound backup report template when migration backup is required") var planPath, approvalPath, backupReportPath, overrideReason string - var breakLock bool + var breakLock, detach bool run := &cobra.Command{ Use: "run [id]", Short: "run one manual job from an inline or saved sealed plan", - Long: "Run one manual job through the canonical lock, fence, local-confirmation and journal boundary.\n\nHumans may pass an id and confirm interactively. Automation should supply a\nsaved --plan and its separately created local-confirmation artifact through\n--approval; migration plans may also require the exact plan-bound --backup-report.", - Args: cobra.MaximumNArgs(1), + Long: "Run one manual job through the canonical lock, fence, local-confirmation and journal boundary.\n\n" + + "A job with schedule configured runs under its installed systemd unit and is\nfollowed by default; Ctrl-C stops following, not the host job. --detach returns\nafter that unit accepts the run. Unscheduled and migration jobs stay attached.\n\n" + + "Humans may pass an id and confirm interactively. Automation should supply a\nsaved --plan and its separately created local-confirmation artifact through\n--approval; migration plans may also require the exact plan-bound --backup-report.", + Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := "" if len(args) == 1 { jobID = args[0] } - return runJob(cmd, g, jobID, planPath, approvalPath, backupReportPath, overrideReason, breakLock) + return runJob(cmd, g, jobID, planPath, approvalPath, backupReportPath, overrideReason, breakLock, detach) }, } run.Flags().StringVar(&planPath, "plan", "", "apply a saved job plan artifact") @@ -53,6 +55,7 @@ func addJobCommand(root *cobra.Command, g *globalFlags) { run.Flags().StringVar(&backupReportPath, "backup-report", "", "apply the backup report bound into the local confirmation") run.Flags().StringVar(&overrideReason, "override-migration-backup", "", "audited break-glass reason (requires --approval)") run.Flags().BoolVar(&breakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") + run.Flags().BoolVar(&detach, "detach", false, "return after the installed host unit accepts the job") group.AddCommand(plan, run) root.AddCommand(group) @@ -163,7 +166,7 @@ func renderJobPlan(cmd *cobra.Command, plan *onebox.JobPlan) { } } -func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, backupReportPath, overrideReason string, breakLock bool) error { +func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, backupReportPath, overrideReason string, breakLock, detach bool) error { if planPath != "" && jobID != "" { return writeEarlyOperationFailure(cmd, g, errors.New("supply either a job id or --plan, not both")) } @@ -217,7 +220,7 @@ func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, b } return runMutation(cmd, g, onebox.ExecuteRequest{ Kind: onebox.KindJobRun, JobPlan: plan, Approval: approval, BreakLock: breakLock, - BackupReport: backupReport, MigrationBackupOverride: override, + BackupReport: backupReport, MigrationBackupOverride: override, Detach: detach, }, "job run") } @@ -245,7 +248,7 @@ func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, b } approval = &grant } - return runMutation(cmd, g, onebox.ExecuteRequest{Kind: onebox.KindJobRun, JobPlan: &plan, Approval: approval, BreakLock: breakLock}, "job run") + return runMutation(cmd, g, onebox.ExecuteRequest{Kind: onebox.KindJobRun, JobPlan: &plan, Approval: approval, BreakLock: breakLock, Detach: detach}, "job run") } func loadJobMigrationOverride( diff --git a/e2e/server_test.go b/e2e/server_test.go index 86375106..3a2522a8 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -401,6 +401,21 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() if err != nil { t.Fatalf("job run failed: %v\n%s", err, out) } + for _, want := range []string{"host run ", "⟳ job chore", "✓ job chore", "ob schedule history chore"} { + if !strings.Contains(out, want) { + t.Fatalf("host-owned job output is missing %q:\n%s", want, out) + } + } + + detachedPlan := filepath.Join(dir, "ob-detached-job-plan.json") + s.mustOb(t, dir, "job", "plan", "chore", "-o", detachedPlan) + out, err = s.obInput(t, dir, s.obHome(t), "y\n", "job", "run", "--plan", detachedPlan, "--detach") + if err != nil { + t.Fatalf("detached job run failed: %v\n%s", err, out) + } + if !strings.Contains(out, "job chore accepted as ") || !strings.Contains(out, "ob schedule history chore") { + t.Fatalf("detached job did not report durable acceptance:\n%s", out) + } }) t.Run("doctor", func(t *testing.T) { diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 54e8523b..fe39b6f8 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -265,6 +265,7 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na "trap 'exit 143' 15", ) lines = append(lines, scheduleRunPreamble(triggerUnit)...) + lines = append(lines, schedulePlannedBindingLines()...) if job.DataEffect != app.DataEffectNone { lines = append(lines, invalidateExecutionCommand(names.AppDir())) } @@ -310,6 +311,7 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names "trap 'exit 143' 15", ) lines = append(lines, scheduleRunPreamble(triggerUnit)...) + lines = append(lines, schedulePlannedBindingLines()...) lines = append(lines, scheduleAttemptLoop(job, compose, container)...) lines = append(lines, "") return strings.Join(lines, "\n") @@ -518,6 +520,8 @@ func scheduleInputsLines(inputsPath string) []string { return []string{ "operation=''", "execution=''", + "expected_release=''", + "expected_runtime=''", "inputs_json=''", "inputs_file=" + q(inputsPath), "if [ -z \"${TRIGGER_UNIT:-}\" ] && [ -f \"$inputs_file\" ]; then", @@ -525,6 +529,8 @@ func scheduleInputsLines(inputsPath string) []string { " case \"$line\" in", " ONEBOX_OPERATION=*) operation=${line#ONEBOX_OPERATION=} ;;", " ONEBOX_EXECUTION=*) execution=${line#ONEBOX_EXECUTION=} ;;", + " ONEBOX_EXPECTED_RELEASE=*) expected_release=${line#ONEBOX_EXPECTED_RELEASE=} ;;", + " ONEBOX_EXPECTED_RUNTIME=*) expected_runtime=${line#ONEBOX_EXPECTED_RUNTIME=} ;;", " [A-Z]*=*) set -- \"$@\" -e \"$line\"; key=${line%%=*}; value=${line#*=}; inputs_json=\"${inputs_json:+$inputs_json,}\\\"$key\\\":\\\"$value\\\"\" ;;", " esac", " done <\"$inputs_file\"", @@ -533,6 +539,24 @@ func scheduleInputsLines(inputsPath string) []string { } } +// schedulePlannedBindingLines makes a sealed manual job plan authoritative at +// the point that owns execution: after the host runner has acquired its locks, +// immediately before it can start the container. Timer firings carry no +// expected binding and pass through unchanged. +const sealedManualJobBindingMarker = "Sealed manual job binding protocol v1." + +func schedulePlannedBindingLines() []string { + return []string{ + "# " + sealedManualJobBindingMarker, + "if [ -n \"$expected_release\" ] && [ \"$release\" != \"$expected_release\" ]; then write_state 0; echo 'onebox: serving release changed after job approval' >&2; exit 74; fi", + "if [ -n \"$expected_runtime\" ]; then", + " runtime_hash=$(sha256sum \"$release_dir/compose.yaml\") || { write_state 0; echo 'onebox: cannot hash the approved job runtime' >&2; exit 74; }", + " runtime_digest=sha256:${runtime_hash%% *}", + " if [ \"$runtime_digest\" != \"$expected_runtime\" ]; then write_state 0; echo 'onebox: serving runtime changed after job approval' >&2; exit 74; fi", + "fi", + } +} + // systemdVersion reads the leading number from `systemd 255 (255.4-1ubuntu8)`. func systemdVersion(firstLine string) (int, bool) { fields := strings.Fields(firstLine) diff --git a/internal/engine/schedule_execution.go b/internal/engine/schedule_execution.go index f28c0737..359dbd6e 100644 --- a/internal/engine/schedule_execution.go +++ b/internal/engine/schedule_execution.go @@ -199,7 +199,7 @@ func (e *Engine) ExecutionResume(ctx context.Context, operation, id string, wait return ScheduleRunResult{}, fmt.Errorf("resume requires the original current release") } // Host runner rechecks the saved definition and compatibility under locks. - return e.scheduleRun(ctx, operation, job, nil, wait, id) + return e.scheduleRun(ctx, operation, job, nil, wait, id, nil) } func (e *Engine) ExecutionAbandon(ctx context.Context, operation, id string) (err error) { diff --git a/internal/engine/schedule_execution_test.go b/internal/engine/schedule_execution_test.go index 67fcbf15..344da6f3 100644 --- a/internal/engine/schedule_execution_test.go +++ b/internal/engine/schedule_execution_test.go @@ -167,7 +167,7 @@ func TestDurableResumeRefusesLegacyRunnerBeforePublishingRequest(t *testing.T) { return base(command) } e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) - _, err := e.scheduleRun(context.Background(), "resume-op", "refresh", nil, false, strings.Repeat("a", 32)) + _, err := e.scheduleRun(context.Background(), "resume-op", "refresh", nil, false, strings.Repeat("a", 32), nil) if err == nil || !strings.Contains(err.Error(), "ob schedule apply") { t.Fatalf("legacy runner accepted: %v", err) } diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index 42542284..534a80f0 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -35,10 +35,29 @@ type ScheduleRunResult struct { // the inputs is the case the sealed plan of `ob job run` exists for, and a // migration or destructive job keeps that path. func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool) (_ ScheduleRunResult, err error) { - return e.scheduleRun(ctx, operationID, name, inputs, wait, "") + return e.scheduleRun(ctx, operationID, name, inputs, wait, "", nil) } -func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool, execution string) (_ ScheduleRunResult, err error) { +// PlannedJobRun submits a sealed manual job plan to the job's installed +// systemd unit. The runner checks the plan's release and runtime digest after +// taking the host-side application exclusion, so releasing the admission lock +// before systemd schedules the unit cannot move the job onto different bytes. +func (e *Engine) PlannedJobRun(ctx context.Context, operationID, name, expectedRelease, expectedRuntime string, wait bool) (_ ScheduleRunResult, err error) { + if expectedRelease == "" || expectedRuntime == "" { + return ScheduleRunResult{}, errors.New("planned job run requires an expected release and runtime digest") + } + return e.scheduleRun(ctx, operationID, name, nil, wait, "", &plannedJobBinding{ + release: expectedRelease, + runtime: expectedRuntime, + }) +} + +type plannedJobBinding struct { + release string + runtime string +} + +func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inputs map[string]string, wait bool, execution string, planned *plannedJobBinding) (_ ScheduleRunResult, err error) { result := ScheduleRunResult{Job: name, Operation: operationID, Inputs: inputs} result.Execution = execution if strings.TrimSpace(operationID) == "" { @@ -54,10 +73,13 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu if execution != "" && (workload.Execution == nil || !scheduleRunID.MatchString(execution) || len(inputs) != 0) { return result, errors.New("resume requires a durable job, a valid execution ID, and no input overrides") } - if workload.DataEffect != app.DataEffectNone { + if workload.DataEffect != app.DataEffectNone && planned == nil { return result, fmt.Errorf("job %s declares data_effect %q; operator-initiated runs of it go through the sealed plan: ob job plan %s, then ob job run", name, workload.DataEffect, name) } + if planned != nil && execution != "" { + return result, errors.New("a planned job run cannot resume a durable execution") + } if err := app.ValidateJobInputValues(workload, inputs); err != nil { return result, err } @@ -70,6 +92,15 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu } unit := e.names().ScheduledJobUnit(name) result.Unit = unit + if planned != nil { + res, err := e.T.Run(ctx, "grep -Fq "+q(sealedManualJobBindingMarker)+" "+q("/etc/systemd/system/"+unit+".run")) + if err != nil { + return result, err + } + if res.ExitCode != 0 { + return result, fmt.Errorf("installed job runner does not support sealed manual-job binding; run `ob schedule apply` before running %s", name) + } + } // Starting an active unit is a no-op to systemd and would consume the // inputs file for a run that never happens; say so instead. @@ -113,7 +144,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu path := e.names().ScheduledJobRunInputs(name) create := "if [ -e " + q(path) + " ]; then exit 73; fi; " + "umask 077 && install -d -m 700 " + q(e.names().AppDir()+"/schedule") + " && set -C && cat > " + q(path) - payload := scheduleInputsFile(operationID, inputs) + payload := scheduleInputsFile(operationID, inputs, planned) if execution != "" { payload += "ONEBOX_EXECUTION=" + execution + "\n" } @@ -139,6 +170,8 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu writer := &journal.Writer{ T: e.T, Names: e.names(), DeployID: operationID, Epoch: epoch, Operator: journal.DefaultOperator(), GitSHA: e.Opts.GitSHA, ConfigHash: e.Opts.ConfigHash, Runner: &e.Opts.Runner, + ApprovalDigest: e.Opts.ApprovalDigest, ApprovalClass: e.Opts.ApprovalClass, + ApprovedBy: e.Opts.ApprovedBy, ApprovalSource: e.Opts.ApprovalSource, } detail := "inputs: defaults" if execution != "" { @@ -148,6 +181,15 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu detail = "inputs: " + scheduleInputsDetail(inputs) } record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + if planned != nil { + // This journal describes host-unit admission, not the job's eventual + // outcome. Keep schedule-run's truthful "started" audit semantics while + // retaining the sealed job plan's authorization evidence. + record.ApprovalDigest = e.Opts.ApprovalDigest + record.ApprovalClass = e.Opts.ApprovalClass + record.ApprovedBy = e.Opts.ApprovedBy + record.ApprovalSource = e.Opts.ApprovalSource + } if err := writer.Append(ctx, record); err != nil { return result, fmt.Errorf("journal schedule run start: %w", err) } @@ -176,6 +218,14 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu if wait { start = "systemctl start " + q(unit+".service") } + finishStep := func(error) {} + if wait { + if planned != nil { + e.ui.Infof("host run %s; Ctrl-C detaches; inspect with `ob schedule history %s`", operationID, name) + } + finishStep = e.ui.Step("job "+name, true) + defer func() { finishStep(err) }() + } res, err = e.mutate(ctx, start) if err != nil { return result, err @@ -187,7 +237,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu // The unit is queued, so the runner owns the file now. pending = false result.Started = true - e.logf("schedule: %s started as %s; ob schedule history %s shows the outcome", name, operationID, name) + e.ui.Successf("job %s accepted as %s; inspect with `ob schedule history %s`", name, operationID, name) return result, nil } // A blocking start that exits non-zero may mean the job failed, which is @@ -263,8 +313,14 @@ func (e *Engine) discardInputs(ctx context.Context, path string) { // id on its reserved line, then one declared override per line. Values were // validated against a charset that has no newline or quote, so the format // needs no escaping. -func scheduleInputsFile(operationID string, inputs map[string]string) string { +func scheduleInputsFile(operationID string, inputs map[string]string, planned *plannedJobBinding) string { lines := []string{app.ReservedInputPrefix + "OPERATION=" + operationID} + if planned != nil { + lines = append(lines, + app.ReservedInputPrefix+"EXPECTED_RELEASE="+planned.release, + app.ReservedInputPrefix+"EXPECTED_RUNTIME="+planned.runtime, + ) + } for _, name := range sortedInputNames(inputs) { lines = append(lines, name+"="+inputs[name]) } diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 9a7d0ae1..5fa9dd27 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -72,6 +72,68 @@ func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testi } } +func TestPlannedJobRunStagesItsExactBindingAndDetachesToSystemd(t *testing.T) { + cfg := testConfig() + cfg.Workloads["refresh"] = app.Workload{ + Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Schedule: &app.JobSchedule{Cron: "0 4 * * 1", Timezone: "UTC", Timeout: "8h"}, + } + f := happyFake() + base := f.Dynamic + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + e := New(cfg, testProject(t), f, Options{ + Out: &bytes.Buffer{}, Sleep: noSleep, + ApprovalDigest: "approval-digest", ApprovalClass: "strong", + ApprovedBy: "operator", ApprovalSource: "local_confirmation", + }) + const ( + operation = "20260914-191943-job_run-9503bc4cfa47" + release = "20260914-190602-deploy-731e31b2d992" + runtime = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ) + result, err := e.PlannedJobRun(context.Background(), operation, "refresh", release, runtime, false) + if err != nil { + t.Fatalf("planned job run: %v\n%s", err, strings.Join(f.Commands, "\n")) + } + if !result.Started || result.Operation != operation || result.Unit != "ob-sample-refresh" { + t.Fatalf("result = %#v", result) + } + written := strings.Join(f.Inputs, "\n") + for _, want := range []string{ + "ONEBOX_OPERATION=" + operation, + "ONEBOX_EXPECTED_RELEASE=" + release, + "ONEBOX_EXPECTED_RUNTIME=" + runtime, + } { + if !strings.Contains(written, want) { + t.Fatalf("planned activation omitted %q:\n%s", want, written) + } + } + commands := strings.Join(f.Commands, "\n") + if !strings.Contains(commands, "grep -Fq '"+sealedManualJobBindingMarker+"' '/etc/systemd/system/ob-sample-refresh.run'") { + t.Fatalf("planned job did not verify the installed runner protocol:\n%s", commands) + } + if !strings.Contains(commands, "systemctl start --no-block 'ob-sample-refresh.service'") { + t.Fatalf("planned job was not detached to systemd:\n%s", commands) + } + for _, want := range []string{`"approval_digest":"approval-digest"`, `"approval_class":"strong"`} { + if !strings.Contains(commands, want) { + t.Fatalf("planned job journal omitted %q:\n%s", want, commands) + } + } +} + func TestScheduleRunRefusals(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index da69f9e4..68e3d0dc 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -201,12 +201,24 @@ func TestScheduledJobUnitContract(t *testing.T) { "compose.yaml", `run --rm --no-deps "$@" --name 'sample-nightly-1'`, "docker rm -f 'sample-nightly-1'", + "ONEBOX_EXPECTED_RELEASE", + "ONEBOX_EXPECTED_RUNTIME", + sealedManualJobBindingMarker, + "sha256sum", + "serving release changed after job approval", + "serving runtime changed after job approval", "nightly", } { if !strings.Contains(runner, want) { t.Errorf("runner is missing %q:\n%s", want, runner) } } + locked := strings.Index(runner, "flock --exclusive --timeout 10") + bound := strings.Index(runner, "serving release changed after job approval") + run := strings.Index(runner, "docker compose") + if locked < 0 || bound < 0 || run < 0 || !(locked < bound && bound < run) { + t.Fatalf("planned binding must be checked under the app lock before the container runs:\n%s", runner) + } command := exec.CommandContext(context.Background(), "sh", "-n") command.Stdin = strings.NewReader(runner) if output, err := command.CombinedOutput(); err != nil { diff --git a/internal/onebox/execute.go b/internal/onebox/execute.go index 94d11445..d0e54b6e 100644 --- a/internal/onebox/execute.go +++ b/internal/onebox/execute.go @@ -117,9 +117,10 @@ func (s *Service) Execute(ctx context.Context, request ExecuteRequest) (Operatio } result.ReleaseID = request.JobPlan.Operation.ReleaseID emitProgress("operation", "started", "") - evidenceID, jobResult, jobErr := s.executeJob(ctx, request, emitProgress) + evidenceID, jobResult, scheduleRun, jobErr := s.executeJob(ctx, request, emitProgress) result.EvidenceID = evidenceID result.JobResult = jobResult + result.ScheduleRun = scheduleRun return finish(jobErr) } diff --git a/internal/onebox/execution_types.go b/internal/onebox/execution_types.go index 29a558e6..030ac43d 100644 --- a/internal/onebox/execution_types.go +++ b/internal/onebox/execution_types.go @@ -293,7 +293,11 @@ type ExecuteRequest struct { BackupReport *BackupReport MigrationBackupOverride *MigrationBackupOverride BreakLock bool - AllowDestructiveMounts bool + // Detach asks a planned manual job to return after its installed host unit + // accepts the run. It is valid only for job_run; the job's schedule history + // remains the outcome authority. + Detach bool + AllowDestructiveMounts bool // Job, Inputs and Wait are the schedule_run arguments: a declared // scheduled job, validated input overrides, and whether to block until // the unit exits and return its run record. @@ -346,6 +350,9 @@ func (request ExecuteRequest) Validate() error { if request.AllowDestructiveMounts && request.Kind != KindServiceApply { return errors.New("allow_destructive_mounts is valid only for service apply") } + if request.Detach && request.Kind != KindJobRun { + return errors.New("detach is valid only for job run") + } if request.BreakMigrationGate && request.Kind != KindAbort { return errors.New("break_migration_gate is valid only for abort") } diff --git a/internal/onebox/job_execute.go b/internal/onebox/job_execute.go index a1f1f077..99d3e1eb 100644 --- a/internal/onebox/job_execute.go +++ b/internal/onebox/job_execute.go @@ -15,74 +15,74 @@ func (s *Service) executeJob( ctx context.Context, request ExecuteRequest, emit func(string, string, string), -) (string, *journal.JobResultEvidence, error) { +) (string, *journal.JobResultEvidence, *engine.ScheduleRunResult, error) { plan := request.JobPlan if err := plan.Validate(); err != nil { - return "", nil, fmt.Errorf("validate executable job plan: %w", err) + return "", nil, nil, fmt.Errorf("validate executable job plan: %w", err) } createdAt, err := parseOperationTime(plan.Operation.CreatedAt, "created_at") if err != nil { - return "", nil, err + return "", nil, nil, err } expiresAt, err := parseOperationTime(plan.Operation.ExpiresAt, "expires_at") if err != nil { - return "", nil, err + return "", nil, nil, err } now := s.now().UTC() if expiresAt.Before(now) { - return "", nil, &PlanExpiredError{Kind: PlanKindJob, Job: plan.Artifact.Job, ExpiresAt: expiresAt} + return "", nil, nil, &PlanExpiredError{Kind: PlanKindJob, Job: plan.Artifact.Job, ExpiresAt: expiresAt} } if createdAt.After(now.Add(time.Minute)) { - return "", nil, errors.New("job plan was created in the future — check the runner clock and re-plan") + return "", nil, nil, errors.New("job plan was created in the future — check the runner clock and re-plan") } emit("binding", "started", "") lp, err := s.loadProject(ctx, false) if err != nil { - return "", nil, fmt.Errorf("load project: %w", err) + return "", nil, nil, fmt.Errorf("load project: %w", err) } if err := ensureEnvironment(lp.resolved, s.environment); err != nil { - return "", nil, err + return "", nil, nil, err } environmentConfig, err := lp.resolved.Environment(s.environment) if err != nil { - return "", nil, err + return "", nil, nil, err } if err := enforceRunnerPolicy(environmentConfig.Policy, s.runner, plan.SchemaVersion); err != nil { - return "", nil, err + return "", nil, nil, err } binding := plan.Operation.Binding if lp.resolved.Name != binding.Application || s.environment != binding.Environment { - return "", nil, errors.New("job plan application or environment changed — re-plan") + return "", nil, nil, errors.New("job plan application or environment changed — re-plan") } if environmentConfig.Route().String() != binding.Server { - return "", nil, errors.New("job plan target changed — re-plan") + return "", nil, nil, errors.New("job plan target changed — re-plan") } if engine.HashBytes(lp.configBytes) != binding.ConfigDigest { - return "", nil, errors.New("configuration changed since job planning — re-plan") + return "", nil, nil, errors.New("configuration changed since job planning — re-plan") } job, ok := lp.resolved.Workloads[plan.Artifact.Job] if !ok || !job.IsJob() || job.When != "manual" || job.DataEffect != plan.Artifact.DataEffect { - return "", nil, errors.New("manual job declaration changed since planning — re-plan") + return "", nil, nil, errors.New("manual job declaration changed since planning — re-plan") } expectedBackup, err := migrationBackupRequirement(lp.resolved, environmentConfig.Policy, plan.Operation.Steps) if err != nil { - return "", nil, err + return "", nil, nil, err } if !reflect.DeepEqual(plan.MigrationBackup, expectedBackup) { - return "", nil, errors.New("migration backup requirement changed since job planning — re-plan") + return "", nil, nil, errors.New("migration backup requirement changed since job planning — re-plan") } if plan.Operation.Approval != ApprovalNone { if request.Approval == nil { - return "", nil, fmt.Errorf("%s approval is required for this exact job plan; record a local confirmation with `ob approve --plan PLAN`", plan.Operation.Approval) + return "", nil, nil, fmt.Errorf("%s approval is required for this exact job plan; record a local confirmation with `ob approve --plan PLAN`", plan.Operation.Approval) } if err := request.Approval.ValidateForPlan(plan, now); err != nil { - return "", nil, fmt.Errorf("validate job approval: %w", err) + return "", nil, nil, fmt.Errorf("validate job approval: %w", err) } } else if request.Approval != nil { if err := request.Approval.ValidateForPlan(plan, now); err != nil { - return "", nil, fmt.Errorf("validate job approval: %w", err) + return "", nil, nil, fmt.Errorf("validate job approval: %w", err) } } backupRequired := plan.MigrationBackup != nil @@ -91,7 +91,7 @@ func (s *Service) executeJob( backupRequired, now, ) if err != nil { - return "", nil, fmt.Errorf("validate migration backup authorization: %w", err) + return "", nil, nil, fmt.Errorf("validate migration backup authorization: %w", err) } e, cleanup, target, err := s.engineWith(ctx, lp, s.environment, func(options *engine.Options) { @@ -114,14 +114,28 @@ func (s *Service) executeJob( options.Progress = emit }) if err != nil { - return "", nil, fmt.Errorf("connect target: %w", err) + return "", nil, nil, fmt.Errorf("connect target: %w", err) } defer cleanup() if target != binding.Server { - return "", nil, fmt.Errorf("target changed from %q to %q — re-plan", binding.Server, target) + return "", nil, nil, fmt.Errorf("target changed from %q to %q — re-plan", binding.Server, target) } emit("binding", "succeeded", "") emit("execute", "started", "") + if job.Schedule != nil && job.DataEffect != DataEffectMigration { + run, err := e.PlannedJobRun(ctx, plan.Operation.ID, plan.Artifact.Job, + plan.Artifact.CurrentRelease, plan.Artifact.RuntimeDigest, !request.Detach) + if err == nil { + emit("execute", "succeeded", "") + } + return plan.Operation.ID, nil, &run, err + } + if request.Detach { + if job.Schedule == nil { + return plan.Operation.ID, nil, nil, fmt.Errorf("job %s has no installed scheduled-job unit; --detach requires a schedule", plan.Artifact.Job) + } + return plan.Operation.ID, nil, nil, fmt.Errorf("migration job %s cannot detach because its result evidence is required by the approved operation", plan.Artifact.Job) + } evidenceID, result, err := e.RunJobWithJournalID(ctx, engine.JobRunRequest{ OperationID: plan.Operation.ID, Job: plan.Artifact.Job, ExpectedRelease: plan.Artifact.CurrentRelease, ExpectedRuntimeDigest: plan.Artifact.RuntimeDigest, @@ -130,5 +144,5 @@ func (s *Service) executeJob( if err == nil { emit("execute", "succeeded", "") } - return evidenceID, result, err + return evidenceID, result, nil, err } diff --git a/internal/onebox/job_plan_test.go b/internal/onebox/job_plan_test.go index 2bf0042f..a0fed4d9 100644 --- a/internal/onebox/job_plan_test.go +++ b/internal/onebox/job_plan_test.go @@ -293,3 +293,75 @@ func TestExecuteJobRunsOnceAndJournalsTerminalResult(t *testing.T) { } } } + +func TestExecuteScheduledDestructiveJobDetachesToHostUnit(t *testing.T) { + current := "R0" + runtime := manualJobRuntime("ghcr.io/example/maintenance@sha256:" + strings.Repeat("ab", 32)) + fake := jobPlanFake(¤t, runtime) + base := fake.Dynamic + fake.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "systemctl --version"): + return transport.Result{Stdout: "systemd 255 (255.4-1ubuntu8)\n"}, true + case strings.Contains(cmd, "systemctl is-active"): + return transport.Result{Stdout: "inactive\n"}, true + case strings.Contains(cmd, "command -v flock"): + return transport.Result{Stdout: "ok\n"}, true + case strings.Contains(cmd, "systemctl start"): + return transport.Result{}, true + } + return base(cmd) + } + now := time.Date(2026, 9, 14, 19, 0, 0, 0, time.UTC) + connects := 0 + path := writeManualJobProject(t, "destructive", false) + encoded, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + project := strings.Replace(string(encoded), + " data_effect: destructive\n", + " data_effect: destructive\n schedule: {cron: '0 4 * * 1', timezone: UTC, timeout: 8h}\n", 1) + if err := os.WriteFile(path, []byte(project), 0o600); err != nil { + t.Fatal(err) + } + service := New(Options{ + ConfigPath: path, + Now: func() time.Time { return now }, + Connect: func(_ context.Context, route transport.Route) (transport.Transport, error) { + connects++ + return fake, nil + }, + }) + plan, err := service.PlanJob(context.Background(), PlanJobRequest{Job: "maintenance"}) + if err != nil { + t.Fatal(err) + } + approval, err := NewApprovalGrant(&plan, nil, "operator@example.test", now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + now = now.Add(2 * time.Minute) + result, err := service.Execute(context.Background(), ExecuteRequest{ + Kind: KindJobRun, JobPlan: &plan, Approval: &approval, Detach: true, + }) + if err != nil { + t.Fatalf("execute detached job: %v", err) + } + if result.Status != OperationStatusSuccess || result.ScheduleRun == nil || !result.ScheduleRun.Started { + t.Fatalf("detached result = %+v", result) + } + commands := strings.Join(fake.Commands, "\n") + if !strings.Contains(commands, "systemctl start --no-block 'ob-demo-maintenance.service'") { + t.Fatalf("job was not submitted to its host unit:\n%s", commands) + } + if strings.Contains(commands, "ONEBOX_RESULT_FILE=/run/onebox/job-result") { + t.Fatalf("detached job also ran in the SSH session:\n%s", commands) + } + written := strings.Join(fake.Inputs, "\n") + for _, want := range []string{"ONEBOX_EXPECTED_RELEASE=R0", "ONEBOX_EXPECTED_RUNTIME=" + plan.Artifact.RuntimeDigest} { + if !strings.Contains(written, want) { + t.Fatalf("host activation omitted %q:\n%s", want, written) + } + } +} diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 09f49d48..8a65da54 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -97,14 +97,18 @@ func (u *UI) Done(label string, d time.Duration, err error) { u.println(u.sOK.Render("✓ "+label) + u.sDim.Render(" "+FmtDur(d))) } -// Step times a step: call the returned func with the outcome. announce prints -// a Begin line for steps long enough that silence reads as a hang. +// Step times a step: call the returned func with the outcome. Announced steps +// get a live elapsed-time spinner on a TTY and a durable Begin line elsewhere. func (u *UI) Step(label string, announce bool) func(error) { + start := u.now() + stop := func() {} if announce { - u.Begin(label) + _, stop = u.Busy(label) + } + return func(err error) { + stop() + u.Done(label, u.now().Sub(start), err) } - start := u.now() - return func(err error) { u.Done(label, u.now().Sub(start), err) } } // Cmd is the forensic command log — verbose only, dimmed. @@ -133,8 +137,10 @@ func FmtDur(d time.Duration) string { return fmt.Sprintf("%.1fs", d.Seconds()) case d < time.Minute: return fmt.Sprintf("%ds", int(d.Seconds())) - default: + case d < time.Hour: return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) + default: + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) } } @@ -201,6 +207,7 @@ func (u *UI) Busy(label string) (update func(string), stop func()) { u.spinLabel, u.spinOn = label, true _, _ = io.WriteString(u.out, hideCursor) // the blinking cursor at line end is just noise u.mu.Unlock() + started := u.now() done := make(chan struct{}) finished := make(chan struct{}) go func() { @@ -219,7 +226,8 @@ func (u *UI) Busy(label string) (update func(string), stop func()) { case <-t.C: u.mu.Lock() if u.spinOn { - _, _ = io.WriteString(u.out, "\r\x1b[K"+u.sDim.Render(spinFrames[i%len(spinFrames)]+" "+u.spinLabel)) + line := fmt.Sprintf("%s %s · %s elapsed", spinFrames[i%len(spinFrames)], u.spinLabel, FmtDur(u.now().Sub(started))) + _, _ = io.WriteString(u.out, "\r\x1b[K"+u.sDim.Render(line)) } u.mu.Unlock() i++ diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index b389209c..1ce28d69 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -59,7 +59,8 @@ func TestFmtDur(t *testing.T) { 12 * time.Second: "12s", 84 * time.Second: "1m24s", 134 * time.Second: "2m14s", - 61 * time.Minute: "61m0s", + 61 * time.Minute: "1h1m", + 8 * time.Hour: "8h0m", } for d, want := range cases { if got := FmtDur(d); got != want { @@ -68,6 +69,25 @@ func TestFmtDur(t *testing.T) { } } +func TestAnnouncedStepIsLineHonestOffTTY(t *testing.T) { + var out bytes.Buffer + u := New(&out, false) + u.now = func() time.Time { return time.Unix(0, 0) } + done := u.Step("job catalog-refresh", true) + u.now = func() time.Time { return time.Unix(61, 0) } + done(nil) + + s := out.String() + if strings.ContainsAny(s, "\r\x1b") { + t.Fatalf("non-TTY step must not emit control sequences: %q", s) + } + for _, want := range []string{"⟳ job catalog-refresh", "✓ job catalog-refresh", "1m1s"} { + if !strings.Contains(s, want) { + t.Fatalf("missing %q in:\n%s", want, s) + } + } +} + func TestStepHelper(t *testing.T) { var out bytes.Buffer u := New(&out, false) diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 47a41fab..28748504 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -523,6 +523,17 @@ ob approve --plan ob-job-plan.json --out ob-job-approval.json ob job run --plan ob-job-plan.json --approval ob-job-approval.json ``` +When the manual job also declares `schedule`, Onebox submits this run to the +same installed systemd unit used by its timer. The host therefore owns the +container, timeout, overlap lock, retries, and run record if the SSH session or +operator terminal disappears. The command follows the unit by default and +shows elapsed time. Ctrl-C stops following but does not stop the host job; use +`--detach` to return immediately after the unit accepts it. Inspect either form +with `ob schedule history ` and `ob schedule logs `. + +Jobs without `schedule` keep the direct foreground path. Migration jobs also +stay attached because their result evidence is part of the approved operation. + `ob job plan` accepts a job only when its resolved `when` is `manual`, which is the default for a job that declares none. A `pre_release` or `post_release` job is refused: it belongs to the deploy graph, and one-shot invocation is reserved diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index 53707452..f7edefa2 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -803,6 +803,10 @@ Global Flags: ``` Run one manual job through the canonical lock, fence, local-confirmation and journal boundary. +A job with schedule configured runs under its installed systemd unit and is +followed by default; Ctrl-C stops following, not the host job. --detach returns +after that unit accepts the run. Unscheduled and migration jobs stay attached. + Humans may pass an id and confirm interactively. Automation should supply a saved --plan and its separately created local-confirmation artifact through --approval; migration plans may also require the exact plan-bound --backup-report. @@ -814,6 +818,7 @@ Flags: --approval string apply a plan-bound local confirmation artifact --backup-report string apply the backup report bound into the local confirmation --break-lock break a stale operation lock after inspecting its holder + --detach return after the installed host unit accepts the job -h, --help help for run --override-migration-backup string audited break-glass reason (requires --approval) --plan string apply a saved job plan artifact