diff --git a/cmd/ob/job.go b/cmd/ob/job.go index 4a7afef8..49a1d9a2 100644 --- a/cmd/ob/job.go +++ b/cmd/ob/job.go @@ -1,8 +1,11 @@ package main import ( + "bytes" "errors" "fmt" + "strconv" + "text/tabwriter" "time" "github.com/spf13/cobra" @@ -14,31 +17,38 @@ import ( func addJobCommand(root *cobra.Command, g *globalFlags) { group := &cobra.Command{ Use: "job", - Short: "plan and run a sealed one-shot manual job", - Long: "Plan and run one declared `when: manual` job against the current serving release.\n\nThe job remains in the release runtime but never runs during deploy. Saved plans\nbind its release, runtime digest, immutable image and data effect so agents can\nobtain a separate approval before execution.", + Short: "plan and run a sealed one-shot operator job", + Long: "Plan and run one declared `operator_run: allowed` job against the current serving release.\n\nDeployment participation is independent: `deployment_phase` may be none, pre_release,\nor post_release. Saved plans bind the release, runtime digest, immutable image,\ndata effect and inputs so agents can obtain separate approval before execution.", Args: cobra.NoArgs, RunE: showCommandHelp, } var planOut, backupReportOut string + var planInputs []string plan := &cobra.Command{ Use: "plan ", Short: "seal a current-release-bound one-shot job plan", Long: "Observe the current serving release and write a short-lived executable job plan.\n\nThe plan binds the exact runtime digest, digest-pinned image, job data effect,\ntarget and expiry. It reads the target and writes only the local plan artifact.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runJobPlan(cmd, g, args[0], planOut, backupReportOut) + inputs, err := parseScheduleInputs(planInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("job_input_invalid", "%v", err)) + } + return runJobPlan(cmd, g, args[0], inputs, planOut, backupReportOut) }, } plan.Flags().StringVarP(&planOut, "out", "o", "ob-job-plan.json", "job plan artifact path") plan.Flags().StringVar(&backupReportOut, "backup-report-out", "", "write a plan-bound backup report template when migration backup is required") + plan.Flags().StringArrayVar(&planInputs, "input", nil, "input override as NAME=VALUE; repeatable and sealed into the plan") var planPath, approvalPath, backupReportPath, overrideReason string + var runInputs []string var breakLock, detach bool run := &cobra.Command{ Use: "run [id]", - Short: "run one manual job from an inline or saved sealed plan", - Long: "Run one manual job through the canonical lock, fence, local-confirmation and journal boundary.\n\n" + + Short: "run one operator job from an inline or saved sealed plan", + Long: "Run one operator job through the canonical lock, fence, local-confirmation and journal boundary.\n\n" + "A job with schedule configured runs under its installed systemd unit and is\nfollowed by default; Ctrl-C stops following, not the host job. --detach returns\nafter that unit accepts the run. Unscheduled and migration jobs stay attached.\n\n" + "Humans may pass an id and confirm interactively. Automation should supply a\nsaved --plan and its separately created local-confirmation artifact through\n--approval; migration plans may also require the exact plan-bound --backup-report.", Args: cobra.MaximumNArgs(1), @@ -47,7 +57,11 @@ func addJobCommand(root *cobra.Command, g *globalFlags) { if len(args) == 1 { jobID = args[0] } - return runJob(cmd, g, jobID, planPath, approvalPath, backupReportPath, overrideReason, breakLock, detach) + inputs, err := parseScheduleInputs(runInputs) + if err != nil { + return writeEarlyOperationFailure(cmd, g, codedError("job_input_invalid", "%v", err)) + } + return runJob(cmd, g, jobID, inputs, planPath, approvalPath, backupReportPath, overrideReason, breakLock, detach) }, } run.Flags().StringVar(&planPath, "plan", "", "apply a saved job plan artifact") @@ -56,13 +70,15 @@ func addJobCommand(root *cobra.Command, g *globalFlags) { run.Flags().StringVar(&overrideReason, "override-migration-backup", "", "audited break-glass reason (requires --approval)") run.Flags().BoolVar(&breakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") run.Flags().BoolVar(&detach, "detach", false, "return after the installed host unit accepts the job") + run.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable and sealed into the inline plan") + addJobReadCommands(group, g) group.AddCommand(plan, run) root.AddCommand(group) } -func runJobPlan(cmd *cobra.Command, g *globalFlags, jobID, outPath, backupReportOut string) error { - plan, err := operationsService(cmd, g).PlanJob(cmd.Context(), onebox.PlanJobRequest{Job: jobID}) +func runJobPlan(cmd *cobra.Command, g *globalFlags, jobID string, inputs map[string]string, outPath, backupReportOut string) error { + plan, err := operationsService(cmd, g).PlanJob(cmd.Context(), onebox.PlanJobRequest{Job: jobID, Inputs: inputs}) if err != nil { return writeStructuredCommandFailure(cmd, g, "job_plan_failed", "job planning failed; inspect stderr for local diagnostics", err) } @@ -166,10 +182,13 @@ func renderJobPlan(cmd *cobra.Command, plan *onebox.JobPlan) { } } -func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, backupReportPath, overrideReason string, breakLock, detach bool) error { +func runJob(cmd *cobra.Command, g *globalFlags, jobID string, inputs map[string]string, planPath, approvalPath, backupReportPath, overrideReason string, breakLock, detach bool) error { if planPath != "" && jobID != "" { return writeEarlyOperationFailure(cmd, g, errors.New("supply either a job id or --plan, not both")) } + if planPath != "" && len(inputs) > 0 { + return writeEarlyOperationFailure(cmd, g, errors.New("--input is sealed into a plan; supply it to ob job plan, not ob job run --plan")) + } if planPath == "" && jobID == "" { return writeEarlyOperationFailure(cmd, g, errors.New("job run requires an id or --plan")) } @@ -224,7 +243,7 @@ func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, b }, "job run") } - plan, err := operationsService(cmd, g).PlanJob(cmd.Context(), onebox.PlanJobRequest{Job: jobID}) + plan, err := operationsService(cmd, g).PlanJob(cmd.Context(), onebox.PlanJobRequest{Job: jobID, Inputs: inputs}) if err != nil { return err } @@ -251,6 +270,96 @@ func runJob(cmd *cobra.Command, g *globalFlags, jobID, planPath, approvalPath, b return runMutation(cmd, g, onebox.ExecuteRequest{Kind: onebox.KindJobRun, JobPlan: &plan, Approval: approval, BreakLock: breakLock, Detach: detach}, "job run") } +func addJobReadCommands(group *cobra.Command, g *globalFlags) { + var historyCount int + history := &cobra.Command{ + Use: "history ", Short: "execution records of one job, newest first", + Long: "Read retained execution records for one job across timer and operator triggers. Host-supervised records come from journald; sealed attached runs come from the operation journal. The result is retention-bounded evidence, so an absent success means no retained success was observed, not that the job never succeeded.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + records, err := e.JobHistory(cmd.Context(), args[0], historyCount) + if err != nil { + return writeStructuredCommandFailure(cmd, g, "job_history_failed", "job history could not be read", err) + } + if isStructuredOutput(g) { + return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) + } + if len(records) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "no retained executions for %s\n", args[0]) + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tOPERATOR\tRUN") + for _, r := range records { + exit := "-" + if r.ExitStatus != nil { + exit = strconv.Itoa(*r.ExitStatus) + } + fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\t%s\n", r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), orDash(r.Operator), r.ID) + } + return w.Flush() + }, + } + history.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest executions to show") + + var logsRun string + logs := &cobra.Command{ + Use: "logs ", Short: "journal of one host-supervised job execution", + Long: "Stream the exact systemd journal of a host-supervised job execution. By default the newest retained run is selected; --run accepts the run id printed by ob job history. Attached sealed executions retain outcome evidence but do not have a separate host log stream.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, p, err := loadAllLenient(cmd.Context(), g) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) + if err != nil { + return writeStructuredReadFailure(cmd, g, err) + } + defer cleanup() + if g.Output == "json" { + var stdout, stderr bytes.Buffer + run, err := e.JobLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) + data := map[string]any{"job": args[0], "run": run, "stdout": stdout.String(), "stderr": stderr.String(), "passthrough_unredacted": true} + if err != nil { + publicErr := publicError(err, "job_logs_failed", "job logs could not be read") + publicErr.Details = data + if writeErr := writeFiniteOutcome(cmd, g, cliOutcomeError, nil, publicErr); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return writeFiniteSuccess(cmd, g, data) + } + if g.Output == "ndjson" { + stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) + run, err := e.JobLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) + data := map[string]any{"job": args[0], "run": run, "passthrough_unredacted": true} + if err != nil { + if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "job_logs_failed", "job logs could not be read")); writeErr != nil { + return writeErr + } + return withExitCode(err, 1) + } + return stream.terminal(cliOutcomeSuccess, data, nil) + } + _, err = e.JobLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()) + return err + }, + } + logs.Flags().StringVar(&logsRun, "run", "", "run id from ob job history; defaults to newest host-supervised run") + group.AddCommand(history, logs) +} + func loadJobMigrationOverride( plan *onebox.JobPlan, approval *onebox.ApprovalGrant, diff --git a/cmd/ob/output.go b/cmd/ob/output.go index cec1a5b6..bb9f88f0 100644 --- a/cmd/ob/output.go +++ b/cmd/ob/output.go @@ -96,6 +96,8 @@ var cliOutputMatrix = map[string]cliOutputClass{ "ob backup verify": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob backup status": {Class: cliClassFiniteEnvelope, JSON: true}, "ob job plan": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job history": {Class: cliClassFiniteEnvelope, JSON: true}, + "ob job logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, "ob job run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, "ob plan": {Class: cliClassFiniteEnvelope, JSON: true}, @@ -105,12 +107,9 @@ var cliOutputMatrix = map[string]cliOutputClass{ "ob resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob rollback": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob schedule apply": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schedule history": {Class: cliClassFiniteEnvelope, JSON: true}, "ob schedule list": {Class: cliClassFiniteEnvelope, JSON: true}, - "ob schedule logs": {Class: cliClassOperatorPassthrough, JSON: true, NDJSON: true}, "ob schedule pause": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob schedule resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, - "ob schedule run": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, "ob execution list": {Class: cliClassFiniteEnvelope, JSON: true}, "ob execution inspect": {Class: cliClassFiniteEnvelope, JSON: true}, "ob execution resume": {Class: cliClassFiniteStream, JSON: true, NDJSON: true}, diff --git a/cmd/ob/output_test.go b/cmd/ob/output_test.go index fdc68a33..fe4b718e 100644 --- a/cmd/ob/output_test.go +++ b/cmd/ob/output_test.go @@ -502,6 +502,8 @@ func TestLeafOutputMatrixIsClosedAndHasNoAliases(t *testing.T) { "ob exec": {Class: "operator_passthrough", NDJSON: true}, "ob init": {Class: "finite_envelope", JSON: true}, "ob job plan": {Class: "finite_envelope", JSON: true}, + "ob job history": {Class: "finite_envelope", JSON: true}, + "ob job logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, "ob backup create": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob backup enable": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob backup disable": {Class: "finite_stream", JSON: true, NDJSON: true}, @@ -519,12 +521,9 @@ func TestLeafOutputMatrixIsClosedAndHasNoAliases(t *testing.T) { "ob resume": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob rollback": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob schedule apply": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schedule history": {Class: "finite_envelope", JSON: true}, "ob schedule list": {Class: "finite_envelope", JSON: true}, - "ob schedule logs": {Class: "operator_passthrough", JSON: true, NDJSON: true}, "ob schedule pause": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob schedule resume": {Class: "finite_stream", JSON: true, NDJSON: true}, - "ob schedule run": {Class: "finite_stream", JSON: true, NDJSON: true}, "ob execution list": {Class: "finite_envelope", JSON: true}, "ob execution inspect": {Class: "finite_envelope", JSON: true}, "ob execution resume": {Class: "finite_stream", JSON: true, NDJSON: true}, diff --git a/cmd/ob/schedule.go b/cmd/ob/schedule.go index 2efa85bf..a50c1e73 100644 --- a/cmd/ob/schedule.go +++ b/cmd/ob/schedule.go @@ -1,9 +1,7 @@ package main import ( - "bytes" "fmt" - "strconv" "strings" "text/tabwriter" @@ -12,16 +10,16 @@ import ( ) // addScheduleCommands wires `ob schedule`: apply reconciles the host units, -// list, history and logs read what the host recorded. The read commands -// connect directly, like `ob status`; they hold no lock and write nothing. +// list reads what the host recorded. The read command connects directly, +// like `ob status`; it holds no lock and writes nothing. func addScheduleCommands(root *cobra.Command, g *globalFlags) { addExecutionCommands(root, g) scheduleCmd := &cobra.Command{Use: "schedule", Short: "manage host timers for scheduled jobs", Long: "Manage the systemd timers generated for scheduled jobs.\n\n" + "Timers outlive the Onebox process and the package installed on the operator\n" + "workstation. `apply` explicitly reconciles their units after a runner or\n" + - "configuration change without deploying a release. `list`, `history` and `logs`\n" + - "read the timer state and the run records the host keeps in its journal.", + "configuration change without deploying a release. `list` reads timer state;\n" + + "job history and logs live under `ob job`.", Args: cobra.NoArgs, RunE: showCommandHelp} var scheduleBreakLock bool @@ -65,7 +63,7 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { return nil } w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "JOB\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT") + fmt.Fprintln(w, "JOB\tDEPLOYMENT\tOPERATOR\tCRON\tTZ\tTIMER\tNEXT\tLAST TRIGGER\tPOLICY\tTIMEOUT\tRETRY BUDGET") for _, j := range jobs { // A paused timer and a broken one are both "inactive" to // systemd. Only one of them is somebody's decision, and this @@ -74,135 +72,14 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { if j.Paused != nil { timer = "paused" } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - j.Name, j.Cron, j.Timezone, timer, orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d attempt(s), %s backoff\n", + j.Name, j.DeploymentPhase, j.OperatorRun, j.Cron, j.Timezone, timer, orDash(j.NextRun), orDash(j.LastTrigger), j.DeployLock, j.Timeout, j.MaxAttempts, j.RetryBudget) } return w.Flush() }, } scheduleCmd.AddCommand(listCmd) - var historyCount int - historyCmd := &cobra.Command{ - Use: "history ", - Short: "run records of one scheduled job, newest first", - Long: "Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs.\n\nRecords live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, err := loadAllLenient(cmd.Context(), g) - if err != nil { - return writeStructuredReadFailure(cmd, g, err) - } - e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) - if err != nil { - return writeStructuredReadFailure(cmd, g, err) - } - defer cleanup() - records, err := e.ScheduleHistory(cmd.Context(), args[0], historyCount) - if err != nil { - return writeStructuredCommandFailure(cmd, g, "schedule_history_failed", "run history could not be read", err) - } - if isStructuredOutput(g) { - return writeFiniteSuccess(cmd, g, map[string]any{"job": args[0], "runs": records}) - } - if len(records) == 0 { - fmt.Fprintf(cmd.OutOrStdout(), "no recorded runs for %s\n", args[0]) - return nil - } - w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "STARTED\tOUTCOME\tDURATION\tATTEMPTS\tEXIT\tTRIGGER\tRELEASE\tRUN") - for _, r := range records { - exit := "-" - if r.ExitStatus != nil { - exit = strconv.Itoa(*r.ExitStatus) - } - fmt.Fprintf(w, "%s\t%s\t%ds\t%d\t%s\t%s\t%s\t%s\n", - r.StartedAt, r.Outcome, r.DurationSeconds, r.Attempts, exit, r.Trigger, orDash(r.Release), r.Run) - } - return w.Flush() - }, - } - historyCmd.Flags().IntVarP(&historyCount, "count", "n", 20, "number of newest runs to show") - scheduleCmd.AddCommand(historyCmd) - - var logsRun string - logsCmd := &cobra.Command{ - Use: "logs ", - Short: "journal of one scheduled run", - Long: "Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only.\n\nLog bytes are operator-controlled and may contain secrets; Onebox does not claim\nto redact passthrough output.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, err := loadAllLenient(cmd.Context(), g) - if err != nil { - return writeStructuredReadFailure(cmd, g, err) - } - e, cleanup, err := connect(cmd, g, cfg, p, newUI(cmd, g)) - if err != nil { - return writeStructuredReadFailure(cmd, g, err) - } - defer cleanup() - if g.Output == "json" { - var stdout, stderr bytes.Buffer - run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, &stdout, &stderr) - data := map[string]any{ - "job": args[0], "run": run, "stdout": stdout.String(), "stderr": stderr.String(), - "passthrough_unredacted": true, - } - if err != nil { - publicErr := publicError(err, "schedule_logs_failed", "run logs could not be read") - publicErr.Details = data - if writeErr := writeFiniteOutcome(cmd, g, cliOutcomeError, nil, publicErr); writeErr != nil { - return writeErr - } - return withExitCode(err, 1) - } - return writeFiniteSuccess(cmd, g, data) - } - if g.Output == "ndjson" { - stream := newCLIRecordStream(cmd.OutOrStdout(), commandName(cmd)) - run, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, stream.channelWriter("stdout"), stream.channelWriter("stderr")) - data := map[string]any{"job": args[0], "run": run, "passthrough_unredacted": true} - if err != nil { - if writeErr := stream.terminal(cliOutcomeError, nil, publicError(err, "schedule_logs_failed", "run logs could not be read")); writeErr != nil { - return writeErr - } - return withExitCode(err, 1) - } - return stream.terminal(cliOutcomeSuccess, data, nil) - } - if _, err := e.ScheduleLogs(cmd.Context(), args[0], logsRun, cmd.OutOrStdout(), cmd.ErrOrStderr()); err != nil { - return writeStructuredCommandFailure(cmd, g, "schedule_logs_failed", "run logs could not be read", err) - } - return nil - }, - } - logsCmd.Flags().StringVar(&logsRun, "run", "", "run id from ob schedule history; default the newest run") - scheduleCmd.AddCommand(logsCmd) - - var runInputs []string - var runWait, runBreakLock bool - runCmd := &cobra.Command{ - Use: "run ", - Short: "start a scheduled job now with declared inputs", - Long: "Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host.\n\n" + - "Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it.\n\n" + - "The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it.", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - inputs, err := parseScheduleInputs(runInputs) - if err != nil { - return writeEarlyOperationFailure(cmd, g, codedError("schedule_input_invalid", "%v", err)) - } - return runMutation(cmd, g, onebox.ExecuteRequest{ - Kind: onebox.KindScheduleRun, Job: args[0], Inputs: inputs, Wait: runWait, BreakLock: runBreakLock, - }, "schedule run") - }, - } - runCmd.Flags().StringArrayVar(&runInputs, "input", nil, "input override as NAME=VALUE; repeatable") - runCmd.Flags().BoolVar(&runWait, "wait", false, "block until the unit exits and report the run record") - runCmd.Flags().BoolVar(&runBreakLock, "break-lock", false, "break a stale operation lock after inspecting its holder") - scheduleCmd.AddCommand(runCmd) - var pauseReason string var pauseBreakLock bool pauseCmd := &cobra.Command{ @@ -228,7 +105,7 @@ func addScheduleCommands(root *cobra.Command, g *globalFlags) { resumeCmd := &cobra.Command{ Use: "resume ", Short: "start a paused scheduled job's timer again", - Long: "Start a paused job's timer and clear the record of the pause.\n\nThe next run is the next scheduled elapse: resuming does not run the job now, and does not make up the firings missed while it was paused. Use `ob schedule run` for an immediate run.", + Long: "Start a paused job's timer and clear the record of the pause.\n\nThe next run is the next scheduled elapse: resuming does not run the job now, and does not make up the firings missed while it was paused. Use `ob job run` for an immediate run.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runMutation(cmd, g, onebox.ExecuteRequest{ diff --git a/docs/onebox.run-v1.schema.json b/docs/onebox.run-v1.schema.json index 6e64dd6a..484319f5 100644 --- a/docs/onebox.run-v1.schema.json +++ b/docs/onebox.run-v1.schema.json @@ -1714,8 +1714,11 @@ "data_effect": { "const": "none" }, - "when": { - "const": "manual" + "deployment_phase": { + "const": "none" + }, + "operator_run": { + "const": "allowed" } }, "required": [ @@ -1897,7 +1900,12 @@ "anyOf": [ { "required": [ - "when" + "deployment_phase" + ] + }, + { + "required": [ + "operator_run" ] }, { @@ -2032,6 +2040,16 @@ ], "type": "string" }, + "deployment_phase": { + "default": "none", + "description": "Deployment phase for this job: none, pre_release, or post_release.", + "enum": [ + "none", + "pre_release", + "post_release" + ], + "type": "string" + }, "domain": { "description": "Domain shorthand for one HTTPS route; requires port and cannot be combined with routes.", "examples": [ @@ -2141,7 +2159,7 @@ }, "execution": { "additionalProperties": false, - "description": "Opt-in durable scheduled execution. Requires a native manual job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume.", + "description": "Opt-in durable scheduled execution. Requires a native operator-runnable phase-none job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume.", "patternProperties": { "^x-": {} }, @@ -2399,7 +2417,7 @@ }, "properties": { "default": { - "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "description": "Value used by a timer firing and by an operator run that does not override it. Must satisfy the input's own constraint.", "type": "string" }, "description": { @@ -2431,7 +2449,7 @@ ], "type": "object" }, - "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob job run may override them.", "propertyNames": { "pattern": "^[A-Z][A-Z0-9_]*$" }, @@ -2510,6 +2528,14 @@ }, "type": "array" }, + "operator_run": { + "description": "Whether an operator may invoke this job outside deployment: allowed or disabled. Defaults to allowed for phase none and disabled otherwise.", + "enum": [ + "allowed", + "disabled" + ], + "type": "string" + }, "persistence": { "additionalProperties": false, "description": "Declares whether this workload holds data that must outlive releases.", @@ -2712,7 +2738,7 @@ }, "schedule": { "additionalProperties": false, - "description": "Host-resident recurring schedule and run policy for a job.", + "description": "Host-resident recurring schedule and run policy for a job, independent of its deployment phase and operator-run policy.", "patternProperties": { "^x-": {} }, @@ -2797,6 +2823,15 @@ }, "type": "object" }, + "shutdown_grace": { + "default": "30s", + "description": "Time allowed for graceful container shutdown after the run deadline before Onebox forces removal. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "45s" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", @@ -2925,16 +2960,6 @@ }, "type": "array" }, - "when": { - "default": "manual", - "description": "When a job runs: manual, pre_release, or post_release.", - "enum": [ - "pre_release", - "post_release", - "manual" - ], - "type": "string" - }, "working_dir": { "description": "Absolute working directory for the container process. Expects an absolute path with no control character or shell metacharacter.", "examples": [ diff --git a/e2e/server_test.go b/e2e/server_test.go index 3a2522a8..f22afdaa 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -237,7 +237,7 @@ ExecStart=/usr/bin/docker compose -p observer -f /var/lib/ob/observer/current/co } // The notifier wrote a record for that run; a hand-started unit has no - // TRIGGER_UNIT, so it is recorded as a manual activation. + // TRIGGER_UNIT, so it is recorded as a operator activation. history := s.mustOb(t, dir, "schedule", "history", "chore", "--output", "json") for _, want := range []string{`"outcome": "success"`, `"trigger": "manual"`, `"attempts": 1`} { if !strings.Contains(history, want) { @@ -255,12 +255,12 @@ ExecStart=/usr/bin/docker compose -p observer -f /var/lib/ob/observer/current/co } } - // A manual run with an input override reaches the container as its + // A operator run with an input override reaches the container as its // environment, is journaled with the operator, and shows up as such. manual := s.mustOb(t, dir, "schedule", "run", "input-chore", "--input", "GREETING=hello", "--wait", "--output", "json") for _, want := range []string{`"GREETING": "hello"`, `"outcome": "success"`, `"trigger": "manual"`} { if !strings.Contains(manual, want) { - t.Fatalf("manual run result is missing %q:\n%s", want, manual) + t.Fatalf("operator run result is missing %q:\n%s", want, manual) } } logs := s.mustOb(t, dir, "schedule", "logs", "input-chore") @@ -269,7 +269,7 @@ ExecStart=/usr/bin/docker compose -p observer -f /var/lib/ob/observer/current/co } audit := s.mustOb(t, dir, "audit") if !strings.Contains(audit, "schedule run") { - t.Fatalf("audit does not list the manual run:\n%s", audit) + t.Fatalf("audit does not list the operator run:\n%s", audit) } list := s.mustOb(t, dir, "schedule", "list") if !strings.Contains(list, "input-chore") || !strings.Contains(list, "active") { @@ -332,11 +332,11 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() } // The record is the verdict, so `systemctl reset-failed` no longer // clears the failure from `ob status`; only a later successful run - // does. A manual run with the input that makes the job finish in time + // does. A operator run with the input that makes the job finish in time // is that run, and it must leave status green for the steps after. s.mustOb(t, dir, "schedule", "run", "timeout-chore", "--input", "SLEEP=0", "--wait") if cleared := s.mustOb(t, dir, "status"); !strings.Contains(cleared, "schedule timeout-chore active") { - t.Fatalf("a successful manual run did not clear the recorded timeout:\n%s", cleared) + t.Fatalf("a successful operator run did not clear the recorded timeout:\n%s", cleared) } }) @@ -401,7 +401,7 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() if err != nil { t.Fatalf("job run failed: %v\n%s", err, out) } - for _, want := range []string{"host run ", "⟳ job chore", "✓ job chore", "ob schedule history chore"} { + for _, want := range []string{"host run ", "⟳ job chore", "✓ job chore", "ob job history chore"} { if !strings.Contains(out, want) { t.Fatalf("host-owned job output is missing %q:\n%s", want, out) } @@ -413,7 +413,7 @@ HTTPServer(("127.0.0.1", 18080), Handler).handle_request() if err != nil { t.Fatalf("detached job run failed: %v\n%s", err, out) } - if !strings.Contains(out, "job chore accepted as ") || !strings.Contains(out, "ob schedule history chore") { + if !strings.Contains(out, "job chore accepted as ") || !strings.Contains(out, "ob job history chore") { t.Fatalf("detached job did not report durable acceptance:\n%s", out) } }) diff --git a/e2e/testdata/postgres/ob.yml.tmpl b/e2e/testdata/postgres/ob.yml.tmpl index 000d8345..6dc07df8 100644 --- a/e2e/testdata/postgres/ob.yml.tmpl +++ b/e2e/testdata/postgres/ob.yml.tmpl @@ -43,7 +43,7 @@ workloads: # A deliberately wedged timer run proves the host-enforced timeout becomes a # recorded timeout that `ob status` exposes. Its annual timer never fires # during the suite; the test starts the service directly. The SLEEP input - # lets a later manual run succeed, which is the only thing that clears a + # lets a later operator run succeed, which is the only thing that clears a # recorded failure. timeout-chore: role: job @@ -64,7 +64,7 @@ workloads: volumes: [{source: /tmp/onebox-e2e-retry, path: /marker}] schedule: { cron: "0 0 1 1 *", timeout: 60s, catch_up: false, retry: {attempts: 2, backoff: 1s} } # A declared input reaches the container as an environment variable: its - # default on a timer-shaped start, an override on a manual run. + # default on a timer-shaped start, an override on a operator run. input-chore: role: job image: public.ecr.aws/docker/library/busybox@sha256:9db7b59979c38555a39def84a31fb98b5296952f9e3afd4f6f11f05b07adfab0 diff --git a/internal/app/canonical_test.go b/internal/app/canonical_test.go index c7721616..945c59b8 100644 --- a/internal/app/canonical_test.go +++ b/internal/app/canonical_test.go @@ -269,8 +269,10 @@ runtime: {env_files: [{file: secrets.env, provider: sops}]} "workloads.web.published_ports[0].bind": "127.0.0.1", "workloads.web.published_ports[0].protocol": "tcp", "workloads.web.persistence.mode": "durable", - "workloads.job.when": "manual", + "workloads.job.deployment_phase": "none", + "workloads.job.operator_run": "allowed", "workloads.job.schedule.timezone": "UTC", + "workloads.job.schedule.shutdown_grace": "30s", "notifications.ops.format": "text", } { if origins[path] != string(OriginDefault) { diff --git a/internal/app/constraints.go b/internal/app/constraints.go index bd75732f..2c4be61e 100644 --- a/internal/app/constraints.go +++ b/internal/app/constraints.go @@ -161,7 +161,8 @@ var ( eNeedCondition = []string{"started", "healthy", "completed"} ePortProtocol = []string{"tcp", "udp"} eStrategy = []string{"rolling", "recreate"} - eJobWhen = []string{"pre_release", "post_release", "manual"} + eJobDeploymentPhase = []string{"none", "pre_release", "post_release"} + eJobOperatorRun = []string{"allowed", "disabled"} eScheduleDeployLock = []string{"exclusive", "pinned"} // The seams the engine actually invokes. An unlisted name loads fine and // never runs, so the set is closed: a hook that silently does not fire is diff --git a/internal/app/defaults.go b/internal/app/defaults.go index 9cc0e63b..38a9fd61 100644 --- a/internal/app/defaults.go +++ b/internal/app/defaults.go @@ -91,6 +91,18 @@ func applyDefaults(p *Spec, raw map[string]any, derived map[string]Origin) { w.Strategy = w.Mode() mark(path + ".strategy") } + if w.IsJob() && w.DeploymentPhase == "" { + w.DeploymentPhase = "none" + mark(path + ".deployment_phase") + } + if w.IsJob() && w.OperatorRun == "" { + if w.DeploymentPhase == "none" { + w.OperatorRun = "allowed" + } else { + w.OperatorRun = "disabled" + } + mark(path + ".operator_run") + } if w.Image != nil && w.Image.Pull == "" { w.Image.Pull = "missing" mark(path + ".image.pull") @@ -99,10 +111,6 @@ func applyDefaults(p *Spec, raw map[string]any, derived map[string]Origin) { w.Drain.Signal = "TERM" mark(path + ".drain.signal") } - if w.IsJob() && w.When == "" { - w.When = "manual" - mark(path + ".when") - } if w.Schedule != nil { if w.Schedule.Timezone == "" { w.Schedule.Timezone = "UTC" @@ -120,6 +128,10 @@ func applyDefaults(p *Spec, raw map[string]any, derived map[string]Origin) { w.Schedule.DeployLock = "exclusive" mark(path + ".schedule.deploy_lock") } + if w.Schedule.ShutdownGrace == "" { + w.Schedule.ShutdownGrace = "30s" + mark(path + ".schedule.shutdown_grace") + } if !stated(raw, path+".schedule.catch_up") { w.Schedule.CatchUp = true mark(path + ".schedule.catch_up") diff --git a/internal/app/generate.go b/internal/app/generate.go index 1ba9e7a9..206605b8 100644 --- a/internal/app/generate.go +++ b/internal/app/generate.go @@ -346,7 +346,7 @@ func (p *Spec) renderWorkload(n Names, name string, w Workload, releaseID string env := stringMap(w.Env) // A declared input's default is part of the release, so a timer firing, a - // manual run without overrides, and a hand-typed `docker compose run` all + // operator run without overrides, and a hand-typed `docker compose run` all // see the same value. Validation refuses a name that is also an env key. if len(w.Inputs) > 0 { if env == nil { diff --git a/internal/app/generate_test.go b/internal/app/generate_test.go index 71f41d56..c6d628e8 100644 --- a/internal/app/generate_test.go +++ b/internal/app/generate_test.go @@ -36,7 +36,7 @@ workloads: role: job image: ghcr.io/acme/ledger:1.4.0 command: [./ledger, migrate] - when: pre_release + deployment_phase: pre_release data_effect: migration needs: [{name: db, condition: healthy}] db: diff --git a/internal/app/job_execution.go b/internal/app/job_execution.go index 52bb1b56..ca0001b5 100644 --- a/internal/app/job_execution.go +++ b/internal/app/job_execution.go @@ -12,8 +12,8 @@ func validateJobExecution(w Workload, path string) error { return nil } p := path + ".execution" - if !w.IsJob() || w.Schedule == nil || (w.When != "" && w.When != "manual") || w.DataEffect != DataEffectNone || w.Compose != "" { - return errf("project_invalid", p, "", "durable execution requires a native scheduled manual job with data_effect none") + if !w.IsJob() || w.Schedule == nil || (w.DeploymentPhase != "" && w.DeploymentPhase != "none") || (w.OperatorRun != "" && w.OperatorRun != "allowed") || w.DataEffect != DataEffectNone || w.Compose != "" { + return errf("project_invalid", p, "", "durable execution requires a native scheduled operator-runnable phase-none job with data_effect none") } if w.Execution.Retention != "" { d, ok := ParseDuration(w.Execution.Retention) diff --git a/internal/app/job_execution_test.go b/internal/app/job_execution_test.go index 8d6b484d..c533597b 100644 --- a/internal/app/job_execution_test.go +++ b/internal/app/job_execution_test.go @@ -8,7 +8,7 @@ import ( func executionWorkload() Workload { return Workload{ - Role: "job", When: "manual", DataEffect: DataEffectNone, + Role: "job", DeploymentPhase: "none", OperatorRun: "allowed", DataEffect: DataEffectNone, Schedule: &JobSchedule{Timeout: "1h"}, Execution: &JobExecution{Steps: []JobStep{ {ID: "sync", Command: []string{"sync"}, Outputs: []string{"RELEASE"}}, @@ -26,13 +26,13 @@ func TestValidateJobExecution(t *testing.T) { {"valid", func(w *Workload) {}, ""}, {"single command", func(w *Workload) { w.Execution.Steps = nil }, ""}, {"disabled", func(w *Workload) { w.Execution = nil; w.Role = "application" }, ""}, - {"default when", func(w *Workload) { w.When = "" }, ""}, + {"default when", func(w *Workload) { w.DeploymentPhase = "" }, ""}, {"retention boundary", func(w *Workload) { w.Execution.Retention = "30d" }, ""}, - {"not job", func(w *Workload) { w.Role = "application" }, "native scheduled manual job"}, - {"not scheduled", func(w *Workload) { w.Schedule = nil }, "native scheduled manual job"}, - {"release hook", func(w *Workload) { w.When = "pre_release" }, "native scheduled manual job"}, - {"migration", func(w *Workload) { w.DataEffect = DataEffectMigration }, "native scheduled manual job"}, - {"compose", func(w *Workload) { w.Compose = "compose.yml#job" }, "native scheduled manual job"}, + {"not job", func(w *Workload) { w.Role = "application" }, "native scheduled operator-runnable phase-none job"}, + {"not scheduled", func(w *Workload) { w.Schedule = nil }, "native scheduled operator-runnable phase-none job"}, + {"release hook", func(w *Workload) { w.DeploymentPhase = "pre_release" }, "native scheduled operator-runnable phase-none job"}, + {"migration", func(w *Workload) { w.DataEffect = DataEffectMigration }, "native scheduled operator-runnable phase-none job"}, + {"compose", func(w *Workload) { w.Compose = "compose.yml#job" }, "native scheduled operator-runnable phase-none job"}, {"retention invalid", func(w *Workload) { w.Execution.Retention = "forever" }, "retention"}, {"retention zero", func(w *Workload) { w.Execution.Retention = "0s" }, "retention"}, {"retention negative", func(w *Workload) { w.Execution.Retention = "-1h" }, "retention"}, diff --git a/internal/app/jsonschema.go b/internal/app/jsonschema.go index 6678e99e..1b519ce7 100644 --- a/internal/app/jsonschema.go +++ b/internal/app/jsonschema.go @@ -340,7 +340,8 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "role"}, enum(eRole)}, {[]string{"workloads", "*", "replicas"}, map[string]any{"minimum": 1}}, {[]string{"workloads", "*", "strategy"}, enum(eStrategy)}, - {[]string{"workloads", "*", "when"}, enum(eJobWhen)}, + {[]string{"workloads", "*", "deployment_phase"}, enum(eJobDeploymentPhase)}, + {[]string{"workloads", "*", "operator_run"}, enum(eJobOperatorRun)}, {[]string{"workloads", "*", "data_effect"}, enum(eDataEffect)}, {[]string{"workloads", "*", "compose"}, pattern(gComposeRef)}, {[]string{"workloads", "*", "port"}, portBounds()}, @@ -428,6 +429,7 @@ var schemaConstraints = []struct { {[]string{"workloads", "*", "schedule", "cron"}, pattern(gCron)}, {[]string{"workloads", "*", "schedule", "timezone"}, pattern(gTZ)}, {[]string{"workloads", "*", "schedule", "timeout"}, pattern(gDur)}, + {[]string{"workloads", "*", "schedule", "shutdown_grace"}, pattern(gDur)}, {[]string{"workloads", "*", "schedule", "deploy_lock"}, enum(eScheduleDeployLock)}, {[]string{"services", "*", "driver"}, pattern(gIdent)}, @@ -533,14 +535,18 @@ func applyRoleRules(doc map[string]any) { map[string]any{"anyOf": anyRequired(sources)}, } - jobOnly := []any{"when", "data_effect", "schedule", "inputs", "execution"} + jobOnly := []any{"deployment_phase", "operator_run", "data_effect", "schedule", "inputs", "execution"} workload["allOf"] = []any{ map[string]any{ "if": map[string]any{"required": []any{"execution"}}, "then": map[string]any{ - "required": []any{"schedule", "data_effect"}, - "properties": map[string]any{"data_effect": map[string]any{"const": "none"}, "when": map[string]any{"const": "manual"}}, - "not": map[string]any{"required": []any{"compose"}}, + "required": []any{"schedule", "data_effect"}, + "properties": map[string]any{ + "data_effect": map[string]any{"const": "none"}, + "deployment_phase": map[string]any{"const": "none"}, + "operator_run": map[string]any{"const": "allowed"}, + }, + "not": map[string]any{"required": []any{"compose"}}, }, }, // Exactly one source. A workload with none cannot run and a workload diff --git a/internal/app/jsonschema_test.go b/internal/app/jsonschema_test.go index d347c320..7d04b1ce 100644 --- a/internal/app/jsonschema_test.go +++ b/internal/app/jsonschema_test.go @@ -124,7 +124,7 @@ func TestPublishedSchemaAcceptsEveryRealProject(t *testing.T) { func TestPublishedSchemaRequiresExecutionStepIDAndCommand(t *testing.T) { schema := compiledSchema(t) for _, step := range []string{`{id: sync, command: [echo, ok]}`, `{command: [echo, ok]}`, `{id: sync}`, `{}`} { - y := "api_version: onebox.run/v1\napp: a\nenvironments: {p: {server: root@h}}\nworkloads:\n sync:\n role: job\n image: busybox\n when: manual\n data_effect: none\n schedule: {cron: '0 * * * *'}\n execution:\n steps: [" + step + "]\n" + y := "api_version: onebox.run/v1\napp: a\nenvironments: {p: {server: root@h}}\nworkloads:\n sync:\n role: job\n image: busybox\n deployment_phase: none\n data_effect: none\n schedule: {cron: '0 * * * *'}\n execution:\n steps: [" + step + "]\n" err := schema.Validate(asJSON(t, y)) valid := strings.Contains(step, "id:") && strings.Contains(step, "command:") if (err == nil) != valid { diff --git a/internal/app/load_test.go b/internal/app/load_test.go index b9c92ce4..cf479c2b 100644 --- a/internal/app/load_test.go +++ b/internal/app/load_test.go @@ -90,7 +90,7 @@ func conformanceCases() []conformanceCase { {"job with data_effect", wl("j: {image: nginx, role: job, data_effect: none}"), true}, {"job data_effect unknown", wl("j: {image: nginx, role: job, data_effect: unknown}"), true}, {"application with data_effect", wl("w: {image: nginx, data_effect: none}"), false}, - {"application with when", wl("w: {image: nginx, when: manual}"), false}, + {"application with when", wl("w: {image: nginx, deployment_phase: none}"), false}, {"worker with schedule", wl("w: {image: nginx, role: worker, schedule: {cron: \"0 3 * * *\"}}"), false}, {"scheduled job", wl("j: {image: nginx, role: job, data_effect: none, schedule: {cron: \"0 4 * * *\"}}"), true}, {"scheduled job run policy", wl("j: {image: nginx, role: job, data_effect: none, schedule: {cron: \"0 4 * * *\", timeout: 45m, catch_up: false}}"), true}, @@ -189,7 +189,7 @@ func conformanceCases() []conformanceCase { {"recreate workload with published host port", wl("w: {image: nginx, strategy: recreate, published_ports: [{host: 8555, container: 8555}]}"), true}, // Manual jobs remain part of the release runtime even though deployment // execution no longer selects them as an automatic release phase. - {"explicit manual job remains a runtime service", wl("j: {image: nginx, role: job, when: manual, data_effect: none}"), true}, + {"explicit operator job remains a runtime service", wl("j: {image: nginx, role: job, deployment_phase: none, data_effect: none}"), true}, {"service scalar", min + "services: {postgres: 18}\n", true}, {"service backup policy", validBackupProject, true}, {"external service connection", validExternalServiceProject, true}, diff --git a/internal/app/names.go b/internal/app/names.go index 0cf6c312..8de7a819 100644 --- a/internal/app/names.go +++ b/internal/app/names.go @@ -230,8 +230,8 @@ func (n Names) ScheduledJobRunState(job string) string { return path.Join(n.AppDir(), "schedule", job+".state") } -// ScheduledJobRunInputs is the one-shot file `ob schedule run` leaves for the -// next manual activation. The runner consumes and deletes it. +// ScheduledJobRunInputs is the one-shot file `ob job run` leaves for the +// next operator activation. The runner consumes and deletes it. func (n Names) ScheduledJobRunInputs(job string) string { return path.Join(n.AppDir(), "schedule", job+".inputs") } diff --git a/internal/app/purity_test.go b/internal/app/purity_test.go index 531c2890..30ecc7f7 100644 --- a/internal/app/purity_test.go +++ b/internal/app/purity_test.go @@ -47,7 +47,7 @@ workloads: role: job image: nginx:1.27 data_effect: migration - when: pre_release + deployment_phase: pre_release services: postgres: 16 proxy: diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 91a47c7a..9d3246af 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -244,7 +244,7 @@ func (p *Spec) ReleaseOrder() []string { // JobOrder is the stable dependency order for every declared job. It describes // the release runtime, not which jobs a deploy executes; callers that execute a -// release phase must use JobOrderFor so manual jobs remain deploy-inert. +// release phase must use JobOrderFor so phase-none jobs remain deploy-inert. func (p *Spec) JobOrder() []string { var jobs []string for _, name := range sortedKeys(p.Workloads) { @@ -262,12 +262,12 @@ func (p *Spec) JobOrder() []string { // JobOrderFor returns only jobs assigned to one automatic release phase while // preserving the dependency order of the complete job graph. In particular, -// when="manual" is never used by deployment execution. -func (p *Spec) JobOrderFor(when string) []string { +// deployment_phase="none" is never used by deployment execution. +func (p *Spec) JobOrderFor(phase string) []string { ordered := p.JobOrder() out := make([]string, 0, len(ordered)) for _, name := range ordered { - if p.Workloads[name].When == when { + if p.Workloads[name].DeploymentPhase == phase { out = append(out, name) } } diff --git a/internal/app/schedule.go b/internal/app/schedule.go index 8f03fe06..1afcc2df 100644 --- a/internal/app/schedule.go +++ b/internal/app/schedule.go @@ -25,12 +25,15 @@ import ( // ScheduledJob is a job workload that runs on a schedule. type ScheduledJob struct { - Name string - Cron string - Timezone string - Timeout string - CatchUp bool - DeployLock string + Name string + Cron string + Timezone string + Timeout string + ShutdownGrace time.Duration + CatchUp bool + DeployLock string + DeploymentPhase string + OperatorRun string // Calendar is the host-side expression the cron translates to. Calendar string // Retry and notify policy, resolved over the defaults so the runner and @@ -39,12 +42,18 @@ type ScheduledJob struct { RetryBackoff time.Duration RetryMaxBackoff time.Duration Notify []string - // Inputs are the declared parameters a manual run may override. + // Inputs are the declared parameters an operator run may override. Inputs map[string]JobInput Execution *JobExecution DataEffect DataEffect } +// RetryBackoffBudget is the maximum time this job can spend sleeping between +// attempts during one activation. Execution time still shares Timeout. +func (j ScheduledJob) RetryBackoffBudget() time.Duration { + return scheduleRetryWorstCase(j.RetryAttempts, j.RetryBackoff, j.RetryMaxBackoff) +} + // ScheduledJobs lists every job with a schedule, in a stable order. func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { var out []ScheduledJob @@ -67,9 +76,11 @@ func (p *Spec) ScheduledJobs() ([]ScheduledJob, error) { deployLock = "exclusive" } attempts, backoff, maxBackoff := w.Schedule.retryPolicy() + shutdownGrace, _ := ParseDuration(w.Schedule.ShutdownGrace) out = append(out, ScheduledJob{ Name: name, Cron: w.Schedule.Cron, Timezone: tz, Calendar: cal, - Timeout: w.Schedule.Timeout, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + Timeout: w.Schedule.Timeout, ShutdownGrace: shutdownGrace, CatchUp: w.Schedule.CatchUp, DeployLock: deployLock, + DeploymentPhase: w.DeploymentPhase, OperatorRun: w.OperatorRun, RetryAttempts: attempts, RetryBackoff: backoff, RetryMaxBackoff: maxBackoff, Notify: w.Schedule.notifyOutcomes(), Inputs: w.Inputs, Execution: w.Execution, DataEffect: w.DataEffect, }) diff --git a/internal/app/testdata/contract-verdicts.json b/internal/app/testdata/contract-verdicts.json index 193ae1b4..3f28f5ae 100644 --- a/internal/app/testdata/contract-verdicts.json +++ b/internal/app/testdata/contract-verdicts.json @@ -145,7 +145,7 @@ "digest": "56cbcd764cb9b8ee0cb78468a1cacc7b54a5afa917ccb1b222194727ddb3434c" }, { - "case": "conformance/explicit manual job remains a runtime service", + "case": "conformance/explicit operator job remains a runtime service", "loads": true, "digest": "13d1b0f2e12b3bee654c9611be6461f90c52efdcbc8294597e53f6378bb70c28" }, diff --git a/internal/app/testdata/corpus/ext-plausible.yml b/internal/app/testdata/corpus/ext-plausible.yml index 35a652c5..53574fe8 100644 --- a/internal/app/testdata/corpus/ext-plausible.yml +++ b/internal/app/testdata/corpus/ext-plausible.yml @@ -32,7 +32,7 @@ workloads: /entrypoint.sh db migrate printf '{"schema_version":"onebox.run/job-result/v1alpha1","changed":true}' > "$ONEBOX_RESULT_FILE" data_effect: migration - when: pre_release + deployment_phase: pre_release env: BASE_URL: "http://stats.example.com" SECRET_KEY_BASE: "0123456789012345678901234567890123456789012345678901234567890123" diff --git a/internal/app/testdata/corpus/pursue.yml b/internal/app/testdata/corpus/pursue.yml index fe2103bb..56fbc7da 100644 --- a/internal/app/testdata/corpus/pursue.yml +++ b/internal/app/testdata/corpus/pursue.yml @@ -17,7 +17,7 @@ workloads: migrate: role: job compose: compose.yaml#migrate - when: pre_release + deployment_phase: pre_release data_effect: migration postgres: role: daemon diff --git a/internal/app/testdata/corpus/recast.yml b/internal/app/testdata/corpus/recast.yml index cd77a656..e5490c98 100644 --- a/internal/app/testdata/corpus/recast.yml +++ b/internal/app/testdata/corpus/recast.yml @@ -22,7 +22,7 @@ workloads: migrate: role: job compose: compose.yaml#migrate - when: pre_release + deployment_phase: pre_release data_effect: migration postgres: role: daemon diff --git a/internal/app/types.go b/internal/app/types.go index 59b73b74..973f33c0 100644 --- a/internal/app/types.go +++ b/internal/app/types.go @@ -168,11 +168,12 @@ type Workload struct { Logging *Logging `json:"logging,omitempty" description:"Container logging driver and driver-specific options."` // Job only. - When string `json:"when,omitempty" description:"When a job runs: manual, pre_release, or post_release." default:"manual"` - DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` - Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job."` - Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them."` - Execution *JobExecution `json:"execution,omitempty" description:"Opt-in durable scheduled execution. Requires a native manual job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume."` + DeploymentPhase string `json:"deployment_phase,omitempty" description:"Deployment phase for this job: none, pre_release, or post_release." default:"none"` + OperatorRun string `json:"operator_run,omitempty" description:"Whether an operator may invoke this job outside deployment: allowed or disabled. Defaults to allowed for phase none and disabled otherwise."` + DataEffect DataEffect `json:"data_effect,omitempty" description:"Job data impact used by rollback and abort gates." example:"migration"` + Schedule *JobSchedule `json:"schedule,omitempty" description:"Host-resident recurring schedule and run policy for a job, independent of its deployment phase and operator-run policy."` + Inputs map[string]JobInput `json:"inputs,omitempty" description:"Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob job run may override them."` + Execution *JobExecution `json:"execution,omitempty" description:"Opt-in durable scheduled execution. Requires a native operator-runnable phase-none job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume."` } type JobExecution struct { @@ -189,13 +190,13 @@ type JobStep struct { } // JobInput is one declared parameter of a scheduled job. The constraint is -// what makes a manual run safe to accept from a command line: a value is +// what makes an operator run safe to accept from a command line: a value is // either one of the listed words or matches the pattern, and never contains a // character the runner would have to escape. type JobInput struct { Enum []string `json:"enum,omitempty" description:"Accepted values." example:"catalog"` Pattern string `json:"pattern,omitempty" description:"Regular expression the whole value must match." example:"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"` - Default string `json:"default" description:"Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint."` + Default string `json:"default" description:"Value used by a timer firing and by an operator run that does not override it. Must satisfy the input's own constraint."` Description string `json:"description,omitempty" description:"What the input controls."` } @@ -292,13 +293,14 @@ type Schedule struct { } type JobSchedule struct { - Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` - Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` - Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` - CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` - DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` - Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` - Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` + Cron string `json:"cron" description:"Five-field cron schedule translated to a host timer." example:"0 2 * * *"` + Timezone string `json:"timezone" description:"IANA timezone used to interpret the cron schedule." default:"UTC" example:"Europe/Berlin"` + Timeout string `json:"timeout" description:"Maximum wall time for one scheduled run before systemd terminates it and records failure." default:"1h" example:"30m"` + ShutdownGrace string `json:"shutdown_grace" description:"Time allowed for graceful container shutdown after the run deadline before Onebox forces removal." default:"30s" example:"45s"` + CatchUp bool `json:"catch_up" description:"Run once after the host returns if an elapsed schedule was missed while it was offline." default:"true"` + DeployLock string `json:"deploy_lock" description:"Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks." default:"exclusive" example:"pinned"` + Retry *JobRetry `json:"retry,omitempty" description:"Bounded retry inside one timer firing. Attempts run under the same locks and the same timeout; a timeout ends the run."` + Notify []string `json:"notify,omitempty" description:"Run outcomes that send the configured notifications: success, failure, timeout, skipped." default:"failure, timeout"` } // JobRetry bounds how a scheduled run recovers from a transient failure. The diff --git a/internal/app/validate.go b/internal/app/validate.go index a48ad225..b15d0b4e 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -434,7 +434,10 @@ func validateWorkload(w Workload, path string) error { } } if w.IsJob() { - if err := checkEnum(path+".when", w.When, eJobWhen); err != nil { + if err := checkEnum(path+".deployment_phase", w.DeploymentPhase, eJobDeploymentPhase); err != nil { + return err + } + if err := checkEnum(path+".operator_run", w.OperatorRun, eJobOperatorRun); err != nil { return err } if err := checkEnum(path+".data_effect", string(w.DataEffect), eDataEffect); err != nil { @@ -461,9 +464,9 @@ func validateWorkload(w Workload, path string) error { if err := validateJobInputs(w, path); err != nil { return err } - } else if w.When != "" || w.DataEffect != "" || w.Schedule != nil || len(w.Inputs) > 0 { + } else if w.DeploymentPhase != "" || w.OperatorRun != "" || w.DataEffect != "" || w.Schedule != nil || len(w.Inputs) > 0 { return errf("project_invalid", path, "", - "when, data_effect, schedule and inputs belong to a job; this workload's role is %q", w.Role) + "deployment_phase, operator_run, data_effect, schedule and inputs belong to a job; this workload's role is %q", w.Role) } return nil } @@ -586,6 +589,9 @@ func validateJobSchedule(s *JobSchedule, path string) error { if err := gDur.check(path+".timeout", s.Timeout); err != nil { return err } + if err := gDur.check(path+".shutdown_grace", s.ShutdownGrace); err != nil { + return err + } if err := validateJobRetry(s, path); err != nil { return err } diff --git a/internal/durable/runner.py b/internal/durable/runner.py index d48791d1..167dc4b2 100644 --- a/internal/durable/runner.py +++ b/internal/durable/runner.py @@ -235,7 +235,7 @@ def container_running(name): ) -def cleanup_container(config, invocation=None): +def cleanup_container(config, invocation=None, shutdown_grace=30, state_path=None): ids = docker( ["ps", "-aq", "--filter", "name=^/" + config["container"] + "$"] ).split() @@ -266,9 +266,29 @@ def cleanup_container(config, invocation=None): labels.get("ob.execution.invocation") == invocation, "container belongs to another invocation", ) + running = row["State"].get("Running") or row["State"].get("Restarting") + forced = False + if running: + docker(["kill", "--signal", "TERM", row["Id"]], capture=False) + deadline = time.monotonic() + shutdown_grace + while container_running(config["container"]): + if time.monotonic() >= deadline: + docker(["kill", "--signal", "KILL", row["Id"]], capture=False) + forced = True + break + time.sleep(min(0.1, max(0, deadline - time.monotonic()))) + if forced and state_path is not None: + with open(state_path, "a", encoding="utf-8") as state: + state.write("forced_kill=true\n") + remaining = docker( + ["ps", "-aq", "--filter", "name=^/" + config["container"] + "$"] + ).split() + if row["Id"] in remaining: + docker(["rm", "-f", row["Id"]], capture=False) + continue # Pre-attempt cleanup never forces removal: Docker must also refuse if # the stopped container starts after our inspection. - docker(["rm"] + (["-f"] if invocation is not None else []) + [row["Id"]]) + docker(["rm", row["Id"]]) def compatibility(config, release_dir): @@ -464,7 +484,7 @@ def update_run_status(store, value, invocation): atomic_bytes(path, ("\n".join(lines) + "\n").encode()) -def execute(store, identity, invocation): +def execute(store, identity, invocation, shutdown_grace=30): value = store.read(identity) require( value["invocation"] == invocation and value["state"] == "running", @@ -583,7 +603,12 @@ def interrupted(_signum, _frame): attempt["reason"] = "output validation or execution failed" print("onebox: " + str(error), file=sys.stderr) finally: - cleanup_container(config, invocation) + cleanup_container( + config, + invocation, + shutdown_grace, + store.root / "schedule" / (config["job"] + ".state"), + ) attempt["finished_at"] = time.time() save_owned(store, value, invocation) if step["state"] == "succeeded": @@ -683,7 +708,7 @@ def abandon(store, identity): def main(args): counts = { "prepare": 8, - "run": 4, + "run": 5, "inspect": 3, "list": 3, "pins": 2, @@ -700,7 +725,7 @@ def main(args): config = json.loads(base64.b64decode(args[2])) print(prepare(store, config, *args[3:7], json.loads(args[7]))) elif command == "run": - return execute(store, *args[2:4]) + return execute(store, *args[2:4], float(args[4])) elif command == "inspect": print(json.dumps(view(store.read(args[2]), store.root))) elif command == "list": diff --git a/internal/durable/runner_test.py b/internal/durable/runner_test.py index c4adbaad..f6391ebd 100644 --- a/internal/durable/runner_test.py +++ b/internal/durable/runner_test.py @@ -352,6 +352,61 @@ def test_cleanup_refuses_another_invocations_container(self): r.cleanup_container(self.config, self.invocation) self.assertEqual(docker.call_count, 2) + def test_owned_running_container_gets_term_before_removal(self): + row = { + "Id": "container", + "State": {"Running": True, "Restarting": False}, + "Config": { + "Labels": { + "ob.execution.job": "refresh", + "ob.execution.invocation": self.invocation, + } + }, + } + with ( + patch.object( + r, + "docker", + side_effect=["container", json.dumps([row]), "", ""], + ) as docker, + patch.object(r, "container_running", return_value=False), + ): + r.cleanup_container(self.config, self.invocation) + calls = [call.args[0] for call in docker.call_args_list] + self.assertIn(["kill", "--signal", "TERM", "container"], calls) + self.assertNotIn(["kill", "--signal", "KILL", "container"], calls) + + def test_owned_running_container_records_forced_kill_after_grace(self): + row = { + "Id": "container", + "State": {"Running": True, "Restarting": False}, + "Config": { + "Labels": { + "ob.execution.job": "refresh", + "ob.execution.invocation": self.invocation, + } + }, + } + state = self.root / "schedule" / "refresh.state" + state.parent.mkdir(parents=True) + state.write_text("phase=stopping\n") + with ( + patch.object( + r, + "docker", + side_effect=["container", json.dumps([row]), "", "", "container", ""], + ) as docker, + patch.object(r, "container_running", return_value=True), + patch.object(r.time, "monotonic", return_value=0), + ): + r.cleanup_container(self.config, self.invocation, 0, state) + calls = [call.args[0] for call in docker.call_args_list] + self.assertLess( + calls.index(["kill", "--signal", "TERM", "container"]), + calls.index(["kill", "--signal", "KILL", "container"]), + ) + self.assertIn("forced_kill=true", state.read_text()) + def test_legacy_cleanup_requires_stopped_matching_compose_job(self): original = { "Id": "legacy", diff --git a/internal/engine/finalize_test.go b/internal/engine/finalize_test.go index 8322f163..f4e6fb5f 100644 --- a/internal/engine/finalize_test.go +++ b/internal/engine/finalize_test.go @@ -152,7 +152,7 @@ var engineProjectWithScheduledJob = strings.Replace(engineProject, "services:", role: job image: ghcr.io/x/app:v2 command: report - when: manual + deployment_phase: none data_effect: none schedule: {cron: "0 2 * * *", timezone: UTC} services:`, 1) diff --git a/internal/engine/findincomplete_test.go b/internal/engine/findincomplete_test.go index 1ade428b..37f5e083 100644 --- a/internal/engine/findincomplete_test.go +++ b/internal/engine/findincomplete_test.go @@ -45,7 +45,7 @@ func TestFindIncompleteIgnoresADeploySupersededByANewerOne(t *testing.T) { } } -// A journal that is not a deploy at all — a manual job, a service apply — must +// A journal that is not a deploy at all — a operator job, a service apply — must // not be mistaken for the newest deploy and hide the incomplete one behind it. func TestFindIncompleteLooksPastNonDeployJournals(t *testing.T) { out := journalMarkerLine + "R1.jsonl\n" + diff --git a/internal/engine/fixtures_test.go b/internal/engine/fixtures_test.go index abdc00a9..667d90d6 100644 --- a/internal/engine/fixtures_test.go +++ b/internal/engine/fixtures_test.go @@ -41,7 +41,7 @@ workloads: role: job image: ghcr.io/x/app:v2 command: migrate - when: pre_release + deployment_phase: pre_release data_effect: unknown services: postgres: diff --git a/internal/engine/gate_test.go b/internal/engine/gate_test.go index b295d342..618a9e24 100644 --- a/internal/engine/gate_test.go +++ b/internal/engine/gate_test.go @@ -153,9 +153,9 @@ func TestJobAutoRunsWithoutHook(t *testing.T) { func TestDeployRunsOnlyAutomaticJobsInTheirDeclaredPhase(t *testing.T) { cfg := testConfig() - cfg.Workloads["cleanup"] = app.Workload{Role: app.RoleJob, When: "post_release", DataEffect: "none"} + cfg.Workloads["cleanup"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "post_release", DataEffect: "none"} cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", } cfg.Hooks["cleanup"] = app.Command{Run: "echo POST_RELEASE_JOB_MARKER"} cfg.Hooks["nightly"] = app.Command{Run: "echo MANUAL_JOB_MARKER"} @@ -166,7 +166,7 @@ func TestDeployRunsOnlyAutomaticJobsInTheirDeclaredPhase(t *testing.T) { } seq := strings.Join(f.Commands, "\n") if strings.Contains(seq, "MANUAL_JOB_MARKER") { - t.Fatalf("manual job executed during deploy:\n%s", seq) + t.Fatalf("operator job executed during deploy:\n%s", seq) } releaseAt := strings.Index(seq, "--force-recreate --timeout 30 worker") postJobAt := strings.Index(seq, "POST_RELEASE_JOB_MARKER") @@ -337,7 +337,7 @@ func TestExpandOnlyPromiseOverridesClosedGate(t *testing.T) { f := gateFake("") // silent migrate cfg := testConfig() cfg.Deployment.MigrationPolicy = "expand-only" - cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, When: "pre_release", DataEffect: "migration"} + cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"} e := New(cfg, testProject(t), f, Options{ Out: &bytes.Buffer{}, Sleep: noSleep, ApprovalDigest: "sha256:approved", ApprovalClass: "strong", AllowUnknownMigration: true, @@ -351,7 +351,7 @@ func TestExpandOnlyPromiseOverridesClosedGate(t *testing.T) { func TestDataEffectNoneOpensGateWithoutResultFile(t *testing.T) { f := gateFake("") cfg := testConfig() - cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, When: "pre_release", DataEffect: "none"} + cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "none"} e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) err := e.Deploy(context.Background(), engineTestDeployReleaseID, t.TempDir()) if err == nil || !strings.Contains(err.Error(), "auto-rolled back") { @@ -363,7 +363,7 @@ func TestExpandOnlyDoesNotCoverUnknownJob(t *testing.T) { f := gateFake("") cfg := testConfig() cfg.Deployment.MigrationPolicy = "expand-only" - cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, When: "pre_release", DataEffect: "unknown"} + cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "unknown"} e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) err := e.Deploy(context.Background(), engineTestDeployReleaseID, t.TempDir()) if err == nil || !strings.Contains(err.Error(), "HALT-AND-PAGE") { @@ -375,7 +375,7 @@ func TestExpandOnlyDoesNotCoverLifecycleHook(t *testing.T) { f := gateFake("") cfg := testConfig() cfg.Deployment.MigrationPolicy = "expand-only" - cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, When: "pre_release", DataEffect: "migration"} + cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"} cfg.Hooks["pre_release"] = app.Command{Run: "true"} e := New(cfg, testProject(t), f, Options{ Out: &bytes.Buffer{}, Sleep: noSleep, diff --git a/internal/engine/job.go b/internal/engine/job.go index c0e9240d..30d340d6 100644 --- a/internal/engine/job.go +++ b/internal/engine/job.go @@ -12,7 +12,7 @@ import ( "github.com/labstack/onebox/internal/release" ) -// RunJobWithJournalID executes one current-release manual job under the same +// RunJobWithJournalID executes one current-release operator job under the same // lock, fence, approval evidence, result protocol, and journal authority used // by deployment jobs. The expected release/runtime checks run after the lock // is held, closing the plan-to-execution race before container creation. @@ -34,8 +34,8 @@ func (e *Engine) RunJobWithJournalID(ctx context.Context, request JobRunRequest) if !ok || !workload.IsJob() { return operationID, nil, fmt.Errorf("unknown job %q", job) } - if workload.When != "manual" { - return operationID, nil, fmt.Errorf("job %q is not a manual job", job) + if workload.OperatorRun != "allowed" { + return operationID, nil, fmt.Errorf("job %q does not allow operator runs", job) } if workload.DataEffect != request.ExpectedDataEffect { return operationID, nil, errors.New("job data effect changed since planning — re-plan") @@ -117,7 +117,7 @@ func (e *Engine) RunJobWithJournalID(ctx context.Context, request JobRunRequest) } start := journal.Record{ Phase: "job", Event: "start", Status: "ok", OperationKind: "job_run", Service: job, - Detail: "release=" + current, + Detail: "release=" + current, ReleaseID: current, DataEffect: string(workload.DataEffect), } if err := writer.Append(ctx, start); err != nil { return operationID, nil, fmt.Errorf("journal job start: %w", err) diff --git a/internal/engine/job_history.go b/internal/engine/job_history.go new file mode 100644 index 00000000..9942a595 --- /dev/null +++ b/internal/engine/job_history.go @@ -0,0 +1,145 @@ +package engine + +import ( + "context" + "fmt" + "io" + "sort" + "time" + + "github.com/labstack/onebox/internal/journal" +) + +// JobHistoryRecord is the common read model for timer, operator-submitted, +// and sealed job executions. The stores remain independent; Operation joins +// the workstation audit lifecycle to a host-supervised activation. +type JobHistoryRecord struct { + ID string `json:"id"` + Run string `json:"run,omitempty"` + Operation string `json:"operation,omitempty"` + Job string `json:"job"` + Trigger string `json:"trigger"` + Release string `json:"release,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at,omitempty"` + DurationSeconds int `json:"duration_s,omitempty"` + Attempts int `json:"attempts,omitempty"` + ExitStatus *int `json:"exit_status,omitempty"` + Outcome string `json:"outcome"` + ForcedKill bool `json:"forced_kill,omitempty"` + Reason string `json:"reason,omitempty"` + Operator string `json:"operator,omitempty"` + Inputs map[string]string `json:"inputs,omitempty"` +} + +// JobHistory merges existing host records at read time. It deliberately does +// not introduce another store or imply completeness beyond journal retention. +func (e *Engine) JobHistory(ctx context.Context, name string, n int) ([]JobHistoryRecord, error) { + workload, ok := e.Spec.Workloads[name] + if !ok || !workload.IsJob() { + return nil, fmt.Errorf("unknown job %q", name) + } + var out []JobHistoryRecord + byOperation := map[string]int{} + if workload.Schedule != nil { + records, err := e.ScheduleHistory(ctx, name, n) + if err != nil { + return nil, err + } + for _, record := range records { + out = append(out, JobHistoryRecord{ + ID: record.Run, Run: record.Run, Operation: record.Operation, Job: name, + Trigger: record.Trigger, Release: record.Release, StartedAt: record.StartedAt, + FinishedAt: record.FinishedAt, DurationSeconds: record.DurationSeconds, + Attempts: record.Attempts, ExitStatus: record.ExitStatus, Outcome: record.Outcome, + ForcedKill: record.ForcedKill, Reason: record.Reason, Inputs: record.Inputs, + }) + if record.Operation != "" { + byOperation[record.Operation] = len(out) - 1 + } + } + } + + ids, journals, err := journal.Journals(ctx, e.T, e.names()) + if err != nil { + return nil, err + } + for _, id := range ids { + records := journals[id] + var start, finish *journal.Record + for i := range records { + record := &records[i] + // Target supports records written before schedule-run populated the + // common Service field. + if (record.Service != name && record.Target != name) || (record.Phase != "job" && record.Phase != "schedule-run") { + continue + } + if record.Event == "start" && start == nil { + start = record + } + if record.Event == "finish" || record.Event == "abort" { + finish = record + } + } + if start == nil { + continue + } + if index, exists := byOperation[start.DeployID]; exists { + out[index].Operator = start.Operator + continue + } + if start.Phase != "job" { + continue // a host record is the outcome authority for schedule-run + } + record := JobHistoryRecord{ + ID: start.DeployID, Operation: start.DeployID, Job: name, Trigger: "operator", + Release: start.ReleaseID, StartedAt: start.TS, Operator: start.Operator, + Attempts: 1, Outcome: "incomplete", + } + if finish != nil { + record.FinishedAt = finish.TS + record.DurationSeconds = elapsedSeconds(start.TS, finish.TS) + switch { + case finish.ErrorCode == "interrupted": + record.Outcome = "interrupted" + case finish.Event == "abort": + record.Outcome = "aborted" + case finish.Status == "ok": + record.Outcome = "success" + default: + record.Outcome = "failure" + } + } + out = append(out, record) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].StartedAt > out[j].StartedAt }) + if n <= 0 { + n = 20 + } + if len(out) > n { + out = out[:n] + } + return out, nil +} + +func elapsedSeconds(started, finished string) int { + start, startErr := time.Parse(time.RFC3339, started) + finish, finishErr := time.Parse(time.RFC3339, finished) + if startErr != nil || finishErr != nil || finish.Before(start) { + return 0 + } + return int(finish.Sub(start).Seconds()) +} + +// JobLogs streams exact logs for host-supervised executions. Sealed attached +// executions have durable outcome evidence but no separate retained log. +func (e *Engine) JobLogs(ctx context.Context, name, run string, stdout, stderr io.Writer) (string, error) { + workload, ok := e.Spec.Workloads[name] + if !ok || !workload.IsJob() { + return "", fmt.Errorf("unknown job %q", name) + } + if workload.Schedule == nil { + return "", fmt.Errorf("job %s has no host-supervised run logs", name) + } + return e.ScheduleLogs(ctx, name, run, stdout, stderr) +} diff --git a/internal/engine/job_history_test.go b/internal/engine/job_history_test.go new file mode 100644 index 00000000..1b2e0d3a --- /dev/null +++ b/internal/engine/job_history_test.go @@ -0,0 +1,50 @@ +package engine + +import ( + "context" + "strings" + "testing" + + "github.com/labstack/onebox/internal/transport" +) + +func TestJobHistoryMergesTimerAndOperatorStoresByOperation(t *testing.T) { + e, f := scheduledFixture(t) + hostRecords := `{"run":"11111111111111111111111111111111","job":"nightly","trigger":"operator","operation":"op-host","release":"release-a","started_at":"2026-09-16T03:00:00Z","finished_at":"2026-09-16T03:00:04Z","duration_s":4,"attempts":1,"exit_status":0,"outcome":"success","inputs":{"SOURCE":"prices"}} +{"run":"22222222222222222222222222222222","job":"nightly","trigger":"timer","release":"release-a","started_at":"2026-09-16T02:00:00Z","finished_at":"2026-09-16T02:00:03Z","duration_s":3,"attempts":1,"exit_status":1,"outcome":"failure","inputs":{}} +` + journals := `@@ob-journal@@op-direct.jsonl +{"deploy_id":"op-direct","phase":"job","event":"start","status":"ok","ts":"2026-09-16T04:00:00Z","operator":"bob@example","service":"nightly","release_id":"release-a"} +{"deploy_id":"op-direct","phase":"job","event":"finish","status":"ok","ts":"2026-09-16T04:00:05Z","service":"nightly"} +@@ob-journal@@op-host.jsonl +{"deploy_id":"op-host","phase":"schedule-run","event":"start","status":"ok","ts":"2026-09-16T02:59:59Z","operator":"alice@example","target":"nightly"} +{"deploy_id":"op-host","phase":"schedule-run","event":"finish","status":"ok","ts":"2026-09-16T03:00:04Z","target":"nightly"} +` + f.Dynamic = func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "journalctl"): + return transport.Result{Stdout: hostRecords}, true + case strings.Contains(cmd, "@@ob-journal@@"): + return transport.Result{Stdout: journals}, true + default: + return transport.Result{}, false + } + } + + records, err := e.JobHistory(context.Background(), "nightly", 20) + if err != nil { + t.Fatal(err) + } + if len(records) != 3 { + t.Fatalf("records = %#v", records) + } + if records[0].ID != "op-direct" || records[0].Trigger != "operator" || records[0].Outcome != "success" || records[0].DurationSeconds != 5 { + t.Fatalf("direct operator record = %#v", records[0]) + } + if records[1].ID != "11111111111111111111111111111111" || records[1].Operator != "alice@example" || records[1].Inputs["SOURCE"] != "prices" { + t.Fatalf("host-supervised operator record was not enriched in place: %#v", records[1]) + } + if records[2].ID != "22222222222222222222222222222222" || records[2].Trigger != "timer" || records[2].Outcome != "failure" { + t.Fatalf("timer record = %#v", records[2]) + } +} diff --git a/internal/engine/job_test.go b/internal/engine/job_test.go index 51121816..921fa63f 100644 --- a/internal/engine/job_test.go +++ b/internal/engine/job_test.go @@ -21,7 +21,8 @@ func manualJobEngineTo(t *testing.T, target *transport.Fake, out *bytes.Buffer) t.Helper() config := testConfig() job := config.Workloads["migrate"] - job.When = "manual" + job.DeploymentPhase = "none" + job.OperatorRun = "allowed" job.DataEffect = "none" config.Workloads["migrate"] = job return New(config, testProject(t), target, Options{ @@ -60,14 +61,14 @@ func TestRunJobRejectsDeclarationDriftBeforeLock(t *testing.T) { want: "unknown job", }, { - name: "not manual", + name: "operator disabled", mutate: func(engine *Engine) { job := engine.Spec.Workloads["migrate"] - job.When = "pre_release" + job.OperatorRun = "disabled" engine.Spec.Workloads["migrate"] = job }, request: JobRunRequest{OperationID: "op-2", Job: "migrate", ExpectedDataEffect: "none"}, - want: "not a manual job", + want: "does not allow operator runs", }, { name: "data effect changed", diff --git a/internal/engine/lock.go b/internal/engine/lock.go index 3055deac..b10487e8 100644 --- a/internal/engine/lock.go +++ b/internal/engine/lock.go @@ -154,7 +154,7 @@ func (e *Engine) acquireLock(ctx context.Context, deployID string, force bool, l return epoch, nil } if useScheduleLock && res.ExitCode == flockConflictExitCode { - return 0, fmt.Errorf("application scheduling rendezvous remained busy — wait for the current scheduled job or application operation to finish") + return 0, fmt.Errorf("scheduling rendezvous remained busy — run `ob status` to see active jobs, then wait for the current job or application operation to finish") } if res.ExitCode != applicationLockHeldExitCode { detail := strings.TrimSpace(res.Stderr) diff --git a/internal/engine/lock_test.go b/internal/engine/lock_test.go index 1474c752..8288726d 100644 --- a/internal/engine/lock_test.go +++ b/internal/engine/lock_test.go @@ -137,7 +137,7 @@ func TestAtomicApplicationLockCreatePublishesOnlyCompleteMetadata(t *testing.T) func TestAcquireLockSerializesWithScheduledJobs(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -151,7 +151,7 @@ func TestAcquireLockSerializesWithScheduledJobs(t *testing.T) { }} e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) _, err := e.AcquireLock(context.Background(), "R9", false) - if err == nil || !strings.Contains(err.Error(), "scheduled job or application operation") { + if err == nil || !strings.Contains(err.Error(), "current job or application operation") { t.Fatalf("error = %v, want schedule-rendezvous contention", err) } seq := strings.Join(f.Commands, "\n") diff --git a/internal/engine/resume_test.go b/internal/engine/resume_test.go index 1d547127..065e52f1 100644 --- a/internal/engine/resume_test.go +++ b/internal/engine/resume_test.go @@ -229,7 +229,7 @@ func TestResumeRestoresOnlyExplicitUnknownMigrationAuthority(t *testing.T) { t.Run(tt.name, func(t *testing.T) { f := interruptedBeforeMigrationFake(tt.allowed) cfg := testConfig() - cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, When: "pre_release", DataEffect: "migration"} + cfg.Workloads["migrate"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"} var out bytes.Buffer e := New(cfg, testProject(t), f, Options{Out: &out, Sleep: noSleep}) err := e.Resume(context.Background()) @@ -283,7 +283,7 @@ func TestAbortUsesInterruptedEffectPolicyAfterConfigEdit(t *testing.T) { cfg.Workloads = map[string]app.Workload{ // The current config now claims this is a covered migration. Abort must // still honor the interrupted journal, which recorded it as uncovered. - "migrate": {Role: app.RoleJob, When: "pre_release", DataEffect: "migration"}, + "migrate": {Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"}, } e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) err := e.Abort(context.Background(), false) @@ -297,7 +297,7 @@ func TestAbortExpandOnlyDoesNotCoverLifecycleHook(t *testing.T) { cfg := testConfig() cfg.Deployment.MigrationPolicy = "expand-only" cfg.Workloads = map[string]app.Workload{ - "migrate": {Role: app.RoleJob, When: "pre_release", DataEffect: "migration"}, + "migrate": {Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"}, } cfg.Hooks["pre_release"] = app.Command{Run: "true"} e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index fe39b6f8..145e66f6 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -257,8 +257,8 @@ func scheduleRunnerScript(application string, job app.ScheduledJob, names app.Na // job runs whatever `current` points at when it starts. "release_dir=$(readlink -f "+q(names.CurrentLink())+" 2>/dev/null || true)", "release=${release_dir##*/}", - scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", + scheduleContainerRemove(container), + "cleanup() { if [ -f \"$state\" ]; then printf 'phase=stopping\\n' >>\"$state\"; fi; "+scheduleContainerStop(container, job.ShutdownGrace)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", @@ -303,8 +303,8 @@ func pinnedScheduleRunnerScript(application string, job app.ScheduledJob, names // Container cleanup and state bookkeeping are per-job work and must not // keep an application operation waiting behind them. "/usr/bin/flock --unlock 8", - scheduleContainerCleanup(container), - "cleanup() { "+scheduleContainerCleanup(container)+"; rm -f \"$tmp\"; }", + scheduleContainerRemove(container), + "cleanup() { if [ -f \"$state\" ]; then printf 'phase=stopping\\n' >>\"$state\"; fi; "+scheduleContainerStop(container, job.ShutdownGrace)+"; rm -f \"$tmp\"; }", "trap cleanup 0", "trap 'exit 129' 1", "trap 'exit 130' 2", @@ -337,9 +337,9 @@ func scheduleLockLines(names app.Names, job, deployLock, applicationLock string, } waitSeconds := strconv.FormatFloat(rendezvousWait.Seconds(), 'f', -1, 64) waitMode := "--timeout " + waitSeconds - busyReason := "the application scheduling lock is busy" + busyReason := "the scheduling rendezvous is busy" if rendezvousWait > 0 { - busyReason = "the application scheduling lock remained busy for " + rendezvousWait.String() + busyReason = "the scheduling rendezvous remained busy for " + rendezvousWait.String() } else { // util-linux documents --timeout 0 as equivalent to --nonblock, but // spelling the mode explicitly makes the zero-budget contract clear. @@ -355,8 +355,8 @@ func scheduleLockLines(names app.Names, job, deployLock, applicationLock string, return []string{ "state=" + q(names.ScheduledJobRunState(job)), "tmp=\"$state.$$\"", - // The operation and inputs of a manual request are kept on the skip - // record too, so `ob schedule run --wait` can find its own outcome. + // The operation and inputs of an operator request are kept on the skip + // record too, so `ob job run` can find its own outcome. // Writing the state requires holding the job lock: it is the run in // flight that owns that file, and overwriting it would replace a real // run's outcome with this one's skip. @@ -411,7 +411,7 @@ func scheduleRendezvousWait(jobTimeout string) time.Duration { // // systemd 252 introduced TRIGGER_UNIT, which is how the runner tells a timer // firing from an operator's start. The floor applies only to a job that -// declares inputs, and to `ob schedule run`; see below for why, and why a host +// declares inputs, and to `ob job run`; see below for why, and why a host // that has been running scheduled jobs for years is not refused one. func (e *Engine) requireScheduleHost(ctx context.Context, jobs []app.ScheduledJob) error { if len(jobs) == 0 { @@ -473,10 +473,29 @@ func needsTriggerUnit(jobs []app.ScheduledJob) bool { return false } -func scheduleContainerCleanup(container string) string { +func scheduleContainerRemove(container string) string { return "/usr/bin/docker rm -f " + q(container) + " >/dev/null 2>&1 || true" } +// scheduleContainerStop gives the container its own TERM grace while the +// runner still owns its flock descriptors. Only Onebox's explicit KILL path +// sets forced_kill; an exit code alone cannot prove how the process stopped. +func scheduleContainerStop(container string, grace time.Duration) string { + seconds := int((grace + time.Second - 1) / time.Second) + if seconds < 1 { + seconds = 1 + } + name := q(container) + return "forced_kill=false; " + + "if [ \"$(/usr/bin/docker inspect -f '{{.State.Running}}' " + name + " 2>/dev/null || true)\" = true ]; then " + + "/usr/bin/docker kill --signal TERM " + name + " >/dev/null 2>&1 || true; " + + "deadline=$(($(date -u '+%s')+" + strconv.Itoa(seconds) + ")); " + + "while [ \"$(/usr/bin/docker inspect -f '{{.State.Running}}' " + name + " 2>/dev/null || true)\" = true ]; do " + + "if [ \"$(date -u '+%s')\" -ge \"$deadline\" ]; then /usr/bin/docker kill --signal KILL " + name + " >/dev/null 2>&1 || true; forced_kill=true; break; fi; sleep 1; done; fi; " + + "/usr/bin/docker rm -f " + name + " >/dev/null 2>&1 || true; " + + "if [ -f \"$state\" ]; then printf 'forced_kill=%s\\n' \"$forced_kill\" >>\"$state\"; fi" +} + // scheduleStateFunction renders the shell function both runners use to record // the run in progress. The notifier reads it after the run ends, so the runner // never removes it: a runner that cleaned up its own state would erase the @@ -485,8 +504,8 @@ func scheduleStateFunction() []string { return []string{ "write_state() {", " umask 077", - " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\ninputs=%s\\n' " + - "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$inputs_json\" >\"$tmp\"", + " printf 'release=%s\\nstarted_at=%s\\nstarted_epoch=%s\\ntrigger=%s\\noperation=%s\\nattempt=%s\\nphase=%s\\ninputs=%s\\n' " + + "\"$release\" \"$started_at\" \"$started_epoch\" \"$trigger\" \"$operation\" \"$1\" \"$phase\" \"$inputs_json\" >\"$tmp\"", " mv -f \"$tmp\" \"$state\"", "}", } @@ -502,18 +521,19 @@ func scheduleRunPreamble(triggerUnit bool) []string { // trigger it cannot observe. otherwise := "unknown" if triggerUnit { - otherwise = "manual" + otherwise = "operator" } return append([]string{ "started_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ')", "started_epoch=$(date -u '+%s')", + "phase=starting", "if [ -n \"${TRIGGER_UNIT:-}\" ]; then trigger=timer; else trigger=" + otherwise + "; fi", }, scheduleStateFunction()...) } -// scheduleInputsLines consumes the one-shot inputs file on a manual +// scheduleInputsLines consumes the one-shot inputs file on an operator // activation. Values reach the container as -e arguments, never as shell -// text, and the file is gone before any lock is taken so a skipped manual run +// text, and the file is gone before any lock is taken so a skipped operator run // cannot hand its inputs to the next timer firing. A timer activation never // opens the file: TRIGGER_UNIT says which one this is. func scheduleInputsLines(inputsPath string) []string { @@ -539,7 +559,7 @@ func scheduleInputsLines(inputsPath string) []string { } } -// schedulePlannedBindingLines makes a sealed manual job plan authoritative at +// schedulePlannedBindingLines makes a sealed operator job plan authoritative at // the point that owns execution: after the host runner has acquired its locks, // immediately before it can start the container. Timer firings carry no // expected binding and pass through unchanged. @@ -573,7 +593,7 @@ func systemdVersion(firstLine string) (int, bool) { // timeout. A single-attempt job gets no loop, so its runner reads as before. func scheduleAttemptLoop(job app.ScheduledJob, compose, container string) []string { if job.RetryAttempts <= 1 { - return []string{"write_state 1", compose} + return []string{"phase=running", "write_state 1", compose} } return []string{ fmt.Sprintf("max_attempts=%d", job.RetryAttempts), @@ -582,16 +602,19 @@ func scheduleAttemptLoop(job app.ScheduledJob, compose, container string) []stri fmt.Sprintf("max_backoff=%d", app.RetryBackoffSeconds(job.RetryMaxBackoff)), "attempt=1", "while :; do", + " phase=running", " write_state \"$attempt\"", // The container name is fixed, so a corpse from the previous attempt // would fail every attempt after it with "name already in use" and // turn one transient failure into all of them. - " " + scheduleContainerCleanup(container), + " " + scheduleContainerRemove(container), " status=0", " " + compose + " || status=$?", " [ \"$status\" -eq 0 ] && exit 0", " if [ \"$attempt\" -ge \"$max_attempts\" ]; then exit \"$status\"; fi", " echo \"onebox: attempt $attempt of $max_attempts exited $status; retrying in ${backoff}s\" >&2", + " phase=backing-off", + " write_state \"$attempt\"", " sleep \"$backoff\"", " backoff=$((backoff * 2))", " [ \"$backoff\" -gt \"$max_backoff\" ] && backoff=$max_backoff", @@ -627,6 +650,7 @@ func scheduleServiceUnit(application string, job app.ScheduledJob, runnerPath, n // runner's state and SERVICE_RESULT, then notifies per the job's policy. "ExecStopPost=/bin/sh " + notifyPath, "TimeoutStartSec=" + job.Timeout, + "TimeoutStopSec=" + (job.ShutdownGrace + 10*time.Second).String(), "", }, "\n") } @@ -654,7 +678,7 @@ const scheduleRunIdentifier = "ob-run" func scheduleRunRecordLines(application, unit, job, state string) []string { return []string{ "state=" + q(state), - "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''; skipped=''; execution=''", + "release=''; started_at=''; started_epoch=''; trigger=''; operation=''; attempt=0; inputs=''; skipped=''; execution=''; forced_kill=false", // A run that stood aside left a note under its own invocation. It // never held the job lock, so the state file belongs to whichever run // is still going: read the note and leave that file alone. @@ -663,6 +687,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { " while IFS= read -r line || [ -n \"$line\" ]; do", " case \"$line\" in", " skipped=*) skipped=${line#skipped=} ;;", + " forced_kill=*) forced_kill=${line#forced_kill=} ;;", " operation=*) operation=${line#operation=} ;;", " inputs=*) inputs=${line#inputs=} ;;", " esac", @@ -680,6 +705,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { " attempt=*) attempt=${line#attempt=} ;;", " inputs=*) inputs=${line#inputs=} ;;", " skipped=*) skipped=${line#skipped=} ;;", + " forced_kill=*) forced_kill=${line#forced_kill=} ;;", " esac", " done <\"$state\"", " rm -f \"$state\"", @@ -690,6 +716,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { // EXIT_STATUS is a signal name when the main process was killed. "case \"$status\" in ''|*[!0-9]*) status=null ;; esac", "case \"$attempt\" in ''|*[!0-9]*) attempt=0 ;; esac", + "case \"$forced_kill\" in true) ;; *) forced_kill=false ;; esac", // A skip is the runner's own word, written before any container ran; // a container that exits non-zero, 75 included, is a failure. "if [ \"$result\" = timeout ]; then outcome=timeout", @@ -703,8 +730,8 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { "[ -z \"$started_at\" ] && started_at=$finished_at", "execution_field=''", "if [ -n \"$execution\" ]; then execution_field=$(printf '\"execution\":\"%s\",' \"$execution\"); fi", - "record=$(printf '{%s\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"reason\":\"%s\",\"inputs\":{%s}}' " + - "\"$execution_field\" \"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$skipped\" \"$inputs\")", + "record=$(printf '{%s\"run\":\"%s\",\"job\":\"%s\",\"trigger\":\"%s\",\"operation\":\"%s\",\"release\":\"%s\",\"started_at\":\"%s\",\"finished_at\":\"%s\",\"duration_s\":%s,\"attempts\":%s,\"exit_status\":%s,\"outcome\":\"%s\",\"forced_kill\":%s,\"reason\":\"%s\",\"inputs\":{%s}}' " + + "\"$execution_field\" \"${INVOCATION_ID:-}\" " + q(job) + " \"$trigger\" \"$operation\" \"$release\" \"$started_at\" \"$finished_at\" \"$duration\" \"$attempt\" \"$status\" \"$outcome\" \"$forced_kill\" \"$skipped\" \"$inputs\")", "printf 'MESSAGE=%s\\nPRIORITY=6\\nSYSLOG_IDENTIFIER=" + scheduleRunIdentifier + "\\nONEBOX_APP=%s\\nONEBOX_UNIT=%s\\nONEBOX_JOB=%s\\n' " + "\"$record\" " + q(application) + " " + q(unit) + " " + q(job) + " | logger --journald || true", } @@ -712,7 +739,7 @@ func scheduleRunRecordLines(application, unit, job, state string) []string { // scheduleNotificationRun marks where the notifier substitutes the run id at // send time. It travels as the payload's deploy_id: the correlation key an -// operator hands to `ob schedule logs --run`. Nothing else about the run goes +// operator hands to `ob job logs --run`. Nothing else about the run goes // into a notification; the notify package redacts diagnostics on purpose, and // attempts, duration and exit status belong to the run record on the host. const scheduleNotificationRun = "__ONEBOX_SCHEDULE_RUN__" @@ -726,9 +753,9 @@ const scheduleNotificationRun = "__ONEBOX_SCHEDULE_RUN__" // contract lives in the notify package and the host has no Onebox to ask at // 2am. Only the timestamp and the run id are filled in on the host. func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { - cleanup := scheduleContainerCleanup(e.names().Container(job.Name, 1)) + cleanup := scheduleContainerRemove(e.names().Container(job.Name, 1)) if job.Execution != nil { - cleanup = durableContainerCleanup(e.names().Container(job.Name, 1)) + cleanup = durableContainerStop(e.names().Container(job.Name, 1), job.ShutdownGrace) } environment := e.Opts.Environment if environment == "" { @@ -738,6 +765,7 @@ func (e *Engine) scheduleNotifier(job app.ScheduledJob) (string, error) { "#!/bin/sh", "# Written by Onebox. Edits are overwritten on the next deploy.", "set -u", + "state=" + q(e.names().ScheduledJobRunState(job.Name)), "exec 9>" + q(e.names().ScheduledJobRunLock(job.Name)), "if /usr/bin/flock --exclusive --nonblock 9; then", " " + cleanup, diff --git a/internal/engine/schedule_execution.go b/internal/engine/schedule_execution.go index 359dbd6e..1cf1440a 100644 --- a/internal/engine/schedule_execution.go +++ b/internal/engine/schedule_execution.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -29,8 +30,8 @@ func invalidateExecutionCommand(root string) string { return "if [ -e " + q(durable.Store(root)) + " ] || [ -L " + q(durable.Store(root)) + " ]; then /usr/bin/python3 " + q(durable.Helper(root)) + " invalidate " + q(root) + "; fi" } -func durableContainerCleanup(container string) string { - return "if [ \"$(/usr/bin/docker inspect --format '{{ index .Config.Labels \"ob.execution.invocation\" }}' " + q(container) + " 2>/dev/null)\" = \"${INVOCATION_ID:-missing}\" ]; then " + scheduleContainerCleanup(container) + "; fi" +func durableContainerStop(container string, grace time.Duration) string { + return "if [ \"$(/usr/bin/docker inspect --format '{{ index .Config.Labels \"ob.execution.invocation\" }}' " + q(container) + " 2>/dev/null)\" = \"${INVOCATION_ID:-missing}\" ]; then " + scheduleContainerStop(container, grace) + "; fi" } type executionDefinition struct { @@ -127,14 +128,15 @@ func (e *Engine) durableScheduleRunner(job app.ScheduledJob, envFiles []app.EnvF "[ \"${release_dir%/*}\" = "+q(n.ReleasesDir())+" ] || exit 1", "exec 7>>\"$release_dir/.ob-schedule.lease\"", "chmod 600 \"$release_dir/.ob-schedule.lease\"", "/usr/bin/flock --shared 7") lines = append(lines, scheduleRunPreamble(true)...) - lines = append(lines, "write_state 1", + lines = append(lines, schedulePlannedBindingLines()...) + lines = append(lines, "phase=running", "write_state 1", "execution=$("+helper+" prepare "+q(n.AppDir())+" "+q(base64.StdEncoding.EncodeToString(encoded))+" \"$release\" \"${INVOCATION_ID:-}\" \"$execution\" \"$operation\" \"{$inputs_json}\")") lines = append(lines, "printf 'execution=%s\\n' \"$execution\" >>\"$state\"") // Publish the durable reference while still inside the retention rendezvous. if job.DeployLock == "pinned" { lines = append(lines, "/usr/bin/flock --unlock 8") } - lines = append(lines, helper+" run "+q(n.AppDir())+" \"$execution\" \"${INVOCATION_ID:-}\"", "") + lines = append(lines, helper+" run "+q(n.AppDir())+" \"$execution\" \"${INVOCATION_ID:-}\" "+strconv.FormatFloat(job.ShutdownGrace.Seconds(), 'f', -1, 64), "") return strings.Join(lines, "\n"), nil } diff --git a/internal/engine/schedule_execution_test.go b/internal/engine/schedule_execution_test.go index 344da6f3..19e1effe 100644 --- a/internal/engine/schedule_execution_test.go +++ b/internal/engine/schedule_execution_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/transport" @@ -31,16 +32,17 @@ func TestRunJournalIncludesExecutionOnlyWhenSet(t *testing.T) { func TestDurableRunnerPublishesCheckpointBeforeReleasingRetentionRendezvous(t *testing.T) { e := New(testConfig(), testProject(t), &transport.Fake{}, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) - runner, err := e.durableScheduleRunner(app.ScheduledJob{Name: "refresh", Timeout: "1h", DeployLock: "pinned", Execution: &app.JobExecution{}}, nil) + runner, err := e.durableScheduleRunner(app.ScheduledJob{Name: "refresh", Timeout: "1h", ShutdownGrace: 30 * time.Second, DeployLock: "pinned", Execution: &app.JobExecution{}}, nil) if err != nil { t.Fatal(err) } lease := strings.Index(runner, "/usr/bin/flock --shared 7") + binding := strings.Index(runner, sealedManualJobBindingMarker) prepare := strings.Index(runner, " prepare ") unlock := strings.Index(runner, "/usr/bin/flock --unlock 8") run := strings.LastIndex(runner, " run ") - if lease < 0 || prepare <= lease || unlock <= prepare || run <= unlock { - t.Fatalf("expected live lease, checkpoint, mutex unlock, then execution:\n%s", runner) + if lease < 0 || binding <= lease || prepare <= binding || unlock <= prepare || run <= unlock { + t.Fatalf("expected live lease, sealed binding, checkpoint, mutex unlock, then execution:\n%s", runner) } } @@ -62,7 +64,7 @@ func TestDurableCleanupCannotRemoveAnotherInvocationsContainer(t *testing.T) { if err := os.WriteFile(stub, []byte(body), 0o700); err != nil { t.Fatal(err) } - command := exec.CommandContext(t.Context(), "sh", "-c", strings.ReplaceAll(durableContainerCleanup("job"), "/usr/bin/docker", q(stub))) + command := exec.CommandContext(t.Context(), "sh", "-c", strings.ReplaceAll(durableContainerStop("job", time.Second), "/usr/bin/docker", q(stub))) command.Env = append(os.Environ(), "INVOCATION_ID="+tc.invocation, "TEST_LABEL="+tc.label) if out, err := command.CombinedOutput(); err != nil { t.Fatalf("cleanup: %v: %s", err, out) @@ -75,6 +77,31 @@ func TestDurableCleanupCannotRemoveAnotherInvocationsContainer(t *testing.T) { } } +func TestDurableNotifierGracefullyStopsOnlyItsInvocation(t *testing.T) { + e := New(testConfig(), testProject(t), &transport.Fake{}, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + script, err := e.scheduleNotifier(app.ScheduledJob{ + Name: "refresh", ShutdownGrace: 12 * time.Second, Execution: &app.JobExecution{}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `ob.execution.invocation`, `${INVOCATION_ID:-missing}`, + `docker kill --signal TERM 'sample-refresh-1'`, + `deadline=$(($(date -u '+%s')+12))`, + `docker kill --signal KILL 'sample-refresh-1'`, + } { + if !strings.Contains(script, want) { + t.Fatalf("durable notifier is missing %q:\n%s", want, script) + } + } + state := strings.Index(script, "state='/var/lib/ob/sample/schedule/refresh.state'") + cleanup := strings.Index(script, "ob.execution.invocation") + if state < 0 || cleanup < 0 || state >= cleanup { + t.Fatalf("durable notifier must initialize state before fallback cleanup:\n%s", script) + } +} + func TestDurablePythonRequirementDoesNotAffectOrdinarySchedules(t *testing.T) { for _, durable := range []bool{false, true} { f := &transport.Fake{Dynamic: func(command string) (transport.Result, bool) { @@ -151,7 +178,7 @@ func TestScheduleFlockProbeExecutesCapabilityChecks(t *testing.T) { func TestDurableResumeRefusesLegacyRunnerBeforePublishingRequest(t *testing.T) { cfg := testConfig() - cfg.Workloads["refresh"] = app.Workload{Role: app.RoleJob, When: "manual", DataEffect: app.DataEffectNone, + cfg.Workloads["refresh"] = app.Workload{Role: app.RoleJob, DeploymentPhase: "none", DataEffect: app.DataEffectNone, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, Execution: &app.JobExecution{}} f := happyFake() base := f.Dynamic diff --git a/internal/engine/schedule_history.go b/internal/engine/schedule_history.go index dbb07461..d9906137 100644 --- a/internal/engine/schedule_history.go +++ b/internal/engine/schedule_history.go @@ -28,6 +28,7 @@ type ScheduleRunRecord struct { Attempts int `json:"attempts"` ExitStatus *int `json:"exit_status"` Outcome string `json:"outcome"` + ForcedKill bool `json:"forced_kill,omitempty"` // Reason is set on a skipped run: what the runner met instead of running. Reason string `json:"reason,omitempty"` Inputs map[string]string `json:"inputs,omitempty"` @@ -35,15 +36,19 @@ type ScheduleRunRecord struct { // ScheduleListing is one declared job beside its timer as the host reports it. type ScheduleListing struct { - Name string `json:"name"` - Unit string `json:"unit"` - Cron string `json:"cron"` - Timezone string `json:"timezone"` - DeployLock string `json:"deploy_lock"` - Timeout string `json:"timeout"` - TimerState string `json:"timer_state"` - NextRun string `json:"next_run,omitempty"` - LastTrigger string `json:"last_trigger,omitempty"` + Name string `json:"name"` + Unit string `json:"unit"` + Cron string `json:"cron"` + Timezone string `json:"timezone"` + DeployLock string `json:"deploy_lock"` + DeploymentPhase string `json:"deployment_phase"` + OperatorRun string `json:"operator_run"` + Timeout string `json:"timeout"` + MaxAttempts int `json:"max_attempts"` + RetryBudget string `json:"retry_backoff_budget"` + TimerState string `json:"timer_state"` + NextRun string `json:"next_run,omitempty"` + LastTrigger string `json:"last_trigger,omitempty"` // Paused is set when an operator stopped this job's timer. Without it an // inactive timer in this table reads the same whether somebody stopped the // job on purpose or it broke. @@ -185,8 +190,10 @@ func (e *Engine) ScheduleList(ctx context.Context) ([]ScheduleListing, error) { values := observed[job.Name] out = append(out, ScheduleListing{ Name: job.Name, Unit: e.names().ScheduledJobUnit(job.Name), Cron: job.Cron, Timezone: job.Timezone, - DeployLock: job.DeployLock, Timeout: job.Timeout, TimerState: values["ActiveState"], - NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], + DeployLock: job.DeployLock, DeploymentPhase: job.DeploymentPhase, OperatorRun: job.OperatorRun, + Timeout: job.Timeout, MaxAttempts: job.RetryAttempts, RetryBudget: job.RetryBackoffBudget().String(), + TimerState: values["ActiveState"], + NextRun: values["NextElapseUSecRealtime"], LastTrigger: values["LastTriggerUSec"], Paused: pauseFrom(paused[job.Name]), }) } diff --git a/internal/engine/schedule_history_test.go b/internal/engine/schedule_history_test.go index 462c59a0..f988415a 100644 --- a/internal/engine/schedule_history_test.go +++ b/internal/engine/schedule_history_test.go @@ -33,7 +33,7 @@ func scheduledFixture(t *testing.T) (*Engine, *transport.Fake) { t.Helper() cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{} @@ -79,7 +79,8 @@ func TestScheduleListReadsTimerState(t *testing.T) { t.Fatal(err) } if len(listing) != 1 || listing[0].Unit != "ob-sample-nightly" || listing[0].TimerState != "active" || - listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || listing[0].Cron != "0 2 * * *" { + listing[0].NextRun != "Sat 2026-09-06 02:00:00 UTC" || listing[0].LastTrigger != "Fri 2026-09-05 02:00:00 UTC" || + listing[0].Cron != "0 2 * * *" || listing[0].MaxAttempts != 1 || listing[0].RetryBudget != "0s" { t.Fatalf("listing = %#v", listing) } } diff --git a/internal/engine/schedule_pause_test.go b/internal/engine/schedule_pause_test.go index f6a6c5ec..7b0f6db9 100644 --- a/internal/engine/schedule_pause_test.go +++ b/internal/engine/schedule_pause_test.go @@ -14,7 +14,7 @@ func pausableFixture(t *testing.T) (*app.Resolved, *transport.Fake) { t.Helper() cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := happyFake() diff --git a/internal/engine/schedule_run.go b/internal/engine/schedule_run.go index 534a80f0..3e168581 100644 --- a/internal/engine/schedule_run.go +++ b/internal/engine/schedule_run.go @@ -38,15 +38,15 @@ func (e *Engine) ScheduleRun(ctx context.Context, operationID, name string, inpu return e.scheduleRun(ctx, operationID, name, inputs, wait, "", nil) } -// PlannedJobRun submits a sealed manual job plan to the job's installed +// PlannedJobRun submits a sealed operator job plan to the job's installed // systemd unit. The runner checks the plan's release and runtime digest after // taking the host-side application exclusion, so releasing the admission lock // before systemd schedules the unit cannot move the job onto different bytes. -func (e *Engine) PlannedJobRun(ctx context.Context, operationID, name, expectedRelease, expectedRuntime string, wait bool) (_ ScheduleRunResult, err error) { +func (e *Engine) PlannedJobRun(ctx context.Context, operationID, name string, inputs map[string]string, expectedRelease, expectedRuntime string, wait bool) (_ ScheduleRunResult, err error) { if expectedRelease == "" || expectedRuntime == "" { return ScheduleRunResult{}, errors.New("planned job run requires an expected release and runtime digest") } - return e.scheduleRun(ctx, operationID, name, nil, wait, "", &plannedJobBinding{ + return e.scheduleRun(ctx, operationID, name, inputs, wait, "", &plannedJobBinding{ release: expectedRelease, runtime: expectedRuntime, }) @@ -88,7 +88,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu // an older one the next timer firing would read the file meant for this // run, and the run itself would be recorded as a firing. if !e.hasTriggerUnit(ctx) { - return result, errors.New("this host's systemd does not set $TRIGGER_UNIT, so a timer firing cannot be told from this run; ob schedule run needs systemd 252 or newer. The timer itself keeps working") + return result, errors.New("this host's systemd does not set $TRIGGER_UNIT, so a timer firing cannot be told from this run; ob job run needs systemd 252 or newer. The timer itself keeps working") } unit := e.names().ScheduledJobUnit(name) result.Unit = unit @@ -98,7 +98,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu return result, err } if res.ExitCode != 0 { - return result, fmt.Errorf("installed job runner does not support sealed manual-job binding; run `ob schedule apply` before running %s", name) + return result, fmt.Errorf("installed job runner does not support sealed operator-job binding; run `ob schedule apply` before running %s", name) } } @@ -110,7 +110,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu } switch state := strings.TrimSpace(active.Stdout); state { case "active", "activating", "deactivating": - return result, fmt.Errorf("job %s is running (%s); wait for it, or read ob schedule history %s", name, state, name) + return result, fmt.Errorf("job %s is running (%s); wait for it, or read ob job history %s", name, state, name) } epoch, err := e.AcquireLock(ctx, operationID, e.Opts.ForceLock) @@ -136,7 +136,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu } } - // noclobber: a second manual run before the first is consumed would + // noclobber: a second operator run before the first is consumed would // otherwise rewrite the file under it and misattribute the inputs. The // existence check in front gives that case its own exit status, so a // host that simply refuses the write is reported as that and not as a @@ -154,7 +154,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu } switch { case res.ExitCode == 73: - return result, fmt.Errorf("a manual run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) + return result, fmt.Errorf("an operator run of %s is already pending (%s exists); wait for it, or remove the file on the host", name, path) case res.ExitCode != 0: return result, fmt.Errorf("cannot write the inputs file %s on the host: %s", path, strings.TrimSpace(res.Stderr)) } @@ -180,7 +180,10 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu if len(inputs) > 0 { detail = "inputs: " + scheduleInputsDetail(inputs) } - record := journal.Record{Phase: "schedule-run", Event: "start", Status: "ok", Target: name, TargetKind: "job", Detail: detail} + record := journal.Record{ + Phase: "schedule-run", Event: "start", Status: "ok", OperationKind: "job_run", + Service: name, Target: name, TargetKind: "job", Detail: detail, + } if planned != nil { // This journal describes host-unit admission, not the job's eventual // outcome. Keep schedule-run's truthful "started" audit semantics while @@ -203,7 +206,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu defer func() { finish := record finish.Event, finish.Status = "finish", "ok" - finish.Detail = "unit started; outcome in ob schedule history " + name + finish.Detail = "unit started; outcome in ob job history " + name if err != nil { finish.Status, finish.Detail = "fail", err.Error() } @@ -221,7 +224,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu finishStep := func(error) {} if wait { if planned != nil { - e.ui.Infof("host run %s; Ctrl-C detaches; inspect with `ob schedule history %s`", operationID, name) + e.ui.Infof("host run %s; Ctrl-C detaches; inspect with `ob job history %s`", operationID, name) } finishStep = e.ui.Step("job "+name, true) defer func() { finishStep(err) }() @@ -237,14 +240,14 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu // The unit is queued, so the runner owns the file now. pending = false result.Started = true - e.ui.Successf("job %s accepted as %s; inspect with `ob schedule history %s`", name, operationID, name) + e.ui.Successf("job %s accepted as %s; inspect with `ob job history %s`", name, operationID, name) return result, nil } // A blocking start that exits non-zero may mean the job failed, which is // an outcome, or that the unit never activated, which is not. Only the // record settles it, and only a record carrying this operation says the // runner read the inputs file: a start that merged into a timer firing - // already in progress leaves that file untouched, for the next manual run + // already in progress leaves that file untouched, for the next operator run // that would otherwise be refused as pending. last, err := e.awaitScheduleRecord(ctx, name, operationID) if err != nil { @@ -267,7 +270,7 @@ func (e *Engine) scheduleRun(ctx context.Context, operationID, name string, inpu // success is a failure of the request, a skip included: the unit exits // cleanly, but the work was not done. if last.Outcome != "success" { - return result, fmt.Errorf("job %s run %s ended %s; see ob schedule logs %s --run %s", name, last.Run, last.Outcome, name, last.Run) + return result, fmt.Errorf("job %s run %s ended %s; see ob job logs %s --run %s", name, last.Run, last.Outcome, name, last.Run) } return result, nil } @@ -296,7 +299,7 @@ func (e *Engine) awaitScheduleRecord(ctx context.Context, name, operationID stri } } } - return nil, fmt.Errorf("no run record for operation %s appeared within %s: the run may still be settling in the host journal, a timer firing may have taken the slot, or the notifier wrote nothing. ob schedule history %s shows what the host has", + return nil, fmt.Errorf("no run record for operation %s appeared within %s: the run may still be settling in the host journal, a timer firing may have taken the slot, or the notifier wrote nothing. ob job history %s shows what the host has", operationID, 10*time.Second, name) } diff --git a/internal/engine/schedule_run_test.go b/internal/engine/schedule_run_test.go index 5fa9dd27..3df4c36b 100644 --- a/internal/engine/schedule_run_test.go +++ b/internal/engine/schedule_run_test.go @@ -13,7 +13,7 @@ import ( func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } @@ -75,7 +75,7 @@ func TestScheduleRunWritesInputsJournalsThenStartsAfterReleasingTheLock(t *testi func TestPlannedJobRunStagesItsExactBindingAndDetachesToSystemd(t *testing.T) { cfg := testConfig() cfg.Workloads["refresh"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "destructive", Schedule: &app.JobSchedule{Cron: "0 4 * * 1", Timezone: "UTC", Timeout: "8h"}, } f := happyFake() @@ -103,7 +103,7 @@ func TestPlannedJobRunStagesItsExactBindingAndDetachesToSystemd(t *testing.T) { release = "20260914-190602-deploy-731e31b2d992" runtime = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ) - result, err := e.PlannedJobRun(context.Background(), operation, "refresh", release, runtime, false) + result, err := e.PlannedJobRun(context.Background(), operation, "refresh", nil, release, runtime, false) if err != nil { t.Fatalf("planned job run: %v\n%s", err, strings.Join(f.Commands, "\n")) } @@ -137,12 +137,12 @@ func TestPlannedJobRunStagesItsExactBindingAndDetachesToSystemd(t *testing.T) { func TestScheduleRunRefusals(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog"}, Default: "catalog"}}, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } cfg.Workloads["prune"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "destructive", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "destructive", Schedule: &app.JobSchedule{Cron: "0 3 * * *", Timezone: "UTC", Timeout: "1h"}, } active := false @@ -185,7 +185,7 @@ func TestScheduleRunRefusals(t *testing.T) { func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } for outcome, wantErr := range map[string]bool{"success": false, "skipped": true, "failure": true} { @@ -224,7 +224,7 @@ func TestScheduleRunWaitReportsTheRecordAndFailsOnAnyOtherOutcome(t *testing.T) func TestScheduleRunDiscardsItsInputsWhenTheStartFails(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"catalog", "prices"}, Default: "catalog"}}, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } @@ -255,7 +255,7 @@ func TestScheduleRunDiscardsItsInputsWhenTheStartFails(t *testing.T) { func TestScheduleRunTellsAPendingFileFromAWriteFailure(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } for name, tc := range map[string]struct { @@ -300,7 +300,7 @@ func TestScheduleRunTellsAPendingFileFromAWriteFailure(t *testing.T) { func TestScheduleRunRefusesAHostThatCannotTellTheTriggerApart(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } f := happyFake() @@ -334,7 +334,7 @@ func TestScheduleRunRefusesAHostThatCannotTellTheTriggerApart(t *testing.T) { func TestScheduleRunJournalsAFailedRequestAsFailed(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } f := happyFake() diff --git a/internal/engine/schedule_status.go b/internal/engine/schedule_status.go index e9e32e1f..8b9bfdc5 100644 --- a/internal/engine/schedule_status.go +++ b/internal/engine/schedule_status.go @@ -16,16 +16,22 @@ import ( // been skipped. systemd contributes the timer's state and next elapse and // whether a run is in progress. type StatusSchedule struct { - Name string `json:"name"` - Unit string `json:"unit"` - TimerState string `json:"timer_state"` - Running bool `json:"running"` - DeployLock string `json:"deploy_lock"` - Timeout string `json:"timeout"` - PinnedRelease string `json:"pinned_release,omitempty"` - StartedAt string `json:"started_at,omitempty"` - Diverged bool `json:"diverged"` - Issues []string `json:"issues,omitempty"` + Name string `json:"name"` + Unit string `json:"unit"` + TimerState string `json:"timer_state"` + Running bool `json:"running"` + Phase string `json:"phase,omitempty"` + Trigger string `json:"trigger,omitempty"` + Release string `json:"release,omitempty"` + ElapsedSeconds int `json:"elapsed_s,omitempty"` + DeployLock string `json:"deploy_lock"` + Timeout string `json:"timeout"` + MaxAttempts int `json:"max_attempts"` + RetryBudget string `json:"retry_backoff_budget"` + PinnedRelease string `json:"pinned_release,omitempty"` + StartedAt string `json:"started_at,omitempty"` + Diverged bool `json:"diverged"` + Issues []string `json:"issues,omitempty"` NextRun string `json:"next_run,omitempty"` Attempt int `json:"attempt,omitempty"` @@ -33,6 +39,11 @@ type StatusSchedule struct { LastReason string `json:"last_reason,omitempty"` LastDurationSeconds int `json:"last_duration_s,omitempty"` LastAttempts int `json:"last_attempts,omitempty"` + LastTimerOutcome string `json:"last_timer_outcome,omitempty"` + LastTimerAt string `json:"last_timer_at,omitempty"` + LastOperatorOutcome string `json:"last_operator_outcome,omitempty"` + LastOperatorAt string `json:"last_operator_at,omitempty"` + LastSuccessAt string `json:"last_success_at,omitempty"` ConsecutiveFailures int `json:"consecutive_failures,omitempty"` ConsecutiveSkips int `json:"consecutive_skips,omitempty"` // JournalPersistent is false when the host keeps its journal in memory, so @@ -69,6 +80,8 @@ type scheduleUnitObservation struct { release string startedAt string attempt string + phase string + trigger string next string history []ScheduleRunRecord pause *SchedulePauseState @@ -132,6 +145,7 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) loadState: values["LoadState"], activeState: values["ActiveState"], result: values["Result"], exitStatus: exit, release: values["release"], startedAt: values["started_at"], attempt: values["attempt"], + phase: values["phase"], trigger: values["trigger"], next: values["NextElapseUSecRealtime"], history: parseScheduleRunRecords(strings.Join(raw, "\n"), name), pause: pauseFrom(values), @@ -177,10 +191,21 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) status := StatusSchedule{ Name: job.Name, Unit: unit, TimerState: timer.activeState, Running: service.activeState == "activating", DeployLock: job.DeployLock, Timeout: job.Timeout, + MaxAttempts: job.RetryAttempts, RetryBudget: job.RetryBackoffBudget().String(), NextRun: timer.next, JournalPersistent: journalPersistent, } if status.Running { status.Attempt, _ = strconv.Atoi(run.attempt) + status.Phase, status.Trigger, status.Release, status.StartedAt = run.phase, run.trigger, run.release, run.startedAt + if status.Phase == "" { + status.Phase = "running" + } + if started, parseErr := time.Parse(time.RFC3339, run.startedAt); parseErr == nil { + status.ElapsedSeconds = int(e.Opts.Now().UTC().Sub(started).Seconds()) + if status.ElapsedSeconds < 0 { + status.ElapsedSeconds = 0 + } + } if job.DeployLock == "pinned" { _, timeErr := time.Parse(time.RFC3339, run.startedAt) if !release.IsID(run.release) || timeErr != nil { @@ -201,6 +226,17 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) status.LastReason = last.Reason status.LastDurationSeconds = last.DurationSeconds status.LastAttempts = last.Attempts + for _, record := range records { + if status.LastTimerOutcome == "" && record.Trigger == "timer" { + status.LastTimerOutcome, status.LastTimerAt = record.Outcome, record.FinishedAt + } + if status.LastOperatorOutcome == "" && (record.Trigger == "operator" || record.Trigger == "manual") { + status.LastOperatorOutcome, status.LastOperatorAt = record.Outcome, record.FinishedAt + } + if status.LastSuccessAt == "" && record.Outcome == "success" { + status.LastSuccessAt = record.FinishedAt + } + } for _, record := range records { if record.Outcome != "skipped" { break @@ -234,7 +270,7 @@ func (e *Engine) scheduleStatuses(ctx context.Context) ([]StatusSchedule, error) } // The record is the verdict: the newest run that actually happened is // the one that counts, and so is a job that keeps being skipped. - if lastRun != nil && (lastRun.Outcome == "failure" || lastRun.Outcome == "timeout") { + if !status.Running && lastRun != nil && (lastRun.Outcome == "failure" || lastRun.Outcome == "timeout") { exit := "?" if lastRun.ExitStatus != nil { exit = strconv.Itoa(*lastRun.ExitStatus) diff --git a/internal/engine/schedule_test.go b/internal/engine/schedule_test.go index 68e3d0dc..63dafd83 100644 --- a/internal/engine/schedule_test.go +++ b/internal/engine/schedule_test.go @@ -23,7 +23,7 @@ import ( func TestSyncSchedulesRetainsManualScheduledJob(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := happyFake() @@ -63,7 +63,7 @@ func TestSyncSchedulesRetainsManualScheduledJob(t *testing.T) { func TestScheduleApplyUpgradesLegacyUnitsUnderRegime(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "45m", CatchUp: false}, } f := happyFake() @@ -123,7 +123,7 @@ func TestScheduleApplyUpgradesLegacyUnitsUnderRegime(t *testing.T) { func TestScheduleApplyRefusesBeforeFirstRelease(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := happyFake() // its current release has no Compose runtime @@ -148,7 +148,7 @@ func TestScheduleApplyRefusesBeforeFirstRelease(t *testing.T) { func TestScheduleApplyStopsBeforeUnitWritesWhenJournalStartFails(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := happyFake() @@ -179,7 +179,8 @@ func TestScheduleApplyStopsBeforeUnitWritesWhenJournalStartFails(t *testing.T) { func TestScheduledJobUnitContract(t *testing.T) { job := app.ScheduledJob{ Name: "nightly", Cron: "0 2 * * *", Timezone: "UTC", - Calendar: "*-*-* 02:00:00", Timeout: "45m", CatchUp: false, DeployLock: "exclusive", + Calendar: "*-*-* 02:00:00", Timeout: "45m", ShutdownGrace: 12 * time.Second, + CatchUp: false, DeployLock: "exclusive", } names := app.Names{App: "sample", BasePath: "/var/lib/ob"} runner := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) @@ -201,6 +202,11 @@ func TestScheduledJobUnitContract(t *testing.T) { "compose.yaml", `run --rm --no-deps "$@" --name 'sample-nightly-1'`, "docker rm -f 'sample-nightly-1'", + "docker kill --signal TERM 'sample-nightly-1'", + "docker kill --signal KILL 'sample-nightly-1'", + "deadline=$(($(date -u '+%s')+12))", + "forced_kill=true", + "phase=stopping", "ONEBOX_EXPECTED_RELEASE", "ONEBOX_EXPECTED_RUNTIME", sealedManualJobBindingMarker, @@ -213,6 +219,15 @@ func TestScheduledJobUnitContract(t *testing.T) { t.Errorf("runner is missing %q:\n%s", want, runner) } } + if !strings.Contains(service, "TimeoutStopSec=22s") { + t.Fatalf("service does not leave systemd enough time for graceful shutdown:\n%s", service) + } + term := strings.Index(runner, "docker kill --signal TERM") + kill := strings.Index(runner, "docker kill --signal KILL") + remove := strings.LastIndex(runner, "docker rm -f") + if term < 0 || kill < 0 || remove < 0 || !(term < kill && kill < remove) { + t.Fatalf("shutdown must order TERM, KILL, then cleanup:\n%s", runner) + } locked := strings.Index(runner, "flock --exclusive --timeout 10") bound := strings.Index(runner, "serving release changed after job approval") run := strings.Index(runner, "docker compose") @@ -270,7 +285,7 @@ func TestScheduleRendezvousWaitReservesShortJobTimeout(t *testing.T) { names := app.Names{App: "sample", BasePath: "/var/lib/ob"} runner := scheduleRunnerScript("sample", app.ScheduledJob{Name: "quick", Timeout: "1s", DeployLock: "pinned"}, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) if !strings.Contains(runner, "flock --shared --nonblock --conflict-exit-code 200 8") || - !strings.Contains(runner, "skip 'the application scheduling lock is busy'") { + !strings.Contains(runner, "skip 'the scheduling rendezvous is busy'") { t.Fatalf("short-timeout runner can outlive its rendezvous budget:\n%s", runner) } } @@ -467,7 +482,7 @@ func TestScheduledJobApplicationRendezvousTimeoutRecordsSkip(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Contains(state, []byte("skipped=the application scheduling lock remained busy for 10s")) { + if !bytes.Contains(state, []byte("skipped=the scheduling rendezvous remained busy for 10s")) { t.Fatalf("timeout state = %q, want application-rendezvous skip", state) } } @@ -665,7 +680,7 @@ func TestPinnedScheduledJobLockProtocol(t *testing.T) { cfg := testConfig() cfg.BasePath = root cfg.Workloads[job.Name] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "6h", CatchUp: true, DeployLock: "pinned"}, } deploy := New(cfg, testProject(t), transport.NewLocal(), Options{Out: &bytes.Buffer{}, Sleep: noSleep}) @@ -758,7 +773,7 @@ func TestScheduledJobFailureNotifierUsesConfiguredWebhooks(t *testing.T) { func TestSyncSchedulesRefusesMissingFlockBeforeInstallingUnits(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -783,7 +798,7 @@ func TestSyncSchedulesRefusesMissingFlockBeforeInstallingUnits(t *testing.T) { func TestScheduleStatusReportsRunningPinnedRelease(t *testing.T) { cfg := testConfig() cfg.Workloads["refresh"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "6h", CatchUp: true, DeployLock: "pinned"}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -799,11 +814,18 @@ ActiveState=active @@refresh:run release=20260828-120000-abc1234 started_at=2026-08-28T12:01:02Z +attempt=2 +phase=backing-off +trigger=operator +@@refresh:history +{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"refresh","trigger":"timer","started_at":"2026-08-28T11:00:01Z","finished_at":"2026-08-28T11:00:02Z","duration_s":1,"attempts":1,"exit_status":1,"outcome":"failure","inputs":{}} `}, true } return transport.Result{}, false }} - e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep}) + e := New(cfg, testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep, Now: func() time.Time { + return time.Date(2026, 8, 28, 12, 2, 2, 0, time.UTC) + }}) statuses, err := e.scheduleStatuses(context.Background()) if err != nil { t.Fatal(err) @@ -813,7 +835,10 @@ started_at=2026-08-28T12:01:02Z } got := statuses[0] if !got.Running || got.DeployLock != "pinned" || got.Timeout != "6h" || - got.PinnedRelease != "20260828-120000-abc1234" || got.StartedAt != "2026-08-28T12:01:02Z" || got.Diverged { + got.PinnedRelease != "20260828-120000-abc1234" || got.Release != "20260828-120000-abc1234" || + got.StartedAt != "2026-08-28T12:01:02Z" || got.ElapsedSeconds != 60 || got.Phase != "backing-off" || + got.Trigger != "operator" || got.Attempt != 2 || got.MaxAttempts != 1 || got.RetryBudget != "0s" || + got.LastTimerOutcome != "failure" || got.Diverged { t.Fatalf("running pinned status was not surfaced: %#v", got) } } @@ -1126,7 +1151,7 @@ func TestScheduledJobNotifierWritesOneRunRecordToTheJournal(t *testing.T) { `outcome=success`, `outcome=failure`, `"run":"%s","job":"%s","trigger":"%s","operation":"%s","release":"%s"`, - `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","reason":"%s","inputs":{%s}`, + `"duration_s":%s,"attempts":%s,"exit_status":%s,"outcome":"%s","forced_kill":%s,"reason":"%s","inputs":{%s}`, `"${INVOCATION_ID:-}" 'nightly'`, `SYSLOG_IDENTIFIER=ob-run\nONEBOX_APP=%s\nONEBOX_UNIT=%s\nONEBOX_JOB=%s`, `"$record" 'sample' 'ob-sample-nightly' 'nightly' | logger --journald`, @@ -1232,6 +1257,7 @@ func runNotifierIn(t *testing.T, base string, job app.ScheduledJob, notification func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { state := "release=20260905-140000-ab12cd\nstarted_at=2026-09-05T15:00:01Z\nstarted_epoch=1\ntrigger=timer\noperation=\nattempt=2\ninputs=\n" + forced := state + "forced_kill=true\n" for name, tc := range map[string]struct { state string env map[string]string @@ -1242,6 +1268,7 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { "success": {state, map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "INVOCATION_ID": "a1b2"}, "success", float64(0), 2}, "failure": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "1"}, "failure", float64(1), 2}, "timeout": {state, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "TERM"}, "timeout", nil, 2}, + "forced kill": {forced, map[string]string{"SERVICE_RESULT": "timeout", "EXIT_STATUS": "KILL"}, "timeout", nil, 2}, "skipped": {"skipped=another run of this job is still in progress\noperation=\ninputs=\n", map[string]string{"SERVICE_RESULT": "success", "EXIT_STATUS": "0", "TRIGGER_UNIT": "ob-sample-nightly.timer"}, "skipped", float64(0), 0}, "job exits 75": {state, map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "75"}, "failure", float64(75), 2}, "no state": {"", map[string]string{"SERVICE_RESULT": "exit-code", "EXIT_STATUS": "3"}, "failure", float64(3), 0}, @@ -1260,6 +1287,9 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { if tc.state == state && (record["release"] != "20260905-140000-ab12cd" || record["trigger"] != "timer" || record["duration_s"].(float64) < 1) { t.Fatalf("state fields not carried: %#v", record) } + if (name == "forced kill") != record["forced_kill"].(bool) { + t.Fatalf("forced-kill evidence wrong: %#v", record) + } if name == "skipped" && (record["trigger"] != "timer" || record["reason"] != "another run of this job is still in progress") { t.Fatalf("skip was not recorded with its trigger and reason: %#v", record) } @@ -1273,7 +1303,7 @@ func TestScheduledJobNotifierRecordsEachOutcomeAndRemovesState(t *testing.T) { func TestScheduleStatusPrefersTheRunRecordOverSystemdResult(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -1320,7 +1350,7 @@ NextElapseUSecRealtime=Sat 2026-09-06 02:00:00 UTC func TestScheduleStatusCountsConsecutiveFailuresFromRecords(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -1509,7 +1539,7 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test if output, err := command.CombinedOutput(); err != nil { t.Fatalf("runner is not valid POSIX shell: %v: %s\n%s", err, output, runner) } - // The consume block precedes the locks so a skipped manual run cannot + // The consume block precedes the locks so a skipped operator run cannot // leave its inputs for the next timer firing. if strings.Index(runner, "inputs_file=") > strings.Index(runner, "exec 9>") { t.Fatalf("inputs are consumed after the lock:\n%s", runner) @@ -1523,7 +1553,7 @@ func TestScheduledJobRunnerConsumesManualInputsWithoutShellInterpolation(t *test func TestSyncSchedulesRequireSystemd252ForEveryScheduledJob(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Inputs: map[string]app.JobInput{"SOURCE": {Enum: []string{"a"}, Default: "a"}}, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } @@ -1601,11 +1631,11 @@ func TestScheduleInputsLinesParseTheFileIntoArguments(t *testing.T) { `json="SOURCE":"prices and more","SINCE":"2026-09-01=ish"` + "\n", } { if !strings.Contains(got, want) { - t.Fatalf("manual activation output is missing %q:\n%s", want, got) + t.Fatalf("operator activation output is missing %q:\n%s", want, got) } } if _, err := os.Stat(inputs); err == nil { - t.Fatal("the inputs file survived a manual activation") + t.Fatal("the inputs file survived a operator activation") } if err := os.WriteFile(inputs, []byte(body), 0o600); err != nil { t.Fatal(err) @@ -1616,7 +1646,7 @@ func TestScheduleInputsLinesParseTheFileIntoArguments(t *testing.T) { func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } skip := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}}` @@ -1660,7 +1690,7 @@ func TestScheduleStatusRaisesAnIssueForASkipStreak(t *testing.T) { func TestScheduleStatusKeepsAFailureVisibleBehindASkip(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } history := `{"run":"a1b2c3d4e5f60718293a4b5c6d7e8f90","job":"nightly","trigger":"timer","started_at":"2026-09-05T02:00:01Z","finished_at":"2026-09-05T02:00:01Z","duration_s":0,"attempts":0,"exit_status":0,"outcome":"skipped","reason":"an application operation holds the deploy lock","inputs":{}} @@ -1691,7 +1721,7 @@ func TestScheduleStatusKeepsAFailureVisibleBehindASkip(t *testing.T) { func TestScheduleStatusDegradesWhenTheJournalCannotBeRead(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -1734,7 +1764,7 @@ func TestScheduledJobRunnerDoesNotClobberARunningJobsState(t *testing.T) { // The other two skips hold the job lock, so the state is theirs to write. for _, want := range []string{ "flock --exclusive --timeout 10 --conflict-exit-code 200 8", - "200) skip 'the application scheduling lock remained busy for 10s'", + "200) skip 'the scheduling rendezvous remained busy for 10s'", "skip 'an application operation holds the deploy lock'", } { if !strings.Contains(runner, want) { @@ -1763,7 +1793,7 @@ func TestScheduledJobRunnerRecordsAnUnknownTriggerOnAnOlderSystemd(t *testing.T) job := app.ScheduledJob{Name: "nightly", Timeout: "1h", DeployLock: "exclusive", RetryAttempts: 1} modern := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, true) older := scheduleRunnerScript("sample", job, names, "/var/lib/ob/sample/lock", nil, 10*time.Minute, false) - if !strings.Contains(modern, "else trigger=manual; fi") { + if !strings.Contains(modern, "else trigger=operator; fi") { t.Fatalf("a host that sets TRIGGER_UNIT must name the operator:\n%s", modern) } if !strings.Contains(older, "else trigger=unknown; fi") { @@ -1784,7 +1814,7 @@ func TestSyncSchedulesRequiresSystemd252OnlyForInputs(t *testing.T) { t.Run(name, func(t *testing.T) { cfg := testConfig() cfg.Workloads["sync"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", Inputs: tc.inputs, + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Inputs: tc.inputs, Schedule: &app.JobSchedule{Cron: "0 * * * *", Timezone: "UTC", Timeout: "1h"}, } f := happyFake() @@ -1885,7 +1915,7 @@ func TestScheduleSkipMarkerIsNamedIdenticallyOnBothSides(t *testing.T) { func TestScheduleStatusFallsBackToSystemdWhenNoRecordsExist(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { @@ -1926,7 +1956,7 @@ ActiveState=active func TestScheduleStatusPrefersRecordsOverSystemdResult(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { diff --git a/internal/engine/status.go b/internal/engine/status.go index 3fa08bba..ecdf8ea4 100644 --- a/internal/engine/status.go +++ b/internal/engine/status.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/labstack/onebox/internal/journal" "github.com/labstack/onebox/internal/release" @@ -180,24 +181,35 @@ func (e *Engine) Status(ctx context.Context) error { continue } } - if schedule.Diverged { + if schedule.Diverged && !schedule.Running { diverged = true e.ui.Println(fmt.Sprintf("schedule %-11s %s", schedule.Name, e.ui.Warn(strings.Join(schedule.Issues, "; ")+" ⚠"))) continue } result := "not recorded" if schedule.Running { - detail := fmt.Sprintf("running; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + detail := fmt.Sprintf("%s; elapsed: %s; trigger: %s; policy: %s; timeout: %s", + orUnknown(schedule.Phase), (time.Duration(schedule.ElapsedSeconds) * time.Second).String(), orUnknown(schedule.Trigger), schedule.DeployLock, schedule.Timeout) + detail += fmt.Sprintf("; retry budget: %d attempt(s), %s backoff", schedule.MaxAttempts, schedule.RetryBudget) if schedule.Attempt > 0 { detail += fmt.Sprintf("; attempt: %d", schedule.Attempt) } - if schedule.PinnedRelease != "" { - detail += fmt.Sprintf("; release: %s; started: %s", schedule.PinnedRelease, schedule.StartedAt) + if schedule.Release != "" { + detail += fmt.Sprintf("; release: %s", schedule.Release) + } + if schedule.StartedAt != "" { + detail += fmt.Sprintf("; started: %s", schedule.StartedAt) + } + if schedule.Diverged { + diverged = true + detail += "; " + strings.Join(schedule.Issues, "; ") + " ⚠" + detail = e.ui.Warn(detail) } fmt.Fprintf(e.Opts.Out, "schedule %-11s %s\n", schedule.Name, detail) continue } - detail := fmt.Sprintf("active; policy: %s; timeout: %s", schedule.DeployLock, schedule.Timeout) + detail := fmt.Sprintf("active; policy: %s; timeout: %s; retry budget: %d attempt(s), %s backoff", + schedule.DeployLock, schedule.Timeout, schedule.MaxAttempts, schedule.RetryBudget) if schedule.NextRun != "" { detail += "; next: " + schedule.NextRun } @@ -208,6 +220,15 @@ func (e *Engine) Status(ctx context.Context) error { result = fmt.Sprintf("%s (%ds, %d attempt(s))", schedule.LastOutcome, schedule.LastDurationSeconds, schedule.LastAttempts) } detail += "; last: " + result + if schedule.LastTimerOutcome != "" { + detail += fmt.Sprintf("; last timer: %s at %s", schedule.LastTimerOutcome, schedule.LastTimerAt) + } + if schedule.LastOperatorOutcome != "" { + detail += fmt.Sprintf("; last operator: %s at %s", schedule.LastOperatorOutcome, schedule.LastOperatorAt) + } + if schedule.LastSuccessAt != "" { + detail += "; last observed success: " + schedule.LastSuccessAt + } if !schedule.JournalPersistent { detail += "; journal: volatile, history since boot only" } @@ -238,6 +259,13 @@ func (e *Engine) Status(ctx context.Context) error { return nil } +func orUnknown(value string) string { + if strings.TrimSpace(value) == "" { + return "unknown" + } + return value +} + // statusWorkloadRevisions reads the active release's own runtime contract only // when an older release label could be a retained workload. The active Compose // is the authority: a local inspection render can contain an authored tag or an diff --git a/internal/engine/status_snapshot_test.go b/internal/engine/status_snapshot_test.go index f8e01e67..7f8a802c 100644 --- a/internal/engine/status_snapshot_test.go +++ b/internal/engine/status_snapshot_test.go @@ -233,7 +233,7 @@ func TestStatusSnapshotReportsObservedDivergenceAndIncompleteDeploy(t *testing.T func TestStatusSnapshotIncludesScheduledJobFailure(t *testing.T) { cfg := testConfig() cfg.Workloads["nightly"] = app.Workload{ - Role: app.RoleJob, When: "manual", DataEffect: "none", + Role: app.RoleJob, DeploymentPhase: "none", DataEffect: "none", Schedule: &app.JobSchedule{Cron: "0 2 * * *", Timezone: "UTC", Timeout: "1h", CatchUp: true}, } f := statusFake("R2", "R2") diff --git a/internal/engine/verify_contract_test.go b/internal/engine/verify_contract_test.go index 2c533130..193c03ea 100644 --- a/internal/engine/verify_contract_test.go +++ b/internal/engine/verify_contract_test.go @@ -188,7 +188,7 @@ func TestVerifyURLSuccessOutputRedactsQuery(t *testing.T) { func TestVerifyMigrationRevisionsMatchesBoundProviderEvidence(t *testing.T) { cfg := testConfig() cfg.Workloads = map[string]app.Workload{ - "migrate": {Role: app.RoleJob, When: "pre_release", DataEffect: "migration"}, + "migrate": {Role: app.RoleJob, DeploymentPhase: "pre_release", DataEffect: "migration"}, } cfg.Checks = app.Checks{Migrations: []app.MigrationCheck{{ Job: "migrate", Provider: "atlas", AppliedRevisions: []string{"r1", "r2"}, diff --git a/internal/journal/journal.go b/internal/journal/journal.go index ef72fcb1..9f6d818f 100644 --- a/internal/journal/journal.go +++ b/internal/journal/journal.go @@ -76,6 +76,8 @@ type Record struct { // do not have. OperationKind string `json:"operation_kind,omitempty"` Service string `json:"service,omitempty"` + ReleaseID string `json:"release_id,omitempty"` + DataEffect string `json:"data_effect,omitempty"` // Exec invocation evidence is intentionally value-free: command bytes and // passthrough output never cross the durable journal boundary. Target string `json:"target,omitempty"` diff --git a/internal/onebox/backup_evidence_test.go b/internal/onebox/backup_evidence_test.go index 95a22069..4b27d187 100644 --- a/internal/onebox/backup_evidence_test.go +++ b/internal/onebox/backup_evidence_test.go @@ -366,7 +366,7 @@ func TestPlanDerivesMigrationBackupRequirementAndExecuteRejectsMissingReportBefo " allow_agent_proposals: true\n migrations: {require_backup: true, backup_max_age: 24h, require_restore_test: true, backup_key_material: [application_encryption_key]}\n", 1) configText = strings.Replace(configText, " database:\n", - " migrate:\n role: job\n image: ghcr.io/example/app:migrate\n when: pre_release\n data_effect: migration\n database:\n", 1) + " migrate:\n role: job\n image: ghcr.io/example/app:migrate\n deployment_phase: pre_release\n data_effect: migration\n database:\n", 1) if err := os.WriteFile(configPath, []byte(configText), 0o600); err != nil { t.Fatal(err) } diff --git a/internal/onebox/execution_types.go b/internal/onebox/execution_types.go index 030ac43d..b490bbfb 100644 --- a/internal/onebox/execution_types.go +++ b/internal/onebox/execution_types.go @@ -293,7 +293,7 @@ type ExecuteRequest struct { BackupReport *BackupReport MigrationBackupOverride *MigrationBackupOverride BreakLock bool - // Detach asks a planned manual job to return after its installed host unit + // Detach asks a planned operator job to return after its installed host unit // accepts the run. It is valid only for job_run; the job's schedule history // remains the outcome authority. Detach bool diff --git a/internal/onebox/job_execute.go b/internal/onebox/job_execute.go index 99d3e1eb..1ec895ce 100644 --- a/internal/onebox/job_execute.go +++ b/internal/onebox/job_execute.go @@ -7,6 +7,7 @@ import ( "reflect" "time" + "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/engine" "github.com/labstack/onebox/internal/journal" ) @@ -62,8 +63,11 @@ func (s *Service) executeJob( return "", nil, nil, errors.New("configuration changed since job planning — re-plan") } job, ok := lp.resolved.Workloads[plan.Artifact.Job] - if !ok || !job.IsJob() || job.When != "manual" || job.DataEffect != plan.Artifact.DataEffect { - return "", nil, nil, errors.New("manual job declaration changed since planning — re-plan") + if !ok || !job.IsJob() || job.OperatorRun != "allowed" || job.DataEffect != plan.Artifact.DataEffect { + return "", nil, nil, errors.New("operator-run job declaration changed since planning — re-plan") + } + if err := app.ValidateJobInputValues(job, plan.Artifact.Inputs); err != nil { + return "", nil, nil, errors.New("job input declaration changed since planning — re-plan") } expectedBackup, err := migrationBackupRequirement(lp.resolved, environmentConfig.Policy, plan.Operation.Steps) if err != nil { @@ -123,7 +127,7 @@ func (s *Service) executeJob( emit("binding", "succeeded", "") emit("execute", "started", "") if job.Schedule != nil && job.DataEffect != DataEffectMigration { - run, err := e.PlannedJobRun(ctx, plan.Operation.ID, plan.Artifact.Job, + run, err := e.PlannedJobRun(ctx, plan.Operation.ID, plan.Artifact.Job, plan.Artifact.Inputs, plan.Artifact.CurrentRelease, plan.Artifact.RuntimeDigest, !request.Detach) if err == nil { emit("execute", "succeeded", "") diff --git a/internal/onebox/job_plan.go b/internal/onebox/job_plan.go index a86bafd9..0ba04afb 100644 --- a/internal/onebox/job_plan.go +++ b/internal/onebox/job_plan.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/labstack/onebox/internal/app" "github.com/labstack/onebox/internal/buildinfo" "github.com/labstack/onebox/internal/engine" "github.com/labstack/onebox/internal/release" @@ -23,14 +24,15 @@ const maxExecutableJobPlanBytes = 1 << 20 var pinnedJobImage = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) type JobArtifact struct { - Application string `json:"application"` - Environment string `json:"environment"` - Server string `json:"server"` - CurrentRelease string `json:"current_release"` - RuntimeDigest string `json:"runtime_digest"` - Job string `json:"job"` - Image string `json:"image"` - DataEffect DataEffectClass `json:"data_effect"` + Application string `json:"application"` + Environment string `json:"environment"` + Server string `json:"server"` + CurrentRelease string `json:"current_release"` + RuntimeDigest string `json:"runtime_digest"` + Job string `json:"job"` + Image string `json:"image"` + DataEffect DataEffectClass `json:"data_effect"` + Inputs map[string]string `json:"inputs,omitempty"` } // JobPlan is a sealed, current-release-bound one-shot operation. It contains @@ -218,7 +220,8 @@ func LoadExecutablePlan(path string) (ExecutablePlan, error) { } type PlanJobRequest struct { - Job string + Job string + Inputs map[string]string } func (s *Service) PlanJob(ctx context.Context, request PlanJobRequest) (JobPlan, error) { @@ -248,8 +251,11 @@ func (s *Service) PlanJob(ctx context.Context, request PlanJobRequest) (JobPlan, if !ok || !job.IsJob() { return JobPlan{}, fmt.Errorf("unknown job %q", jobID) } - if job.When != "manual" { - return JobPlan{}, fmt.Errorf("job %q is %s; one-shot invocation is reserved for when: manual jobs", jobID, job.When) + if job.OperatorRun != "allowed" { + return JobPlan{}, fmt.Errorf("job %q has operator_run %s", jobID, job.OperatorRun) + } + if err := app.ValidateJobInputValues(job, request.Inputs); err != nil { + return JobPlan{}, err } e, cleanup, target, err := s.engine(ctx, lp, s.environment) if err != nil { @@ -292,7 +298,7 @@ func (s *Service) PlanJob(ctx context.Context, request PlanJobRequest) (JobPlan, artifact := JobArtifact{ Application: lp.resolved.Name, Environment: s.environment, Server: target, CurrentRelease: hostState.CurrentRelease, RuntimeDigest: runtimeDigest, - Job: jobID, Image: image, DataEffect: effect, + Job: jobID, Image: image, DataEffect: effect, Inputs: request.Inputs, } stateDigest, err := jobArtifactDigest(artifact) if err != nil { diff --git a/internal/onebox/job_plan_test.go b/internal/onebox/job_plan_test.go index a0fed4d9..7d0b5f8e 100644 --- a/internal/onebox/job_plan_test.go +++ b/internal/onebox/job_plan_test.go @@ -20,7 +20,7 @@ func writeManualJobProject(t *testing.T, effect string, requireBackup bool) stri } project := strings.Replace(string(encoded), " database:\n", - " maintenance:\n role: job\n image: ghcr.io/example/maintenance:v1\n when: manual\n data_effect: "+effect+"\n database:\n", 1) + " maintenance:\n role: job\n image: ghcr.io/example/maintenance:v1\n deployment_phase: none\n data_effect: "+effect+"\n database:\n", 1) if requireBackup { project = strings.Replace(project, " allow_agent_proposals: true\n", diff --git a/internal/onebox/operation_errors.go b/internal/onebox/operation_errors.go index 2574d5ad..7da9caa7 100644 --- a/internal/onebox/operation_errors.go +++ b/internal/onebox/operation_errors.go @@ -131,6 +131,18 @@ var operationFailureDefinitions = map[string]OperationFailure{ Message: "the one-shot job plan could not be produced", Command: "ob job plan --output json", }, + "job_history_failed": { + Message: "the job's retained execution records could not be read", + Command: "ob status --output json", + }, + "job_input_invalid": { + Message: "an --input flag is not NAME=VALUE, or names the same input twice", + Command: "ob canonical --output json", + }, + "job_logs_failed": { + Message: "the host-supervised job journal could not be read", + Command: "ob job history --output json", + }, "logs_failed": { Message: "log retrieval failed", Command: "ob status --output json", @@ -139,22 +151,10 @@ var operationFailureDefinitions = map[string]OperationFailure{ Message: "a release manifest is not valid closed JSON for its schema", Command: "ob status --output json", }, - "schedule_history_failed": { - Message: "the scheduled job's run records could not be read from the host journal", - Command: "ob status --output json", - }, - "schedule_input_invalid": { - Message: "an --input flag is not NAME=VALUE, or names the same input twice", - Command: "ob canonical --output json", - }, "schedule_list_failed": { Message: "the scheduled jobs' timer state could not be read", Command: "ob status --output json", }, - "schedule_logs_failed": { - Message: "the scheduled run's journal could not be read", - Command: "ob schedule history --output json", - }, "manifest_missing": { Message: "a release directory carries no manifest, so its lifecycle state is unknown", Command: "ob status --output json", diff --git a/internal/onebox/operation_graph_test.go b/internal/onebox/operation_graph_test.go index ee9caf1d..443f0b2f 100644 --- a/internal/onebox/operation_graph_test.go +++ b/internal/onebox/operation_graph_test.go @@ -53,7 +53,7 @@ func TestDeploymentGraphIsDeterministicAndOrdered(t *testing.T) { } for _, step := range first { if step.ID == "job:nightly" { - t.Fatal("manual job entered the deploy operation graph") + t.Fatal("operator job entered the deploy operation graph") } } } @@ -124,10 +124,10 @@ environments: {production: {server: root@h}} workloads: web: {role: application, image: x:1, strategy: rolling, health: {http: /healthz, port: 8080}} worker: {role: worker, image: x:1, strategy: recreate} - migrate: {role: job, image: x:1, command: "echo JOB_SECRET", when: pre_release, data_effect: migration} - assets: {role: job, image: x:1, when: pre_release, data_effect: none} - cleanup: {role: job, image: x:1, when: post_release, data_effect: none} - nightly: {role: job, image: x:1, when: manual, data_effect: none, schedule: {cron: "0 2 * * *"}} + migrate: {role: job, image: x:1, command: "echo JOB_SECRET", deployment_phase: pre_release, data_effect: migration} + assets: {role: job, image: x:1, deployment_phase: pre_release, data_effect: none} + cleanup: {role: job, image: x:1, deployment_phase: post_release, data_effect: none} + nightly: {role: job, image: x:1, deployment_phase: none, data_effect: none, schedule: {cron: "0 2 * * *"}} deployment: order: [worker, web] hooks: diff --git a/site/public/onebox.run-v1.schema.json b/site/public/onebox.run-v1.schema.json index 6e64dd6a..484319f5 100644 --- a/site/public/onebox.run-v1.schema.json +++ b/site/public/onebox.run-v1.schema.json @@ -1714,8 +1714,11 @@ "data_effect": { "const": "none" }, - "when": { - "const": "manual" + "deployment_phase": { + "const": "none" + }, + "operator_run": { + "const": "allowed" } }, "required": [ @@ -1897,7 +1900,12 @@ "anyOf": [ { "required": [ - "when" + "deployment_phase" + ] + }, + { + "required": [ + "operator_run" ] }, { @@ -2032,6 +2040,16 @@ ], "type": "string" }, + "deployment_phase": { + "default": "none", + "description": "Deployment phase for this job: none, pre_release, or post_release.", + "enum": [ + "none", + "pre_release", + "post_release" + ], + "type": "string" + }, "domain": { "description": "Domain shorthand for one HTTPS route; requires port and cannot be combined with routes.", "examples": [ @@ -2141,7 +2159,7 @@ }, "execution": { "additionalProperties": false, - "description": "Opt-in durable scheduled execution. Requires a native manual job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume.", + "description": "Opt-in durable scheduled execution. Requires a native operator-runnable phase-none job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume.", "patternProperties": { "^x-": {} }, @@ -2399,7 +2417,7 @@ }, "properties": { "default": { - "description": "Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint.", + "description": "Value used by a timer firing and by an operator run that does not override it. Must satisfy the input's own constraint.", "type": "string" }, "description": { @@ -2431,7 +2449,7 @@ ], "type": "object" }, - "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them.", + "description": "Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob job run may override them.", "propertyNames": { "pattern": "^[A-Z][A-Z0-9_]*$" }, @@ -2510,6 +2528,14 @@ }, "type": "array" }, + "operator_run": { + "description": "Whether an operator may invoke this job outside deployment: allowed or disabled. Defaults to allowed for phase none and disabled otherwise.", + "enum": [ + "allowed", + "disabled" + ], + "type": "string" + }, "persistence": { "additionalProperties": false, "description": "Declares whether this workload holds data that must outlive releases.", @@ -2712,7 +2738,7 @@ }, "schedule": { "additionalProperties": false, - "description": "Host-resident recurring schedule and run policy for a job.", + "description": "Host-resident recurring schedule and run policy for a job, independent of its deployment phase and operator-run policy.", "patternProperties": { "^x-": {} }, @@ -2797,6 +2823,15 @@ }, "type": "object" }, + "shutdown_grace": { + "default": "30s", + "description": "Time allowed for graceful container shutdown after the run deadline before Onebox forces removal. Expects a duration such as 30s, 5m, 1h30m or 14d.", + "examples": [ + "45s" + ], + "pattern": "^(([0-9]+([.][0-9]+)?(ns|us|µs|ms|s|m|h))+|[0-9]+d)$", + "type": "string" + }, "timeout": { "default": "1h", "description": "Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d.", @@ -2925,16 +2960,6 @@ }, "type": "array" }, - "when": { - "default": "manual", - "description": "When a job runs: manual, pre_release, or post_release.", - "enum": [ - "pre_release", - "post_release", - "manual" - ], - "type": "string" - }, "working_dir": { "description": "Absolute working directory for the container process. Expects an absolute path with no control character or shell metacharacter.", "examples": [ diff --git a/site/src/content/docs/guides/run-migrations.mdx b/site/src/content/docs/guides/run-migrations.mdx index 1031ffc1..a9baa904 100644 --- a/site/src/content/docs/guides/run-migrations.mdx +++ b/site/src/content/docs/guides/run-migrations.mdx @@ -19,7 +19,7 @@ workloads: image: ghcr.io/acme/shop:1.4.0 command: ["./bin/migrate"] data_effect: migration - when: pre_release + deployment_phase: pre_release needs: [{name: postgres, condition: healthy}] ``` @@ -33,8 +33,9 @@ gates read: | `destructive` | Removes data. | | `unknown` | You cannot state it. Treated as the most cautious case. | -`when` decides when it happens: `manual` (default), `pre_release`, or -`post_release`. +`deployment_phase` decides whether it runs during deployment: `none` (default), +`pre_release`, or `post_release`. `operator_run` independently controls explicit +invocation (`allowed` or `disabled`); release-phase jobs default to `disabled`. ## Report what actually happened diff --git a/site/src/content/docs/guides/schedule-a-job.mdx b/site/src/content/docs/guides/schedule-a-job.mdx index 28748504..72677ff0 100644 --- a/site/src/content/docs/guides/schedule-a-job.mdx +++ b/site/src/content/docs/guides/schedule-a-job.mdx @@ -174,9 +174,9 @@ Read the records back from the workstation: ```sh ob schedule list # every job, its timer state and next elapse -ob schedule history nightly-dump # records, newest first; -n 50 for more -ob schedule logs nightly-dump # the journal of the newest run -ob schedule logs nightly-dump --run a3f9… +ob job history nightly-dump # records, newest first; -n 50 for more +ob job logs nightly-dump # the journal of the newest run +ob job logs nightly-dump --run a3f9… ``` `ob status` reads the same records. Each scheduled job's line carries the next @@ -208,7 +208,7 @@ schedule: Each selected outcome sends a bounded, fail-open POST to every webhook whose own `on` list accepts that class of outcome. The payload carries the run id as -its `deploy_id`, so `ob schedule logs --run ` finds the run; it +its `deploy_id`, so `ob job logs --run ` finds the run; it carries nothing else about the run, because notifications cross the host trust boundary and diagnostics stay on the host. Onebox writes the webhook handler root-only beside the unit, so credentials in webhook paths do not appear in @@ -279,7 +279,7 @@ lock for its whole run and a crashed deploy leaves one behind. Resuming starts the timer again; the next run is the next scheduled elapse. It does not run the job now, and does not make up firings missed while paused — -`ob schedule run` is how you ask for an immediate run. +`ob job run` is how you ask for an immediate run. Deleting a job from the project clears its pause along with its units, so a name reused later does not come back stopped for a reason from a previous @@ -313,7 +313,7 @@ workloads: ``` ```sh -ob schedule run source-sync --input SOURCE=prices --input SINCE=2026-09-01 --wait +ob job run source-sync --input SOURCE=prices --input SINCE=2026-09-01 ``` Names are upper-case identifiers outside Onebox's `ONEBOX_` namespace and may @@ -323,21 +323,20 @@ value. Whatever the pattern allows, a value may not contain a double quote, a backslash, or a control character, and is at most 256 bytes. Those rules are what let the runner hand values to the container without escaping anything. -`ob schedule run` validates every value on the workstation, refuses if the -unit is already running or a manual run is still pending, journals the request +`ob job run` validates every value on the workstation, refuses if the +unit is already running or an operator run is still pending, journals the request as `schedule_run` with the operator and the inputs, and starts the unit. It holds the application lock only while writing and journaling, because the runner skips a run that meets that lock. The host record of the run carries the -operation id, so `ob audit` and `ob schedule history` join on it. Without -`--wait` the command returns as soon as the unit is started; with it, the -command blocks until the unit exits and reports the record. +operation id, so `ob audit` and `ob job history` join on it. The command follows +the host-supervised unit by default; `--detach` returns once the unit accepts +the run. -Only a job with `data_effect: none` accepts inputs or manual runs this way. A -`migration` or `destructive` job is still run by its timer as declared, but an -operator-initiated run of it keeps the sealed plan, approval and backup-report -gates of `ob job run` below. +Only a job with `data_effect: none` accepts inputs. Every operator invocation +uses a sealed plan; migration jobs additionally retain their approval and +backup-report gates and remain attached so Onebox can capture result evidence. -The runner tells a manual activation from a timer firing by the `TRIGGER_UNIT` +The runner tells an operator activation from a timer firing by the `TRIGGER_UNIT` variable systemd sets on timer activations, which that project introduced in version 252. Ubuntu 24.04 and Debian 12 qualify; Ubuntu 22.04 and Debian 11 do not. @@ -347,7 +346,7 @@ Scheduled jobs still run on those older hosts, unchanged. Two things narrow: - A job that declares `inputs` is refused, by `ob preflight` and by `ob deploy`, before anything is staged. Without `TRIGGER_UNIT` the next timer firing would read the file meant for an operator's run. -- `ob schedule run` is refused for the same reason. The timer keeps firing. +- `ob job run` is refused for the same reason. The timer keeps firing. The records on such a host say `trigger: unknown`, because the runner cannot observe what started it and will not guess. @@ -357,7 +356,7 @@ observe what started it and will not guess. Opt into durable execution when a failed job needs to continue with its original inputs. Onebox saves orchestration state on the managed host; the application keeps responsibility for domain checkpoints and idempotent effects. This requires -a native Onebox job with `schedule`, `when: manual` (the default), and +a native Onebox job with `schedule`, `deployment_phase: none` (the default), and `data_effect: none`. Adopted Compose jobs and release-phase jobs are refused. For one command, add `execution: {retention: 168h}` to the job. Onebox treats the @@ -452,7 +451,7 @@ must fit below `schedule.timeout`. ### Inspect and recover ```sh -ob schedule run refresh --wait # start a new execution +ob job run refresh # start a new execution ob execution list # newest executions; -n 50 for more ob execution inspect # original inputs, steps, attempts, eligibility ob execution resume --wait # skip completed steps; use saved outputs @@ -460,13 +459,13 @@ ob execution abandon # end resumability and release retention ho ``` Resume cannot change inputs. To use different inputs, start a new execution with -`ob schedule run --input NAME=VALUE`. Timer firings also create new executions; +`ob job run --input NAME=VALUE`. Timer firings also create new executions; they do not resume failed work automatically. After a reboot, inspect interrupted work and request resume explicitly. A pending failed execution does not stop future timer firings; use `ob schedule pause --reason "..."` if needed. Inspection includes each activation's systemd invocation ID. Read its logs with -`ob schedule logs refresh --run `. The execution ID stays the same +`ob job logs refresh --run `. The execution ID stays the same across activations; an invocation ID identifies one activation's journal output. Run logs keep the journal's retention independently of saved execution state. @@ -523,23 +522,24 @@ ob approve --plan ob-job-plan.json --out ob-job-approval.json ob job run --plan ob-job-plan.json --approval ob-job-approval.json ``` -When the manual job also declares `schedule`, Onebox submits this run to the +When the operator job also declares `schedule`, Onebox submits this run to the same installed systemd unit used by its timer. The host therefore owns the container, timeout, overlap lock, retries, and run record if the SSH session or operator terminal disappears. The command follows the unit by default and shows elapsed time. Ctrl-C stops following but does not stop the host job; use `--detach` to return immediately after the unit accepts it. Inspect either form -with `ob schedule history ` and `ob schedule logs `. +with `ob job history ` and `ob job logs `. Jobs without `schedule` keep the direct foreground path. Migration jobs also stay attached because their result evidence is part of the approved operation. -`ob job plan` accepts a job only when its resolved `when` is `manual`, which is -the default for a job that declares none. A `pre_release` or `post_release` job -is refused: it belongs to the deploy graph, and one-shot invocation is reserved -for the jobs that do not. +`ob job plan` accepts a job when `operator_run` is `allowed`. This property is +independent of deployment: `deployment_phase` decides whether the job runs in +the deploy graph, while `operator_run` decides whether an operator may invoke +it. By default, phase `none` allows operator runs; release-phase jobs disable +them unless explicitly enabled. -A declared `when: manual` job remains in the digest-pinned release runtime but +A declared `deployment_phase: none` job remains in the digest-pinned release runtime but never joins the deploy graph. Its job plan binds the current serving release, runtime digest, immutable image, data effect, target, and expiry. Automation supplies the saved plan and its separately recorded local confirmation; @@ -561,12 +561,15 @@ The reason is durable metadata, so do not put a secret in it. For a job that participates automatically in a deploy, choose a release phase: ```yaml -when: pre_release # or post_release; manual is invoked only through job plan/run +deployment_phase: pre_release +operator_run: allowed # optional: also permit ob job plan/run ``` -`schedule` and `when` are orthogonal, and both fire. A job declaring `schedule:` -**and** `when: pre_release` runs on the host timer at its cron time *and* again -on every deploy. Declaring both is how you ask for both; for the timer alone, -leave `when` at its `manual` default. +The three properties are independent. `deployment_phase` controls deploy hooks, +`schedule` controls timer activation, and `operator_run` controls explicit +invocation. A job declaring `schedule:` and `deployment_phase: pre_release` +runs both at its cron time and on every deploy. For a timer-only job, leave +`deployment_phase` at `none`; set `operator_run: disabled` too if operators +must not start it explicitly. See [`workloads`](/reference/fields/workloads) for every field. diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index f7edefa2..dbedff09 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -61,7 +61,7 @@ Available Commands: execution inspect and recover durable job executions help Help about any command init scaffold ob.yml from the compose file + rollability doctor - job plan and run a sealed one-shot manual job + job plan and run a sealed one-shot operator job logs compose logs from the current release plan refresh → rendered diff + pinned images + command list → plan artifact preflight ask the server whether this project could be deployed (changes nothing) @@ -749,19 +749,21 @@ Global Flags: ## ob job ``` -Plan and run one declared `when: manual` job against the current serving release. +Plan and run one declared `operator_run: allowed` job against the current serving release. -The job remains in the release runtime but never runs during deploy. Saved plans -bind its release, runtime digest, immutable image and data effect so agents can -obtain a separate approval before execution. +Deployment participation is independent: `deployment_phase` may be none, pre_release, +or post_release. Saved plans bind the release, runtime digest, immutable image, +data effect and inputs so agents can obtain separate approval before execution. Usage: ob job [flags] ob job [command] Available Commands: + history execution records of one job, newest first + logs journal of one host-supervised job execution plan seal a current-release-bound one-shot job plan - run run one manual job from an inline or saved sealed plan + run run one operator job from an inline or saved sealed plan Flags: -h, --help help for job @@ -775,6 +777,44 @@ Global Flags: Use "ob job [command] --help" for more information about a command. ``` +### ob job history + +``` +Read retained execution records for one job across timer and operator triggers. Host-supervised records come from journald; sealed attached runs come from the operation journal. The result is retention-bounded evidence, so an absent success means no retained success was observed, not that the job never succeeded. + +Usage: + ob job history [flags] + +Flags: + -n, --count int number of newest executions to show (default 20) + -h, --help help for history + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + +### ob job logs + +``` +Stream the exact systemd journal of a host-supervised job execution. By default the newest retained run is selected; --run accepts the run id printed by ob job history. Attached sealed executions retain outcome evidence but do not have a separate host log stream. + +Usage: + ob job logs [flags] + +Flags: + -h, --help help for logs + --run string run id from ob job history; defaults to newest host-supervised run + +Global Flags: + -c, --config string path to the project YAML file (default "ob.yml") + -e, --env string environment name (default "production") + --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") + -v, --verbose print every remote command +``` + ### ob job plan ``` @@ -789,6 +829,7 @@ Usage: Flags: --backup-report-out string write a plan-bound backup report template when migration backup is required -h, --help help for plan + --input stringArray input override as NAME=VALUE; repeatable and sealed into the plan -o, --out string job plan artifact path (default "ob-job-plan.json") Global Flags: @@ -801,7 +842,7 @@ Global Flags: ### ob job run ``` -Run one manual job through the canonical lock, fence, local-confirmation and journal boundary. +Run one operator job through the canonical lock, fence, local-confirmation and journal boundary. A job with schedule configured runs under its installed systemd unit and is followed by default; Ctrl-C stops following, not the host job. --detach returns @@ -820,6 +861,7 @@ Flags: --break-lock break a stale operation lock after inspecting its holder --detach return after the installed host unit accepts the job -h, --help help for run + --input stringArray input override as NAME=VALUE; repeatable and sealed into the inline plan --override-migration-backup string audited break-glass reason (requires --approval) --plan string apply a saved job plan artifact @@ -1038,8 +1080,8 @@ Manage the systemd timers generated for scheduled jobs. Timers outlive the Onebox process and the package installed on the operator workstation. `apply` explicitly reconciles their units after a runner or -configuration change without deploying a release. `list`, `history` and `logs` -read the timer state and the run records the host keeps in its journal. +configuration change without deploying a release. `list` reads timer state; +job history and logs live under `ob job`. Usage: ob schedule [flags] @@ -1047,12 +1089,9 @@ Usage: Available Commands: apply reconcile scheduled-job units without deploying a release - history run records of one scheduled job, newest first list declared scheduled jobs with timer state and next elapse - logs journal of one scheduled run pause stop a scheduled job's timer until it is resumed resume start a paused scheduled job's timer again - run start a scheduled job now with declared inputs Flags: -h, --help help for schedule @@ -1087,27 +1126,6 @@ Global Flags: -v, --verbose print every remote command ``` -### ob schedule history - -``` -Read the run records the host wrote for one scheduled job. Each record is one activation: run id, trigger, release, start and end, attempts, exit status, outcome and, for a manual run, its inputs. - -Records live in the host journal with syslog identifier ob-run and the job's unit in their ONEBOX_UNIT field; retention is the journal's. Reads only. - -Usage: - ob schedule history [flags] - -Flags: - -n, --count int number of newest runs to show (default 20) - -h, --help help for history - -Global Flags: - -c, --config string path to the project YAML file (default "ob.yml") - -e, --env string environment name (default "production") - --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") - -v, --verbose print every remote command -``` - ### ob schedule list ``` @@ -1126,28 +1144,6 @@ Global Flags: -v, --verbose print every remote command ``` -### ob schedule logs - -``` -Stream the host journal for one run of a scheduled job: by default the newest recorded run, or the run named with --run. The run id is systemd's invocation id, so the output is exactly that activation. Reads only. - -Log bytes are operator-controlled and may contain secrets; Onebox does not claim -to redact passthrough output. - -Usage: - ob schedule logs [flags] - -Flags: - -h, --help help for logs - --run string run id from ob schedule history; default the newest run - -Global Flags: - -c, --config string path to the project YAML file (default "ob.yml") - -e, --env string environment name (default "production") - --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") - -v, --verbose print every remote command -``` - ### ob schedule pause ``` @@ -1175,7 +1171,7 @@ Global Flags: ``` Start a paused job's timer and clear the record of the pause. -The next run is the next scheduled elapse: resuming does not run the job now, and does not make up the firings missed while it was paused. Use `ob schedule run` for an immediate run. +The next run is the next scheduled elapse: resuming does not run the job now, and does not make up the firings missed while it was paused. Use `ob job run` for an immediate run. Usage: ob schedule resume [flags] @@ -1191,31 +1187,6 @@ Global Flags: -v, --verbose print every remote command ``` -### ob schedule run - -``` -Start one scheduled job's unit now, with values for its declared inputs. Values are validated on the workstation against the declaration; an undeclared name or a value outside its enum or pattern is refused before anything reaches the host. - -Only a job with data_effect none may run this way; a migration or destructive job keeps the sealed plan of ob job run. The request is journaled as schedule_run with the operator and inputs, and the host record carries the operation id, so ob audit and ob schedule history join on it. - -The outcome is the run record: ob schedule history , or --wait to block until the unit exits and print it. - -Usage: - ob schedule run [flags] - -Flags: - --break-lock break a stale operation lock after inspecting its holder - -h, --help help for run - --input stringArray input override as NAME=VALUE; repeatable - --wait block until the unit exits and report the run record - -Global Flags: - -c, --config string path to the project YAML file (default "ob.yml") - -e, --env string environment name (default "production") - --output string output mode for supported commands: human|json|ndjson (see the CLI reference) (default "human") - -v, --verbose print every remote command -``` - ## ob schema ``` diff --git a/site/src/content/docs/reference/errors.mdx b/site/src/content/docs/reference/errors.mdx index b3f3b3aa..96e1e4ac 100644 --- a/site/src/content/docs/reference/errors.mdx +++ b/site/src/content/docs/reference/errors.mdx @@ -137,6 +137,9 @@ step to complete rather than a line to run verbatim. | `host_environment_mismatch` | this host is claimed by a different environment of the same application, which would share its container and volume names | diagnostic | `ob preflight --output json` | | `host_owner_mismatch` | this host is owned by a different Onebox application, and one host has one owner | diagnostic | `ob preflight --output json` | | `interrupted` | the operation's client went away before its outcome could be recorded | diagnostic | `ob audit --output json` | +| `job_history_failed` | the job's retained execution records could not be read | diagnostic | `ob status --output json` | +| `job_input_invalid` | an --input flag is not NAME=VALUE, or names the same input twice | diagnostic | `ob canonical --output json` | +| `job_logs_failed` | the host-supervised job journal could not be read | next | `ob job history --output json` | | `job_plan_failed` | the one-shot job plan could not be produced | next | `ob job plan --output json` | | `logs_failed` | log retrieval failed | diagnostic | `ob status --output json` | | `manifest_invalid` | a release manifest is not valid closed JSON for its schema | diagnostic | `ob status --output json` | @@ -156,10 +159,7 @@ step to complete rather than a line to run verbatim. | `preflight_failed` | a target readiness check failed before any mutation | — | — | | `recovery_incomplete` | recovery did not reach its verified terminal state | resolving | `ob resume --output ndjson` | | `rollback_target_missing` | no previously serving release is recorded as a rollback target | next | `ob plan --output json` | -| `schedule_history_failed` | the scheduled job's run records could not be read from the host journal | diagnostic | `ob status --output json` | -| `schedule_input_invalid` | an --input flag is not NAME=VALUE, or names the same input twice | diagnostic | `ob canonical --output json` | | `schedule_list_failed` | the scheduled jobs' timer state could not be read | diagnostic | `ob status --output json` | -| `schedule_logs_failed` | the scheduled run's journal could not be read | next | `ob schedule history --output json` | | `secret_cleanup_pending` | the rotation is applied and verified, but removing the retired generation did not finish | resolving | `ob secrets push --output ndjson` | | `secret_declaration_not_deployed` | the deployed release does not declare this secret graph | next | `ob plan --output json` | | `secret_entry_not_selected` | more than one editable secret source exists, so an entry identifier is required | diagnostic | `ob secrets list --output json` | diff --git a/site/src/content/docs/reference/fields/workloads.mdx b/site/src/content/docs/reference/fields/workloads.mdx index df58dcd7..1e21301a 100644 --- a/site/src/content/docs/reference/fields/workloads.mdx +++ b/site/src/content/docs/reference/fields/workloads.mdx @@ -19,7 +19,7 @@ cannot drift from what `ob validate` accepts. ## Fields on this page -`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `default` · `deploy_lock` · `description` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `enum` · `env` · `env_files` · `exec` · `execution` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `id` · `image` · `init` · `inputs` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `options` · `outputs` · `path` · `pattern` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retention` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `signal` · `source` · `start_period` · `stdin_open` · `steps` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `when` · `within` · `working_dir` +`args` · `attempts` · `backoff` · `bind` · `build` · `catch_up` · `command` · `compose` · `condition` · `container` · `context` · `cpus` · `cron` · `data_effect` · `default` · `deploy_lock` · `deployment_phase` · `description` · `dockerfile` · `domain` · `drain` · `driver` · `entrypoint` · `enum` · `env` · `env_files` · `exec` · `execution` · `extra_hosts` · `file` · `grace` · `health` · `host` · `hostname` · `http` · `id` · `image` · `init` · `inputs` · `interval` · `labels` · `logging` · `max_backoff` · `memory` · `middlewares` · `mode` · `name` · `needs` · `notify` · `operator_run` · `options` · `outputs` · `path` · `pattern` · `persistence` · `port` · `protocol` · `provider` · `published_ports` · `pull` · `reference` · `replicas` · `resources` · `retention` · `retries` · `retry` · `role` · `routes` · `schedule` · `scheme` · `shutdown_grace` · `signal` · `source` · `start_period` · `stdin_open` · `steps` · `strategy` · `target` · `tcp` · `timeout` · `timezone` · `tls` · `tty` · `user` · `volumes` · `wait` · `within` · `working_dir` ## Reference @@ -33,6 +33,7 @@ cannot drift from what `ob validate` accepts. | `.command` | list | — | Container command as a shell string or argument list. Also accepts a command line or argument list. | | `.compose` | string | — | Existing Compose service to adopt, as repository path#service. Expects a reference of the form path/to/compose.yaml#service. | | `.data_effect` | `none` · `migration` · `destructive` · `unknown` | — | Job data impact used by rollback and abort gates. | +| `.deployment_phase` | `none` · `pre_release` · `post_release` | `none` | Deployment phase for this job: none, pre_release, or post_release. | | `.domain` | string | — | Domain shorthand for one HTTPS route; requires port and cannot be combined with routes. | | `.drain` | object | — | Signal and timing used to remove a container from traffic before stopping it. | | `.drain.grace` | string | — | Maximum graceful-shutdown time before forced termination, at most 7d. Expects a duration such as 30s, 5m, 1h30m or 14d. | @@ -43,7 +44,7 @@ cannot drift from what `ob validate` accepts. | `.env_files` | list | — | Workload-specific ordered environment-file list. Replaces broader defaults when present. | | `.env_files[].file` `*` | string | — | Repository-relative environment file path. Expects a path inside the repository, with no control character or shell metacharacter. | | `.env_files[].provider` | `sops` | — | Decryptor used before staging the file. The supported encrypted provider is sops. | -| `.execution` | object | — | Opt-in durable scheduled execution. Requires a native manual job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume. | +| `.execution` | object | — | Opt-in durable scheduled execution. Requires a native operator-runnable phase-none job with data_effect none. Stores non-secret checkpoints on the host and permits explicit same-release resume. | | `.execution.retention` | string | `168h` | Time from creation during which an unsuccessful execution may be resumed, at most 30d. Active executions remain protected. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.execution.steps` | list | — | Optional ordered steps using this job's image and entrypoint. Omit to execute the job command as one step. At most 32 steps. | | `.execution.steps[].command` `*` | list | — | Argument vector passed to the job image's entrypoint. No shell evaluation is performed. | @@ -69,8 +70,8 @@ cannot drift from what `ob validate` accepts. | `.image.pull` | `always` · `missing` · `never` | `missing` | When to fetch the image from the registry: missing fetches only what the host does not already hold, always fetches every release, never fetches at all and fails on a missing image. | | `.image.reference` | string | — | Complete container image reference, optionally tagged or digest-pinned. Expects a registry reference such as nginx:1.27 or ghcr.io/acme/app@sha256:…. | | `.init` | boolean | — | Run a minimal init process as PID 1 inside the container. | -| `.inputs` | map | — | Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob schedule run may override them. | -| `.inputs..default` `*` | string | — | Value used by a timer firing and by a manual run that does not override it. Must satisfy the input's own constraint. | +| `.inputs` | map | — | Declared parameters of a scheduled job, exposed as environment variables. Names are upper-case identifiers; each declares exactly one of enum or pattern and a default. A timer firing uses the defaults; ob job run may override them. | +| `.inputs..default` `*` | string | — | Value used by a timer firing and by an operator run that does not override it. Must satisfy the input's own constraint. | | `.inputs..description` | string | — | What the input controls. | | `.inputs..enum` | list | — | Accepted values. | | `.inputs..pattern` | string | — | Regular expression the whole value must match. | @@ -82,6 +83,7 @@ cannot drift from what `ob validate` accepts. | `.needs[].condition` | `started` · `healthy` · `completed` | — | Prerequisite condition: started, healthy, or completed. | | `.needs[].env` | map | — | Maps application environment-variable names to service connection parts such as host, port, user, password, database, or url. | | `.needs[].name` | string | — | Name of a workload or supporting service that must start first. Expects lower-case letters, digits and hyphens, starting with a letter, at most 40 characters. | +| `.operator_run` | `allowed` · `disabled` | — | Whether an operator may invoke this job outside deployment: allowed or disabled. Defaults to allowed for phase none and disabled otherwise. | | `.persistence` | object | — | Declares whether this workload holds data that must outlive releases. | | `.persistence.mode` | `durable` · `ephemeral` · `external` | `durable` | Data lifetime: durable, ephemeral, or external. | | `.port` | integer | — | Container port used with domain shorthand and as the default HTTP health port. | @@ -104,7 +106,7 @@ cannot drift from what `ob validate` accepts. | `.routes[].protocol` | `http` · `tcp` | `http` | Routing protocol: http, tcp, or udp. | | `.routes[].scheme` | `http` · `https` · `h2c` | `http` | Backend connection scheme: http, https, h2c, tcp, or udp. | | `.routes[].tls` | `terminate` · `passthrough` · `none` | `terminate` | TLS handling: terminate, passthrough, or none. | -| `.schedule` | object | — | Host-resident recurring schedule and run policy for a job. | +| `.schedule` | object | — | Host-resident recurring schedule and run policy for a job, independent of its deployment phase and operator-run policy. | | `.schedule.catch_up` | boolean | `true` | Run once after the host returns if an elapsed schedule was missed while it was offline. | | `.schedule.cron` | string | — | Five-field cron schedule translated to a host timer. Expects five cron fields. | | `.schedule.deploy_lock` | `exclusive` · `pinned` | `exclusive` | Deployment coordination policy: exclusive blocks application operations for the full run; pinned leases the immutable starting release and permits only deployments without data-changing jobs or untyped hooks. | @@ -113,6 +115,7 @@ cannot drift from what `ob validate` accepts. | `.schedule.retry.attempts` | integer | `1` | Total attempts including the first, 1 to 10. | | `.schedule.retry.backoff` | string | `30s` | Sleep before the second attempt; it doubles after each failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.retry.max_backoff` | string | `10m` | Upper bound for the doubling sleep. Expects a duration such as 30s, 5m, 1h30m or 14d. | +| `.schedule.shutdown_grace` | string | `30s` | Time allowed for graceful container shutdown after the run deadline before Onebox forces removal. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timeout` | string | `1h` | Maximum wall time for one scheduled run before systemd terminates it and records failure. Expects a duration such as 30s, 5m, 1h30m or 14d. | | `.schedule.timezone` | string | `UTC` | IANA timezone used to interpret the cron schedule. Expects an IANA zone name such as UTC or Europe/Berlin. | | `.stdin_open` | boolean | — | Keep standard input open for the container. | @@ -124,7 +127,6 @@ cannot drift from what `ob validate` accepts. | `.volumes[].name` | string | — | Stable logical name of a Onebox-managed volume. Expects lower-case letters, digits and hyphens, starting with a letter, at most 40 characters. | | `.volumes[].path` | string | — | Absolute container path where the volume or bind mount is attached. Expects an absolute path with no control character or shell metacharacter. | | `.volumes[].source` | string | — | Bind mount source. An absolute path is external host state that outlives releases. A dot-prefixed repository path is read-only release content, kept for as long as a container still mounts it. Expects an absolute host path or a dot-prefixed path inside the repository, with no colon, control character or shell metacharacter. | -| `.when` | `pre_release` · `post_release` · `manual` | `manual` | When a job runs: manual, pre_release, or post_release. | | `.working_dir` | string | — | Absolute working directory for the container process. Expects an absolute path with no control character or shell metacharacter. | `*` marks a field that is required within its own object. diff --git a/site/src/content/docs/reference/policies.mdx b/site/src/content/docs/reference/policies.mdx index ee335c56..444eb6a4 100644 --- a/site/src/content/docs/reference/policies.mdx +++ b/site/src/content/docs/reference/policies.mdx @@ -64,7 +64,7 @@ for a deploy, the job name for a job run. A deploy mints the release it names, while a job runs inside the release already serving — which every job planned against that release shares — so the job is what identifies it. -A manual job carrying one of those effects asks whether or not approval policy +A operator job carrying one of those effects asks whether or not approval policy is enabled. Policy decides only whether a deploy that is not critical asks at all. @@ -103,9 +103,9 @@ redacted. | Class | JSON | NDJSON | Commands | | --- | --- | --- | --- | -| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob execution list` · `ob execution inspect` · `ob init` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schedule history` · `ob schedule list` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | -| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob execution resume` · `ob execution abandon` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob schedule pause` · `ob schedule resume` · `ob schedule run` · `ob secrets push` · `ob service apply` | -| Operator passthrough | finite only | yes | `ob logs` · `ob schedule logs` | +| Finite envelope | yes | no | `ob approve` · `ob audit` · `ob backup status` · `ob canonical` · `ob doctor` · `ob eject` · `ob execution list` · `ob execution inspect` · `ob init` · `ob job history` · `ob job plan` · `ob plan` · `ob preflight` · `ob preview` · `ob schedule list` · `ob schema` · `ob secrets list` · `ob status` · `ob validate` · `ob version` | +| Finite operation stream | yes | yes | `ob abort` · `ob backup create` · `ob backup enable` · `ob backup disable` · `ob backup drill` · `ob backup prune` · `ob backup restore` · `ob backup verify` · `ob bootstrap` · `ob deploy` · `ob destroy` · `ob execution resume` · `ob execution abandon` · `ob job run` · `ob proxy apply` · `ob resume` · `ob rollback` · `ob schedule apply` · `ob schedule pause` · `ob schedule resume` · `ob secrets push` · `ob service apply` | +| Operator passthrough | finite only | yes | `ob job logs` · `ob logs` | | Operator passthrough | no | yes | `ob exec` | | Trusted editor | yes, after exit | no | `ob secrets edit` | diff --git a/site/src/content/docs/status/capabilities.mdx b/site/src/content/docs/status/capabilities.mdx index e034eb27..304069ed 100644 --- a/site/src/content/docs/status/capabilities.mdx +++ b/site/src/content/docs/status/capabilities.mdx @@ -29,9 +29,9 @@ This page is the reconciliation. Three states: state-bound plans, image pinning, rendered diffs. - Scheduled jobs as host timers with exact cron translation, bounded retry inside one firing, one run record per activation in the host journal - (`ob schedule history`, `ob schedule logs`, `ob status`), per-outcome + (`ob job history`, `ob job logs`, `ob status`), per-outcome notifications, and declared inputs for operator-initiated runs - (`ob schedule run`). + (`ob job run`). - Opt-in durable job executions with named linear steps, declared string outputs, per-step retries, and host-local checkpoints. `ob execution list`, `inspect`, `resume`, and `abandon` expose recovery state. Resume requires the original