jobs: add support for one-shot and scheduled jobs

cmd/compose/build.go

@@ -183,5 +183,6 @@ func runBuild(ctx context.Context, dockerCli command.Cli, backendOptions *Backen
 	}
 	apiBuildOptions.Attestations = true
 
-	return backend.Build(ctx, project, apiBuildOptions)
+	err = backend.Build(ctx, project, apiBuildOptions)
+	return jobTargetErrOr(ctx, dockerCli, opts.ProjectOptions, services, err)
 }

cmd/compose/compose.go

@@ -175,6 +175,14 @@ func (o *ProjectOptions) WithServices(dockerCli command.Cli, fn ProjectServicesF
 
 		project, metrics, err := o.ToProject(ctx, dockerCli, backend, services, warnUnsupportedAttributes, cli.WithoutEnvironmentResolution)
 		if err != nil {
+			// a service name among services can genuinely be a declared job:
+			// every WithServices caller has nothing to act on for it, so
+			// report that clearly instead of the raw "no such service" below
+			// — same translation projectOrName already centralizes for its
+			// own callers (start/stop/down/...).
+			if jobErr, replaced := jobTargetErr(ctx, dockerCli, o, services, err); replaced {
+				return jobErr
+			}
 			return err
 		}
 

cmd/compose/create.go

@@ -31,7 +31,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
-	"github.com/docker/compose/v5/cmd/display"
 	"github.com/docker/compose/v5/pkg/api"
 	"github.com/docker/compose/v5/pkg/compose"
 )
@@ -72,18 +71,9 @@ func createCommand(p *ProjectOptions, dockerCli command.Cli, backendOptions *Bac
 			}
 			return nil
 		}),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			err := p.WithServices(dockerCli, func(ctx context.Context, project *types.Project, services []string) error {
-				return runCreate(ctx, dockerCli, backendOptions, opts, buildOpts, project, services)
-			})(cmd, args)
-			if jobErr, replaced := jobTargetErr(cmd.Context(), dockerCli, p, args, err); replaced {
-				if display.Mode == display.ModeJSON {
-					return makeJSONError(jobErr)
-				}
-				return jobErr
-			}
-			return err
-		},
+		RunE: p.WithServices(dockerCli, func(ctx context.Context, project *types.Project, services []string) error {
+			return runCreate(ctx, dockerCli, backendOptions, opts, buildOpts, project, services)
+		}),
 		ValidArgsFunction: completeServiceNames(dockerCli, p),
 	}
 	flags := cmd.Flags()
@@ -108,11 +98,6 @@ func createCommand(p *ProjectOptions, dockerCli command.Cli, backendOptions *Bac
 }
 
 func runCreate(ctx context.Context, dockerCli command.Cli, backendOptions *BackendOptions, createOpts createOptions, buildOpts buildOptions, project *types.Project, services []string) error {
-	// same contract as up: an active scheduled job is refused before any
-	// resource is created — silently not scheduling would break expectations
-	if err := rejectScheduledJobs(project); err != nil {
-		return err
-	}
 	if err := createOpts.Apply(project); err != nil {
 		return err
 	}

cmd/compose/pull.go

@@ -106,7 +106,7 @@ func runPull(ctx context.Context, dockerCli command.Cli, backendOptions *Backend
 
 	project, _, err := opts.ToProject(ctx, dockerCli, backend, services, warnUnsupportedAttributes, cli.WithoutEnvironmentResolution)
 	if err != nil {
-		return err
+		return jobTargetErrOr(ctx, dockerCli, opts.ProjectOptions, services, err)
 	}
 
 	project, err = opts.apply(project, services)

cmd/compose/push.go

@@ -64,7 +64,7 @@ func runPush(ctx context.Context, dockerCli command.Cli, backendOptions *Backend
 
 	project, _, err := opts.ToProject(ctx, dockerCli, backend, services, warnUnsupportedAttributes)
 	if err != nil {
-		return err
+		return jobTargetErrOr(ctx, dockerCli, opts.ProjectOptions, services, err)
 	}
 
 	if !opts.IncludeDeps {

cmd/compose/run.go

@@ -365,6 +365,17 @@ func jobTargetErr(ctx context.Context, dockerCli command.Cli, p *ProjectOptions,
 	return err, false
 }
 
+// jobTargetErrOr applies jobTargetErr's translation to err when it matches,
+// and returns err unchanged otherwise — the wrap every caller loading a
+// project outside WithServices/projectOrName's own centralized handling
+// (build, pull, push) needs around its own selection error.
+func jobTargetErrOr(ctx context.Context, dockerCli command.Cli, p *ProjectOptions, names []string, err error) error {
+	if jobErr, replaced := jobTargetErr(ctx, dockerCli, p, names, err); replaced {
+		return jobErr
+	}
+	return err
+}
+
 func runRun(ctx context.Context, backend api.Compose, project *types.Project, options runOptions, createOpts createOptions, buildOpts buildOptions, dockerCli command.Cli) error {
 	project, err := options.apply(project)
 	if err != nil {
@@ -435,6 +446,22 @@ func runRun(ctx context.Context, backend api.Compose, project *types.Project, op
 		Index:             0,
 	}
 
+	if _, ok := project.AllJobs()[options.Service]; ok {
+		exitCode, err := backend.RunJob(ctx, project, options.Service, runOpts)
+		// An engine without the jobs API runs the manual job like a one-off
+		// service: fall through, the job is already materialized as one.
+		if !errors.Is(err, api.ErrJobsNotSupported) {
+			if exitCode != 0 {
+				errMsg := ""
+				if err != nil {
+					errMsg = err.Error()
+				}
+				return cli.StatusError{StatusCode: exitCode, Status: errMsg, Cause: err}
+			}
+			return err
+		}
+	}
+
 	for name, service := range project.Services {
 		if name == options.Service {
 			service.StdinOpen = options.interactive
@@ -472,8 +499,8 @@ func materializeManualJob(project *types.Project, name string) (*types.Project,
 	if !ok {
 		return project, nil
 	}
-	if job.Triggers != nil && job.Triggers.Manual != nil && !*job.Triggers.Manual {
-		return nil, fmt.Errorf("job %q is declared with manual: false, it cannot be run manually", name)
+	if compose.ManualTriggerDisabled(job) {
+		return nil, compose.ManualTriggerDisabledErr(name)
 	}
 	project, err := project.WithSelectedJob(name)
 	if err != nil {
@@ -488,7 +515,7 @@ func materializeManualJob(project *types.Project, name string) (*types.Project,
 	if err := materializeJobClosure(project, jobs, job, map[string]bool{name: true}); err != nil {
 		return nil, err
 	}
-	project.Services[name] = jobAsService(project, name, job)
+	project.Services[name] = compose.JobAsService(project, name, job)
 	return project, nil
 }
 
@@ -509,39 +536,13 @@ func materializeJobClosure(project *types.Project, jobs types.Jobs, job types.Jo
 		if !isJob {
 			continue
 		}
-		if depJob.Triggers != nil && depJob.Triggers.Manual != nil && !*depJob.Triggers.Manual {
+		if compose.ManualTriggerDisabled(depJob) {
 			return fmt.Errorf("job %q is declared with manual: false, it cannot be triggered even as a dependency of another job", dep)
 		}
 		if err := materializeJobClosure(project, jobs, depJob, seen); err != nil {
 			return err
 		}
-		project.Services[dep] = jobAsService(project, dep, depJob)
+		project.Services[dep] = compose.JobAsService(project, dep, depJob)
 	}
 	return nil
 }
-
-// jobAsService materializes a job as a service for the one-off machinery: a
-// job is a ContainerSpec+WorkloadSpec, the same layers a service is made of.
-// It carries the standard custom labels the loader stamps on every service —
-// materialization happens after loading, so without them the containers
-// created for a dependency job would be invisible to every label-driven
-// path: start would silently skip them, ps/down would not see them, and the
-// dependency wait would report the job as a missing dependency.
-func jobAsService(project *types.Project, name string, job types.JobConfig) types.ServiceConfig {
-	svc := types.ServiceConfig{
-		Name:          name,
-		Profiles:      job.Profiles,
-		Extensions:    job.Extensions,
-		ContainerSpec: job.ContainerSpec,
-		WorkloadSpec:  job.WorkloadSpec,
-	}
-	svc.CustomLabels = types.Labels{
-		api.ProjectLabel:     project.Name,
-		api.ServiceLabel:     name,
-		api.VersionLabel:     api.ComposeVersion,
-		api.WorkingDirLabel:  project.WorkingDir,
-		api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","),
-		api.OneoffLabel:      "False",
-	}
-	return svc
-}

cmd/compose/run_test.go

@@ -17,13 +17,20 @@
 package compose
 
 import (
+	"context"
 	"errors"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/compose-spec/compose-go/v2/types"
+	"github.com/docker/cli/cli"
+	cliopts "github.com/docker/cli/opts"
+	"go.uber.org/mock/gomock"
 	"gotest.tools/v3/assert"
+
+	"github.com/docker/compose/v5/pkg/api"
+	"github.com/docker/compose/v5/pkg/mocks"
 )
 
 func TestMaterializeManualJob(t *testing.T) {
@@ -209,3 +216,58 @@ func TestJobTargetErr(t *testing.T) {
 		assert.Equal(t, err, original)
 	})
 }
+
+// run targets a job through the engine's jobs API when it has one, and
+// degrades to a one-off service container, exactly like a service run, when
+// it does not.
+func TestRunRunJob(t *testing.T) {
+	yes := true
+	newProject := func() *types.Project {
+		return &types.Project{
+			Name:     "prj",
+			Services: types.Services{},
+			Jobs: types.Jobs{"migrate": {
+				Name:          "migrate",
+				Triggers:      &types.TriggerConfig{Manual: &yes},
+				ContainerSpec: types.ContainerSpec{Image: "migrator"},
+			}},
+		}
+	}
+	run := func(t *testing.T, ctrl *gomock.Controller, backend api.Compose) error {
+		t.Helper()
+		return runRun(t.Context(), backend, newProject(),
+			runOptions{composeOptions: &composeOptions{ProjectOptions: &ProjectOptions{}}, Service: "migrate", noTty: true, capAdd: cliopts.NewListOpts(nil), capDrop: cliopts.NewListOpts(nil)},
+			createOptions{}, buildOptions{ProjectOptions: &ProjectOptions{}}, mocks.NewMockCli(ctrl))
+	}
+
+	t.Run("an engine with the jobs API runs the job through it", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		backend := mocks.NewMockCompose(ctrl)
+		backend.EXPECT().RunJob(gomock.Any(), gomock.Any(), "migrate", gomock.Any()).Return(0, nil)
+		assert.NilError(t, run(t, ctrl, backend))
+	})
+
+	t.Run("a failing job reports its exit code", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		backend := mocks.NewMockCompose(ctrl)
+		backend.EXPECT().RunJob(gomock.Any(), gomock.Any(), "migrate", gomock.Any()).Return(3, nil)
+		err := run(t, ctrl, backend)
+		var statusErr cli.StatusError
+		assert.Assert(t, errors.As(err, &statusErr), err)
+		assert.Equal(t, statusErr.StatusCode, 3)
+	})
+
+	t.Run("an engine without the jobs API runs the job like a one-off service", func(t *testing.T) {
+		ctrl := gomock.NewController(t)
+		backend := mocks.NewMockCompose(ctrl)
+		backend.EXPECT().RunJob(gomock.Any(), gomock.Any(), "migrate", gomock.Any()).Return(0, api.ErrJobsNotSupported)
+		backend.EXPECT().RunOneOffContainer(gomock.Any(), gomock.Any(), gomock.Any()).
+			DoAndReturn(func(_ context.Context, project *types.Project, opts api.RunOptions) (int, error) {
+				assert.Equal(t, opts.Service, "migrate")
+				_, materialized := project.Services["migrate"]
+				assert.Assert(t, materialized, "the job must be materialized as a service")
+				return 0, nil
+			})
+		assert.NilError(t, run(t, ctrl, backend))
+	})
+}

cmd/compose/start.go

@@ -56,12 +56,6 @@ func runStart(ctx context.Context, dockerCli command.Cli, backendOptions *Backen
 	if err != nil {
 		return err
 	}
-	if project != nil {
-		// with a file, refuse active scheduled jobs like up and create do
-		if err := rejectScheduledJobs(project); err != nil {
-			return err
-		}
-	}
 
 	var timeout time.Duration
 	if opts.waitTimeout > 0 {

cmd/compose/up.go

@@ -242,9 +242,6 @@ func runUp(
 		return err
 	}
 
-	if err := rejectScheduledJobs(project); err != nil {
-		return err
-	}
 	warnIgnoredJobs(project)
 
 	err := createOptions.Apply(project)
@@ -297,7 +294,10 @@ func runUp(
 	}
 
 	if upOptions.noStart {
-		return backend.Create(ctx, project, create)
+		return backend.Up(ctx, project, api.UpOptions{
+			Create: create,
+			Start:  api.StartOptions{Project: project, NoStart: true},
+		})
 	}
 
 	var consumer api.LogConsumer
@@ -356,34 +356,32 @@ func runUp(
 	})
 }
 
-// warnIgnoredJobs names the declared jobs up will not act on: manual jobs
-// wait for an explicit `compose run <job>` trigger.
+// warnIgnoredJobs names the declared manual jobs up will not act on: they
+// wait for an explicit `compose run <job>` trigger. Scheduled jobs are not
+// ignored — they are registered with the engine (see pkg/compose Up).
 func warnIgnoredJobs(project *types.Project) {
-	jobs := project.Jobs
-	if len(jobs) == 0 {
+	names := manualJobNames(project)
+	if len(names) == 0 {
 		return
 	}
-	names := make([]string, 0, len(jobs))
-	for name := range jobs {
-		names = append(names, name)
-	}
-	sort.Strings(names)
 	logrus.Warnf("jobs are not started by up; trigger them with `docker compose run`: %s", strings.Join(names, ", "))
 }
 
-// rejectScheduledJobs refuses to bring a project up when it declares active
-// scheduled jobs: silently not scheduling them would break the user's
-// expectations, unlike manual jobs which simply wait for an explicit trigger.
-func rejectScheduledJobs(project *types.Project) error {
+// manualJobNames returns the sorted names of the project's jobs that up
+// leaves untouched: a job with a schedule is registered with the engine
+// (see pkg/compose Up), so only jobs without one wait for an explicit
+// `compose run <job>` trigger — unless they opt out with `manual: false`.
+func manualJobNames(project *types.Project) []string {
 	names := make([]string, 0, len(project.Jobs))
 	for name, job := range project.Jobs {
-		if job.Triggers != nil && len(job.Triggers.Schedule) > 0 {
-			names = append(names, name)
+		if compose.HasSchedule(job) {
+			continue
 		}
-	}
-	if len(names) == 0 {
-		return nil
+		if compose.ManualTriggerDisabled(job) {
+			continue
+		}
+		names = append(names, name)
 	}
 	sort.Strings(names)
-	return fmt.Errorf("scheduled jobs are not supported in this version: %s", strings.Join(names, ", "))
+	return names
 }

cmd/compose/up_test.go

@@ -180,20 +180,19 @@ services:
 	assert.Assert(t, !strings.Contains(fmt.Sprint(err), "invalid ip address"), fmt.Sprint(err))
 }
 
-func TestRejectScheduledJobs(t *testing.T) {
+func TestManualJobNames(t *testing.T) {
 	yes := true
 	manual := types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}}
 	scheduled := types.JobConfig{Triggers: &types.TriggerConfig{
 		Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}},
 	}}
 
-	assert.NilError(t, rejectScheduledJobs(&types.Project{}))
-	assert.NilError(t, rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"migrate": manual}}))
-	// profile-disabled scheduled jobs don't block up
-	assert.NilError(t, rejectScheduledJobs(&types.Project{DisabledJobs: types.Jobs{"backup": scheduled}}))
-
-	err := rejectScheduledJobs(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": scheduled, "migrate": manual}})
-	assert.Error(t, err, "scheduled jobs are not supported in this version: backup, sync")
+	assert.DeepEqual(t, manualJobNames(&types.Project{}), []string{})
+	assert.DeepEqual(t, manualJobNames(&types.Project{Jobs: types.Jobs{"backup": scheduled}}), []string{})
+	assert.DeepEqual(t,
+		manualJobNames(&types.Project{Jobs: types.Jobs{"backup": scheduled, "sync": manual, "migrate": manual}}),
+		[]string{"migrate", "sync"},
+	)
 }
 
 // warnIgnoredJobs must only name profile-enabled jobs: a job disabled by
@@ -203,9 +202,11 @@ func TestRejectScheduledJobs(t *testing.T) {
 func TestWarnIgnoredJobs(t *testing.T) {
 	hook := logrustest.NewGlobal()
 
+	yes := true
+	manual := types.JobConfig{Triggers: &types.TriggerConfig{Manual: &yes}}
 	warnIgnoredJobs(&types.Project{
-		Jobs:         types.Jobs{"migrate": {}},
-		DisabledJobs: types.Jobs{"backup": {}},
+		Jobs:         types.Jobs{"migrate": manual},
+		DisabledJobs: types.Jobs{"backup": manual},
 	})
 
 	assert.Equal(t, len(hook.AllEntries()), 1)