adopt compose-go jobs and container-spec layering

cmd/compose/compose_test.go

@@ -31,10 +31,9 @@ func TestFilterServices(t *testing.T) {
 				Links: []string{"bar"},
 			},
 			"bar": {
-				Name: "bar",
-				DependsOn: map[string]types.ServiceDependency{
+				Name: "bar", WorkloadSpec: types.WorkloadSpec{DependsOn: map[string]types.ServiceDependency{
 					"zot": {},
-				},
+				}},
 			},
 			"zot": {
 				Name: "zot",

cmd/compose/config.go

@@ -260,7 +260,7 @@ func imagesOnly(project *types.Project) *types.Project {
 	digests := types.Services{}
 	for name, config := range project.Services {
 		service := types.ServiceConfig{
-			Image: config.Image,
+			ContainerSpec: types.ContainerSpec{Image: config.Image},
 		}
 		for _, vol := range config.Volumes {
 			if vol.Type == types.VolumeTypeImage {
@@ -384,7 +384,7 @@ func resolveImageDigests(ctx context.Context, dockerCli command.Cli, model map[s
 		}
 		for _, hook := range preStartHooks(service) {
 			image, _ := hook["image"].(string)
-			config.PreStart = append(config.PreStart, types.ServiceHook{Image: image})
+			config.PreStart = append(config.PreStart, types.PreStartHook{ContainerSpec: types.ContainerSpec{Image: image}})
 		}
 		for _, volume := range imageVolumes(service) {
 			source, _ := volume["source"].(string)

cmd/compose/config_test.go

@@ -139,14 +139,17 @@ func TestImagesOnly(t *testing.T) {
 		Name: "test",
 		Services: types.Services{
 			"test": types.ServiceConfig{
-				Name:    "test",
-				Image:   "docker.io/library/nginx@" + testDigest,
-				Command: types.ShellCommand{"echo", "hello"},
+				Name: "test",
+
 				// hooks can't be overridden element-wise on merge, so the lock must not carry them
-				PreStart: []types.ServiceHook{{Image: "docker.io/library/hookimage@" + testDigest}},
-				Volumes: []types.ServiceVolumeConfig{
-					{Type: types.VolumeTypeImage, Source: "docker.io/library/someimage@" + testDigest, Target: "/data"},
-					{Type: types.VolumeTypeBind, Source: "/host", Target: "/bind"},
+				PreStart: []types.PreStartHook{{ContainerSpec: types.ContainerSpec{Image: "docker.io/library/hookimage@" + testDigest}}}, ContainerSpec: types.ContainerSpec{
+					Image:   "docker.io/library/nginx@" + testDigest,
+					Command: types.ShellCommand{"echo", "hello"},
+
+					Volumes: []types.ServiceVolumeConfig{
+						{Type: types.VolumeTypeImage, Source: "docker.io/library/someimage@" + testDigest, Target: "/data"},
+						{Type: types.VolumeTypeBind, Source: "/host", Target: "/bind"},
+					},
 				},
 			},
 		},
@@ -158,9 +161,11 @@ func TestImagesOnly(t *testing.T) {
 	assert.DeepEqual(t, locked, &types.Project{
 		Services: types.Services{
 			"test": types.ServiceConfig{
-				Image: "docker.io/library/nginx@" + testDigest,
-				Volumes: []types.ServiceVolumeConfig{
-					{Type: types.VolumeTypeImage, Source: "docker.io/library/someimage@" + testDigest, Target: "/data"},
+				ContainerSpec: types.ContainerSpec{
+					Image: "docker.io/library/nginx@" + testDigest,
+					Volumes: []types.ServiceVolumeConfig{
+						{Type: types.VolumeTypeImage, Source: "docker.io/library/someimage@" + testDigest, Target: "/data"},
+					},
 				},
 			},
 		},
@@ -177,8 +182,8 @@ func TestWarnHooksNotLockable(t *testing.T) {
 
 	warnHooksNotLockable(&types.Project{
 		Services: types.Services{
-			"with-hook-image": types.ServiceConfig{PreStart: []types.ServiceHook{{Image: "alpine:latest"}}},
-			"inline-hook":     types.ServiceConfig{PreStart: []types.ServiceHook{{Command: types.ShellCommand{"echo"}}}},
+			"with-hook-image": types.ServiceConfig{PreStart: []types.PreStartHook{{ContainerSpec: types.ContainerSpec{Image: "alpine:latest"}}}},
+			"inline-hook":     types.ServiceConfig{PreStart: []types.PreStartHook{{ContainerSpec: types.ContainerSpec{Command: types.ShellCommand{"echo"}}}}},
 			"without-hook":    types.ServiceConfig{},
 		},
 	})

cmd/compose/options_test.go

@@ -38,17 +38,18 @@ func TestApplyPlatforms_InferFromRuntime(t *testing.T) {
 		return &types.Project{
 			Services: types.Services{
 				"test": {
-					Name:  "test",
-					Image: "foo",
-					Build: &types.BuildConfig{
+					Name: "test", ContainerSpec: types.ContainerSpec{
+						Image: "foo",
+
+						Platform: "alice/32",
+					}, WorkloadSpec: types.WorkloadSpec{Build: &types.BuildConfig{
 						Context: ".",
 						Platforms: []string{
 							"linux/amd64",
 							"linux/arm64",
 							"alice/32",
 						},
-					},
-					Platform: "alice/32",
+					}},
 				},
 			},
 		}
@@ -75,15 +76,13 @@ func TestApplyPlatforms_DockerDefaultPlatform(t *testing.T) {
 			},
 			Services: types.Services{
 				"test": {
-					Name:  "test",
-					Image: "foo",
-					Build: &types.BuildConfig{
+					Name: "test", ContainerSpec: types.ContainerSpec{Image: "foo"}, WorkloadSpec: types.WorkloadSpec{Build: &types.BuildConfig{
 						Context: ".",
 						Platforms: []string{
 							"linux/amd64",
 							"linux/arm64",
 						},
-					},
+					}},
 				},
 			},
 		}
@@ -110,15 +109,13 @@ func TestApplyPlatforms_UnsupportedPlatform(t *testing.T) {
 			},
 			Services: types.Services{
 				"test": {
-					Name:  "test",
-					Image: "foo",
-					Build: &types.BuildConfig{
+					Name: "test", ContainerSpec: types.ContainerSpec{Image: "foo"}, WorkloadSpec: types.WorkloadSpec{Build: &types.BuildConfig{
 						Context: ".",
 						Platforms: []string{
 							"linux/amd64",
 							"linux/arm64",
 						},
-					},
+					}},
 				},
 			},
 		}

cmd/compose/pullOptions_test.go

@@ -27,22 +27,19 @@ func TestApplyPullOptions(t *testing.T) {
 	project := &types.Project{
 		Services: types.Services{
 			"must-build": {
-				Name: "must-build",
+				Name: "must-build", WorkloadSpec:
 				// No image, local build only
-				Build: &types.BuildConfig{
+				types.WorkloadSpec{Build: &types.BuildConfig{
 					Context: ".",
-				},
+				}},
 			},
 			"has-build": {
-				Name:  "has-build",
-				Image: "registry.example.com/myservice",
-				Build: &types.BuildConfig{
+				Name: "has-build", ContainerSpec: types.ContainerSpec{Image: "registry.example.com/myservice"}, WorkloadSpec: types.WorkloadSpec{Build: &types.BuildConfig{
 					Context: ".",
-				},
+				}},
 			},
 			"must-pull": {
-				Name:  "must-pull",
-				Image: "registry.example.com/another-service",
+				Name: "must-pull", ContainerSpec: types.ContainerSpec{Image: "registry.example.com/another-service"},
 			},
 		},
 	}

cmd/compose/run.go

@@ -71,6 +71,11 @@ type runOptions struct {
 }
 
 func (options runOptions) apply(project *types.Project) (*types.Project, error) {
+	project, err := materializeManualJob(project, options.Service)
+	if err != nil {
+		return nil, err
+	}
+
 	if options.noDeps {
 		var err error
 		project, err = project.WithSelectedServices([]string{options.Service}, types.IgnoreDependencies)
@@ -83,7 +88,6 @@ func (options runOptions) apply(project *types.Project) (*types.Project, error)
 	if err != nil {
 		return nil, err
 	}
-
 	target.Tty = !options.noTty
 	target.StdinOpen = options.interactive
 
@@ -272,6 +276,11 @@ func normalizeRunFlags(f *pflag.FlagSet, name string) pflag.NormalizedName {
 // their containers.
 func runProject(ctx context.Context, dockerCli command.Cli, backend api.Compose, p *ProjectOptions, service string) (*types.Project, error) {
 	project, _, err := p.ToProject(ctx, dockerCli, backend, []string{service}, composecli.WithoutEnvironmentResolution)
+	if err != nil && strings.Contains(err.Error(), "no such service") {
+		// the run target may be a job, which the service selector cannot
+		// resolve: reload unselected and let materializeManualJob decide
+		project, _, err = p.ToProject(ctx, dockerCli, backend, nil, composecli.WithoutEnvironmentResolution)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -366,3 +375,31 @@ func runRun(ctx context.Context, backend api.Compose, project *types.Project, op
 	}
 	return err
 }
+
+// materializeManualJob lets run target a job exactly like a service: per the
+// spec, any job can be triggered manually regardless of its automated
+// triggers, unless it explicitly opts out with `triggers.manual: false`.
+// A job is a ContainerSpec+WorkloadSpec — the same layers a service is made
+// of — so it materializes as a service for the one-off machinery: its
+// profile is activated and the project narrowed to its dependencies by
+// WithSelectedJob, then the job joins Services under its own name.
+func materializeManualJob(project *types.Project, name string) (*types.Project, error) {
+	job, ok := project.AllJobs()[name]
+	if !ok {
+		return project, nil
+	}
+	if job.Triggers != nil && job.Triggers.Manual != nil && !*job.Triggers.Manual {
+		return nil, fmt.Errorf("job %q is declared with manual: false, it cannot be run manually", name)
+	}
+	project, err := project.WithSelectedJob(name)
+	if err != nil {
+		return nil, err
+	}
+	project.Services[name] = types.ServiceConfig{
+		Name:          name,
+		Profiles:      job.Profiles,
+		ContainerSpec: job.ContainerSpec,
+		WorkloadSpec:  job.WorkloadSpec,
+	}
+	return project, nil
+}

cmd/compose/run_test.go

@@ -0,0 +1,86 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package compose
+
+import (
+	"testing"
+
+	"github.com/compose-spec/compose-go/v2/types"
+	"gotest.tools/v3/assert"
+)
+
+func TestMaterializeManualJob(t *testing.T) {
+	yes, no := true, false
+	base := func() *types.Project {
+		return &types.Project{
+			Services: types.Services{
+				"db": {Name: "db", ContainerSpec: types.ContainerSpec{Image: "postgres"}},
+			},
+			Jobs: types.Jobs{
+				"migrate": {
+					Name:          "migrate",
+					Triggers:      &types.TriggerConfig{Manual: &yes},
+					ContainerSpec: types.ContainerSpec{Image: "migrator", Command: types.ShellCommand{"migrate"}},
+					WorkloadSpec:  types.WorkloadSpec{DependsOn: types.DependsOnConfig{"db": {Condition: types.ServiceConditionStarted, Required: true}}},
+				},
+				"backup": {
+					Name:          "backup",
+					Triggers:      &types.TriggerConfig{Schedule: []types.ScheduleConfig{{Cron: "0 3 * * *"}}},
+					ContainerSpec: types.ContainerSpec{Image: "backup-tool"},
+				},
+				"rotation": {
+					Name:     "rotation",
+					Triggers: &types.TriggerConfig{Manual: &no, Schedule: []types.ScheduleConfig{{Cron: "0 3 1 * *"}}},
+				},
+			},
+		}
+	}
+
+	t.Run("a service name passes through", func(t *testing.T) {
+		p := base()
+		got, err := materializeManualJob(p, "db")
+		assert.NilError(t, err)
+		assert.Equal(t, got, p)
+	})
+
+	t.Run("a manual job materializes as a service with its spec and deps", func(t *testing.T) {
+		got, err := materializeManualJob(base(), "migrate")
+		assert.NilError(t, err)
+		svc, err := got.GetService("migrate")
+		assert.NilError(t, err)
+		assert.Equal(t, svc.Image, "migrator")
+		assert.DeepEqual(t, []string(svc.Command), []string{"migrate"})
+		_, hasDep := svc.DependsOn["db"]
+		assert.Check(t, hasDep)
+		// the dependency is kept in the narrowed project
+		_, err = got.GetService("db")
+		assert.NilError(t, err)
+	})
+
+	t.Run("a scheduled job without explicit manual opt-out can be run", func(t *testing.T) {
+		got, err := materializeManualJob(base(), "backup")
+		assert.NilError(t, err)
+		svc, err := got.GetService("backup")
+		assert.NilError(t, err)
+		assert.Equal(t, svc.Image, "backup-tool")
+	})
+
+	t.Run("manual: false explicitly forbids manual execution", func(t *testing.T) {
+		_, err := materializeManualJob(base(), "rotation")
+		assert.Error(t, err, `job "rotation" is declared with manual: false, it cannot be run manually`)
+	})
+}

cmd/compose/up.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sort"
 	"strings"
 	"time"
 
@@ -241,6 +242,11 @@ func runUp(
 		return err
 	}
 
+	if err := rejectScheduledJobs(project); err != nil {
+		return err
+	}
+	warnIgnoredJobs(project)
+
 	err := createOptions.Apply(project)
 	if err != nil {
 		return err
@@ -349,3 +355,35 @@ func runUp(
 		},
 	})
 }
+
+// warnIgnoredJobs names the declared jobs up will not act on: manual jobs
+// wait for an explicit `compose run <job>` trigger.
+func warnIgnoredJobs(project *types.Project) {
+	jobs := project.AllJobs()
+	if len(jobs) == 0 {
+		return
+	}
+	names := make([]string, 0, len(jobs))
+	for name := range jobs {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	logrus.Warnf("jobs are not started by up; trigger them with `docker compose run`: %s", strings.Join(names, ", "))
+}
+
+// rejectScheduledJobs refuses to bring a project up when it declares active
+// scheduled jobs: silently not scheduling them would break the user's
+// expectations, unlike manual jobs which simply wait for an explicit trigger.
+func rejectScheduledJobs(project *types.Project) error {
+	names := make([]string, 0, len(project.Jobs))
+	for name, job := range project.Jobs {
+		if job.Triggers != nil && len(job.Triggers.Schedule) > 0 {
+			names = append(names, name)
+		}
+	}
+	if len(names) == 0 {
+		return nil
+	}
+	sort.Strings(names)
+	return fmt.Errorf("scheduled jobs are not supported in this version: %s", strings.Join(names, ", "))
+}