From 7f999b94863768c122481039c9c66c8a28c7b7e9 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 13 Sep 2026 22:48:32 -0700 Subject: [PATCH] refactor(backup): bundle wal-g with postgres image --- .github/workflows/postgres-image.yml | 2 + build/package/postgres.Dockerfile | 27 ++- cmd/ob/backup.go | 2 +- .../2026-09-13-core-engine-boundary-review.md | 176 ++++++++++++++ docs/product.md | 4 + internal/app/backup_artifacts.go | 3 +- internal/app/backup_walg.go | 96 ++------ internal/app/backup_walg_test.go | 8 +- internal/app/names.go | 42 ++-- internal/app/services.go | 4 +- internal/engine/backup_image_test.go | 31 ++- internal/engine/backup_postgres.go | 228 +++--------------- internal/engine/backup_restore.go | 8 +- internal/engine/backup_schedule.go | 6 +- internal/engine/backup_trust_store_test.go | 31 +-- internal/engine/service_apply.go | 8 +- internal/onebox/backup_enable.go | 6 +- .../docs/guides/back-up-a-database.mdx | 20 +- site/src/content/docs/reference/cli.mdx | 2 +- site/src/content/docs/status/capabilities.mdx | 4 + 20 files changed, 351 insertions(+), 357 deletions(-) create mode 100644 docs/plans/2026-09-13-core-engine-boundary-review.md diff --git a/.github/workflows/postgres-image.yml b/.github/workflows/postgres-image.yml index 57cefcc4..c672cd96 100644 --- a/.github/workflows/postgres-image.yml +++ b/.github/workflows/postgres-image.yml @@ -51,6 +51,7 @@ jobs: fi sleep 1 done + docker run --rm onebox-postgres:test wal-g --version docker exec "$container" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \ -c 'CREATE EXTENSION vector' \ -c 'CREATE EXTENSION vectorscale' \ @@ -153,6 +154,7 @@ jobs: -c 'CREATE TABLE onebox_hypopg(id bigint)' \ -c "SELECT indexrelid IS NOT NULL FROM hypopg_create_index('CREATE INDEX ON onebox_hypopg(id)')" docker exec "$container" pg_repack --version + docker exec "$container" wal-g --version docker rm -f "$container" trap - EXIT done diff --git a/build/package/postgres.Dockerfile b/build/package/postgres.Dockerfile index ad32378d..ba545ebe 100644 --- a/build/package/postgres.Dockerfile +++ b/build/package/postgres.Dockerfile @@ -2,6 +2,29 @@ ARG PG_MAJOR=18 ARG DEBIAN_CODENAME=trixie + +FROM debian:trixie-slim AS walg-build + +# The official release has separate Linux binaries. This mapping is build-time +# only: Buildx selects TARGETARCH and the resulting multi-arch image manifest +# selects the matching image at the host. ob neither detects an architecture +# nor transfers a WAL-G binary. +ARG TARGETARCH +ARG WALG_VERSION=v3.0.8 +ARG WALG_AMD64_SHA256=f30544c5ce93cf83b87578e3c4a2e9c0e0ffc3d160ef89ecddaf75f397d98deb +ARG WALG_ARM64_SHA256=794d1a81f0c27825a1603bd39c0f2cf5dd8bed7cc36b598ca05d8d963c3d5fcf +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + case "${TARGETARCH}" in \ + amd64) walg_asset="wal-g-pg-22.04-amd64"; walg_sha="${WALG_AMD64_SHA256}" ;; \ + arm64) walg_asset="wal-g-pg-22.04-aarch64"; walg_sha="${WALG_ARM64_SHA256}" ;; \ + *) echo "unsupported WAL-G architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + install -d /out && \ + curl -fsSL "https://github.com/wal-g/wal-g/releases/download/${WALG_VERSION}/${walg_asset}" -o /out/wal-g && \ + echo "${walg_sha} /out/wal-g" | sha256sum -c - && \ + chmod 0755 /out/wal-g + FROM postgres:${PG_MAJOR}-${DEBIAN_CODENAME} ARG PG_MAJOR @@ -23,6 +46,8 @@ LABEL org.opencontainers.image.source="https://github.com/labstack/onebox" \ org.opencontainers.image.description="PostgreSQL for Onebox-managed applications" \ org.opencontainers.image.licenses="PostgreSQL AND GPL-2.0-or-later" +COPY --from=walg-build /out/wal-g /usr/local/bin/wal-g + # The full commit is immutable. BuildKit checks out that exact source instead # of trusting a movable release tag, and the final image contains no compiler. ADD https://github.com/pgvector/pgvector.git#${PGVECTOR_COMMIT} /tmp/pgvector @@ -77,7 +102,7 @@ RUN apt-get update && \ echo "4e204f7b0aa175af0a3b38c3bc56852954adf0110e25babe94eab6e35eeef114 /usr/share/doc/pgvectorscale/NOTICE" | sha256sum -c - && \ cd / && \ rm -rf /tmp/pgvector /tmp/pgvectorscale-package "/tmp/${pgvectorscale_archive}" && \ - apt-get remove -y build-essential ca-certificates curl postgresql-server-dev-${PG_MAJOR} unzip && \ + apt-get remove -y build-essential curl postgresql-server-dev-${PG_MAJOR} unzip && \ apt-get autoremove -y && \ apt-mark unhold locales && \ rm -rf /var/lib/apt/lists/* diff --git a/cmd/ob/backup.go b/cmd/ob/backup.go index 7a6ac6aa..7779b5bd 100644 --- a/cmd/ob/backup.go +++ b/cmd/ob/backup.go @@ -42,7 +42,7 @@ func addBackupCommands(root *cobra.Command, g *globalFlags) { Short: "establish backup — restarts the service archiving and takes the first backup", Long: "Make a declared backup policy real.\n\n" + "The order is forced: the credentials are checked, the image is pinned by\n" + - "registry digest, the verified wal-g binary is staged on the host, and only\n" + + "registry digest, the generated credential adapter is staged on the host, and only\n" + "then does the server restart with archiving on.\n\n" + "The restart is a real restart of the database. It is not complete until the\n" + "first base backup exists, because WAL archiving with nothing to replay onto\n" + diff --git a/docs/plans/2026-09-13-core-engine-boundary-review.md b/docs/plans/2026-09-13-core-engine-boundary-review.md new file mode 100644 index 00000000..8f735efe --- /dev/null +++ b/docs/plans/2026-09-13-core-engine-boundary-review.md @@ -0,0 +1,176 @@ +# Core engine boundary review + +## Decision + +Onebox should be the **transactional operating engine for one declared +application on one host**, not a general host-management agent or a catalogue +of containers it happens to be able to start. + +The core owns a capability only when it can model the desired state, execute a +bounded lifecycle, verify the promised outcome, retain evidence, and run a +tested recovery path. Everything else is either a user-defined workload or a +documented integration. A container being easy to launch is not enough to make +its semantics a Onebox responsibility. + +This retains the product's strongest property: a request is not presented as a +guarantee until the engine has established and checked that guarantee. It is +also the scaling boundary. Adding more integrations does not make the core a +larger pile of special cases unless the integration can pass the same ownership +test. + +## Existing evidence + +The repository already has the correct foundation: + +- the lifecycle engine takes a host lock and fence, journals each phase, and + refuses a stale runner before it can mutate (`internal/engine/deploy.go`); +- plans, approval binding, derived Compose, release retention, and health-gated + convergence make deployment an evidence-backed transaction rather than a + remote shell script (`docs/product.md`); +- Postgres earns a backup contract through an executable archive, base-backup, + verification, restore, and drill path; generic workload volumes are + explicitly not claimed as backed up (`docs/product.md`); +- durable executions have already exposed the cost of extending the host + runtime: a Python helper, systemd units, Docker/Compose inspection, host + files, locks, and crash-state rules (`internal/durable/runner.py`, + `docs/plans/2026-09-06-durable-job-executions.md`). + +The current product statement is therefore a better guide than a feature list: +Onebox owns explicit operational state and resources, while Docker itself, +Linux, and arbitrary application behavior remain user-owned. + +## Independent-review finding: agentless is not stateless + +The correct contract is **no resident Onebox management/control daemon and no +network control port**, not “nothing is installed or persists on the host.” +The shipped system already writes release state, journals, systemd units and +timers, generated runners/notifiers, a durable-execution helper, backup +credentials/wrappers, and run records. Those are legitimate application +operations state; denying their existence creates an artificial constraint on +safe implementation. + +Keep SSH as the control channel and systemd as the supervisor for unattended +work. Replace host-language and generated-script complexity with a finite, +versioned on-demand helper when doing so materially improves correctness. Do +not introduce a persistent management daemon unless a concrete always-on +requirement cannot be met by systemd plus a short-lived helper. A daemon would +add an upgrade protocol, durable schema migration, local authorization, +availability monitoring, and an incident-response obligation without improving +the single-host availability boundary. + +## Ownership test + +A proposed capability is **Managed** only if every answer below is yes. + +| Test | Required evidence | +| --- | --- | +| Intent | A small declarative model describes what must happen without embedding arbitrary shell or vendor configuration. | +| Preconditions | The engine can identify supported hosts, storage, versions, and exclusive ownership before mutation. | +| Execution | The action is bounded, idempotent or safely resumable, fenced from stale actors, and has clear concurrency rules. | +| Verification | A machine-checkable observation proves the promised result, not merely that a command exited zero. | +| Recovery | Restore/rollback is specified, exercised in CI, and does not overwrite the surviving state before the replacement is proven. | +| Lifecycle | Upgrade, credential rotation, retention, removal, and failure reporting are defined. | +| Support matrix | The supported engine, filesystem, container/runtime version, and security model are finite and tested. | + +If any test is no, the capability is **Run** or **External**. Onebox may still +wire its image, mounts, secrets, resource limits, logs, and health check, but +must say that backup/recovery semantics belong to the user and tool vendor. + +## What the core should own + +| Area | Core responsibility | Boundary | +| --- | --- | --- | +| Intent and policy | Parse, validate, canonicalize, plan, approve, and classify risk. | Do not become a general IaC or scripting language. | +| Transaction execution | Locks, fences, journals, resumability, cancellation, and auditable results. | Do not promise rollback for unmodelled external effects. | +| Application runtime | Generate and reconcile the declared Compose runtime, names, networks, release directories, health gates, and release retention. | Do not install or upgrade Docker, Linux, or host networking. | +| Secrets | Resolve declared secrets, stage them privately, rotate them through a defined flow, and avoid logging them. | Do not become a general secret manager or expose unscoped host credentials. | +| Selected service drivers | Own a small set of database/service drivers with a pinned image, migration path, health semantics, backup, restore, and drill. | Unknown services remain Run/External. | +| Scheduled work | Schedule declared jobs, prevent overlap, retain evidence, and coordinate jobs with deployment. | Job business effects and idempotency remain application-owned. | +| Observation | Report the observed state, drift relevant to owned resources, health, backup evidence, and actionable refusal reasons. | Do not imply full host monitoring, SIEM, or universal log management. | + +## What should remain integration-first + +| Need | Onebox role | User/tool role | +| --- | --- | --- | +| Generic workload-volume backup | Initially document a reference deployment and mark it user-owned. Promote only per storage/consistency driver. | Choose the tool, RPO, consistency method, retention, and recovery procedure. | +| Continuous file replication | Run the declared container and expose its health/logs. | Own watcher behavior, versioning, ransomware protection, and restore semantics. | +| Host storage snapshots | Detect a declared supported backend only after a driver exists. | ZFS/Btrfs/LVM layout and replication remain external until tested end-to-end. | +| Monitoring, log shipping, malware scanning | Provide ordinary workload wiring. | Own alert routes, data retention, query semantics, and incident response. | +| Arbitrary installers and host tuning | Execute an explicit bootstrap hook under the normal journal/lock boundary. | Own package lifecycle, distro matrix, CVE response, and rollback. | + +This is not a refusal to support those outcomes. It is a rule against silently +turning an image pull into a support commitment. + +## Backup implications + +Restic is suitable for efficient scheduled file snapshots: it is a command-line +program, not a daemon, and delegates scheduling to systemd/cron. [Restic's +documentation](https://restic.readthedocs.io/en/stable/040_backup.html) also +notes that recurring invocations need overlap control. It is therefore a good +engine for a bounded snapshot driver, but not evidence of continuous protection +or application consistency. + +Storage-native drivers can offer a stronger contract. For example, zrepl takes +periodic ZFS snapshots and incrementally replicates them, with hooks for +quiescing selected applications. Its own documentation frames the interval as +the RPO and requires replication health to be monitored. [zrepl snapshotting](https://zrepl.github.io/configuration/snapshotting.html) +and [continuous-server example](https://zrepl.github.io/quickstart/continuous_server_backup.html) +show both the promise and the prerequisite: this is a ZFS product feature, not +a generic bind-mount feature. + +The appropriate progression is: + +1. Document user-defined backup/replication workloads now, clearly outside the + Onebox backup guarantee. +2. Add a **snapshot-volume** driver only with explicit consistency modes + (`quiesced`, then app-aware hooks), restore into a fresh destination, and a + CI restore drill. +3. Add a **ZFS replication** driver only on a finite ZFS host profile. +4. Keep database log shipping in the database driver; it has different recovery + semantics from a filesystem copy. + +## Host runtime and delivery + +Use OCI images as the standard distribution format for Onebox-owned executable +components, with immutable multi-architecture digests. That is a delivery +choice, not an ownership claim. The image should contain a narrow, versioned +runtime API and be invoked only for operations the corresponding driver owns. + +`onebox-kit` is appropriate for ephemeral helpers. If a resident controller is +actually needed later, name it `onebox-agent` and give it a separate lifecycle, +database migration, health, upgrade, and incident-response contract. Do not +quietly turn a helper image into a daemon. + +An agent that controls Docker is privileged: Docker documents that a user with +daemon access can create containers that alter arbitrary host paths. [Docker +Engine security](https://docs.docker.com/engine/security/) Thus an agent or +worker receiving the Docker socket is a core trusted-computing-base component, +not a normal workload. Prefer generated static mounts for backup workers; grant +the socket only where the operation demonstrably needs host runtime control. + +## Recommendation + +Keep the current transactional engine as the product core and establish the +ownership test as the admission rule for every new service, backup, or host +tool. Do not make generic continuous backup a core promise now. Deliver a +documented user-defined container pattern first; add managed drivers one +storage/application class at a time as their verification and restore evidence +exist. + +Revisit a resident `onebox-agent` only when the roadmap contains at least two +funded, always-on capabilities that cannot be expressed as a systemd-timed +operation plus an ephemeral worker. At that point the agent is not delivery +plumbing: it is a deliberate replacement for part of the engine's execution +model and must be designed and operated as such. + +## Sources + +1. Onebox, [Product direction](../product.md) and repository lifecycle sources + cited above, accessed 2026-09-13. +2. Restic, [Backing up](https://restic.readthedocs.io/en/stable/040_backup.html), + accessed 2026-09-13. +3. zrepl, [Taking snapshots](https://zrepl.github.io/configuration/snapshotting.html) + and [Continuous backup of a server](https://zrepl.github.io/quickstart/continuous_server_backup.html), + accessed 2026-09-13. +4. Docker, [Engine security](https://docs.docker.com/engine/security/), accessed + 2026-09-13. diff --git a/docs/product.md b/docs/product.md index 18a03c79..bcae0456 100644 --- a/docs/product.md +++ b/docs/product.md @@ -55,6 +55,10 @@ unattended full restore drills, and log rotation.** Onebox says so rather than implying otherwise — `ob doctor` reports every durable workload or service that has no executable backup contract, because silence there would read as approval. +PostgreSQL backup is delivered as part of the pinned Onebox PostgreSQL image: +the image owns the compatible WAL-G executable; Onebox owns policy, generated +credential adaptation, scheduling, verification, and recovery orchestration. + The distinction matters more than it looks. A product direction that reads as a capability list is how an operator ends up believing their database is backed up by something that has never taken a backup. diff --git a/internal/app/backup_artifacts.go b/internal/app/backup_artifacts.go index 01e65d71..694c2e3d 100644 --- a/internal/app/backup_artifacts.go +++ b/internal/app/backup_artifacts.go @@ -17,8 +17,7 @@ type BackupEffectiveProjection struct { // None of it ever had a caller, and the design it described no longer exists: // the schedules are systemd units derived from the policy, retention is applied // by the prune command from the same policy, and the provenance that matters is -// the wal-g checksum pinned in this binary and verified before the binary is -// ever placed on a host. +// the digest of the PostgreSQL image that contains WAL-G. // // Drift is now asked of the target directly rather than of a descriptor written // beside it — see VerifyBackupRuntime. A second description of the truth is diff --git a/internal/app/backup_walg.go b/internal/app/backup_walg.go index d7031bea..f10e365d 100644 --- a/internal/app/backup_walg.go +++ b/internal/app/backup_walg.go @@ -15,19 +15,9 @@ import ( // and a copy of a running data directory is the generic live-volume archive the // contract refuses outright. // -// It runs from a verified binary staged on the host and mounted into the stock -// PostgreSQL image, rather than from a PostgreSQL image Onebox builds and -// publishes. That is the whole reason this file is short. wal-g links against -// libc and nothing else, and takes its entire configuration from the -// environment — so there is no image to maintain, no configuration file to -// place, and no second copy of anything to keep in step with the project. -// -// pgBackRest was implemented first and replaced. It is a fine tool, but it -// needs 41 shared libraries, so it cannot be dropped into the official image -// and forces a derived one; and it is configured by a file, which brought the -// file's own problems — an atomically replaced config vanishing from a running -// container, credential names colliding with its option namespace, and a -// restore_command it writes as the absolute path of its own binary. +// It runs from the Onebox PostgreSQL image, where it is versioned, verified and +// published with the server that invokes it. Onebox mounts only a small +// generated adapter for project-specific credential names. // PgDataPath is the data directory the postgres driver runs with. Every wal-g // command that touches the cluster needs it exactly: the driver sets PGDATA to @@ -35,12 +25,16 @@ import ( // initdb, and pointing a backup at the volume root captures the wrong tree. const PgDataPath = "/var/lib/postgresql/data/pgdata" -// WalgMountPath is where the staged binary and its wrapper are mounted inside -// the container, read-only. Outside /usr/local/bin deliberately: the mount must -// not shadow anything the official image ships. +// WalgMountPath is where Onebox's generated adapter is mounted inside the +// container, read-only. It deliberately does not shadow image-owned binaries. const WalgMountPath = "/opt/onebox/backup" -// WalgBinary is the wrapper Onebox stages beside wal-g, and what every caller +// WalgExecutable is bundled into the pinned onebox-postgres image. The +// per-service wrapper remains a generated, read-only mount because it maps the +// project's credential entry names without baking credentials into the image. +const WalgExecutable = "/usr/local/bin/wal-g" + +// WalgBinary is the wrapper Onebox stages beside its configuration, and what every caller // invokes. It exists because wal-g reads its credentials from fixed AWS_* names // while a backup target names its own entries, so something has to bridge the // two — and doing it here keeps the project's vocabulary out of wal-g's and @@ -231,51 +225,13 @@ func RenderWalgWrapper(target BackupTarget) []byte { b.WriteString(" export " + name + "\n") } b.WriteString("fi\n") - b.WriteString("exec " + WalgMountPath + "/wal-g \"$@\"\n") + b.WriteString("exec " + WalgExecutable + " \"$@\"\n") return []byte(b.String()) } -// WalgVersion is the wal-g release Onebox stages, pinned in the binary rather -// than resolved at run time, together with the checksum of each architecture's -// asset. The checksums are the provenance: the binary is verified against these -// before it is ever placed on a host, so a compromised release page cannot -// substitute one. They were taken from the release and confirmed against the -// binary this was validated with. -const WalgVersion = "v3.0.8" - -// walgChecksums maps the target's `uname -m` to the published asset and its -// SHA-256. A host reporting anything else is refused rather than guessed at. -var walgChecksums = map[string]struct{ Asset, SHA256 string }{ - "x86_64": { - Asset: "wal-g-pg-22.04-amd64", - SHA256: "f30544c5ce93cf83b87578e3c4a2e9c0e0ffc3d160ef89ecddaf75f397d98deb", - }, - "aarch64": { - Asset: "wal-g-pg-22.04-aarch64", - SHA256: "794d1a81f0c27825a1603bd39c0f2cf5dd8bed7cc36b598ca05d8d963c3d5fcf", - }, -} - -// WalgAssetFor returns the download name and expected checksum for a target's -// machine architecture. -// -// The assets are built against Ubuntu 22.04 and link against glibc, which is -// what the official Debian-based PostgreSQL images provide. An Alpine variant -// would not run them, which is why the driver's image is not a matter of taste. -func WalgAssetFor(machine string) (asset, sha256 string, err error) { - entry, ok := walgChecksums[normalizeMachine(machine)] - if !ok { - return "", "", fmt.Errorf( - "no verified wal-g build for machine architecture %q; backup supports x86_64 and aarch64", machine) - } - return entry.Asset, entry.SHA256, nil -} - -// WalgDownloadURL is where the pinned asset comes from. Check is by the -// checksum above, not by trusting this location. -func WalgDownloadURL(asset string) string { - return "https://github.com/wal-g/wal-g/releases/download/" + WalgVersion + "/" + asset -} +// BackupAdapterFormat versions the host-mounted wrapper contract. It does not +// version WAL-G: the PostgreSQL image digest owns that executable version. +const BackupAdapterFormat = "v1" // serviceBackup is everything renderService needs to run a service under // backup. It is derived from observed durable lifecycle state rather than @@ -284,7 +240,7 @@ func WalgDownloadURL(asset string) string { // an archive_command pointing at a repository that was never initialised would // take the database down at its next WAL switch. type serviceBackup struct { - RuntimeHostDir string + AdapterHostDir string CredentialFile string ArchiveCommand string ArchiveTimeout string @@ -343,7 +299,7 @@ func (r *Resolved) backupForRender(n Names, serviceName string) (*serviceBackup, environment["ONEBOX_S3_KEY_ENTRY"] = projection.Target.Credentials.AccessKeyEntry environment["ONEBOX_S3_SECRET_ENTRY"] = projection.Target.Credentials.SecretKeyEntry return &serviceBackup{ - RuntimeHostDir: n.BackupRuntimeDir(serviceName), + AdapterHostDir: n.BackupAdapterDir(serviceName), CredentialFile: n.BackupCredentialFile(serviceName, projection.Policy.Target), ArchiveCommand: WalgArchiveCommand(), ArchiveTimeout: fmt.Sprintf("%ds", int(maximumDataLoss.Seconds())), @@ -420,24 +376,6 @@ func (r *Resolved) BackupRepository(serviceName string) (string, error) { return WalgPrefix(projection.Target, r.Spec.Name, serviceName, state.BackupRepositoryGeneration), nil } -// normalizeMachine folds the spellings of one architecture onto a single name. -// -// `uname -m` is not standardised: Linux says aarch64 where Darwin says arm64, -// and amd64 appears for x86_64. The architecture that matters is the one the -// *container* runs, which is Linux — so a Darwin host reporting arm64 still -// needs the Linux aarch64 build, and folding the names is exactly right rather -// than merely convenient. -func normalizeMachine(machine string) string { - switch strings.TrimSpace(strings.ToLower(machine)) { - case "aarch64", "arm64", "armv8l": - return "aarch64" - case "x86_64", "amd64", "x64": - return "x86_64" - default: - return strings.TrimSpace(machine) - } -} - // ValidateWalgCredentials checks decrypted credential material against what the // repository needs, before any of it reaches the target. // diff --git a/internal/app/backup_walg_test.go b/internal/app/backup_walg_test.go index 07e68c65..d12f9f2d 100644 --- a/internal/app/backup_walg_test.go +++ b/internal/app/backup_walg_test.go @@ -165,16 +165,16 @@ func TestTheWrapperPointsWalgAtTheStagedTrustStore(t *testing.T) { func runWrapper(t *testing.T, target BackupTarget, env []string) (stdout, stderr string, code int) { t.Helper() dir := t.TempDir() - // A stub standing in for the staged binary, reporting whether the wrapper - // handed it an encryption key. The wrapper execs wal-g by its absolute - // staged path, so only that one line is redirected; everything the guard + // A stub standing in for the image binary, reporting whether the wrapper + // handed it an encryption key. The wrapper execs WAL-G by its absolute image + // path, so only that one line is redirected; everything the guard // does above it runs verbatim. stub := filepath.Join(dir, "wal-g") body := "#!/bin/sh\nprintf 'libsodium=[%s]\\n' \"${WALG_LIBSODIUM_KEY-}\"\n" if err := os.WriteFile(stub, []byte(body), 0o755); err != nil { t.Fatal(err) } - rendered := strings.Replace(string(RenderWalgWrapper(target)), WalgMountPath+"/wal-g", stub, 1) + rendered := strings.Replace(string(RenderWalgWrapper(target)), WalgExecutable, stub, 1) script := filepath.Join(dir, "wrapper.sh") if err := os.WriteFile(script, []byte(rendered), 0o755); err != nil { t.Fatal(err) diff --git a/internal/app/names.go b/internal/app/names.go index 56d4a7d1..0cf6c312 100644 --- a/internal/app/names.go +++ b/internal/app/names.go @@ -92,9 +92,9 @@ func (n Names) ServiceFile(service string) string { return path.Join(n.ServiceDir(), service+".yaml") } -// BackupRuntimeDir holds what a protected service needs at run time: the -// verified wal-g binary and the generated wrapper that puts its credentials in -// scope. The whole directory is mounted read-only into the container. +// BackupAdapterDir holds generated backup configuration: a credential wrapper +// and optional additional certificate authorities. The whole directory is +// mounted read-only; WAL-G itself belongs to the PostgreSQL image. // // A directory rather than two file mounts, and that is not tidiness. Onebox // replaces generated files atomically, by writing a temporary file and renaming @@ -103,38 +103,24 @@ func (n Names) ServiceFile(service string) string { // disappears from inside the running container: the mount still points at the // inode that was unlinked. Mounting the directory keeps the mount stable. // -// It is keyed by wal-g version, so upgrading the pinned version changes the -// mount path and the container is recreated onto the new binary rather than -// having it swapped underneath a running server. -func (n Names) BackupRuntimeDir(service string) string { - return path.Join(n.AppDir(), "backup", "runtime", service, WalgVersion) -} - -// BackupBinaryFile is the verified wal-g binary on the target. -func (n Names) BackupBinaryFile(service string) string { - return path.Join(n.BackupRuntimeDir(service), "wal-g") +// It is keyed by the adapter format, not WAL-G's version. Image and tool +// versions are selected together by the image digest. +func (n Names) BackupAdapterDir(service string) string { + return path.Join(n.AppDir(), "backup", "runtime", service, BackupAdapterFormat) } // BackupWrapperFile is the generated credential wrapper. It sits beside the -// binary and holds no secret: it names the credential entries and reads their -// values from the environment. +// optional trust store and holds no secret: it names credential entries and +// reads their values from the environment. func (n Names) BackupWrapperFile(service string) string { - return path.Join(n.BackupRuntimeDir(service), "ob-wal-g") + return path.Join(n.BackupAdapterDir(service), "ob-wal-g") } -// BackupTrustStoreFile is the host's certificate authority bundle, copied in -// beside the binary. -// -// wal-g runs inside the driver's image, and the official PostgreSQL images -// carry no trust store: `postgres:18` has no /etc/ssl/certs/ca-certificates.crt -// at all. Since every S3-compatible target is required to be HTTPS, a wal-g -// with nothing to verify against cannot upload anywhere — it fails the -// handshake with "x509: certificate signed by unknown authority" after the -// base backup has already been written, and archiving has already been turned -// on. The trust store therefore travels the same way the binary does, through -// the directory that is already mounted read-only into the container. +// BackupTrustStoreFile is an optional host certificate authority bundle. It is +// mounted only when present, supplementing the public certificate roots in the +// image for private backup endpoints. func (n Names) BackupTrustStoreFile(service string) string { - return path.Join(n.BackupRuntimeDir(service), "ca-certificates.crt") + return path.Join(n.BackupAdapterDir(service), "ca-certificates.crt") } // ServiceSecretFile holds the credential Onebox generates on the target. It is diff --git a/internal/app/services.go b/internal/app/services.go index 630f4813..fdcdf223 100644 --- a/internal/app/services.go +++ b/internal/app/services.go @@ -455,11 +455,11 @@ func (p *Spec) renderService(n Names, name string, s Service, selectedImage stri full := n.ServiceVolume(name, vol) mounts := []string{full + ":" + d.dataPath} if backup != nil { - // The directory, not the files — see BackupRuntimeDir for why + // The directory, not the files — see BackupAdapterDir for why // an atomically replaced file vanishes from a running container. // Read-only, because a container that could rewrite the binary it // archives with could send the archive anywhere. - mounts = append(mounts, backup.RuntimeHostDir+":"+WalgMountPath+":ro") + mounts = append(mounts, backup.AdapterHostDir+":"+WalgMountPath+":ro") } svc["volumes"] = mounts volumes[full] = map[string]any{ diff --git a/internal/engine/backup_image_test.go b/internal/engine/backup_image_test.go index 577e61dd..0ed74e01 100644 --- a/internal/engine/backup_image_test.go +++ b/internal/engine/backup_image_test.go @@ -81,7 +81,7 @@ func TestReEnableRejectsPinFromDifferentRepository(t *testing.T) { case strings.Contains(cmd, "docker pull"): return transport.Result{}, true case strings.Contains(cmd, "RepoDigests"): - return transport.Result{Stdout: managedPin + "\n"}, true + return transport.Result{Stdout: `["` + managedPin + `"]` + "\n"}, true } return transport.Result{Stdout: "absent\n"}, true }} @@ -116,7 +116,7 @@ func TestReEnableDoesNotCreatePinFromDifferentRepository(t *testing.T) { case strings.Contains(cmd, "docker pull"): return transport.Result{}, true case strings.Contains(cmd, "RepoDigests"): - return transport.Result{Stdout: managedPin + "\n"}, true + return transport.Result{Stdout: `["` + managedPin + `"]` + "\n"}, true } return transport.Result{Stdout: "absent\n"}, true }} @@ -151,7 +151,7 @@ func TestADeclaredVersionChangeStillResolvesThroughTheRegistry(t *testing.T) { case strings.Contains(cmd, "docker pull"): return transport.Result{}, true case strings.Contains(cmd, "RepoDigests"): - return transport.Result{Stdout: "ghcr.io/labstack/onebox-postgres@sha256:" + strings.Repeat("b", 64) + "\n"}, true + return transport.Result{Stdout: `["ghcr.io/labstack/onebox-postgres@sha256:` + strings.Repeat("b", 64) + `"]` + "\n"}, true } return transport.Result{Stdout: "absent\n"}, true }} @@ -176,6 +176,31 @@ func TestADeclaredVersionChangeStillResolvesThroughTheRegistry(t *testing.T) { } } +func TestProtectedImageSelectsTheDigestForThePulledRepository(t *testing.T) { + const wanted = "ghcr.io/labstack/onebox-postgres@sha256:16cad38a5d9f5d24b4d83d86def30795d5e4b757fedbf5281172b576dedcd942" + fake := &transport.Fake{Dynamic: func(cmd string) (transport.Result, bool) { + switch { + case strings.Contains(cmd, "docker pull"): + return transport.Result{}, true + case strings.Contains(cmd, "RepoDigests"): + return transport.Result{Stdout: `[` + + `"onebox-postgres@sha256:06cad38a5d9f5d24b4d83d86def30795d5e4b757fedbf5281172b576dedcd941",` + + `"` + wanted + `"]` + "\n"}, true + } + return transport.Result{Stdout: "absent\n"}, true + }} + + got, err := protectedImageTestEngine(fake).ResolveProtectedImage( + context.Background(), "database", "", "", + ) + if err != nil { + t.Fatalf("resolving an image with several repository digests: %v", err) + } + if got != wanted { + t.Fatalf("resolved %q, want %q", got, wanted) + } +} + func protectedImageTestEngine(fake *transport.Fake) *Engine { spec := &app.Spec{ Name: "shop", diff --git a/internal/engine/backup_postgres.go b/internal/engine/backup_postgres.go index 7db2ca24..fff3f4fc 100644 --- a/internal/engine/backup_postgres.go +++ b/internal/engine/backup_postgres.go @@ -2,14 +2,10 @@ package engine import ( "context" - "crypto/sha256" - "encoding/hex" + "encoding/json" + "errors" "fmt" - "io" - "net/http" - "os" "path" - "path/filepath" "regexp" "strconv" "strings" @@ -34,45 +30,18 @@ import ( // produces an ordinary server rather than one archiving to a repository nobody // initialised. -// StageBackupRuntime places the verified wal-g binary and its generated -// wrapper on the target, then makes them readable by the service. -// -// The binary is fetched here — on the machine running `ob` — and uploaded, -// rather than downloaded by the target. That keeps the agentless model intact -// and, more importantly, keeps verification on this side of the trust boundary: -// the checksum is pinned in the Onebox binary, so a host with no outbound -// internet still gets backup, and a compromised release page cannot -// substitute a binary that a target-side `curl | sha256sum` would happily -// accept against a checksum from the same source. +// StageBackupRuntime writes the generated wrapper and trust store the bundled +// image needs. WAL-G itself is bundled into the pinned multi-arch +// onebox-postgres image, so no operator-side download or target upload exists. func (e *Engine) StageBackupRuntime(ctx context.Context, service string, wrapper []byte) error { - machine, err := e.targetMachine(ctx) - if err != nil { - return err - } - asset, expected, err := app.WalgAssetFor(machine) - if err != nil { - return err - } n := e.names() - destination := n.BackupBinaryFile(service) - - present, err := e.fileHasChecksum(ctx, destination, expected) + adapterDir := n.BackupAdapterDir(service) + res, err := e.T.Run(ctx, "mkdir -p "+q(adapterDir)) if err != nil { return err } - if !present { - st := e.ui.Step("backup runtime wal-g "+app.WalgVersion+" ("+machine+")", false) - staged, cleanup, err := fetchVerifiedBinary(ctx, app.WalgDownloadURL(asset), expected) - if err != nil { - st(err) - return err - } - defer cleanup() - if err := e.uploadBackupBinary(ctx, n.BackupRuntimeDir(service), staged, destination); err != nil { - st(err) - return err - } - st(nil) + if res.ExitCode != 0 { + return fmt.Errorf("cannot create backup adapter directory %s: %s", adapterDir, strings.TrimSpace(res.Stderr)) } // The wrapper is passed in rather than rendered here, because enablement @@ -94,7 +63,7 @@ func (e *Engine) StageBackupRuntime(ctx context.Context, service string, wrapper if err := e.stageTrustStore(ctx, service); err != nil { return err } - return e.chmodPath(ctx, n.BackupRuntimeDir(service), "0755") + return e.chmodPath(ctx, n.BackupAdapterDir(service), "0755") } // trustStoreCandidates are the certificate authority bundles a Linux host is @@ -107,19 +76,10 @@ var trustStoreCandidates = []string{ "/etc/ssl/cert.pem", } -// stageTrustStore copies the host's certificate authorities in beside the -// binary, because wal-g runs in the driver's image and that image has none. -// -// `postgres:18` ships two entries under /etc/ssl/certs and no bundle among -// them, so every upload to the HTTPS endpoint an s3-compatible target is -// required to declare fails verification. It failed *late*: the base backup -// completed first, so the error arrived a quarter of an hour in, against a -// server whose archiving was already on. -// -// A host with no bundle is refused here rather than discovered there. The -// alternative is staging nothing, letting the wrapper fall back to the image's -// empty store, and reproducing exactly the failure this exists to prevent — -// only later, and with the database already archiving. +// stageTrustStore optionally copies the host's certificate authorities beside +// the wrapper. The image carries public roots; a host bundle lets a private +// endpoint use its additional trusted roots. When the host has +// no bundle, remove a stale staged copy so the wrapper uses the image default. func (e *Engine) stageTrustStore(ctx context.Context, service string) error { destination := e.names().BackupTrustStoreFile(service) // Copied on the target rather than uploaded from here: the bundle that @@ -140,18 +100,15 @@ func (e *Engine) stageTrustStore(ctx context.Context, service string) error { probe.WriteString(" exit 0\n") probe.WriteString("fi\n") } - probe.WriteString("exit 1\n") + probe.WriteString("rm -f " + q(destination) + "\n") + probe.WriteString("exit 0\n") res, err := e.T.Run(ctx, probe.String()) if err != nil { return err } if res.ExitCode != 0 { - return fmt.Errorf( - "service %s: the target holds no certificate authority bundle at any of %s, "+ - "so wal-g cannot verify the backup endpoint from inside the container; "+ - "install the host's CA certificates (on Debian and Ubuntu: apt-get install ca-certificates)", - service, strings.Join(trustStoreCandidates, ", ")) + return fmt.Errorf("service %s: stage host certificate authorities: %s", service, res.Stderr) } return nil } @@ -311,108 +268,6 @@ func postgresControlSystemIdentifier(output string) string { var databaseSystemIdentifier = regexp.MustCompile(`^[0-9]{1,20}$`) -func (e *Engine) targetMachine(ctx context.Context) (string, error) { - res, err := e.T.Run(ctx, "uname -m") - if err != nil { - return "", err - } - machine := strings.TrimSpace(res.Stdout) - if res.ExitCode != 0 || machine == "" { - return "", fmt.Errorf("cannot determine the target's machine architecture") - } - return machine, nil -} - -// fileHasChecksum reports whether the target already holds exactly the expected -// bytes. Re-uploading 60MB on every enable would be the kind of cost that makes -// people avoid running the command. -func (e *Engine) fileHasChecksum(ctx context.Context, remotePath, expected string) (bool, error) { - res, err := e.T.Run(ctx, "sha256sum "+q(remotePath)+" 2>/dev/null | cut -d' ' -f1") - if err != nil { - return false, err - } - return strings.TrimSpace(res.Stdout) == expected, nil -} - -// fetchVerifiedBinary downloads an asset and refuses it unless it hashes to the -// pinned value. The file is never made executable and never leaves the -// temporary directory until it has matched. -func fetchVerifiedBinary(ctx context.Context, url, expected string) (string, func(), error) { - dir, err := os.MkdirTemp("", "ob-backup-runtime-") - if err != nil { - return "", nil, fmt.Errorf("create staging directory: %w", err) - } - cleanup := func() { os.RemoveAll(dir) } - - request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - cleanup() - return "", nil, err - } - client := &http.Client{Timeout: 10 * time.Minute} - response, err := client.Do(request) - if err != nil { - cleanup() - return "", nil, fmt.Errorf("fetch %s: %w", url, err) - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - cleanup() - return "", nil, fmt.Errorf("fetch %s: %s", url, response.Status) - } - - staged := filepath.Join(dir, "wal-g") - file, err := os.OpenFile(staged, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o600) - if err != nil { - cleanup() - return "", nil, err - } - digest := sha256.New() - if _, err := io.Copy(io.MultiWriter(file, digest), response.Body); err != nil { - file.Close() - cleanup() - return "", nil, fmt.Errorf("download %s: %w", url, err) - } - if err := file.Close(); err != nil { - cleanup() - return "", nil, err - } - if observed := hex.EncodeToString(digest.Sum(nil)); observed != expected { - cleanup() - return "", nil, fmt.Errorf( - "the wal-g download does not match its pinned checksum (expected %s, got %s); refusing to place it on the host", - expected, observed) - } - return staged, cleanup, nil -} - -// uploadBackupBinary moves the verified binary into place. Upload writes a -// directory, so the staged file is placed alone in one and moved across. -func (e *Engine) uploadBackupBinary(ctx context.Context, runtimeDir, staged, destination string) error { - res, err := e.T.Run(ctx, "mkdir -p "+q(runtimeDir)) - if err != nil { - return err - } - if res.ExitCode != 0 { - return fmt.Errorf("cannot create the backup runtime directory: %s", strings.TrimSpace(res.Stderr)) - } - remoteStaging := runtimeDir + "/.staging" - if err := e.T.Upload(ctx, filepath.Dir(staged), remoteStaging); err != nil { - return fmt.Errorf("upload the wal-g binary: %w", err) - } - install := "mv -f " + q(remoteStaging+"/"+filepath.Base(staged)) + " " + q(destination) + - " && chmod 0755 " + q(destination) + - " && rm -rf " + q(remoteStaging) - res, err = e.T.Run(ctx, install) - if err != nil { - return err - } - if res.ExitCode != 0 { - return fmt.Errorf("cannot install the wal-g binary: %s", strings.TrimSpace(res.Stderr)) - } - return nil -} - func (e *Engine) chmodPath(ctx context.Context, target, mode string) error { res, err := e.T.Run(ctx, "chmod "+mode+" "+q(target)) if err != nil { @@ -470,9 +325,9 @@ func (e *Engine) RebindServiceRuntimeStates(states map[string]app.ServiceRuntime // ResolveProtectedImage pins the service image by the digest the host actually // has, after pulling it. // -// WAL-G is mounted beside the PostgreSQL runtime image rather than baked into -// it, but the image is still pinned: the bytes running over a live data -// directory must not change because a tag moved. +// WAL-G is baked into the PostgreSQL runtime image, so the image is pinned: +// the bytes running over a live data directory must not change because a tag +// moved. // recordedPin and recordedReference come from the service's lifecycle record: // the digest it was last bound with, and the reference that produced it. When // the project still declares that same reference and the host still holds those @@ -539,24 +394,33 @@ func (e *Engine) ResolveProtectedImage(ctx context.Context, service, recordedPin st(err) return "", err } - res, err = e.T.Run(ctx, "docker image inspect --format '{{index .RepoDigests 0}}' "+q(resolutionReference)) + res, err = e.T.Run(ctx, "docker image inspect --format '{{json .RepoDigests}}' "+q(resolutionReference)) if err != nil { st(err) return "", err } - pinned := strings.TrimSpace(res.Stdout) - if res.ExitCode != 0 || !containsDigest(pinned) { + var repoDigests []string + if res.ExitCode == 0 { + if err := json.Unmarshal([]byte(strings.TrimSpace(res.Stdout)), &repoDigests); err != nil { + err := errors.New("docker returned invalid repository digest metadata") + st(err) + return "", err + } + } + pinned := "" + for _, candidate := range repoDigests { + if containsDigest(candidate) && sameImageRepository(candidate, resolutionReference) { + pinned = candidate + break + } + } + if pinned == "" { err := fmt.Errorf( "%s has no registry digest on this host; a protected service runs an image pinned by digest, so it must come from a registry rather than a local build", resolutionReference) st(err) return "", err } - if !sameImageRepository(pinned, resolutionReference) { - err := fmt.Errorf("resolved digest %s does not belong to image repository %s", pinned, resolutionReference) - st(err) - return "", err - } st(nil) return pinned, nil } @@ -619,24 +483,6 @@ func (e *Engine) VerifyBackupRuntime(ctx context.Context, service string) ([]str n := e.names() var issues []string - machine, err := e.targetMachine(ctx) - if err != nil { - return nil, err - } - _, expected, err := app.WalgAssetFor(machine) - if err != nil { - return nil, err - } - matches, err := e.fileHasChecksum(ctx, n.BackupBinaryFile(service), expected) - if err != nil { - return nil, err - } - if !matches { - issues = append(issues, fmt.Sprintf( - "the wal-g binary at %s is not the %s build this release pins; re-run `ob service apply` to replace it", - n.BackupBinaryFile(service), app.WalgVersion)) - } - wrappers, err := e.Spec.RenderServiceBackupWrappers(e.Opts.Environment) if err != nil { return nil, err diff --git a/internal/engine/backup_restore.go b/internal/engine/backup_restore.go index 13e049a8..475b1336 100644 --- a/internal/engine/backup_restore.go +++ b/internal/engine/backup_restore.go @@ -242,8 +242,8 @@ func (e *Engine) discardRecoveryStaging(ctx context.Context, container, staging return nil } -// startRecoveryContainer runs the protected image with the staged wal-g mounted -// and the data directory empty, doing nothing. The server is started later, by +// startRecoveryContainer runs the protected image with the staged adapter +// mounted and the data directory empty, doing nothing. The server is started later, by // hand, so recovery configuration is in place before it reads anything. func (e *Engine) startRecoveryContainer(ctx context.Context, container, staging, image string, environment map[string]any, service, credentialTarget string) error { n := e.names() @@ -252,7 +252,7 @@ func (e *Engine) startRecoveryContainer(ctx context.Context, container, staging, "--network", q(n.ServiceNetwork()), "--entrypoint", "sleep", "-v", q(staging + ":/var/lib/postgresql/data"), - "-v", q(n.BackupRuntimeDir(service) + ":" + app.WalgMountPath + ":ro"), + "-v", q(n.BackupAdapterDir(service) + ":" + app.WalgMountPath + ":ro"), // The target-managed PostgreSQL credential is needed only inside the // recovery container. Passing the file by name keeps its value out of this // command, transport logs, recovery evidence, and process arguments. @@ -329,7 +329,7 @@ func (e *Engine) fetchRecoveryBase(ctx context.Context, container, service, targ // requested point, which is the only kind replay can carry forward to it. // // Read from the recovery container rather than the live service: it already has -// the staged wal-g and the repository credentials, and a recovery must not +// the image-owned WAL-G, adapter and repository credentials, and a recovery must not // depend on the database it may be about to replace. func (e *Engine) baseBackupFor(ctx context.Context, container, service, targetTime string) (string, error) { target, err := time.Parse(time.RFC3339, targetTime) diff --git a/internal/engine/backup_schedule.go b/internal/engine/backup_schedule.go index 80d299eb..8116afcd 100644 --- a/internal/engine/backup_schedule.go +++ b/internal/engine/backup_schedule.go @@ -25,9 +25,9 @@ import ( // - the drill schedule verifies the archived WAL forms an unbroken // chain, which is the check a green backup does not imply. // -// They run wal-g directly rather than through `ob`, because there is no `ob` on -// the target — Onebox is agentless, and the only thing it has already placed -// there is the verified binary these units invoke. +// They run WAL-G through the generated wrapper rather than through `ob`, +// because there is no `ob` on the target. The wrapper invokes the image-owned +// executable with the target's credential file. // // That agentlessness is also why the drill schedule verifies rather than // actually restoring. A real drill recovers into a throwaway volume and proves diff --git a/internal/engine/backup_trust_store_test.go b/internal/engine/backup_trust_store_test.go index ed970fa7..e1165b28 100644 --- a/internal/engine/backup_trust_store_test.go +++ b/internal/engine/backup_trust_store_test.go @@ -2,18 +2,14 @@ package engine import ( "context" - "regexp" "strings" "testing" "github.com/labstack/onebox/internal/transport" ) -// wal-g executes inside the driver's image. `postgres:18` carries no -// certificate authorities, so unless the host's bundle travels with the binary -// every upload to the HTTPS endpoint an s3-compatible target must declare -// fails with "certificate signed by unknown authority" — after the base backup -// has been written and archiving is already on. +// The image provides public certificate authorities. When the host has a +// bundle, stage it so private endpoint roots work as they do for Docker. func TestStagingTheRuntimeCopiesTheHostTrustStoreInBesideTheBinary(t *testing.T) { fake := &transport.Fake{} engine := backupLockTestEngine(fake) @@ -37,23 +33,16 @@ func TestStagingTheRuntimeCopiesTheHostTrustStoreInBesideTheBinary(t *testing.T) } } -// A target with no bundle anywhere is refused while the service is still -// exactly as it was. Staging nothing and letting the wrapper fall back to the -// image's empty store reproduces the original failure, only a quarter of an -// hour later and with the database already archiving. -func TestATargetWithNoTrustStoreIsRefusedBeforeArchivingIsTurnedOn(t *testing.T) { - fake := &transport.Fake{Script: []transport.Rule{ - {Match: regexp.MustCompile("ca-certificates|ca-bundle|cert.pem"), Result: transport.Result{ExitCode: 1}}, - }} +func TestStagingTheRuntimeCreatesTheAdapterDirectory(t *testing.T) { + fake := &transport.Fake{} engine := backupLockTestEngine(fake) - err := engine.stageTrustStore(context.Background(), "database") - if err == nil { - t.Fatal("a target with no certificate authorities was accepted") + if err := engine.StageBackupRuntime(context.Background(), "database", []byte("#!/bin/sh\n")); err != nil { + t.Fatalf("staging the backup adapter: %v", err) } - for _, want := range []string{"/etc/ssl/certs/ca-certificates.crt", "ca-certificates"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the error does not tell the operator what to install (%q): %v", want, err) - } + commands := strings.Join(fake.Commands, "\n") + want := engine.names().BackupAdapterDir("database") + if !strings.Contains(commands, "mkdir -p") || !strings.Contains(commands, want) { + t.Errorf("adapter directory %s was not created:\n%s", want, commands) } } diff --git a/internal/engine/service_apply.go b/internal/engine/service_apply.go index f261271a..99dcf491 100644 --- a/internal/engine/service_apply.go +++ b/internal/engine/service_apply.go @@ -96,14 +96,14 @@ func (e *Engine) ServiceApply(ctx context.Context, releaseID string, allowDestru if strings.Contains(src[1], "/releases/") { continue } - // Everything Onebox stages for backup — the verified wal-g - // binary and the generated credential wrapper — is mounted + // Everything Onebox stages for backup — generated configuration + // and its credential wrapper — is mounted // read-only and replaced from the project on every apply. Treating // it as data would make each apply of a protected service demand // --allow-destructive-mounts to detach files Onebox wrote itself, // which teaches operators to pass that flag by reflex — the exact - // habit it exists to prevent. The path is keyed by wal-g version, - // so an upgrade legitimately changes it. + // habit it exists to prevent. The path is keyed by adapter format, + // so a wrapper-contract upgrade legitimately changes it. if strings.HasPrefix(src[1], path.Join(n.AppDir(), "backup")+"/") { continue } diff --git a/internal/onebox/backup_enable.go b/internal/onebox/backup_enable.go index 09093e29..18ff5a21 100644 --- a/internal/onebox/backup_enable.go +++ b/internal/onebox/backup_enable.go @@ -17,7 +17,7 @@ import ( // // The order is forced: check the credentials, pin the image, record the state // that makes rendering produce a protected server, restart under it — which is -// also what places the verified wal-g binary and turns archive_mode on — and +// also what mounts the credential adapter and turns archive_mode on — and // only then take the base backup the recovery window is measured from. // // Re-running it on an already-enabled service re-converges rather than @@ -195,8 +195,8 @@ func executeBackupEnable(ctx context.Context, e *engine.Engine, resolved *app.Re if err := e.RebindServiceRuntimeStates(map[string]app.ServiceRuntimeState{service: runtime}); err != nil { return err } - // ApplyServices stages the verified wal-g binary and the generated wrapper - // before starting anything that mounts them, then restarts the server with + // ApplyServices stages the generated wrapper before starting anything that + // mounts it, then restarts the server with // archive_mode on. if err := e.ApplyServices(ctx); err != nil { failure := fmt.Errorf("service %s could not restart under backup: %w", service, err) diff --git a/site/src/content/docs/guides/back-up-a-database.mdx b/site/src/content/docs/guides/back-up-a-database.mdx index 929ed71d..ba7dc3cb 100644 --- a/site/src/content/docs/guides/back-up-a-database.mdx +++ b/site/src/content/docs/guides/back-up-a-database.mdx @@ -49,20 +49,20 @@ nothing archives. ```console $ ob backup enable database -✓ protected image postgres:18 -✓ backup runtime wal-g v3.0.8 (aarch64) +✓ protected image ghcr.io/labstack/onebox-postgres:18 ✓ service database → backup schedule: ob-backup-shop-production-database-backup at 0 2 * * * → backup schedule: ob-backup-shop-production-database-verify at 0 4 * * * ✓ backup database ``` -That one command pins the image by registry digest, stages a checksum-verified -wal-g onto the host, decrypts and installs the destination credentials, restarts -the server with WAL archiving on, installs the timers, and takes the first base -backup. **It is not finished until that base backup exists** — WAL archiving with -nothing to replay onto recovers nothing, and reporting success there would be -telling you the database is protected at the moment it is not. +That one command pins the image by registry digest. That image contains the +compatible WAL-G executable; Onebox stages only the generated credential adapter +on the host. It decrypts and installs the destination credentials, restarts the +server with WAL archiving on, installs the timers, and takes the first base +backup. **It is not finished until that base backup exists** — WAL archiving +with nothing to replay onto recovers nothing, and reporting success there would +be telling you the database is protected at the moment it is not. The restart is a real restart. Enabling is a maintenance action, not a configuration change. @@ -119,8 +119,8 @@ A repository younger than its window says so rather than implying otherwise: yet`. That is not a fault on a service enabled this morning, but it is not the promise either, and only the report can tell you which one you are looking at. -Status also asks the server whether it is still archiving. A correct wal-g -binary on the host says the tooling is in place; it says nothing about +Status also asks the server whether it is still archiving. A correct WAL-G +binary in the pinned PostgreSQL image says the tooling is in place; it says nothing about `archive_mode` still being on or `archive_command` still being the one Onebox installed. Anything that drifted is printed as a `drift` line above the figures. diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index f32e99f7..53707452 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -292,7 +292,7 @@ Global Flags: Make a declared backup policy real. The order is forced: the credentials are checked, the image is pinned by -registry digest, the verified wal-g binary is staged on the host, and only +registry digest, the generated credential adapter is staged on the host, and only then does the server restart with archiving on. The restart is a real restart of the database. It is not complete until the diff --git a/site/src/content/docs/status/capabilities.mdx b/site/src/content/docs/status/capabilities.mdx index 5313f4b3..e034eb27 100644 --- a/site/src/content/docs/status/capabilities.mdx +++ b/site/src/content/docs/status/capabilities.mdx @@ -114,6 +114,10 @@ the backup reference. Only the `postgres` driver has an executable contract today; every other driver declares `policy_qualified: false` and its backup policy is refused rather than accepted and ignored. +For PostgreSQL, the digest-pinned Onebox image carries the compatible WAL-G +executable. The host receives generated configuration only; no backup binary is +installed by Onebox. + **Restore proof.** `ob backup drill` proves it on demand: it recovers the repository into a throwaway volume, waits for the cluster to promote, and makes it answer a query — the same code a real restore runs, stopped before the last