From 13d26828375627fddc81c593885e340e0b8e3510 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:12:58 +0200 Subject: [PATCH 01/63] fix(repo): match LICENSE to the AGPL-3.0 text on gnu.org --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 0ad25db..be3f7b2 100644 --- a/LICENSE +++ b/LICENSE @@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, From 30cb6b557dcb06fac4a2a9ec65fcf0c05a76097a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:46:32 +0200 Subject: [PATCH 02/63] style: indent version.json with tabs --- .editorconfig | 2 +- scripts/check-upstream.sh | 2 +- src/version.json | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index 66abfd4..56e2f56 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,6 +16,6 @@ space_redirects = true [Dockerfile] indent_style = space -[*.{json,yml,yaml,md}] +[*.{yml,yaml,md}] indent_style = space indent_size = 2 diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 43f4f1f..6ef017c 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -51,7 +51,7 @@ download_amalgamation() { } write_version_json() { - jq -n --arg version "$version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ + jq -n --tab --arg version "$version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ '{$version, $url, $size, $sha3_256, $source_id}' > src/version.json ok "src/version.json pins $BOLD$version$RESET with url, size, SHA3-256 and source id $source_id" } diff --git a/src/version.json b/src/version.json index 244dae8..59bd974 100644 --- a/src/version.json +++ b/src/version.json @@ -1,7 +1,7 @@ { - "version": "3.54.1", - "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", - "size": 2990966, - "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", - "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" + "version": "3.54.1", + "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", + "size": 2990966, + "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", + "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" } From 1bc73826f350ee2a5997e93e9b0386f95b0e928a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:51:15 +0200 Subject: [PATCH 03/63] refactor(image): replace uid and gid ARGs with 65532:65532 --- src/Dockerfile | 10 +--------- src/rootfs.sh | 6 +++--- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index cafd42e..6531736 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -13,10 +13,6 @@ ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 -# Runtime user, nonroot 65532 like in distroless images -ARG SQLITE_UID=65532 -ARG SQLITE_GID=65532 - # ARGs set by the release workflow ARG BUILD_DATE ARG GIT_SHA @@ -26,8 +22,6 @@ FROM ${BASE_IMAGE} AS build ARG SQLITE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG SQLITE_UID -ARG SQLITE_GID # curl, jq and openssl are only used during the build stage, so they are not pinned to a specific version # hadolint ignore=DL3018 @@ -86,8 +80,6 @@ ARG SQLITE_VERSION ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG SQLITE_UID -ARG SQLITE_GID ARG BUILD_DATE ARG GIT_SHA @@ -118,7 +110,7 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ SQLITE_PAGE_SIZE=4096 WORKDIR /var/lib/sqlite -USER ${SQLITE_UID}:${SQLITE_GID} +USER 65532:65532 STOPSIGNAL SIGTERM ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] diff --git a/src/rootfs.sh b/src/rootfs.sh index 823fe14..f82b6c5 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -31,12 +31,12 @@ install_binaries() { create_user() { install -d /rootfs/etc - echo "nonroot:x:$SQLITE_UID:$SQLITE_GID::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd - echo "nonroot:x:$SQLITE_GID:" > /rootfs/etc/group + echo "nonroot:x:65532:65532::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd + echo "nonroot:x:65532:" > /rootfs/etc/group } create_directories() { - install -d -m 0700 -o "$SQLITE_UID" -g "$SQLITE_GID" /rootfs/var/lib/sqlite + install -d -m 0700 -o 65532 -g 65532 /rootfs/var/lib/sqlite install -d -m 1777 /rootfs/tmp } From 1387e087947b9e155ee73ed240052192f31f2ec2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:28:18 +0200 Subject: [PATCH 04/63] chore(image): name the final stage --- src/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dockerfile b/src/Dockerfile index 6531736..22fdbf1 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -74,7 +74,7 @@ COPY rootfs.sh /src/ RUN sh /src/rootfs.sh -FROM scratch +FROM scratch AS final ARG SQLITE_VERSION ARG IMAGE_VERSION From 7c8cc1ec2d5373f1299fdefda351106a15150dcb Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:35:58 +0200 Subject: [PATCH 05/63] feat(release): make builds reproducible - the same commit builds the same image digest (the same image, byte for byte) - the created date is now the commit date --- .github/workflows/release.yaml | 15 +++++++++++---- src/Dockerfile | 7 ++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1168a8d..8d60cd6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,7 +25,6 @@ jobs: image: ${{ steps.version.outputs.image }} draft: ${{ steps.version.outputs.draft }} tags: ${{ steps.version.outputs.tags }} - created: ${{ steps.version.outputs.created }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: @@ -55,7 +54,6 @@ jobs: echo "image=$image" echo "draft=$draft" echo "tags=$release $sqlite ${sqlite%.*} ${sqlite%%.*} latest" - echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" } >> "$GITHUB_OUTPUT" build: @@ -84,6 +82,14 @@ jobs: with: persist-credentials: false + - name: Read the commit date + id: commit + run: | + { + echo "epoch=$(git log -1 --format=%ct)" + echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_OUTPUT" + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 if: matrix.qemu @@ -108,9 +114,10 @@ jobs: tags: | ghcr.io/${{ github.repository }} docker.io/${{ github.repository }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true + outputs: type=image,push-by-digest=true,name-canonical=true,push=true,rewrite-timestamp=true build-args: | - BUILD_DATE=${{ needs.version.outputs.created }} + SOURCE_DATE_EPOCH=${{ steps.commit.outputs.epoch }} + CREATED=${{ steps.commit.outputs.created }} GIT_SHA=${{ github.sha }} provenance: mode=max sbom: true diff --git a/src/Dockerfile b/src/Dockerfile index 22fdbf1..1c97c86 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -14,7 +14,8 @@ ARG TINI_VERSION=0.19.0-r3 ARG BUSYBOX_VERSION=1.37.0-r31 # ARGs set by the release workflow -ARG BUILD_DATE +ARG SOURCE_DATE_EPOCH +ARG CREATED ARG GIT_SHA FROM ${BASE_IMAGE} AS build @@ -80,7 +81,7 @@ ARG SQLITE_VERSION ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG BUILD_DATE +ARG CREATED ARG GIT_SHA COPY --from=build /rootfs/ / @@ -88,7 +89,7 @@ COPY --from=build /rootfs/ / LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.description="Minimal, production-ready, hardened and secure SQLite Docker image." \ org.opencontainers.image.version="${SQLITE_VERSION}-v${IMAGE_VERSION}" \ - org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.revision="${GIT_SHA}" \ org.opencontainers.image.licenses="blessing AND MIT AND GPL-2.0-only AND AGPL-3.0-or-later" \ org.opencontainers.image.authors="peakimages" \ From 102ac642de4698fe213cec09d87c860d43dd13a1 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:53:08 +0200 Subject: [PATCH 06/63] feat(image): add the shell options sqlite.org recommends - .recover, .dbinfo and the sqlite_dbpage and sqlite_stmt virtual tables --- scripts/test.sh | 20 +++++++++++++++++++- src/Dockerfile | 10 ++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 9868347..01b7e88 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -30,6 +30,7 @@ main() { check_rejected_settings heading "Data" seed_database + recover_database backup_database restore_database check_stop @@ -128,8 +129,25 @@ seed_database() { [ "$(sql app.sqlite "SELECT json_extract('{\"a\":1}','\$.a') + sqrt(16);")" = 5.0 ] || die "JSON or math functions" [ "$(sql app.sqlite "SELECT soundex('Robert');")" = R163 ] || die "soundex()" [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM dbstat;')" = 1 ] || die "dbstat" + [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM sqlite_dbpage;')" = 1 ] || die "dbpage" + [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM sqlite_stmt;')" = 1 ] || die "stmt" [ "$(sql app.sqlite 'PRAGMA integrity_check;')" = ok ] || die "integrity check after seeding" - ok "FTS3, FTS4, FTS5, R*Tree, Geopoly, JSON, math functions, soundex() and dbstat work" + ok "FTS3, FTS4, FTS5, R*Tree, Geopoly, JSON, math functions, soundex(), dbstat, dbpage and stmt work" +} + +recover_database() { + local page result + sql app.sqlite "VACUUM INTO 'damaged.sqlite';" + [ "$(sql damaged.sqlite .dbinfo | sed -nE 's/^database page size: +//p')" = 4096 ] || die ".dbinfo does not print the page size 4096" + sql damaged.sqlite 'CREATE TABLE junk(id INTEGER PRIMARY KEY, filler BLOB); INSERT INTO junk(filler) SELECT randomblob(500) FROM t;' + page="$(sql damaged.sqlite "SELECT rootpage FROM sqlite_master WHERE name = 'junk';")" + sql -cmd '.dbconfig defensive off' damaged.sqlite "UPDATE sqlite_dbpage SET data = randomblob(4096) WHERE pgno = $page;" > /dev/null + [ "$(sql damaged.sqlite 'PRAGMA integrity_check;' 2>&1 | head -n 1)" != ok ] || die "overwriting page $page of damaged.sqlite left it intact" + docker exec "$CONTAINER" sh -c 'sqlite3 damaged.sqlite .recover | sqlite3 recovered.sqlite > /dev/null && rm damaged.sqlite' || die ".recover failed on the damaged database" + result="$(sql recovered.sqlite 'PRAGMA integrity_check; SELECT count(*) FROM t;' | tr '\n' ' ')" + [ "$result" = 'ok 3 ' ] || die "the recovered database is not intact and complete, got '$result'" + docker exec "$CONTAINER" rm recovered.sqlite + ok ".dbinfo prints the page size, .recover rebuilds the rows from a database with a destroyed page" } backup_database() { diff --git a/src/Dockerfile b/src/Dockerfile index 1c97c86..3fcbd5f 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -33,7 +33,8 @@ COPY version.json verify.sh ./ RUN curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url version.json)" \ && sh verify.sh -# Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: https://sqlite.org/compile.html#recommended_compile_time_options +# Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: +# https://sqlite.org/compile.html#recommended_compile_time_options and https://sqlite.org/cli.html#compiling_the_command_line_interface WORKDIR /src/sqlite RUN gcc -static-pie -fPIE -Os \ -ffunction-sections -fdata-sections -Wl,--gc-sections \ @@ -59,6 +60,11 @@ RUN gcc -static-pie -fPIE -Os \ -DSQLITE_ENABLE_GEOPOLY \ -DSQLITE_ENABLE_MATH_FUNCTIONS \ -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ -DSQLITE_SOUNDEX \ -DHAVE_USLEEP=1 \ -DHAVE_FDATASYNC=1 \ @@ -99,7 +105,7 @@ LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.documentation="https://github.com/peakimages/sqlite#readme" \ org.opencontainers.image.base.name="scratch" \ dev.peakimages.sqlite.version="${SQLITE_VERSION}" \ - dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,soundex" \ + dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,dbpage,stmt,soundex" \ dev.peakimages.tini.version="${TINI_VERSION}" \ dev.peakimages.busybox.version="${BUSYBOX_VERSION}" \ dev.peakimages.image.version="${IMAGE_VERSION}" From 31742406aaed3d1d044557a21adec9901a59571c Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:13:05 +0200 Subject: [PATCH 07/63] feat(release): verify the check-in of a new SQLite release --- scripts/check-upstream.sh | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 6ef017c..0a18a4f 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -3,7 +3,7 @@ set -euo pipefail WORK_DIR="$(mktemp -d)" -BOLD=$'\033[1m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } @@ -21,26 +21,27 @@ main() { download_amalgamation write_version_json verify_amalgamation + verify_release_check_in update_dockerfile } read_latest_release() { - local release + local release path fetch "$WORK_DIR/download.html" https://sqlite.org/download.html release="$(grep -Em 1 '^PRODUCT,[0-9.]+,[0-9]{4}/sqlite-amalgamation-[0-9]+\.zip,[0-9]+,[0-9a-f]{64}$' "$WORK_DIR/download.html")" \ || die "no amalgamation listed on sqlite.org/download.html" - IFS=, read -r _ version path size sha3 <<< "$release" + IFS=, read -r _ new_version path size sha3 <<< "$release" url="https://sqlite.org/$path" } compare_with_pinned() { - local pinned - pinned="$(jq -r .version src/version.json)" - if [ "$version" = "$pinned" ]; then - ok "SQLite $BOLD$pinned$RESET is pinned and the current release on sqlite.org" + local current_version + current_version="$(jq -r .version src/version.json)" + if [ "$new_version" = "$current_version" ]; then + ok "SQLite $BOLD$current_version$RESET is pinned and the current release on sqlite.org" exit 0 fi - info "sqlite.org lists SQLite $BOLD$version$RESET, $pinned is pinned" + info "sqlite.org lists SQLite $BOLD$new_version$RESET, $current_version is pinned" } download_amalgamation() { @@ -51,21 +52,31 @@ download_amalgamation() { } write_version_json() { - jq -n --tab --arg version "$version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ + jq -n --tab --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ '{$version, $url, $size, $sha3_256, $source_id}' > src/version.json - ok "src/version.json pins $BOLD$version$RESET with url, size, SHA3-256 and source id $source_id" + ok "src/version.json pins $BOLD$new_version$RESET with url, size, SHA3-256 and source id $source_id" } verify_amalgamation() { cp src/verify.sh src/version.json "$WORK_DIR" - SQLITE_VERSION="$version" sh "$WORK_DIR/verify.sh" > /dev/null + SQLITE_VERSION="$new_version" sh "$WORK_DIR/verify.sh" > /dev/null ok "src/verify.sh accepts the download against the new src/version.json" } +verify_release_check_in() { + local check_in tag + check_in="${source_id##* }" + fetch "$WORK_DIR/manifest" "https://sqlite.org/src/raw/$check_in" + for tag in release "version-$new_version"; do + grep -qxF "T +sym-$tag *" "$WORK_DIR/manifest" || die "check-in ${check_in:0:10} on sqlite.org/src is not tagged $tag" + done + ok "check-in ${check_in:0:10} is tagged release and ${BOLD}version-$new_version$RESET on sqlite.org/src" +} + update_dockerfile() { - sed -E "s|^ARG SQLITE_VERSION=.*|ARG SQLITE_VERSION=$version|" src/Dockerfile > "$WORK_DIR/Dockerfile" + sed -E "s|^ARG SQLITE_VERSION=.*|ARG SQLITE_VERSION=$new_version|" src/Dockerfile > "$WORK_DIR/Dockerfile" cp "$WORK_DIR/Dockerfile" src/Dockerfile - ok "src/Dockerfile sets SQLITE_VERSION $BOLD$version$RESET" + ok "src/Dockerfile sets SQLITE_VERSION $BOLD$new_version$RESET" } fetch() { From b3dfcb9fc1a92cb0d3dc5c51ad11e7ced4e1ac22 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:22:49 +0200 Subject: [PATCH 08/63] ci(release): improve check-upstream.yaml - run the upstream check twice per day - add a concurrency group, cache-mode none and a job name - set bash as the default shell so a failing command in a pipe fails the step - improve the pull request title and description - label the pull request with the bot/upstream label - run the tests on the branch of the pull request - close open pull requests for older versions when a new one is opened --- .github/workflows/check-upstream.yaml | 39 ++++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index c8aaad5..9d06f06 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -2,16 +2,28 @@ name: Check upstream on: schedule: - - cron: "30 8 * * *" + - cron: "30 6,18 * * *" workflow_dispatch: permissions: {} +cache-mode: none + +concurrency: + group: check-upstream + cancel-in-progress: false + +defaults: + run: + shell: bash + jobs: check: + name: Check for a new SQLite release permissions: - contents: write - pull-requests: write + actions: write # run the tests on the branch of the pull request + contents: write # push the branch of the pull request + pull-requests: write # open the pull request runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: @@ -26,22 +38,25 @@ jobs: GH_TOKEN: ${{ github.token }} run: | git diff --quiet && exit 0 - version="$(jq -r .version src/version.json)" + current_version="$(git show HEAD:src/version.json | jq -r .version)" + new_version="$(jq -r .version src/version.json)" source_id="$(jq -r .source_id src/version.json)" - branch="feat/sqlite-v$version" - title="feat(release): bump SQLite version to v$version" + check_in="${source_id##* }" + branch="feat/sqlite-v$new_version" + title="feat(release): bump SQLite version from v$current_version to v$new_version" if [ -n "$(gh pr list --head "$branch" --json number --jq '.[0].number')" ]; then echo "a pull request from $branch is open already" exit 0 fi cat > "$RUNNER_TEMP/body.md" < [!IMPORTANT] > Squash merge this PR to keep the commit history clean. - BODY gh auth setup-git git config user.name "github-actions[bot]" @@ -49,4 +64,8 @@ jobs: git switch --create "$branch" git commit --all --quiet --message "$title" git push --quiet --set-upstream origin "$branch" - gh pr create --base main --head "$branch" --title "$title" --body-file "$RUNNER_TEMP/body.md" + new_pr="$(gh pr create --base main --head "$branch" --title "$title" --label bot/upstream --body-file "$RUNNER_TEMP/body.md")" + gh workflow run tests.yaml --ref "$branch" + for open_pr in $(gh pr list --label bot/upstream --json url --jq '.[].url'); do + [ "$open_pr" = "$new_pr" ] || gh pr close "$open_pr" --delete-branch --comment "Superseded by $new_pr" + done From c4750fb651b7959af57271d37f32bac3ec0f5a21 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:51:18 +0200 Subject: [PATCH 09/63] refactor: move the SQLite pin to upstream.json --- .github/workflows/check-upstream.yaml | 6 +++--- .github/workflows/release.yaml | 2 +- README.md | 2 +- scripts/check-upstream.sh | 17 +++++++++-------- scripts/release.sh | 2 +- scripts/test.sh | 6 +++--- src/Dockerfile | 4 ++-- src/upstream.json | 11 +++++++++++ src/verify.sh | 10 +++++----- src/version.json | 7 ------- 10 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 src/upstream.json delete mode 100644 src/version.json diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index 9d06f06..72d8083 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -38,9 +38,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: | git diff --quiet && exit 0 - current_version="$(git show HEAD:src/version.json | jq -r .version)" - new_version="$(jq -r .version src/version.json)" - source_id="$(jq -r .source_id src/version.json)" + current_version="$(git show HEAD:src/upstream.json | jq -r .sqlite.latest.version)" + new_version="$(jq -r .sqlite.latest.version src/upstream.json)" + source_id="$(jq -r .sqlite.latest.source_id src/upstream.json)" check_in="${source_id##* }" branch="feat/sqlite-v$new_version" title="feat(release): bump SQLite version from v$current_version to v$new_version" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8d60cd6..6a18f72 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,7 +35,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - sqlite="$(jq -r .version src/version.json)" + sqlite="$(jq -r .sqlite.latest.version src/upstream.json)" image="$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)" release="$sqlite-v$image" publish=true diff --git a/README.md b/README.md index 5f37277..a46b653 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SQLite [![Tests](https://img.shields.io/github/actions/workflow/status/peakimages/sqlite/tests.yaml?branch=main&label=tests)](https://github.com/peakimages/sqlite/actions/workflows/tests.yaml) -[![SQLite](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Fpeakimages%2Fsqlite%2Fmain%2Fsrc%2Fversion.json&query=%24.version&label=sqlite)](https://github.com/peakimages/sqlite/releases) +[![SQLite](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Fpeakimages%2Fsqlite%2Fmain%2Fsrc%2Fupstream.json&query=%24.sqlite.latest.version&label=sqlite)](https://github.com/peakimages/sqlite/releases) [![License](https://img.shields.io/github/license/peakimages/sqlite)](LICENSE) [![Docker pulls](https://img.shields.io/docker/pulls/peakimages/sqlite)](https://hub.docker.com/r/peakimages/sqlite) [![Image size](https://img.shields.io/docker/image-size/peakimages/sqlite/latest)](https://hub.docker.com/r/peakimages/sqlite) diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 0a18a4f..feeebfa 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -19,7 +19,7 @@ main() { read_latest_release compare_with_pinned download_amalgamation - write_version_json + write_upstream_json verify_amalgamation verify_release_check_in update_dockerfile @@ -36,7 +36,7 @@ read_latest_release() { compare_with_pinned() { local current_version - current_version="$(jq -r .version src/version.json)" + current_version="$(jq -r .sqlite.latest.version src/upstream.json)" if [ "$new_version" = "$current_version" ]; then ok "SQLite $BOLD$current_version$RESET is pinned and the current release on sqlite.org" exit 0 @@ -51,16 +51,17 @@ download_amalgamation() { ok "downloaded ${url##*/}, $size bytes" } -write_version_json() { - jq -n --tab --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ - '{$version, $url, $size, $sha3_256, $source_id}' > src/version.json - ok "src/version.json pins $BOLD$new_version$RESET with url, size, SHA3-256 and source id $source_id" +write_upstream_json() { + jq --tab --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ + '.sqlite.latest = {$version, $url, $size, $sha3_256, $source_id}' src/upstream.json > "$WORK_DIR/upstream.json" + cp "$WORK_DIR/upstream.json" src/upstream.json + ok "src/upstream.json pins $BOLD$new_version$RESET with url, size, SHA3-256 and source id $source_id" } verify_amalgamation() { - cp src/verify.sh src/version.json "$WORK_DIR" + cp src/verify.sh src/upstream.json "$WORK_DIR" SQLITE_VERSION="$new_version" sh "$WORK_DIR/verify.sh" > /dev/null - ok "src/verify.sh accepts the download against the new src/version.json" + ok "src/verify.sh accepts the download against the new src/upstream.json" } verify_release_check_in() { diff --git a/scripts/release.sh b/scripts/release.sh index f45b933..4a461bb 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -90,7 +90,7 @@ commit() { push() { local tag - tag="$(jq -r .version src/version.json)-v$new_version" + tag="$(jq -r .sqlite.latest.version src/upstream.json)-v$new_version" if confirm "Push to origin/main?"; then git push --quiet origin main ok "pushed to origin/main, the tests run and then the release workflow publishes $BOLD$tag$RESET" diff --git a/scripts/test.sh b/scripts/test.sh index 01b7e88..cc015a0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -40,10 +40,10 @@ main() { check_version() { local printed version source_id printed="$(docker run --rm "$IMAGE" --version)" - version="$(jq -r .version src/version.json)" - source_id="$(jq -r .source_id src/version.json)" + version="$(jq -r .sqlite.latest.version src/upstream.json)" + source_id="$(jq -r .sqlite.latest.source_id src/upstream.json)" [[ "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected '$version $source_id'" - ok "sqlite3 --version prints $version with the source id from src/version.json" + ok "sqlite3 --version prints $version with the source id from src/upstream.json" } check_default_database() { diff --git a/src/Dockerfile b/src/Dockerfile index 3fcbd5f..7c0b4dc 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -29,8 +29,8 @@ ARG BUSYBOX_VERSION RUN apk add --no-cache build-base curl jq openssl tini-static=${TINI_VERSION} busybox-static=${BUSYBOX_VERSION} WORKDIR /src -COPY version.json verify.sh ./ -RUN curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url version.json)" \ +COPY upstream.json verify.sh ./ +RUN curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .sqlite.latest.url upstream.json)" \ && sh verify.sh # Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: diff --git a/src/upstream.json b/src/upstream.json new file mode 100644 index 0000000..888a2e4 --- /dev/null +++ b/src/upstream.json @@ -0,0 +1,11 @@ +{ + "sqlite": { + "latest": { + "version": "3.54.1", + "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", + "size": 2990966, + "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", + "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" + } + } +} diff --git a/src/verify.sh b/src/verify.sh index 9591ba0..cd2f9d4 100755 --- a/src/verify.sh +++ b/src/verify.sh @@ -1,13 +1,13 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" - die() { echo "verify: $*" >&2 exit 1 } +cd "$(dirname "$0")" + main() { check_version check_download @@ -17,13 +17,13 @@ main() { } check_version() { - [ "$(pin version)" = "$SQLITE_VERSION" ] || die "version.json pins $(pin version), the Dockerfile says $SQLITE_VERSION" + [ "$(pin version)" = "$SQLITE_VERSION" ] || die "upstream.json pins $(pin version), the Dockerfile says $SQLITE_VERSION" } check_download() { size="$(wc -c < sqlite.zip | tr -d ' ')" [ "$size" = "$(pin size)" ] || die "sqlite.zip has $size bytes, expected $(pin size)" - [ "$(openssl dgst -sha3-256 -r sqlite.zip | cut -d ' ' -f 1)" = "$(pin sha3_256)" ] || die "SHA3-256 of sqlite.zip does not match version.json" + [ "$(openssl dgst -sha3-256 -r sqlite.zip | cut -d ' ' -f 1)" = "$(pin sha3_256)" ] || die "SHA3-256 of sqlite.zip does not match upstream.json" } unpack() { @@ -37,7 +37,7 @@ check_header() { } pin() { - jq -r ".$1" version.json + jq -r ".sqlite.latest.$1" upstream.json } main diff --git a/src/version.json b/src/version.json deleted file mode 100644 index 59bd974..0000000 --- a/src/version.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": "3.54.1", - "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", - "size": 2990966, - "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", - "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" -} From 1dafe76781c69b683494311a4e97b4009b132836 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:12:34 +0200 Subject: [PATCH 10/63] feat(release): add new release.json --- release.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 release.json diff --git a/release.json b/release.json new file mode 100644 index 0000000..2f2969c --- /dev/null +++ b/release.json @@ -0,0 +1,8 @@ +{ + "stable": { + "image": "0.1.0" + }, + "preview": { + "image": "0.1.0" + } +} From 2ea253d7897c327c7a415152b8a98559305c07c4 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:22:37 +0200 Subject: [PATCH 11/63] docs: add 3 new rows and update comparison.md - add reproducible builds, PIE and detached runs to comparison.md - change the SQLite version row to SQLite updates, which now says how each image gets a new release --- docs/comparison.md | 71 ++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/comparison.md b/docs/comparison.md index ff4363c..b342c4a 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -2,42 +2,51 @@ `peakimages/sqlite` compared to other popular SQLite Docker images. -| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | -| -------------------------------- | ------------------------------------------------------- | ------------------------------------ | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | -| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | -| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | -| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | -| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | -| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | -| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | -| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | -| Recommended compile options[^2] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Build[^3] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | -| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Init[^4] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | -| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | -| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | -| No runtime extension loading[^5] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Compressed[^6] | 1.55 MB | 0.57 MB | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | -| On disk[^6] | 2.7 MB | 0.57 MB | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | -| Platforms | amd64, arm64, arm/v7, riscv64 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | -| Extensions[^7] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | -| SQLite | 3.53.4 | 3.53.4 | 3.53.4 | 3.53.4 | 3.53.4 | 3.34.1 (2021) | 3.46.1 (2024) | 3.28.0 (2019) | -| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | +| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | +| -------------------------------- | ------------------------------------------------------- | --------------------------------------- | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | +| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | +| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | +| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | +| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | +| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | +| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | +| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | +| Reproducible build[^2] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Recommended compile options[^3] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Build[^4] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | +| PIE[^5] | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Runs detached[^6] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Init[^7] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | +| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | +| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | +| No runtime extension loading[^8] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Compressed[^9] | 1.55 MB | 0.57 MB | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | +| On disk[^9] | 2.7 MB | 0.57 MB | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | +| Platforms | amd64, arm64, arm/v7, riscv64 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | +| Extensions[^10] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | +| SQLite updates | ✅ (checked twice a day, merged by hand) | ✅ (checked daily, built automatically) | ⚠️ (checked weekly, merged by hand) | ⚠️ (follows Alpine releases) | ⚠️ (policy not public) | ❌ (3.34.1, 2021) | ❌ (3.46.1, 2024) | ❌ (3.28.0, 2019) | +| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | [^1]: `sqlite3 database.sqlite` writes into the working directory, and a new named volume takes the owner of the directory it is mounted on. `keinos/sqlite3` and `cleanstart/sqlite3` run their user in `/`, which root owns, so both fail with `unable to open database file`. -[^2]: sqlite.org lists [these options](https://sqlite.org/compile.html#recommended_compile_time_options) "for applications that are able to use them" to save CPU cycles and memory, and to turn off legacy misfeatures. `SQLITE_THREADSAFE=0` alone makes SQLite about 2% faster and smaller. +[^2]: Building the same commit again gives the same image byte for byte, so a rebuild shows whether a published image really comes from its source. The image manifest of each platform keeps its digest, while the digest of a tag changes with every build because the provenance attestation records the build run. -[^3]: A static binary carries its own libc and needs nothing from the OS, so the same binary runs on every Linux distribution and there are no shared libraries to keep patched. +[^3]: sqlite.org lists [these options](https://sqlite.org/compile.html#recommended_compile_time_options) "for applications that are able to use them" to save CPU cycles and memory, and to turn off legacy misfeatures. `SQLITE_THREADSAFE=0` alone makes SQLite about 2% faster and smaller. -[^4]: PID 1 ignores signals it has no handler for, so without an init `docker stop` waits out its grace period and then kills the container. +[^4]: A static binary carries its own libc and needs nothing from the OS, so the same binary runs on any Linux distribution and there are no shared libraries to keep patched. -[^5]: `load_extension()` lets any SQL that reaches the shell run native code from a file. +[^5]: A position-independent executable is loaded at a random memory address on every start, so an exploit cannot rely on known memory addresses. -[^6]: linux/arm64 where published, linux/amd64 otherwise. +[^6]: Most of the other images run the interactive `sqlite3` shell as their main process, which exits when its input ends. With `docker run -d` or as a Compose service, the container stops with exit code 0 unless `-i` or `-t` (`stdin_open` or `tty` in Compose) is set. A running container allows database health checks, scheduled tasks, backups and debugging through its shell. -[^7]: JSON is built into SQLite since 3.38.0 and therefore not listed. A distribution package ships the extensions the distribution chose, which is why the Red Hat build has fewer than the amalgamation offers. +[^7]: PID 1 ignores signals it has no handler for, so without an init, `docker stop` waits out its grace period and then kills the container. + +[^8]: `load_extension()` lets any SQL that reaches the shell run native code from a file. + +[^9]: Measured for `linux/amd64`. + +[^10]: JSON has been built into SQLite since 3.38.0 and is not listed. A distribution package ships the extensions its distribution chose, and SQLite before 3.35.0 has no math functions. From 6cd943eb62ffe05913a61a31667d5439a7a474d5 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:35:48 +0200 Subject: [PATCH 12/63] feat(image): let any uid write SQLite databases --- scripts/test.sh | 27 +++++++++++++++---- src/rootfs.sh | 2 +- src/rootfs/usr/local/bin/docker-entrypoint.sh | 1 + 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index cc015a0..9a4aedb 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -26,6 +26,7 @@ main() { check_databases check_healthcheck check_restart + check_other_user check_creation_settings check_rejected_settings heading "Data" @@ -68,12 +69,12 @@ start_container() { } check_databases() { - local owner + local file [ "$(sql app.sqlite 'PRAGMA journal_mode;')" = wal ] || die "app.sqlite is not in WAL mode" [ "$(sql jobs.sqlite 'PRAGMA journal_mode;')" = wal ] || die "jobs.sqlite is not in WAL mode" - owner="$(docker exec "$CONTAINER" stat -c '%u:%g' jobs.sqlite)" - [ "$owner" = 65532:65532 ] || die "jobs.sqlite is owned by $owner, expected 65532:65532" - ok "app.sqlite and jobs.sqlite from SQLITE_DATABASES are created in WAL mode, owned by $owner" + file="$(docker exec "$CONTAINER" stat -c '%a %u:%g' jobs.sqlite)" + [ "$file" = '666 65532:65532' ] || die "jobs.sqlite has mode and owner '$file', expected 666 65532:65532" + ok "app.sqlite and jobs.sqlite from SQLITE_DATABASES are created in WAL mode, mode 666, owned by 65532:65532" } check_healthcheck() { @@ -91,6 +92,22 @@ check_restart() { ok "a restart leaves an existing database as it is" } +check_other_user() { + local rows + docker run -d --name "$CONTAINER-user" --user 1000:1000 -v "$CONTAINER:/var/lib/sqlite" "$IMAGE" \ + sh -c 'printf "CREATE TABLE shared(id INTEGER PRIMARY KEY); INSERT INTO shared DEFAULT VALUES;\n" | { cat; sleep 60; } | sqlite3 jobs.sqlite' > /dev/null + for _ in $(seq 30); do + rows="$(sql jobs.sqlite 'SELECT count(*) FROM shared;' 2> /dev/null || true)" + [ "$rows" = 1 ] && break + sleep 1 + done + [ "$rows" = 1 ] || die "a container running as uid 1000 could not write to jobs.sqlite: $(docker logs "$CONTAINER-user" 2>&1 | tail -n 1)" + [ "$(docker exec "$CONTAINER" stat -c '%a' jobs.sqlite-shm)" = 666 ] || die "jobs.sqlite-shm is not mode 666 while uid 1000 holds a connection" + [ "$(sql jobs.sqlite 'INSERT INTO shared DEFAULT VALUES; SELECT count(*) FROM shared;')" = 2 ] || die "the image user cannot write jobs.sqlite while uid 1000 holds a connection" + docker rm -f "$CONTAINER-user" > /dev/null + ok "a container running as uid 1000 writes jobs.sqlite, the image user writes it too while that connection is open" +} + check_creation_settings() { local files settings docker run -d --name "$CONTAINER-settings" \ @@ -191,7 +208,7 @@ sql() { } cleanup() { - docker rm -f "$CONTAINER" "$CONTAINER-default" "$CONTAINER-settings" > /dev/null 2>&1 || true + docker rm -f "$CONTAINER" "$CONTAINER-default" "$CONTAINER-settings" "$CONTAINER-user" > /dev/null 2>&1 || true docker volume rm -f "$CONTAINER" > /dev/null 2>&1 || true rm -rf "$WORK_DIR" } diff --git a/src/rootfs.sh b/src/rootfs.sh index f82b6c5..05121af 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -36,7 +36,7 @@ create_user() { } create_directories() { - install -d -m 0700 -o 65532 -g 65532 /rootfs/var/lib/sqlite + install -d -m 0777 -o 65532 -g 65532 /rootfs/var/lib/sqlite install -d -m 1777 /rootfs/tmp } diff --git a/src/rootfs/usr/local/bin/docker-entrypoint.sh b/src/rootfs/usr/local/bin/docker-entrypoint.sh index 5181fdf..852f322 100755 --- a/src/rootfs/usr/local/bin/docker-entrypoint.sh +++ b/src/rootfs/usr/local/bin/docker-entrypoint.sh @@ -60,6 +60,7 @@ validate_databases() { create_database() { sqlite3 "$DATA_DIR/$1" "PRAGMA page_size=$page_size; PRAGMA auto_vacuum=$auto_vacuum; PRAGMA user_version=0; PRAGMA journal_mode=$journal_mode;" > /dev/null + chmod 0666 "$DATA_DIR/$1" echo "docker-entrypoint: created $1, page size $page_size, auto_vacuum $auto_vacuum, journal mode $journal_mode" } From 6433054000666f33f3e6575bb17bf59505f784ce Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:30:59 +0200 Subject: [PATCH 13/63] build(image): bump Alpine from v3.24.2 to v3.24.3 --- src/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 7c0b4dc..029bf0c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -6,8 +6,8 @@ ARG SQLITE_VERSION=3.54.1 ARG IMAGE_VERSION=0.1.0 # Build dependencies -# https://hub.docker.com/_/alpine/tags?name=3.24.2 -ARG BASE_IMAGE=alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 +# https://hub.docker.com/_/alpine/tags?name=3.24.3 +ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a # https://pkgs.alpinelinux.org/package/v3.24/community/x86_64/tini-static ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static From d51dcf099d7e7b0c21e33aa31d879adbc5cb6712 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:33:12 +0200 Subject: [PATCH 14/63] docs: put the scope first in SECURITY.md - say where to report a vulnerability in SQLite, BusyBox or tini - update the example tags to 3.54.1-v0.2.0 --- SECURITY.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 207cc25..12cd03f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,23 +1,25 @@ # Security -This page outlines our support policy and how to report vulnerabilities. +This page outlines what is in scope, our support policy and how to report vulnerabilities. + +## Scope + +Report a vulnerability to the project that maintains the affected code: + +- **The image** is built in this repository: the Dockerfile and its compile-time options, the entrypoint and health check scripts, the root filesystem and the workflows that test and publish it. Report it here, see [Reporting a Vulnerability](#reporting-a-vulnerability). +- **SQLite** is maintained at [sqlite.org](https://sqlite.org). Report it to the SQLite team, see [sqlite.org/cves.html](https://sqlite.org/cves.html). Every SQLite release becomes a new image, usually within a day. +- **BusyBox and tini** come from pinned Alpine packages. Report it to Alpine or the upstream project, the fix reaches the image with its next release. ## Support Policy -Below you can find the support status for different versions of the image. The image version is the `v` part of an image tag, `peakimages/sqlite:3.53.4-v0.1.0` is image version `v0.1.0`. +Below you can find the support status for different versions of the image. The image version is the `v` part of an image tag, `peakimages/sqlite:3.54.1-v0.2.0` is image version `v0.2.0`. | Version | Supported | Support Status | | ------- | ------------------ | ------------------ | | **1.x** | :white_check_mark: | Unreleased | | **0.x** | :white_check_mark: | Active Development | -Immutable tags such as `3.53.4-v0.1.0` are never rebuilt. A fix ships as a new image version, and the floating tags `3.53.4`, `3.53`, `3` and `latest` move to it. Deployments that pin an immutable tag receive fixes by moving to the newer tag. - -### Scope - -- **This repository** builds the image: the Dockerfile and its compile-time options, the entrypoint and health check scripts, the root filesystem and the workflows that test and publish. Report vulnerabilities in these here. -- **SQLite itself** is developed at [sqlite.org](https://sqlite.org). Report vulnerabilities in SQLite to the SQLite team, see [sqlite.org/cves.html](https://sqlite.org/cves.html). Every new SQLite release becomes a new image, usually within a day. -- **BusyBox and tini** come from pinned Alpine packages. Alpine's fixes reach the image with the next release. +Immutable tags such as `3.54.1-v0.2.0` are never rebuilt. A fix ships as a new image version, and the floating tags `3.54.1`, `3.54`, `3` and `latest` move to it. Deployments that pin an immutable tag receive fixes by moving to the newer tag. ## Reporting a Vulnerability From 507caea684982ab16a84a3aaddb148504261f9f9 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:53:31 +0200 Subject: [PATCH 15/63] docs: prepare README.md for v0.2.0 - show the amd64 image size in the size badge and switch tests to the release workflow - describe the image as a central home for SQLite databases in the introduction - update what makes this image different - explain that the application still talks to the database through its own SQLite engine, version and extensions, and what this image does next to it - drop the uid 65532 note and the user line from the Compose example - add a section on copying the static sqlite3 shell into other images - update the example tags to 3.54.1-v0.2.0 and add the preview tag --- README.md | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a46b653..9a17b8d 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # SQLite -[![Tests](https://img.shields.io/github/actions/workflow/status/peakimages/sqlite/tests.yaml?branch=main&label=tests)](https://github.com/peakimages/sqlite/actions/workflows/tests.yaml) +[![Release](https://img.shields.io/github/actions/workflow/status/peakimages/sqlite/release.yaml?branch=main&label=release)](https://github.com/peakimages/sqlite/actions/workflows/release.yaml) [![SQLite](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Fpeakimages%2Fsqlite%2Fmain%2Fsrc%2Fupstream.json&query=%24.sqlite.latest.version&label=sqlite)](https://github.com/peakimages/sqlite/releases) [![License](https://img.shields.io/github/license/peakimages/sqlite)](LICENSE) [![Docker pulls](https://img.shields.io/docker/pulls/peakimages/sqlite)](https://hub.docker.com/r/peakimages/sqlite) -[![Image size](https://img.shields.io/docker/image-size/peakimages/sqlite/latest)](https://hub.docker.com/r/peakimages/sqlite) +[![Image size](https://img.shields.io/docker/image-size/peakimages/sqlite/latest?arch=amd64)](https://hub.docker.com/r/peakimages/sqlite) ## Introduction -Minimal, production-ready, hardened and secure SQLite Docker image. Made for SQLite database deployments on [Coolify](https://coolify.io) and any other container host. +A central home for one or more SQLite databases. One minimal and hardened container creates every database you name, health checks them and brings the `sqlite3` shell for backups, restores and maintenance. Made for [Coolify](https://coolify.io?ref=peakimages.dev) and any other container host. > [!IMPORTANT] -> `peakimages/sqlite` is currently **v0** and subject to breaking changes prior to `v1.0.0`. Pin to a specific version like `peakimages/sqlite:3.53.4-v0.1.0` to avoid breaking changes. +> `peakimages/sqlite` is currently **v0** and subject to breaking changes prior to `v1.0.0`. Pin to a specific version like `peakimages/sqlite:3.54.1-v0.2.0` to avoid breaking changes. ### What Makes This Image Different? @@ -22,9 +22,9 @@ SQLite has no server and the database is just a file. Application images rarely - ✅ Zero Config Required: Production-ready defaults, customizable with environment variables - ✅ Multi-Database Support: One container creates and health checks every database you name - ✅ Verified Source: Size, SHA3-256 and source id of the official amalgamation are checked before each compilation -- ✅ Batteries Included: `FTS3`, `FTS4`, `FTS5`, `R*Tree`, `Geopoly`, `JSON`, `math`, `dbstat` and `soundex` SQLite extensions -- ✅ Modern Architecture: Native health checks, `tini` init, OCI labels, provenance and SBOM attestations -- ✅ Always Current: sqlite.org is checked daily and every new SQLite release becomes a new image +- ✅ Batteries Included: `FTS3`, `FTS4`, `FTS5`, `R*Tree`, `Geopoly`, `math`, `dbstat`, `dbpage`, `stmt` and `soundex` SQLite extensions +- ✅ Modern Architecture: Native health checks, `tini` init, OCI labels, reproducible builds, provenance and SBOM attestations +- ✅ Always Current: sqlite.org is checked twice a day and new SQLite releases become new images - ✅ Multi-Architecture: Builds for `linux/amd64`, `linux/arm64`, `linux/arm/v7` and `linux/riscv64` - ✅ Tested: Every release is tested on all four platforms before it is published @@ -34,6 +34,8 @@ To see a comparison of this image with other popular SQLite Docker images, see [ ## Getting Started +SQLite is not a server and the database is just a file. The application reads and writes it with the SQLite engine it bundles, such as `pdo_sqlite` for PHP or `rusqlite` for Rust, so the SQLite version and extensions of that engine apply, not the ones of this image. This container shares the volume with the application and looks after the files: it creates and configures them, health checks them and brings the `sqlite3` shell for backups, restores and maintenance. + ### Docker Start the container with a named volume for the default database: @@ -46,13 +48,10 @@ docker run -d --name sqlite -v sqlite-data:/var/lib/sqlite peakimages/sqlite Add the SQLite container next to the application that needs a SQLite database, both mounting the same volume: -> [!IMPORTANT] -> The application must run as uid `65532`, the `nonroot` user of distroless images, or as root to write to the database. - ```yaml services: sqlite: - image: peakimages/sqlite:3.53.4-v0.1.0 + image: peakimages/sqlite:3.54.1-v0.2.0 environment: SQLITE_DATABASES: app.sqlite, jobs.sqlite volumes: @@ -60,7 +59,6 @@ services: app: image: your-app - user: "65532:65532" depends_on: sqlite: condition: service_healthy @@ -73,6 +71,14 @@ volumes: sqlite-data: ``` +### Shell for Other Images + +`sqlite3` is a static binary without dependencies. You can copy it into any image with a single line, and it works on `scratch`, distroless, Alpine, Debian and more, for backups, restores and maintenance without a package manager: + +```dockerfile +COPY --from=peakimages/sqlite:3.54.1-v0.2.0 /usr/local/bin/sqlite3 /usr/local/bin/sqlite3 +``` + ## Available Image Variations For now there is just one image variation. All tags follow this pattern: @@ -83,15 +89,16 @@ peakimages/sqlite:{{sqlite-version}}-v{{image-version}} | Variation | Best For | Example | | --------- | ------------------------------------------------------------------- | ------------------------------------------------------------ | -| default | Every SQLite deployment, from a Coolify service to a Kubernetes pod | `peakimages/sqlite:3.53.4-v0.1.0` `peakimages/sqlite:3.53.4` | +| default | Every SQLite deployment, from a Coolify service to a Kubernetes pod | `peakimages/sqlite:3.54.1-v0.2.0` `peakimages/sqlite:3.54.1` | ### Tags -| Tag | Meaning | -| --------------------- | ------------------------------------------------------------- | -| `3.53.4-v0.1.0` | Immutable: SQLite 3.53.4 built as image v0.1.0, never rebuilt | -| `3.53.4`, `3.53`, `3` | Newest build of that SQLite patch, minor or major version | -| `latest` | Newest build | +| Tag | Meaning | +| --------------------- | ----------------------------------------------------------------------------- | +| `3.54.1-v0.2.0` | Immutable: SQLite 3.54.1 built as image v0.2.0, never rebuilt | +| `3.54.1`, `3.54`, `3` | Newest release of that SQLite patch, minor or major version | +| `latest` | Newest release | +| `preview` | Newest prerelease, like `3.54.1-v0.3.0-beta.1`, until its release replaces it | ### Registries @@ -114,6 +121,6 @@ The defaults are set in the image, override them with `-e` or the `environment` ## Releases -Every SQLite release becomes a new version of the image. A check for new SQLite versions is run daily. +Every SQLite release becomes a new version of the image. A check for new SQLite versions runs twice a day. The image version follows semantic versioning independently of SQLite. Changes to the image are listed in [CHANGELOG.md](CHANGELOG.md). From 5d3c780b0ecfa91e83d91d71b61100cc1680674a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:26:06 +0100 Subject: [PATCH 16/63] feat(scripts): stop on arguments a script does not take --- scripts/lint.sh | 10 ++++------ scripts/test.sh | 4 +++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/lint.sh b/scripts/lint.sh index 071eec7..0ec40ef 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 SHFMT=mvdan/shfmt:v3.14.1 @@ -22,13 +23,10 @@ cd "$(dirname "$0")/.." failed=0 main() { + [[ "$OPTION" =~ ^(--fix)?$ ]] || die "'$OPTION' is not an option, expected --fix or none" command -v docker > /dev/null || die "docker is not installed" docker info > /dev/null 2>&1 || die "docker is not running" - case "${1:-}" in - "") ;; - --fix) format ;; - *) die "usage: scripts/lint.sh [--fix]" ;; - esac + [ "$OPTION" != --fix ] || format heading "Linting" check src/Dockerfile hadolint "$HADOLINT" hadolint src/Dockerfile check "the shell scripts" ShellCheck "$SHELLCHECK" scripts/*.sh src/*.sh src/rootfs/usr/local/bin/*.sh @@ -65,4 +63,4 @@ tool() { docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/mnt" -w /mnt "$@" } -main "$@" +main diff --git a/scripts/test.sh b/scripts/test.sh index 9a4aedb..8ee5fd9 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE="${1:?usage: scripts/test.sh IMAGE}" +OPTION="$*" +IMAGE="${1:-}" CONTAINER=sqlite-test WORK_DIR="$(mktemp -d)" @@ -18,6 +19,7 @@ cd "$(dirname "$0")/.." trap cleanup EXIT main() { + [[ "$OPTION" =~ ^[^-][^[:space:]]*$ ]] || die "'$OPTION' is not an option, expected an image" heading "Testing $IMAGE" check_version check_default_database From dce37e732028395c7c4a878f00ccbd7f3ea10e5b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:21:51 +0100 Subject: [PATCH 17/63] feat(release): check and support older SQLite lines --- scripts/check-upstream.sh | 66 +++++++++++++++++++++++++-------------- scripts/test.sh | 6 ++-- src/Dockerfile | 3 +- src/verify.sh | 7 +---- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index feeebfa..61944c2 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" +LINE="${1:-latest}" +VERSION_PATTERN='[0-9]+\.[0-9]+\.[0-9]+' WORK_DIR="$(mktemp -d)" BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' @@ -16,52 +19,54 @@ cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"' EXIT main() { - read_latest_release + [[ "$OPTION" =~ ^(latest|[0-9]+(\.[0-9]+)*)?$ ]] || die "'$OPTION' is not an option, expected a line such as 3.53, latest or none" + read_newest_release compare_with_pinned download_amalgamation + verify_release_notes + verify_release_check_in write_upstream_json verify_amalgamation - verify_release_check_in update_dockerfile } -read_latest_release() { - local release path - fetch "$WORK_DIR/download.html" https://sqlite.org/download.html - release="$(grep -Em 1 '^PRODUCT,[0-9.]+,[0-9]{4}/sqlite-amalgamation-[0-9]+\.zip,[0-9]+,[0-9a-f]{64}$' "$WORK_DIR/download.html")" \ - || die "no amalgamation listed on sqlite.org/download.html" - IFS=, read -r _ new_version path size sha3 <<< "$release" - url="https://sqlite.org/$path" +read_newest_release() { + local release year number + fetch "$WORK_DIR/chronology.html" https://sqlite.org/chronology.html + release="$(sed -nE "s|.*>([0-9]{4})-[0-9]{2}-[0-9]{2}\$|year \1|p; s|.*data-sortkey='([0-9]+)'>($VERSION_PATTERN).*|version \1 \2|p" "$WORK_DIR/chronology.html" \ + | awk -v line="$LINE." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ + | sort -k3,3V | tail -n 1)" + [ -n "$release" ] || die "sqlite.org/chronology.html lists no SQLite $LINE release" + read -r year number new_version <<< "$release" + url="https://sqlite.org/$year/sqlite-amalgamation-$number.zip" } compare_with_pinned() { local current_version - current_version="$(jq -r .sqlite.latest.version src/upstream.json)" + current_version="$(jq -r --arg line "$LINE" '.sqlite[$line].version // empty' src/upstream.json)" if [ "$new_version" = "$current_version" ]; then - ok "SQLite $BOLD$current_version$RESET is pinned and the current release on sqlite.org" + ok "SQLite $BOLD$current_version$RESET is pinned as $LINE and the newest release of it on sqlite.org" exit 0 fi - info "sqlite.org lists SQLite $BOLD$new_version$RESET, $current_version is pinned" + info "sqlite.org lists SQLite $BOLD$new_version$RESET, ${current_version:-no version} is pinned as $LINE" } download_amalgamation() { fetch "$WORK_DIR/sqlite.zip" "$url" + size="$(wc -c < "$WORK_DIR/sqlite.zip" | tr -d ' ')" + sha3="$(openssl dgst -sha3-256 -r "$WORK_DIR/sqlite.zip" | cut -d ' ' -f 1)" source_id="$(unzip -p "$WORK_DIR/sqlite.zip" '*/sqlite3.h' | sed -nE 's/^#define SQLITE_SOURCE_ID +"([^"]+)".*/\1/p')" [ -n "$source_id" ] || die "sqlite3.h in the download does not define SQLITE_SOURCE_ID" ok "downloaded ${url##*/}, $size bytes" } -write_upstream_json() { - jq --tab --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ - '.sqlite.latest = {$version, $url, $size, $sha3_256, $source_id}' src/upstream.json > "$WORK_DIR/upstream.json" - cp "$WORK_DIR/upstream.json" src/upstream.json - ok "src/upstream.json pins $BOLD$new_version$RESET with url, size, SHA3-256 and source id $source_id" -} - -verify_amalgamation() { - cp src/verify.sh src/upstream.json "$WORK_DIR" - SQLITE_VERSION="$new_version" sh "$WORK_DIR/verify.sh" > /dev/null - ok "src/verify.sh accepts the download against the new src/upstream.json" +verify_release_notes() { + local sqlite3_c + fetch "$WORK_DIR/release.html" "https://sqlite.org/releaselog/${new_version//./_}.html" + sqlite3_c="$(unzip -p "$WORK_DIR/sqlite.zip" '*/sqlite3.c' | openssl dgst -sha3-256 -r | cut -d ' ' -f 1)" + grep -qxF "
  • SQLITE_SOURCE_ID: $source_id" "$WORK_DIR/release.html" || die "the release notes of SQLite $new_version list another source id than $source_id" + grep -qxF "
  • SHA3-256 for sqlite3.c: $sqlite3_c" "$WORK_DIR/release.html" || die "the release notes of SQLite $new_version list another SHA3-256 for sqlite3.c than $sqlite3_c" + ok "the release notes of SQLite $new_version list the source id and the SHA3-256 of sqlite3.c in the download" } verify_release_check_in() { @@ -74,7 +79,22 @@ verify_release_check_in() { ok "check-in ${check_in:0:10} is tagged release and ${BOLD}version-$new_version$RESET on sqlite.org/src" } +write_upstream_json() { + jq --tab --arg line "$LINE" --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ + '.sqlite[$line] = {$version, $url, $size, $sha3_256, $source_id}' src/upstream.json > "$WORK_DIR/upstream.json" + cp "$WORK_DIR/upstream.json" src/upstream.json + ok "src/upstream.json pins $BOLD$new_version$RESET as $LINE with url, size, SHA3-256 and source id $source_id" +} + +verify_amalgamation() { + cp src/verify.sh "$WORK_DIR" + jq --arg line "$LINE" '.sqlite[$line]' src/upstream.json > "$WORK_DIR/sqlite.json" + SQLITE_VERSION="$new_version" sh "$WORK_DIR/verify.sh" > /dev/null + ok "src/verify.sh accepts the download against the new pin" +} + update_dockerfile() { + [ "$LINE" = latest ] || return 0 sed -E "s|^ARG SQLITE_VERSION=.*|ARG SQLITE_VERSION=$new_version|" src/Dockerfile > "$WORK_DIR/Dockerfile" cp "$WORK_DIR/Dockerfile" src/Dockerfile ok "src/Dockerfile sets SQLITE_VERSION $BOLD$new_version$RESET" diff --git a/scripts/test.sh b/scripts/test.sh index 8ee5fd9..f20fdc8 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -43,10 +43,10 @@ main() { check_version() { local printed version source_id printed="$(docker run --rm "$IMAGE" --version)" - version="$(jq -r .sqlite.latest.version src/upstream.json)" - source_id="$(jq -r .sqlite.latest.source_id src/upstream.json)" + version="$(docker image inspect --format '{{ index .Config.Labels "dev.peakimages.sqlite.version" }}' "$IMAGE")" + source_id="$(jq -r --arg version "$version" '.sqlite[] | select(.version == $version) | .source_id' src/upstream.json)" [[ "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected '$version $source_id'" - ok "sqlite3 --version prints $version with the source id from src/upstream.json" + ok "sqlite3 --version prints $version with the source id src/upstream.json pins for it" } check_default_database() { diff --git a/src/Dockerfile b/src/Dockerfile index 029bf0c..432ef33 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -30,7 +30,8 @@ RUN apk add --no-cache build-base curl jq openssl tini-static=${TINI_VERSION} bu WORKDIR /src COPY upstream.json verify.sh ./ -RUN curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .sqlite.latest.url upstream.json)" \ +RUN jq --arg version "$SQLITE_VERSION" '.sqlite[] | select(.version == $version)' upstream.json > sqlite.json \ + && curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url sqlite.json)" \ && sh verify.sh # Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: diff --git a/src/verify.sh b/src/verify.sh index cd2f9d4..ace0755 100755 --- a/src/verify.sh +++ b/src/verify.sh @@ -9,17 +9,12 @@ die() { cd "$(dirname "$0")" main() { - check_version check_download unpack check_header echo "verify: SQLite $SQLITE_VERSION, $(pin size) bytes, SHA3-256 and source id match" } -check_version() { - [ "$(pin version)" = "$SQLITE_VERSION" ] || die "upstream.json pins $(pin version), the Dockerfile says $SQLITE_VERSION" -} - check_download() { size="$(wc -c < sqlite.zip | tr -d ' ')" [ "$size" = "$(pin size)" ] || die "sqlite.zip has $size bytes, expected $(pin size)" @@ -37,7 +32,7 @@ check_header() { } pin() { - jq -r ".sqlite.latest.$1" upstream.json + jq -r ".$1" sqlite.json } main From dc8c8aa86b733bd329fca5f760bc2945662bb562 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:28:38 +0100 Subject: [PATCH 18/63] feat(release): adapt release.sh to the new release process - bump stable.image or preview.image in release.json instead of IMAGE_VERSION in the Dockerfile - release from main, next, release and maintenance branches - allow prereleases with an alpha, beta or rc stage - offer the versions the branch allows and preselect the bump the commits need - skip a version another branch released and warn about one a release branch holds - date the CHANGELOG.md entries only for a stable release --- scripts/release.sh | 226 ++++++++++++++++++++++++++++++++-------- scripts/undo-release.sh | 26 +++-- 2 files changed, 200 insertions(+), 52 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 4a461bb..db16134 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,14 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" +VERSION_FILE=release.json +ARTIFACT=image +UPSTREAM=sqlite +BREAKING_CHANGE='^([a-z]+(\([^)]*\))?!|BREAKING[ -]CHANGE): ' WORK_DIR="$(mktemp -d)" -BOLD=$'\033[1m' DIM=$'\033[2m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } -hint() { printf ' %-26s%s%s%s\n' "$1" "$DIM" "$2" "$RESET"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } die() { printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 exit 1 @@ -19,84 +24,162 @@ confirm() { read -r answer [[ "$answer" =~ ^[Yy]$ ]] } +hint() { printf ' %-26s%s%s%s\n' "$1" "$DIM" "$2" "$RESET"; } cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"; tput cnorm 2> /dev/null || true' EXIT main() { + [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository - read_version - select_bump + read_versions + read_changes + select_stage + select_version heading "Releasing v$new_version" update_changelog - update_dockerfile + update_version commit push } check_repository() { - local branch ahead behind entries word + local counts behind ahead entries heading "Checking the repository" git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" - branch="$(git rev-parse --abbrev-ref HEAD)" - [ "$branch" = main ] || die "releases are made from main, the current branch is $branch" + branch="$(git symbolic-ref --quiet --short HEAD)" || die "HEAD is detached, expected a branch" + [[ "$branch" =~ ^(main|next|v[0-9]+(\.[0-9]+)?\.x|release/v([0-9]+\.[0-9]+))$ ]] \ + || die "$branch releases nothing" + line="${BASH_REMATCH[3]}" + channel=stable + [[ ! "$branch" =~ ^(next|release/) ]] || channel=preview [ -z "$(git status --porcelain)" ] || die "the working tree is not clean, commit or stash first" - git fetch --quiet origin || die "could not fetch origin" - ahead="$(git rev-list --count origin/main..HEAD)" - behind="$(git rev-list --count HEAD..origin/main)" - [ "$ahead" = 0 ] || die "main is $ahead commit(s) ahead of origin/main, push first" - [ "$behind" = 0 ] || die "main is $behind commit(s) behind origin/main, pull first" - ok "main is clean and in sync with origin/main" - entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- /' CHANGELOG.md | wc -l | tr -d ' ')" + [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" + git fetch --quiet --prune --tags origin || die "could not fetch origin" + counts="$(git rev-list --left-right --count "origin/$branch...HEAD" 2> /dev/null)" || die "origin has no branch $branch, push it first" + read -r behind ahead <<< "$counts" + [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" + ok "$branch is clean and in sync with origin/$branch" + entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- / { count++ } END { print count + 0 }' CHANGELOG.md)" [ "$entries" != 0 ] || die "CHANGELOG.md has no entries under ## [Unreleased], describe the changes first" - word=entries - [ "$entries" = 1 ] && word=entry - ok "CHANGELOG.md has $BOLD$entries$RESET unreleased $word" + ok "CHANGELOG.md lists $BOLD$entries$RESET change(s) under [Unreleased], the notes of this release" +} + +read_versions() { + local released + version="$(artifact_version "$channel")" + released="$(artifact_version stable)" + [ -n "$version" ] && [ -n "$released" ] || die "$VERSION_FILE has no versions of $ARTIFACT" + stable="$(release_tags --merged HEAD | tag_versions | awk '!/-/' | sort -V | tail -n 1)" + stable="${stable:-$released}" + ! newer "$released" "$stable" || die "$VERSION_FILE sets the stable version $released, $branch released $stable last, publish $released first" + ok "$VERSION_FILE sets the $channel version $BOLD$version$RESET" +} + +read_changes() { + local tag range hash subject + tag="$(git describe --tags --abbrev=0 --match "*v$stable" 2> /dev/null)" || true + range="${tag:+$tag..}HEAD" + required="patch" + if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then + required=major + [ "${stable%%.*}" != 0 ] || required=minor + elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): / { found = 1 } END { exit !found }'; then + required=minor + fi + info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" + git log -E --grep="$BREAKING_CHANGE" --format='%h %s' "$range" | while read -r hash subject; do + warn "$DIM$hash$RESET $subject is a breaking change" + done } -read_version() { - current_version="$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)" - [[ "$current_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "IMAGE_VERSION in src/Dockerfile is '$current_version', expected major.minor.patch" +select_stage() { + local stages=(alpha beta rc) selected=0 i + stage= + [ "$channel" = preview ] || return 0 + for i in "${!stages[@]}"; do + [[ "$version" != *-"${stages[$i]}".* ]] || selected="$i" + done + heading "Select the stage" + stage="${stages[$(menu "$selected" "${stages[@]}")]}" } -select_bump() { - local major minor patch options choice - IFS=. read -r major minor patch <<< "$current_version" - options=("$major.$minor.$((patch + 1))" "$major.$((minor + 1)).0" "$((major + 1)).0.0") - heading "Current version v$current_version, select the bump" - choice="$(menu "patch v${options[0]}" "minor v${options[1]}" "major v${options[2]}")" - new_version="${options[$choice]}" +select_version() { + local bases=() base candidate options=() selected=0 + if [ -n "$line" ]; then + bases=("$line.0") + elif [ "$channel" = preview ]; then + bases=("$(next_version "$stable" minor)" "$(next_version "$stable" major)") + [ "$required" != major ] || selected=1 + if [[ "$version" == *-* ]] && newer "${version%%-*}" "$stable"; then + if taken "${version%%-*}"; then + info "${version%%-*} is taken by $holder" + else + bases=("${version%%-*}" "${bases[@]}") + selected=0 + fi + fi + else + bases=("$(next_version "$stable" patch)") + [[ "$branch" == v*.*.x ]] || bases+=("$(next_version "$stable" minor)") + [ "$branch" != main ] || bases+=("$(next_version "$stable" major)") + case "$required" in + minor) selected=1 ;; + major) selected=2 ;; + esac + fi + for base in "${bases[@]}"; do + candidate="$base" + if [ -n "$stage" ]; then + candidate="$base-$stage.1" + [ "$base-$stage" != "${version%.*}" ] || candidate="$base-$stage.$((${version##*.} + 1))" + fi + if newer "$candidate" "$version" && [[ " ${options[*]:-} " != *" $candidate "* ]]; then + options+=("$candidate") + ! in_progress "$base" || warn "$base is in progress, held by $holder" + fi + done + [ -n "${options[*]:-}" ] || die "$branch is at $version, no $stage prerelease follows it" + [ "$selected" -lt "${#options[@]}" ] || selected=$((${#options[@]} - 1)) + heading "Select the version" + new_version="${options[$(menu "$selected" "${options[@]/#/v}")]}" } update_changelog() { local today + if [ "$channel" = preview ]; then + info "CHANGELOG.md keeps the entries under [Unreleased] until the stable release dates them" + return + fi today="$(date +%Y-%m-%d)" - awk -v heading="## [v$new_version] - $today" '{ print } /^## \[Unreleased\]$/ { print ""; print heading }' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" + awk -v heading="## [v$new_version] - $today" ' + /^#/ && NR > 1 && previous != "" { print "" } + { print; previous = $0 } + /^## \[Unreleased\]$/ { print ""; print heading; previous = heading } + ' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" cp "$WORK_DIR/CHANGELOG.md" CHANGELOG.md ok "CHANGELOG.md dates the unreleased entries as ${BOLD}v$new_version$RESET, $today" } -update_dockerfile() { - sed -E "s/^ARG IMAGE_VERSION=.*/ARG IMAGE_VERSION=$new_version/" src/Dockerfile > "$WORK_DIR/Dockerfile" - cp "$WORK_DIR/Dockerfile" src/Dockerfile - ok "src/Dockerfile sets IMAGE_VERSION $current_version → $BOLD$new_version$RESET" +update_version() { + jq --tab --arg channel "$channel" --arg artifact "$ARTIFACT" --arg version "$new_version" '.[$channel][$artifact] = $version' "$VERSION_FILE" > "$WORK_DIR/release.json" + cp "$WORK_DIR/release.json" "$VERSION_FILE" + ok "$VERSION_FILE sets the $channel version $version → $BOLD$new_version$RESET" } commit() { - git add src/Dockerfile CHANGELOG.md - git commit --quiet -S --message "feat(release): bump image version to v$new_version" + git add "$VERSION_FILE" CHANGELOG.md + git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } push() { - local tag - tag="$(jq -r .sqlite.latest.version src/upstream.json)-v$new_version" - if confirm "Push to origin/main?"; then - git push --quiet origin main - ok "pushed to origin/main, the tests run and then the release workflow publishes $BOLD$tag$RESET" - heading "Next" - hint "gh run watch" "follow the workflow run" - hint "Releases on GitHub" "review the draft release once the workflow run is green and then publish it" + local runs + if confirm "Push to origin/$branch?"; then + git push --quiet origin "$branch" + ok "pushed ${BOLD}v$new_version$RESET to origin/$branch" + runs="$(git remote get-url origin | sed -E 's#^(ssh://)?git@github\.com[:/]#https://github.com/#; s#\.git$##')/actions/workflows/release.yaml?query=branch%3A$branch" + info "the release workflow runs at $runs" else info "not pushed" heading "Next" @@ -105,8 +188,63 @@ push() { fi } +artifact_version() { + jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "$VERSION_FILE" +} + +release_tags() { + git tag --list "$@" | grep -E "^${UPSTREAM:+[^-]+-}v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true +} + +tag_versions() { + sed 's/.*v//' +} + +next_version() { + local major minor patch + IFS=. read -r major minor patch <<< "${1%%-*}" + while true; do + case "$2" in + patch) patch=$((patch + 1)) ;; + minor) minor=$((minor + 1)) patch=0 ;; + major) major=$((major + 1)) minor=0 patch=0 ;; + esac + taken "$major.$minor.$patch" || break + done + echo "$major.$minor.$patch" +} + +taken() { + local tag + for tag in $(release_tags "*v$1"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + [ "$channel" = preview ] && in_progress "$1" +} + +in_progress() { + local tag release="release/v${1%.0}" + if [ "$release" != "$branch" ] && git rev-parse --quiet --verify "origin/$release" > /dev/null; then + holder="$release" + [ "$channel" = stable ] || return 0 + git merge-base --is-ancestor "origin/$release" HEAD || return 0 + fi + for tag in $(release_tags "*v$1-*"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + return 1 +} + +newer() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$1" "$2" | sed 's/-/~/' | sort -V | tail -n 1 | sed 's/~/-/')" = "$1" ] +} + menu() { - local options=("$@") selected=0 key i + local selected="$1" options key i + shift + options=("$@") tput civis >&2 2> /dev/null || true while true; do for i in "${!options[@]}"; do diff --git a/scripts/undo-release.sh b/scripts/undo-release.sh index 6a51358..843e107 100755 --- a/scripts/undo-release.sh +++ b/scripts/undo-release.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -BOLD=$'\033[1m' DIM=$'\033[2m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +OPTION="$*" +VERSION_FILE=release.json + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } @@ -20,27 +23,34 @@ confirm() { cd "$(dirname "$0")/.." main() { + [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository heading "Undoing $(git log -1 --format='%h %s')" - info "resets the commit and restores src/Dockerfile and CHANGELOG.md" + info "resets the commit and restores $VERSION_FILE and CHANGELOG.md" confirm "Continue?" || die "aborted, nothing changed" undo_commit } check_repository() { - local subject + local branch git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" - subject="$(git log -1 --format=%s)" - [[ "$subject" == "feat(release): bump image version to v"* ]] || die "the last commit is not a release commit: $subject" + branch="$(git rev-parse --abbrev-ref HEAD)" + git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" || die "the last commit changes more than $VERSION_FILE and CHANGELOG.md, expected a release commit" + [ "$(versions_at HEAD~1)" != "$(versions_at HEAD)" ] || die "the last commit keeps the versions in $VERSION_FILE at $(versions_at HEAD), expected a release commit" + [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md)" ] || die "$VERSION_FILE or CHANGELOG.md has uncommitted changes, commit or stash them first" git fetch --quiet origin || die "could not fetch origin" - [ "$(git rev-list --count origin/main..HEAD)" -ge 1 ] || die "the release commit is on origin/main already, nothing to undo locally" + [ "$(git rev-list --count "origin/$branch..HEAD")" -ge 1 ] || die "the release commit is on origin/$branch already, nothing to undo locally" } undo_commit() { git reset --quiet --soft HEAD~1 - git restore --staged --worktree src/Dockerfile CHANGELOG.md + git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md ok "reset to $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" - ok "src/Dockerfile is back to IMAGE_VERSION $BOLD$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)$RESET, the entries are back under [Unreleased]" + ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the entries are back under [Unreleased]" +} + +versions_at() { + git show "$1:$VERSION_FILE" | jq -r '[paths(scalars) as $path | "\($path | join(".")) \(getpath($path))"] | join(", ")' } main From 88f98760f104cfbf3cf1753e3859373120e00a08 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:29:15 +0100 Subject: [PATCH 19/63] feat(release): add version.sh to decide releases and tags --- scripts/version.sh | 277 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100755 scripts/version.sh diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100755 index 0000000..84e296f --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +BRANCH="${BRANCH:-}" +VERSION_FILE=release.json +ARTIFACT=image +UPSTREAM_FILE=src/upstream.json +UPSTREAM=sqlite + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +info() { [ -n "$OPTION" ] || printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*" >&2; } +ok() { [ -n "$OPTION" ] || printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*" >&2; } +warn() { [ -n "$OPTION" ] || printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*" >&2; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." + +main() { + [[ "$OPTION" =~ ^(--build|--release|--tags)?$ ]] || die "'$OPTION' is not an option, expected --build, --release, --tags or none" + read_version + fetch_origin + print_tags + check_branch + check_version + select_releases + check_unreleased + print_upcoming_versions + print_releases +} + +read_version() { + branch="${BRANCH:-$(git branch --show-current)}" + channel=stable + [[ ! "$branch" =~ ^(next|release/v[0-9]+\.[0-9]+)$ ]] || channel=preview + version="$(artifact_version "$channel")" + [ -n "$version" ] || die "$VERSION_FILE has no $channel version of $ARTIFACT" + if [ "$OPTION" = --build ]; then + echo "$version" + exit 0 + fi +} + +fetch_origin() { + local current behind + [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" + git fetch --quiet --prune --tags origin || die "could not fetch origin" + current="$(git branch --show-current)" + if [ -n "$current" ] && behind="$(git rev-list --count "HEAD..origin/$current" 2> /dev/null)"; then + [ "$behind" = 0 ] || die "$current is $behind commit(s) behind origin/$current, pull first" + fi + tags="$(release_tags | sort -V)" +} + +print_tags() { + local released + [ "$OPTION" = --tags ] || return 0 + released="$(release_tags --points-at HEAD)" + moved_tags "$(grep -vxF -f <(echo "$released") <<< "$tags" || true)" "$tags" \ + | jq -Rnc '[inputs | split(" ") | {(.[0]): .[1]}] | add // {}' + exit 0 +} + +check_branch() { + [[ "$branch" =~ ^(main|next|v([0-9]+(\.[0-9]+)?)\.x|release/v([0-9]+\.[0-9]+))$ ]] || die "${branch:-a detached HEAD} releases nothing" + line="${BASH_REMATCH[2]}${BASH_REMATCH[4]}" + case "$branch" in + main) ok "main releases stable versions" ;; + next) ok "next releases prereleases" ;; + release/*) ok "$branch releases prereleases of $line.0" ;; + *) ok "$branch releases stable $line.x versions" ;; + esac +} + +check_version() { + local from + if [ "$channel" = stable ]; then + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "$VERSION_FILE sets '$version', expected major.minor.patch" + [ -z "$line" ] || [[ "$version" == "$line".* ]] || die "$VERSION_FILE sets $version, expected a $line.x version" + else + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$ ]] \ + || die "$VERSION_FILE sets '$version', expected major.minor.patch with an optional -alpha.N, -beta.N or -rc.N" + [ -z "$line" ] || [ "${version%%-*}" = "$line.0" ] || die "$VERSION_FILE sets $version, expected a prerelease of $line.0" + fi + last="$(release_tags --merged HEAD | tag_versions | awk -v channel="$channel" 'channel == "preview" || !/-/' | newest)" + earlier="$(with_version "$version" <<< "$tags" | tail -n 1)" + publish=false + if newer "$last" "$version"; then + info "$VERSION_FILE sets $version, $branch released $last already" + elif [ "$channel" = preview ] && [[ ! "$version" =~ -[a-z]+\.[1-9] ]]; then + info "$VERSION_FILE sets $version, $branch publishes numbered prereleases only" + elif [ "$channel" = preview ] && [ -n "$earlier" ]; then + ok "$VERSION_FILE sets $BOLD$version$RESET" + info "$earlier is published, $branch publishes a prerelease once" + else + from="${last:+$last → }" + [ "$version" != "$last" ] || from= + ok "$VERSION_FILE sets $from$BOLD$version$RESET" + publish=true + fi + if [ "$publish" = true ] && taken "${version%%-*}"; then + die "$VERSION_FILE sets $version, ${version%%-*} is taken by $holder" + fi + if [ "$publish" = true ] && in_progress "${version%%-*}"; then + warn "${version%%-*} is in progress, held by $holder" + fi +} + +select_releases() { + local name sqlite tag previous older moves listed added="" + releases= + [ "$publish" = true ] || return 0 + while read -r name sqlite; do + tag="${sqlite:+$sqlite-}v$version" + if grep -qxF "$tag" <<< "$tags"; then + info "$tag is published" + continue + fi + previous= + while read -r older; do + ! newer "$sqlite" "$older" || previous="$older" + done < <(with_version "$version" <<< "$tags" | sed -n 's/-v.*//p') + added+="$tag"$'\n' + releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ + '{$name, $version, $image, $tag, $previous}')"$'\n' + done < <(upstream_lines) + moves="$(moved_tags "$tags" "$tags"$'\n'"$added")" + for tag in $added; do + listed="$(moved_to "$tag")" + ok "$BOLD$tag$RESET is new, a push publishes it${listed:+ and moves $listed}" + done + awk 'NF { print $2 }' <<< "$moves" | sort -u | while read -r tag; do + grep -qxF "$tag" <<< "$added" || info "a push moves $(moved_to "$tag") to $tag" + done +} + +check_unreleased() { + local count + [ -n "$releases" ] && [ -n "$earlier" ] || return 0 + count="$(git rev-list --no-merges "$earlier..HEAD" -- src | grep -cvxF -f <(git rev-list "$earlier..HEAD" -- "$UPSTREAM_FILE") || true)" + [ "$count" = 0 ] || die "$count commit(s) since $earlier change src/ and would ship as $version, release them first with scripts/release.sh" +} + +print_upcoming_versions() { + local base stable upcoming + [ -z "$releases" ] || return 0 + if [ "$channel" = stable ]; then + base="$version" + ! newer "$last" "$base" || base="$last" + case "$branch" in + main) upcoming="$(next_version "$base" patch), $(next_version "$base" minor) or $(next_version "$base" major)" ;; + v*.*.x) upcoming="$(next_version "$base" patch)" ;; + *) upcoming="$(next_version "$base" patch) or $(next_version "$base" minor)" ;; + esac + elif [ -n "$line" ]; then + upcoming="a prerelease of $line.0" + else + stable="$(release_tags --merged HEAD | tag_versions | awk '!/-/' | newest)" + stable="${stable:-$(artifact_version stable)}" + upcoming="a prerelease of $(next_version "$stable" minor) or $(next_version "$stable" major)" + fi + info "the upcoming version on $branch is $upcoming, scripts/release.sh sets it" +} + +print_releases() { + [ "$OPTION" = --release ] || return 0 + jq -sc . <<< "$releases" +} + +artifact_version() { + jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "$VERSION_FILE" +} + +upstream_lines() { + if [ -z "$UPSTREAM" ]; then + echo latest + return + fi + jq -r --arg upstream "$UPSTREAM" '.[$upstream] | to_entries[] | "\(.key) \(.value.version)"' "$UPSTREAM_FILE" +} + +next_version() { + local major minor patch + IFS=. read -r major minor patch <<< "${1%%-*}" + while true; do + case "$2" in + patch) patch=$((patch + 1)) ;; + minor) minor=$((minor + 1)) patch=0 ;; + major) major=$((major + 1)) minor=0 patch=0 ;; + esac + taken "$major.$minor.$patch" || break + done + echo "$major.$minor.$patch" +} + +taken() { + local tag + for tag in $(release_tags "*v$1"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + [ "$channel" = preview ] && in_progress "$1" +} + +in_progress() { + local tag release="release/v${1%.0}" + if [ "$release" != "$branch" ] && git rev-parse --quiet --verify "origin/$release" > /dev/null; then + holder="$release" + [ "$channel" = stable ] || return 0 + git merge-base --is-ancestor "origin/$release" HEAD || return 0 + fi + for tag in $(release_tags "*v$1-*"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + return 1 +} + +moved_tags() { + moving_tags <<< "$2" | grep -vxF -f <(moving_tags <<< "$1") || true +} + +moving_tags() { + local tags releases latest preview base image sqlite prefix seen=" " + tags="$(cat)" + releases="$(sed -nE 's/^((.*)-)?v([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" + latest="$(awk 'NR == 1 && NF { print ($2 == "" ? "" : $2 "-") "v" $1 }' <<< "$releases")" + preview="$latest" + while read -r base; do + if newer "$base" "${latest##*v}"; then + preview="$(tag_versions <<< "$tags" | grep "^$base-" | newest)" + preview="$(with_version "$preview" <<< "$tags" | sort -V | tail -n 1)" + break + fi + done < <(tag_versions <<< "$tags" | sed -n 's/-.*//p' | sort -uV) + [ -z "$latest" ] || echo "latest $latest" + [ -z "$preview" ] || echo "preview $preview" + while read -r image sqlite; do + prefix="$sqlite" + while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do + echo "$prefix $sqlite-v$image" + seen+="$prefix " + [[ "$prefix" == *.* ]] || break + prefix="${prefix%.*}" + done + done <<< "$releases" +} + +release_tags() { + git tag --list "$@" | grep -E "^${UPSTREAM:+[^-]+-}v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true +} + +tag_versions() { + sed 's/.*v//' +} + +with_version() { + awk -v version="$1" '{ tagged = $0; sub(/.*v/, "", tagged) } tagged == version' +} + +moved_to() { + awk -v tag="$1" '$2 == tag { print $1 }' <<< "$moves" | paste -sd , - | sed 's/,/, /g; s/\(.*\), /\1 and /' +} + +newer() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$1" "$2" | newest)" = "$1" ] +} + +newest() { + sed 's/-/~/' | sort -V | sed 's/~/-/' | tail -n 1 +} + +main From 804798e25074ad7b2de106aea5bf79b231eba305 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:29:48 +0100 Subject: [PATCH 20/63] chore(scripts): add build.sh for local builds --- scripts/build.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 scripts/build.sh diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..01845ee --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +IMAGE="${1:-peakimages/sqlite:local}" +PLATFORM="${PLATFORM:-}" + +BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' + +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." + +main() { + [[ "$OPTION" =~ ^([^-][^[:space:]]*)?$ ]] || die "'$OPTION' is not an option, expected an image or none" + build +} + +build() { + local image_version + [ -z "${CI:-}" ] || die "scripts/build.sh is for local builds, CI builds with docker/build-push-action" + image_version="$(scripts/version.sh --build)" + docker buildx build ${PLATFORM:+--platform "$PLATFORM"} --build-arg IMAGE_VERSION="$image_version" \ + --provenance mode=max --sbom true --load --tag "$IMAGE" src + ok "$IMAGE is built for ${PLATFORM:-the platform of the Docker host} with image version $BOLD$image_version$RESET" +} + +main From d1eee699449349b5c1b40a317851662f4809ccb0 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:32:41 +0100 Subject: [PATCH 21/63] ci: merge main and release branches forward --- .gitattributes | 1 + .github/workflows/merge-forward.yaml | 92 ++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 .github/workflows/merge-forward.yaml diff --git a/.gitattributes b/.gitattributes index 3ecad78..8a01f67 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,7 @@ * text=auto eol=lf *.md diff=markdown +CHANGELOG.md merge=union /.github export-ignore .editorconfig export-ignore diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml new file mode 100644 index 0000000..77a197c --- /dev/null +++ b/.github/workflows/merge-forward.yaml @@ -0,0 +1,92 @@ +name: Merge forward + +on: + schedule: + - cron: "40 8,19 * * *" + workflow_dispatch: + workflow_call: + +permissions: {} + +cache-mode: none + +concurrency: + group: merge-forward + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + merge: + name: Merge forward + permissions: + contents: write # push the merges + pull-requests: write # open a pull request when a merge conflicts + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Merge main into the release branches and next, the release branches into next + env: + GH_TOKEN: ${{ github.token }} + BODY: | + The branches do not merge without conflicts. Resolve them on this branch, with the Resolve conflicts button of this pull request or with a merge of the base branch on a checkout of it. Both leave the source branch as it is. + + Then merge this pull request with a merge commit. A squash merge leaves the source branch unmerged, so the next merge conflicts again. + run: | + gh auth setup-git + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + push() { + if [ "$(git rev-parse HEAD)" != "$(git rev-parse "origin/$1")" ]; then + awk 'NR > 1 && /^#/ && previous != "" { print "" } { print; previous = $0 }' CHANGELOG.md > "$RUNNER_TEMP/CHANGELOG.md" + cp "$RUNNER_TEMP/CHANGELOG.md" CHANGELOG.md + git diff --quiet || git commit --quiet --amend --no-edit --all + fi + git push --quiet origin "HEAD:$2" + } + merge() { + local head="chore/merge-${1//\//-}-into-${2//\//-}" tag title conflicts pull_request + git switch --quiet --detach "origin/$2" + if git merge-base --is-ancestor "origin/$1" HEAD; then + echo "$2 contains $1 already" + return + fi + tag="$(git tag --points-at "origin/$1" | sed -n 1p)" + title="chore: merge ${tag:-$1} into $2" + if git merge --quiet --no-edit --message "$title" "origin/$1"; then + push "$1" "$2" + return + fi + conflicts="$(git diff --name-only --diff-filter=U)" + git merge --abort + if [ "$conflicts" = release.json ] && git merge --quiet --no-edit --strategy-option ours --message "$title" "origin/$1"; then + push "$1" "$2" + return + fi + if git rev-parse --verify --quiet "origin/$head" > /dev/null && ! git merge-base --is-ancestor "origin/$head" "origin/$1"; then + echo "::notice::$head has commits of its own, $1 follows once its pull request is merged" + return + fi + git push --quiet origin "origin/$1:refs/heads/$head" + pull_request="$(gh pr list --head "$head" --base "$2" --json url --jq '.[0].url')" + if [ -n "$pull_request" ]; then + gh pr edit "$pull_request" --title "$title" + else + gh pr create --base "$2" --head "$head" --title "$title" --body "$BODY" + fi + } + releases="$(git branch --remotes --list 'origin/release/*' --format='%(refname:lstrip=3)' | grep -vE '\.0$' | sort -V || true)" + for release in $releases; do + merge main "$release" + done + for release in $releases; do + merge "$release" next + done + merge main next From 71ec9dbfa80cc949e9eb8de1089040d36c0fe5d3 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:31:41 +0100 Subject: [PATCH 22/63] ci: use the new release process - split the build, test and release workflows - release on a push that changes release.json or src/upstream.json - select the releases with scripts/version.sh from release.json and src/upstream.json - build and test every upstream line for every platform - publish the tested images instead of a second build - publish the prereleases of next and the release branches as GitHub prereleases - take the notes of a first release from CHANGELOG.md and publish it right away, not as a draft - move the tags scripts/version.sh names on both registries and mark the GitHub release of latest - read the Docker Hub credentials only in jobs of the release environment - queue release runs instead of cancelling them, so no run is lost and no two runs move tags at once - run without the GitHub Actions cache, so no run restores a cache another run poisoned - run every step with bash, so a failing command in a pipe fails the step - merge forward after a release --- .github/workflows/build.yaml | 127 ++++++++++++++ .github/workflows/check-upstream.yaml | 4 +- .github/workflows/release.yaml | 237 ++++++++++++-------------- .github/workflows/tests.yaml | 69 ++++---- src/Dockerfile | 5 +- 5 files changed, 272 insertions(+), 170 deletions(-) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..6034b8a --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,127 @@ +name: Build + +on: + pull_request: + schedule: + - cron: "50 7 * * 1" + workflow_dispatch: + workflow_call: + inputs: + lines: + description: The upstream lines to build as JSON, every line of src/upstream.json when empty + type: string + default: "" + +permissions: {} + +cache-mode: none + +concurrency: + group: build-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + versions: + name: Read the versions + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + image: ${{ steps.versions.outputs.image }} + lines: ${{ inputs.lines || steps.versions.outputs.lines }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + persist-credentials: false + + - id: versions + env: + BRANCH: ${{ github.base_ref || github.ref_name }} + run: | + image="$(scripts/version.sh --build)" + lines="$(jq -c '[.sqlite | to_entries[] | {name: .key, version: .value.version}]' src/upstream.json)" + printf 'image=%s\nlines=%s\n' "$image" "$lines" >> "$GITHUB_OUTPUT" + + build: + name: Build ${{ matrix.line.name }} ${{ matrix.platform }} + needs: versions + permissions: + contents: read + runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images + timeout-minutes: 60 + strategy: + matrix: + line: ${{ fromJSON(needs.versions.outputs.lines) }} + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/riscv64] + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-26.04 + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-26.04-arm + - platform: linux/arm/v7 + arch: armv7 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/riscv64 + arch: riscv64 + runner: ubuntu-26.04 + qemu: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + persist-credentials: false + + - name: Set the build arguments + id: args + run: | + { + echo "epoch=$(git log -1 --format=%ct)" + echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_OUTPUT" + + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + if: ${{ matrix.qemu }} + with: + cache-image: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + with: + cache-binary: false + + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + with: + context: src + platforms: ${{ matrix.platform }} + tags: peakimages/sqlite:test + outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true + build-args: | + SQLITE_VERSION=${{ matrix.line.version }} + IMAGE_VERSION=${{ needs.versions.outputs.image }} + SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} + CREATED=${{ steps.args.outputs.created }} + GIT_SHA=${{ github.sha }} + provenance: mode=max + sbom: true + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # https://github.com/actions/upload-artifact/releases/tag/v7.0.1 + with: + name: image-${{ matrix.line.name }}-${{ matrix.arch }} + path: ${{ runner.temp }}/image + if-no-files-found: error + retention-days: 1 + + tests: + name: Tests + needs: [versions, build] + permissions: + contents: read + uses: $/.github/workflows/tests.yaml + with: + lines: ${{ needs.versions.outputs.lines }} diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index 72d8083..c4583e9 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -21,7 +21,7 @@ jobs: check: name: Check for a new SQLite release permissions: - actions: write # run the tests on the branch of the pull request + actions: write # start the build workflow on the branch of the pull request contents: write # push the branch of the pull request pull-requests: write # open the pull request runs-on: ubuntu-26.04 # https://github.com/actions/runner-images @@ -65,7 +65,7 @@ jobs: git commit --all --quiet --message "$title" git push --quiet --set-upstream origin "$branch" new_pr="$(gh pr create --base main --head "$branch" --title "$title" --label bot/upstream --body-file "$RUNNER_TEMP/body.md")" - gh workflow run tests.yaml --ref "$branch" + gh workflow run build.yaml --ref "$branch" for open_pr in $(gh pr list --label bot/upstream --json url --jq '.[].url'); do [ "$open_pr" = "$new_pr" ] || gh pr close "$open_pr" --delete-branch --comment "Superseded by $new_pr" done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6a18f72..4b00e79 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,99 +1,87 @@ name: Release on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true + push: + branches: + - main + - next + - v[0-9]+.x + - v[0-9]+.[0-9]+.x + - release/v[0-9]+.[0-9]+ + paths: + - release.json + - src/upstream.json + workflow_dispatch: permissions: {} -concurrency: release +cache-mode: none + +concurrency: + group: release + cancel-in-progress: false + queue: max + +defaults: + run: + shell: bash jobs: version: + name: Select the releases + if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images - timeout-minutes: 30 + timeout-minutes: 10 outputs: - publish: ${{ steps.version.outputs.publish }} - release: ${{ steps.version.outputs.release }} - sqlite: ${{ steps.version.outputs.sqlite }} - image: ${{ steps.version.outputs.image }} - draft: ${{ steps.version.outputs.draft }} - tags: ${{ steps.version.outputs.tags }} + releases: ${{ steps.version.outputs.releases }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: + fetch-depth: 0 persist-credentials: false - - name: Read the versions - id: version + - id: version env: - GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.ref_name }} run: | - sqlite="$(jq -r .sqlite.latest.version src/upstream.json)" - image="$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)" - release="$sqlite-v$image" - publish=true - if gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$release" > /dev/null 2>&1 || gh release view "$release" > /dev/null 2>&1; then - echo "::notice::$release is published already, nothing to release" - publish=false - fi - # the first release of an image version is a draft with the notes from CHANGELOG.md - released="$(gh api --paginate "repos/$GITHUB_REPOSITORY/tags" --jq ".[].name | select(endswith(\"-v$image\"))")" - draft=false - [ -n "$released" ] || draft=true - { - echo "publish=$publish" - echo "release=$release" - echo "sqlite=$sqlite" - echo "image=$image" - echo "draft=$draft" - echo "tags=$release $sqlite ${sqlite%.*} ${sqlite%%.*} latest" - } >> "$GITHUB_OUTPUT" + releases="$(scripts/version.sh --release)" + echo "releases=$releases" | tee -a "$GITHUB_OUTPUT" build: + name: Build needs: version - if: needs.version.outputs.publish == 'true' + if: ${{ needs.version.outputs.releases != '[]' }} permissions: contents: read - packages: write - runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images - timeout-minutes: 60 + uses: $/.github/workflows/build.yaml + with: + lines: ${{ needs.version.outputs.releases }} + + publish: + name: Publish ${{ matrix.release.tag }} + needs: [version, build] + permissions: + contents: write # create the GitHub release and its tag + packages: write # push the tested images to ghcr.io + environment: + name: release + deployment: false + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 strategy: matrix: - include: - - platform: linux/amd64 - runner: ubuntu-26.04 - - platform: linux/arm64 - runner: ubuntu-26.04-arm - - platform: linux/arm/v7 - runner: ubuntu-26.04-arm - qemu: true - - platform: linux/riscv64 - runner: ubuntu-26.04 - qemu: true + release: ${{ fromJSON(needs.version.outputs.releases) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: persist-credentials: false - - name: Read the commit date - id: commit - run: | - { - echo "epoch=$(git log -1 --format=%ct)" - echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" - } >> "$GITHUB_OUTPUT" - - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 - if: matrix.qemu - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # https://github.com/actions/download-artifact/releases/tag/v8.0.1 + with: + pattern: image-${{ matrix.release.name }}-* + path: ${{ runner.temp }}/images - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 with: @@ -106,56 +94,59 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 - id: build - with: - context: src - platforms: ${{ matrix.platform }} - tags: | - ghcr.io/${{ github.repository }} - docker.io/${{ github.repository }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true,rewrite-timestamp=true - build-args: | - SOURCE_DATE_EPOCH=${{ steps.commit.outputs.epoch }} - CREATED=${{ steps.commit.outputs.created }} - GIT_SHA=${{ github.sha }} - provenance: mode=max - sbom: true - - - name: Save the digest + - name: Push the tested images env: - DIGEST: ${{ steps.build.outputs.digest }} + TAG: ${{ matrix.release.tag }} run: | - mkdir "$RUNNER_TEMP/digests" - touch "$RUNNER_TEMP/digests/${DIGEST#sha256:}" + sources=() + for index in "$RUNNER_TEMP"/images/*/index.json; do + sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") + done + for repository in ghcr.io/peakimages/sqlite docker.io/peakimages/sqlite; do + docker buildx imagetools create --tag "$repository:$TAG" "${sources[@]}" + done - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # https://github.com/actions/upload-artifact/releases/tag/v7.0.1 - with: - name: digest-${{ strategy.job-index }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ matrix.release.tag }} + IMAGE_VERSION: ${{ matrix.release.image }} + SQLITE_VERSION: ${{ matrix.release.version }} + PREVIOUS: ${{ matrix.release.previous }} + run: | + args=(--title "$TAG" --target "$GITHUB_SHA" --notes-file "$RUNNER_TEMP/notes.md" --latest=false) + heading="## [v$IMAGE_VERSION]" + [[ "$IMAGE_VERSION" != *-* ]] || { args+=(--prerelease); heading="## [Unreleased]"; } + if [ -z "$PREVIOUS" ]; then + awk -v heading="$heading" 'index($0, heading) == 1 { inside = 1; next } /^## / { inside = 0 } inside { sub(/^### /, "## "); print }' CHANGELOG.md > "$RUNNER_TEMP/notes.md" + else + cat > "$RUNNER_TEMP/notes.md" < "$RUNNER_TEMP/notes.md" - args=(--draft) - else - echo "SQLite $SQLITE, release notes: https://sqlite.org/releaselog/${SQLITE//./_}.html" > "$RUNNER_TEMP/notes.md" - fi - gh release create "$RELEASE" "${args[@]}" --target "$GITHUB_SHA" --title "$RELEASE" --notes-file "$RUNNER_TEMP/notes.md" + merge: + name: Merge forward + needs: publish + permissions: + contents: write # push the merges + pull-requests: write # open a pull request when a merge conflicts + uses: $/.github/workflows/merge-forward.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d74c4c9..0050b67 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,27 +1,24 @@ name: Tests on: - push: - branches: [main, next] - paths: - - src/** - - scripts/** - - .github/** - - .editorconfig - pull_request: - paths: - - src/** - - scripts/** - - .github/** - - .editorconfig - schedule: - - cron: "30 6 * * 1" - workflow_dispatch: + workflow_call: + inputs: + lines: + description: The upstream lines the build workflow built as JSON + type: string + required: true permissions: {} +cache-mode: none + +defaults: + run: + shell: bash + jobs: lint: + name: Lint permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images @@ -34,23 +31,29 @@ jobs: - run: scripts/lint.sh test: - needs: lint + name: Test ${{ matrix.line.name }} ${{ matrix.platform }} permissions: contents: read runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images - timeout-minutes: 60 + timeout-minutes: 30 strategy: fail-fast: false matrix: + line: ${{ fromJSON(inputs.lines) }} + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/riscv64] include: - platform: linux/amd64 + arch: amd64 runner: ubuntu-26.04 - platform: linux/arm64 + arch: arm64 runner: ubuntu-26.04-arm - platform: linux/arm/v7 + arch: armv7 runner: ubuntu-26.04-arm qemu: true - platform: linux/riscv64 + arch: riscv64 runner: ubuntu-26.04 qemu: true steps: @@ -58,27 +61,17 @@ jobs: with: persist-credentials: false - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 - if: matrix.qemu - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # https://github.com/actions/download-artifact/releases/tag/v8.0.1 + with: + name: image-${{ matrix.line.name }}-${{ matrix.arch }} + path: ${{ runner.temp }}/image - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + if: ${{ matrix.qemu }} with: - context: src - platforms: ${{ matrix.platform }} - load: true - tags: peakimages/sqlite:test + cache-image: false - - run: scripts/test.sh peakimages/sqlite:test + - name: Load the image + run: tar -cf - -C "$RUNNER_TEMP/image" . | docker load - release: - needs: test - if: github.repository == 'peakimages/sqlite' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - permissions: - contents: write - packages: write - uses: $/.github/workflows/release.yaml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - run: scripts/test.sh peakimages/sqlite:test diff --git a/src/Dockerfile b/src/Dockerfile index 432ef33..6449fd5 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -2,8 +2,6 @@ # Set by scripts/check-upstream.sh ARG SQLITE_VERSION=3.54.1 -# Set by scripts/release.sh -ARG IMAGE_VERSION=0.1.0 # Build dependencies # https://hub.docker.com/_/alpine/tags?name=3.24.3 @@ -13,7 +11,8 @@ ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 -# ARGs set by the release workflow +# Set by the build workflow +ARG IMAGE_VERSION ARG SOURCE_DATE_EPOCH ARG CREATED ARG GIT_SHA From 416684479df4637dc368706dfaa844fbcdf990a8 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:32:26 +0100 Subject: [PATCH 23/63] ci(release): check every SQLite line in check-upstream.yaml --- .github/workflows/check-upstream.yaml | 57 +++++++++++++++++++++------ 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index c4583e9..bc9eefe 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -2,7 +2,7 @@ name: Check upstream on: schedule: - - cron: "30 6,18 * * *" + - cron: "20 7,18 * * *" workflow_dispatch: permissions: {} @@ -18,42 +18,73 @@ defaults: shell: bash jobs: + lines: + name: List the lines of every branch + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + lines: ${{ steps.lines.outputs.lines }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - id: lines + run: | + for branch in $(git branch --remotes --format='%(refname:lstrip=3)' | grep -xE 'main|v[0-9]+\.x|release/v[0-9]+\.0'); do + git show "origin/$branch:src/upstream.json" | jq -c --arg branch "$branch" '.sqlite | keys[] | {$branch, line: .}' + done | jq -sc '"lines=\(.)"' -r >> "$GITHUB_OUTPUT" + check: - name: Check for a new SQLite release + name: Check SQLite ${{ matrix.line }} on ${{ matrix.branch }} + needs: lines permissions: actions: write # start the build workflow on the branch of the pull request contents: write # push the branch of the pull request pull-requests: write # open the pull request runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.lines.outputs.lines) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: + ref: ${{ matrix.branch }} persist-credentials: false - - run: scripts/check-upstream.sh + - env: + LINE: ${{ matrix.line }} + run: scripts/check-upstream.sh "$LINE" - name: Open a pull request env: GH_TOKEN: ${{ github.token }} + BASE: ${{ matrix.branch }} + LINE: ${{ matrix.line }} run: | git diff --quiet && exit 0 - current_version="$(git show HEAD:src/upstream.json | jq -r .sqlite.latest.version)" - new_version="$(jq -r .sqlite.latest.version src/upstream.json)" - source_id="$(jq -r .sqlite.latest.source_id src/upstream.json)" - check_in="${source_id##* }" - branch="feat/sqlite-v$new_version" - title="feat(release): bump SQLite version from v$current_version to v$new_version" + pin() { jq -r --arg line "$LINE" ".sqlite[\$line].$1"; } + current="$(git show HEAD:src/upstream.json | pin version)" + version="$(pin version < src/upstream.json)" + source_id="$(pin source_id < src/upstream.json)" + branch="feat/sqlite-v$version" + [ "$BASE" = main ] || branch="$branch-${BASE//\//-}" + title="feat(release): bump SQLite version from v$current to v$version" if [ -n "$(gh pr list --head "$branch" --json number --jq '.[0].number')" ]; then echo "a pull request from $branch is open already" exit 0 fi cat > "$RUNNER_TEMP/body.md" < [!IMPORTANT] > Squash merge this PR to keep the commit history clean. @@ -64,8 +95,8 @@ jobs: git switch --create "$branch" git commit --all --quiet --message "$title" git push --quiet --set-upstream origin "$branch" - new_pr="$(gh pr create --base main --head "$branch" --title "$title" --label bot/upstream --body-file "$RUNNER_TEMP/body.md")" + new_pr="$(gh pr create --base "$BASE" --head "$branch" --title "$title" --label bot/upstream --body-file "$RUNNER_TEMP/body.md")" gh workflow run build.yaml --ref "$branch" - for open_pr in $(gh pr list --label bot/upstream --json url --jq '.[].url'); do + for open_pr in $(gh pr list --label bot/upstream --base "$BASE" --json url,title | jq -r --arg from "from v$current to" '.[] | select(.title | contains($from)) | .url'); do [ "$open_pr" = "$new_pr" ] || gh pr close "$open_pr" --delete-branch --comment "Superseded by $new_pr" done From 82208f057d1e38ef43c69c674d98b7d207a901ce Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:46:35 +0200 Subject: [PATCH 24/63] chore(lint): run jactionlint pedantic and docker build checks --- .github/jactionlint.yaml | 1 + scripts/lint.sh | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 .github/jactionlint.yaml diff --git a/.github/jactionlint.yaml b/.github/jactionlint.yaml new file mode 100644 index 0000000..07d42f9 --- /dev/null +++ b/.github/jactionlint.yaml @@ -0,0 +1 @@ +profile: pedantic diff --git a/scripts/lint.sh b/scripts/lint.sh index 0ec40ef..ec8eded 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -2,25 +2,28 @@ set -euo pipefail OPTION="$*" +# https://github.com/hadolint/hadolint/releases/tag/v2.15.1 HADOLINT=hadolint/hadolint:v2.15.1 +# https://github.com/koalaman/shellcheck/releases/tag/v0.11.0 SHELLCHECK=koalaman/shellcheck:v0.11.0 +# https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -ACTIONLINT=kjanat/actionlint:1.17.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.2.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.2.0 +# https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 -BOLD=$'\033[1m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } -err() { printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2; } die() { - err "$@" + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 exit 1 } cd "$(dirname "$0")/.." -failed=0 main() { [[ "$OPTION" =~ ^(--fix)?$ ]] || die "'$OPTION' is not an option, expected --fix or none" @@ -28,12 +31,12 @@ main() { docker info > /dev/null 2>&1 || die "docker is not running" [ "$OPTION" != --fix ] || format heading "Linting" + run_build_checks check src/Dockerfile hadolint "$HADOLINT" hadolint src/Dockerfile check "the shell scripts" ShellCheck "$SHELLCHECK" scripts/*.sh src/*.sh src/rootfs/usr/local/bin/*.sh check "the shell scripts" shfmt "$SHFMT" -d scripts src - check "the workflows" actionlint "$ACTIONLINT" + check "the workflows" jactionlint "$JACTIONLINT" --strict-exit check "every file" editorconfig-checker "$EDITORCONFIG_CHECKER" editorconfig-checker -exclude '^LICENSE$' - [ "$failed" = 0 ] || die "$failed of 5 checks failed" heading "All checks passed" } @@ -44,15 +47,16 @@ format() { ok "shfmt and editorconfig-checker wrote their fixes" } +run_build_checks() { + docker build --check --progress=quiet src | sed 1d || die "src/Dockerfile failed docker build --check" + ok "src/Dockerfile passed docker build --check" +} + check() { local files="$1" linter="$2" image="$3" shift 3 - if tool "$image" "$@"; then - ok "$files passed $linter" - else - err "$files failed $linter" - failed=$((failed + 1)) - fi + tool "$image" "$@" || die "$files failed $linter" + ok "$files passed $linter" } tool() { From 03cf82ac4f9d80cf8c5cbb33ed2e2d4409e6dace Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:19:37 +0100 Subject: [PATCH 25/63] style: format Markdown with prettier --- .editorconfig | 3 +++ SECURITY.md | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.editorconfig b/.editorconfig index 56e2f56..d843ad9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,3 +19,6 @@ indent_style = space [*.{yml,yaml,md}] indent_style = space indent_size = 2 + +[*.md] +indent_size = unset diff --git a/SECURITY.md b/SECURITY.md index 12cd03f..4734f57 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,7 +28,7 @@ If you find a security vulnerability in a currently supported version, please fo 1. **DO NOT** disclose the vulnerability publicly. 2. Report it via [GitHub Security Advisories](https://github.com/peakimages/sqlite/security/advisories/new). 3. Include in your report: - - **Explanation:** Detailed explanation of the vulnerability. - - **Affected Versions:** Which image tags are affected. - - **Reproduction:** Clear steps to reproduce the vulnerability. - - **Impact:** Assessment of the potential impact of the vulnerability. + - **Explanation:** Detailed explanation of the vulnerability. + - **Affected Versions:** Which image tags are affected. + - **Reproduction:** Clear steps to reproduce the vulnerability. + - **Impact:** Assessment of the potential impact of the vulnerability. From 743f427cea8517014324a0642a85764d54122c07 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:33:51 +0100 Subject: [PATCH 26/63] docs: add RELEASE.md documenting the new release process --- RELEASE.md | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..a9bc84e --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,138 @@ +# Releases + +Every release is tagged `-v`, such as `3.54.1-v1.4.0-beta.1`. + +> [!IMPORTANT] +> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/version.sh` first. + +## Rules + +| Branch | Takes | Releases | +| -------------- | ---------------------------------------------------------- | -------------------------------------- | +| `main` | fixes, and `next` or a release branch for a new version | stable versions | +| `next` | features | prereleases of the next minor or major | +| `release/v1.4` | the minor 1.4, split off `next` so `next` can go on to 1.5 | prereleases of `1.4.0` | +| `release/v2.0` | the next major, cut from `next` | prereleases of `2.0.0` | +| `v1.x` | backports, once `main` is on v2 | stable `1.x` versions | +| `v1.2.x` | backports and hotfixes, cut from the newest tag of v1.2 | stable `1.2.x` versions | + +- `main` takes fixes only. A new minor or major reaches it as a merge of `next` or a release branch +- `next` and the release branches publish prereleases only, their stable versions are released on `main` +- Name a branch after what it changes: `fix/`, `feat/`, and `fix/-v1.2.2` for a backport, after the version that releases it +- Squash merge pull requests from contributors and pull requests with a single change, such as SQLite updates +- Merge everything else with a merge commit: team pull requests with several commits, backports, `next`, the release branches and `v1.3.x` into `main` +- The merge forward workflow merges `main` into `release/v1.4` and `next`, and `release/v1.4` into `next` +- While `main` is on 1.3, `v1.1.x` and `v1.2.x` are never merged into another branch +- Delete `release/v1.4` once `main` has released `1.4.0` +- Delete `v1.3.x` once it is merged into `main` +- Delete `v1.1.x` and `v1.2.x` once their release is published, the next backport cuts them again from the newest tag + +> [!CAUTION] +> Never squash merge `next` or a release branch. Their commits stay unmerged and come back as conflicts in every later merge. + +## Create a release + +> [!NOTE] +> `next` publishes prereleases only. For its stable version, merge `next` into `main` with a merge commit and create the release on `main`. + +1. Describe the changes under `## [Unreleased]` in `CHANGELOG.md`, they become the release notes +2. Check what the branch releases: + + ```sh + scripts/version.sh + ``` + +3. Select the stage and the version, the script commits them and asks to push: + + ```sh + scripts/release.sh + ``` + +4. The pushed commit changes `release.json`, so it runs the release workflow, which builds, tests and publishes the release and moves its tags + +> [!TIP] +> To take the release commit back before the push: +> +> ```sh +> scripts/undo-release.sh +> ``` + +## Backport a fix + +`main` is on 1.3, and 1.1 and 1.2 still get fixes. A fix reaches each of them through a patch branch and a pull request of its own: + +1. Cut the patch branch from the newest tag of the minor, unless it exists: + + ```sh + git switch -c v1.2.x 3.54.1-v1.2.1 && git push -u origin v1.2.x + ``` + +2. Put the fix on a branch named after the version that releases it, cherry-picked from `main` when it is there already: + + ```sh + git switch -c fix/-v1.2.2 origin/v1.2.x && git cherry-pick -x + ``` + +3. Merge a pull request from `fix/-v1.2.2` into `v1.2.x` with a merge commit +4. Create the release on your checkout of `v1.2.x`: + + ```sh + git switch v1.2.x && git pull && scripts/release.sh + ``` + +5. Repeat with `v1.1.x` and `fix/-v1.1.4`. 1.0 gets no backports + +> [!IMPORTANT] +> A hotfix for 1.3 while `main` holds unreleased changes goes through `v1.3.x` the same way. Merge `v1.3.x` into `main` with a merge commit afterwards and delete it. `v1.1.x` and `v1.2.x` are never merged, their fixes come from `main`. + +## Add an older SQLite line + +1. Pin the line on a branch from `main`: + + ```sh + git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh 3.53 + ``` + +2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `3.53.4-v1.3.0` +3. To stop the line, remove `3.53` from `src/upstream.json`, its tags stay where they are + +## Release a major version + +1. Cut `release/v2.0` from `next` and create its prereleases there with `scripts/release.sh`: + + ```sh + git switch -c release/v2.0 origin/next && git push -u origin release/v2.0 + ``` + +2. Merge `main` into `release/v2.0`, so v2 has every fix of v1 and the pull request into `main` has no conflicts: + + ```sh + git switch release/v2.0 && git pull && git merge origin/main && git push + ``` + +3. Keep v1 on a branch of its own: + + ```sh + git switch -c v1.x origin/main && git push -u origin v1.x + ``` + +4. Merge a pull request from `release/v2.0` into `main` with a merge commit +5. Create the release on `main` and select `2.0.0`: + + ```sh + git switch main && git pull && scripts/release.sh + ``` + +6. Delete `release/v2.0` + +> [!IMPORTANT] +> Cut `v1.x` from `main`, or bring an existing one up to date with `main`, before `release/v2.0` is merged into `main`. Afterwards `main` is v2 and v1 lives on `v1.x` alone. + +## Tags + +| Tag | Points at | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `3.54.1-v1.3.0` | one release, it never moves | +| `latest`, `3.54.1`, `3.54`, `3` | `3.54.1-v1.3.0`, the newest SQLite version of the newest stable image version | +| `3.53.4`, `3.53` | the newest stable image version released with SQLite 3.53, also once the line stops | +| `preview` | the newest prerelease of the lowest version in prerelease above `latest`, else `latest` | From 411a00c48ef7fdd235101403f400233e396397fd Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:32:20 +0100 Subject: [PATCH 27/63] build!: compress the image layers with zstd - zstd is faster, more modern and compresses better than gzip, pulling needs Docker Engine 23.0 or later --- .github/workflows/build.yaml | 2 +- README.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6034b8a..c761be4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -100,7 +100,7 @@ jobs: context: src platforms: ${{ matrix.platform }} tags: peakimages/sqlite:test - outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true + outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true build-args: | SQLITE_VERSION=${{ matrix.line.version }} IMAGE_VERSION=${{ needs.versions.outputs.image }} diff --git a/README.md b/README.md index 9a17b8d..7b20e27 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,9 @@ To see a comparison of this image with other popular SQLite Docker images, see [ SQLite is not a server and the database is just a file. The application reads and writes it with the SQLite engine it bundles, such as `pdo_sqlite` for PHP or `rusqlite` for Rust, so the SQLite version and extensions of that engine apply, not the ones of this image. This container shares the volume with the application and looks after the files: it creates and configures them, health checks them and brings the `sqlite3` shell for backups, restores and maintenance. +> [!NOTE] +> The layers of the image are compressed with zstd, pulling it needs Docker Engine 23.0 or later. + ### Docker Start the container with a named volume for the default database: From 9b5897082c1506473f137af2e536cdc78f2dde12 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:35:08 +0100 Subject: [PATCH 28/63] fix(image): list SQLite, tini and BusyBox in the SBOM - a scratch image has no package database, sbom.sh writes a syft JSON document into /var/lib/db/sbom for the scanner - a build without IMAGE_VERSION stops in the build stage --- src/Dockerfile | 7 +++++ src/sbom.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/sbom.sh diff --git a/src/Dockerfile b/src/Dockerfile index 6449fd5..b039552 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -20,6 +20,7 @@ ARG GIT_SHA FROM ${BASE_IMAGE} AS build ARG SQLITE_VERSION +ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION @@ -80,6 +81,12 @@ COPY rootfs/ /rootfs/ COPY rootfs.sh /src/ RUN sh /src/rootfs.sh +# A scratch image has no package database for the SBOM scanner to read, this syft JSON document takes its place +COPY sbom.sh /src/ +RUN sh /src/sbom.sh "peakimages/sqlite:${SQLITE_VERSION}-v${IMAGE_VERSION:?}" \ + "generic|sqlite|${SQLITE_VERSION}|blessing|$(jq -r .url /src/sqlite.json)|sqlite:sqlite" \ + "apk|tini-static|tini_project:tini|/sbin/tini" \ + "apk|busybox-static|busybox:busybox|/bin/busybox" FROM scratch AS final diff --git a/src/sbom.sh b/src/sbom.sh new file mode 100644 index 0000000..f448274 --- /dev/null +++ b/src/sbom.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -eu + +USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" +IMAGE="${1:-}" +REPOSITORY="${IMAGE%:*}" +OUTPUT_FILE="/rootfs/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" + +die() { + echo "sbom: $*" >&2 + exit 1 +} + +main() { + [ "$#" -gt 1 ] || die "$USAGE" + shift + write_document "$@" + verify_files + echo "sbom: $OUTPUT_FILE lists $(jq -r '[.artifacts[] | "\(.name) \(.version)"] | join(", ")' "$OUTPUT_FILE")" +} + +write_document() { + mkdir -p "${OUTPUT_FILE%/*}" + cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" ' + ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os + | [ + split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries + | select(.Status | . == null or endswith(" installed")) + ] as $installed + | def installed($key; $name): first($installed[] | select(.[$key] == $name)) // error("sbom: \($name) is not installed"); + def purl($path; $version; $qualifiers): "pkg:\($path)@\($version)?" + ([$qualifiers | to_entries[] | select(.value | length > 0) | "\(.key)=\(.value | @uri)"] | join("&")); + def os_purl($type; $name; $version; $arch; $upstream): purl("\($type)/\($os.ID)/\($name)"; $version; { arch: $arch, distro: "\($os.ID)-\($os.VERSION_ID)", upstream: $upstream }); + def package($type; $name; $version; $license; $cpe; $purl): { + name: $name, + version: $version, + type: $type, + licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], + cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], + purl: $purl + }; + def apk($name; $cpe; $paths): + installed("P"; $name) + | package("apk"; .P; .V; .L; $cpe; os_purl("apk"; .P; .V; .A; if .o != .P then .o else "" end)) + { + metadataType: "apk-db-entry", + metadata: { package: .P, originPackage: .o, version: .V, architecture: .A, files: [$paths[] | { path: . }] } + }; + def deb($name; $license; $cpe; $paths): + installed("Package"; $name) + | package("deb"; .Package; .Version; $license; $cpe; os_purl("deb"; .Package; .Version; .Architecture; .Source // "" | sub(" \\((?.*)\\)$"; "@\(.version)"))) + { + metadataType: "dpkg-db-entry", + metadata: { package: .Package, source: (.Source // "" | sub(" .*"; "")), version: .Version, architecture: .Architecture, files: [$paths[] | { path: . }] } + }; + def generic($name; $version; $license; $url; $cpe): + package("binary"; $name; $version; $license; $cpe; purl("generic/\($name)"; $version; { download_url: $url })); + { + artifacts: [ + $ARGS.positional[] | split("|") | . as [$type, $name] + | if $type == "apk" and length > 3 then apk($name; .[2]; .[3:]) + elif $type == "deb" and length > 4 then deb($name; .[2]; .[3]; .[4:]) + elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) + else error("sbom: \(join("|")) is not a component, \($usage)") end + ], + schema: { version: "16.1.10", url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.1.10.json" } + }' --args "$@" > "$OUTPUT_FILE" +} + +verify_files() { + jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do + [ -f "/rootfs$path" ] || die "$path is not a file in /rootfs" + done +} + +main "$@" From 3e493bb7128797136497d84a6524b408288295f2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:37:42 +0100 Subject: [PATCH 29/63] docs: explain the UPX packing of grepular/sqlite3 --- docs/comparison.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/comparison.md b/docs/comparison.md index b342c4a..4266844 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -23,10 +23,10 @@ | No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | | No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | | No runtime extension loading[^8] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Compressed[^9] | 1.55 MB | 0.57 MB | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | -| On disk[^9] | 2.7 MB | 0.57 MB | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | +| Compressed[^9] | 1.55 MB | 0.57 MB[^10] | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | +| On disk[^9] | 2.7 MB | 0.57 MB[^10] | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | | Platforms | amd64, arm64, arm/v7, riscv64 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | -| Extensions[^10] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | +| Extensions[^11] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | | SQLite updates | ✅ (checked twice a day, merged by hand) | ✅ (checked daily, built automatically) | ⚠️ (checked weekly, merged by hand) | ⚠️ (follows Alpine releases) | ⚠️ (policy not public) | ❌ (3.34.1, 2021) | ❌ (3.46.1, 2024) | ❌ (3.28.0, 2019) | | Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | | OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | @@ -49,4 +49,6 @@ [^9]: Measured for `linux/amd64`. -[^10]: JSON has been built into SQLite since 3.38.0 and is not listed. A distribution package ships the extensions its distribution chose, and SQLite before 3.35.0 has no math functions. +[^10]: `grepular/sqlite3` packs its binary with UPX, which unpacks it into memory on every start: every `sqlite3` run, a health check too, starts about 200 times slower (25 ms instead of 0.1 ms on arm64), every process keeps its own unpacked copy in memory instead of sharing one, and antivirus scanners may flag the file, as malware often uses UPX. It pays on every start to save on size, on a download that happens once. + +[^11]: JSON has been built into SQLite since 3.38.0 and is not listed. A distribution package ships the extensions its distribution chose, and SQLite before 3.35.0 has no math functions. From 5322df8dd67cfb0dadf5e948fe24993fc7611e67 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:54:29 +0100 Subject: [PATCH 30/63] ci: bump GitHub Actions to the latest releases --- .github/workflows/build.yaml | 2 +- .github/workflows/release.yaml | 10 +++++----- .github/workflows/tests.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c761be4..bca8880 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -110,7 +110,7 @@ jobs: provenance: mode=max sbom: true - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # https://github.com/actions/upload-artifact/releases/tag/v7.0.1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4b00e79..a4eb822 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,18 +78,18 @@ jobs: with: persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # https://github.com/actions/download-artifact/releases/tag/v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 with: pattern: image-${{ matrix.release.name }}-* path: ${{ runner.temp }}/images - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -147,13 +147,13 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0050b67..1369136 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,7 +61,7 @@ jobs: with: persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # https://github.com/actions/download-artifact/releases/tag/v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image From c99924fe46c77960b10a471ebcc9f112f5f535dc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:54:32 +0100 Subject: [PATCH 31/63] chore(lint): bump jactionlint from v2.2.0 to v2.3.0 --- scripts/lint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lint.sh b/scripts/lint.sh index ec8eded..9f65424 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -8,8 +8,8 @@ HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 # https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -# https://github.com/jdx/jactionlint/releases/tag/v2.2.0 -JACTIONLINT=ghcr.io/jdx/jactionlint:2.2.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.3.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.3.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 From 4ee4abd369a4953c892ff974acfa4abdd29b66e3 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:31:10 +0100 Subject: [PATCH 32/63] docs: add v0.2.0 changes to changelog --- CHANGELOG.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3937d9..072fcb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,68 @@ All notable changes to this project will be documented in this file. SQLite vers ## [Unreleased] +### Breaking Changes + +- Compress the layers with zstd instead of gzip, zstd is faster, more modern and compresses better than gzip, pulling needs Docker Engine 23.0 or later + +### Added + +- Add `.recover` and `.dbinfo` to the shell and the `sqlite_dbpage` and `sqlite_stmt` virtual tables, the shell is built with the options sqlite.org recommends for a full-featured shell +- Build reproducibly, a rebuild of the same commit gives the same image digest per platform, and the `created` date of the image and its `org.opencontainers.image.created` label are the date of that commit +- Publish prereleases from `next` and the release branches, tagged like `3.54.1-v0.2.0-beta.1`, and the moving `preview` tag +- Publish older SQLite lines such as `3.53` next to the newest, built with the same image version and tagged like `3.53.4-v0.2.0`, `3.53.4` and `3.53` + +### Changed + +- Let any uid write the databases, `/var/lib/sqlite` is `0777` and the entrypoint creates databases with mode `0666`, which their `-wal` and `-shm` files take, so the application can run as any user and not only as `65532` or root +- Replace the release process, `RELEASE.md` describes it + - Keep the image versions in `release.json`, `stable.image` for `main` and the maintenance branches and `preview.image` for `next` and the release branches + - Pin SQLite in `src/upstream.json`, one entry per line, instead of `src/version.json` + - Release from `main`, `next`, release branches such as `release/v0.3` and maintenance branches such as `v1.x` and `v0.1.x` + - Select the stage and the version with `scripts/release.sh`, which offers the versions the branch allows and preselects the bump the commits need + - Decide the image version, the releases and the moving tags with `scripts/version.sh`, which also reports what a push publishes + - Publish every release a push of `release.json` or `src/upstream.json` names and whose tag does not exist yet, never as a draft + - Move `latest`, `preview` and every prefix of a SQLite version to their place among all published tags, so a release of an older line never moves a tag back + - Check every SQLite line of `main`, the maintenance branches and the major release branches twice a day and open one pull request per bump, which supersedes the older ones + - Verify a new SQLite release against its release notes and its tagged check-in on sqlite.org + - Merge `main` forward into `next` and the release branches twice a day and after every release +- Rework the CI workflows for the new release process + - Split the build, test and release workflows, the release workflow calls the build workflow and the build workflow calls the tests + - Build every SQLite line for every platform once, test those builds and publish exactly them, not a second build + - Write the notes of the first release of an image version from `CHANGELOG.md`, later releases name the SQLite update + - Mark the GitHub release that `latest` points at as the latest release +- Harden the CI workflows + - Grant every job the least permissions it needs and state the reason for every write permission + - Keep the Docker Hub credentials in the `release` environment + - Run without the GitHub Actions cache, layer caches and cached tool binaries, so no run restores what another run wrote + - Run every step with bash, so a failing command in a pipe fails the step + - Queue release runs instead of cancelling them, so no two runs publish or move tags at the same time + - Lint the workflows with jactionlint pedantic, which includes the audits of zizmor, and fail on every warning +- Remove the `SQLITE_UID` and `SQLITE_GID` build arguments, the user is `65532:65532` +- Rename the build argument `BUILD_DATE` to `CREATED`, add `SOURCE_DATE_EPOCH` and require `IMAGE_VERSION` +- Bump Alpine in the build stage from 3.24.2 to 3.24.3 + +### Fixed + +- List SQLite, tini and BusyBox in the SBOM attestation with their versions, licenses, package URLs and CPEs, from an SBOM document in `/var/lib/db/sbom` that takes the place of the package database a scratch image lacks +- Match `LICENSE` to the AGPL-3.0 text on gnu.org + +### Refactored + +- Name the final stage of the Dockerfile `final` + +### Maintenance + +- Add `scripts/build.sh` to build the image locally with the image version CI builds with, for the platform of the Docker host or the platforms `PLATFORM` names +- Run docker build checks in `scripts/lint.sh` and stop at the first failed check +- Stop every script on arguments it does not take +- Indent JSON with tabs and format Markdown with prettier +- Bump `actions/download-artifact` to v8.0.2, `actions/upload-artifact` to v7.0.2 and `docker/login-action` to v4.7.0 +- Document the release process in `RELEASE.md` +- Describe the image in `README.md` as a central home for SQLite databases, explain that the application keeps its own SQLite engine, add copying the static `sqlite3` shell into other images and drop the uid `65532` note +- Add reproducible builds, PIE, detached runs and SQLite updates to `docs/comparison.md` and explain the UPX packing of `grepular/sqlite3` +- Put the scope first in `SECURITY.md` + ## [v0.1.0] - 2026-09-23 ### Added From 6defe78baa92abf3cafd65f5be99fec0c1d1fe7e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:32:50 +0100 Subject: [PATCH 33/63] feat(release): bump image version to v0.2.0-beta.1 --- release.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.json b/release.json index 2f2969c..81e6a8d 100644 --- a/release.json +++ b/release.json @@ -3,6 +3,6 @@ "image": "0.1.0" }, "preview": { - "image": "0.1.0" + "image": "0.2.0-beta.1" } } From 59bccc88c7139e95024f27b769537e1b92f9ea41 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:14:55 +0100 Subject: [PATCH 34/63] feat(release): release nothing while release.json sets 0.0.0 --- scripts/release.sh | 2 ++ scripts/version.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/scripts/release.sh b/scripts/release.sh index db16134..dc8bb6f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -119,6 +119,8 @@ select_version() { selected=0 fi fi + elif [ "$stable" = 0.0.0 ]; then + bases=("$(next_version "$stable" minor)" "$(next_version "$stable" major)") else bases=("$(next_version "$stable" patch)") [[ "$branch" == v*.*.x ]] || bases+=("$(next_version "$stable" minor)") diff --git a/scripts/version.sh b/scripts/version.sh index 84e296f..f451eee 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -91,6 +91,8 @@ check_version() { publish=false if newer "$last" "$version"; then info "$VERSION_FILE sets $version, $branch released $last already" + elif [ "$version" = 0.0.0 ]; then + info "$VERSION_FILE sets 0.0.0, $branch releases nothing before scripts/release.sh sets the first version" elif [ "$channel" = preview ] && [[ ! "$version" =~ -[a-z]+\.[1-9] ]]; then info "$VERSION_FILE sets $version, $branch publishes numbered prereleases only" elif [ "$channel" = preview ] && [ -n "$earlier" ]; then @@ -156,6 +158,7 @@ print_upcoming_versions() { v*.*.x) upcoming="$(next_version "$base" patch)" ;; *) upcoming="$(next_version "$base" patch) or $(next_version "$base" minor)" ;; esac + [ "$base" != 0.0.0 ] || upcoming="$(next_version "$base" minor) or $(next_version "$base" major)" elif [ -n "$line" ]; then upcoming="a prerelease of $line.0" else From aedb22d4c5229b197cab2735c0626f987a7dfdbd Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:16:55 +0100 Subject: [PATCH 35/63] build: arm/v6, ppc64le, s390x and 386 images --- .github/workflows/build.yaml | 17 ++++++++++- .github/workflows/tests.yaml | 17 ++++++++++- README.md | 4 +-- docs/comparison.md | 56 ++++++++++++++++++------------------ 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bca8880..9316342 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -57,7 +57,7 @@ jobs: strategy: matrix: line: ${{ fromJSON(needs.versions.outputs.lines) }} - platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/riscv64] + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6, linux/riscv64, linux/ppc64le, linux/s390x, linux/386] include: - platform: linux/amd64 arch: amd64 @@ -69,10 +69,25 @@ jobs: arch: armv7 runner: ubuntu-26.04-arm qemu: true + - platform: linux/arm/v6 + arch: armv6 + runner: ubuntu-26.04-arm + qemu: true - platform: linux/riscv64 arch: riscv64 runner: ubuntu-26.04 qemu: true + - platform: linux/ppc64le + arch: ppc64le + runner: ubuntu-26.04 + qemu: true + - platform: linux/s390x + arch: s390x + runner: ubuntu-26.04 + qemu: true + - platform: linux/386 + arch: "386" + runner: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1369136..a43b035 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -40,7 +40,7 @@ jobs: fail-fast: false matrix: line: ${{ fromJSON(inputs.lines) }} - platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/riscv64] + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6, linux/riscv64, linux/ppc64le, linux/s390x, linux/386] include: - platform: linux/amd64 arch: amd64 @@ -52,10 +52,25 @@ jobs: arch: armv7 runner: ubuntu-26.04-arm qemu: true + - platform: linux/arm/v6 + arch: armv6 + runner: ubuntu-26.04-arm + qemu: true - platform: linux/riscv64 arch: riscv64 runner: ubuntu-26.04 qemu: true + - platform: linux/ppc64le + arch: ppc64le + runner: ubuntu-26.04 + qemu: true + - platform: linux/s390x + arch: s390x + runner: ubuntu-26.04 + qemu: true + - platform: linux/386 + arch: "386" + runner: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 with: diff --git a/README.md b/README.md index 7b20e27..fa53581 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ SQLite has no server and the database is just a file. Application images rarely - ✅ Batteries Included: `FTS3`, `FTS4`, `FTS5`, `R*Tree`, `Geopoly`, `math`, `dbstat`, `dbpage`, `stmt` and `soundex` SQLite extensions - ✅ Modern Architecture: Native health checks, `tini` init, OCI labels, reproducible builds, provenance and SBOM attestations - ✅ Always Current: sqlite.org is checked twice a day and new SQLite releases become new images -- ✅ Multi-Architecture: Builds for `linux/amd64`, `linux/arm64`, `linux/arm/v7` and `linux/riscv64` -- ✅ Tested: Every release is tested on all four platforms before it is published +- ✅ Multi-Architecture: Builds for `linux/amd64`, `linux/arm64`, `linux/arm/v7`, `linux/arm/v6`, `linux/riscv64`, `linux/ppc64le`, `linux/s390x` and `linux/386` +- ✅ Tested: Every release is tested on all eight platforms before it is published ## Comparison diff --git a/docs/comparison.md b/docs/comparison.md index 4266844..399f5d5 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -2,34 +2,34 @@ `peakimages/sqlite` compared to other popular SQLite Docker images. -| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | -| -------------------------------- | ------------------------------------------------------- | --------------------------------------- | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | -| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | -| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | -| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | -| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | -| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | -| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | -| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | -| Reproducible build[^2] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Recommended compile options[^3] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Build[^4] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | -| PIE[^5] | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | -| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Runs detached[^6] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Init[^7] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | -| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | -| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | -| No runtime extension loading[^8] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Compressed[^9] | 1.55 MB | 0.57 MB[^10] | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | -| On disk[^9] | 2.7 MB | 0.57 MB[^10] | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | -| Platforms | amd64, arm64, arm/v7, riscv64 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | -| Extensions[^11] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | -| SQLite updates | ✅ (checked twice a day, merged by hand) | ✅ (checked daily, built automatically) | ⚠️ (checked weekly, merged by hand) | ⚠️ (follows Alpine releases) | ⚠️ (policy not public) | ❌ (3.34.1, 2021) | ❌ (3.46.1, 2024) | ❌ (3.28.0, 2019) | -| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | +| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | +| -------------------------------- | ---------------------------------------------------------- | --------------------------------------- | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | +| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | +| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | +| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | +| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | +| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | +| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | +| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | +| Reproducible build[^2] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Recommended compile options[^3] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Build[^4] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | +| PIE[^5] | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Runs detached[^6] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Init[^7] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | +| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | +| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | +| No runtime extension loading[^8] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Compressed[^9] | 1.55 MB | 0.57 MB[^10] | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | +| On disk[^9] | 2.7 MB | 0.57 MB[^10] | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | +| Platforms | amd64, arm64, arm/v7, arm/v6, riscv64, ppc64le, s390x, 386 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | +| Extensions[^11] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | +| SQLite updates | ✅ (checked twice a day, merged by hand) | ✅ (checked daily, built automatically) | ⚠️ (checked weekly, merged by hand) | ⚠️ (follows Alpine releases) | ⚠️ (policy not public) | ❌ (3.34.1, 2021) | ❌ (3.46.1, 2024) | ❌ (3.28.0, 2019) | +| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | [^1]: `sqlite3 database.sqlite` writes into the working directory, and a new named volume takes the owner of the directory it is mounted on. `keinos/sqlite3` and `cleanstart/sqlite3` run their user in `/`, which root owns, so both fail with `unable to open database file`. From 5e351fe67e1d168a53095e9527b8eaeb6d9f0cea Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:17:28 +0100 Subject: [PATCH 36/63] ci(build): skip pull requests from branches that release --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9316342..f69837c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,6 +27,7 @@ defaults: jobs: versions: name: Read the versions + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images From a56a587fe6dedafa1e9f689427acf3303a9dd740 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:18:21 +0100 Subject: [PATCH 37/63] docs: note that the changelog follows Keep a Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 072fcb9..3a2da19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. SQLite version bumps are not listed, every release tag names the SQLite version it ships. +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + ## [Unreleased] ### Breaking Changes From cd54fe15514c2b4686be0866f2e7a737fe9a4f9f Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:40:46 +0100 Subject: [PATCH 38/63] ci: pin the binfmt image by tag and digest - it runs privileged and installs QEMU of every emulated build --- .github/workflows/build.yaml | 1 + .github/workflows/tests.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f69837c..24d370a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -105,6 +105,7 @@ jobs: - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 if: ${{ matrix.qemu }} with: + image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a43b035..d3bd10a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -84,6 +84,7 @@ jobs: - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 if: ${{ matrix.qemu }} with: + image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false - name: Load the image From 2f3c2f8973a899a75df0bf9833ffe3473d7f6981 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:44:25 +0100 Subject: [PATCH 39/63] feat(release): add leading v to git tags and releases --- .github/workflows/check-upstream.yaml | 2 +- .github/workflows/release.yaml | 6 +++--- RELEASE.md | 6 +++--- scripts/release.sh | 4 ++-- scripts/version.sh | 12 ++++++------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index bc9eefe..d9ea4fb 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -80,7 +80,7 @@ jobs: exit 0 fi cat > "$RUNNER_TEMP/body.md" < "$RUNNER_TEMP/notes.md" else cat > "$RUNNER_TEMP/notes.md" <-v`, such as `3.54.1-v1.4.0-beta.1`. +Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. > [!IMPORTANT] > A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/version.sh` first. @@ -64,7 +64,7 @@ Every release is tagged `-v`, such as `3.54.1-v1. 1. Cut the patch branch from the newest tag of the minor, unless it exists: ```sh - git switch -c v1.2.x 3.54.1-v1.2.1 && git push -u origin v1.2.x + git switch -c v1.2.x v3.54.1-v1.2.1 && git push -u origin v1.2.x ``` 2. Put the fix on a branch named after the version that releases it, cherry-picked from `main` when it is there already: @@ -93,7 +93,7 @@ Every release is tagged `-v`, such as `3.54.1-v1. git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh 3.53 ``` -2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `3.53.4-v1.3.0` +2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `v3.53.4-v1.3.0` 3. To stop the line, remove `3.53` from `src/upstream.json`, its tags stay where they are ## Release a major version diff --git a/scripts/release.sh b/scripts/release.sh index dc8bb6f..efe71f6 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -78,7 +78,7 @@ read_versions() { read_changes() { local tag range hash subject - tag="$(git describe --tags --abbrev=0 --match "*v$stable" 2> /dev/null)" || true + tag="$(git describe --tags --abbrev=0 --match "v${UPSTREAM:+*-v}$stable" 2> /dev/null)" || true range="${tag:+$tag..}HEAD" required="patch" if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then @@ -195,7 +195,7 @@ artifact_version() { } release_tags() { - git tag --list "$@" | grep -E "^${UPSTREAM:+[^-]+-}v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true + git tag --list "$@" | grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true } tag_versions() { diff --git a/scripts/version.sh b/scripts/version.sh index f451eee..3d00f44 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -117,7 +117,7 @@ select_releases() { releases= [ "$publish" = true ] || return 0 while read -r name sqlite; do - tag="${sqlite:+$sqlite-}v$version" + tag="v${sqlite:+$sqlite-v}$version" if grep -qxF "$tag" <<< "$tags"; then info "$tag is published" continue @@ -125,7 +125,7 @@ select_releases() { previous= while read -r older; do ! newer "$sqlite" "$older" || previous="$older" - done < <(with_version "$version" <<< "$tags" | sed -n 's/-v.*//p') + done < <(with_version "$version" <<< "$tags" | sed -nE 's/^v(.*)-v.*/\1/p') added+="$tag"$'\n' releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ '{$name, $version, $image, $tag, $previous}')"$'\n' @@ -230,8 +230,8 @@ moved_tags() { moving_tags() { local tags releases latest preview base image sqlite prefix seen=" " tags="$(cat)" - releases="$(sed -nE 's/^((.*)-)?v([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" - latest="$(awk 'NR == 1 && NF { print ($2 == "" ? "" : $2 "-") "v" $1 }' <<< "$releases")" + releases="$(sed -nE 's/^v((.*)-v)?([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" + latest="$(awk 'NR == 1 && NF { print "v" ($2 == "" ? "" : $2 "-v") $1 }' <<< "$releases")" preview="$latest" while read -r base; do if newer "$base" "${latest##*v}"; then @@ -245,7 +245,7 @@ moving_tags() { while read -r image sqlite; do prefix="$sqlite" while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do - echo "$prefix $sqlite-v$image" + echo "$prefix v$sqlite-v$image" seen+="$prefix " [[ "$prefix" == *.* ]] || break prefix="${prefix%.*}" @@ -254,7 +254,7 @@ moving_tags() { } release_tags() { - git tag --list "$@" | grep -E "^${UPSTREAM:+[^-]+-}v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true + git tag --list "$@" | grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true } tag_versions() { From 9699f4aee9971b056d36469dcf0be94937503d88 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:55:09 +0100 Subject: [PATCH 40/63] fix(security): set the mode of every COPY from the checkout - without --chmod the image takes the modes of the checkout, which follow the umask of whoever cloned it, so a checkout with umask 000 makes /usr/local/bin writable for every user --- src/Dockerfile | 8 ++++---- src/rootfs.sh | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index b039552..8878e64 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -29,7 +29,7 @@ ARG BUSYBOX_VERSION RUN apk add --no-cache build-base curl jq openssl tini-static=${TINI_VERSION} busybox-static=${BUSYBOX_VERSION} WORKDIR /src -COPY upstream.json verify.sh ./ +COPY --chmod=0644 upstream.json verify.sh ./ RUN jq --arg version "$SQLITE_VERSION" '.sqlite[] | select(.version == $version)' upstream.json > sqlite.json \ && curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url sqlite.json)" \ && sh verify.sh @@ -77,12 +77,12 @@ RUN gcc -static-pie -fPIE -Os \ && strip --strip-all sqlite3 # Assemble the root filesystem of the final image in /rootfs -COPY rootfs/ /rootfs/ -COPY rootfs.sh /src/ +COPY --chmod=0755 rootfs/ /rootfs/ +COPY --chmod=0644 rootfs.sh /src/ RUN sh /src/rootfs.sh # A scratch image has no package database for the SBOM scanner to read, this syft JSON document takes its place -COPY sbom.sh /src/ +COPY --chmod=0644 sbom.sh /src/ RUN sh /src/sbom.sh "peakimages/sqlite:${SQLITE_VERSION}-v${IMAGE_VERSION:?}" \ "generic|sqlite|${SQLITE_VERSION}|blessing|$(jq -r .url /src/sqlite.json)|sqlite:sqlite" \ "apk|tini-static|tini_project:tini|/sbin/tini" \ diff --git a/src/rootfs.sh b/src/rootfs.sh index 05121af..5a61243 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -26,7 +26,6 @@ install_binaries() { install -D -m 0755 /src/sqlite/sqlite3 /rootfs/usr/local/bin/sqlite3 install -D -m 0755 /sbin/tini-static /rootfs/sbin/tini install -D -m 0755 /bin/busybox.static /rootfs/bin/busybox - chmod 0755 /rootfs/usr/local/bin/*.sh } create_user() { From 8263b34356bc65740c0998fa868f1712a876e27d Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:12:20 +0100 Subject: [PATCH 41/63] feat(scripts): add size.sh to measure image sizes --- scripts/size.sh | 200 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100755 scripts/size.sh diff --git a/scripts/size.sh b/scripts/size.sh new file mode 100755 index 0000000..b37ee98 --- /dev/null +++ b/scripts/size.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +IMAGE="${1:-}" +BASELINE="${2:-}" +PLATFORM="${PLATFORM:-}" +IMAGE_NAME="${IMAGE_NAME:-}" +OUTPUT_FILE="${OUTPUT_FILE:-}" +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^[^-][^[:space:]]*( [^-][^[:space:]]*)?$ ]] || die "'$OPTION' is not an option, expected an image and an optional baseline" + [ -z "$OUTPUT_FILE" ] || [ -n "$BASELINE" ] || die "OUTPUT_FILE is set without a baseline, expected an image and a baseline to compare" + measure_image image "$IMAGE" + [ -z "$BASELINE" ] || measure_image baseline "$BASELINE" + [ -z "$BASELINE" ] || compare_with_baseline + [ -z "$OUTPUT_FILE" ] || write_report +} + +measure_image() { + heading "Sizes of $2" + mkdir "$WORK_DIR/$1" + if [ -d "$2" ]; then + read_layouts "$1" "$2" + else + read_repository "$1" "$2" + fi + measure_manifests "$1" "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1.tsv" ] || die "$2 has no image for ${PLATFORM:-a platform}" + sort -o "$WORK_DIR/$1.tsv" "$WORK_DIR/$1.tsv" +} + +read_layouts() { + layout="$2" + find "$2" -name index.json -exec jq -s '{manifests: map(.manifests[])}' {} + > "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1/index.json" ] || die "$2 holds no OCI layout, expected an index.json in it" +} + +read_repository() { + local name="${2#docker.io/}" reference + layout="" + repository="${name%%[:@]*}" + reference="${name:${#repository}+1}" + [[ "$repository" == */* ]] || repository="library/$repository" + read_object manifests "${reference:-latest}" > "$WORK_DIR/$1/index.json" 2> /dev/null || die "$2 is not on Docker Hub or not public" +} + +measure_manifests() { + local digest + if jq -e 'has("manifests")' "$2" > /dev/null; then + for digest in $(jq -r '.manifests[] | select(.platform.os != "unknown") | .digest' "$2"); do + read_object manifests "$digest" > "$WORK_DIR/$1/$digest" + measure_manifests "$1" "$WORK_DIR/$1/$digest" + done + else + measure_platform "$1" "$2" + fi +} + +measure_platform() { + local platform compressed unpacked=0 media_type digest blocks + read_object blobs "$(jq -r .config.digest "$2")" > "$2.config" + platform="$(jq -r '[.os, .architecture, .variant // empty] | join("/")' "$2.config")" + [ -z "$PLATFORM" ] || [ "$platform" = "$PLATFORM" ] || return 0 + compressed=$(($(wc -c < "$2") + $(jq '.config.size + ([.layers[].size] | add // 0)' "$2"))) + while read -r media_type digest; do + blocks="$(read_object blobs "$digest" | decompress "$media_type" | count_blocks)" + unpacked=$((unpacked + blocks)) + done < <(jq -r '.layers[] | "\(.mediaType) \(.digest)"' "$2") + printf '%s\t%s\t%s\t%s\n' "$platform" "$compressed" $((compressed + unpacked)) \ + "$(jq -r '.config.Labels["org.opencontainers.image.version"] // ""' "$2.config")" >> "$WORK_DIR/$1.tsv" + ok "$BOLD$platform$RESET is $(format_size "$compressed") compressed and $(format_size $((compressed + unpacked))) on disk, $(format_size "$unpacked") of it unpacked" +} + +decompress() { + case "$1" in + *+zstd) zstd -dc ;; + *+gzip | *.tar.gzip) gzip -dc ;; + *.tar) cat ;; + *) die "a layer is $1, expected a tar archive, plain or compressed with gzip or zstd" ;; + esac +} + +count_blocks() { + tar -tvf - | awk ' + BEGIN { blocks = 1 } + /^-/ { blocks += int((($2 ~ /\// ? $3 : $5) + 4095) / 4096) } + /^d/ { blocks++ } + /^l/ && length(substr($0, index($0, " -> ") + 4)) >= 60 { blocks++ } + END { print blocks * 4096 }' +} + +compare_with_baseline() { + local platform compressed on_disk base_compressed base_on_disk compressed_change on_disk_change + heading "Changes from $BASELINE" + while IFS=$'\t' read -r platform compressed on_disk base_compressed base_on_disk; do + compressed_change="$(format_change "$compressed" "$base_compressed")" + on_disk_change="$(format_change "$on_disk" "$base_on_disk")" + if [ "$compressed" = - ]; then + warn "$BOLD$platform$RESET is gone, $IMAGE has no image for it" + elif [ "$base_compressed" = - ]; then + ok "$BOLD$platform$RESET is new, $BASELINE has no image for it" + else + ok "$BOLD$platform$RESET changes by $compressed_change compressed and $on_disk_change on disk" + fi + printf '| %s | %s | %s | %s | %s | %s | %s |\n' "$platform" "$(format_size "$compressed")" "$(format_size "$base_compressed")" "$compressed_change" \ + "$(format_size "$on_disk")" "$(format_size "$base_on_disk")" "$on_disk_change" >> "$WORK_DIR/rows.md" + done < <(join -t $'\t' -a 1 -a 2 -e - -o 0,1.2,1.3,2.2,2.3 "$WORK_DIR/image.tsv" "$WORK_DIR/baseline.tsv") +} + +format_change() { + awk -v size="$1" -v base="$2" 'BEGIN { + if (size == "-" || base == "-") { print "-"; exit } + delta = size - base + if (delta == 0) { print "±0"; exit } + if (delta <= -1000000 || delta >= 1000000) amount = sprintf("%+.2f MB", delta / 1000000) + else if (delta <= -1000 || delta >= 1000) amount = sprintf("%+.1f kB", delta / 1000) + else amount = sprintf("%+d B", delta) + printf "%s (%+.1f %%)\n", amount, delta * 100 / base + }' +} + +write_report() { + local image_name baseline_name + image_name="$(short_name image "$IMAGE")" + baseline_name="$(short_name baseline "$BASELINE")" + if [ "$image_name" = "$baseline_name" ]; then + image_name="\`$IMAGE\`" + baseline_name="\`$BASELINE\`" + fi + { + printf '## Image size\n\n' + printf '%s compared with %s%s.\n\n' "$image_name" "$baseline_name" "$(format_version baseline "$BASELINE")" + printf '| Platform | Compressed %s | Compressed %s | Change | On disk %s | On disk %s | Change |\n' "$image_name" "$baseline_name" "$image_name" "$baseline_name" + printf '| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n' + cat "$WORK_DIR/rows.md" + printf "\nCompressed is what \`docker pull\` downloads, the CONTENT SIZE of the platform in \`docker image ls --tree\`. " + printf "On disk adds the unpacked files in 4 KiB blocks of ext4, the DISK USAGE that Docker Desktop shows, as the containerd image store keeps both. " + printf "Neither counts the SBOM and provenance attestations.\n" + } > "$OUTPUT_FILE" + ok "$OUTPUT_FILE holds the Markdown report" +} + +short_name() { + if [ "$1" = image ] && [ -n "$IMAGE_NAME" ]; then + printf "\`%s\`" "$IMAGE_NAME" + elif [ -d "$2" ]; then + printf 'the build' + else + printf "\`%s\`" "${2##*:}" + fi +} + +format_version() { + local version + [ ! -d "$2" ] || return 0 + version="$(head -n 1 "$WORK_DIR/$1.tsv" | cut -f 4)" + [ -z "$version" ] || printf " (\`%s\`)" "$version" +} + +read_object() { + local file + if [ -z "$layout" ]; then + fetch -H "Authorization: Bearer $(fetch "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$repository:pull" | jq -r .token)" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/$repository/$1/$2" + else + file="$(find "$layout" -path "*/blobs/${2/://}" -print -quit)" + [ -n "$file" ] || die "$layout has no blob $2" + cat "$file" + fi +} + +fetch() { + curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-all-errors -L "$@" +} + +format_size() { + awk -v bytes="$1" 'BEGIN { + if (bytes == "-") print "-" + else if (bytes >= 1000000) printf "%.2f MB\n", bytes / 1000000 + else if (bytes >= 1000) printf "%.1f kB\n", bytes / 1000 + else printf "%d B\n", bytes + }' +} + +main From 9a12f568905bf6f87ec82aef9e8021fa33f3ee83 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:11:27 +0100 Subject: [PATCH 42/63] ci(size): comment the size change on pull requests --- .github/workflows/size.yaml | 139 ++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 .github/workflows/size.yaml diff --git a/.github/workflows/size.yaml b/.github/workflows/size.yaml new file mode 100644 index 0000000..5a97caf --- /dev/null +++ b/.github/workflows/size.yaml @@ -0,0 +1,139 @@ +name: Size + +on: + pull_request: + paths: + - src/** + - scripts/size.sh + - .github/workflows/size.yaml + +permissions: {} + +cache-mode: none + +concurrency: + group: size-${{ github.event.pull_request.number }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build: + name: Build ${{ matrix.platform }} + if: ${{ github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} + permissions: + contents: read + runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images + timeout-minutes: 60 + strategy: + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-26.04 + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-26.04-arm + - platform: linux/arm/v7 + arch: armv7 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/arm/v6 + arch: armv6 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/riscv64 + arch: riscv64 + runner: ubuntu-26.04 + qemu: true + - platform: linux/ppc64le + arch: ppc64le + runner: ubuntu-26.04 + qemu: true + - platform: linux/s390x + arch: s390x + runner: ubuntu-26.04 + qemu: true + - platform: linux/386 + arch: "386" + runner: ubuntu-26.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + persist-credentials: false + + - name: Set the build arguments + id: args + env: + BRANCH: ${{ github.base_ref }} + run: | + image_version="$(scripts/version.sh --build)" + { + echo "image-version=$image_version" + echo "epoch=$(git log -1 --format=%ct)" + echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_OUTPUT" + + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + if: ${{ matrix.qemu }} + with: + image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 + cache-image: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + with: + cache-binary: false + + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + with: + context: src + platforms: ${{ matrix.platform }} + outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true + build-args: | + IMAGE_VERSION=${{ steps.args.outputs.image-version }} + SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} + CREATED=${{ steps.args.outputs.created }} + GIT_SHA=${{ github.sha }} + provenance: mode=max + sbom: true + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 + with: + name: image-${{ matrix.arch }} + path: ${{ runner.temp }}/image + if-no-files-found: error + retention-days: 1 + + report: + name: Compare with the latest release + needs: build + permissions: + contents: read + pull-requests: write # comment the size report on the pull request + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + with: + pattern: image-* + path: ${{ runner.temp }}/images + + - name: Compare the size with the latest release + env: + OUTPUT_FILE: ${{ runner.temp }}/size.md + run: | + scripts/size.sh "$RUNNER_TEMP/images" peakimages/sqlite:latest + cat "$OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment the size on the pull request + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: gh pr comment "$PULL_REQUEST" --body-file "$RUNNER_TEMP/size.md" --edit-last --create-if-none From 9429fa35a0880c1692c4209206b64583b96cefc9 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:35:26 +0100 Subject: [PATCH 43/63] ci: publish an image for every pull request - the pull request workflow now calls the build workflow, which no longer runs on pull_request - a pull request from a fork is built but not published --- .github/workflows/build.yaml | 3 +- .github/workflows/pull-request.yaml | 57 +++++++++++++++++++++++++++++ RELEASE.md | 9 +++++ scripts/version.sh | 45 ++++++++++++++++++++--- 4 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/pull-request.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 24d370a..84fc47b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,7 +1,6 @@ name: Build on: - pull_request: schedule: - cron: "50 7 * * 1" workflow_dispatch: @@ -27,7 +26,6 @@ defaults: jobs: versions: name: Read the versions - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images @@ -43,6 +41,7 @@ jobs: - id: versions env: BRANCH: ${{ github.base_ref || github.ref_name }} + PULL_REQUEST: ${{ github.event.pull_request.number }} run: | image="$(scripts/version.sh --build)" lines="$(jq -c '[.sqlite | to_entries[] | {name: .key, version: .value.version}]' src/upstream.json)" diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..9885bb0 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,57 @@ +name: Pull request + +on: + pull_request: + +permissions: {} + +cache-mode: none + +concurrency: + group: pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build: + name: Build + if: ${{ github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} + permissions: + contents: read + uses: $/.github/workflows/build.yaml + + publish: + name: Publish pr-${{ github.event.pull_request.number }} + needs: build + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} + permissions: + packages: write # push the image of the pull request to ghcr.io + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + with: + pattern: image-latest-* + path: ${{ runner.temp }}/images + + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push the image of the pull request + env: + TAG: pr-${{ github.event.pull_request.number }} + run: | + sources=() + for index in "$RUNNER_TEMP"/images/*/index.json; do + sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") + done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" + docker buildx imagetools create "${annotations[@]}" --tag "ghcr.io/peakimages/sqlite:$TAG" "${sources[@]}" diff --git a/RELEASE.md b/RELEASE.md index a2c72b6..8631a1e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -128,6 +128,14 @@ Every release is tagged `v-v`, such as `v3.54.1-v > [!IMPORTANT] > Cut `v1.x` from `main`, or bring an existing one up to date with `main`, before `release/v2.0` is merged into `main`. Afterwards `main` is v2 and v1 lives on `v1.x` alone. +## Pull requests + +The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. + +- Pull requests from forks and Dependabot are built, not published +- Pull requests from `main`, `next`, a release branch or a patch branch are not built, the release workflow builds their commits +- ghcr.io only, no git tag and no GitHub release + ## Tags | Tag | Points at | @@ -136,3 +144,4 @@ Every release is tagged `v-v`, such as `v3.54.1-v | `latest`, `3.54.1`, `3.54`, `3` | `3.54.1-v1.3.0`, the newest SQLite version of the newest stable image version | | `3.53.4`, `3.53` | the newest stable image version released with SQLite 3.53, also once the line stops | | `preview` | the newest prerelease of the lowest version in prerelease above `latest`, else `latest` | +| `pr-123` | the image of the newest push to an open pull request, on ghcr.io only | diff --git a/scripts/version.sh b/scripts/version.sh index 3d00f44..1675a45 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -3,6 +3,7 @@ set -euo pipefail OPTION="$*" BRANCH="${BRANCH:-}" +PULL_REQUEST="${PULL_REQUEST:-}" VERSION_FILE=release.json ARTIFACT=image UPSTREAM_FILE=src/upstream.json @@ -23,6 +24,7 @@ cd "$(dirname "$0")/.." main() { [[ "$OPTION" =~ ^(--build|--release|--tags)?$ ]] || die "'$OPTION' is not an option, expected --build, --release, --tags or none" read_version + print_build_version fetch_origin print_tags check_branch @@ -39,10 +41,21 @@ read_version() { [[ ! "$branch" =~ ^(next|release/v[0-9]+\.[0-9]+)$ ]] || channel=preview version="$(artifact_version "$channel")" [ -n "$version" ] || die "$VERSION_FILE has no $channel version of $ARTIFACT" - if [ "$OPTION" = --build ]; then - echo "$version" - exit 0 +} + +print_build_version() { + [ "$OPTION" = --build ] || return 0 + if [ -n "$PULL_REQUEST" ] && [ "$channel" = preview ]; then + tags="$(git ls-remote --tags --refs origin | sed 's|.*refs/tags/||' | filter_release_tags | sort -V)" || die "could not list the tags of origin" + line="${branch#release/v}" + [ "$line" != "$branch" ] || line="" + revision=HEAD + select_newest_stable + select_preview_base + version="$base" fi + echo "$version${PULL_REQUEST:+-pr.$PULL_REQUEST}" + exit 0 } fetch_origin() { @@ -174,8 +187,26 @@ print_releases() { jq -sc . <<< "$releases" } +select_newest_stable() { + stable="$(tag_versions <<< "$tags" | awk '!/-/' | newest)" + stable="${stable:-$(artifact_version stable)}" +} + +select_preview_base() { + version="$(artifact_version preview <(git show "$revision:$VERSION_FILE"))" + if [ -n "$line" ]; then + base="$line.0-alpha.0" + [[ "$version" != "$line.0-"* ]] || base="$version" + elif newer "$version" "$stable"; then + base="$version" + [[ "$version" == *-* ]] || base="$version-alpha.0" + else + base="$(next_version "$stable" minor)-alpha.0" + fi +} + artifact_version() { - jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "$VERSION_FILE" + jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "${2:-$VERSION_FILE}" } upstream_lines() { @@ -254,7 +285,11 @@ moving_tags() { } release_tags() { - git tag --list "$@" | grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true + git tag --list "$@" | filter_release_tags +} + +filter_release_tags() { + grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true } tag_versions() { From 0e1788c06047dbc516470f0a3c6e64e50b50eb0a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:02:24 +0100 Subject: [PATCH 44/63] feat(scripts): add clean-manifests.sh to delete unused manifests --- scripts/clean-manifests.sh | 263 +++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100755 scripts/clean-manifests.sh diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh new file mode 100755 index 0000000..49b3949 --- /dev/null +++ b/scripts/clean-manifests.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +LIMIT="${2:-}" +IMAGE=peakimages/sqlite +KEEP_TAG_PATTERN='^(latest|preview|[0-9]+(\.[0-9]+){0,2}|[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' +KEEP_TAG_EXCEPTIONS="" +DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" +DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +TO_DELETE_FILE=scripts/manifests-to-delete.tsv +DELETED_FILE=scripts/deleted-manifests.tsv +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" + read_ghcr_versions + read_dockerhub_tags + select_kept_tags + read_manifests + select_kept_manifests + heading "Selecting what to delete" + find_dockerhub_only_manifests + write_list + delete_manifests + verify_kept_manifests +} + +read_ghcr_versions() { + local state + for state in active deleted; do + gh api --paginate "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&state=$state" \ + --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' > "$WORK_DIR/ghcr-$state.tsv" 2> /dev/null \ + || die "gh could not list the $state versions of ghcr.io/$IMAGE, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" + done + cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and lists $(count < "$WORK_DIR/ghcr-deleted.tsv") deleted ones" +} + +read_dockerhub_tags() { + local token page last="" + token="$(registry_token docker.io pull)" + while :; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ + || die "could not list the tags of docker.io/$IMAGE" + [ -n "$page" ] || break + echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" + last="$(tail -n 1 <<< "$page")" + done + request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ + > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" +} + +select_kept_tags() { + { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" + git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ + || die "could not list the git tags of github.com/$IMAGE" + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + || die "gh could not list the open pull requests of $IMAGE" + tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + { + grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true + grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true + } | sort -u > "$WORK_DIR/kept-tags.txt" + grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, every git tag of github.com/$IMAGE and the tags of open pull requests" +} + +read_manifests() { + local digest tag + mkdir "$WORK_DIR/manifests" + request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ + || die "could not read the manifests of ghcr.io/$IMAGE" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" + awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" + while IFS=$'\t' read -r digest tag; do + curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ + || die "could not read the manifests $tag points at on docker.io" + done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ + >> "$WORK_DIR/children.tsv" + manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" + manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ + > "$WORK_DIR/kinds.tsv" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" +} + +select_kept_manifests() { + local kinds + kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" + awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ + || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" + kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" +} + +find_dockerhub_only_manifests() { + awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" + request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ + > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ + || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" +} + +write_list() { + awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + FILENAME == ARGV[1] { kept[$1]; next } + FILENAME == ARGV[2] { kind[$1] = $2; next } + FILENAME == ARGV[3] { size[$1] = $2; next } + FILENAME == ARGV[4] { size[$1] += $3; next } + FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } + FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } + FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } + FILENAME == ARGV[8] { if ($1 == 200) print "docker.io", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } + !($1 in kept) { print "ghcr.io and docker.io", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k2,2r -k5,5 > "$WORK_DIR/rows.tsv" + write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" + ok "$(awk -F '\t' 'END { print $8 " to delete, " $2 ", they free " $6 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' '$1 == "docker.io"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" + info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" +} + +delete_manifests() { + local registries type tags version digest row name deleted status + if [ -z "$LIMIT" ]; then + info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + return + fi + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" + heading "Deleting" + write_table /dev/null "$DELETED_FILE" + while IFS='|' read -r registries type tags version digest row; do + name="$digest${tags:+ ($tags)}" + deleted="" + if [ "$registries" != docker.io ]; then + delete_ghcr_version "$version" "$type $name" || continue + deleted=ghcr.io + fi + status="$(delete_dockerhub_manifest "$digest")" + [[ "$status" != 2?? ]] || deleted="${deleted:+$deleted and }docker.io" + if [ -n "$deleted" ]; then + ok "deleted the $type $BOLD$name$RESET from $deleted" + printf '%s\t%s\n' "$deleted" "${row#*$'\t'}" >> "$WORK_DIR/deleted.tsv" + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + fi + case "$status" in + 2?? | 404) ;; + 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; + *) die "could not delete the $type $name from docker.io, it answered $status" ;; + esac + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ print $1, $2, $4, $7, $8, $0 }') + ok "$(awk -F '\t' 'END { print "deleted " $8 ", " $2 ", they free " $6 }' "$DELETED_FILE")" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" +} + +verify_kept_manifests() { + local registry missing + [ -n "$LIMIT" ] || return 0 + for registry in ghcr.io docker.io; do + missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ + || die "could not check the kept manifests on $registry" + [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + done + ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" +} + +request_manifests() { + local registry="$1" list="$2" chunk token + shift 2 + [ -s "$list" ] || return 0 + split -l 1000 "$list" "$WORK_DIR/chunk-" + for chunk in "$WORK_DIR"/chunk-*; do + token="$(registry_token "$registry" pull)" + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + rm "$chunk" + done +} + +manifest_fields() { + find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + +} + +kept_manifests() { + awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } + FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[2] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" +} + +write_table() { + awk -F '\t' -v OFS='\t' 'function human(bytes, unit) { for (unit = 1; bytes >= 1000 && unit < 5; unit++) bytes /= 1000; return sprintf(unit == 1 ? "%d %s" : "%.1f %s", bytes, units[unit]) } + function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + BEGIN { split("B kB MB GB TB", units, " "); print "registries", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } + FILENAME == ARGV[1] { removed[$8]; types[$2]++; manifests++; if ($6 != "") $6 = human($6); print; next } + FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2] } + END { + for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + print "total", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), + "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" +} + +delete_ghcr_version() { + local error + until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do + case "$error" in + *"rate limit"*) sleep 60 ;; + *downloads*) + warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + return 1 + ;; + *) die "could not delete the $2 from ghcr.io: $error" ;; + esac + done + sleep 1 +} + +delete_dockerhub_manifest() { + local status + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ + "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + sleep 60 + done + echo "$status" +} + +registry_token() { + case "$1" in + ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + esac | jq -r .token +} + +count() { awk 'END { print NR }'; } + +main From da4529e5336429174c63125795c44f9fb2045dc0 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:28:15 +0100 Subject: [PATCH 45/63] ci: add a workflow to delete unused manifests --- .github/workflows/clean-manifests.yaml | 47 ++++++++++++++++++++++++++ RELEASE.md | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/clean-manifests.yaml diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml new file mode 100644 index 0000000..d38fc29 --- /dev/null +++ b/.github/workflows/clean-manifests.yaml @@ -0,0 +1,47 @@ +name: Clean manifests + +on: + workflow_dispatch: + +permissions: {} + +cache-mode: none + +concurrency: + group: clean-manifests + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + clean: + name: Delete unused manifests + if: ${{ github.repository == 'peakimages/sqlite' }} + permissions: + contents: read + environment: + name: cleanup + deployment: false + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + with: + persist-credentials: false + + - name: Delete the unused manifests + env: + GH_TOKEN: ${{ secrets.GHCR_CLEANUP_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CLEANUP_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} + run: scripts/clean-manifests.sh --delete 200 + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 + if: ${{ always() }} + with: + name: deleted-manifests + path: scripts/deleted-manifests.tsv + if-no-files-found: ignore + retention-days: 90 diff --git a/RELEASE.md b/RELEASE.md index 8631a1e..766a195 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -134,7 +134,7 @@ The pull request workflow builds every pull request and publishes the image of t - Pull requests from forks and Dependabot are built, not published - Pull requests from `main`, `next`, a release branch or a patch branch are not built, the release workflow builds their commits -- ghcr.io only, no git tag and no GitHub release +- ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open ## Tags From 0220c92eec1c3fee4ea66ef027af625c17f37f41 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:59:50 +0100 Subject: [PATCH 46/63] ci: lock the actions in actions.lock --- .editorconfig | 4 ++ .github/workflows/actions.lock | 72 ++++++++++++++++++++++++++ .github/workflows/build.yaml | 12 ++--- .github/workflows/check-upstream.yaml | 4 +- .github/workflows/clean-manifests.yaml | 4 +- .github/workflows/merge-forward.yaml | 2 +- .github/workflows/pull-request.yaml | 4 +- .github/workflows/release.yaml | 16 +++--- .github/workflows/size.yaml | 14 ++--- .github/workflows/tests.yaml | 8 +-- 10 files changed, 108 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/actions.lock diff --git a/.editorconfig b/.editorconfig index d843ad9..687f4cd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,7 @@ indent_size = 2 [*.md] indent_size = unset + +[*.lock] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock new file mode 100644 index 0000000..53758ae --- /dev/null +++ b/.github/workflows/actions.lock @@ -0,0 +1,72 @@ +# This file is machine-generated by `gh actions-lock`. +# Do not edit by hand; run `gh actions-lock` to update. +# Docs: https://gh.io/actions-lockfile +version: 'v0.0.3' +workflows: + '.github/workflows/build.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.2' + - 'docker/build-push-action@v7.4.0' + - 'docker/setup-buildx-action@v4.4.1' + - 'docker/setup-qemu-action@v4.4.0' + '.github/workflows/check-upstream.yaml': + - 'actions/checkout@v7.0.1' + '.github/workflows/clean-manifests.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.2' + '.github/workflows/merge-forward.yaml': + - 'actions/checkout@v7.0.1' + '.github/workflows/pull-request.yaml': + - 'actions/download-artifact@v8.0.2' + - 'docker/login-action@v4.7.0' + '.github/workflows/release.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'docker/login-action@v4.7.0' + '.github/workflows/size.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'actions/upload-artifact@v7.0.2' + - 'docker/build-push-action@v7.4.0' + - 'docker/setup-buildx-action@v4.4.1' + - 'docker/setup-qemu-action@v4.4.0' + '.github/workflows/tests.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'docker/setup-qemu-action@v4.4.0' +dependencies: + 'actions/checkout@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 + 'actions/download-artifact@v8.0.2': + ref: 'v8.0.2' + commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' + owner_id: 44036562 + repo_id: 192626254 + 'actions/upload-artifact@v7.0.2': + ref: 'v7.0.2' + commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9' + owner_id: 44036562 + repo_id: 192625955 + 'docker/build-push-action@v7.4.0': + ref: 'v7.4.0' + commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc' + owner_id: 5429470 + repo_id: 241092383 + 'docker/login-action@v4.7.0': + ref: 'v4.7.0' + commit: 'sha1-81c6f93d4bc991e510b342a00b419b3bb4efc5c6' + owner_id: 5429470 + repo_id: 287743349 + 'docker/setup-buildx-action@v4.4.1': + ref: 'v4.4.1' + commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069' + owner_id: 5429470 + repo_id: 288485773 + 'docker/setup-qemu-action@v4.4.0': + ref: 'v4.4.0' + commit: 'sha1-99012661954931238ded8c8b007157a8430204e1' + owner_id: 5429470 + repo_id: 288487894 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 84fc47b..e252452 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,7 +34,7 @@ jobs: image: ${{ steps.versions.outputs.image }} lines: ${{ inputs.lines || steps.versions.outputs.lines }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -89,7 +89,7 @@ jobs: arch: "386" runner: ubuntu-26.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -101,17 +101,17 @@ jobs: echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" } >> "$GITHUB_OUTPUT" - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + - uses: docker/setup-qemu-action@v4.4.0 if: ${{ matrix.qemu }} with: image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - uses: docker/setup-buildx-action@v4.4.1 with: cache-binary: false - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + - uses: docker/build-push-action@v7.4.0 with: context: src platforms: ${{ matrix.platform }} @@ -126,7 +126,7 @@ jobs: provenance: mode=max sbom: true - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 + - uses: actions/upload-artifact@v7.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index d9ea4fb..0c030b5 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -27,7 +27,7 @@ jobs: outputs: lines: ${{ steps.lines.outputs.lines }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -52,7 +52,7 @@ jobs: matrix: include: ${{ fromJSON(needs.lines.outputs.lines) }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: ref: ${{ matrix.branch }} persist-credentials: false diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index d38fc29..1467dc3 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 60 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -38,7 +38,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} run: scripts/clean-manifests.sh --delete 200 - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 + - uses: actions/upload-artifact@v7.0.2 if: ${{ always() }} with: name: deleted-manifests diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml index 77a197c..a482087 100644 --- a/.github/workflows/merge-forward.yaml +++ b/.github/workflows/merge-forward.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 9885bb0..43406fc 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -32,12 +32,12 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + - uses: actions/download-artifact@v8.0.2 with: pattern: image-latest-* path: ${{ runner.temp }}/images - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + - uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5a5a4e4..6795c44 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,7 +37,7 @@ jobs: outputs: releases: ${{ steps.version.outputs.releases }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -74,22 +74,22 @@ jobs: matrix: release: ${{ fromJSON(needs.version.outputs.releases) }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + - uses: actions/download-artifact@v8.0.2 with: pattern: image-${{ matrix.release.name }}-* path: ${{ runner.temp }}/images - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + - uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + - uses: docker/login-action@v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -142,18 +142,18 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + - uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # https://github.com/docker/login-action/releases/tag/v4.7.0 + - uses: docker/login-action@v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/size.yaml b/.github/workflows/size.yaml index 5a97caf..b52d7ff 100644 --- a/.github/workflows/size.yaml +++ b/.github/workflows/size.yaml @@ -60,7 +60,7 @@ jobs: arch: "386" runner: ubuntu-26.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -76,17 +76,17 @@ jobs: echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" } >> "$GITHUB_OUTPUT" - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + - uses: docker/setup-qemu-action@v4.4.0 if: ${{ matrix.qemu }} with: image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - uses: docker/setup-buildx-action@v4.4.1 with: cache-binary: false - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + - uses: docker/build-push-action@v7.4.0 with: context: src platforms: ${{ matrix.platform }} @@ -99,7 +99,7 @@ jobs: provenance: mode=max sbom: true - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # https://github.com/actions/upload-artifact/releases/tag/v7.0.2 + - uses: actions/upload-artifact@v7.0.2 with: name: image-${{ matrix.arch }} path: ${{ runner.temp }}/image @@ -115,11 +115,11 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + - uses: actions/download-artifact@v8.0.2 with: pattern: image-* path: ${{ runner.temp }}/images diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d3bd10a..e59f463 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -72,16 +72,16 @@ jobs: arch: "386" runner: ubuntu-26.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # https://github.com/actions/download-artifact/releases/tag/v8.0.2 + - uses: actions/download-artifact@v8.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 + - uses: docker/setup-qemu-action@v4.4.0 if: ${{ matrix.qemu }} with: image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 From 9577562d39e0908768103205783100acd5844880 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:00:24 +0100 Subject: [PATCH 47/63] ci: verify action lockfile during tests --- .github/workflows/tests.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e59f463..af6d930 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,6 +30,14 @@ jobs: - run: scripts/lint.sh + - name: Verify the action lockfile + env: + GH_TOKEN: ${{ github.token }} + # https://github.com/github/gh-actions-lock/releases/tag/v0.4.0 + run: | + gh extension install github/gh-actions-lock --pin v0.4.0 + gh actions-lock --verify + test: name: Test ${{ matrix.line.name }} ${{ matrix.platform }} permissions: From 88936635bda0efd88608ecc84714f364f36c6670 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:22:16 +0100 Subject: [PATCH 48/63] fix(release): copy the image labels to the release index - ghcr.io reads the description and the license of an image of several platforms from the annotations of its index, not from the labels of each image --- .github/workflows/release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6795c44..46960bd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -102,8 +102,11 @@ jobs: for index in "$RUNNER_TEMP"/images/*/index.json; do sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" for repository in ghcr.io/peakimages/sqlite docker.io/peakimages/sqlite; do - docker buildx imagetools create --tag "$repository:${TAG#v}" "${sources[@]}" + docker buildx imagetools create "${annotations[@]}" --tag "$repository:${TAG#v}" "${sources[@]}" done - name: Create the GitHub release From b8c6ebb81de567b9579f0551225fbd768d3c334e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:48:35 +0100 Subject: [PATCH 49/63] fix(image): stop sbom.sh on a malformed CPE --- src/sbom.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/sbom.sh b/src/sbom.sh index f448274..318262f 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -30,14 +30,16 @@ write_document() { | def installed($key; $name): first($installed[] | select(.[$key] == $name)) // error("sbom: \($name) is not installed"); def purl($path; $version; $qualifiers): "pkg:\($path)@\($version)?" + ([$qualifiers | to_entries[] | select(.value | length > 0) | "\(.key)=\(.value | @uri)"] | join("&")); def os_purl($type; $name; $version; $arch; $upstream): purl("\($type)/\($os.ID)/\($name)"; $version; { arch: $arch, distro: "\($os.ID)-\($os.VERSION_ID)", upstream: $upstream }); - def package($type; $name; $version; $license; $cpe; $purl): { - name: $name, - version: $version, - type: $type, - licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], - cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], - purl: $purl - }; + def package($type; $name; $version; $license; $cpe; $purl): + ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe + | { + name: $name, + version: $version, + type: $type, + licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], + cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], + purl: $purl + }; def apk($name; $cpe; $paths): installed("P"; $name) | package("apk"; .P; .V; .L; $cpe; os_purl("apk"; .P; .V; .A; if .o != .P then .o else "" end)) + { From 40aff91ab04b420cb6c9187cebd773dba295143a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:56:03 +0100 Subject: [PATCH 50/63] ci: delete unused manifests every month - the pull request workflow leaves the images of every closed pull request behind on ghcr.io --- .github/workflows/clean-manifests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index 1467dc3..e50bf5c 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -1,6 +1,8 @@ name: Clean manifests on: + schedule: + - cron: "20 13 1 * *" workflow_dispatch: permissions: {} From 028034a665de82c5800f0ac05b69850016ab352f Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:38:19 +0100 Subject: [PATCH 51/63] feat(scripts): give each registry a column in the lists - docker_hub is undefined for a manifest ghcr.io has, as a HEAD on docker.io for every one of them could trigger rate limits --- scripts/clean-manifests.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 49b3949..e488086 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -134,18 +134,18 @@ write_list() { FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } - FILENAME == ARGV[8] { if ($1 == 200) print "docker.io", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "ghcr.io and docker.io", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } + !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k2,2r -k5,5 > "$WORK_DIR/rows.tsv" + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $8 " to delete, " $2 ", they free " $6 }' "$TO_DELETE_FILE")" - ok "$(awk -F '\t' '$1 == "docker.io"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" + ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" } delete_manifests() { - local registries type tags version digest row name deleted status + local on_ghcr type tags version digest rest name deleted ghcr docker_hub status if [ -z "$LIMIT" ]; then info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" return @@ -153,18 +153,20 @@ delete_manifests() { [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" heading "Deleting" write_table /dev/null "$DELETED_FILE" - while IFS='|' read -r registries type tags version digest row; do + while IFS='|' read -r on_ghcr type tags version digest rest; do name="$digest${tags:+ ($tags)}" - deleted="" - if [ "$registries" != docker.io ]; then + deleted="" ghcr=no docker_hub=no + if [ "$on_ghcr" = yes ]; then delete_ghcr_version "$version" "$type $name" || continue - deleted=ghcr.io + deleted=ghcr.io ghcr=yes fi status="$(delete_dockerhub_manifest "$digest")" - [[ "$status" != 2?? ]] || deleted="${deleted:+$deleted and }docker.io" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\n' "$deleted" "${row#*$'\t'}" >> "$WORK_DIR/deleted.tsv" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" fi case "$status" in @@ -172,8 +174,8 @@ delete_manifests() { 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ print $1, $2, $4, $7, $8, $0 }') - ok "$(awk -F '\t' 'END { print "deleted " $8 ", " $2 ", they free " $6 }' "$DELETED_FILE")" + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') + ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" } @@ -216,13 +218,13 @@ kept_manifests() { write_table() { awk -F '\t' -v OFS='\t' 'function human(bytes, unit) { for (unit = 1; bytes >= 1000 && unit < 5; unit++) bytes /= 1000; return sprintf(unit == 1 ? "%d %s" : "%.1f %s", bytes, units[unit]) } function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } - BEGIN { split("B kB MB GB TB", units, " "); print "registries", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } - FILENAME == ARGV[1] { removed[$8]; types[$2]++; manifests++; if ($6 != "") $6 = human($6); print; next } + BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } + FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } { blob[$2] = $3; if (!($1 in removed)) used[$2] } END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } - print "total", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), + print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" } From 57fc8f74d9e1f62568f27366457b553727506620 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:03:09 +0100 Subject: [PATCH 52/63] feat(scripts): limit docker.io checks to twice the count - ghcr.io lists deleted versions for 30 days, a HEAD on docker.io for each would reach thousands per run, which could trigger rate limits - twice the count covers the last two runs, and manifests only docker.io has sort first, so a copy docker.io refused is deleted within two runs --- scripts/clean-manifests.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index e488086..964f394 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -42,14 +42,13 @@ main() { } read_ghcr_versions() { - local state - for state in active deleted; do - gh api --paginate "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&state=$state" \ - --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' > "$WORK_DIR/ghcr-$state.tsv" 2> /dev/null \ - || die "gh could not list the $state versions of ghcr.io/$IMAGE, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" - done + local page window=$((${LIMIT:-0} * 2)) + list_ghcr_versions "state=active" --paginate > "$WORK_DIR/ghcr-active.tsv" + for ((page = 1; page <= (window + 99) / 100; page++)); do + list_ghcr_versions "state=deleted&page=$page" + done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" - ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and lists $(count < "$WORK_DIR/ghcr-deleted.tsv") deleted ones" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" } read_dockerhub_tags() { @@ -137,7 +136,7 @@ write_list() { FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/rows.tsv" + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" @@ -190,6 +189,14 @@ verify_kept_manifests() { ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" } +list_ghcr_versions() { + local query="$1" + shift + gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ + --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ + || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" +} + request_manifests() { local registry="$1" list="$2" chunk token shift 2 From 4e0f932c03cde335f3660d25ced1dd80aa1e752e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:03:09 +0100 Subject: [PATCH 53/63] ci: compare the size change in the pull request workflow - reads the layouts the build workflow built, so no second build is needed to get the pull request image sizes --- .github/workflows/actions.lock | 8 +- .github/workflows/pull-request.yaml | 33 +++++++ .github/workflows/size.yaml | 139 ---------------------------- 3 files changed, 34 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/size.yaml diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 53758ae..b7ef44f 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -17,19 +17,13 @@ workflows: '.github/workflows/merge-forward.yaml': - 'actions/checkout@v7.0.1' '.github/workflows/pull-request.yaml': + - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - 'docker/login-action@v4.7.0' '.github/workflows/release.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - 'docker/login-action@v4.7.0' - '.github/workflows/size.yaml': - - 'actions/checkout@v7.0.1' - - 'actions/download-artifact@v8.0.2' - - 'actions/upload-artifact@v7.0.2' - - 'docker/build-push-action@v7.4.0' - - 'docker/setup-buildx-action@v4.4.1' - - 'docker/setup-qemu-action@v4.4.0' '.github/workflows/tests.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 43406fc..1fb14e7 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -55,3 +55,36 @@ jobs: | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" mapfile -t annotations <<< "$labels" docker buildx imagetools create "${annotations[@]}" --tag "ghcr.io/peakimages/sqlite:$TAG" "${sources[@]}" + + size: + name: Compare the size with the latest release + needs: build + permissions: + contents: read + pull-requests: write # comment the size report on the pull request + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@v8.0.2 + with: + pattern: image-latest-* + path: ${{ runner.temp }}/images + + - name: Compare the size with the latest release + env: + IMAGE_NAME: pr-${{ github.event.pull_request.number }} + OUTPUT_FILE: ${{ runner.temp }}/size.md + run: | + scripts/size.sh "$RUNNER_TEMP/images" peakimages/sqlite:latest + cat "$OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment the size on the pull request + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: gh pr comment "$PULL_REQUEST" --body-file "$RUNNER_TEMP/size.md" --edit-last --create-if-none diff --git a/.github/workflows/size.yaml b/.github/workflows/size.yaml deleted file mode 100644 index b52d7ff..0000000 --- a/.github/workflows/size.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: Size - -on: - pull_request: - paths: - - src/** - - scripts/size.sh - - .github/workflows/size.yaml - -permissions: {} - -cache-mode: none - -concurrency: - group: size-${{ github.event.pull_request.number }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - build: - name: Build ${{ matrix.platform }} - if: ${{ github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} - permissions: - contents: read - runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images - timeout-minutes: 60 - strategy: - matrix: - include: - - platform: linux/amd64 - arch: amd64 - runner: ubuntu-26.04 - - platform: linux/arm64 - arch: arm64 - runner: ubuntu-26.04-arm - - platform: linux/arm/v7 - arch: armv7 - runner: ubuntu-26.04-arm - qemu: true - - platform: linux/arm/v6 - arch: armv6 - runner: ubuntu-26.04-arm - qemu: true - - platform: linux/riscv64 - arch: riscv64 - runner: ubuntu-26.04 - qemu: true - - platform: linux/ppc64le - arch: ppc64le - runner: ubuntu-26.04 - qemu: true - - platform: linux/s390x - arch: s390x - runner: ubuntu-26.04 - qemu: true - - platform: linux/386 - arch: "386" - runner: ubuntu-26.04 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: Set the build arguments - id: args - env: - BRANCH: ${{ github.base_ref }} - run: | - image_version="$(scripts/version.sh --build)" - { - echo "image-version=$image_version" - echo "epoch=$(git log -1 --format=%ct)" - echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" - } >> "$GITHUB_OUTPUT" - - - uses: docker/setup-qemu-action@v4.4.0 - if: ${{ matrix.qemu }} - with: - image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 - cache-image: false - - - uses: docker/setup-buildx-action@v4.4.1 - with: - cache-binary: false - - - uses: docker/build-push-action@v7.4.0 - with: - context: src - platforms: ${{ matrix.platform }} - outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true - build-args: | - IMAGE_VERSION=${{ steps.args.outputs.image-version }} - SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} - CREATED=${{ steps.args.outputs.created }} - GIT_SHA=${{ github.sha }} - provenance: mode=max - sbom: true - - - uses: actions/upload-artifact@v7.0.2 - with: - name: image-${{ matrix.arch }} - path: ${{ runner.temp }}/image - if-no-files-found: error - retention-days: 1 - - report: - name: Compare with the latest release - needs: build - permissions: - contents: read - pull-requests: write # comment the size report on the pull request - runs-on: ubuntu-26.04 # https://github.com/actions/runner-images - timeout-minutes: 30 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - uses: actions/download-artifact@v8.0.2 - with: - pattern: image-* - path: ${{ runner.temp }}/images - - - name: Compare the size with the latest release - env: - OUTPUT_FILE: ${{ runner.temp }}/size.md - run: | - scripts/size.sh "$RUNNER_TEMP/images" peakimages/sqlite:latest - cat "$OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY" - - - name: Comment the size on the pull request - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - env: - GH_TOKEN: ${{ github.token }} - PULL_REQUEST: ${{ github.event.pull_request.number }} - run: gh pr comment "$PULL_REQUEST" --body-file "$RUNNER_TEMP/size.md" --edit-last --create-if-none From 0976c9bc62165cd774754046c0bf373b1a9d55ac Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:53:23 +0100 Subject: [PATCH 54/63] fix(release): preselect a minor for feat(release) changes - only the commits that bump the image or the SQLite version stay ignored, so a feat change to the release process asks for a minor --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index efe71f6..7eaa44f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -84,7 +84,7 @@ read_changes() { if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then required=major [ "${stable%%.*}" != 0 ] || required=minor - elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): / { found = 1 } END { exit !found }'; then + elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): bump / { found = 1 } END { exit !found }'; then required=minor fi info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" From 18f9a66ae672e58339b4d669bc52996adcc090f0 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:40:43 +0100 Subject: [PATCH 55/63] docs: update CHANGELOG.md and add a Security section --- CHANGELOG.md | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2da19..b3c4ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Compress the layers with zstd instead of gzip, zstd is faster, more modern and compresses better than gzip, pulling needs Docker Engine 23.0 or later +### Security + +- Harden the image against a checkout with a permissive umask, every file the build copies from the repository gets its mode from the Dockerfile, so no build can make `/usr`, `/usr/local` or `/usr/local/bin` writable for every user. +- Warn in `README.md` that a bind mount opens the databases to every user on the host who can reach the directory, and show how to close it +- Make release tags immutable + - Refuse to overwrite or delete a release tag such as `3.54.1-v0.2.0` on Docker Hub by using the ag mutability setting on docker hub with a strict patter. + - Refuse to updated, force push or delete the git tag of a release by using tag protection rule on GitHub +- Refuse to delete or force-push `main`, `next`, release and maintenance branches by using branch protection rules on GitHub +- Harden the CI workflows + - Grant every job the least permissions it needs and state the reason for every write permission + - Keep the Docker Hub credentials in the `release` environment, which only the branches that release can use, with a token that can push but not delete + - Keep the tokens that delete images in the `cleanup` environment, which only `main` can use and a maintainer approves for every run + - Lock every action and the actions it calls to a commit in `.github/workflows/actions.lock`, name actions by their release tag, and fail the lint job when the lockfile is stale + - Pin the binfmt image by tag and digest, it runs privileged and installs the QEMU of every emulated build + - Allow the workflows only the events they use, so `pull_request_target`, `workflow_run` and comment events start no run + - Run without the GitHub Actions cache, layer caches and cached tool binaries, so no run restores what another run wrote + - Lint the workflows with jactionlint pedantic, which includes the audits of zizmor, and fail on every warning + ### Added -- Add `.recover` and `.dbinfo` to the shell and the `sqlite_dbpage` and `sqlite_stmt` virtual tables, the shell is built with the options sqlite.org recommends for a full-featured shell +- Add the shell options sqlite.org recommends for a full-featured shell: `.recover` and `.dbinfo`, the `sqlite_dbpage` and `sqlite_stmt` virtual tables, the `sqlite_offset()` function, comments in `EXPLAIN` output and `EXPLAIN` of statements that call functions the shell does not know - Build reproducibly, a rebuild of the same commit gives the same image digest per platform, and the `created` date of the image and its `org.opencontainers.image.created` label are the date of that commit - Publish prereleases from `next` and the release branches, tagged like `3.54.1-v0.2.0-beta.1`, and the moving `preview` tag - Publish older SQLite lines such as `3.53` next to the newest, built with the same image version and tagged like `3.53.4-v0.2.0`, `3.53.4` and `3.53` +- Builds for `linux/arm/v6`, `linux/ppc64le`, `linux/s390x` and `linux/386`, eight platforms in total +- Publish the image of every pull request from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, so a change can be tried before it is merged ### Changed @@ -25,30 +45,29 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Pin SQLite in `src/upstream.json`, one entry per line, instead of `src/version.json` - Release from `main`, `next`, release branches such as `release/v0.3` and maintenance branches such as `v1.x` and `v0.1.x` - Select the stage and the version with `scripts/release.sh`, which offers the versions the branch allows and preselects the bump the commits need + - Publish nothing while `release.json` sets `0.0.0`, the version a new repository starts with, `scripts/release.sh` offers `0.1.0` or `1.0.0` as its first release - Decide the image version, the releases and the moving tags with `scripts/version.sh`, which also reports what a push publishes - Publish every release a push of `release.json` or `src/upstream.json` names and whose tag does not exist yet, never as a draft - Move `latest`, `preview` and every prefix of a SQLite version to their place among all published tags, so a release of an older line never moves a tag back - Check every SQLite line of `main`, the maintenance branches and the major release branches twice a day and open one pull request per bump, which supersedes the older ones - Verify a new SQLite release against its release notes and its tagged check-in on sqlite.org - Merge `main` forward into `next` and the release branches twice a day and after every release + - Tag releases in git and on GitHub with a leading `v`, `v3.54.1-v0.2.0`, the image tags stay `3.54.1-v0.2.0`, and rename the tags and releases of the earlier versions to match - Rework the CI workflows for the new release process - Split the build, test and release workflows, the release workflow calls the build workflow and the build workflow calls the tests - Build every SQLite line for every platform once, test those builds and publish exactly them, not a second build - Write the notes of the first release of an image version from `CHANGELOG.md`, later releases name the SQLite update - Mark the GitHub release that `latest` points at as the latest release -- Harden the CI workflows - - Grant every job the least permissions it needs and state the reason for every write permission - - Keep the Docker Hub credentials in the `release` environment - - Run without the GitHub Actions cache, layer caches and cached tool binaries, so no run restores what another run wrote - Run every step with bash, so a failing command in a pipe fails the step - Queue release runs instead of cancelling them, so no two runs publish or move tags at the same time - - Lint the workflows with jactionlint pedantic, which includes the audits of zizmor, and fail on every warning - Remove the `SQLITE_UID` and `SQLITE_GID` build arguments, the user is `65532:65532` - Rename the build argument `BUILD_DATE` to `CREATED`, add `SOURCE_DATE_EPOCH` and require `IMAGE_VERSION` - Bump Alpine in the build stage from 3.24.2 to 3.24.3 ### Fixed +- Push nothing but the release index, its images and their attestations, the old release workflow pushed every platform as an index of its own by digest, which stayed behind untagged and unused next to the release index on ghcr.io and docker.io +- Copy the labels of the image to the annotations of the release index, ghcr.io reads the description and the license of an image of several platforms from there and showed neither - List SQLite, tini and BusyBox in the SBOM attestation with their versions, licenses, package URLs and CPEs, from an SBOM document in `/var/lib/db/sbom` that takes the place of the package database a scratch image lacks - Match `LICENSE` to the AGPL-3.0 text on gnu.org @@ -58,6 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Maintenance +- Add `scripts/clean-manifests.sh` and a monthly workflow that delete the manifests no tag needs from ghcr.io and docker.io, such as the untagged images the old release workflow left behind and the images of closed pull requests, and keep every release and every image a tag points at +- Add `scripts/size.sh` to measure the compressed and the on-disk size of every platform, and comment the size change of every pull request compared with the latest release - Add `scripts/build.sh` to build the image locally with the image version CI builds with, for the platform of the Docker host or the platforms `PLATFORM` names - Run docker build checks in `scripts/lint.sh` and stop at the first failed check - Stop every script on arguments it does not take From 3e8f97d6fa3ae492df9335a394f2f50fd6cc14e9 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:01:04 +0100 Subject: [PATCH 56/63] chore(repo): improve default PR template --- .github/PULL_REQUEST_TEMPLATE/default.md | 6 +++++- CHANGELOG.md | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/default.md b/.github/PULL_REQUEST_TEMPLATE/default.md index 049e486..33f8af0 100644 --- a/.github/PULL_REQUEST_TEMPLATE/default.md +++ b/.github/PULL_REQUEST_TEMPLATE/default.md @@ -1,9 +1,13 @@ - + ## Changes - +## Why + + + ## Issues - Fixes diff --git a/CHANGELOG.md b/CHANGELOG.md index b3c4ed7..fe0820d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Describe the image in `README.md` as a central home for SQLite databases, explain that the application keeps its own SQLite engine, add copying the static `sqlite3` shell into other images and drop the uid `65532` note - Add reproducible builds, PIE, detached runs and SQLite updates to `docs/comparison.md` and explain the UPX packing of `grepular/sqlite3` - Put the scope first in `SECURITY.md` +- Add a Why section to the pull request template ## [v0.1.0] - 2026-09-23 From 45c503c4e24a758ba81634f3e508bbaee9c9b0ab Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:29:23 +0100 Subject: [PATCH 57/63] refactor(release): rename version.sh to plan-version.sh --- .github/workflows/build.yaml | 2 +- .github/workflows/release.yaml | 4 ++-- CHANGELOG.md | 2 +- RELEASE.md | 4 ++-- scripts/build.sh | 2 +- scripts/{version.sh => plan-version.sh} | 0 6 files changed, 7 insertions(+), 7 deletions(-) rename scripts/{version.sh => plan-version.sh} (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e252452..9ee467b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -43,7 +43,7 @@ jobs: BRANCH: ${{ github.base_ref || github.ref_name }} PULL_REQUEST: ${{ github.event.pull_request.number }} run: | - image="$(scripts/version.sh --build)" + image="$(scripts/plan-version.sh --build)" lines="$(jq -c '[.sqlite | to_entries[] | {name: .key, version: .value.version}]' src/upstream.json)" printf 'image=%s\nlines=%s\n' "$image" "$lines" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 46960bd..9c1b8f0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -46,7 +46,7 @@ jobs: env: BRANCH: ${{ github.ref_name }} run: | - releases="$(scripts/version.sh --release)" + releases="$(scripts/plan-version.sh --release)" echo "releases=$releases" | tee -a "$GITHUB_OUTPUT" build: @@ -165,7 +165,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - tags="$(scripts/version.sh --tags)" + tags="$(scripts/plan-version.sh --tags)" jq -r 'to_entries[] | "\(.key) \(.value)"' <<< "$tags" | while read -r tag release; do for repository in ghcr.io/peakimages/sqlite docker.io/peakimages/sqlite; do docker buildx imagetools create --tag "$repository:${tag#v}" "$repository:${release#v}" diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0820d..76ec8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Release from `main`, `next`, release branches such as `release/v0.3` and maintenance branches such as `v1.x` and `v0.1.x` - Select the stage and the version with `scripts/release.sh`, which offers the versions the branch allows and preselects the bump the commits need - Publish nothing while `release.json` sets `0.0.0`, the version a new repository starts with, `scripts/release.sh` offers `0.1.0` or `1.0.0` as its first release - - Decide the image version, the releases and the moving tags with `scripts/version.sh`, which also reports what a push publishes + - Plan the image version, the releases and the moving tags with `scripts/plan-version.sh`, which also reports what a push publishes - Publish every release a push of `release.json` or `src/upstream.json` names and whose tag does not exist yet, never as a draft - Move `latest`, `preview` and every prefix of a SQLite version to their place among all published tags, so a release of an older line never moves a tag back - Check every SQLite line of `main`, the maintenance branches and the major release branches twice a day and open one pull request per bump, which supersedes the older ones diff --git a/RELEASE.md b/RELEASE.md index 766a195..7f62bdb 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,7 +3,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. > [!IMPORTANT] -> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/version.sh` first. +> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/plan-version.sh` first. ## Rules @@ -39,7 +39,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v 2. Check what the branch releases: ```sh - scripts/version.sh + scripts/plan-version.sh ``` 3. Select the stage and the version, the script commits them and asks to push: diff --git a/scripts/build.sh b/scripts/build.sh index 01845ee..0814a69 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,7 +23,7 @@ main() { build() { local image_version [ -z "${CI:-}" ] || die "scripts/build.sh is for local builds, CI builds with docker/build-push-action" - image_version="$(scripts/version.sh --build)" + image_version="$(scripts/plan-version.sh --build)" docker buildx build ${PLATFORM:+--platform "$PLATFORM"} --build-arg IMAGE_VERSION="$image_version" \ --provenance mode=max --sbom true --load --tag "$IMAGE" src ok "$IMAGE is built for ${PLATFORM:-the platform of the Docker host} with image version $BOLD$image_version$RESET" diff --git a/scripts/version.sh b/scripts/plan-version.sh similarity index 100% rename from scripts/version.sh rename to scripts/plan-version.sh From 55316c3c4174b4be25f1870169065fd53ccc118b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:30:56 +0100 Subject: [PATCH 58/63] ci(release): plan the moving tags in a step of their own --- .github/workflows/release.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9c1b8f0..db54ffd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -150,6 +150,12 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Plan the moving tags + id: plan + run: | + tags="$(scripts/plan-version.sh --tags)" + echo "tags=$tags" | tee -a "$GITHUB_OUTPUT" + - uses: docker/login-action@v4.7.0 with: registry: ghcr.io @@ -161,18 +167,22 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Move the tags the releases take over + - name: Move the tags on ghcr.io and docker.io env: - GH_TOKEN: ${{ github.token }} + TAGS: ${{ steps.plan.outputs.tags }} run: | - tags="$(scripts/plan-version.sh --tags)" - jq -r 'to_entries[] | "\(.key) \(.value)"' <<< "$tags" | while read -r tag release; do + jq -r 'to_entries[] | "\(.key) \(.value)"' <<< "$TAGS" | while read -r tag release; do for repository in ghcr.io/peakimages/sqlite docker.io/peakimages/sqlite; do docker buildx imagetools create --tag "$repository:${tag#v}" "$repository:${release#v}" done done - latest="$(jq -r '.latest // empty' <<< "$tags")" - [ -z "$latest" ] || gh release edit "$latest" --latest + + - name: Mark the latest GitHub release + if: ${{ fromJSON(steps.plan.outputs.tags).latest }} + env: + GH_TOKEN: ${{ github.token }} + LATEST: ${{ fromJSON(steps.plan.outputs.tags).latest }} + run: gh release edit "$LATEST" --latest merge: name: Merge forward From 79563554e6e22a4771b2ffa0aae98d975162b90a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:31:23 +0100 Subject: [PATCH 59/63] ci: simplify the PR description of a conflicting merge --- .github/workflows/merge-forward.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml index a482087..fa35e31 100644 --- a/.github/workflows/merge-forward.yaml +++ b/.github/workflows/merge-forward.yaml @@ -35,10 +35,6 @@ jobs: - name: Merge main into the release branches and next, the release branches into next env: GH_TOKEN: ${{ github.token }} - BODY: | - The branches do not merge without conflicts. Resolve them on this branch, with the Resolve conflicts button of this pull request or with a merge of the base branch on a checkout of it. Both leave the source branch as it is. - - Then merge this pull request with a merge commit. A squash merge leaves the source branch unmerged, so the next merge conflicts again. run: | gh auth setup-git git config user.name "github-actions[bot]" @@ -78,9 +74,15 @@ jobs: pull_request="$(gh pr list --head "$head" --base "$2" --json url --jq '.[0].url')" if [ -n "$pull_request" ]; then gh pr edit "$pull_request" --title "$title" - else - gh pr create --base "$2" --head "$head" --title "$title" --body "$BODY" + return fi + cat > "$RUNNER_TEMP/body.md" < [!IMPORTANT] + > Merge this PR with a merge commit, not a squash merge, so the next merge forward does not conflict again. + BODY + gh pr create --base "$2" --head "$head" --title "$title" --body-file "$RUNNER_TEMP/body.md" } releases="$(git branch --remotes --list 'origin/release/*' --format='%(refname:lstrip=3)' | grep -vE '\.0$' | sort -V || true)" for release in $releases; do From 96c68a1836fff124a361c8b161f3ecc11c9a2956 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:34:18 +0100 Subject: [PATCH 60/63] ci: name every step and job after what it does --- .github/workflows/build.yaml | 43 +++++++++++++---------- .github/workflows/check-upstream.yaml | 18 ++++++---- .github/workflows/clean-manifests.yaml | 8 +++-- .github/workflows/merge-forward.yaml | 5 +-- .github/workflows/pull-request.yaml | 16 +++++---- .github/workflows/release.yaml | 47 +++++++++++++++----------- .github/workflows/tests.yaml | 22 +++++++----- 7 files changed, 96 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ee467b..b212639 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,21 +24,23 @@ defaults: shell: bash jobs: - versions: - name: Read the versions + plan: + name: Plan build permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 outputs: - image: ${{ steps.versions.outputs.image }} - lines: ${{ inputs.lines || steps.versions.outputs.lines }} + image: ${{ steps.plan.outputs.image }} + lines: ${{ inputs.lines || steps.plan.outputs.lines }} steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - id: versions + - name: Plan the image version of each SQLite line + id: plan env: BRANCH: ${{ github.base_ref || github.ref_name }} PULL_REQUEST: ${{ github.event.pull_request.number }} @@ -48,15 +50,15 @@ jobs: printf 'image=%s\nlines=%s\n' "$image" "$lines" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.line.name }} ${{ matrix.platform }} - needs: versions + name: Build image (${{ matrix.line.name }}, ${{ matrix.platform }}) + needs: plan permissions: contents: read runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images timeout-minutes: 60 strategy: matrix: - line: ${{ fromJSON(needs.versions.outputs.lines) }} + line: ${{ fromJSON(needs.plan.outputs.lines) }} platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6, linux/riscv64, linux/ppc64le, linux/s390x, linux/386] include: - platform: linux/amd64 @@ -89,11 +91,12 @@ jobs: arch: "386" runner: ubuntu-26.04 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - name: Set the build arguments + - name: Read the commit date id: args run: | { @@ -101,17 +104,20 @@ jobs: echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" } >> "$GITHUB_OUTPUT" - - uses: docker/setup-qemu-action@v4.4.0 + - name: Set up QEMU if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@v4.4.0 with: image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false - - uses: docker/setup-buildx-action@v4.4.1 + - name: Set up Buildx + uses: docker/setup-buildx-action@v4.4.1 with: cache-binary: false - - uses: docker/build-push-action@v7.4.0 + - name: Build the image + uses: docker/build-push-action@v7.4.0 with: context: src platforms: ${{ matrix.platform }} @@ -119,14 +125,15 @@ jobs: outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true build-args: | SQLITE_VERSION=${{ matrix.line.version }} - IMAGE_VERSION=${{ needs.versions.outputs.image }} + IMAGE_VERSION=${{ needs.plan.outputs.image }} SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} CREATED=${{ steps.args.outputs.created }} GIT_SHA=${{ github.sha }} provenance: mode=max sbom: true - - uses: actions/upload-artifact@v7.0.2 + - name: Upload the image + uses: actions/upload-artifact@v7.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image @@ -135,9 +142,9 @@ jobs: tests: name: Tests - needs: [versions, build] + needs: [plan, build] permissions: contents: read uses: $/.github/workflows/tests.yaml with: - lines: ${{ needs.versions.outputs.lines }} + lines: ${{ needs.plan.outputs.lines }} diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index 0c030b5..0acc25d 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -19,7 +19,7 @@ defaults: jobs: lines: - name: List the lines of every branch + name: List SQLite lines permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images @@ -27,19 +27,21 @@ jobs: outputs: lines: ${{ steps.lines.outputs.lines }} steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false - - id: lines + - name: List the lines of main, the maintenance and the major release branches + id: lines run: | for branch in $(git branch --remotes --format='%(refname:lstrip=3)' | grep -xE 'main|v[0-9]+\.x|release/v[0-9]+\.0'); do git show "origin/$branch:src/upstream.json" | jq -c --arg branch "$branch" '.sqlite | keys[] | {$branch, line: .}' done | jq -sc '"lines=\(.)"' -r >> "$GITHUB_OUTPUT" check: - name: Check SQLite ${{ matrix.line }} on ${{ matrix.branch }} + name: Check sqlite.org (${{ matrix.branch }}, ${{ matrix.line }}) needs: lines permissions: actions: write # start the build workflow on the branch of the pull request @@ -52,16 +54,18 @@ jobs: matrix: include: ${{ fromJSON(needs.lines.outputs.lines) }} steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the branch + uses: actions/checkout@v7.0.1 with: ref: ${{ matrix.branch }} persist-credentials: false - - env: + - name: Check sqlite.org and pin a new SQLite release + env: LINE: ${{ matrix.line }} run: scripts/check-upstream.sh "$LINE" - - name: Open a pull request + - name: Open a pull request for the new release env: GH_TOKEN: ${{ github.token }} BASE: ${{ matrix.branch }} diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index e50bf5c..4a1ee32 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -19,7 +19,7 @@ defaults: jobs: clean: - name: Delete unused manifests + name: Clean manifests if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read @@ -29,7 +29,8 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 60 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -40,8 +41,9 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} run: scripts/clean-manifests.sh --delete 200 - - uses: actions/upload-artifact@v7.0.2 + - name: Upload the list of deleted manifests if: ${{ always() }} + uses: actions/upload-artifact@v7.0.2 with: name: deleted-manifests path: scripts/deleted-manifests.tsv diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml index fa35e31..d5e327c 100644 --- a/.github/workflows/merge-forward.yaml +++ b/.github/workflows/merge-forward.yaml @@ -27,12 +27,13 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false - - name: Merge main into the release branches and next, the release branches into next + - name: Merge main into the release branches and next, and each release branch into next env: GH_TOKEN: ${{ github.token }} run: | diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 1fb14e7..81ee0f8 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -24,7 +24,7 @@ jobs: uses: $/.github/workflows/build.yaml publish: - name: Publish pr-${{ github.event.pull_request.number }} + name: Publish image needs: build if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} permissions: @@ -32,12 +32,14 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/download-artifact@v8.0.2 + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 with: pattern: image-latest-* path: ${{ runner.temp }}/images - - uses: docker/login-action@v4.7.0 + - name: Log in to ghcr.io + uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -57,7 +59,7 @@ jobs: docker buildx imagetools create "${annotations[@]}" --tag "ghcr.io/peakimages/sqlite:$TAG" "${sources[@]}" size: - name: Compare the size with the latest release + name: Compare size with latest needs: build permissions: contents: read @@ -65,11 +67,13 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@v8.0.2 + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 with: pattern: image-latest-* path: ${{ runner.temp }}/images diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index db54ffd..f50019a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,22 +27,24 @@ defaults: shell: bash jobs: - version: - name: Select the releases + plan: + name: Plan releases if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 outputs: - releases: ${{ steps.version.outputs.releases }} + releases: ${{ steps.plan.outputs.releases }} steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false - - id: version + - name: Plan the releases + id: plan env: BRANCH: ${{ github.ref_name }} run: | @@ -51,17 +53,17 @@ jobs: build: name: Build - needs: version - if: ${{ needs.version.outputs.releases != '[]' }} + needs: plan + if: ${{ needs.plan.outputs.releases != '[]' }} permissions: contents: read uses: $/.github/workflows/build.yaml with: - lines: ${{ needs.version.outputs.releases }} + lines: ${{ needs.plan.outputs.releases }} publish: - name: Publish ${{ matrix.release.tag }} - needs: [version, build] + name: Publish release (${{ matrix.release.tag }}) + needs: [plan, build] permissions: contents: write # create the GitHub release and its tag packages: write # push the tested images to ghcr.io @@ -72,24 +74,28 @@ jobs: timeout-minutes: 30 strategy: matrix: - release: ${{ fromJSON(needs.version.outputs.releases) }} + release: ${{ fromJSON(needs.plan.outputs.releases) }} steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@v8.0.2 + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 with: pattern: image-${{ matrix.release.name }}-* path: ${{ runner.temp }}/images - - uses: docker/login-action@v4.7.0 + - name: Log in to ghcr.io + uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@v4.7.0 + - name: Log in to docker.io + uses: docker/login-action@v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -134,7 +140,7 @@ jobs: gh release create "$TAG" "${args[@]}" tags: - name: Move the tags + name: Move tags needs: publish permissions: contents: write # mark the GitHub release that latest points at as the latest release @@ -145,7 +151,8 @@ jobs: runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 10 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -156,13 +163,15 @@ jobs: tags="$(scripts/plan-version.sh --tags)" echo "tags=$tags" | tee -a "$GITHUB_OUTPUT" - - uses: docker/login-action@v4.7.0 + - name: Log in to ghcr.io + uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@v4.7.0 + - name: Log in to docker.io + uses: docker/login-action@v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index af6d930..5322ecc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,17 +18,19 @@ defaults: jobs: lint: - name: Lint + name: Lint repository permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - run: scripts/lint.sh + - name: Lint the repository + run: scripts/lint.sh - name: Verify the action lockfile env: @@ -39,7 +41,7 @@ jobs: gh actions-lock --verify test: - name: Test ${{ matrix.line.name }} ${{ matrix.platform }} + name: Test image (${{ matrix.line.name }}, ${{ matrix.platform }}) permissions: contents: read runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images @@ -80,17 +82,20 @@ jobs: arch: "386" runner: ubuntu-26.04 steps: - - uses: actions/checkout@v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/download-artifact@v8.0.2 + - name: Download the image + uses: actions/download-artifact@v8.0.2 with: name: image-${{ matrix.line.name }}-${{ matrix.arch }} path: ${{ runner.temp }}/image - - uses: docker/setup-qemu-action@v4.4.0 + - name: Set up QEMU if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@v4.4.0 with: image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 cache-image: false @@ -98,4 +103,5 @@ jobs: - name: Load the image run: tar -cf - -C "$RUNNER_TEMP/image" . | docker load - - run: scripts/test.sh peakimages/sqlite:test + - name: Test the image + run: scripts/test.sh peakimages/sqlite:test From 23a1019633487e6ed2a07f40719abceea4f98c72 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:07:41 +0100 Subject: [PATCH 61/63] chore(lint): bump jactionlint from v2.3.0 to v2.4.0 --- scripts/lint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lint.sh b/scripts/lint.sh index 9f65424..0624685 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -8,8 +8,8 @@ HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 # https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -# https://github.com/jdx/jactionlint/releases/tag/v2.3.0 -JACTIONLINT=ghcr.io/jdx/jactionlint:2.3.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.4.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.4.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 From a05377eea8099a7b05fd15abeef65d2782572104 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:25:14 +0100 Subject: [PATCH 62/63] ci: merge forward only after a release - next gets one merge per release, named after its tag, and no merge of main commits that are not released yet --- .github/workflows/merge-forward.yaml | 2 -- CHANGELOG.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml index d5e327c..9d948ea 100644 --- a/.github/workflows/merge-forward.yaml +++ b/.github/workflows/merge-forward.yaml @@ -1,8 +1,6 @@ name: Merge forward on: - schedule: - - cron: "40 8,19 * * *" workflow_dispatch: workflow_call: diff --git a/CHANGELOG.md b/CHANGELOG.md index 76ec8a2..422445d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Move `latest`, `preview` and every prefix of a SQLite version to their place among all published tags, so a release of an older line never moves a tag back - Check every SQLite line of `main`, the maintenance branches and the major release branches twice a day and open one pull request per bump, which supersedes the older ones - Verify a new SQLite release against its release notes and its tagged check-in on sqlite.org - - Merge `main` forward into `next` and the release branches twice a day and after every release + - Merge `main` forward into `next` and the release branches after every release - Tag releases in git and on GitHub with a leading `v`, `v3.54.1-v0.2.0`, the image tags stay `3.54.1-v0.2.0`, and rename the tags and releases of the earlier versions to match - Rework the CI workflows for the new release process - Split the build, test and release workflows, the release workflow calls the build workflow and the build workflow calls the tests From cd76036db729f33afd7053d88488f8ba1a7a2add Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:05:28 +0100 Subject: [PATCH 63/63] ci: build pull requests from branches that release - a push that changes release.json or src/upstream.json stays unbuilt as the release workflow builds it --- .github/workflows/pull-request.yaml | 30 ++++++++++++++++++++++++++++- RELEASE.md | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 81ee0f8..d488618 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -16,9 +16,37 @@ defaults: shell: bash jobs: + detect: + name: Detect release push + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + release: ${{ steps.detect.outputs.release }} + steps: + - name: Check whether the push releases + id: detect + env: + GH_TOKEN: ${{ github.token }} + ACTION: ${{ github.event.action }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_BRANCH: ${{ github.head_ref }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + release=false + if [ "$ACTION" = synchronize ] && [ "$HEAD_REPOSITORY" = "$GITHUB_REPOSITORY" ] \ + && [[ "$HEAD_BRANCH" =~ ^(main|next|v[0-9]+(\.[0-9]+)?\.x|release/v[0-9]+\.[0-9]+)$ ]]; then + files="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$AFTER" --jq '.files[].filename')" + ! grep -qxE 'release\.json|src/upstream\.json' <<< "$files" || release=true + fi + echo "release=$release" | tee -a "$GITHUB_OUTPUT" + build: name: Build - if: ${{ github.event.pull_request.head.repo.full_name != github.repository || !(github.head_ref == 'main' || github.head_ref == 'next' || startsWith(github.head_ref, 'release/') || endsWith(github.head_ref, '.x')) }} + needs: detect + if: ${{ needs.detect.outputs.release == 'false' }} permissions: contents: read uses: $/.github/workflows/build.yaml diff --git a/RELEASE.md b/RELEASE.md index 7f62bdb..80e9246 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -133,7 +133,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. - Pull requests from forks and Dependabot are built, not published -- Pull requests from `main`, `next`, a release branch or a patch branch are not built, the release workflow builds their commits +- A push to a pull request from `main`, `next`, a release branch or a patch branch that changes `release.json` or `src/upstream.json` is not built, the release workflow builds it - ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open ## Tags