diff --git a/.editorconfig b/.editorconfig index 66abfd4..687f4cd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,6 +16,13 @@ space_redirects = true [Dockerfile] indent_style = space -[*.{json,yml,yaml,md}] +[*.{yml,yaml,md}] indent_style = space indent_size = 2 + +[*.md] +indent_size = unset + +[*.lock] +indent_style = space +indent_size = 4 diff --git a/.gitattributes b/.gitattributes index 3ecad78..8a01f67 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,7 @@ * text=auto eol=lf *.md diff=markdown +CHANGELOG.md merge=union /.github export-ignore .editorconfig export-ignore diff --git a/.github/PULL_REQUEST_TEMPLATE/default.md b/.github/PULL_REQUEST_TEMPLATE/default.md index 049e486..33f8af0 100644 --- a/.github/PULL_REQUEST_TEMPLATE/default.md +++ b/.github/PULL_REQUEST_TEMPLATE/default.md @@ -1,9 +1,13 @@ - + ## Changes - +## Why + + + ## Issues - Fixes diff --git a/.github/jactionlint.yaml b/.github/jactionlint.yaml new file mode 100644 index 0000000..07d42f9 --- /dev/null +++ b/.github/jactionlint.yaml @@ -0,0 +1 @@ +profile: pedantic diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock new file mode 100644 index 0000000..b7ef44f --- /dev/null +++ b/.github/workflows/actions.lock @@ -0,0 +1,66 @@ +# This file is machine-generated by `gh actions-lock`. +# Do not edit by hand; run `gh actions-lock` to update. +# Docs: https://gh.io/actions-lockfile +version: 'v0.0.3' +workflows: + '.github/workflows/build.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.2' + - 'docker/build-push-action@v7.4.0' + - 'docker/setup-buildx-action@v4.4.1' + - 'docker/setup-qemu-action@v4.4.0' + '.github/workflows/check-upstream.yaml': + - 'actions/checkout@v7.0.1' + '.github/workflows/clean-manifests.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.2' + '.github/workflows/merge-forward.yaml': + - 'actions/checkout@v7.0.1' + '.github/workflows/pull-request.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'docker/login-action@v4.7.0' + '.github/workflows/release.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'docker/login-action@v4.7.0' + '.github/workflows/tests.yaml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' + - 'docker/setup-qemu-action@v4.4.0' +dependencies: + 'actions/checkout@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 + 'actions/download-artifact@v8.0.2': + ref: 'v8.0.2' + commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333' + owner_id: 44036562 + repo_id: 192626254 + 'actions/upload-artifact@v7.0.2': + ref: 'v7.0.2' + commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9' + owner_id: 44036562 + repo_id: 192625955 + 'docker/build-push-action@v7.4.0': + ref: 'v7.4.0' + commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc' + owner_id: 5429470 + repo_id: 241092383 + 'docker/login-action@v4.7.0': + ref: 'v4.7.0' + commit: 'sha1-81c6f93d4bc991e510b342a00b419b3bb4efc5c6' + owner_id: 5429470 + repo_id: 287743349 + 'docker/setup-buildx-action@v4.4.1': + ref: 'v4.4.1' + commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069' + owner_id: 5429470 + repo_id: 288485773 + 'docker/setup-qemu-action@v4.4.0': + ref: 'v4.4.0' + commit: 'sha1-99012661954931238ded8c8b007157a8430204e1' + owner_id: 5429470 + repo_id: 288487894 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..b212639 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,150 @@ +name: Build + +on: + schedule: + - cron: "50 7 * * 1" + workflow_dispatch: + workflow_call: + inputs: + lines: + description: The upstream lines to build as JSON, every line of src/upstream.json when empty + type: string + default: "" + +permissions: {} + +cache-mode: none + +concurrency: + group: build-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + plan: + name: Plan build + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + image: ${{ steps.plan.outputs.image }} + lines: ${{ inputs.lines || steps.plan.outputs.lines }} + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Plan the image version of each SQLite line + id: plan + env: + BRANCH: ${{ github.base_ref || github.ref_name }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: | + image="$(scripts/plan-version.sh --build)" + lines="$(jq -c '[.sqlite | to_entries[] | {name: .key, version: .value.version}]' src/upstream.json)" + printf 'image=%s\nlines=%s\n' "$image" "$lines" >> "$GITHUB_OUTPUT" + + build: + name: Build image (${{ matrix.line.name }}, ${{ matrix.platform }}) + needs: plan + permissions: + contents: read + runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images + timeout-minutes: 60 + strategy: + matrix: + line: ${{ fromJSON(needs.plan.outputs.lines) }} + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6, linux/riscv64, linux/ppc64le, linux/s390x, linux/386] + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-26.04 + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-26.04-arm + - platform: linux/arm/v7 + arch: armv7 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/arm/v6 + arch: armv6 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/riscv64 + arch: riscv64 + runner: ubuntu-26.04 + qemu: true + - platform: linux/ppc64le + arch: ppc64le + runner: ubuntu-26.04 + qemu: true + - platform: linux/s390x + arch: s390x + runner: ubuntu-26.04 + qemu: true + - platform: linux/386 + arch: "386" + runner: ubuntu-26.04 + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Read the commit date + id: args + run: | + { + echo "epoch=$(git log -1 --format=%ct)" + echo "created=$(TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" + } >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@v4.4.0 + with: + image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 + cache-image: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@v4.4.1 + with: + cache-binary: false + + - name: Build the image + uses: docker/build-push-action@v7.4.0 + with: + context: src + platforms: ${{ matrix.platform }} + tags: peakimages/sqlite:test + outputs: type=oci,dest=${{ runner.temp }}/image,tar=false,rewrite-timestamp=true,compression=zstd,compression-level=11,force-compression=true + build-args: | + SQLITE_VERSION=${{ matrix.line.version }} + IMAGE_VERSION=${{ needs.plan.outputs.image }} + SOURCE_DATE_EPOCH=${{ steps.args.outputs.epoch }} + CREATED=${{ steps.args.outputs.created }} + GIT_SHA=${{ github.sha }} + provenance: mode=max + sbom: true + + - name: Upload the image + uses: actions/upload-artifact@v7.0.2 + with: + name: image-${{ matrix.line.name }}-${{ matrix.arch }} + path: ${{ runner.temp }}/image + if-no-files-found: error + retention-days: 1 + + tests: + name: Tests + needs: [plan, build] + permissions: + contents: read + uses: $/.github/workflows/tests.yaml + with: + lines: ${{ needs.plan.outputs.lines }} diff --git a/.github/workflows/check-upstream.yaml b/.github/workflows/check-upstream.yaml index c8aaad5..0acc25d 100644 --- a/.github/workflows/check-upstream.yaml +++ b/.github/workflows/check-upstream.yaml @@ -2,46 +2,96 @@ name: Check upstream on: schedule: - - cron: "30 8 * * *" + - cron: "20 7,18 * * *" workflow_dispatch: permissions: {} +cache-mode: none + +concurrency: + group: check-upstream + cancel-in-progress: false + +defaults: + run: + shell: bash + jobs: + lines: + name: List SQLite lines + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + lines: ${{ steps.lines.outputs.lines }} + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: List the lines of main, the maintenance and the major release branches + id: lines + run: | + for branch in $(git branch --remotes --format='%(refname:lstrip=3)' | grep -xE 'main|v[0-9]+\.x|release/v[0-9]+\.0'); do + git show "origin/$branch:src/upstream.json" | jq -c --arg branch "$branch" '.sqlite | keys[] | {$branch, line: .}' + done | jq -sc '"lines=\(.)"' -r >> "$GITHUB_OUTPUT" + check: + name: Check sqlite.org (${{ matrix.branch }}, ${{ matrix.line }}) + needs: lines permissions: - contents: write - pull-requests: write + actions: write # start the build workflow on the branch of the pull request + contents: write # push the branch of the pull request + pull-requests: write # open the pull request runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.lines.outputs.lines) }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - name: Check out the branch + uses: actions/checkout@v7.0.1 with: + ref: ${{ matrix.branch }} persist-credentials: false - - run: scripts/check-upstream.sh + - name: Check sqlite.org and pin a new SQLite release + env: + LINE: ${{ matrix.line }} + run: scripts/check-upstream.sh "$LINE" - - name: Open a pull request + - name: Open a pull request for the new release env: GH_TOKEN: ${{ github.token }} + BASE: ${{ matrix.branch }} + LINE: ${{ matrix.line }} run: | git diff --quiet && exit 0 - version="$(jq -r .version src/version.json)" - source_id="$(jq -r .source_id src/version.json)" + pin() { jq -r --arg line "$LINE" ".sqlite[\$line].$1"; } + current="$(git show HEAD:src/upstream.json | pin version)" + version="$(pin version < src/upstream.json)" + source_id="$(pin source_id < src/upstream.json)" branch="feat/sqlite-v$version" - title="feat(release): bump SQLite version to v$version" + [ "$BASE" = main ] || branch="$branch-${BASE//\//-}" + title="feat(release): bump SQLite version from v$current to v$version" if [ -n "$(gh pr list --head "$branch" --json number --jq '.[0].number')" ]; then echo "a pull request from $branch is open already" exit 0 fi cat > "$RUNNER_TEMP/body.md" < [!IMPORTANT] > Squash merge this PR to keep the commit history clean. - BODY gh auth setup-git git config user.name "github-actions[bot]" @@ -49,4 +99,8 @@ jobs: git switch --create "$branch" git commit --all --quiet --message "$title" git push --quiet --set-upstream origin "$branch" - gh pr create --base main --head "$branch" --title "$title" --body-file "$RUNNER_TEMP/body.md" + new_pr="$(gh pr create --base "$BASE" --head "$branch" --title "$title" --label bot/upstream --body-file "$RUNNER_TEMP/body.md")" + gh workflow run build.yaml --ref "$branch" + for open_pr in $(gh pr list --label bot/upstream --base "$BASE" --json url,title | jq -r --arg from "from v$current to" '.[] | select(.title | contains($from)) | .url'); do + [ "$open_pr" = "$new_pr" ] || gh pr close "$open_pr" --delete-branch --comment "Superseded by $new_pr" + done diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml new file mode 100644 index 0000000..4a1ee32 --- /dev/null +++ b/.github/workflows/clean-manifests.yaml @@ -0,0 +1,51 @@ +name: Clean manifests + +on: + schedule: + - cron: "20 13 1 * *" + workflow_dispatch: + +permissions: {} + +cache-mode: none + +concurrency: + group: clean-manifests + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + clean: + name: Clean manifests + if: ${{ github.repository == 'peakimages/sqlite' }} + permissions: + contents: read + environment: + name: cleanup + deployment: false + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 60 + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Delete the unused manifests + env: + GH_TOKEN: ${{ secrets.GHCR_CLEANUP_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CLEANUP_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} + run: scripts/clean-manifests.sh --delete 200 + + - name: Upload the list of deleted manifests + if: ${{ always() }} + uses: actions/upload-artifact@v7.0.2 + with: + name: deleted-manifests + path: scripts/deleted-manifests.tsv + if-no-files-found: ignore + retention-days: 90 diff --git a/.github/workflows/merge-forward.yaml b/.github/workflows/merge-forward.yaml new file mode 100644 index 0000000..9d948ea --- /dev/null +++ b/.github/workflows/merge-forward.yaml @@ -0,0 +1,93 @@ +name: Merge forward + +on: + workflow_dispatch: + workflow_call: + +permissions: {} + +cache-mode: none + +concurrency: + group: merge-forward + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + merge: + name: Merge forward + permissions: + contents: write # push the merges + pull-requests: write # open a pull request when a merge conflicts + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Merge main into the release branches and next, and each release branch into next + env: + GH_TOKEN: ${{ github.token }} + run: | + gh auth setup-git + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + push() { + if [ "$(git rev-parse HEAD)" != "$(git rev-parse "origin/$1")" ]; then + awk 'NR > 1 && /^#/ && previous != "" { print "" } { print; previous = $0 }' CHANGELOG.md > "$RUNNER_TEMP/CHANGELOG.md" + cp "$RUNNER_TEMP/CHANGELOG.md" CHANGELOG.md + git diff --quiet || git commit --quiet --amend --no-edit --all + fi + git push --quiet origin "HEAD:$2" + } + merge() { + local head="chore/merge-${1//\//-}-into-${2//\//-}" tag title conflicts pull_request + git switch --quiet --detach "origin/$2" + if git merge-base --is-ancestor "origin/$1" HEAD; then + echo "$2 contains $1 already" + return + fi + tag="$(git tag --points-at "origin/$1" | sed -n 1p)" + title="chore: merge ${tag:-$1} into $2" + if git merge --quiet --no-edit --message "$title" "origin/$1"; then + push "$1" "$2" + return + fi + conflicts="$(git diff --name-only --diff-filter=U)" + git merge --abort + if [ "$conflicts" = release.json ] && git merge --quiet --no-edit --strategy-option ours --message "$title" "origin/$1"; then + push "$1" "$2" + return + fi + if git rev-parse --verify --quiet "origin/$head" > /dev/null && ! git merge-base --is-ancestor "origin/$head" "origin/$1"; then + echo "::notice::$head has commits of its own, $1 follows once its pull request is merged" + return + fi + git push --quiet origin "origin/$1:refs/heads/$head" + pull_request="$(gh pr list --head "$head" --base "$2" --json url --jq '.[0].url')" + if [ -n "$pull_request" ]; then + gh pr edit "$pull_request" --title "$title" + return + fi + cat > "$RUNNER_TEMP/body.md" < [!IMPORTANT] + > Merge this PR with a merge commit, not a squash merge, so the next merge forward does not conflict again. + BODY + gh pr create --base "$2" --head "$head" --title "$title" --body-file "$RUNNER_TEMP/body.md" + } + releases="$(git branch --remotes --list 'origin/release/*' --format='%(refname:lstrip=3)' | grep -vE '\.0$' | sort -V || true)" + for release in $releases; do + merge main "$release" + done + for release in $releases; do + merge "$release" next + done + merge main next diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..d488618 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,122 @@ +name: Pull request + +on: + pull_request: + +permissions: {} + +cache-mode: none + +concurrency: + group: pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + detect: + name: Detect release push + permissions: + contents: read + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 10 + outputs: + release: ${{ steps.detect.outputs.release }} + steps: + - name: Check whether the push releases + id: detect + env: + GH_TOKEN: ${{ github.token }} + ACTION: ${{ github.event.action }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_BRANCH: ${{ github.head_ref }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + release=false + if [ "$ACTION" = synchronize ] && [ "$HEAD_REPOSITORY" = "$GITHUB_REPOSITORY" ] \ + && [[ "$HEAD_BRANCH" =~ ^(main|next|v[0-9]+(\.[0-9]+)?\.x|release/v[0-9]+\.[0-9]+)$ ]]; then + files="$(gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$AFTER" --jq '.files[].filename')" + ! grep -qxE 'release\.json|src/upstream\.json' <<< "$files" || release=true + fi + echo "release=$release" | tee -a "$GITHUB_OUTPUT" + + build: + name: Build + needs: detect + if: ${{ needs.detect.outputs.release == 'false' }} + permissions: + contents: read + uses: $/.github/workflows/build.yaml + + publish: + name: Publish image + needs: build + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} + permissions: + packages: write # push the image of the pull request to ghcr.io + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 + with: + pattern: image-latest-* + path: ${{ runner.temp }}/images + + - name: Log in to ghcr.io + uses: docker/login-action@v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push the image of the pull request + env: + TAG: pr-${{ github.event.pull_request.number }} + run: | + sources=() + for index in "$RUNNER_TEMP"/images/*/index.json; do + sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") + done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" + docker buildx imagetools create "${annotations[@]}" --tag "ghcr.io/peakimages/sqlite:$TAG" "${sources[@]}" + + size: + name: Compare size with latest + needs: build + permissions: + contents: read + pull-requests: write # comment the size report on the pull request + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 + steps: + - name: Check out the repository + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 + with: + pattern: image-latest-* + path: ${{ runner.temp }}/images + + - name: Compare the size with the latest release + env: + IMAGE_NAME: pr-${{ github.event.pull_request.number }} + OUTPUT_FILE: ${{ runner.temp }}/size.md + run: | + scripts/size.sh "$RUNNER_TEMP/images" peakimages/sqlite:latest + cat "$OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY" + + - name: Comment the size on the pull request + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: gh pr comment "$PULL_REQUEST" --body-file "$RUNNER_TEMP/size.md" --edit-last --create-if-none diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1168a8d..f50019a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,190 +1,202 @@ name: Release on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true + push: + branches: + - main + - next + - v[0-9]+.x + - v[0-9]+.[0-9]+.x + - release/v[0-9]+.[0-9]+ + paths: + - release.json + - src/upstream.json + workflow_dispatch: permissions: {} -concurrency: release +cache-mode: none + +concurrency: + group: release + cancel-in-progress: false + queue: max + +defaults: + run: + shell: bash jobs: - version: + plan: + name: Plan releases + if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images - timeout-minutes: 30 + timeout-minutes: 10 outputs: - publish: ${{ steps.version.outputs.publish }} - release: ${{ steps.version.outputs.release }} - sqlite: ${{ steps.version.outputs.sqlite }} - image: ${{ steps.version.outputs.image }} - draft: ${{ steps.version.outputs.draft }} - tags: ${{ steps.version.outputs.tags }} - created: ${{ steps.version.outputs.created }} + releases: ${{ steps.plan.outputs.releases }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: + fetch-depth: 0 persist-credentials: false - - name: Read the versions - id: version + - name: Plan the releases + id: plan env: - GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.ref_name }} run: | - sqlite="$(jq -r .version src/version.json)" - image="$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)" - release="$sqlite-v$image" - publish=true - if gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$release" > /dev/null 2>&1 || gh release view "$release" > /dev/null 2>&1; then - echo "::notice::$release is published already, nothing to release" - publish=false - fi - # the first release of an image version is a draft with the notes from CHANGELOG.md - released="$(gh api --paginate "repos/$GITHUB_REPOSITORY/tags" --jq ".[].name | select(endswith(\"-v$image\"))")" - draft=false - [ -n "$released" ] || draft=true - { - echo "publish=$publish" - echo "release=$release" - echo "sqlite=$sqlite" - echo "image=$image" - echo "draft=$draft" - echo "tags=$release $sqlite ${sqlite%.*} ${sqlite%%.*} latest" - echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" - } >> "$GITHUB_OUTPUT" + releases="$(scripts/plan-version.sh --release)" + echo "releases=$releases" | tee -a "$GITHUB_OUTPUT" build: - needs: version - if: needs.version.outputs.publish == 'true' + name: Build + needs: plan + if: ${{ needs.plan.outputs.releases != '[]' }} permissions: contents: read - packages: write - runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images - timeout-minutes: 60 + uses: $/.github/workflows/build.yaml + with: + lines: ${{ needs.plan.outputs.releases }} + + publish: + name: Publish release (${{ matrix.release.tag }}) + needs: [plan, build] + permissions: + contents: write # create the GitHub release and its tag + packages: write # push the tested images to ghcr.io + environment: + name: release + deployment: false + runs-on: ubuntu-26.04 # https://github.com/actions/runner-images + timeout-minutes: 30 strategy: matrix: - include: - - platform: linux/amd64 - runner: ubuntu-26.04 - - platform: linux/arm64 - runner: ubuntu-26.04-arm - - platform: linux/arm/v7 - runner: ubuntu-26.04-arm - qemu: true - - platform: linux/riscv64 - runner: ubuntu-26.04 - qemu: true + release: ${{ fromJSON(needs.plan.outputs.releases) }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 - if: matrix.qemu - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - name: Download the tested images + uses: actions/download-artifact@v8.0.2 + with: + pattern: image-${{ matrix.release.name }}-* + path: ${{ runner.temp }}/images - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - name: Log in to ghcr.io + uses: docker/login-action@v4.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # https://github.com/docker/login-action/releases/tag/v4.6.0 + - name: Log in to docker.io + uses: docker/login-action@v4.7.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 - id: build - with: - context: src - platforms: ${{ matrix.platform }} - tags: | - ghcr.io/${{ github.repository }} - docker.io/${{ github.repository }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true - build-args: | - BUILD_DATE=${{ needs.version.outputs.created }} - GIT_SHA=${{ github.sha }} - provenance: mode=max - sbom: true - - - name: Save the digest + - name: Push the tested images env: - DIGEST: ${{ steps.build.outputs.digest }} + TAG: ${{ matrix.release.tag }} run: | - mkdir "$RUNNER_TEMP/digests" - touch "$RUNNER_TEMP/digests/${DIGEST#sha256:}" + sources=() + for index in "$RUNNER_TEMP"/images/*/index.json; do + sources+=("oci-layout://${index%/index.json}@$(jq -r '.manifests[0].digest' "$index")") + done + labels="$(docker buildx imagetools inspect "${sources[0]}" --format '{{ json .Image.Config.Labels }}' \ + | jq -r 'to_entries[] | "--annotation=index:\(.key)=\(.value)"')" + mapfile -t annotations <<< "$labels" + for repository in ghcr.io/peakimages/sqlite docker.io/peakimages/sqlite; do + docker buildx imagetools create "${annotations[@]}" --tag "$repository:${TAG#v}" "${sources[@]}" + done - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # https://github.com/actions/upload-artifact/releases/tag/v7.0.1 - with: - name: digest-${{ strategy.job-index }} - path: ${{ runner.temp }}/digests/* - if-no-files-found: error - retention-days: 1 + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ matrix.release.tag }} + IMAGE_VERSION: ${{ matrix.release.image }} + SQLITE_VERSION: ${{ matrix.release.version }} + PREVIOUS: ${{ matrix.release.previous }} + run: | + args=(--title "$TAG" --target "$GITHUB_SHA" --notes-file "$RUNNER_TEMP/notes.md" --latest=false) + heading="## [v$IMAGE_VERSION]" + [[ "$IMAGE_VERSION" != *-* ]] || { args+=(--prerelease); heading="## [Unreleased]"; } + if [ -z "$PREVIOUS" ]; then + awk -v heading="$heading" 'index($0, heading) == 1 { inside = 1; next } /^## / { inside = 0 } inside { sub(/^### /, "## "); print }' CHANGELOG.md > "$RUNNER_TEMP/notes.md" + else + cat > "$RUNNER_TEMP/notes.md" < "$RUNNER_TEMP/notes.md" - args=(--draft) - else - echo "SQLite $SQLITE, release notes: https://sqlite.org/releaselog/${SQLITE//./_}.html" > "$RUNNER_TEMP/notes.md" - fi - gh release create "$RELEASE" "${args[@]}" --target "$GITHUB_SHA" --title "$RELEASE" --notes-file "$RUNNER_TEMP/notes.md" + LATEST: ${{ fromJSON(steps.plan.outputs.tags).latest }} + run: gh release edit "$LATEST" --latest + + merge: + name: Merge forward + needs: publish + permissions: + contents: write # push the merges + pull-requests: write # open a pull request when a merge conflicts + uses: $/.github/workflows/merge-forward.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d74c4c9..5322ecc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,84 +1,107 @@ name: Tests on: - push: - branches: [main, next] - paths: - - src/** - - scripts/** - - .github/** - - .editorconfig - pull_request: - paths: - - src/** - - scripts/** - - .github/** - - .editorconfig - schedule: - - cron: "30 6 * * 1" - workflow_dispatch: + workflow_call: + inputs: + lines: + description: The upstream lines the build workflow built as JSON + type: string + required: true permissions: {} +cache-mode: none + +defaults: + run: + shell: bash + jobs: lint: + name: Lint repository permissions: contents: read runs-on: ubuntu-26.04 # https://github.com/actions/runner-images timeout-minutes: 30 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - run: scripts/lint.sh + - name: Lint the repository + run: scripts/lint.sh + + - name: Verify the action lockfile + env: + GH_TOKEN: ${{ github.token }} + # https://github.com/github/gh-actions-lock/releases/tag/v0.4.0 + run: | + gh extension install github/gh-actions-lock --pin v0.4.0 + gh actions-lock --verify test: - needs: lint + name: Test image (${{ matrix.line.name }}, ${{ matrix.platform }}) permissions: contents: read runs-on: ${{ matrix.runner }} # https://github.com/actions/runner-images - timeout-minutes: 60 + timeout-minutes: 30 strategy: fail-fast: false matrix: + line: ${{ fromJSON(inputs.lines) }} + platform: [linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6, linux/riscv64, linux/ppc64le, linux/s390x, linux/386] include: - platform: linux/amd64 + arch: amd64 runner: ubuntu-26.04 - platform: linux/arm64 + arch: arm64 runner: ubuntu-26.04-arm - platform: linux/arm/v7 + arch: armv7 + runner: ubuntu-26.04-arm + qemu: true + - platform: linux/arm/v6 + arch: armv6 runner: ubuntu-26.04-arm qemu: true - platform: linux/riscv64 + arch: riscv64 + runner: ubuntu-26.04 + qemu: true + - platform: linux/ppc64le + arch: ppc64le + runner: ubuntu-26.04 + qemu: true + - platform: linux/s390x + arch: s390x runner: ubuntu-26.04 qemu: true + - platform: linux/386 + arch: "386" + runner: ubuntu-26.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # https://github.com/actions/checkout/releases/tag/v7.0.1 + - name: Check out the repository + uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # https://github.com/docker/setup-qemu-action/releases/tag/v4.4.0 - if: matrix.qemu - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # https://github.com/docker/setup-buildx-action/releases/tag/v4.4.1 + - name: Download the image + uses: actions/download-artifact@v8.0.2 + with: + name: image-${{ matrix.line.name }}-${{ matrix.arch }} + path: ${{ runner.temp }}/image - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # https://github.com/docker/build-push-action/releases/tag/v7.4.0 + - name: Set up QEMU + if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@v4.4.0 with: - context: src - platforms: ${{ matrix.platform }} - load: true - tags: peakimages/sqlite:test + image: docker.io/tonistiigi/binfmt:qemu-v10.2.3-68@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 # https://github.com/tonistiigi/binfmt/releases/tag/deploy/v10.2.3-68 + cache-image: false - - run: scripts/test.sh peakimages/sqlite:test + - name: Load the image + run: tar -cf - -C "$RUNNER_TEMP/image" . | docker load - release: - needs: test - if: github.repository == 'peakimages/sqlite' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - permissions: - contents: write - packages: write - uses: $/.github/workflows/release.yaml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Test the image + run: scripts/test.sh peakimages/sqlite:test diff --git a/CHANGELOG.md b/CHANGELOG.md index d3937d9..422445d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,94 @@ All notable changes to this project will be documented in this file. SQLite version bumps are not listed, every release tag names the SQLite version it ships. +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + ## [Unreleased] +### Breaking Changes + +- Compress the layers with zstd instead of gzip, zstd is faster, more modern and compresses better than gzip, pulling needs Docker Engine 23.0 or later + +### Security + +- Harden the image against a checkout with a permissive umask, every file the build copies from the repository gets its mode from the Dockerfile, so no build can make `/usr`, `/usr/local` or `/usr/local/bin` writable for every user. +- Warn in `README.md` that a bind mount opens the databases to every user on the host who can reach the directory, and show how to close it +- Make release tags immutable + - Refuse to overwrite or delete a release tag such as `3.54.1-v0.2.0` on Docker Hub by using the ag mutability setting on docker hub with a strict patter. + - Refuse to updated, force push or delete the git tag of a release by using tag protection rule on GitHub +- Refuse to delete or force-push `main`, `next`, release and maintenance branches by using branch protection rules on GitHub +- Harden the CI workflows + - Grant every job the least permissions it needs and state the reason for every write permission + - Keep the Docker Hub credentials in the `release` environment, which only the branches that release can use, with a token that can push but not delete + - Keep the tokens that delete images in the `cleanup` environment, which only `main` can use and a maintainer approves for every run + - Lock every action and the actions it calls to a commit in `.github/workflows/actions.lock`, name actions by their release tag, and fail the lint job when the lockfile is stale + - Pin the binfmt image by tag and digest, it runs privileged and installs the QEMU of every emulated build + - Allow the workflows only the events they use, so `pull_request_target`, `workflow_run` and comment events start no run + - Run without the GitHub Actions cache, layer caches and cached tool binaries, so no run restores what another run wrote + - Lint the workflows with jactionlint pedantic, which includes the audits of zizmor, and fail on every warning + +### Added + +- Add the shell options sqlite.org recommends for a full-featured shell: `.recover` and `.dbinfo`, the `sqlite_dbpage` and `sqlite_stmt` virtual tables, the `sqlite_offset()` function, comments in `EXPLAIN` output and `EXPLAIN` of statements that call functions the shell does not know +- Build reproducibly, a rebuild of the same commit gives the same image digest per platform, and the `created` date of the image and its `org.opencontainers.image.created` label are the date of that commit +- Publish prereleases from `next` and the release branches, tagged like `3.54.1-v0.2.0-beta.1`, and the moving `preview` tag +- Publish older SQLite lines such as `3.53` next to the newest, built with the same image version and tagged like `3.53.4-v0.2.0`, `3.53.4` and `3.53` +- Builds for `linux/arm/v6`, `linux/ppc64le`, `linux/s390x` and `linux/386`, eight platforms in total +- Publish the image of every pull request from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, so a change can be tried before it is merged + +### Changed + +- Let any uid write the databases, `/var/lib/sqlite` is `0777` and the entrypoint creates databases with mode `0666`, which their `-wal` and `-shm` files take, so the application can run as any user and not only as `65532` or root +- Replace the release process, `RELEASE.md` describes it + - Keep the image versions in `release.json`, `stable.image` for `main` and the maintenance branches and `preview.image` for `next` and the release branches + - Pin SQLite in `src/upstream.json`, one entry per line, instead of `src/version.json` + - Release from `main`, `next`, release branches such as `release/v0.3` and maintenance branches such as `v1.x` and `v0.1.x` + - Select the stage and the version with `scripts/release.sh`, which offers the versions the branch allows and preselects the bump the commits need + - Publish nothing while `release.json` sets `0.0.0`, the version a new repository starts with, `scripts/release.sh` offers `0.1.0` or `1.0.0` as its first release + - Plan the image version, the releases and the moving tags with `scripts/plan-version.sh`, which also reports what a push publishes + - Publish every release a push of `release.json` or `src/upstream.json` names and whose tag does not exist yet, never as a draft + - Move `latest`, `preview` and every prefix of a SQLite version to their place among all published tags, so a release of an older line never moves a tag back + - Check every SQLite line of `main`, the maintenance branches and the major release branches twice a day and open one pull request per bump, which supersedes the older ones + - Verify a new SQLite release against its release notes and its tagged check-in on sqlite.org + - Merge `main` forward into `next` and the release branches after every release + - Tag releases in git and on GitHub with a leading `v`, `v3.54.1-v0.2.0`, the image tags stay `3.54.1-v0.2.0`, and rename the tags and releases of the earlier versions to match +- Rework the CI workflows for the new release process + - Split the build, test and release workflows, the release workflow calls the build workflow and the build workflow calls the tests + - Build every SQLite line for every platform once, test those builds and publish exactly them, not a second build + - Write the notes of the first release of an image version from `CHANGELOG.md`, later releases name the SQLite update + - Mark the GitHub release that `latest` points at as the latest release + - Run every step with bash, so a failing command in a pipe fails the step + - Queue release runs instead of cancelling them, so no two runs publish or move tags at the same time +- Remove the `SQLITE_UID` and `SQLITE_GID` build arguments, the user is `65532:65532` +- Rename the build argument `BUILD_DATE` to `CREATED`, add `SOURCE_DATE_EPOCH` and require `IMAGE_VERSION` +- Bump Alpine in the build stage from 3.24.2 to 3.24.3 + +### Fixed + +- Push nothing but the release index, its images and their attestations, the old release workflow pushed every platform as an index of its own by digest, which stayed behind untagged and unused next to the release index on ghcr.io and docker.io +- Copy the labels of the image to the annotations of the release index, ghcr.io reads the description and the license of an image of several platforms from there and showed neither +- List SQLite, tini and BusyBox in the SBOM attestation with their versions, licenses, package URLs and CPEs, from an SBOM document in `/var/lib/db/sbom` that takes the place of the package database a scratch image lacks +- Match `LICENSE` to the AGPL-3.0 text on gnu.org + +### Refactored + +- Name the final stage of the Dockerfile `final` + +### Maintenance + +- Add `scripts/clean-manifests.sh` and a monthly workflow that delete the manifests no tag needs from ghcr.io and docker.io, such as the untagged images the old release workflow left behind and the images of closed pull requests, and keep every release and every image a tag points at +- Add `scripts/size.sh` to measure the compressed and the on-disk size of every platform, and comment the size change of every pull request compared with the latest release +- Add `scripts/build.sh` to build the image locally with the image version CI builds with, for the platform of the Docker host or the platforms `PLATFORM` names +- Run docker build checks in `scripts/lint.sh` and stop at the first failed check +- Stop every script on arguments it does not take +- Indent JSON with tabs and format Markdown with prettier +- Bump `actions/download-artifact` to v8.0.2, `actions/upload-artifact` to v7.0.2 and `docker/login-action` to v4.7.0 +- Document the release process in `RELEASE.md` +- Describe the image in `README.md` as a central home for SQLite databases, explain that the application keeps its own SQLite engine, add copying the static `sqlite3` shell into other images and drop the uid `65532` note +- Add reproducible builds, PIE, detached runs and SQLite updates to `docs/comparison.md` and explain the UPX packing of `grepular/sqlite3` +- Put the scope first in `SECURITY.md` +- Add a Why section to the pull request template + ## [v0.1.0] - 2026-09-23 ### Added diff --git a/LICENSE b/LICENSE index 0ad25db..be3f7b2 100644 --- a/LICENSE +++ b/LICENSE @@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, diff --git a/README.md b/README.md index 5f37277..fa53581 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # SQLite -[![Tests](https://img.shields.io/github/actions/workflow/status/peakimages/sqlite/tests.yaml?branch=main&label=tests)](https://github.com/peakimages/sqlite/actions/workflows/tests.yaml) -[![SQLite](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Fpeakimages%2Fsqlite%2Fmain%2Fsrc%2Fversion.json&query=%24.version&label=sqlite)](https://github.com/peakimages/sqlite/releases) +[![Release](https://img.shields.io/github/actions/workflow/status/peakimages/sqlite/release.yaml?branch=main&label=release)](https://github.com/peakimages/sqlite/actions/workflows/release.yaml) +[![SQLite](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Fpeakimages%2Fsqlite%2Fmain%2Fsrc%2Fupstream.json&query=%24.sqlite.latest.version&label=sqlite)](https://github.com/peakimages/sqlite/releases) [![License](https://img.shields.io/github/license/peakimages/sqlite)](LICENSE) [![Docker pulls](https://img.shields.io/docker/pulls/peakimages/sqlite)](https://hub.docker.com/r/peakimages/sqlite) -[![Image size](https://img.shields.io/docker/image-size/peakimages/sqlite/latest)](https://hub.docker.com/r/peakimages/sqlite) +[![Image size](https://img.shields.io/docker/image-size/peakimages/sqlite/latest?arch=amd64)](https://hub.docker.com/r/peakimages/sqlite) ## Introduction -Minimal, production-ready, hardened and secure SQLite Docker image. Made for SQLite database deployments on [Coolify](https://coolify.io) and any other container host. +A central home for one or more SQLite databases. One minimal and hardened container creates every database you name, health checks them and brings the `sqlite3` shell for backups, restores and maintenance. Made for [Coolify](https://coolify.io?ref=peakimages.dev) and any other container host. > [!IMPORTANT] -> `peakimages/sqlite` is currently **v0** and subject to breaking changes prior to `v1.0.0`. Pin to a specific version like `peakimages/sqlite:3.53.4-v0.1.0` to avoid breaking changes. +> `peakimages/sqlite` is currently **v0** and subject to breaking changes prior to `v1.0.0`. Pin to a specific version like `peakimages/sqlite:3.54.1-v0.2.0` to avoid breaking changes. ### What Makes This Image Different? @@ -22,11 +22,11 @@ SQLite has no server and the database is just a file. Application images rarely - ✅ Zero Config Required: Production-ready defaults, customizable with environment variables - ✅ Multi-Database Support: One container creates and health checks every database you name - ✅ Verified Source: Size, SHA3-256 and source id of the official amalgamation are checked before each compilation -- ✅ Batteries Included: `FTS3`, `FTS4`, `FTS5`, `R*Tree`, `Geopoly`, `JSON`, `math`, `dbstat` and `soundex` SQLite extensions -- ✅ Modern Architecture: Native health checks, `tini` init, OCI labels, provenance and SBOM attestations -- ✅ Always Current: sqlite.org is checked daily and every new SQLite release becomes a new image -- ✅ Multi-Architecture: Builds for `linux/amd64`, `linux/arm64`, `linux/arm/v7` and `linux/riscv64` -- ✅ Tested: Every release is tested on all four platforms before it is published +- ✅ Batteries Included: `FTS3`, `FTS4`, `FTS5`, `R*Tree`, `Geopoly`, `math`, `dbstat`, `dbpage`, `stmt` and `soundex` SQLite extensions +- ✅ Modern Architecture: Native health checks, `tini` init, OCI labels, reproducible builds, provenance and SBOM attestations +- ✅ Always Current: sqlite.org is checked twice a day and new SQLite releases become new images +- ✅ Multi-Architecture: Builds for `linux/amd64`, `linux/arm64`, `linux/arm/v7`, `linux/arm/v6`, `linux/riscv64`, `linux/ppc64le`, `linux/s390x` and `linux/386` +- ✅ Tested: Every release is tested on all eight platforms before it is published ## Comparison @@ -34,6 +34,11 @@ To see a comparison of this image with other popular SQLite Docker images, see [ ## Getting Started +SQLite is not a server and the database is just a file. The application reads and writes it with the SQLite engine it bundles, such as `pdo_sqlite` for PHP or `rusqlite` for Rust, so the SQLite version and extensions of that engine apply, not the ones of this image. This container shares the volume with the application and looks after the files: it creates and configures them, health checks them and brings the `sqlite3` shell for backups, restores and maintenance. + +> [!NOTE] +> The layers of the image are compressed with zstd, pulling it needs Docker Engine 23.0 or later. + ### Docker Start the container with a named volume for the default database: @@ -46,13 +51,10 @@ docker run -d --name sqlite -v sqlite-data:/var/lib/sqlite peakimages/sqlite Add the SQLite container next to the application that needs a SQLite database, both mounting the same volume: -> [!IMPORTANT] -> The application must run as uid `65532`, the `nonroot` user of distroless images, or as root to write to the database. - ```yaml services: sqlite: - image: peakimages/sqlite:3.53.4-v0.1.0 + image: peakimages/sqlite:3.54.1-v0.2.0 environment: SQLITE_DATABASES: app.sqlite, jobs.sqlite volumes: @@ -60,7 +62,6 @@ services: app: image: your-app - user: "65532:65532" depends_on: sqlite: condition: service_healthy @@ -73,6 +74,14 @@ volumes: sqlite-data: ``` +### Shell for Other Images + +`sqlite3` is a static binary without dependencies. You can copy it into any image with a single line, and it works on `scratch`, distroless, Alpine, Debian and more, for backups, restores and maintenance without a package manager: + +```dockerfile +COPY --from=peakimages/sqlite:3.54.1-v0.2.0 /usr/local/bin/sqlite3 /usr/local/bin/sqlite3 +``` + ## Available Image Variations For now there is just one image variation. All tags follow this pattern: @@ -83,15 +92,16 @@ peakimages/sqlite:{{sqlite-version}}-v{{image-version}} | Variation | Best For | Example | | --------- | ------------------------------------------------------------------- | ------------------------------------------------------------ | -| default | Every SQLite deployment, from a Coolify service to a Kubernetes pod | `peakimages/sqlite:3.53.4-v0.1.0` `peakimages/sqlite:3.53.4` | +| default | Every SQLite deployment, from a Coolify service to a Kubernetes pod | `peakimages/sqlite:3.54.1-v0.2.0` `peakimages/sqlite:3.54.1` | ### Tags -| Tag | Meaning | -| --------------------- | ------------------------------------------------------------- | -| `3.53.4-v0.1.0` | Immutable: SQLite 3.53.4 built as image v0.1.0, never rebuilt | -| `3.53.4`, `3.53`, `3` | Newest build of that SQLite patch, minor or major version | -| `latest` | Newest build | +| Tag | Meaning | +| --------------------- | ----------------------------------------------------------------------------- | +| `3.54.1-v0.2.0` | Immutable: SQLite 3.54.1 built as image v0.2.0, never rebuilt | +| `3.54.1`, `3.54`, `3` | Newest release of that SQLite patch, minor or major version | +| `latest` | Newest release | +| `preview` | Newest prerelease, like `3.54.1-v0.3.0-beta.1`, until its release replaces it | ### Registries @@ -114,6 +124,6 @@ The defaults are set in the image, override them with `-e` or the `environment` ## Releases -Every SQLite release becomes a new version of the image. A check for new SQLite versions is run daily. +Every SQLite release becomes a new version of the image. A check for new SQLite versions runs twice a day. The image version follows semantic versioning independently of SQLite. Changes to the image are listed in [CHANGELOG.md](CHANGELOG.md). diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..80e9246 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,147 @@ +# Releases + +Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. + +> [!IMPORTANT] +> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/plan-version.sh` first. + +## Rules + +| Branch | Takes | Releases | +| -------------- | ---------------------------------------------------------- | -------------------------------------- | +| `main` | fixes, and `next` or a release branch for a new version | stable versions | +| `next` | features | prereleases of the next minor or major | +| `release/v1.4` | the minor 1.4, split off `next` so `next` can go on to 1.5 | prereleases of `1.4.0` | +| `release/v2.0` | the next major, cut from `next` | prereleases of `2.0.0` | +| `v1.x` | backports, once `main` is on v2 | stable `1.x` versions | +| `v1.2.x` | backports and hotfixes, cut from the newest tag of v1.2 | stable `1.2.x` versions | + +- `main` takes fixes only. A new minor or major reaches it as a merge of `next` or a release branch +- `next` and the release branches publish prereleases only, their stable versions are released on `main` +- Name a branch after what it changes: `fix/`, `feat/`, and `fix/-v1.2.2` for a backport, after the version that releases it +- Squash merge pull requests from contributors and pull requests with a single change, such as SQLite updates +- Merge everything else with a merge commit: team pull requests with several commits, backports, `next`, the release branches and `v1.3.x` into `main` +- The merge forward workflow merges `main` into `release/v1.4` and `next`, and `release/v1.4` into `next` +- While `main` is on 1.3, `v1.1.x` and `v1.2.x` are never merged into another branch +- Delete `release/v1.4` once `main` has released `1.4.0` +- Delete `v1.3.x` once it is merged into `main` +- Delete `v1.1.x` and `v1.2.x` once their release is published, the next backport cuts them again from the newest tag + +> [!CAUTION] +> Never squash merge `next` or a release branch. Their commits stay unmerged and come back as conflicts in every later merge. + +## Create a release + +> [!NOTE] +> `next` publishes prereleases only. For its stable version, merge `next` into `main` with a merge commit and create the release on `main`. + +1. Describe the changes under `## [Unreleased]` in `CHANGELOG.md`, they become the release notes +2. Check what the branch releases: + + ```sh + scripts/plan-version.sh + ``` + +3. Select the stage and the version, the script commits them and asks to push: + + ```sh + scripts/release.sh + ``` + +4. The pushed commit changes `release.json`, so it runs the release workflow, which builds, tests and publishes the release and moves its tags + +> [!TIP] +> To take the release commit back before the push: +> +> ```sh +> scripts/undo-release.sh +> ``` + +## Backport a fix + +`main` is on 1.3, and 1.1 and 1.2 still get fixes. A fix reaches each of them through a patch branch and a pull request of its own: + +1. Cut the patch branch from the newest tag of the minor, unless it exists: + + ```sh + git switch -c v1.2.x v3.54.1-v1.2.1 && git push -u origin v1.2.x + ``` + +2. Put the fix on a branch named after the version that releases it, cherry-picked from `main` when it is there already: + + ```sh + git switch -c fix/-v1.2.2 origin/v1.2.x && git cherry-pick -x + ``` + +3. Merge a pull request from `fix/-v1.2.2` into `v1.2.x` with a merge commit +4. Create the release on your checkout of `v1.2.x`: + + ```sh + git switch v1.2.x && git pull && scripts/release.sh + ``` + +5. Repeat with `v1.1.x` and `fix/-v1.1.4`. 1.0 gets no backports + +> [!IMPORTANT] +> A hotfix for 1.3 while `main` holds unreleased changes goes through `v1.3.x` the same way. Merge `v1.3.x` into `main` with a merge commit afterwards and delete it. `v1.1.x` and `v1.2.x` are never merged, their fixes come from `main`. + +## Add an older SQLite line + +1. Pin the line on a branch from `main`: + + ```sh + git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh 3.53 + ``` + +2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `v3.53.4-v1.3.0` +3. To stop the line, remove `3.53` from `src/upstream.json`, its tags stay where they are + +## Release a major version + +1. Cut `release/v2.0` from `next` and create its prereleases there with `scripts/release.sh`: + + ```sh + git switch -c release/v2.0 origin/next && git push -u origin release/v2.0 + ``` + +2. Merge `main` into `release/v2.0`, so v2 has every fix of v1 and the pull request into `main` has no conflicts: + + ```sh + git switch release/v2.0 && git pull && git merge origin/main && git push + ``` + +3. Keep v1 on a branch of its own: + + ```sh + git switch -c v1.x origin/main && git push -u origin v1.x + ``` + +4. Merge a pull request from `release/v2.0` into `main` with a merge commit +5. Create the release on `main` and select `2.0.0`: + + ```sh + git switch main && git pull && scripts/release.sh + ``` + +6. Delete `release/v2.0` + +> [!IMPORTANT] +> Cut `v1.x` from `main`, or bring an existing one up to date with `main`, before `release/v2.0` is merged into `main`. Afterwards `main` is v2 and v1 lives on `v1.x` alone. + +## Pull requests + +The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. + +- Pull requests from forks and Dependabot are built, not published +- A push to a pull request from `main`, `next`, a release branch or a patch branch that changes `release.json` or `src/upstream.json` is not built, the release workflow builds it +- ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open + +## Tags + +| Tag | Points at | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `3.54.1-v1.3.0` | one release, it never moves | +| `latest`, `3.54.1`, `3.54`, `3` | `3.54.1-v1.3.0`, the newest SQLite version of the newest stable image version | +| `3.53.4`, `3.53` | the newest stable image version released with SQLite 3.53, also once the line stops | +| `preview` | the newest prerelease of the lowest version in prerelease above `latest`, else `latest` | +| `pr-123` | the image of the newest push to an open pull request, on ghcr.io only | diff --git a/SECURITY.md b/SECURITY.md index 207cc25..4734f57 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,23 +1,25 @@ # Security -This page outlines our support policy and how to report vulnerabilities. +This page outlines what is in scope, our support policy and how to report vulnerabilities. + +## Scope + +Report a vulnerability to the project that maintains the affected code: + +- **The image** is built in this repository: the Dockerfile and its compile-time options, the entrypoint and health check scripts, the root filesystem and the workflows that test and publish it. Report it here, see [Reporting a Vulnerability](#reporting-a-vulnerability). +- **SQLite** is maintained at [sqlite.org](https://sqlite.org). Report it to the SQLite team, see [sqlite.org/cves.html](https://sqlite.org/cves.html). Every SQLite release becomes a new image, usually within a day. +- **BusyBox and tini** come from pinned Alpine packages. Report it to Alpine or the upstream project, the fix reaches the image with its next release. ## Support Policy -Below you can find the support status for different versions of the image. The image version is the `v` part of an image tag, `peakimages/sqlite:3.53.4-v0.1.0` is image version `v0.1.0`. +Below you can find the support status for different versions of the image. The image version is the `v` part of an image tag, `peakimages/sqlite:3.54.1-v0.2.0` is image version `v0.2.0`. | Version | Supported | Support Status | | ------- | ------------------ | ------------------ | | **1.x** | :white_check_mark: | Unreleased | | **0.x** | :white_check_mark: | Active Development | -Immutable tags such as `3.53.4-v0.1.0` are never rebuilt. A fix ships as a new image version, and the floating tags `3.53.4`, `3.53`, `3` and `latest` move to it. Deployments that pin an immutable tag receive fixes by moving to the newer tag. - -### Scope - -- **This repository** builds the image: the Dockerfile and its compile-time options, the entrypoint and health check scripts, the root filesystem and the workflows that test and publish. Report vulnerabilities in these here. -- **SQLite itself** is developed at [sqlite.org](https://sqlite.org). Report vulnerabilities in SQLite to the SQLite team, see [sqlite.org/cves.html](https://sqlite.org/cves.html). Every new SQLite release becomes a new image, usually within a day. -- **BusyBox and tini** come from pinned Alpine packages. Alpine's fixes reach the image with the next release. +Immutable tags such as `3.54.1-v0.2.0` are never rebuilt. A fix ships as a new image version, and the floating tags `3.54.1`, `3.54`, `3` and `latest` move to it. Deployments that pin an immutable tag receive fixes by moving to the newer tag. ## Reporting a Vulnerability @@ -26,7 +28,7 @@ If you find a security vulnerability in a currently supported version, please fo 1. **DO NOT** disclose the vulnerability publicly. 2. Report it via [GitHub Security Advisories](https://github.com/peakimages/sqlite/security/advisories/new). 3. Include in your report: - - **Explanation:** Detailed explanation of the vulnerability. - - **Affected Versions:** Which image tags are affected. - - **Reproduction:** Clear steps to reproduce the vulnerability. - - **Impact:** Assessment of the potential impact of the vulnerability. + - **Explanation:** Detailed explanation of the vulnerability. + - **Affected Versions:** Which image tags are affected. + - **Reproduction:** Clear steps to reproduce the vulnerability. + - **Impact:** Assessment of the potential impact of the vulnerability. diff --git a/docs/comparison.md b/docs/comparison.md index ff4363c..399f5d5 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -2,42 +2,53 @@ `peakimages/sqlite` compared to other popular SQLite Docker images. -| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | -| -------------------------------- | ------------------------------------------------------- | ------------------------------------ | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | -| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | -| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | -| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | -| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | -| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | -| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | -| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | -| Recommended compile options[^2] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Build[^3] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | -| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Init[^4] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | -| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | -| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | -| No runtime extension loading[^5] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | -| Compressed[^6] | 1.55 MB | 0.57 MB | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | -| On disk[^6] | 2.7 MB | 0.57 MB | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | -| Platforms | amd64, arm64, arm/v7, riscv64 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | -| Extensions[^7] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | -| SQLite | 3.53.4 | 3.53.4 | 3.53.4 | 3.53.4 | 3.53.4 | 3.34.1 (2021) | 3.46.1 (2024) | 3.28.0 (2019) | -| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | +| | `peakimages/sqlite` | `grepular/sqlite3` | `keinos/sqlite3` | `alpine/sqlite` | `cleanstart/sqlite3` | `rapidfort/sqlite-ib` | `nouchka/sqlite3` | `dockerpinata/sqlite` | +| -------------------------------- | ---------------------------------------------------------- | --------------------------------------- | -------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---------------------------------------- | +| Base | scratch | scratch | Alpine 3.24 | Alpine 3.24 | CleanStart Linux, distroless | Red Hat UBI 9 | Debian 13 | Alpine 3.10 | +| Rootless | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | +| User | 65532 (nonroot) | root | 100 (sqlite) | root | 65532 (clnstrt) | 1000 (dbuser) | root | root | +| Writable working directory[^1] | ✅ (`/var/lib/sqlite`, owned by the user) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ⚠️ (`/`, as root) | ❌ (`/`, owned by root) | ✅ (`/home/dbuser`, owned by the user) | ⚠️ (`/root/db`, as root) | ⚠️ (`/`, as root) | +| Open source | ✅ (`AGPL-3.0`) | ✅ (`AGPL-3.0`) | ✅ (`MIT`) | ❌ (no public source) | ❌ (proprietary) | ✅ (`MIT`, build scripts) | ⚠️ (source public, no license) | ❌ (no public source) | +| Custom build | ✅ (official amalgamation) | ✅ (official amalgamation) | ✅ (official source) | ❌ (Alpine package) | ❌ (unknown, build not public) | ❌ (Red Hat package) | ❌ (Debian package) | ❌ (Alpine package) | +| Verified source | ✅ (size, SHA3-256, source id) | ❌ (unchecked download) | ❌ (unchecked download) | ✅ (by Alpine) | ❌ (build not public) | ✅ (by Red Hat) | ✅ (by Debian) | ✅ (by Alpine) | +| Reproducible build[^2] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Recommended compile options[^3] | ✅ | ⚠️ (`THREADSAFE=0` only) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Build[^4] | static | static | static | dynamic | dynamic | dynamic | dynamic | dynamic | +| PIE[^5] | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Native health check | ✅ | ❌ | ⚠️ (checks the binary, not a database) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Customizable and flexible | ✅ (environment variables) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Multiple databases per container | ✅ (created and health checked) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Runs detached[^6] | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Init[^7] | ✅ (tini) | ❌ | ✅ (tini) | ❌ | ❌ | ❌ | ❌ | ❌ | +| No shell | ❌ (BusyBox) | ✅ | ❌ (BusyBox) | ❌ (BusyBox) | ✅ | ❌ (bash) | ❌ (bash) | ❌ (BusyBox) | +| No package manager | ✅ | ✅ | ❌ (apk) | ❌ (apk) | ✅ | ✅ | ❌ (apt) | ❌ (apk) | +| No runtime extension loading[^8] | ✅ (omitted at compile time) | ✅ (side effect of the static build) | ✅ (side effect of the static build) | ❌ | ❌ | ❌ | ❌ | ❌ | +| Compressed[^9] | 1.55 MB | 0.57 MB[^10] | 4.8 MB | 5.6 MB | 33.2 MB | 6.3 MB | 30.4 MB | 5.6 MB | +| On disk[^9] | 2.7 MB | 0.57 MB[^10] | 10.2 MB | 11.8 MB | 39.1 MB | 16.5 MB | 81.9 MB | 13.2 MB | +| Platforms | amd64, arm64, arm/v7, arm/v6, riscv64, ppc64le, s390x, 386 | amd64, arm64 | amd64, arm64, arm/v6, arm/v7 | amd64, arm64, arm/v6, arm/v7, ppc64le, s390x | amd64, arm64 | amd64 | amd64 | amd64 | +| Extensions[^11] | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex | FTS3, FTS4, FTS5, RTree | ❌ | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, dbstat, session | FTS3, FTS4, FTS5, RTree, math, dbstat, soundex, session | FTS3, FTS4, FTS5, RTree, Geopoly, dbstat | +| SQLite updates | ✅ (checked twice a day, merged by hand) | ✅ (checked daily, built automatically) | ⚠️ (checked weekly, merged by hand) | ⚠️ (follows Alpine releases) | ⚠️ (policy not public) | ❌ (3.34.1, 2021) | ❌ (3.46.1, 2024) | ❌ (3.28.0, 2019) | +| Maintained | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| OCI labels | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | [^1]: `sqlite3 database.sqlite` writes into the working directory, and a new named volume takes the owner of the directory it is mounted on. `keinos/sqlite3` and `cleanstart/sqlite3` run their user in `/`, which root owns, so both fail with `unable to open database file`. -[^2]: sqlite.org lists [these options](https://sqlite.org/compile.html#recommended_compile_time_options) "for applications that are able to use them" to save CPU cycles and memory, and to turn off legacy misfeatures. `SQLITE_THREADSAFE=0` alone makes SQLite about 2% faster and smaller. +[^2]: Building the same commit again gives the same image byte for byte, so a rebuild shows whether a published image really comes from its source. The image manifest of each platform keeps its digest, while the digest of a tag changes with every build because the provenance attestation records the build run. -[^3]: A static binary carries its own libc and needs nothing from the OS, so the same binary runs on every Linux distribution and there are no shared libraries to keep patched. +[^3]: sqlite.org lists [these options](https://sqlite.org/compile.html#recommended_compile_time_options) "for applications that are able to use them" to save CPU cycles and memory, and to turn off legacy misfeatures. `SQLITE_THREADSAFE=0` alone makes SQLite about 2% faster and smaller. -[^4]: PID 1 ignores signals it has no handler for, so without an init `docker stop` waits out its grace period and then kills the container. +[^4]: A static binary carries its own libc and needs nothing from the OS, so the same binary runs on any Linux distribution and there are no shared libraries to keep patched. -[^5]: `load_extension()` lets any SQL that reaches the shell run native code from a file. +[^5]: A position-independent executable is loaded at a random memory address on every start, so an exploit cannot rely on known memory addresses. -[^6]: linux/arm64 where published, linux/amd64 otherwise. +[^6]: Most of the other images run the interactive `sqlite3` shell as their main process, which exits when its input ends. With `docker run -d` or as a Compose service, the container stops with exit code 0 unless `-i` or `-t` (`stdin_open` or `tty` in Compose) is set. A running container allows database health checks, scheduled tasks, backups and debugging through its shell. -[^7]: JSON is built into SQLite since 3.38.0 and therefore not listed. A distribution package ships the extensions the distribution chose, which is why the Red Hat build has fewer than the amalgamation offers. +[^7]: PID 1 ignores signals it has no handler for, so without an init, `docker stop` waits out its grace period and then kills the container. + +[^8]: `load_extension()` lets any SQL that reaches the shell run native code from a file. + +[^9]: Measured for `linux/amd64`. + +[^10]: `grepular/sqlite3` packs its binary with UPX, which unpacks it into memory on every start: every `sqlite3` run, a health check too, starts about 200 times slower (25 ms instead of 0.1 ms on arm64), every process keeps its own unpacked copy in memory instead of sharing one, and antivirus scanners may flag the file, as malware often uses UPX. It pays on every start to save on size, on a download that happens once. + +[^11]: JSON has been built into SQLite since 3.38.0 and is not listed. A distribution package ships the extensions its distribution chose, and SQLite before 3.35.0 has no math functions. diff --git a/release.json b/release.json new file mode 100644 index 0000000..81e6a8d --- /dev/null +++ b/release.json @@ -0,0 +1,8 @@ +{ + "stable": { + "image": "0.1.0" + }, + "preview": { + "image": "0.2.0-beta.1" + } +} diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..0814a69 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +IMAGE="${1:-peakimages/sqlite:local}" +PLATFORM="${PLATFORM:-}" + +BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' + +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." + +main() { + [[ "$OPTION" =~ ^([^-][^[:space:]]*)?$ ]] || die "'$OPTION' is not an option, expected an image or none" + build +} + +build() { + local image_version + [ -z "${CI:-}" ] || die "scripts/build.sh is for local builds, CI builds with docker/build-push-action" + image_version="$(scripts/plan-version.sh --build)" + docker buildx build ${PLATFORM:+--platform "$PLATFORM"} --build-arg IMAGE_VERSION="$image_version" \ + --provenance mode=max --sbom true --load --tag "$IMAGE" src + ok "$IMAGE is built for ${PLATFORM:-the platform of the Docker host} with image version $BOLD$image_version$RESET" +} + +main diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 43f4f1f..61944c2 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" +LINE="${1:-latest}" +VERSION_PATTERN='[0-9]+\.[0-9]+\.[0-9]+' WORK_DIR="$(mktemp -d)" -BOLD=$'\033[1m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } @@ -16,56 +19,85 @@ cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"' EXIT main() { - read_latest_release + [[ "$OPTION" =~ ^(latest|[0-9]+(\.[0-9]+)*)?$ ]] || die "'$OPTION' is not an option, expected a line such as 3.53, latest or none" + read_newest_release compare_with_pinned download_amalgamation - write_version_json + verify_release_notes + verify_release_check_in + write_upstream_json verify_amalgamation update_dockerfile } -read_latest_release() { - local release - fetch "$WORK_DIR/download.html" https://sqlite.org/download.html - release="$(grep -Em 1 '^PRODUCT,[0-9.]+,[0-9]{4}/sqlite-amalgamation-[0-9]+\.zip,[0-9]+,[0-9a-f]{64}$' "$WORK_DIR/download.html")" \ - || die "no amalgamation listed on sqlite.org/download.html" - IFS=, read -r _ version path size sha3 <<< "$release" - url="https://sqlite.org/$path" +read_newest_release() { + local release year number + fetch "$WORK_DIR/chronology.html" https://sqlite.org/chronology.html + release="$(sed -nE "s|.*>([0-9]{4})-[0-9]{2}-[0-9]{2}\$|year \1|p; s|.*data-sortkey='([0-9]+)'>($VERSION_PATTERN).*|version \1 \2|p" "$WORK_DIR/chronology.html" \ + | awk -v line="$LINE." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ + | sort -k3,3V | tail -n 1)" + [ -n "$release" ] || die "sqlite.org/chronology.html lists no SQLite $LINE release" + read -r year number new_version <<< "$release" + url="https://sqlite.org/$year/sqlite-amalgamation-$number.zip" } compare_with_pinned() { - local pinned - pinned="$(jq -r .version src/version.json)" - if [ "$version" = "$pinned" ]; then - ok "SQLite $BOLD$pinned$RESET is pinned and the current release on sqlite.org" + local current_version + current_version="$(jq -r --arg line "$LINE" '.sqlite[$line].version // empty' src/upstream.json)" + if [ "$new_version" = "$current_version" ]; then + ok "SQLite $BOLD$current_version$RESET is pinned as $LINE and the newest release of it on sqlite.org" exit 0 fi - info "sqlite.org lists SQLite $BOLD$version$RESET, $pinned is pinned" + info "sqlite.org lists SQLite $BOLD$new_version$RESET, ${current_version:-no version} is pinned as $LINE" } download_amalgamation() { fetch "$WORK_DIR/sqlite.zip" "$url" + size="$(wc -c < "$WORK_DIR/sqlite.zip" | tr -d ' ')" + sha3="$(openssl dgst -sha3-256 -r "$WORK_DIR/sqlite.zip" | cut -d ' ' -f 1)" source_id="$(unzip -p "$WORK_DIR/sqlite.zip" '*/sqlite3.h' | sed -nE 's/^#define SQLITE_SOURCE_ID +"([^"]+)".*/\1/p')" [ -n "$source_id" ] || die "sqlite3.h in the download does not define SQLITE_SOURCE_ID" ok "downloaded ${url##*/}, $size bytes" } -write_version_json() { - jq -n --arg version "$version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ - '{$version, $url, $size, $sha3_256, $source_id}' > src/version.json - ok "src/version.json pins $BOLD$version$RESET with url, size, SHA3-256 and source id $source_id" +verify_release_notes() { + local sqlite3_c + fetch "$WORK_DIR/release.html" "https://sqlite.org/releaselog/${new_version//./_}.html" + sqlite3_c="$(unzip -p "$WORK_DIR/sqlite.zip" '*/sqlite3.c' | openssl dgst -sha3-256 -r | cut -d ' ' -f 1)" + grep -qxF "
  • SQLITE_SOURCE_ID: $source_id" "$WORK_DIR/release.html" || die "the release notes of SQLite $new_version list another source id than $source_id" + grep -qxF "
  • SHA3-256 for sqlite3.c: $sqlite3_c" "$WORK_DIR/release.html" || die "the release notes of SQLite $new_version list another SHA3-256 for sqlite3.c than $sqlite3_c" + ok "the release notes of SQLite $new_version list the source id and the SHA3-256 of sqlite3.c in the download" +} + +verify_release_check_in() { + local check_in tag + check_in="${source_id##* }" + fetch "$WORK_DIR/manifest" "https://sqlite.org/src/raw/$check_in" + for tag in release "version-$new_version"; do + grep -qxF "T +sym-$tag *" "$WORK_DIR/manifest" || die "check-in ${check_in:0:10} on sqlite.org/src is not tagged $tag" + done + ok "check-in ${check_in:0:10} is tagged release and ${BOLD}version-$new_version$RESET on sqlite.org/src" +} + +write_upstream_json() { + jq --tab --arg line "$LINE" --arg version "$new_version" --arg url "$url" --argjson size "$size" --arg sha3_256 "$sha3" --arg source_id "$source_id" \ + '.sqlite[$line] = {$version, $url, $size, $sha3_256, $source_id}' src/upstream.json > "$WORK_DIR/upstream.json" + cp "$WORK_DIR/upstream.json" src/upstream.json + ok "src/upstream.json pins $BOLD$new_version$RESET as $LINE with url, size, SHA3-256 and source id $source_id" } verify_amalgamation() { - cp src/verify.sh src/version.json "$WORK_DIR" - SQLITE_VERSION="$version" sh "$WORK_DIR/verify.sh" > /dev/null - ok "src/verify.sh accepts the download against the new src/version.json" + cp src/verify.sh "$WORK_DIR" + jq --arg line "$LINE" '.sqlite[$line]' src/upstream.json > "$WORK_DIR/sqlite.json" + SQLITE_VERSION="$new_version" sh "$WORK_DIR/verify.sh" > /dev/null + ok "src/verify.sh accepts the download against the new pin" } update_dockerfile() { - sed -E "s|^ARG SQLITE_VERSION=.*|ARG SQLITE_VERSION=$version|" src/Dockerfile > "$WORK_DIR/Dockerfile" + [ "$LINE" = latest ] || return 0 + sed -E "s|^ARG SQLITE_VERSION=.*|ARG SQLITE_VERSION=$new_version|" src/Dockerfile > "$WORK_DIR/Dockerfile" cp "$WORK_DIR/Dockerfile" src/Dockerfile - ok "src/Dockerfile sets SQLITE_VERSION $BOLD$version$RESET" + ok "src/Dockerfile sets SQLITE_VERSION $BOLD$new_version$RESET" } fetch() { diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh new file mode 100755 index 0000000..964f394 --- /dev/null +++ b/scripts/clean-manifests.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +LIMIT="${2:-}" +IMAGE=peakimages/sqlite +KEEP_TAG_PATTERN='^(latest|preview|[0-9]+(\.[0-9]+){0,2}|[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' +KEEP_TAG_EXCEPTIONS="" +DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" +DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +TO_DELETE_FILE=scripts/manifests-to-delete.tsv +DELETED_FILE=scripts/deleted-manifests.tsv +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" + read_ghcr_versions + read_dockerhub_tags + select_kept_tags + read_manifests + select_kept_manifests + heading "Selecting what to delete" + find_dockerhub_only_manifests + write_list + delete_manifests + verify_kept_manifests +} + +read_ghcr_versions() { + local page window=$((${LIMIT:-0} * 2)) + list_ghcr_versions "state=active" --paginate > "$WORK_DIR/ghcr-active.tsv" + for ((page = 1; page <= (window + 99) / 100; page++)); do + list_ghcr_versions "state=deleted&page=$page" + done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" + cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" +} + +read_dockerhub_tags() { + local token page last="" + token="$(registry_token docker.io pull)" + while :; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ + || die "could not list the tags of docker.io/$IMAGE" + [ -n "$page" ] || break + echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" + last="$(tail -n 1 <<< "$page")" + done + request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ + > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" +} + +select_kept_tags() { + { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" + git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ + || die "could not list the git tags of github.com/$IMAGE" + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + || die "gh could not list the open pull requests of $IMAGE" + tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + { + grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true + grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true + } | sort -u > "$WORK_DIR/kept-tags.txt" + grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, every git tag of github.com/$IMAGE and the tags of open pull requests" +} + +read_manifests() { + local digest tag + mkdir "$WORK_DIR/manifests" + request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ + || die "could not read the manifests of ghcr.io/$IMAGE" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" + awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" + while IFS=$'\t' read -r digest tag; do + curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ + || die "could not read the manifests $tag points at on docker.io" + done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ + >> "$WORK_DIR/children.tsv" + manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" + manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ + > "$WORK_DIR/kinds.tsv" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" +} + +select_kept_manifests() { + local kinds + kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" + awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ + || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" + kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" +} + +find_dockerhub_only_manifests() { + awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" + request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ + > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ + || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" +} + +write_list() { + awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + FILENAME == ARGV[1] { kept[$1]; next } + FILENAME == ARGV[2] { kind[$1] = $2; next } + FILENAME == ARGV[3] { size[$1] = $2; next } + FILENAME == ARGV[4] { size[$1] += $3; next } + FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } + FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } + FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } + FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } + !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ + "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" + write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" + ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" + ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" + info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" +} + +delete_manifests() { + local on_ghcr type tags version digest rest name deleted ghcr docker_hub status + if [ -z "$LIMIT" ]; then + info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + return + fi + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" + heading "Deleting" + write_table /dev/null "$DELETED_FILE" + while IFS='|' read -r on_ghcr type tags version digest rest; do + name="$digest${tags:+ ($tags)}" + deleted="" ghcr=no docker_hub=no + if [ "$on_ghcr" = yes ]; then + delete_ghcr_version "$version" "$type $name" || continue + deleted=ghcr.io ghcr=yes + fi + status="$(delete_dockerhub_manifest "$digest")" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + fi + if [ -n "$deleted" ]; then + ok "deleted the $type $BOLD$name$RESET from $deleted" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + fi + case "$status" in + 2?? | 404) ;; + 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; + *) die "could not delete the $type $name from docker.io, it answered $status" ;; + esac + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') + ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" +} + +verify_kept_manifests() { + local registry missing + [ -n "$LIMIT" ] || return 0 + for registry in ghcr.io docker.io; do + missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ + || die "could not check the kept manifests on $registry" + [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + done + ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" +} + +list_ghcr_versions() { + local query="$1" + shift + gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ + --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ + || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" +} + +request_manifests() { + local registry="$1" list="$2" chunk token + shift 2 + [ -s "$list" ] || return 0 + split -l 1000 "$list" "$WORK_DIR/chunk-" + for chunk in "$WORK_DIR"/chunk-*; do + token="$(registry_token "$registry" pull)" + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + rm "$chunk" + done +} + +manifest_fields() { + find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + +} + +kept_manifests() { + awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } + FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[2] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" +} + +write_table() { + awk -F '\t' -v OFS='\t' 'function human(bytes, unit) { for (unit = 1; bytes >= 1000 && unit < 5; unit++) bytes /= 1000; return sprintf(unit == 1 ? "%d %s" : "%.1f %s", bytes, units[unit]) } + function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } + FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } + FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2] } + END { + for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), + "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" +} + +delete_ghcr_version() { + local error + until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do + case "$error" in + *"rate limit"*) sleep 60 ;; + *downloads*) + warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + return 1 + ;; + *) die "could not delete the $2 from ghcr.io: $error" ;; + esac + done + sleep 1 +} + +delete_dockerhub_manifest() { + local status + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ + "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + sleep 60 + done + echo "$status" +} + +registry_token() { + case "$1" in + ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + esac | jq -r .token +} + +count() { awk 'END { print NR }'; } + +main diff --git a/scripts/lint.sh b/scripts/lint.sh index 071eec7..0624685 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,41 +1,42 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" +# https://github.com/hadolint/hadolint/releases/tag/v2.15.1 HADOLINT=hadolint/hadolint:v2.15.1 +# https://github.com/koalaman/shellcheck/releases/tag/v0.11.0 SHELLCHECK=koalaman/shellcheck:v0.11.0 +# https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -ACTIONLINT=kjanat/actionlint:1.17.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.4.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.4.0 +# https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 -BOLD=$'\033[1m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } -err() { printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2; } die() { - err "$@" + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 exit 1 } cd "$(dirname "$0")/.." -failed=0 main() { + [[ "$OPTION" =~ ^(--fix)?$ ]] || die "'$OPTION' is not an option, expected --fix or none" command -v docker > /dev/null || die "docker is not installed" docker info > /dev/null 2>&1 || die "docker is not running" - case "${1:-}" in - "") ;; - --fix) format ;; - *) die "usage: scripts/lint.sh [--fix]" ;; - esac + [ "$OPTION" != --fix ] || format heading "Linting" + run_build_checks check src/Dockerfile hadolint "$HADOLINT" hadolint src/Dockerfile check "the shell scripts" ShellCheck "$SHELLCHECK" scripts/*.sh src/*.sh src/rootfs/usr/local/bin/*.sh check "the shell scripts" shfmt "$SHFMT" -d scripts src - check "the workflows" actionlint "$ACTIONLINT" + check "the workflows" jactionlint "$JACTIONLINT" --strict-exit check "every file" editorconfig-checker "$EDITORCONFIG_CHECKER" editorconfig-checker -exclude '^LICENSE$' - [ "$failed" = 0 ] || die "$failed of 5 checks failed" heading "All checks passed" } @@ -46,15 +47,16 @@ format() { ok "shfmt and editorconfig-checker wrote their fixes" } +run_build_checks() { + docker build --check --progress=quiet src | sed 1d || die "src/Dockerfile failed docker build --check" + ok "src/Dockerfile passed docker build --check" +} + check() { local files="$1" linter="$2" image="$3" shift 3 - if tool "$image" "$@"; then - ok "$files passed $linter" - else - err "$files failed $linter" - failed=$((failed + 1)) - fi + tool "$image" "$@" || die "$files failed $linter" + ok "$files passed $linter" } tool() { @@ -65,4 +67,4 @@ tool() { docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/mnt" -w /mnt "$@" } -main "$@" +main diff --git a/scripts/plan-version.sh b/scripts/plan-version.sh new file mode 100755 index 0000000..1675a45 --- /dev/null +++ b/scripts/plan-version.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +BRANCH="${BRANCH:-}" +PULL_REQUEST="${PULL_REQUEST:-}" +VERSION_FILE=release.json +ARTIFACT=image +UPSTREAM_FILE=src/upstream.json +UPSTREAM=sqlite + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +info() { [ -n "$OPTION" ] || printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*" >&2; } +ok() { [ -n "$OPTION" ] || printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*" >&2; } +warn() { [ -n "$OPTION" ] || printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*" >&2; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +cd "$(dirname "$0")/.." + +main() { + [[ "$OPTION" =~ ^(--build|--release|--tags)?$ ]] || die "'$OPTION' is not an option, expected --build, --release, --tags or none" + read_version + print_build_version + fetch_origin + print_tags + check_branch + check_version + select_releases + check_unreleased + print_upcoming_versions + print_releases +} + +read_version() { + branch="${BRANCH:-$(git branch --show-current)}" + channel=stable + [[ ! "$branch" =~ ^(next|release/v[0-9]+\.[0-9]+)$ ]] || channel=preview + version="$(artifact_version "$channel")" + [ -n "$version" ] || die "$VERSION_FILE has no $channel version of $ARTIFACT" +} + +print_build_version() { + [ "$OPTION" = --build ] || return 0 + if [ -n "$PULL_REQUEST" ] && [ "$channel" = preview ]; then + tags="$(git ls-remote --tags --refs origin | sed 's|.*refs/tags/||' | filter_release_tags | sort -V)" || die "could not list the tags of origin" + line="${branch#release/v}" + [ "$line" != "$branch" ] || line="" + revision=HEAD + select_newest_stable + select_preview_base + version="$base" + fi + echo "$version${PULL_REQUEST:+-pr.$PULL_REQUEST}" + exit 0 +} + +fetch_origin() { + local current behind + [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" + git fetch --quiet --prune --tags origin || die "could not fetch origin" + current="$(git branch --show-current)" + if [ -n "$current" ] && behind="$(git rev-list --count "HEAD..origin/$current" 2> /dev/null)"; then + [ "$behind" = 0 ] || die "$current is $behind commit(s) behind origin/$current, pull first" + fi + tags="$(release_tags | sort -V)" +} + +print_tags() { + local released + [ "$OPTION" = --tags ] || return 0 + released="$(release_tags --points-at HEAD)" + moved_tags "$(grep -vxF -f <(echo "$released") <<< "$tags" || true)" "$tags" \ + | jq -Rnc '[inputs | split(" ") | {(.[0]): .[1]}] | add // {}' + exit 0 +} + +check_branch() { + [[ "$branch" =~ ^(main|next|v([0-9]+(\.[0-9]+)?)\.x|release/v([0-9]+\.[0-9]+))$ ]] || die "${branch:-a detached HEAD} releases nothing" + line="${BASH_REMATCH[2]}${BASH_REMATCH[4]}" + case "$branch" in + main) ok "main releases stable versions" ;; + next) ok "next releases prereleases" ;; + release/*) ok "$branch releases prereleases of $line.0" ;; + *) ok "$branch releases stable $line.x versions" ;; + esac +} + +check_version() { + local from + if [ "$channel" = stable ]; then + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "$VERSION_FILE sets '$version', expected major.minor.patch" + [ -z "$line" ] || [[ "$version" == "$line".* ]] || die "$VERSION_FILE sets $version, expected a $line.x version" + else + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$ ]] \ + || die "$VERSION_FILE sets '$version', expected major.minor.patch with an optional -alpha.N, -beta.N or -rc.N" + [ -z "$line" ] || [ "${version%%-*}" = "$line.0" ] || die "$VERSION_FILE sets $version, expected a prerelease of $line.0" + fi + last="$(release_tags --merged HEAD | tag_versions | awk -v channel="$channel" 'channel == "preview" || !/-/' | newest)" + earlier="$(with_version "$version" <<< "$tags" | tail -n 1)" + publish=false + if newer "$last" "$version"; then + info "$VERSION_FILE sets $version, $branch released $last already" + elif [ "$version" = 0.0.0 ]; then + info "$VERSION_FILE sets 0.0.0, $branch releases nothing before scripts/release.sh sets the first version" + elif [ "$channel" = preview ] && [[ ! "$version" =~ -[a-z]+\.[1-9] ]]; then + info "$VERSION_FILE sets $version, $branch publishes numbered prereleases only" + elif [ "$channel" = preview ] && [ -n "$earlier" ]; then + ok "$VERSION_FILE sets $BOLD$version$RESET" + info "$earlier is published, $branch publishes a prerelease once" + else + from="${last:+$last → }" + [ "$version" != "$last" ] || from= + ok "$VERSION_FILE sets $from$BOLD$version$RESET" + publish=true + fi + if [ "$publish" = true ] && taken "${version%%-*}"; then + die "$VERSION_FILE sets $version, ${version%%-*} is taken by $holder" + fi + if [ "$publish" = true ] && in_progress "${version%%-*}"; then + warn "${version%%-*} is in progress, held by $holder" + fi +} + +select_releases() { + local name sqlite tag previous older moves listed added="" + releases= + [ "$publish" = true ] || return 0 + while read -r name sqlite; do + tag="v${sqlite:+$sqlite-v}$version" + if grep -qxF "$tag" <<< "$tags"; then + info "$tag is published" + continue + fi + previous= + while read -r older; do + ! newer "$sqlite" "$older" || previous="$older" + done < <(with_version "$version" <<< "$tags" | sed -nE 's/^v(.*)-v.*/\1/p') + added+="$tag"$'\n' + releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ + '{$name, $version, $image, $tag, $previous}')"$'\n' + done < <(upstream_lines) + moves="$(moved_tags "$tags" "$tags"$'\n'"$added")" + for tag in $added; do + listed="$(moved_to "$tag")" + ok "$BOLD$tag$RESET is new, a push publishes it${listed:+ and moves $listed}" + done + awk 'NF { print $2 }' <<< "$moves" | sort -u | while read -r tag; do + grep -qxF "$tag" <<< "$added" || info "a push moves $(moved_to "$tag") to $tag" + done +} + +check_unreleased() { + local count + [ -n "$releases" ] && [ -n "$earlier" ] || return 0 + count="$(git rev-list --no-merges "$earlier..HEAD" -- src | grep -cvxF -f <(git rev-list "$earlier..HEAD" -- "$UPSTREAM_FILE") || true)" + [ "$count" = 0 ] || die "$count commit(s) since $earlier change src/ and would ship as $version, release them first with scripts/release.sh" +} + +print_upcoming_versions() { + local base stable upcoming + [ -z "$releases" ] || return 0 + if [ "$channel" = stable ]; then + base="$version" + ! newer "$last" "$base" || base="$last" + case "$branch" in + main) upcoming="$(next_version "$base" patch), $(next_version "$base" minor) or $(next_version "$base" major)" ;; + v*.*.x) upcoming="$(next_version "$base" patch)" ;; + *) upcoming="$(next_version "$base" patch) or $(next_version "$base" minor)" ;; + esac + [ "$base" != 0.0.0 ] || upcoming="$(next_version "$base" minor) or $(next_version "$base" major)" + elif [ -n "$line" ]; then + upcoming="a prerelease of $line.0" + else + stable="$(release_tags --merged HEAD | tag_versions | awk '!/-/' | newest)" + stable="${stable:-$(artifact_version stable)}" + upcoming="a prerelease of $(next_version "$stable" minor) or $(next_version "$stable" major)" + fi + info "the upcoming version on $branch is $upcoming, scripts/release.sh sets it" +} + +print_releases() { + [ "$OPTION" = --release ] || return 0 + jq -sc . <<< "$releases" +} + +select_newest_stable() { + stable="$(tag_versions <<< "$tags" | awk '!/-/' | newest)" + stable="${stable:-$(artifact_version stable)}" +} + +select_preview_base() { + version="$(artifact_version preview <(git show "$revision:$VERSION_FILE"))" + if [ -n "$line" ]; then + base="$line.0-alpha.0" + [[ "$version" != "$line.0-"* ]] || base="$version" + elif newer "$version" "$stable"; then + base="$version" + [[ "$version" == *-* ]] || base="$version-alpha.0" + else + base="$(next_version "$stable" minor)-alpha.0" + fi +} + +artifact_version() { + jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "${2:-$VERSION_FILE}" +} + +upstream_lines() { + if [ -z "$UPSTREAM" ]; then + echo latest + return + fi + jq -r --arg upstream "$UPSTREAM" '.[$upstream] | to_entries[] | "\(.key) \(.value.version)"' "$UPSTREAM_FILE" +} + +next_version() { + local major minor patch + IFS=. read -r major minor patch <<< "${1%%-*}" + while true; do + case "$2" in + patch) patch=$((patch + 1)) ;; + minor) minor=$((minor + 1)) patch=0 ;; + major) major=$((major + 1)) minor=0 patch=0 ;; + esac + taken "$major.$minor.$patch" || break + done + echo "$major.$minor.$patch" +} + +taken() { + local tag + for tag in $(release_tags "*v$1"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + [ "$channel" = preview ] && in_progress "$1" +} + +in_progress() { + local tag release="release/v${1%.0}" + if [ "$release" != "$branch" ] && git rev-parse --quiet --verify "origin/$release" > /dev/null; then + holder="$release" + [ "$channel" = stable ] || return 0 + git merge-base --is-ancestor "origin/$release" HEAD || return 0 + fi + for tag in $(release_tags "*v$1-*"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + return 1 +} + +moved_tags() { + moving_tags <<< "$2" | grep -vxF -f <(moving_tags <<< "$1") || true +} + +moving_tags() { + local tags releases latest preview base image sqlite prefix seen=" " + tags="$(cat)" + releases="$(sed -nE 's/^v((.*)-v)?([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" + latest="$(awk 'NR == 1 && NF { print "v" ($2 == "" ? "" : $2 "-v") $1 }' <<< "$releases")" + preview="$latest" + while read -r base; do + if newer "$base" "${latest##*v}"; then + preview="$(tag_versions <<< "$tags" | grep "^$base-" | newest)" + preview="$(with_version "$preview" <<< "$tags" | sort -V | tail -n 1)" + break + fi + done < <(tag_versions <<< "$tags" | sed -n 's/-.*//p' | sort -uV) + [ -z "$latest" ] || echo "latest $latest" + [ -z "$preview" ] || echo "preview $preview" + while read -r image sqlite; do + prefix="$sqlite" + while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do + echo "$prefix v$sqlite-v$image" + seen+="$prefix " + [[ "$prefix" == *.* ]] || break + prefix="${prefix%.*}" + done + done <<< "$releases" +} + +release_tags() { + git tag --list "$@" | filter_release_tags +} + +filter_release_tags() { + grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true +} + +tag_versions() { + sed 's/.*v//' +} + +with_version() { + awk -v version="$1" '{ tagged = $0; sub(/.*v/, "", tagged) } tagged == version' +} + +moved_to() { + awk -v tag="$1" '$2 == tag { print $1 }' <<< "$moves" | paste -sd , - | sed 's/,/, /g; s/\(.*\), /\1 and /' +} + +newer() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$1" "$2" | newest)" = "$1" ] +} + +newest() { + sed 's/-/~/' | sort -V | sed 's/~/-/' | tail -n 1 +} + +main diff --git a/scripts/release.sh b/scripts/release.sh index f45b933..7eaa44f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,14 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +OPTION="$*" +VERSION_FILE=release.json +ARTIFACT=image +UPSTREAM=sqlite +BREAKING_CHANGE='^([a-z]+(\([^)]*\))?!|BREAKING[ -]CHANGE): ' WORK_DIR="$(mktemp -d)" -BOLD=$'\033[1m' DIM=$'\033[2m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } -hint() { printf ' %-26s%s%s%s\n' "$1" "$DIM" "$2" "$RESET"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } die() { printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 exit 1 @@ -19,84 +24,164 @@ confirm() { read -r answer [[ "$answer" =~ ^[Yy]$ ]] } +hint() { printf ' %-26s%s%s%s\n' "$1" "$DIM" "$2" "$RESET"; } cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"; tput cnorm 2> /dev/null || true' EXIT main() { + [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository - read_version - select_bump + read_versions + read_changes + select_stage + select_version heading "Releasing v$new_version" update_changelog - update_dockerfile + update_version commit push } check_repository() { - local branch ahead behind entries word + local counts behind ahead entries heading "Checking the repository" git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" - branch="$(git rev-parse --abbrev-ref HEAD)" - [ "$branch" = main ] || die "releases are made from main, the current branch is $branch" + branch="$(git symbolic-ref --quiet --short HEAD)" || die "HEAD is detached, expected a branch" + [[ "$branch" =~ ^(main|next|v[0-9]+(\.[0-9]+)?\.x|release/v([0-9]+\.[0-9]+))$ ]] \ + || die "$branch releases nothing" + line="${BASH_REMATCH[3]}" + channel=stable + [[ ! "$branch" =~ ^(next|release/) ]] || channel=preview [ -z "$(git status --porcelain)" ] || die "the working tree is not clean, commit or stash first" - git fetch --quiet origin || die "could not fetch origin" - ahead="$(git rev-list --count origin/main..HEAD)" - behind="$(git rev-list --count HEAD..origin/main)" - [ "$ahead" = 0 ] || die "main is $ahead commit(s) ahead of origin/main, push first" - [ "$behind" = 0 ] || die "main is $behind commit(s) behind origin/main, pull first" - ok "main is clean and in sync with origin/main" - entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- /' CHANGELOG.md | wc -l | tr -d ' ')" + [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" + git fetch --quiet --prune --tags origin || die "could not fetch origin" + counts="$(git rev-list --left-right --count "origin/$branch...HEAD" 2> /dev/null)" || die "origin has no branch $branch, push it first" + read -r behind ahead <<< "$counts" + [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" + ok "$branch is clean and in sync with origin/$branch" + entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- / { count++ } END { print count + 0 }' CHANGELOG.md)" [ "$entries" != 0 ] || die "CHANGELOG.md has no entries under ## [Unreleased], describe the changes first" - word=entries - [ "$entries" = 1 ] && word=entry - ok "CHANGELOG.md has $BOLD$entries$RESET unreleased $word" + ok "CHANGELOG.md lists $BOLD$entries$RESET change(s) under [Unreleased], the notes of this release" +} + +read_versions() { + local released + version="$(artifact_version "$channel")" + released="$(artifact_version stable)" + [ -n "$version" ] && [ -n "$released" ] || die "$VERSION_FILE has no versions of $ARTIFACT" + stable="$(release_tags --merged HEAD | tag_versions | awk '!/-/' | sort -V | tail -n 1)" + stable="${stable:-$released}" + ! newer "$released" "$stable" || die "$VERSION_FILE sets the stable version $released, $branch released $stable last, publish $released first" + ok "$VERSION_FILE sets the $channel version $BOLD$version$RESET" +} + +read_changes() { + local tag range hash subject + tag="$(git describe --tags --abbrev=0 --match "v${UPSTREAM:+*-v}$stable" 2> /dev/null)" || true + range="${tag:+$tag..}HEAD" + required="patch" + if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then + required=major + [ "${stable%%.*}" != 0 ] || required=minor + elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): bump / { found = 1 } END { exit !found }'; then + required=minor + fi + info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" + git log -E --grep="$BREAKING_CHANGE" --format='%h %s' "$range" | while read -r hash subject; do + warn "$DIM$hash$RESET $subject is a breaking change" + done } -read_version() { - current_version="$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)" - [[ "$current_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "IMAGE_VERSION in src/Dockerfile is '$current_version', expected major.minor.patch" +select_stage() { + local stages=(alpha beta rc) selected=0 i + stage= + [ "$channel" = preview ] || return 0 + for i in "${!stages[@]}"; do + [[ "$version" != *-"${stages[$i]}".* ]] || selected="$i" + done + heading "Select the stage" + stage="${stages[$(menu "$selected" "${stages[@]}")]}" } -select_bump() { - local major minor patch options choice - IFS=. read -r major minor patch <<< "$current_version" - options=("$major.$minor.$((patch + 1))" "$major.$((minor + 1)).0" "$((major + 1)).0.0") - heading "Current version v$current_version, select the bump" - choice="$(menu "patch v${options[0]}" "minor v${options[1]}" "major v${options[2]}")" - new_version="${options[$choice]}" +select_version() { + local bases=() base candidate options=() selected=0 + if [ -n "$line" ]; then + bases=("$line.0") + elif [ "$channel" = preview ]; then + bases=("$(next_version "$stable" minor)" "$(next_version "$stable" major)") + [ "$required" != major ] || selected=1 + if [[ "$version" == *-* ]] && newer "${version%%-*}" "$stable"; then + if taken "${version%%-*}"; then + info "${version%%-*} is taken by $holder" + else + bases=("${version%%-*}" "${bases[@]}") + selected=0 + fi + fi + elif [ "$stable" = 0.0.0 ]; then + bases=("$(next_version "$stable" minor)" "$(next_version "$stable" major)") + else + bases=("$(next_version "$stable" patch)") + [[ "$branch" == v*.*.x ]] || bases+=("$(next_version "$stable" minor)") + [ "$branch" != main ] || bases+=("$(next_version "$stable" major)") + case "$required" in + minor) selected=1 ;; + major) selected=2 ;; + esac + fi + for base in "${bases[@]}"; do + candidate="$base" + if [ -n "$stage" ]; then + candidate="$base-$stage.1" + [ "$base-$stage" != "${version%.*}" ] || candidate="$base-$stage.$((${version##*.} + 1))" + fi + if newer "$candidate" "$version" && [[ " ${options[*]:-} " != *" $candidate "* ]]; then + options+=("$candidate") + ! in_progress "$base" || warn "$base is in progress, held by $holder" + fi + done + [ -n "${options[*]:-}" ] || die "$branch is at $version, no $stage prerelease follows it" + [ "$selected" -lt "${#options[@]}" ] || selected=$((${#options[@]} - 1)) + heading "Select the version" + new_version="${options[$(menu "$selected" "${options[@]/#/v}")]}" } update_changelog() { local today + if [ "$channel" = preview ]; then + info "CHANGELOG.md keeps the entries under [Unreleased] until the stable release dates them" + return + fi today="$(date +%Y-%m-%d)" - awk -v heading="## [v$new_version] - $today" '{ print } /^## \[Unreleased\]$/ { print ""; print heading }' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" + awk -v heading="## [v$new_version] - $today" ' + /^#/ && NR > 1 && previous != "" { print "" } + { print; previous = $0 } + /^## \[Unreleased\]$/ { print ""; print heading; previous = heading } + ' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" cp "$WORK_DIR/CHANGELOG.md" CHANGELOG.md ok "CHANGELOG.md dates the unreleased entries as ${BOLD}v$new_version$RESET, $today" } -update_dockerfile() { - sed -E "s/^ARG IMAGE_VERSION=.*/ARG IMAGE_VERSION=$new_version/" src/Dockerfile > "$WORK_DIR/Dockerfile" - cp "$WORK_DIR/Dockerfile" src/Dockerfile - ok "src/Dockerfile sets IMAGE_VERSION $current_version → $BOLD$new_version$RESET" +update_version() { + jq --tab --arg channel "$channel" --arg artifact "$ARTIFACT" --arg version "$new_version" '.[$channel][$artifact] = $version' "$VERSION_FILE" > "$WORK_DIR/release.json" + cp "$WORK_DIR/release.json" "$VERSION_FILE" + ok "$VERSION_FILE sets the $channel version $version → $BOLD$new_version$RESET" } commit() { - git add src/Dockerfile CHANGELOG.md - git commit --quiet -S --message "feat(release): bump image version to v$new_version" + git add "$VERSION_FILE" CHANGELOG.md + git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } push() { - local tag - tag="$(jq -r .version src/version.json)-v$new_version" - if confirm "Push to origin/main?"; then - git push --quiet origin main - ok "pushed to origin/main, the tests run and then the release workflow publishes $BOLD$tag$RESET" - heading "Next" - hint "gh run watch" "follow the workflow run" - hint "Releases on GitHub" "review the draft release once the workflow run is green and then publish it" + local runs + if confirm "Push to origin/$branch?"; then + git push --quiet origin "$branch" + ok "pushed ${BOLD}v$new_version$RESET to origin/$branch" + runs="$(git remote get-url origin | sed -E 's#^(ssh://)?git@github\.com[:/]#https://github.com/#; s#\.git$##')/actions/workflows/release.yaml?query=branch%3A$branch" + info "the release workflow runs at $runs" else info "not pushed" heading "Next" @@ -105,8 +190,63 @@ push() { fi } +artifact_version() { + jq -r --arg channel "$1" --arg artifact "$ARTIFACT" '.[$channel][$artifact] // empty' "$VERSION_FILE" +} + +release_tags() { + git tag --list "$@" | grep -E "^v${UPSTREAM:+[^-]+-v}[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$" || true +} + +tag_versions() { + sed 's/.*v//' +} + +next_version() { + local major minor patch + IFS=. read -r major minor patch <<< "${1%%-*}" + while true; do + case "$2" in + patch) patch=$((patch + 1)) ;; + minor) minor=$((minor + 1)) patch=0 ;; + major) major=$((major + 1)) minor=0 patch=0 ;; + esac + taken "$major.$minor.$patch" || break + done + echo "$major.$minor.$patch" +} + +taken() { + local tag + for tag in $(release_tags "*v$1"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + [ "$channel" = preview ] && in_progress "$1" +} + +in_progress() { + local tag release="release/v${1%.0}" + if [ "$release" != "$branch" ] && git rev-parse --quiet --verify "origin/$release" > /dev/null; then + holder="$release" + [ "$channel" = stable ] || return 0 + git merge-base --is-ancestor "origin/$release" HEAD || return 0 + fi + for tag in $(release_tags "*v$1-*"); do + holder="$tag, which $branch does not contain" + git merge-base --is-ancestor "$tag" HEAD || return 0 + done + return 1 +} + +newer() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$1" "$2" | sed 's/-/~/' | sort -V | tail -n 1 | sed 's/~/-/')" = "$1" ] +} + menu() { - local options=("$@") selected=0 key i + local selected="$1" options key i + shift + options=("$@") tput civis >&2 2> /dev/null || true while true; do for i in "${!options[@]}"; do diff --git a/scripts/size.sh b/scripts/size.sh new file mode 100755 index 0000000..b37ee98 --- /dev/null +++ b/scripts/size.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +set -euo pipefail + +OPTION="$*" +IMAGE="${1:-}" +BASELINE="${2:-}" +PLATFORM="${PLATFORM:-}" +IMAGE_NAME="${IMAGE_NAME:-}" +OUTPUT_FILE="${OUTPUT_FILE:-}" +WORK_DIR="$(mktemp -d)" + +BOLD=$'\033[1m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +warn() { printf ' %s⚠%s %s\n' "$YELLOW" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} + +trap 'rm -rf "$WORK_DIR"' EXIT + +main() { + [[ "$OPTION" =~ ^[^-][^[:space:]]*( [^-][^[:space:]]*)?$ ]] || die "'$OPTION' is not an option, expected an image and an optional baseline" + [ -z "$OUTPUT_FILE" ] || [ -n "$BASELINE" ] || die "OUTPUT_FILE is set without a baseline, expected an image and a baseline to compare" + measure_image image "$IMAGE" + [ -z "$BASELINE" ] || measure_image baseline "$BASELINE" + [ -z "$BASELINE" ] || compare_with_baseline + [ -z "$OUTPUT_FILE" ] || write_report +} + +measure_image() { + heading "Sizes of $2" + mkdir "$WORK_DIR/$1" + if [ -d "$2" ]; then + read_layouts "$1" "$2" + else + read_repository "$1" "$2" + fi + measure_manifests "$1" "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1.tsv" ] || die "$2 has no image for ${PLATFORM:-a platform}" + sort -o "$WORK_DIR/$1.tsv" "$WORK_DIR/$1.tsv" +} + +read_layouts() { + layout="$2" + find "$2" -name index.json -exec jq -s '{manifests: map(.manifests[])}' {} + > "$WORK_DIR/$1/index.json" + [ -s "$WORK_DIR/$1/index.json" ] || die "$2 holds no OCI layout, expected an index.json in it" +} + +read_repository() { + local name="${2#docker.io/}" reference + layout="" + repository="${name%%[:@]*}" + reference="${name:${#repository}+1}" + [[ "$repository" == */* ]] || repository="library/$repository" + read_object manifests "${reference:-latest}" > "$WORK_DIR/$1/index.json" 2> /dev/null || die "$2 is not on Docker Hub or not public" +} + +measure_manifests() { + local digest + if jq -e 'has("manifests")' "$2" > /dev/null; then + for digest in $(jq -r '.manifests[] | select(.platform.os != "unknown") | .digest' "$2"); do + read_object manifests "$digest" > "$WORK_DIR/$1/$digest" + measure_manifests "$1" "$WORK_DIR/$1/$digest" + done + else + measure_platform "$1" "$2" + fi +} + +measure_platform() { + local platform compressed unpacked=0 media_type digest blocks + read_object blobs "$(jq -r .config.digest "$2")" > "$2.config" + platform="$(jq -r '[.os, .architecture, .variant // empty] | join("/")' "$2.config")" + [ -z "$PLATFORM" ] || [ "$platform" = "$PLATFORM" ] || return 0 + compressed=$(($(wc -c < "$2") + $(jq '.config.size + ([.layers[].size] | add // 0)' "$2"))) + while read -r media_type digest; do + blocks="$(read_object blobs "$digest" | decompress "$media_type" | count_blocks)" + unpacked=$((unpacked + blocks)) + done < <(jq -r '.layers[] | "\(.mediaType) \(.digest)"' "$2") + printf '%s\t%s\t%s\t%s\n' "$platform" "$compressed" $((compressed + unpacked)) \ + "$(jq -r '.config.Labels["org.opencontainers.image.version"] // ""' "$2.config")" >> "$WORK_DIR/$1.tsv" + ok "$BOLD$platform$RESET is $(format_size "$compressed") compressed and $(format_size $((compressed + unpacked))) on disk, $(format_size "$unpacked") of it unpacked" +} + +decompress() { + case "$1" in + *+zstd) zstd -dc ;; + *+gzip | *.tar.gzip) gzip -dc ;; + *.tar) cat ;; + *) die "a layer is $1, expected a tar archive, plain or compressed with gzip or zstd" ;; + esac +} + +count_blocks() { + tar -tvf - | awk ' + BEGIN { blocks = 1 } + /^-/ { blocks += int((($2 ~ /\// ? $3 : $5) + 4095) / 4096) } + /^d/ { blocks++ } + /^l/ && length(substr($0, index($0, " -> ") + 4)) >= 60 { blocks++ } + END { print blocks * 4096 }' +} + +compare_with_baseline() { + local platform compressed on_disk base_compressed base_on_disk compressed_change on_disk_change + heading "Changes from $BASELINE" + while IFS=$'\t' read -r platform compressed on_disk base_compressed base_on_disk; do + compressed_change="$(format_change "$compressed" "$base_compressed")" + on_disk_change="$(format_change "$on_disk" "$base_on_disk")" + if [ "$compressed" = - ]; then + warn "$BOLD$platform$RESET is gone, $IMAGE has no image for it" + elif [ "$base_compressed" = - ]; then + ok "$BOLD$platform$RESET is new, $BASELINE has no image for it" + else + ok "$BOLD$platform$RESET changes by $compressed_change compressed and $on_disk_change on disk" + fi + printf '| %s | %s | %s | %s | %s | %s | %s |\n' "$platform" "$(format_size "$compressed")" "$(format_size "$base_compressed")" "$compressed_change" \ + "$(format_size "$on_disk")" "$(format_size "$base_on_disk")" "$on_disk_change" >> "$WORK_DIR/rows.md" + done < <(join -t $'\t' -a 1 -a 2 -e - -o 0,1.2,1.3,2.2,2.3 "$WORK_DIR/image.tsv" "$WORK_DIR/baseline.tsv") +} + +format_change() { + awk -v size="$1" -v base="$2" 'BEGIN { + if (size == "-" || base == "-") { print "-"; exit } + delta = size - base + if (delta == 0) { print "±0"; exit } + if (delta <= -1000000 || delta >= 1000000) amount = sprintf("%+.2f MB", delta / 1000000) + else if (delta <= -1000 || delta >= 1000) amount = sprintf("%+.1f kB", delta / 1000) + else amount = sprintf("%+d B", delta) + printf "%s (%+.1f %%)\n", amount, delta * 100 / base + }' +} + +write_report() { + local image_name baseline_name + image_name="$(short_name image "$IMAGE")" + baseline_name="$(short_name baseline "$BASELINE")" + if [ "$image_name" = "$baseline_name" ]; then + image_name="\`$IMAGE\`" + baseline_name="\`$BASELINE\`" + fi + { + printf '## Image size\n\n' + printf '%s compared with %s%s.\n\n' "$image_name" "$baseline_name" "$(format_version baseline "$BASELINE")" + printf '| Platform | Compressed %s | Compressed %s | Change | On disk %s | On disk %s | Change |\n' "$image_name" "$baseline_name" "$image_name" "$baseline_name" + printf '| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n' + cat "$WORK_DIR/rows.md" + printf "\nCompressed is what \`docker pull\` downloads, the CONTENT SIZE of the platform in \`docker image ls --tree\`. " + printf "On disk adds the unpacked files in 4 KiB blocks of ext4, the DISK USAGE that Docker Desktop shows, as the containerd image store keeps both. " + printf "Neither counts the SBOM and provenance attestations.\n" + } > "$OUTPUT_FILE" + ok "$OUTPUT_FILE holds the Markdown report" +} + +short_name() { + if [ "$1" = image ] && [ -n "$IMAGE_NAME" ]; then + printf "\`%s\`" "$IMAGE_NAME" + elif [ -d "$2" ]; then + printf 'the build' + else + printf "\`%s\`" "${2##*:}" + fi +} + +format_version() { + local version + [ ! -d "$2" ] || return 0 + version="$(head -n 1 "$WORK_DIR/$1.tsv" | cut -f 4)" + [ -z "$version" ] || printf " (\`%s\`)" "$version" +} + +read_object() { + local file + if [ -z "$layout" ]; then + fetch -H "Authorization: Bearer $(fetch "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$repository:pull" | jq -r .token)" \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://registry-1.docker.io/v2/$repository/$1/$2" + else + file="$(find "$layout" -path "*/blobs/${2/://}" -print -quit)" + [ -n "$file" ] || die "$layout has no blob $2" + cat "$file" + fi +} + +fetch() { + curl --proto '=https' --tlsv1.2 -sSf --retry 3 --retry-all-errors -L "$@" +} + +format_size() { + awk -v bytes="$1" 'BEGIN { + if (bytes == "-") print "-" + else if (bytes >= 1000000) printf "%.2f MB\n", bytes / 1000000 + else if (bytes >= 1000) printf "%.1f kB\n", bytes / 1000 + else printf "%d B\n", bytes + }' +} + +main diff --git a/scripts/test.sh b/scripts/test.sh index 9868347..f20fdc8 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -IMAGE="${1:?usage: scripts/test.sh IMAGE}" +OPTION="$*" +IMAGE="${1:-}" CONTAINER=sqlite-test WORK_DIR="$(mktemp -d)" @@ -18,6 +19,7 @@ cd "$(dirname "$0")/.." trap cleanup EXIT main() { + [[ "$OPTION" =~ ^[^-][^[:space:]]*$ ]] || die "'$OPTION' is not an option, expected an image" heading "Testing $IMAGE" check_version check_default_database @@ -26,10 +28,12 @@ main() { check_databases check_healthcheck check_restart + check_other_user check_creation_settings check_rejected_settings heading "Data" seed_database + recover_database backup_database restore_database check_stop @@ -39,10 +43,10 @@ main() { check_version() { local printed version source_id printed="$(docker run --rm "$IMAGE" --version)" - version="$(jq -r .version src/version.json)" - source_id="$(jq -r .source_id src/version.json)" + version="$(docker image inspect --format '{{ index .Config.Labels "dev.peakimages.sqlite.version" }}' "$IMAGE")" + source_id="$(jq -r --arg version "$version" '.sqlite[] | select(.version == $version) | .source_id' src/upstream.json)" [[ "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected '$version $source_id'" - ok "sqlite3 --version prints $version with the source id from src/version.json" + ok "sqlite3 --version prints $version with the source id src/upstream.json pins for it" } check_default_database() { @@ -67,12 +71,12 @@ start_container() { } check_databases() { - local owner + local file [ "$(sql app.sqlite 'PRAGMA journal_mode;')" = wal ] || die "app.sqlite is not in WAL mode" [ "$(sql jobs.sqlite 'PRAGMA journal_mode;')" = wal ] || die "jobs.sqlite is not in WAL mode" - owner="$(docker exec "$CONTAINER" stat -c '%u:%g' jobs.sqlite)" - [ "$owner" = 65532:65532 ] || die "jobs.sqlite is owned by $owner, expected 65532:65532" - ok "app.sqlite and jobs.sqlite from SQLITE_DATABASES are created in WAL mode, owned by $owner" + file="$(docker exec "$CONTAINER" stat -c '%a %u:%g' jobs.sqlite)" + [ "$file" = '666 65532:65532' ] || die "jobs.sqlite has mode and owner '$file', expected 666 65532:65532" + ok "app.sqlite and jobs.sqlite from SQLITE_DATABASES are created in WAL mode, mode 666, owned by 65532:65532" } check_healthcheck() { @@ -90,6 +94,22 @@ check_restart() { ok "a restart leaves an existing database as it is" } +check_other_user() { + local rows + docker run -d --name "$CONTAINER-user" --user 1000:1000 -v "$CONTAINER:/var/lib/sqlite" "$IMAGE" \ + sh -c 'printf "CREATE TABLE shared(id INTEGER PRIMARY KEY); INSERT INTO shared DEFAULT VALUES;\n" | { cat; sleep 60; } | sqlite3 jobs.sqlite' > /dev/null + for _ in $(seq 30); do + rows="$(sql jobs.sqlite 'SELECT count(*) FROM shared;' 2> /dev/null || true)" + [ "$rows" = 1 ] && break + sleep 1 + done + [ "$rows" = 1 ] || die "a container running as uid 1000 could not write to jobs.sqlite: $(docker logs "$CONTAINER-user" 2>&1 | tail -n 1)" + [ "$(docker exec "$CONTAINER" stat -c '%a' jobs.sqlite-shm)" = 666 ] || die "jobs.sqlite-shm is not mode 666 while uid 1000 holds a connection" + [ "$(sql jobs.sqlite 'INSERT INTO shared DEFAULT VALUES; SELECT count(*) FROM shared;')" = 2 ] || die "the image user cannot write jobs.sqlite while uid 1000 holds a connection" + docker rm -f "$CONTAINER-user" > /dev/null + ok "a container running as uid 1000 writes jobs.sqlite, the image user writes it too while that connection is open" +} + check_creation_settings() { local files settings docker run -d --name "$CONTAINER-settings" \ @@ -128,8 +148,25 @@ seed_database() { [ "$(sql app.sqlite "SELECT json_extract('{\"a\":1}','\$.a') + sqrt(16);")" = 5.0 ] || die "JSON or math functions" [ "$(sql app.sqlite "SELECT soundex('Robert');")" = R163 ] || die "soundex()" [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM dbstat;')" = 1 ] || die "dbstat" + [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM sqlite_dbpage;')" = 1 ] || die "dbpage" + [ "$(sql app.sqlite 'SELECT count(*) > 0 FROM sqlite_stmt;')" = 1 ] || die "stmt" [ "$(sql app.sqlite 'PRAGMA integrity_check;')" = ok ] || die "integrity check after seeding" - ok "FTS3, FTS4, FTS5, R*Tree, Geopoly, JSON, math functions, soundex() and dbstat work" + ok "FTS3, FTS4, FTS5, R*Tree, Geopoly, JSON, math functions, soundex(), dbstat, dbpage and stmt work" +} + +recover_database() { + local page result + sql app.sqlite "VACUUM INTO 'damaged.sqlite';" + [ "$(sql damaged.sqlite .dbinfo | sed -nE 's/^database page size: +//p')" = 4096 ] || die ".dbinfo does not print the page size 4096" + sql damaged.sqlite 'CREATE TABLE junk(id INTEGER PRIMARY KEY, filler BLOB); INSERT INTO junk(filler) SELECT randomblob(500) FROM t;' + page="$(sql damaged.sqlite "SELECT rootpage FROM sqlite_master WHERE name = 'junk';")" + sql -cmd '.dbconfig defensive off' damaged.sqlite "UPDATE sqlite_dbpage SET data = randomblob(4096) WHERE pgno = $page;" > /dev/null + [ "$(sql damaged.sqlite 'PRAGMA integrity_check;' 2>&1 | head -n 1)" != ok ] || die "overwriting page $page of damaged.sqlite left it intact" + docker exec "$CONTAINER" sh -c 'sqlite3 damaged.sqlite .recover | sqlite3 recovered.sqlite > /dev/null && rm damaged.sqlite' || die ".recover failed on the damaged database" + result="$(sql recovered.sqlite 'PRAGMA integrity_check; SELECT count(*) FROM t;' | tr '\n' ' ')" + [ "$result" = 'ok 3 ' ] || die "the recovered database is not intact and complete, got '$result'" + docker exec "$CONTAINER" rm recovered.sqlite + ok ".dbinfo prints the page size, .recover rebuilds the rows from a database with a destroyed page" } backup_database() { @@ -173,7 +210,7 @@ sql() { } cleanup() { - docker rm -f "$CONTAINER" "$CONTAINER-default" "$CONTAINER-settings" > /dev/null 2>&1 || true + docker rm -f "$CONTAINER" "$CONTAINER-default" "$CONTAINER-settings" "$CONTAINER-user" > /dev/null 2>&1 || true docker volume rm -f "$CONTAINER" > /dev/null 2>&1 || true rm -rf "$WORK_DIR" } diff --git a/scripts/undo-release.sh b/scripts/undo-release.sh index 6a51358..843e107 100755 --- a/scripts/undo-release.sh +++ b/scripts/undo-release.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -BOLD=$'\033[1m' DIM=$'\033[2m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m' +OPTION="$*" +VERSION_FILE=release.json + +BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } info() { printf ' %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; } @@ -20,27 +23,34 @@ confirm() { cd "$(dirname "$0")/.." main() { + [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository heading "Undoing $(git log -1 --format='%h %s')" - info "resets the commit and restores src/Dockerfile and CHANGELOG.md" + info "resets the commit and restores $VERSION_FILE and CHANGELOG.md" confirm "Continue?" || die "aborted, nothing changed" undo_commit } check_repository() { - local subject + local branch git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" - subject="$(git log -1 --format=%s)" - [[ "$subject" == "feat(release): bump image version to v"* ]] || die "the last commit is not a release commit: $subject" + branch="$(git rev-parse --abbrev-ref HEAD)" + git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" || die "the last commit changes more than $VERSION_FILE and CHANGELOG.md, expected a release commit" + [ "$(versions_at HEAD~1)" != "$(versions_at HEAD)" ] || die "the last commit keeps the versions in $VERSION_FILE at $(versions_at HEAD), expected a release commit" + [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md)" ] || die "$VERSION_FILE or CHANGELOG.md has uncommitted changes, commit or stash them first" git fetch --quiet origin || die "could not fetch origin" - [ "$(git rev-list --count origin/main..HEAD)" -ge 1 ] || die "the release commit is on origin/main already, nothing to undo locally" + [ "$(git rev-list --count "origin/$branch..HEAD")" -ge 1 ] || die "the release commit is on origin/$branch already, nothing to undo locally" } undo_commit() { git reset --quiet --soft HEAD~1 - git restore --staged --worktree src/Dockerfile CHANGELOG.md + git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md ok "reset to $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" - ok "src/Dockerfile is back to IMAGE_VERSION $BOLD$(sed -nE 's/^ARG IMAGE_VERSION=(.*)$/\1/p' src/Dockerfile)$RESET, the entries are back under [Unreleased]" + ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the entries are back under [Unreleased]" +} + +versions_at() { + git show "$1:$VERSION_FILE" | jq -r '[paths(scalars) as $path | "\($path | join(".")) \(getpath($path))"] | join(", ")' } main diff --git a/src/Dockerfile b/src/Dockerfile index cafd42e..8878e64 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -2,43 +2,40 @@ # Set by scripts/check-upstream.sh ARG SQLITE_VERSION=3.54.1 -# Set by scripts/release.sh -ARG IMAGE_VERSION=0.1.0 # Build dependencies -# https://hub.docker.com/_/alpine/tags?name=3.24.2 -ARG BASE_IMAGE=alpine:3.24.2@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 +# https://hub.docker.com/_/alpine/tags?name=3.24.3 +ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a # https://pkgs.alpinelinux.org/package/v3.24/community/x86_64/tini-static ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 -# Runtime user, nonroot 65532 like in distroless images -ARG SQLITE_UID=65532 -ARG SQLITE_GID=65532 - -# ARGs set by the release workflow -ARG BUILD_DATE +# Set by the build workflow +ARG IMAGE_VERSION +ARG SOURCE_DATE_EPOCH +ARG CREATED ARG GIT_SHA FROM ${BASE_IMAGE} AS build ARG SQLITE_VERSION +ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG SQLITE_UID -ARG SQLITE_GID # curl, jq and openssl are only used during the build stage, so they are not pinned to a specific version # hadolint ignore=DL3018 RUN apk add --no-cache build-base curl jq openssl tini-static=${TINI_VERSION} busybox-static=${BUSYBOX_VERSION} WORKDIR /src -COPY version.json verify.sh ./ -RUN curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url version.json)" \ +COPY --chmod=0644 upstream.json verify.sh ./ +RUN jq --arg version "$SQLITE_VERSION" '.sqlite[] | select(.version == $version)' upstream.json > sqlite.json \ + && curl --proto '=https' --tlsv1.2 -sSf --retry 3 -o sqlite.zip "$(jq -r .url sqlite.json)" \ && sh verify.sh -# Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: https://sqlite.org/compile.html#recommended_compile_time_options +# Compile SQLite and the sqlite3 shell into one static binary using recommended compile time options: +# https://sqlite.org/compile.html#recommended_compile_time_options and https://sqlite.org/cli.html#compiling_the_command_line_interface WORKDIR /src/sqlite RUN gcc -static-pie -fPIE -Os \ -ffunction-sections -fdata-sections -Wl,--gc-sections \ @@ -64,6 +61,11 @@ RUN gcc -static-pie -fPIE -Os \ -DSQLITE_ENABLE_GEOPOLY \ -DSQLITE_ENABLE_MATH_FUNCTIONS \ -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ -DSQLITE_SOUNDEX \ -DHAVE_USLEEP=1 \ -DHAVE_FDATASYNC=1 \ @@ -75,20 +77,24 @@ RUN gcc -static-pie -fPIE -Os \ && strip --strip-all sqlite3 # Assemble the root filesystem of the final image in /rootfs -COPY rootfs/ /rootfs/ -COPY rootfs.sh /src/ +COPY --chmod=0755 rootfs/ /rootfs/ +COPY --chmod=0644 rootfs.sh /src/ RUN sh /src/rootfs.sh +# A scratch image has no package database for the SBOM scanner to read, this syft JSON document takes its place +COPY --chmod=0644 sbom.sh /src/ +RUN sh /src/sbom.sh "peakimages/sqlite:${SQLITE_VERSION}-v${IMAGE_VERSION:?}" \ + "generic|sqlite|${SQLITE_VERSION}|blessing|$(jq -r .url /src/sqlite.json)|sqlite:sqlite" \ + "apk|tini-static|tini_project:tini|/sbin/tini" \ + "apk|busybox-static|busybox:busybox|/bin/busybox" -FROM scratch +FROM scratch AS final ARG SQLITE_VERSION ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG SQLITE_UID -ARG SQLITE_GID -ARG BUILD_DATE +ARG CREATED ARG GIT_SHA COPY --from=build /rootfs/ / @@ -96,7 +102,7 @@ COPY --from=build /rootfs/ / LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.description="Minimal, production-ready, hardened and secure SQLite Docker image." \ org.opencontainers.image.version="${SQLITE_VERSION}-v${IMAGE_VERSION}" \ - org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.revision="${GIT_SHA}" \ org.opencontainers.image.licenses="blessing AND MIT AND GPL-2.0-only AND AGPL-3.0-or-later" \ org.opencontainers.image.authors="peakimages" \ @@ -106,7 +112,7 @@ LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.documentation="https://github.com/peakimages/sqlite#readme" \ org.opencontainers.image.base.name="scratch" \ dev.peakimages.sqlite.version="${SQLITE_VERSION}" \ - dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,soundex" \ + dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,dbpage,stmt,soundex" \ dev.peakimages.tini.version="${TINI_VERSION}" \ dev.peakimages.busybox.version="${BUSYBOX_VERSION}" \ dev.peakimages.image.version="${IMAGE_VERSION}" @@ -118,7 +124,7 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ SQLITE_PAGE_SIZE=4096 WORKDIR /var/lib/sqlite -USER ${SQLITE_UID}:${SQLITE_GID} +USER 65532:65532 STOPSIGNAL SIGTERM ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] diff --git a/src/rootfs.sh b/src/rootfs.sh index 823fe14..5a61243 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -26,17 +26,16 @@ install_binaries() { install -D -m 0755 /src/sqlite/sqlite3 /rootfs/usr/local/bin/sqlite3 install -D -m 0755 /sbin/tini-static /rootfs/sbin/tini install -D -m 0755 /bin/busybox.static /rootfs/bin/busybox - chmod 0755 /rootfs/usr/local/bin/*.sh } create_user() { install -d /rootfs/etc - echo "nonroot:x:$SQLITE_UID:$SQLITE_GID::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd - echo "nonroot:x:$SQLITE_GID:" > /rootfs/etc/group + echo "nonroot:x:65532:65532::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd + echo "nonroot:x:65532:" > /rootfs/etc/group } create_directories() { - install -d -m 0700 -o "$SQLITE_UID" -g "$SQLITE_GID" /rootfs/var/lib/sqlite + install -d -m 0777 -o 65532 -g 65532 /rootfs/var/lib/sqlite install -d -m 1777 /rootfs/tmp } diff --git a/src/rootfs/usr/local/bin/docker-entrypoint.sh b/src/rootfs/usr/local/bin/docker-entrypoint.sh index 5181fdf..852f322 100755 --- a/src/rootfs/usr/local/bin/docker-entrypoint.sh +++ b/src/rootfs/usr/local/bin/docker-entrypoint.sh @@ -60,6 +60,7 @@ validate_databases() { create_database() { sqlite3 "$DATA_DIR/$1" "PRAGMA page_size=$page_size; PRAGMA auto_vacuum=$auto_vacuum; PRAGMA user_version=0; PRAGMA journal_mode=$journal_mode;" > /dev/null + chmod 0666 "$DATA_DIR/$1" echo "docker-entrypoint: created $1, page size $page_size, auto_vacuum $auto_vacuum, journal mode $journal_mode" } diff --git a/src/sbom.sh b/src/sbom.sh new file mode 100644 index 0000000..318262f --- /dev/null +++ b/src/sbom.sh @@ -0,0 +1,75 @@ +#!/bin/sh +set -eu + +USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" +IMAGE="${1:-}" +REPOSITORY="${IMAGE%:*}" +OUTPUT_FILE="/rootfs/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" + +die() { + echo "sbom: $*" >&2 + exit 1 +} + +main() { + [ "$#" -gt 1 ] || die "$USAGE" + shift + write_document "$@" + verify_files + echo "sbom: $OUTPUT_FILE lists $(jq -r '[.artifacts[] | "\(.name) \(.version)"] | join(", ")' "$OUTPUT_FILE")" +} + +write_document() { + mkdir -p "${OUTPUT_FILE%/*}" + cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" ' + ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os + | [ + split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries + | select(.Status | . == null or endswith(" installed")) + ] as $installed + | def installed($key; $name): first($installed[] | select(.[$key] == $name)) // error("sbom: \($name) is not installed"); + def purl($path; $version; $qualifiers): "pkg:\($path)@\($version)?" + ([$qualifiers | to_entries[] | select(.value | length > 0) | "\(.key)=\(.value | @uri)"] | join("&")); + def os_purl($type; $name; $version; $arch; $upstream): purl("\($type)/\($os.ID)/\($name)"; $version; { arch: $arch, distro: "\($os.ID)-\($os.VERSION_ID)", upstream: $upstream }); + def package($type; $name; $version; $license; $cpe; $purl): + ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe + | { + name: $name, + version: $version, + type: $type, + licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], + cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], + purl: $purl + }; + def apk($name; $cpe; $paths): + installed("P"; $name) + | package("apk"; .P; .V; .L; $cpe; os_purl("apk"; .P; .V; .A; if .o != .P then .o else "" end)) + { + metadataType: "apk-db-entry", + metadata: { package: .P, originPackage: .o, version: .V, architecture: .A, files: [$paths[] | { path: . }] } + }; + def deb($name; $license; $cpe; $paths): + installed("Package"; $name) + | package("deb"; .Package; .Version; $license; $cpe; os_purl("deb"; .Package; .Version; .Architecture; .Source // "" | sub(" \\((?.*)\\)$"; "@\(.version)"))) + { + metadataType: "dpkg-db-entry", + metadata: { package: .Package, source: (.Source // "" | sub(" .*"; "")), version: .Version, architecture: .Architecture, files: [$paths[] | { path: . }] } + }; + def generic($name; $version; $license; $url; $cpe): + package("binary"; $name; $version; $license; $cpe; purl("generic/\($name)"; $version; { download_url: $url })); + { + artifacts: [ + $ARGS.positional[] | split("|") | . as [$type, $name] + | if $type == "apk" and length > 3 then apk($name; .[2]; .[3:]) + elif $type == "deb" and length > 4 then deb($name; .[2]; .[3]; .[4:]) + elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) + else error("sbom: \(join("|")) is not a component, \($usage)") end + ], + schema: { version: "16.1.10", url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.1.10.json" } + }' --args "$@" > "$OUTPUT_FILE" +} + +verify_files() { + jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do + [ -f "/rootfs$path" ] || die "$path is not a file in /rootfs" + done +} + +main "$@" diff --git a/src/upstream.json b/src/upstream.json new file mode 100644 index 0000000..888a2e4 --- /dev/null +++ b/src/upstream.json @@ -0,0 +1,11 @@ +{ + "sqlite": { + "latest": { + "version": "3.54.1", + "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", + "size": 2990966, + "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", + "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" + } + } +} diff --git a/src/verify.sh b/src/verify.sh index 9591ba0..ace0755 100755 --- a/src/verify.sh +++ b/src/verify.sh @@ -1,29 +1,24 @@ #!/bin/sh set -eu -cd "$(dirname "$0")" - die() { echo "verify: $*" >&2 exit 1 } +cd "$(dirname "$0")" + main() { - check_version check_download unpack check_header echo "verify: SQLite $SQLITE_VERSION, $(pin size) bytes, SHA3-256 and source id match" } -check_version() { - [ "$(pin version)" = "$SQLITE_VERSION" ] || die "version.json pins $(pin version), the Dockerfile says $SQLITE_VERSION" -} - check_download() { size="$(wc -c < sqlite.zip | tr -d ' ')" [ "$size" = "$(pin size)" ] || die "sqlite.zip has $size bytes, expected $(pin size)" - [ "$(openssl dgst -sha3-256 -r sqlite.zip | cut -d ' ' -f 1)" = "$(pin sha3_256)" ] || die "SHA3-256 of sqlite.zip does not match version.json" + [ "$(openssl dgst -sha3-256 -r sqlite.zip | cut -d ' ' -f 1)" = "$(pin sha3_256)" ] || die "SHA3-256 of sqlite.zip does not match upstream.json" } unpack() { @@ -37,7 +32,7 @@ check_header() { } pin() { - jq -r ".$1" version.json + jq -r ".$1" sqlite.json } main diff --git a/src/version.json b/src/version.json deleted file mode 100644 index 244dae8..0000000 --- a/src/version.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": "3.54.1", - "url": "https://sqlite.org/2026/sqlite-amalgamation-3540100.zip", - "size": 2990966, - "sha3_256": "92281c86e2957f9b9305890f9ac04204caa7b677cc868dc326d942689df28e30", - "source_id": "2026-10-16 14:45:20 555e07ef33f69a0957d24d8e555b702464a131d6f1607f4c3c1e4551e8e6ca76" -}