From c797777319370e2d39b44b725f135f6ebc57b295 Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Sun, 25 Oct 2026 12:01:09 +0100 Subject: [PATCH] ci: skip packages that are already on NuGet.org instead of re-pushing them A master push packs every project and pushes every artifact. `--skip-duplicate` does not avoid the PUT: a duplicate is still a request, and NuGet.org counts it, so a burst of merges exhausts the push quota and the run dies with `403 Quota Exceeded` (retry after 822s) long after the release itself shipped. That is what turned the four dependabot merges after #475 red: 2.1.1 was on the feed, the publish job was not. push_layer now reads id and version out of each nupkg nuspec, asks the flatcontainer index, and only pushes what is missing. Anything already published is skipped without touching the push endpoint, so an unchanged tree costs one GET per package and zero pushes, and the 30s indexing wait only happens for a layer that actually pushed something. The step summary now reports pushed vs skipped and says plainly when a run shipped nothing. `ls *.nupkg | wc -l` never did: it read "Published 28 package(s)" on runs that pushed none. The catch-all `push_layer "Remaining packages" "*.nupkg"` also re-pushed the whole set after the layered passes - 50 already-exists conflicts in a single run - so every release paid for 28 extra requests. It still catches packages the layers do not name; the feed check makes it free. The same skip logic goes into publish-manual.yml, so a re-dispatch after a partial publish costs one request per package instead of a full re-push. Validated: both workflows parse as YAML; the two helpers run against the live feed (published version skipped, unpublished pushed, unknown package id pushed, 404 index pushed, nuspec 2.1.0-RC.3 matching the feed lowercase 2.1.0-rc.3); nuspec extraction returns id/version from a nuspec fragment. --- .github/workflows/ci.yml | 80 +++++++++++++++++++++++++--- .github/workflows/publish-manual.yml | 48 +++++++++++++++-- 2 files changed, 116 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 290032b7..a9c055c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,26 +344,76 @@ jobs: run: | set -euo pipefail + # NuGet.org rate-limits pushes and answers 403 "Quota Exceeded" once the budget is gone, which + # turns a master push red after the release itself has already shipped. `--skip-duplicate` does + # not help: a duplicate is still a PUT, and it still counts against the quota. So ask the feed + # first and only push what is actually missing. Everything already on NuGet.org is skipped + # without a request to the push endpoint. + PUSHED_FILE=/tmp/pushed.txt + SKIPPED_FILE=/tmp/skipped.txt + SEEN_FILE=/tmp/seen.txt + : > "$PUSHED_FILE" + : > "$SKIPPED_FILE" + : > "$SEEN_FILE" + + nuspec_field() { + # $1 = nupkg, $2 = nuspec element name (id, version). `|| true` keeps `pipefail` from + # aborting the step when head closes the pipe early; an empty result fails the caller's guard. + unzip -p "$1" '*.nuspec' 2>/dev/null | grep -oP "(?<=<$2>)[^<]+" | head -n1 || true + } + + already_on_feed() { + # $1 = package id, $2 = version. 0 = present, 1 = absent. + local id_lower code + id_lower="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" + code="$(curl -s -o /tmp/nuget-versions.json -w '%{http_code}' \ + "https://api.nuget.org/v3-flatcontainer/${id_lower}/index.json")" + [ "$code" = "200" ] || return 1 + # Case-insensitive: the feed lowercases pre-release labels ("2.1.0-rc.3"), the nuspec + # declares them as authored ("2.1.0-RC.3"). + grep -qi "\"$2\"" /tmp/nuget-versions.json + } + push_layer() { local layer_name="$1" shift - local found_any=false + local pushed_any=false echo "::group::Publishing $layer_name" for pattern in "$@"; do for pkg in ./artifacts/${pattern}; do [ -f "$pkg" ] || continue - found_any=true - echo " Publishing $(basename "$pkg")" + # The catch-all layer re-visits packages the layers above already handled. Visit each + # artifact once: a no-op run then costs one GET per package, not two. + local base="$(basename "$pkg")" + if grep -qxF "$base" "$SEEN_FILE"; then + continue + fi + echo "$base" >> "$SEEN_FILE" + local id version + id="$(nuspec_field "$pkg" id)" + version="$(nuspec_field "$pkg" version)" + if [ -z "$id" ] || [ -z "$version" ]; then + echo " Cannot read id/version from $(basename "$pkg")" >&2 + exit 1 + fi + if already_on_feed "$id" "$version"; then + echo " Skip $id $version (already on NuGet.org)" + echo "$id $version" >> "$SKIPPED_FILE" + continue + fi + echo " Publishing $id $version" dotnet nuget push "$pkg" \ --api-key "$NUGET_API_KEY" \ --source "$NUGET_SOURCE" \ --skip-duplicate + echo "$id $version" >> "$PUSHED_FILE" + pushed_any=true done done echo "::endgroup::" - if [ "$found_any" = true ]; then + if [ "$pushed_any" = true ]; then echo "Waiting 30s for NuGet.org to index $layer_name..." sleep 30 fi @@ -403,19 +453,33 @@ jobs: "SharpCoreDB.Functional.EntityFrameworkCore.*.nupkg" \ "SharpCoreDB.Functional.Linq2DB.*.nupkg" + # Catch-all for anything the layers above do not name. It no longer re-PUTs the packages it + # shares with them: the feed check in push_layer skips what is already published. push_layer "Remaining packages" \ "*.nupkg" - echo "All packages published in dependency order." + echo "Push complete: $(wc -l < "$PUSHED_FILE") published, $(wc -l < "$SKIPPED_FILE") already on NuGet.org." - name: Create release summary run: | + pushed=$(wc -l < /tmp/pushed.txt 2>/dev/null || echo 0) + skipped=$(wc -l < /tmp/skipped.txt 2>/dev/null || echo 0) echo "## NuGet Packages Published (Dependency-Ordered)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "Published $(ls -1 ./artifacts/*.nupkg | wc -l) package(s) to NuGet.org" >> $GITHUB_STEP_SUMMARY + echo "$pushed package(s) pushed, $skipped already on NuGet.org" >> $GITHUB_STEP_SUMMARY + if [ "$pushed" = "0" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "Nothing was pushed: every package in this run is already published at the version this" >> $GITHUB_STEP_SUMMARY + echo "tree declares. Bump the version (or dispatch publish-manual.yml) to actually release." >> $GITHUB_STEP_SUMMARY + fi echo "" >> $GITHUB_STEP_SUMMARY - echo "### Packages" >> $GITHUB_STEP_SUMMARY - ls -1 ./artifacts/*.nupkg | xargs -I {} basename {} >> $GITHUB_STEP_SUMMARY + if [ "$pushed" -gt 0 ]; then + echo "### Pushed" >> $GITHUB_STEP_SUMMARY + sed 's/^/- /' /tmp/pushed.txt >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + echo "### Skipped (already on NuGet.org)" >> $GITHUB_STEP_SUMMARY + sed 's/^/- /' /tmp/skipped.txt >> $GITHUB_STEP_SUMMARY # Core-package-only publish (workflow_dispatch). Carried over from ci-net11.yml so the RC workflow's # ability to push just SharpCoreDB to NuGet.org survives the promotion; the full package set goes diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml index d1b7c962..fb7036f9 100644 --- a/.github/workflows/publish-manual.yml +++ b/.github/workflows/publish-manual.yml @@ -82,26 +82,66 @@ jobs: run: | set -euo pipefail + # NuGet.org rate-limits pushes (403 "Quota Exceeded") and `--skip-duplicate` does not avoid the + # PUT — a duplicate still counts. Ask the feed first and push only what is missing, so a + # re-dispatch after a partial run costs one request per package instead of a full re-push. + SEEN_FILE=/tmp/seen.txt + : > "$SEEN_FILE" + + nuspec_field() { + # $1 = nupkg, $2 = nuspec element name (id, version). `|| true` keeps `pipefail` from + # aborting the step when head closes the pipe early; an empty result fails the caller's guard. + unzip -p "$1" '*.nuspec' 2>/dev/null | grep -oP "(?<=<$2>)[^<]+" | head -n1 || true + } + + already_on_feed() { + # $1 = package id, $2 = version. 0 = present, 1 = absent. + local id_lower code + id_lower="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" + code="$(curl -s -o /tmp/nuget-versions.json -w '%{http_code}' \ + "https://api.nuget.org/v3-flatcontainer/${id_lower}/index.json")" + [ "$code" = "200" ] || return 1 + grep -qi "\"$2\"" /tmp/nuget-versions.json + } + push_layer() { local layer_name="$1" shift - local found_any=false + local pushed_any=false echo "::group::Publishing $layer_name" for pattern in "$@"; do for pkg in ./artifacts/${pattern}; do [ -f "$pkg" ] || continue - found_any=true - echo " ↗ Pushing $(basename "$pkg")" + # The catch-all layer re-visits packages the layers above already handled. Visit each + # artifact once: a no-op run then costs one GET per package, not two. + local base="$(basename "$pkg")" + if grep -qxF "$base" "$SEEN_FILE"; then + continue + fi + echo "$base" >> "$SEEN_FILE" + local id version + id="$(nuspec_field "$pkg" id)" + version="$(nuspec_field "$pkg" version)" + if [ -z "$id" ] || [ -z "$version" ]; then + echo " Cannot read id/version from $(basename "$pkg")" >&2 + exit 1 + fi + if already_on_feed "$id" "$version"; then + echo " ⏭ Skipping $id $version (already on NuGet.org)" + continue + fi + echo " ↗ Pushing $id $version" dotnet nuget push "$pkg" \ --api-key "$NUGET_API_KEY" \ --source "$NUGET_SOURCE" \ --skip-duplicate + pushed_any=true done done echo "::endgroup::" - if [ "$found_any" = true ]; then + if [ "$pushed_any" = true ]; then echo "⏳ Waiting 30s for NuGet.org to index $layer_name..." sleep 30 fi