ci: introducing-jitter

Author: darklight3it

.github/workflows/test-on-push-and-pr.yml

@@ -189,6 +189,15 @@ jobs:
     name: "${{ matrix.distro }} ${{ matrix.distro_version }} / python ${{ matrix.runtime_version }}"
 
     steps:
+      - name: Jitter start (stagger matrix shards to spare public.ecr.aws rate limit)
+        # Every shard is scheduled within milliseconds of the others and pulls
+        # its base images from public.ecr.aws inside the same rate-limit window,
+        # so together they trip the throttle (429 Too Many Requests).
+        run: |
+          jitter=$((RANDOM % 61))
+          echo "Sleeping ${jitter}s to stagger this shard"
+          sleep "${jitter}"
+
       - uses: actions/checkout@v4
 
       - name: Extract RIE
@@ -215,7 +224,7 @@ jobs:
             echo "RUN apk add curl" >> "$TMPFILE"
           fi
           echo "COPY .scratch/${RIE} /usr/bin/${RIE}" >> "$TMPFILE"
-          docker build . \
+          tests/integration/docker-retry.sh docker build . \
             -f "$TMPFILE" \
             -t ric-test \
             --build-arg RUNTIME_VERSION=${{ matrix.runtime_version }} \

tests/integration/docker-retry.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Run a docker command, retrying when it fails (e.g. public.ecr.aws throttling
+# the base image pulls with "429 Too Many Requests").
+# Usage: docker-retry.sh <command> [args...]
+
+set -uo pipefail
+
+# Ladder of base wait times (in seconds) between successive attempts.
+# Default: 1 min, 2 min, 5 min — 4 attempts total (initial + 3 retries).
+# public.ecr.aws rate-limit windows recover on a minute-scale timer, so short
+# exponential backoff tends to exhaust its budget before the slot reopens.
+IFS=' ' read -r -a DELAYS <<< "${RETRY_DELAYS:-60 120 300}"
+JITTER_MAX="${RETRY_JITTER_MAX:-30}"
+MAX_ATTEMPTS=$(( ${#DELAYS[@]} + 1 ))
+
+if (( $# == 0 )); then
+    >&2 echo "usage: docker-retry.sh <command> [args...]"
+    exit 2
+fi
+
+attempt=1
+while true; do
+    "$@"
+    status=$?
+    if (( status == 0 || attempt >= MAX_ATTEMPTS )); then
+        exit "$status"
+    fi
+
+    # Base delay from the ladder + additive jitter in [0, JITTER_MAX] so
+    # concurrent shards don't retry in the exact same second.
+    base=${DELAYS[$((attempt - 1))]}
+    jitter=$(( RANDOM % (JITTER_MAX + 1) ))
+    delay=$(( base + jitter ))
+
+    >&2 echo "docker-retry: '$*' failed (exit ${status}); attempt ${attempt}/${MAX_ATTEMPTS}, retrying in ${delay}s (${base}s + ${jitter}s jitter)."
+    sleep "$delay"
+    attempt=$(( attempt + 1 ))
+done