Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci-reap.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ jobs:

- name: Reap stale CI workspaces and template versions
env:
# Must stay above the integration jobs' 75-minute timeout-minutes, or the
# janitor deletes workspaces that live jobs are still using.
# Must stay above the longest time a job keeps its workspace (under an
# hour), or the janitor deletes workspaces that live jobs still use.
AGE_MINUTES: ${{ github.event.inputs.age_minutes || '80' }}
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
run: ./scripts/ci-reap-staging.sh $([[ "$DRY_RUN" == "true" ]] || echo --force)
55 changes: 40 additions & 15 deletions .github/workflows/drupal-contrib-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,32 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

# Docs-only changes skip this workflow: docs-check.yml covers them, and a
# docs PR has no reason to queue heavy workspaces on the shared staging box.
# A mixed change still runs, because paths-ignore skips only when every
# changed file matches.
on:
schedule:
- cron: '0 5 * * *'
push:
branches: [main]
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
pull_request:
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
workflow_dispatch:
inputs:
debug_enabled:
Expand All @@ -67,9 +87,10 @@ jobs:
# To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
if: false
runs-on: [self-hosted, sysbox]
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
matrix:
include:
Expand Down Expand Up @@ -256,9 +277,10 @@ jobs:
# To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}`
if: false
runs-on: [self-hosted, sysbox]
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
defaults:
run:
shell: bash -euo pipefail {0}
Expand Down Expand Up @@ -463,9 +485,10 @@ jobs:
name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH)
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
# PRs run a single module (smoke); push-to-main and nightly run the full
# set. Cuts per-PR workspace load on staging. See ci-reap.yml.
Expand Down Expand Up @@ -689,9 +712,10 @@ jobs:
name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH)
if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
defaults:
run:
shell: bash -euo pipefail {0}
Expand Down Expand Up @@ -899,9 +923,10 @@ jobs:
name: Contrib ${{ vars.CONTRIB_TEST_GITLAB_PROJECT || 'ui_icons' }} issue fork (GitLab-native, GH)
if: ${{ vars.CONTRIB_TEST_GITLAB_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
defaults:
run:
shell: bash -euo pipefail {0}
Expand Down
48 changes: 36 additions & 12 deletions .github/workflows/drupal-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,32 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

# Docs-only changes skip this workflow: docs-check.yml covers them, and a
# docs PR has no reason to queue heavy workspaces on the shared staging box.
# A mixed change still runs, because paths-ignore skips only when every
# changed file matches.
on:
schedule:
- cron: '0 4 * * *'
push:
branches: [main]
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
pull_request:
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
workflow_dispatch:
inputs:
debug_enabled:
Expand All @@ -72,9 +92,10 @@ jobs:
# To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
if: false
runs-on: [self-hosted, sysbox]
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
matrix:
drupal_version: ["12", "11"]
Expand Down Expand Up @@ -243,9 +264,10 @@ jobs:
name: Drupal ${{ matrix.drupal_version }} (plain, GH)
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
# PRs run a single Drupal version (smoke); push-to-main and nightly run the
# full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml.
Expand Down Expand Up @@ -383,9 +405,10 @@ jobs:
# To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
if: false
runs-on: [self-hosted, sysbox]
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
defaults:
run:
shell: bash -euo pipefail {0}
Expand Down Expand Up @@ -579,9 +602,10 @@ jobs:
name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH)
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
defaults:
run:
shell: bash -euo pipefail {0}
Expand Down
32 changes: 26 additions & 6 deletions .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,34 @@ name: Integration Tests
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

# Docs-only changes skip this workflow: docs-check.yml covers them, and a
# docs PR has no reason to queue heavy workspaces on the shared staging box.
# A mixed change still runs, because paths-ignore skips only when every
# changed file matches.
on:
push:
branches: [main]
paths-ignore:
- 'drupal-core/**'
- 'drupal-contrib/**'
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
pull_request:
paths-ignore:
- 'drupal-core/**'
- 'drupal-contrib/**'
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
schedule:
- cron: '0 3 * * *'
workflow_dispatch:
Expand Down Expand Up @@ -87,9 +105,10 @@ jobs:
# To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}`
if: false
runs-on: [self-hosted, sysbox]
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
matrix:
include:
Expand Down Expand Up @@ -306,9 +325,10 @@ jobs:
name: Integration test GH (${{ matrix.template }})
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
runs-on: ubuntu-latest
# Long enough to cover queued time behind other box-provisioning jobs
# (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime.
timeout-minutes: 75
# GitHub's maximum for hosted runners. A job waits in the
# ci-acquire-staging-lock.sh queue as long as it takes instead of failing;
# the work after the lock takes under an hour.
timeout-minutes: 360
strategy:
matrix:
include:
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/staging-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,30 @@ name: Staging Push Validation
# Repository secret: OP_SERVICE_ACCOUNT_TOKEN - 1Password service account with read access
# 1Password item: op://test-secrets/TEST_CODER_SESSION_TOKEN/credential

# Docs-only changes skip this workflow: docs-check.yml covers them, and a
# docs PR has no reason to push templates to the shared staging box.
# A mixed change still runs, because paths-ignore skips only when every
# changed file matches.
on:
push:
branches: [main]
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
pull_request:
paths-ignore:
- 'docs/**'
- '*.md'
- '*/README.md'
- 'mkdocs.yml'
- '.readthedocs.yaml'
- '.markdownlint.jsonc'
- '.linkspector.yml'
workflow_dispatch:
inputs:
debug_enabled:
Expand Down
15 changes: 8 additions & 7 deletions scripts/ci-acquire-staging-lock.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,20 @@
# Environment:
# CI_LOCK_SLOTS Number of concurrent slots (default: 2)
# CI_OWNER Coder username that owns CI workspaces (default: ci-bot)
# MAX_WAIT_SECONDS Give up and fail after this long (default: 2700 = 45m)
# MAX_WAIT_SECONDS Give up and fail after this long (default: 0 = never;
# the job's own timeout-minutes still applies)
# POLL_SECONDS Delay between full passes over all slots (default: 30)
# STALE_MINUTES Force-reclaim a held slot older than this (default: 80;
# must stay above the jobs' 75-minute timeout-minutes so
# a slot held by a live job is never reclaimed)
# must stay above the longest time a live job holds its
# slot, about 45 minutes, so a live slot is never reclaimed)
# STAGGER_SECONDS Minimum gap enforced between two workspace starts, even
# when slots are available (default: 90)

set -uo pipefail

CI_LOCK_SLOTS="${CI_LOCK_SLOTS:-2}"
CI_OWNER="${CI_OWNER:-ci-bot}"
MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-2700}"
MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-0}"
POLL_SECONDS="${POLL_SECONDS:-30}"
STALE_MINUTES="${STALE_MINUTES:-80}"
STAGGER_SECONDS="${STAGGER_SECONDS:-90}"
Expand Down Expand Up @@ -122,7 +123,7 @@ stagger_if_needed() {

# Fail fast if the ci-lock template itself is missing (e.g. never pushed to
# this Coder deployment) rather than retrying a doomed `coder create` for the
# full MAX_WAIT_SECONDS -- that failure mode wastes 45 minutes per job with no
# whole job timeout -- that failure mode would waste hours per job with no
# indication of the real problem. See docs/admin/server-setup.md for the
# one-time `make push-all-templates` setup step.
if ! templates_json=$(coder templates list --output json 2>&1); then
Expand Down Expand Up @@ -155,13 +156,13 @@ while true; do
continue # a slot just freed up -- retry now instead of sleeping
fi

if [[ "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then
if [[ "$MAX_WAIT_SECONDS" -gt 0 && "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then
echo "ERROR: all $CI_LOCK_SLOTS staging box lock slot(s) still busy after ${MAX_WAIT_SECONDS}s -- giving up" >&2
echo "Last error: $last_err" >&2
exit 1
fi

echo "All $CI_LOCK_SLOTS staging box lock slot(s) busy; waiting ${POLL_SECONDS}s (${waited}s/${MAX_WAIT_SECONDS}s so far)..."
echo "All $CI_LOCK_SLOTS staging box lock slot(s) busy; waiting ${POLL_SECONDS}s (${waited}s so far)..."
sleep "$POLL_SECONDS"
waited=$((waited + POLL_SECONDS))
done
21 changes: 10 additions & 11 deletions scripts/ci-reap-staging.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,18 @@
#
# Why the version-age floor matters: a job pushes an inactive template
# version, then may wait behind ci-acquire-staging-lock.sh for a free slot
# (up to MAX_WAIT_SECONDS there, currently 45m) before creating a workspace
# from it. Archiving "unused" versions the instant this janitor runs -- as it
# (as long as it takes, up to the job's 360-minute timeout) before creating
# a workspace from it. Archiving "unused" versions the instant this janitor runs -- as it
# used to -- can archive a version that's mid-wait, not abandoned, causing
# `coder create` to fail with "template version archived". VERSION_AGE_MINUTES
# must stay comfortably above that worst-case wait.
# must stay above that worst-case wait.
#
# Why the running-workspace age floor matters: a test job holds its workspace
# (and its ci-slot-* lock) for as long as the job runs, and jobs run up to
# their 75-minute `timeout-minutes`. A Drupal setup with slow Composer
# downloads can spend 20+ minutes in `ddev poser` alone. A lower threshold
# deletes workspaces out from under live jobs, which then fail with
# "agent is shutting down". AGE_MINUTES must stay above the job timeout, so
# anything older than it cannot belong to a running job.
# (and its ci-slot-* lock) from lock to cleanup, which takes up to about
# 45 minutes in passing jobs. A Drupal contrib setup spends about 20 minutes in
# `ddev poser` alone. A lower threshold deletes workspaces out from under live
# jobs, which then fail with "agent is shutting down". AGE_MINUTES must stay
# above that runtime, with margin, so anything older is stuck or orphaned.
#
# By default runs in dry-run mode (prints what would be deleted).
# Pass --force (or set DRY_RUN=false) to actually delete/archive.
Expand All @@ -49,7 +48,7 @@
# Environment overrides:
# CI_OWNER Coder username that owns CI workspaces (default: ci-bot)
# AGE_MINUTES Age threshold in minutes for running workspaces (default: 80)
# VERSION_AGE_MINUTES Age threshold in minutes for unused template versions (default: 60)
# VERSION_AGE_MINUTES Age threshold in minutes for unused template versions (default: 380)
# DRY_RUN true|false (default: true; --force sets false)
# TEMPLATES space-separated template names to archive versions for
#
Expand All @@ -62,7 +61,7 @@ set -euo pipefail

CI_OWNER="${CI_OWNER:-ci-bot}"
AGE_MINUTES="${AGE_MINUTES:-80}"
VERSION_AGE_MINUTES="${VERSION_AGE_MINUTES:-60}"
VERSION_AGE_MINUTES="${VERSION_AGE_MINUTES:-380}"
DRY_RUN="${DRY_RUN:-true}"
TEMPLATES="${TEMPLATES:-drupal-core drupal-contrib freeform}"

Expand Down
Loading