diff --git a/.github/workflows/ci-reap.yml b/.github/workflows/ci-reap.yml index 8155201..89dceb2 100644 --- a/.github/workflows/ci-reap.yml +++ b/.github/workflows/ci-reap.yml @@ -65,8 +65,8 @@ jobs: - name: Reap stale CI workspaces and template versions env: - # Must stay above the integration jobs' 75-minute timeout-minutes, or the - # janitor deletes workspaces that live jobs are still using. + # Must stay above the longest time a job keeps its workspace (under an + # hour), or the janitor deletes workspaces that live jobs still use. AGE_MINUTES: ${{ github.event.inputs.age_minutes || '80' }} DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} run: ./scripts/ci-reap-staging.sh $([[ "$DRY_RUN" == "true" ]] || echo --force) diff --git a/.github/workflows/drupal-contrib-integration-test.yml b/.github/workflows/drupal-contrib-integration-test.yml index 7a2cf0d..92440d6 100644 --- a/.github/workflows/drupal-contrib-integration-test.yml +++ b/.github/workflows/drupal-contrib-integration-test.yml @@ -46,12 +46,32 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true +# Docs-only changes skip this workflow: docs-check.yml covers them, and a +# docs PR has no reason to queue heavy workspaces on the shared staging box. +# A mixed change still runs, because paths-ignore skips only when every +# changed file matches. on: schedule: - cron: '0 5 * * *' push: branches: [main] + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' workflow_dispatch: inputs: debug_enabled: @@ -67,9 +87,10 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: matrix: include: @@ -256,9 +277,10 @@ jobs: # To re-enable: restore `if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }}` if: false runs-on: [self-hosted, sysbox] - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 defaults: run: shell: bash -euo pipefail {0} @@ -463,9 +485,10 @@ jobs: name: Contrib ${{ matrix.project }} D${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: # PRs run a single module (smoke); push-to-main and nightly run the full # set. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -689,9 +712,10 @@ jobs: name: Contrib ${{ vars.CONTRIB_TEST_PROJECT || 'token' }} issue fork (GH) if: ${{ vars.CONTRIB_TEST_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 defaults: run: shell: bash -euo pipefail {0} @@ -899,9 +923,10 @@ jobs: name: Contrib ${{ vars.CONTRIB_TEST_GITLAB_PROJECT || 'ui_icons' }} issue fork (GitLab-native, GH) if: ${{ vars.CONTRIB_TEST_GITLAB_ISSUE_FORK != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner) }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 defaults: run: shell: bash -euo pipefail {0} diff --git a/.github/workflows/drupal-integration-test.yml b/.github/workflows/drupal-integration-test.yml index 252b43d..6bb5ac4 100644 --- a/.github/workflows/drupal-integration-test.yml +++ b/.github/workflows/drupal-integration-test.yml @@ -51,12 +51,32 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: true +# Docs-only changes skip this workflow: docs-check.yml covers them, and a +# docs PR has no reason to queue heavy workspaces on the shared staging box. +# A mixed change still runs, because paths-ignore skips only when every +# changed file matches. on: schedule: - cron: '0 4 * * *' push: branches: [main] + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' workflow_dispatch: inputs: debug_enabled: @@ -72,9 +92,10 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: matrix: drupal_version: ["12", "11"] @@ -243,9 +264,10 @@ jobs: name: Drupal ${{ matrix.drupal_version }} (plain, GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: # PRs run a single Drupal version (smoke); push-to-main and nightly run the # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml. @@ -383,9 +405,10 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 defaults: run: shell: bash -euo pipefail {0} @@ -579,9 +602,10 @@ jobs: name: Drupal issue fork ${{ vars.DRUPAL_TEST_ISSUE_FORK || '3585397' }} (GH) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 defaults: run: shell: bash -euo pipefail {0} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index e652939..f5ea3d2 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -41,16 +41,34 @@ name: Integration Tests env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +# Docs-only changes skip this workflow: docs-check.yml covers them, and a +# docs PR has no reason to queue heavy workspaces on the shared staging box. +# A mixed change still runs, because paths-ignore skips only when every +# changed file matches. on: push: branches: [main] paths-ignore: - 'drupal-core/**' - 'drupal-contrib/**' + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' pull_request: paths-ignore: - 'drupal-core/**' - 'drupal-contrib/**' + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' schedule: - cron: '0 3 * * *' workflow_dispatch: @@ -87,9 +105,10 @@ jobs: # To re-enable: restore `if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}` if: false runs-on: [self-hosted, sysbox] - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: matrix: include: @@ -306,9 +325,10 @@ jobs: name: Integration test GH (${{ matrix.template }}) if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} runs-on: ubuntu-latest - # Long enough to cover queued time behind other box-provisioning jobs - # (see scripts/ci-acquire-staging-lock.sh) plus this job's own runtime. - timeout-minutes: 75 + # GitHub's maximum for hosted runners. A job waits in the + # ci-acquire-staging-lock.sh queue as long as it takes instead of failing; + # the work after the lock takes under an hour. + timeout-minutes: 360 strategy: matrix: include: diff --git a/.github/workflows/staging-push.yml b/.github/workflows/staging-push.yml index 31ed90f..d9e3ba3 100644 --- a/.github/workflows/staging-push.yml +++ b/.github/workflows/staging-push.yml @@ -14,10 +14,30 @@ name: Staging Push Validation # Repository secret: OP_SERVICE_ACCOUNT_TOKEN - 1Password service account with read access # 1Password item: op://test-secrets/TEST_CODER_SESSION_TOKEN/credential +# Docs-only changes skip this workflow: docs-check.yml covers them, and a +# docs PR has no reason to push templates to the shared staging box. +# A mixed change still runs, because paths-ignore skips only when every +# changed file matches. on: push: branches: [main] + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + - '*/README.md' + - 'mkdocs.yml' + - '.readthedocs.yaml' + - '.markdownlint.jsonc' + - '.linkspector.yml' workflow_dispatch: inputs: debug_enabled: diff --git a/scripts/ci-acquire-staging-lock.sh b/scripts/ci-acquire-staging-lock.sh index b891a0f..d17bbf3 100755 --- a/scripts/ci-acquire-staging-lock.sh +++ b/scripts/ci-acquire-staging-lock.sh @@ -35,11 +35,12 @@ # Environment: # CI_LOCK_SLOTS Number of concurrent slots (default: 2) # CI_OWNER Coder username that owns CI workspaces (default: ci-bot) -# MAX_WAIT_SECONDS Give up and fail after this long (default: 2700 = 45m) +# MAX_WAIT_SECONDS Give up and fail after this long (default: 0 = never; +# the job's own timeout-minutes still applies) # POLL_SECONDS Delay between full passes over all slots (default: 30) # STALE_MINUTES Force-reclaim a held slot older than this (default: 80; -# must stay above the jobs' 75-minute timeout-minutes so -# a slot held by a live job is never reclaimed) +# must stay above the longest time a live job holds its +# slot, about 45 minutes, so a live slot is never reclaimed) # STAGGER_SECONDS Minimum gap enforced between two workspace starts, even # when slots are available (default: 90) @@ -47,7 +48,7 @@ set -uo pipefail CI_LOCK_SLOTS="${CI_LOCK_SLOTS:-2}" CI_OWNER="${CI_OWNER:-ci-bot}" -MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-2700}" +MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-0}" POLL_SECONDS="${POLL_SECONDS:-30}" STALE_MINUTES="${STALE_MINUTES:-80}" STAGGER_SECONDS="${STAGGER_SECONDS:-90}" @@ -122,7 +123,7 @@ stagger_if_needed() { # Fail fast if the ci-lock template itself is missing (e.g. never pushed to # this Coder deployment) rather than retrying a doomed `coder create` for the -# full MAX_WAIT_SECONDS -- that failure mode wastes 45 minutes per job with no +# whole job timeout -- that failure mode would waste hours per job with no # indication of the real problem. See docs/admin/server-setup.md for the # one-time `make push-all-templates` setup step. if ! templates_json=$(coder templates list --output json 2>&1); then @@ -155,13 +156,13 @@ while true; do continue # a slot just freed up -- retry now instead of sleeping fi - if [[ "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then + if [[ "$MAX_WAIT_SECONDS" -gt 0 && "$waited" -ge "$MAX_WAIT_SECONDS" ]]; then echo "ERROR: all $CI_LOCK_SLOTS staging box lock slot(s) still busy after ${MAX_WAIT_SECONDS}s -- giving up" >&2 echo "Last error: $last_err" >&2 exit 1 fi - echo "All $CI_LOCK_SLOTS staging box lock slot(s) busy; waiting ${POLL_SECONDS}s (${waited}s/${MAX_WAIT_SECONDS}s so far)..." + echo "All $CI_LOCK_SLOTS staging box lock slot(s) busy; waiting ${POLL_SECONDS}s (${waited}s so far)..." sleep "$POLL_SECONDS" waited=$((waited + POLL_SECONDS)) done diff --git a/scripts/ci-reap-staging.sh b/scripts/ci-reap-staging.sh index c8e9bd5..3484fa7 100755 --- a/scripts/ci-reap-staging.sh +++ b/scripts/ci-reap-staging.sh @@ -27,19 +27,18 @@ # # Why the version-age floor matters: a job pushes an inactive template # version, then may wait behind ci-acquire-staging-lock.sh for a free slot -# (up to MAX_WAIT_SECONDS there, currently 45m) before creating a workspace -# from it. Archiving "unused" versions the instant this janitor runs -- as it +# (as long as it takes, up to the job's 360-minute timeout) before creating +# a workspace from it. Archiving "unused" versions the instant this janitor runs -- as it # used to -- can archive a version that's mid-wait, not abandoned, causing # `coder create` to fail with "template version archived". VERSION_AGE_MINUTES -# must stay comfortably above that worst-case wait. +# must stay above that worst-case wait. # # Why the running-workspace age floor matters: a test job holds its workspace -# (and its ci-slot-* lock) for as long as the job runs, and jobs run up to -# their 75-minute `timeout-minutes`. A Drupal setup with slow Composer -# downloads can spend 20+ minutes in `ddev poser` alone. A lower threshold -# deletes workspaces out from under live jobs, which then fail with -# "agent is shutting down". AGE_MINUTES must stay above the job timeout, so -# anything older than it cannot belong to a running job. +# (and its ci-slot-* lock) from lock to cleanup, which takes up to about +# 45 minutes in passing jobs. A Drupal contrib setup spends about 20 minutes in +# `ddev poser` alone. A lower threshold deletes workspaces out from under live +# jobs, which then fail with "agent is shutting down". AGE_MINUTES must stay +# above that runtime, with margin, so anything older is stuck or orphaned. # # By default runs in dry-run mode (prints what would be deleted). # Pass --force (or set DRY_RUN=false) to actually delete/archive. @@ -49,7 +48,7 @@ # Environment overrides: # CI_OWNER Coder username that owns CI workspaces (default: ci-bot) # AGE_MINUTES Age threshold in minutes for running workspaces (default: 80) -# VERSION_AGE_MINUTES Age threshold in minutes for unused template versions (default: 60) +# VERSION_AGE_MINUTES Age threshold in minutes for unused template versions (default: 380) # DRY_RUN true|false (default: true; --force sets false) # TEMPLATES space-separated template names to archive versions for # @@ -62,7 +61,7 @@ set -euo pipefail CI_OWNER="${CI_OWNER:-ci-bot}" AGE_MINUTES="${AGE_MINUTES:-80}" -VERSION_AGE_MINUTES="${VERSION_AGE_MINUTES:-60}" +VERSION_AGE_MINUTES="${VERSION_AGE_MINUTES:-380}" DRY_RUN="${DRY_RUN:-true}" TEMPLATES="${TEMPLATES:-drupal-core drupal-contrib freeform}"