feat: run the scheduled jobs one after another from 10:00 on weekdays

README.md

@@ -12,8 +12,8 @@ share for translations. One package, `session_ops`, deployed to one self-hosted
 | `zendesk-relay` | Drafts and sends Zendesk replies from `claude:` private notes | [zendesk-relay](docs/jobs/zendesk-relay.md) |
 | `github-prs-digest` | Weekday Discord digest of open pull requests from outside contributors | [github-prs-digest](docs/jobs/github-prs-digest.md) |
 | `crowdin-duplicates` | Crowdin string slots holding more than one translation, as they open and close | [crowdin-duplicates](docs/jobs/crowdin-duplicates.md) |
-| `crowdin-sync` | Weekly: Crowdin translations into pull requests on iOS, Android and the localization module | [crowdin-sync](docs/jobs/crowdin-sync.md) |
-| `snode-list` | Daily: the fallback service node list into session-ios | [snode-list](docs/jobs/snode-list.md) |
+| `crowdin-sync` | Weekday: Crowdin translations into pull requests on iOS, Android and the localization module | [crowdin-sync](docs/jobs/crowdin-sync.md) |
+| `snode-list` | Weekday: the fallback service node list into session-ios | [snode-list](docs/jobs/snode-list.md) |
 | `release-stats` | On demand: download counts of the latest releases | [release-stats](docs/jobs/release-stats.md) |
 | `session-ops-silence` | Discord alerts for a job that failed, or stopped running | [session-ops-silence](docs/jobs/session-ops-silence.md) |
 

deploy/README.md

@@ -7,6 +7,7 @@ does the work: accounts, venv, env files, units, timers, and migrating an older
 | Unit | What it is |
 | --- | --- |
 | `session-ops@<job>.timer` → `.service` | One per job; a generated drop-in sets its account, env files and schedule. |
+| `session-ops-queue.timer` → `.service` | Starts the jobs in `jobs.toml`'s `[queue]`, which then run one at a time in its order. |
 | `zendesk-relay.service` | Always on, `127.0.0.1:8080`: Zendesk's `claude:` note webhooks. |
 | `session-ops-alert@.service` | Every unit's `OnFailure=` backstop; see [session-ops-silence](../docs/jobs/session-ops-silence.md). |
 
@@ -59,8 +60,10 @@ installed from [`env/`](env/), saying what goes in it.
 ## Checking
 
 ```bash
-systemctl list-timers 'session-ops@*'
+systemctl list-timers 'session-ops*'
 systemctl start session-ops@<job>.service && journalctl -fu session-ops@<job>
+# Re-run one job by its own service: starting session-ops-queue.service again re-runs
+# every queued job that has already finished today.
 systemctl start session-ops-alert@test.service          # posts to the alerts channel
 curl -sS -o /dev/null -w '%{http_code}\n' -X POST 127.0.0.1:8080/zendesk/notes \
   -H 'Content-Type: application/json' -d '{"ticket_id":"1"}'   # expect 401
@@ -80,7 +83,7 @@ To end it, stop the timers, then close the rehearsal pull requests:
 for link in /etc/systemd/system/timers.target.wants/session-ops@*.timer; do
   [ -L "$link" ] && systemctl disable --now "${link##*/}"
 done
-systemctl disable --now zendesk-relay.service
+systemctl disable --now session-ops-queue.timer zendesk-relay.service
 for repo in session-android session-ios session-localization; do
   gh pr list -R "session-foundation/$repo" --state open --json number,headRefName \
     -q '.[] | select(.headRefName | startswith("rehearsal/")) | .number' |

deploy/install.sh

@@ -5,7 +5,7 @@
 # It creates the accounts, builds the venv, creates any missing env file empty, moves
 # state and env files from the layout before session-ops@ units, installs the units
 # and each job's drop-ins, and enables every job whose env files have content and no
-# other. It never edits nginx, which certbot owns; see deploy/README.md for the route.
+# other: on its own timer, or on the queue's. It never edits nginx, which certbot owns; see deploy/README.md for the route.
 set -eu
 
 ROOT=$(cd "$(dirname "$0")/.." && pwd)
@@ -106,27 +106,50 @@ rm -f "$UNITS/crowdin-relay.service"
 
 install -m 644 "$ROOT"/deploy/*.service "$ROOT"/deploy/*.timer "$UNITS/"
 # Only the generated files go, so a drop-in added by hand survives.
-rm -f "$UNITS"/session-ops@*.service.d/job.conf "$UNITS"/session-ops@*.timer.d/schedule.conf
+rm -f "$UNITS"/session-ops@*.service.d/job.conf "$UNITS"/session-ops@*.timer.d/schedule.conf \
+    "$UNITS"/session-ops-queue.timer.d/schedule.conf
 "$OPS" units --out "$UNITS" >/dev/null
-systemctl daemon-reload
 
 READY=$("$OPS" list --ready)
-# A job removed from jobs.toml, or whose env file was emptied, stops being scheduled.
+QUEUED=$("$OPS" list --queued)
+listed() { printf '%s\n' $2 | grep -qxF "$1"; }
+# What the queue's timer starts: its ready jobs, rebuilt from scratch each install.
+WANTS="$UNITS/session-ops-queue.service.wants"
+rm -rf "$WANTS"
+for job in $QUEUED; do
+    if listed "$job" "$READY"; then
+        install -d -m 755 "$WANTS"
+        ln -s "$UNITS/session-ops@.service" "$WANTS/session-ops@$job.service"
+    fi
+done
+systemctl daemon-reload
+
+# A job removed from jobs.toml, whose env file was emptied, or now queued, loses its timer.
 for link in "$UNITS"/timers.target.wants/session-ops@*.timer; do
     [ -L "$link" ] || continue
     job=${link##*/session-ops@}
     job=${job%.timer}
-    if ! printf '%s\n' $READY | grep -qxF "$job"; then
+    if ! listed "$job" "$READY" || listed "$job" "$QUEUED"; then
         systemctl disable --now "session-ops@$job.timer" >/dev/null
-        echo "disabled session-ops@$job.timer (no longer a ready job)"
+        echo "disabled session-ops@$job.timer (no longer a ready job with a timer of its own)"
     fi
 done
 for job in $READY; do
-    systemctl enable --now "session-ops@$job.timer" >/dev/null
-    echo "enabled session-ops@$job.timer"
+    if listed "$job" "$QUEUED"; then
+        echo "queued session-ops@$job.service"
+    else
+        systemctl enable --now "session-ops@$job.timer" >/dev/null
+        echo "enabled session-ops@$job.timer"
+    fi
 done
+if [ -d "$WANTS" ]; then
+    systemctl enable --now session-ops-queue.timer >/dev/null
+    echo "enabled session-ops-queue.timer"
+else
+    systemctl disable --now session-ops-queue.timer 2>/dev/null || true
+fi
 for job in $("$OPS" list --not-ready); do
-    echo "not enabled: session-ops@$job.timer (its env file is empty)"
+    echo "not enabled: session-ops@$job (its env file is empty)"
 done
 if [ -s "$ETC/zendesk.env" ]; then
     systemctl enable zendesk-relay.service >/dev/null

deploy/session-ops-queue.service

@@ -0,0 +1,10 @@
+# Its timer's run of the jobs in jobs.toml's [queue]: install.sh links each ready one
+# into session-ops-queue.service.wants/, and their generated After= runs them in order.
+[Unit]
+Description=Start the queued session-ops jobs
+Documentation=https://github.com/session-foundation/session-shared-scripts
+OnFailure=session-ops-alert@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/true

deploy/session-ops-queue.timer

@@ -0,0 +1,11 @@
+# The schedule comes from the drop-in (OnCalendar=), written from jobs.toml's [queue].
+[Unit]
+Description=Schedule for the queued session-ops jobs
+Documentation=https://github.com/session-foundation/session-shared-scripts
+
+[Timer]
+Persistent=yes
+RandomizedDelaySec=2min
+
+[Install]
+WantedBy=timers.target

docs/jobs/crowdin-duplicates.md

@@ -6,7 +6,7 @@ for a plural string, is what gets exported. A slot is (string, locale, plural ca
 
 | | |
 | --- | --- |
-| Runs | `session-ops@crowdin-duplicates.timer`, daily 03:00 UTC |
+| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, last |
 | Secrets | `/etc/session-ops/crowdin.env`: a read-only `CROWDIN_API_TOKEN` and the channel's webhook |
 | Dry run | `session-ops run crowdin-duplicates --dry-run -- --locales de` |
 | Re-run | `systemctl start session-ops@crowdin-duplicates.service` |
@@ -18,10 +18,10 @@ The open slots live in `/var/lib/session-ops/crowdin-duplicates/duplicates.json`
 only what changed: slots newly holding 2+ translations, and slots that no longer do.
 Nothing changed, nothing is posted.
 
-Reconciliation judges every string of every locale once a day, so a new duplicate is
-posted within a day, well before the weekly export. A slot whose string was deleted,
-or whose locale left the project, resolves. One reconciliation runs at a time: one
-started while another holds the state's lock exits. The state is written only once
+Reconciliation judges every string of every locale each weekday, after that day's
+export, so a new duplicate is posted before the next one. A slot whose string was
+deleted, or whose locale left the project, resolves. One reconciliation runs at a time:
+one started while another holds the state's lock exits. The state is written only once
 Discord accepted every message, so a failed post is repeated in full rather than lost.
 
 Losing the state is not harmless the way a digest's dedup file is: every open slot

docs/jobs/crowdin-sync.md

@@ -6,7 +6,7 @@ straight onto session-localization's `main`, the TypeScript module Desktop and Q
 
 | | |
 | --- | --- |
-| Runs | `session-ops@crowdin-sync.timer`, Mondays 13:00 Australia/Melbourne |
+| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `zendesk-digest` |
 | Secrets | `/etc/session-ops/crowdin.env`: a read-only `CROWDIN_API_TOKEN`; `/etc/session-ops/publish.env` and the GitHub App key, to publish |
 | Dry run | `session-ops run crowdin-sync --dry-run`: everything but the push, with each platform's diff in the journal |
 | Re-run | `systemctl start session-ops@crowdin-sync.service`; one platform with `session-ops run crowdin-sync -- --only ios` |

docs/jobs/github-prs-digest.md

@@ -7,7 +7,7 @@ has a reason to already know about:
 
 | | |
 | --- | --- |
-| Runs | `session-ops@github-prs-digest.timer`, Mon–Fri 09:30 Australia/Melbourne |
+| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, first in the queue |
 | Secrets | `/etc/session-ops/github-prs.env`: `GITHUB_PRS_TOKEN` with no scopes at all, and the channel's webhook |
 | Dry run | `session-ops run github-prs-digest --dry-run`; `uv run github-prs-digest --dry-run` from a checkout |
 | Re-run | `systemctl start session-ops@github-prs-digest.service` |
@@ -30,7 +30,7 @@ much is waiting.
 
 ## Weekdays, and the state file
 
-The timer runs `Mon..Fri`, so Monday's run has to cover the weekend — hence a 72-hour
+The queue runs `Mon..Fri`, so Monday's run has to cover the weekend — hence a 72-hour
 window rather than a daily one. That window overlaps itself by two days on every run,
 and [`--state`](../../src/session_ops/github_prs/digest.py) is what stops the overlap being noise: it records
 which PRs reached Discord and what each one's `updated_at` was at the time.

docs/jobs/session-ops-silence.md

@@ -28,4 +28,4 @@ through a whole schedule: nothing fails, so nothing alerts. Each scheduled unit
 the first check that found it missing, so a job left disabled alerts once that is older
 than its `max_age_hours`: every scheduled job is meant to run.
 
-A job in `jobs.toml` with a `schedule` is watched; the `session-ops@` template stamps it.
+A job in `jobs.toml` with a `schedule`, or in its `[queue]`, is watched; the `session-ops@` template stamps it.

docs/jobs/snode-list.md

@@ -7,7 +7,7 @@ cannot reach the seed nodes. A change opens, or updates, a pull request from
 
 | | |
 | --- | --- |
-| Runs | `session-ops@snode-list.timer`, daily 13:00 Australia/Melbourne; the source updates at 10:00 UTC |
+| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `crowdin-sync`; the source updates at 10:00 UTC |
 | Secrets | `/etc/session-ops/publish.env` and the GitHub App key; `CROWDIN_DISCORD_WEBHOOK_URL` from `crowdin.env` for the summary |
 | Dry run | `session-ops run snode-list --dry-run` |
 | Re-run | `systemctl start session-ops@snode-list.service` |

docs/jobs/zendesk-digest.md

@@ -4,7 +4,7 @@ Claude reviews the Zendesk tickets awaiting a reply — `new` and `open`, no app
 
 | | |
 | --- | --- |
-| Runs | `session-ops@zendesk-digest.timer`, Mon–Fri 10:00 Australia/Melbourne: `zendesk-resolve-reviews --apply`, then `zendesk-triage` |
+| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `github-prs-digest`: `zendesk-resolve-reviews --apply`, then `zendesk-triage` |
 | Secrets | `/etc/session-ops/zendesk.env`: the Zendesk API token, the triage channel's webhook, and the Claude Code CLI login of the `zendesk` account |
 | Dry run | `session-ops run zendesk-digest --dry-run`; `uv run zendesk-triage --window-hours 72 --dry-run` from a checkout |
 | Re-run | `systemctl start session-ops@zendesk-digest.service` |
@@ -162,7 +162,7 @@ If a single request ever does hit the ceiling, the JSON never closes and no `str
 
 ## Schedule
 
-Runs **Monday to Friday at 10:00 Melbourne** over a 72h window (~70 tickets) — 00:00 UTC in winter, 23:00 UTC the previous day under AEDT. The cron this replaces had to pin UTC+10 year-round and drift an hour against local time, because GitHub cron is UTC-only; `OnCalendar=` takes a named zone, which tracks daylight saving and keeps the day-of-week local as well. The timezone belongs inside the expression; there is no `Timezone=` key in a `[Timer]` and systemd ignores one silently, so check any change with `systemd-analyze calendar`. Unlike the cron, a host that was asleep at 10:00 still gets its digest once on the next boot (`Persistent=yes`).
+Runs **Monday to Friday from 10:00 Melbourne**, second in the queue, over a 72h window (~70 tickets) — 00:00 UTC in winter, 23:00 UTC the previous day under AEDT. The cron this replaces had to pin UTC+10 year-round and drift an hour against local time, because GitHub cron is UTC-only; `OnCalendar=` takes a named zone, which tracks daylight saving and keeps the day-of-week local as well. The timezone belongs inside the expression; there is no `Timezone=` key in a `[Timer]` and systemd ignores one silently, so check any change with `systemd-analyze calendar`. Unlike the cron, a host that was asleep at 10:00 still gets its digest once on the next boot (`Persistent=yes`).
 
 The window is on `updated>`, not `created>`, so a ticket the requester adds detail to days after opening it is fetched again — a created-window would never see it. 72h rather than the 24h between runs so a failed run doesn't drop a day and Monday still reaches back past the weekend. Neither the overlap nor the wider net duplicates posts, because of the dedup state above.
 

src/session_ops/crowdin/generate_language_list.py

@@ -117,7 +117,7 @@ def generate_languages_ts(parsed_data: Dict[str, Any], output_path: str):
     keys = locale_keys(parsed_data)
     names, english, territories, unnamed = build_language_data(keys)
 
-    # Reported rather than raised: this runs in the weekly translation job, and a language nobody
+    # Reported rather than raised: this runs in the translation job, and a language nobody
     # has named yet must not hold up everyone else's strings. The fix is an override in
     # languageList.ts, which is a decision somebody makes rather than one this script can.
     if unnamed:

src/session_ops/crowdin/sync.py

@@ -1,5 +1,5 @@
 """
-Weekly: download the approved translations from Crowdin, validate them, generate each
+Weekdays: download the approved translations from Crowdin, validate them, generate each
 platform's strings, and publish them.
 
 - session-android and session-ios get a pull request from

src/session_ops/jobs.toml

@@ -11,9 +11,18 @@
 #                  without one reports to ALERT_DISCORD_WEBHOOK_URL
 #   unit           further [Service] lines, over the template's hardening
 #
+#   max_age_hours  for a queued job, its queue's: the weekend's 72 h (73 h across
+#                  April's DST change), plus every job ahead of it running to its timeout
+#
 # Timezones go in the schedule itself: systemd has no Timezone= key and ignores one.
 # Check a schedule with `systemd-analyze calendar "<schedule>"`.
 
+# One timer starts these together and each runs once the one before it has ended,
+# whether or not it succeeded, so no two of them share the host or their APIs.
+[queue]
+schedule = "Mon..Fri 10:00 Australia/Melbourne"
+jobs = ["github-prs-digest", "zendesk-digest", "crowdin-sync", "snode-list", "crowdin-duplicates"]
+
 [[job]]
 name = "github-prs-digest"
 description = "Daily digest of contributor pull requests"
@@ -23,9 +32,8 @@ user = "ghdigest"
 env_files = ["/etc/session-ops/github-prs.env"]
 env = ["GITHUB_PRS_TOKEN", "GITHUB_PRS_DISCORD_WEBHOOK_URL"]
 channel_env = "GITHUB_PRS_DISCORD_WEBHOOK_URL"
-# Weekdays, half an hour before the Zendesk digest. The window spans the weekend; a longer
-# gap (April's 73 h DST weekend, a missed run) reaches back through covered_until in --state.
-schedule = "Mon..Fri 09:30 Australia/Melbourne"
+# The window spans the weekend; a longer gap (April's 73 h DST weekend, a missed run)
+# reaches back through covered_until in --state.
 max_age_hours = 80
 timeout = "15min"
 
@@ -38,7 +46,6 @@ user = "zendesk"
 env_files = ["/etc/session-ops/zendesk.env"]
 env = ["ZENDESK_SUBDOMAIN", "ZENDESK_EMAIL", "ZENDESK_API_TOKEN", "ZENDESK_DISCORD_WEBHOOK_URL"]
 channel_env = "ZENDESK_DISCORD_WEBHOOK_URL"
-schedule = "Mon..Fri 10:00 Australia/Melbourne"
 max_age_hours = 80
 # A full 1000-ticket resolve is ten bulk jobs polled for up to 300 s each.
 timeout = "90min"
@@ -61,8 +68,7 @@ user = "crowdin"
 env_files = ["/etc/session-ops/crowdin.env"]
 env = ["CROWDIN_API_TOKEN", "CROWDIN_DISCORD_WEBHOOK_URL"]
 channel_env = "CROWDIN_DISCORD_WEBHOOK_URL"
-schedule = "*-*-* 03:00 UTC"
-max_age_hours = 28
+max_age_hours = 80
 # About 9 minutes; a full scan without --croql is ~110k requests, about 85 minutes.
 timeout = "2h"
 
@@ -87,8 +93,7 @@ user = "publisher"
 env_files = ["/etc/session-ops/crowdin.env", "/etc/session-ops/publish.env"]
 env = ["CROWDIN_API_TOKEN", "PUBLISH_GIT_AUTHOR"]
 channel_env = "CROWDIN_DISCORD_WEBHOOK_URL"
-schedule = "Mon 13:00 Australia/Melbourne"
-max_age_hours = 176
+max_age_hours = 80
 timeout = "1h"
 # Readable by this unit alone, at $CREDENTIALS_DIRECTORY; publishing needs it.
 unit = ["LoadCredential=github-app.pem:/etc/session-ops/github-app.pem"]
@@ -101,8 +106,7 @@ user = "publisher"
 env_files = ["/etc/session-ops/crowdin.env", "/etc/session-ops/publish.env"]
 env = ["PUBLISH_GIT_AUTHOR", "CROWDIN_DISCORD_WEBHOOK_URL"]
 channel_env = "CROWDIN_DISCORD_WEBHOOK_URL"
-schedule = "*-*-* 13:00 Australia/Melbourne"
-max_age_hours = 28
+max_age_hours = 80
 timeout = "10min"
 unit = ["LoadCredential=github-app.pem:/etc/session-ops/github-app.pem"]
 

src/session_ops/monitor/silence.py

@@ -43,8 +43,8 @@
 
 def load_jobs(path):
     """The scheduled jobs in the registry: nothing is late that has no schedule."""
-    return [{"name": job.name, "max_age_hours": job.max_age_hours}
-            for job in registry.load(path) if job.schedule]
+    return [{"name": job.name, "max_age_hours": job.max_age_hours, "timer": job.timer}
+            for job in registry.load(path) if job.scheduled]
 
 
 def stamp_time(stamps_dir, name):
@@ -121,7 +121,7 @@ def build_message(due, host, now):
                 else f"no success recorded since {utc(since)}")
         lines.append(f"• **{name}**: silent for **{duration(now - since)}** "
                      f"(allowed {job['max_age_hours']}h), {seen}.")
-        lines.append(f"  `systemctl list-timers session-ops@{name}.timer` · "
+        lines.append(f"  `systemctl list-timers {job['timer']}` · "
                      f"`journalctl -u session-ops@{name}.service -n 50 --no-pager`")
     return "\n".join(lines)