From ac80ca9954560a0cae2b0e8cce3e395991bde902 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 5 Oct 2026 12:07:55 +1100 Subject: [PATCH 1/4] feat: bump a client's submodule from a sparse clone submodules.bump clones only .gitmodules, moves the gitlink with update-index and publishes it through publish.pull_request, so the same branch rules apply: rebuilt from the base, not pushed when unchanged, retired when the base is already there. --- src/session_ops/platforms/submodules.py | 27 +++++++++++++++++++++++++ src/session_ops/shared/git.py | 11 ++++++++++ 2 files changed, 38 insertions(+) create mode 100644 src/session_ops/platforms/submodules.py diff --git a/src/session_ops/platforms/submodules.py b/src/session_ops/platforms/submodules.py new file mode 100644 index 0000000..6939cdf --- /dev/null +++ b/src/session_ops/platforms/submodules.py @@ -0,0 +1,27 @@ +"""Pull requests that move a client's submodule to a commit a job just published. + +Only the gitlink changes: the client is cloned sparse, and the submodule itself is +never checked out. +""" +import os +from dataclasses import dataclass + +from session_ops.platforms import publish +from session_ops.shared.git import Repo + + +@dataclass(frozen=True) +class Submodule: + repo: str + base: str + path: str + + +def bump(submodule, sha, work, token, api, branch, title, body, author, dry_run): + """Publish `submodule` at `sha` to `branch`. Returns a one-line result.""" + repo = Repo.sparse_clone(f"{publish.GITHUB}/{publish.ORG}/{submodule.repo}", + submodule.base, os.path.join(work, submodule.repo), + ["/.gitmodules"], token) + repo.set_gitlink(submodule.path, sha) + return publish.pull_request(repo, api, f"{publish.ORG}/{submodule.repo}", submodule.base, + branch, title, body, author, dry_run) diff --git a/src/session_ops/shared/git.py b/src/session_ops/shared/git.py index 6bea77d..5ad7f0c 100644 --- a/src/session_ops/shared/git.py +++ b/src/session_ops/shared/git.py @@ -86,5 +86,16 @@ def remote_tree(self, branch): def tree(self): return self.git("rev-parse", "HEAD^{tree}").stdout.strip() + def head(self): + return self.git("rev-parse", "HEAD").stdout.strip() + + def set_gitlink(self, path, sha): + """Point the submodule at `path` to `sha`, without checking it out.""" + if self.git("ls-files", "--stage", "--", path).stdout.split()[:1] != ["160000"]: + raise RuntimeError(f"{path} is not a submodule") + self.git("update-index", "--cacheinfo", f"160000,{sha},{path}") + # update-index drops skip-worktree, leaving the never-checked-out path a deletion. + self.git("update-index", "--skip-worktree", "--", path) + def push(self, branch, force=False): self.git("push", *(["--force"] if force else []), "origin", f"HEAD:refs/heads/{branch}") From 468a6c1a70319930602fe420699a561ec76a78c7 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 5 Oct 2026 12:07:55 +1100 Subject: [PATCH 2/4] feat: generate the snode list from the seed nodes and publish it everywhere snode-list now asks the seed nodes itself, as session-desktop-dynamic-assets' populate_cache.ts did, and publishes the result in one run: a commit onto dynamic-assets' main, a pull request bumping Desktop's dynamic_assets submodule to it, and the iOS copy. iOS no longer gets a list up to 13 hours old. The Discord summary lists each repo's result, its links in <> so Discord shows no preview. --- docs/jobs/snode-list.md | 36 +++-- src/session_ops/jobs.toml | 2 +- src/session_ops/platforms/snode_list.py | 186 +++++++++++++++++------- tests/goldens/units/dropins.txt | 2 +- 4 files changed, 160 insertions(+), 66 deletions(-) diff --git a/docs/jobs/snode-list.md b/docs/jobs/snode-list.md index 087bd7b..06d3ab3 100644 --- a/docs/jobs/snode-list.md +++ b/docs/jobs/snode-list.md @@ -1,24 +1,32 @@ # Static Snode List -Copies `service-nodes-cache.json` from session-desktop-dynamic-assets into session-ios -as `Session/Meta/service-nodes-cache.json`: the list a new client falls back on when it -cannot reach the seed nodes. A change opens, or updates, a pull request from -`feature/update-static-snode-list`. +Fetches the service node list from the seed nodes and publishes it wherever a client +bundles it, as the list a new client falls back on when it cannot reach them: + +| Target | Repo | What it gets | +| --- | --- | --- | +| `dynamic-assets` | session-desktop-dynamic-assets | `service-nodes-cache.json`, committed straight onto `main` | +| `desktop` | session-desktop | a pull request from `update-dynamic-assets` moving its `dynamic_assets` submodule to that commit | +| `ios` | session-ios | `Session/Meta/service-nodes-cache.json`, in a pull request from `feature/update-static-snode-list` | | | | | --- | --- | -| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `crowdin-sync`; the source updates at 10:00 UTC | -| Secrets | `/etc/session-ops/publish.env` and the GitHub App key; `CROWDIN_DISCORD_WEBHOOK_URL` from `crowdin.env` for the summary | +| Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `crowdin-sync` | +| Secrets | `/etc/session-ops/publish.env` and the GitHub App key, installed on the three repos above; `CROWDIN_DISCORD_WEBHOOK_URL` from `crowdin.env` for the summary | | Dry run | `session-ops run snode-list --dry-run` | | Re-run | `systemctl start session-ops@snode-list.service` | | Logs | `journalctl -u session-ops@snode-list -n 50 --no-pager` | -The file is committed exactly as fetched, but only once it holds service nodes, each -with an IP and a key: an error page or an empty list published as the fallback would -strand exactly the clients it exists for. The -branch follows the same rules as [the translation sync's](crowdin-sync.md): rebuilt -from `dev`, not pushed when unchanged, retired when `dev` already has the list. +The seeds are asked in turn until one answers with at least 20 active nodes, each +carrying every field the clients read; anything less publishes nothing, since an empty +or broken fallback would strand exactly the clients it exists for. Nodes without a +public IP are dropped, and the rest sorted by `pubkey_ed25519` so a run's diff is only +what changed on the network. + +The targets are independent, except that Desktop's bump needs dynamic-assets to have +published. The pull requests follow [the translation sync's](crowdin-sync.md) rules: +rebuilt from `dev`, not pushed when unchanged, retired when `dev` already has it. -Each run posts one line to the translations channel: how many nodes in the list have -`requested_unlock_height` set, meaning they asked to exit, and what happened to the pull -request. A failed run reports there too. +Each run posts one message to the translations channel: how many nodes have +`requested_unlock_height` set, meaning they asked to exit, then what happened in each +repo. A failed target is reported there too. diff --git a/src/session_ops/jobs.toml b/src/session_ops/jobs.toml index b23b351..f42cffc 100644 --- a/src/session_ops/jobs.toml +++ b/src/session_ops/jobs.toml @@ -100,7 +100,7 @@ unit = ["LoadCredential=github-app.pem:/etc/session-ops/github-app.pem"] [[job]] name = "snode-list" -description = "Fallback service node list into session-ios" +description = "Fallback service node list into dynamic-assets, Desktop and iOS" entry = "session_ops.platforms.snode_list:main" user = "publisher" env_files = ["/etc/session-ops/crowdin.env", "/etc/session-ops/publish.env"] diff --git a/src/session_ops/platforms/snode_list.py b/src/session_ops/platforms/snode_list.py index f24b4bd..c1318b5 100644 --- a/src/session_ops/platforms/snode_list.py +++ b/src/session_ops/platforms/snode_list.py @@ -1,64 +1,124 @@ """ -Daily: copy the service node list from session-desktop-dynamic-assets into session-ios, -as the fallback a new client uses when it cannot reach the seed nodes, and open a pull -request from feature/update-static-snode-list when it changed. Each run posts how many -nodes are requesting exit to the translations channel. +Weekdays: fetch the service node list from the seed nodes and publish it wherever a +client bundles it, as the fallback a new client uses when it cannot reach them. -The list is published as fetched, byte for byte, but only once it holds service nodes, -each with an IP and a key: a fallback that is empty or unreadable would strand exactly -the clients it exists for. +- session-desktop-dynamic-assets gets it as a commit straight onto `main`. +- session-desktop gets a pull request from update-dynamic-assets moving its + dynamic_assets submodule to that commit. +- session-ios gets a copy in a pull request from feature/update-static-snode-list. + +Each target is independent, except that Desktop's bump waits on dynamic-assets having +published. Each run posts how many nodes are requesting exit, and what happened in each +repo, to the translations channel. session-ops run snode-list [--dry-run] Environment: CROWDIN_DISCORD_WEBHOOK_URL where the summary goes (not needed with --dry-run) + PUBLISH_GIT_AUTHOR "Name " the commits are authored as + plus what shared/github.py reads to publish (not needed with --dry-run) """ import argparse import json import os +import re import tempfile -from session_ops.ops.runner import step -from session_ops.platforms import publish +from session_ops.ops.runner import Outcome, call_target, step +from session_ops.platforms import publish, submodules from session_ops.shared import discord, github, http from session_ops.shared.env import get_env from session_ops.shared.git import Repo -SOURCE = ("https://raw.githubusercontent.com/session-foundation/" - "session-desktop-dynamic-assets/main/service-nodes-cache.json") -REPO = "session-ios" -PATH = "Session/Meta/service-nodes-cache.json" -BRANCH = "feature/update-static-snode-list" -TITLE = "[Automated] Update fallback static snode list" -BODY = """[Automated] +SEEDS = ("https://seed1.getsession.org/json_rpc", + "https://seed2.getsession.org/json_rpc", + "https://seed3.getsession.org/json_rpc") +# Each node's fields as the clients read them, in this order. +FIELDS = {"public_ip": str, "storage_port": int, "pubkey_ed25519": str, + "pubkey_x25519": str, "requested_unlock_height": int, "storage_lmq_port": int, + "storage_server_version": list, "swarm": str} +# The network has over a thousand; fewer means a seed answered from a broken view of it. +MIN_NODES = 20 + +ASSETS = "session-desktop-dynamic-assets" +ASSETS_PATH = "service-nodes-cache.json" +ASSETS_MESSAGE = "chore: update snode cache" + +DESKTOP = submodules.Submodule("session-desktop", "dev", "dynamic_assets") +DESKTOP_BRANCH = "update-dynamic-assets" +DESKTOP_TITLE = "chore: Update dynamic assets submodule" +DESKTOP_BODY = """[Automated] +Moves the `session-desktop-dynamic-assets` submodule to the latest service node list. +""" + +IOS = "session-ios" +IOS_PATH = "Session/Meta/service-nodes-cache.json" +IOS_BRANCH = "feature/update-static-snode-list" +IOS_TITLE = "[Automated] Update fallback static snode list" +IOS_BODY = """[Automated] This PR updates the static service node list which is used as a fallback when a new client is unable to contact the seed nodes """ +TARGETS = ("dynamic-assets", "desktop", "ios") -def fetch(session): - resp = session.request("GET", SOURCE) + +def ask(session, seed): + resp = session.request("POST", seed, json={"method": "get_service_nodes", "params": { + "active_only": True, "fields": {**dict.fromkeys(FIELDS, True), "height": True}}}) if resp.status_code != 200: - raise RuntimeError(f"{SOURCE} answered {resp.status_code}") + raise RuntimeError(f"answered {resp.status_code}") try: - nodes = json.loads(resp.content).get("service_node_states") - except (ValueError, AttributeError) as exc: - raise RuntimeError(f"{SOURCE} is not a JSON object: {exc}") from exc - # An empty list parses as well as a full one, and strands clients just the same. - if not nodes or not isinstance(nodes, list) or not all( - isinstance(node, dict) and node.get("public_ip") and node.get("pubkey_ed25519") - for node in nodes): - raise RuntimeError(f"{SOURCE} holds no usable service_node_states") - return resp.content - - -def summary(content, result): + result = resp.json()["result"] + nodes, height = result["service_node_states"], result["height"] + except (ValueError, KeyError, TypeError) as exc: + raise RuntimeError(f"no service_node_states in its answer ({exc!r})") from exc + if not isinstance(nodes, list) or not isinstance(height, int): + raise RuntimeError("no service_node_states in its answer") + for node in nodes: + if not isinstance(node, dict) or not all( + isinstance(node.get(name), kind) for name, kind in FIELDS.items()): + raise RuntimeError(f"a node lacks one of {', '.join(FIELDS)}: {node!r:.200}") + nodes = [node for node in nodes if node["public_ip"] not in ("", "0.0.0.0")] + if len(nodes) < MIN_NODES: + raise RuntimeError(f"only {len(nodes)} usable nodes, fewer than {MIN_NODES}") + return nodes, height + + +def render(nodes, height): + """The list as the clients bundle it: sorted by key, so a run's diff is only what + changed on the network.""" + nodes = sorted(nodes, key=lambda node: node["pubkey_ed25519"]) + return json.dumps({"service_node_states": [{name: node[name] for name in FIELDS} + for node in nodes], + "height": height}, indent=2, ensure_ascii=False).encode() + + +def fetch(session): + """The first seed's list that holds enough nodes, each with every field.""" + errors = [] + for seed in SEEDS: + try: + return render(*ask(session, seed)) + except (RuntimeError, OSError) as exc: + errors.append(f"{seed}: {exc}") + raise RuntimeError("no seed node gave a usable list: " + "; ".join(errors)) + + +def write(repo, path, content): + with open(os.path.join(repo.path, path), "wb") as handle: + handle.write(content) + + +def summary(content, results): """The channel's line: nodes asking to exit (`requested_unlock_height` set), then - what happened to the pull request.""" + what happened in each repo.""" data = json.loads(content) nodes = data["service_node_states"] exiting = sum(1 for node in nodes if node.get("requested_unlock_height")) - return (f"🛰️ **snode-list**: {exiting} of {len(nodes)} service nodes are requesting " - f"exit at height {data.get('height')}.\n{result}") + # keeps Discord from unfurling a preview of each pull request. + lines = [re.sub(r"(https?://\S+)", r"<\1>", result) for result in results] + return "\n".join([f"🛰️ **snode-list**: {exiting} of {len(nodes)} service nodes are " + f"requesting exit at height {data.get('height')}.", *lines]) def main(argv=None): @@ -73,22 +133,48 @@ def main(argv=None): work = os.environ.get("SESSION_OPS_WORK_DIR") or tempfile.mkdtemp(prefix="snode-list-") step("fetch") content = fetch(http.Session()) - token = None if args.dry_run else github.publish_token(publish.ORG, [REPO]) - step("checkout") - repo = Repo.sparse_clone(f"{publish.GITHUB}/{publish.ORG}/{REPO}", "dev", - os.path.join(work, REPO), [f"/{PATH}"], token) - with open(os.path.join(repo.path, PATH), "wb") as handle: - handle.write(content) - step("publish") - result = publish.pull_request(repo, github.session(token) if token else None, - f"{publish.ORG}/{REPO}", "dev", BRANCH, TITLE, BODY, author, - args.dry_run) - print(result) + token = None if args.dry_run else github.publish_token(publish.ORG, + [ASSETS, DESKTOP.repo, IOS]) + api = github.session(token) if token else None + results, published = [], {} + + def assets(_): + step("dynamic-assets: publish") + repo = Repo.sparse_clone(f"{publish.GITHUB}/{publish.ORG}/{ASSETS}", "main", + os.path.join(work, ASSETS), [f"/{ASSETS_PATH}"], token) + write(repo, ASSETS_PATH, content) + results.append(publish.direct_push(repo, api, f"{publish.ORG}/{ASSETS}", "main", + ASSETS_MESSAGE, "", author, args.dry_run)) + published["sha"] = repo.head() + + def desktop(_): + step("desktop: publish") + if "sha" not in published: + raise RuntimeError(f"{ASSETS} did not publish, so there is nothing to bump to") + results.append(submodules.bump(DESKTOP, published["sha"], work, token, api, + DESKTOP_BRANCH, DESKTOP_TITLE, DESKTOP_BODY, author, + args.dry_run)) + + def ios(_): + step("ios: publish") + repo = Repo.sparse_clone(f"{publish.GITHUB}/{publish.ORG}/{IOS}", "dev", + os.path.join(work, IOS), [f"/{IOS_PATH}"], token) + write(repo, IOS_PATH, content) + results.append(publish.pull_request(repo, api, f"{publish.ORG}/{IOS}", "dev", + IOS_BRANCH, IOS_TITLE, IOS_BODY, author, + args.dry_run)) + + errors = {name: call_target(function, []) + for name, function in zip(TARGETS, (assets, desktop, ios))} + for result in results: + print(result) step("report") - message = summary(content, result) + message = summary(content, results + [f"{name}: failed, see the alert" + for name, error in errors.items() if error]) if args.dry_run: print(message) - return - payload = {"content": message, "allowed_mentions": {"parse": []}} - if discord.post_to_discord(http.Session(), webhook, [payload]) != 1: - raise RuntimeError("Discord did not accept the summary") + else: + payload = {"content": message, "allowed_mentions": {"parse": []}} + if discord.post_to_discord(http.Session(), webhook, [payload]) != 1: + raise RuntimeError("Discord did not accept the summary") + return Outcome(targets=errors) diff --git a/tests/goldens/units/dropins.txt b/tests/goldens/units/dropins.txt index 27a9306..1d63f13 100644 --- a/tests/goldens/units/dropins.txt +++ b/tests/goldens/units/dropins.txt @@ -78,7 +78,7 @@ OnCalendar=hourly # Generated by `session-ops units` from jobs.toml. Edit the registry, not this. [Unit] -Description=Fallback service node list into session-ios +Description=Fallback service node list into dynamic-assets, Desktop and iOS After=session-ops@github-prs-digest.service session-ops@zendesk-digest.service session-ops@crowdin-sync.service [Service] From 300aab4693f7d13d4f8dd868dc4e5397d12c75f0 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 5 Oct 2026 12:07:56 +1100 Subject: [PATCH 3/4] feat: bump session-localization in every client after the Crowdin sync crowdin-sync now opens an update-localization pull request in session-desktop, session-app, session-website, session-appium and session-playwright, moving each one's submodule to the commit it just pushed to session-localization. --- README.md | 4 +- deploy/README.md | 3 +- docs/jobs/crowdin-sync.md | 22 +++- src/session_ops/crowdin/sync.py | 67 ++++++++-- tests/test_publish.py | 220 +++++++++++++++++++++++++------- 5 files changed, 254 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 1d5224b..a1a9ce3 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ share for translations. One package, `session_ops`, deployed to one self-hosted | `zendesk-relay` | Drafts and sends Zendesk replies from `claude:` private notes | [zendesk-relay](docs/jobs/zendesk-relay.md) | | `github-prs-digest` | Weekday Discord digest of open pull requests from outside contributors | [github-prs-digest](docs/jobs/github-prs-digest.md) | | `crowdin-duplicates` | Crowdin string slots holding more than one translation, as they open and close | [crowdin-duplicates](docs/jobs/crowdin-duplicates.md) | -| `crowdin-sync` | Weekday: Crowdin translations into pull requests on iOS, Android and the localization module | [crowdin-sync](docs/jobs/crowdin-sync.md) | -| `snode-list` | Weekday: the fallback service node list into session-ios | [snode-list](docs/jobs/snode-list.md) | +| `crowdin-sync` | Weekday: Crowdin translations into iOS, Android and the localization module, and its submodule bumped in each client | [crowdin-sync](docs/jobs/crowdin-sync.md) | +| `snode-list` | Weekday: the fallback service node list from the seed nodes into dynamic-assets, Desktop and iOS | [snode-list](docs/jobs/snode-list.md) | | `release-stats` | On demand: download counts of the latest releases | [release-stats](docs/jobs/release-stats.md) | | `session-ops-silence` | Discord alerts for a job that failed, or stopped running | [session-ops-silence](docs/jobs/session-ops-silence.md) | diff --git a/deploy/README.md b/deploy/README.md index e0fe969..14837db 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -84,7 +84,8 @@ for link in /etc/systemd/system/timers.target.wants/session-ops@*.timer; do [ -L "$link" ] && systemctl disable --now "${link##*/}" done systemctl disable --now session-ops-queue.timer zendesk-relay.service -for repo in session-android session-ios session-localization; do +for repo in session-android session-ios session-localization session-desktop-dynamic-assets \ + session-desktop session-app session-website session-appium session-playwright; do gh pr list -R "session-foundation/$repo" --state open --json number,headRefName \ -q '.[] | select(.headRefName | startswith("rehearsal/")) | .number' | xargs -r -I{} gh pr close -R "session-foundation/$repo" {} --delete-branch diff --git a/docs/jobs/crowdin-sync.md b/docs/jobs/crowdin-sync.md index 320676d..5585924 100644 --- a/docs/jobs/crowdin-sync.md +++ b/docs/jobs/crowdin-sync.md @@ -3,13 +3,23 @@ Downloads the approved translations from Crowdin, validates them, and publishes each platform's strings: a pull request on session-android and session-ios, and a commit straight onto session-localization's `main`, the TypeScript module Desktop and QA use. +Each client holding that module as a submodule then gets a pull request moving it to the +commit just pushed: + +| Target | Repo | Base | Submodule | +| --- | --- | --- | --- | +| `desktop` | session-desktop | `dev` | `ts/localization` | +| `app` | session-app | `main` | `packages/localization/src/localization-src` | +| `website` | session-website | `main` | `lib/app_localization` | +| `appium` | session-appium | `main` | `run/localizer/lib` | +| `playwright` | session-playwright | `main` | `tests/localization/lib` | | | | | --- | --- | | Runs | `session-ops-queue.timer`, Mon–Fri from 10:00 Australia/Melbourne, after `zendesk-digest` | | Secrets | `/etc/session-ops/crowdin.env`: a read-only `CROWDIN_API_TOKEN`; `/etc/session-ops/publish.env` and the GitHub App key, to publish | | Dry run | `session-ops run crowdin-sync --dry-run`: everything but the push, with each platform's diff in the journal | -| Re-run | `systemctl start session-ops@crowdin-sync.service`; one platform with `session-ops run crowdin-sync -- --only ios` | +| Re-run | `systemctl start session-ops@crowdin-sync.service`; one target with `session-ops run crowdin-sync -- --only ios` | | Logs | `journalctl -u session-ops@crowdin-sync -n 100 --no-pager`; the run's downloads, parsed JSON and validation report under `/var/lib/session-ops/crowdin-sync/runs/`, for 14 days | One process, in order: @@ -21,16 +31,22 @@ One process, in order: 3. **For each platform**, independently, so one failing does not stop the others: a shallow, sparse checkout of just the paths its generator writes, the generator, then publishing. The alert says which platform failed and at which step. +4. **Bump each client's submodule** to the commit localization published, cloning only + `.gitmodules` and moving the gitlink. A bump is skipped, and reported, when + localization fails; with `--only` leaving localization out, it moves to `main`'s tip. The pull requests come from `feature/update-crowdin-translations`, rebuilt from `dev` each run and force-pushed: the branch is the job's, and nobody else commits to it. An unchanged tree is not pushed again, and a run with nothing to change closes the pull request and deletes the branch. Android's own CI validates its pull request, so no -Gradle build runs here. +Gradle build runs here. The bumps follow the same rules from `update-localization`, +rebuilt from each client's base; appium's and playwright's assertions follow the +strings, so their pull request may need test fixes before it merges. ## Publishing -Publishing authenticates as the GitHub App: `GITHUB_APP_ID` in `publish.env` and its +Publishing authenticates as the GitHub App, which must be installed on every repo above +with contents and pull requests write: `GITHUB_APP_ID` in `publish.env` and its key in `/etc/session-ops/github-app.pem`, and a run missing either exits naming it (a dry run needs neither); see [publish.env.example](../../deploy/env/publish.env.example). Commits are authored as `PUBLISH_GIT_AUTHOR`. diff --git a/src/session_ops/crowdin/sync.py b/src/session_ops/crowdin/sync.py index f3e774d..5e437ea 100644 --- a/src/session_ops/crowdin/sync.py +++ b/src/session_ops/crowdin/sync.py @@ -5,10 +5,14 @@ - session-android and session-ios get a pull request from feature/update-crowdin-translations, rebuilt from `dev` each run. - session-localization gets a commit straight onto `main`. +- The clients holding session-localization as a submodule (session-desktop, + session-app, session-website, session-appium, session-playwright) each get a pull request from + update-localization moving it to that commit, or to `main`'s tip when this run + leaves localization out. Each platform is checked out shallow and sparse, only the paths its generator writes. -The three are independent targets: one failing to publish does not stop the others, -and the alert says which. Android's own CI validates its pull request, so no Gradle +The targets are independent: one failing to publish does not stop the others, and the +alert says which; only the submodule bumps wait on localization, when it runs. Android's own CI validates its pull request, so no Gradle build runs here. The run's inputs, outputs and validation report are kept under the job's @@ -19,7 +23,7 @@ PUBLISH_GIT_AUTHOR "Name " the commits are authored as plus what shared/github.py reads to publish (not needed with --dry-run) - session-ops run crowdin-sync [--dry-run] [-- --only android] + session-ops run crowdin-sync [--dry-run] [-- --only android desktop] """ import argparse import os @@ -31,7 +35,7 @@ generate_android_strings, generate_ios_strings, generate_language_list, parse_xliff) from session_ops.ops.runner import Outcome, call_target, step -from session_ops.platforms import publish +from session_ops.platforms import publish, submodules from session_ops.shared import github from session_ops.shared.git import Repo @@ -50,9 +54,25 @@ IOS_TRANSLATIONS = "Session/Meta/Translations" IOS_CONSTANTS = "SessionUIKit/Style Guide/Constants.swift" -TARGETS = ("android", "ios", "localization") +SUBMODULE_BRANCH = "update-localization" +SUBMODULE_TITLE = "chore: Update localization submodule" +SUBMODULE_BODY = """[Automated] +Moves the `session-localization` submodule to the latest translations from Crowdin. +""" +CLIENTS = { + "desktop": submodules.Submodule("session-desktop", "dev", "ts/localization"), + "app": submodules.Submodule("session-app", "main", + "packages/localization/src/localization-src"), + "website": submodules.Submodule("session-website", "main", "lib/app_localization"), + "appium": submodules.Submodule("session-appium", "main", "run/localizer/lib"), + "playwright": submodules.Submodule("session-playwright", "main", + "tests/localization/lib"), +} + +TARGETS = ("android", "ios", "localization", *CLIENTS) REPOS = {"android": "session-android", "ios": "session-ios", - "localization": "session-localization"} + "localization": "session-localization", + **{target: client.repo for target, client in CLIENTS.items()}} def checkout(target, work, token): @@ -94,13 +114,31 @@ def publish_target(target, repo, api, author, dry_run): dry_run) -def sync_target(target, work, parsed, token, api, author, dry_run): +def sync_target(target, work, parsed, token, api, author, dry_run, published): step(f"{target}: checkout") repo = checkout(target, work, token) step(f"{target}: generate") generate(target, repo, parsed) step(f"{target}: publish") print(publish_target(target, repo, api, author, dry_run)) + published[target] = repo.head() + + +def localization_tip(work, token): + url = f"{publish.GITHUB}/{publish.ORG}/{REPOS['localization']}" + listed = Repo(work, token).git("ls-remote", url, "refs/heads/main").stdout.split() + if not listed: + raise RuntimeError(f"{url} has no main branch") + return listed[0] + + +def bump_target(target, work, token, api, author, dry_run, published, localization_ran): + step(f"{target}: publish") + if localization_ran and "localization" not in published: + raise RuntimeError("localization did not publish, so there is nothing to bump to") + sha = published.get("localization") or localization_tip(work, token) + print(submodules.bump(CLIENTS[target], sha, work, token, api, SUBMODULE_BRANCH, + SUBMODULE_TITLE, SUBMODULE_BODY, author, dry_run)) def keep(work, runs_dir, names): @@ -148,7 +186,16 @@ def main(argv=None): token = None if args.dry_run else github.publish_token( publish.ORG, [REPOS[t] for t in args.only]) api = github.session(token) if token else None - results = {target: call_target(lambda _, target=target: sync_target( - target, work, parsed, token, api, author, args.dry_run), []) - for target in args.only} + published, localization_ran = {}, "localization" in args.only + + def run_target(target): + if target in CLIENTS: + bump_target(target, work, token, api, author, args.dry_run, published, + localization_ran) + else: + sync_target(target, work, parsed, token, api, author, args.dry_run, published) + + # TARGETS' order, whatever --only's: the bumps follow localization. + results = {target: call_target(lambda _, target=target: run_target(target), []) + for target in TARGETS if target in args.only} return Outcome(targets=results) diff --git a/tests/test_publish.py b/tests/test_publish.py index 02dfaa6..a7b1c7c 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -8,6 +8,7 @@ import io import json import os +import shutil import subprocess import tempfile import unittest @@ -18,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from session_ops.crowdin import sync -from session_ops.platforms import publish, snode_list +from session_ops.platforms import publish, snode_list, submodules from session_ops.shared import github from session_ops.shared.git import Repo from session_ops.shared.testing import FakeResponse, FakeSession @@ -31,8 +32,9 @@ def git(*args, cwd=None): text=True).stdout.strip() -def bare_repo(root, name, branch, files): - """A bare repository `name` whose `branch` holds `files`.""" +def bare_repo(root, name, branch, files, gitlinks=None): + """A bare repository `name` whose `branch` holds `files`, and a submodule at each + of `gitlinks`' paths pinned to its commit.""" seed = os.path.join(root, f"seed-{name}") os.makedirs(seed) git("init", "-q", "-b", branch, cwd=seed) @@ -41,6 +43,8 @@ def bare_repo(root, name, branch, files): with open(os.path.join(seed, path), "w", encoding="utf-8") as handle: handle.write(content) git("add", "-A", cwd=seed) + for path, sha in (gitlinks or {}).items(): + git("update-index", "--add", "--cacheinfo", f"160000,{sha},{path}", cwd=seed) git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "seed", cwd=seed) bare = os.path.join(root, "session-foundation", name) git("clone", "-q", "--bare", seed, bare) @@ -356,46 +360,145 @@ def test_nothing_configured_names_both(self): self.assertIn("private key", message) -class TestSnodeList(RepoTest): - def run_job(self, body, *argv): - bare_repo(self.root, "session-ios", "dev", - {snode_list.PATH: '{"old": true}\n', "Other.swift": "x"}) - fetched = FakeSession([FakeResponse(None)]) - fetched._responses[0].text = body - with mock.patch.object(snode_list.http, "Session", lambda: fetched), \ - mock.patch.dict(os.environ, {"SESSION_OPS_WORK_DIR": self.work}), \ - contextlib.redirect_stdout(io.StringIO()) as out: - snode_list.main(list(argv)) - return out.getvalue() - - NODE = {"public_ip": "203.0.113.7", "pubkey_ed25519": "ab" * 32} - - def test_the_fetched_list_is_committed_byte_for_byte(self): - body = json.dumps({"service_node_states": [self.NODE, self.NODE], "height": 1}) - self.assertIn("would push", self.run_job(body, "--dry-run")) - with open(os.path.join(self.work, "session-ios", snode_list.PATH), - encoding="utf-8") as handle: - self.assertEqual(handle.read(), body) +def gitlink(bare, branch, path): + return git("ls-tree", branch, path, cwd=bare).split()[2] + + +def tip(bare, branch): + return git("rev-parse", branch, cwd=bare) + + +OLD_SHA = "1" * 40 + + +class TestSubmoduleBump(RepoTest): + CLIENT = submodules.Submodule("client", "dev", "lib/strings") + + def setUp(self): + super().setUp() + self.bare = bare_repo(self.root, "client", "dev", + {".gitmodules": "[submodule]\n", "src/App.ts": "x"}, + {"lib/strings": OLD_SHA}) + + def bump(self, sha, api): + with contextlib.redirect_stdout(io.StringIO()): + return submodules.bump(self.CLIENT, sha, self.work, None, api, "bump", "Bump", + "body", AUTHOR, False) + + def test_only_the_pointer_moves(self): + api = GitHubFake() + self.assertIn("https://github.com/pr/7", self.bump("2" * 40, api)) + self.assertEqual(gitlink(self.bare, "bump", "lib/strings"), "2" * 40) + self.assertEqual(git("diff", "--name-only", "dev", "bump", cwd=self.bare), + "lib/strings") + self.assertEqual([c[2]["json"]["title"] for c in api.calls if c[0] == "POST"], + ["Bump"]) + + def test_a_client_already_there_gets_no_pull_request(self): + self.assertIn("no changes", self.bump(OLD_SHA, GitHubFake())) + self.assertNotIn("bump", self.branches("client")) + + def test_a_path_that_is_no_submodule_is_refused(self): + client = submodules.Submodule("client", "dev", "src/App.ts") + with self.assertRaisesRegex(RuntimeError, "not a submodule"): + submodules.bump(client, "2" * 40, self.work, None, None, "bump", "Bump", "body", + AUTHOR, True) + - def test_the_summary_counts_the_nodes_requesting_exit(self): - exiting = dict(self.NODE, requested_unlock_height=2212248) - staying = dict(self.NODE, requested_unlock_height=0) - body = json.dumps({"service_node_states": [exiting, staying, self.NODE], - "height": 2210944}) - self.assertIn("1 of 3 service nodes are requesting exit at height 2210944", - self.run_job(body, "--dry-run")) +def node(index, **fields): + return {"public_ip": f"203.0.113.{index}", "storage_port": 22100, + "pubkey_ed25519": f"{index:064x}", "pubkey_x25519": "cd" * 32, + "requested_unlock_height": 0, "storage_lmq_port": 20200, + "storage_server_version": [2, 11, 3], "swarm": "60ffffffffffffff", **fields} - def test_a_body_that_is_not_json_is_never_published(self): - with self.assertRaises(RuntimeError): - self.run_job("rate limited", "--dry-run") - def test_an_empty_or_malformed_list_is_never_published(self): - for body in ("[]", "{}", '{"service_node_states": []}', - '{"service_node_states": [{"public_ip": "203.0.113.7"}]}'): - response = FakeResponse(None) - response.text = body - with self.subTest(body=body), self.assertRaises(RuntimeError): - snode_list.fetch(FakeSession([response])) +def seed_answer(nodes, height=2210944): + return FakeResponse({"result": {"service_node_states": nodes, "height": height, + "status": "OK"}}) + + +NODES = [node(index) for index in range(snode_list.MIN_NODES, 0, -1)] + + +class TestSnodeListFetch(unittest.TestCase): + def test_the_list_is_sorted_by_key_with_the_clients_fields_only(self): + nodes = [dict(n, extra=1) for n in NODES] + [node(99, public_ip="0.0.0.0")] + data = json.loads(snode_list.fetch(FakeSession([seed_answer(nodes)]))) + keys = [n["pubkey_ed25519"] for n in data["service_node_states"]] + self.assertEqual(keys, sorted(n["pubkey_ed25519"] for n in NODES)) + self.assertEqual(list(data["service_node_states"][0]), list(snode_list.FIELDS)) + self.assertEqual(data["height"], 2210944) + + def test_a_seed_that_fails_is_passed_over_for_the_next(self): + session = FakeSession([FakeResponse({}, status_code=502), seed_answer(NODES)]) + self.assertTrue(snode_list.fetch(session)) + self.assertEqual([c[1] for c in session.calls], list(snode_list.SEEDS[:2])) + + def test_a_short_or_malformed_list_is_never_published(self): + broken = dict(NODES[0]) + del broken["pubkey_x25519"] + for answer in (seed_answer(NODES[:-1]), seed_answer(NODES[1:] + [broken]), + FakeResponse({"error": "busy"}), seed_answer(NODES, height=None)): + with self.subTest(answer=answer.text[:60]), \ + self.assertRaisesRegex(RuntimeError, "no seed node gave a usable list"): + snode_list.fetch(FakeSession([answer] * len(snode_list.SEEDS))) + + +class TestSnodeList(RepoTest): + def setUp(self): + super().setUp() + self.assets = bare_repo(self.root, snode_list.ASSETS, "main", + {snode_list.ASSETS_PATH: "{}", "GeoLite2-Country.mmdb": "x"}) + self.desktop = bare_repo(self.root, "session-desktop", "dev", + {".gitmodules": "[submodule]\n"}, + {"dynamic_assets": tip(self.assets, "main")}) + self.ios = bare_repo(self.root, "session-ios", "dev", + {snode_list.IOS_PATH: '{"old": true}\n', "Other.swift": "x"}) + + def run_job(self, *argv): + api, posted = GitHubFake(), [] + exiting = [node(1, requested_unlock_height=2212248)] + NODES[:-1] + with mock.patch.object(snode_list.http, "Session", + lambda: FakeSession([seed_answer(exiting)])), \ + mock.patch.object(snode_list.github, "publish_token", lambda owner, repos: "t"), \ + mock.patch.object(snode_list.github, "session", lambda token: api), \ + mock.patch.object(snode_list.discord, "post_to_discord", + lambda session, url, messages: posted.extend(messages) or 1), \ + mock.patch.dict(os.environ, {"SESSION_OPS_WORK_DIR": self.work, + "CROWDIN_DISCORD_WEBHOOK_URL": "https://hook"}), \ + contextlib.redirect_stdout(io.StringIO()) as out, \ + contextlib.redirect_stderr(io.StringIO()): + outcome = snode_list.main(list(argv)) + return outcome, posted, out.getvalue() + + def test_every_repo_bundling_the_list_gets_it(self): + outcome, posted, _ = self.run_job() + self.assertEqual(outcome.failures(), {}) + published = git("show", f"main:{snode_list.ASSETS_PATH}", cwd=self.assets) + self.assertEqual(len(json.loads(published)["service_node_states"]), len(NODES)) + self.assertEqual(gitlink(self.desktop, snode_list.DESKTOP_BRANCH, "dynamic_assets"), + tip(self.assets, "main")) + self.assertEqual(git("show", f"{snode_list.IOS_BRANCH}:{snode_list.IOS_PATH}", + cwd=self.ios), published) + message = posted[0]["content"] + self.assertIn(f"1 of {len(NODES)} service nodes are requesting exit at height 2210944", + message) + self.assertIn("session-foundation/session-desktop: ", message) + self.assertNotIn(" https://", message) + + def test_desktop_is_not_bumped_when_dynamic_assets_did_not_publish(self): + shutil.rmtree(self.assets) + outcome, posted, _ = self.run_job() + self.assertEqual(set(outcome.failures()), {"dynamic-assets", "desktop"}) + self.assertIn(snode_list.IOS_BRANCH, self.branches("session-ios")) + self.assertIn("desktop: failed, see the alert", posted[0]["content"]) + + def test_a_dry_run_pushes_nothing(self): + _, posted, out = self.run_job("--dry-run") + self.assertEqual(posted, []) + self.assertIn("would push to main", out) + self.assertEqual(self.branches("session-desktop"), ["dev"]) + self.assertEqual(self.branches("session-ios"), ["dev"]) class TestCrowdinSync(RepoTest): @@ -409,7 +512,12 @@ def setUp(self): bare_repo(self.root, "session-ios", "dev", {f"{sync.IOS_TRANSLATIONS}/Localizable.xcstrings": "old", sync.IOS_CONSTANTS: "k"}) - bare_repo(self.root, "session-localization", "main", {"generated/english.ts": "old"}) + self.localization = bare_repo(self.root, "session-localization", "main", + {"generated/english.ts": "old"}) + self.clients = {target: bare_repo(self.root, client.repo, client.base, + {".gitmodules": "[submodule]\n"}, + {client.path: OLD_SHA}) + for target, client in sync.CLIENTS.items()} def fake_generate(self, target, repo, parsed): if target == "ios": @@ -420,7 +528,7 @@ def fake_generate(self, target, repo, parsed): with open(os.path.join(repo.path, path), "w", encoding="utf-8") as handle: handle.write(json.dumps(target)) - def test_each_target_publishes_on_its_own(self): + def run_sync(self, *argv): env = {"SESSION_OPS_WORK_DIR": self.work, "CROWDIN_API_TOKEN": "t", "PUBLISH_GIT_AUTHOR": AUTHOR} api = GitHubFake() @@ -432,15 +540,35 @@ def test_each_target_publishes_on_its_own(self): mock.patch.dict(os.environ, env), \ contextlib.redirect_stdout(io.StringIO()), \ contextlib.redirect_stderr(io.StringIO()): - outcome = sync.main([]) + return sync.main(list(argv)), api + + def bumped(self, target): + client = sync.CLIENTS[target] + return gitlink(self.clients[target], sync.SUBMODULE_BRANCH, client.path) + + def test_each_target_publishes_on_its_own(self): + outcome, api = self.run_sync() self.assertEqual(outcome.failures(), {"ios": "RuntimeError: catalog write failed"}) self.assertIn(sync.BOT_BRANCH, self.branches("session-android")) self.assertEqual(git("show", "main:generated/english.ts", cwd=os.path.join(self.root, "session-foundation", "session-localization")), '"localization"') - opened = [c for c in api.calls if c[0] == "POST"] - self.assertEqual(len(opened), 1) - self.assertEqual(opened[0][2]["json"]["title"], sync.TITLE) + for target in sync.CLIENTS: + self.assertEqual(self.bumped(target), tip(self.localization, "main")) + opened = [c[2]["json"]["title"] for c in api.calls if c[0] == "POST"] + self.assertEqual(opened, [sync.TITLE] + [sync.SUBMODULE_TITLE] * len(sync.CLIENTS)) + + def test_a_bump_alone_moves_to_localizations_tip(self): + outcome, _ = self.run_sync("--only", "playwright") + self.assertEqual(outcome.failures(), {}) + self.assertEqual(self.bumped("playwright"), tip(self.localization, "main")) + self.assertEqual(self.branches("session-desktop"), ["dev"]) + + def test_nothing_is_bumped_when_localization_fails(self): + shutil.rmtree(self.localization) + outcome, _ = self.run_sync("--only", "localization", "desktop") + self.assertEqual(set(outcome.failures()), {"localization", "desktop"}) + self.assertEqual(self.branches("session-desktop"), ["dev"]) def crowdin_language(lang_id, locale, name): From e3f05a32228890af5fd87bb994dbc4d306b1a381 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 5 Oct 2026 12:44:54 +1100 Subject: [PATCH 4/4] feat: refuse any write the App is not meant to make PUBLISHABLE in shared/github.py lists each repo's branches the jobs write, as a bot branch (force-pushed) or a direct one (fast-forward only). Repo.push checks the clone's own origin against it, ensure_pull and retire_branch check their branch, a token is minted only for listed repos, and publish_session refuses every API call publishing does not make, merging and reviewing included. --- docs/jobs/crowdin-sync.md | 5 ++ src/session_ops/crowdin/sync.py | 2 +- src/session_ops/platforms/publish.py | 7 +- src/session_ops/platforms/snode_list.py | 2 +- src/session_ops/shared/git.py | 8 +++ src/session_ops/shared/github.py | 85 +++++++++++++++++++++++++ tests/test_publish.py | 73 ++++++++++++++++++++- 7 files changed, 176 insertions(+), 6 deletions(-) diff --git a/docs/jobs/crowdin-sync.md b/docs/jobs/crowdin-sync.md index 5585924..625a191 100644 --- a/docs/jobs/crowdin-sync.md +++ b/docs/jobs/crowdin-sync.md @@ -51,6 +51,11 @@ key in `/etc/session-ops/github-app.pem`, and a run missing either exits naming dry run needs neither); see [publish.env.example](../../deploy/env/publish.env.example). Commits are authored as `PUBLISH_GIT_AUTHOR`. +Whatever the App's installation allows, session-ops writes only the branches listed in +`PUBLISHABLE` in `shared/github.py`: a push, a pull request or a branch deletion +anywhere else is refused before it reaches GitHub, as is any API call publishing does +not make (merging, reviewing, settings). A job writing a new branch adds it there. + ### Crowdin token scopes Crowdin scopes personal access tokens per endpoint family, and each scope can be diff --git a/src/session_ops/crowdin/sync.py b/src/session_ops/crowdin/sync.py index 5e437ea..e43416d 100644 --- a/src/session_ops/crowdin/sync.py +++ b/src/session_ops/crowdin/sync.py @@ -185,7 +185,7 @@ def main(argv=None): token = None if args.dry_run else github.publish_token( publish.ORG, [REPOS[t] for t in args.only]) - api = github.session(token) if token else None + api = github.publish_session(token) if token else None published, localization_ran = {}, "localization" in args.only def run_target(target): diff --git a/src/session_ops/platforms/publish.py b/src/session_ops/platforms/publish.py index 362903d..b1baa68 100644 --- a/src/session_ops/platforms/publish.py +++ b/src/session_ops/platforms/publish.py @@ -10,8 +10,8 @@ from session_ops.shared.env import rehearsing GITHUB = "https://github.com" -ORG = "session-foundation" -REHEARSAL_PREFIX = "rehearsal/" +ORG = github.ORG +REHEARSAL_PREFIX = github.REHEARSAL_PREFIX REHEARSAL_NOTE = ("**Rehearsal of session-ops: do not merge.** Close it and delete the branch " "once reviewed; production publishes to the branch without the " f"`{REHEARSAL_PREFIX}` prefix.\n\n") @@ -21,6 +21,8 @@ def pull_request(repo, api, name, base, branch, title, body, author, dry_run): """Publish `repo`'s uncommitted changes to `branch`. Returns a one-line result.""" if rehearsing(): branch, title, body = REHEARSAL_PREFIX + branch, f"[Rehearsal] {title}", REHEARSAL_NOTE + body + # Before anything else, so a dry run catches a branch the App may not write. + github.require_publishable(name, branch, force=True) if not repo.changed(): if not dry_run: github.retire_branch(api, name, branch) @@ -42,6 +44,7 @@ def direct_push(repo, api, name, branch, message, body, author, dry_run): if rehearsing(): return pull_request(repo, api, name, branch, f"direct-push-to-{branch}", message, body, author, dry_run) + github.require_publishable(name, branch) if not repo.changed(): return f"{name}: no changes on {branch}" repo.commit(message, author) diff --git a/src/session_ops/platforms/snode_list.py b/src/session_ops/platforms/snode_list.py index c1318b5..9720462 100644 --- a/src/session_ops/platforms/snode_list.py +++ b/src/session_ops/platforms/snode_list.py @@ -135,7 +135,7 @@ def main(argv=None): content = fetch(http.Session()) token = None if args.dry_run else github.publish_token(publish.ORG, [ASSETS, DESKTOP.repo, IOS]) - api = github.session(token) if token else None + api = github.publish_session(token) if token else None results, published = [], {} def assets(_): diff --git a/src/session_ops/shared/git.py b/src/session_ops/shared/git.py index 5ad7f0c..852a686 100644 --- a/src/session_ops/shared/git.py +++ b/src/session_ops/shared/git.py @@ -8,6 +8,8 @@ import os import subprocess +from session_ops.shared import github + def reason(stderr): """git's own error line, rather than whatever a wrapper printed before it. @@ -97,5 +99,11 @@ def set_gitlink(self, path, sha): # update-index drops skip-worktree, leaving the never-checked-out path a deletion. self.git("update-index", "--skip-worktree", "--", path) + def origin(self): + """The remote's "owner/name", from its URL.""" + url = self.git("remote", "get-url", "origin").stdout.strip() + return "/".join(url.removesuffix(".git").rstrip("/").split("/")[-2:]) + def push(self, branch, force=False): + github.require_publishable(self.origin(), branch, force) self.git("push", *(["--force"] if force else []), "origin", f"HEAD:refs/heads/{branch}") diff --git a/src/session_ops/shared/github.py b/src/session_ops/shared/github.py index 584036c..84c556a 100644 --- a/src/session_ops/shared/github.py +++ b/src/session_ops/shared/github.py @@ -10,6 +10,7 @@ CREDENTIALS_DIRECTORY set by systemd's LoadCredential=; holds github-app.pem """ import os +import re import time import jwt @@ -21,6 +22,85 @@ # Where the publishing units' LoadCredential= reads it from. APP_KEY_PATH = "/etc/session-ops/github-app.pem" +ORG = "session-foundation" +REHEARSAL_PREFIX = "rehearsal/" +BOT, DIRECT = "bot", "direct" +# Every branch the App may write, and how: a BOT branch is the job's own, rebuilt and +# force-pushed; a DIRECT one only ever fast-forwards. Anything else is refused before it +# reaches GitHub, whatever the App's installation would allow. A rehearsal writes the +# same branches under rehearsal/, a direct push as rehearsal/direct-push-to-. +PUBLISHABLE = { + "session-android": {"feature/update-crowdin-translations": BOT}, + "session-ios": {"feature/update-crowdin-translations": BOT, + "feature/update-static-snode-list": BOT}, + "session-localization": {"main": DIRECT}, + "session-desktop-dynamic-assets": {"main": DIRECT}, + "session-desktop": {"update-dynamic-assets": BOT, "update-localization": BOT}, + "session-app": {"update-localization": BOT}, + "session-website": {"update-localization": BOT}, + "session-appium": {"update-localization": BOT}, + "session-playwright": {"update-localization": BOT}, +} + + +def repo_name(repo): + """`repo`'s name in PUBLISHABLE, from "owner/name"; None outside the org.""" + owner, _, name = repo.rpartition("/") + return name if owner.rpartition("/")[2] == ORG else None + + +def require_publishable(repo, branch, force=False): + """Refuse to write `branch` of `repo` ("owner/name") unless PUBLISHABLE allows it.""" + allowed = PUBLISHABLE.get(repo_name(repo), {}) + kind = allowed.get(branch) + if kind is None and branch.startswith(REHEARSAL_PREFIX): + rehearsed = branch.removeprefix(REHEARSAL_PREFIX) + kind = BOT if (allowed.get(rehearsed) + or allowed.get(rehearsed.removeprefix("direct-push-to-")) == DIRECT) \ + else None + if kind is None or (force and kind == DIRECT): + how = "force-push" if force and kind else "write" + raise PermissionError(f"session-ops may not {how} {repo}:{branch}") + + +# What publishing may ask of the API, as (method, path under /repos//). +ENDPOINTS = (("GET", re.compile(r"/pulls")), ("POST", re.compile(r"/pulls")), + ("PATCH", re.compile(r"/pulls/\d+")), + ("DELETE", re.compile(r"/git/refs/heads/(?P.+)"))) +PULL_EDITS = {"title", "body", "state"} + + +def require_endpoint(method, url, payload): + """Refuse any call publishing does not make: merging, reviewing, settings.""" + match = re.fullmatch(re.escape(API) + r"/repos/([^/]+/[^/]+)(/.*)", url) + repo, path = match.groups() if match else (None, url) + if repo and repo_name(repo) in PUBLISHABLE: + for allowed_method, pattern in ENDPOINTS: + route = pattern.fullmatch(path) + if method != allowed_method or not route: + continue + if method == "POST": + require_publishable(repo, (payload or {}).get("head", "")) + elif method == "PATCH" and (set(payload or {}) - PULL_EDITS + or (payload or {}).get("state", "closed") != "closed"): + break + elif method == "DELETE": + require_publishable(repo, route["branch"]) + return + raise PermissionError(f"session-ops may not call {method} {url}") + + +def publish_session(token): + """session() for the App's installation token, refusing what require_endpoint does.""" + api = session(token) + send = api.request + + def request(method, url, **kwargs): + require_endpoint(method, url, kwargs.get("json")) + return send(method, url, **kwargs) + api.request = request + return api + def session(token): s = http.Session() @@ -71,6 +151,9 @@ def app_private_key(): def publish_token(owner, repositories): """An installation token for `repositories`; exits naming whatever is missing.""" + unlisted = [name for name in repositories if name not in PUBLISHABLE] + if owner != ORG or unlisted: + raise PermissionError(f"session-ops may not publish to {owner}/{unlisted or '*'}") app_id, key = os.environ.get("GITHUB_APP_ID"), app_private_key() missing = [] if not app_id: @@ -93,6 +176,7 @@ def open_pull(api, repo, head): def ensure_pull(api, repo, head, base, title, body): """Open a pull request from `head`, or retitle the one already open. Returns its URL.""" + require_publishable(repo, head) existing = open_pull(api, repo, head) if existing: check(api.request("PATCH", f"{API}/repos/{repo}/pulls/{existing['number']}", @@ -108,6 +192,7 @@ def ensure_pull(api, repo, head, base, title, body): def retire_branch(api, repo, branch): """Close the pull request from `branch` and delete it: nothing is left to merge.""" + require_publishable(repo, branch) existing = open_pull(api, repo, branch) if existing: check(api.request("PATCH", f"{API}/repos/{repo}/pulls/{existing['number']}", diff --git a/tests/test_publish.py b/tests/test_publish.py index a7b1c7c..c03f1f9 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -72,6 +72,10 @@ def request(self, method, url, params=None, json=None, **kwargs): raise AssertionError((method, url)) +TEST_REPOS = {"app": {"bot": github.BOT, "main": github.DIRECT}, + "module": {"main": github.DIRECT}, "client": {"bump": github.BOT}} + + class RepoTest(unittest.TestCase): def setUp(self): self.root = tempfile.mkdtemp() @@ -79,6 +83,9 @@ def setUp(self): patcher = mock.patch.object(publish, "GITHUB", f"file://{self.root}") patcher.start() self.addCleanup(patcher.stop) + allowed = mock.patch.dict(github.PUBLISHABLE, TEST_REPOS) + allowed.start() + self.addCleanup(allowed.stop) def clone(self, name, branch, patterns): return Repo.sparse_clone(f"file://{self.root}/session-foundation/{name}", branch, @@ -113,7 +120,8 @@ def test_a_relative_path_lands_where_it_names(self): def test_the_token_travels_in_the_environment_not_the_command_line(self): """Any account on the box can read another process's argv from `ps`.""" import base64 - with mock.patch("session_ops.shared.git.subprocess.run") as run: + with mock.patch("session_ops.shared.git.subprocess.run") as run, \ + mock.patch.object(github, "require_publishable"): run.return_value = subprocess.CompletedProcess([], 0, "", "") Repo("/tmp", token="ghs_secret").push("bot", force=True) argv, env = run.call_args.args[0], run.call_args.kwargs["env"] @@ -230,7 +238,7 @@ def test_a_change_lands_on_the_branch_without_force(self): with open(os.path.join(repo.path, "generated/english.ts"), "w") as handle: handle.write("b") with contextlib.redirect_stdout(io.StringIO()): - self.assertIn("pushed", publish.direct_push(repo, None, "m", "main", "T", "B", + self.assertIn("pushed", publish.direct_push(repo, None, "session-foundation/module", "main", "T", "B", AUTHOR, False)) self.assertEqual(git("show", "main:generated/english.ts", cwd=os.path.join(self.root, "session-foundation", "module")), "b") @@ -420,6 +428,67 @@ def seed_answer(nodes, height=2210944): NODES = [node(index) for index in range(snode_list.MIN_NODES, 0, -1)] +class TestPublishable(RepoTest): + """The App writes only where PUBLISHABLE says, however its caller asks.""" + + def test_only_listed_branches_are_written(self): + github.require_publishable("session-foundation/session-desktop", "update-localization") + for repo, branch in (("session-foundation/session-desktop", "dev"), + ("session-foundation/session-ios", "update-localization"), + ("someone/session-desktop", "update-localization"), + ("session-foundation/unlisted", "main")): + with self.subTest(repo=repo, branch=branch), self.assertRaises(PermissionError): + github.require_publishable(repo, branch) + + def test_a_direct_branch_is_never_forced_and_never_a_bot_branch(self): + github.require_publishable("session-foundation/session-localization", "main") + with self.assertRaises(PermissionError): + github.require_publishable("session-foundation/session-localization", "main", + force=True) + + def test_a_rehearsal_writes_only_its_own_prefixed_branches(self): + for branch in ("rehearsal/update-localization", "rehearsal/direct-push-to-main"): + github.require_publishable("session-foundation/session-desktop-dynamic-assets" + if "main" in branch else + "session-foundation/session-app", branch, force=True) + with self.assertRaises(PermissionError): + github.require_publishable("session-foundation/session-app", "rehearsal/other") + + def test_a_push_is_checked_against_the_clones_own_origin(self): + bare_repo(self.root, "app", "dev", {"a.txt": "a"}) + repo = self.clone("app", "dev", ["/a.txt"]) + with self.assertRaisesRegex(PermissionError, "session-foundation/app:dev"): + repo.push("dev", force=True) + with self.assertRaisesRegex(PermissionError, "force-push"): + repo.push("main", force=True) + + def test_the_api_refuses_every_call_publishing_does_not_make(self): + api = FakeSession([FakeResponse({})] * 3) + with mock.patch.object(github, "session", lambda token: api): + guarded = github.publish_session("t") + pulls = f"{github.API}/repos/session-foundation/session-app/pulls" + guarded.request("GET", pulls, params={"head": "x"}) + guarded.request("POST", pulls, json={"head": "update-localization", "base": "main"}) + guarded.request("PATCH", f"{pulls}/5", json={"state": "closed"}) + for method, url, payload in ( + ("PUT", f"{pulls}/5/merge", None), + ("POST", f"{pulls}/5/reviews", {"event": "APPROVE"}), + ("POST", pulls, {"head": "dev", "base": "main"}), + ("PATCH", f"{pulls}/5", {"base": "release"}), + ("PATCH", f"{pulls}/5", {"state": "open"}), + ("DELETE", f"{github.API}/repos/session-foundation/session-app/git/refs/heads/main", + None), + ("GET", f"{github.API}/repos/session-foundation/unlisted/pulls", None), + ("PATCH", f"{github.API}/repos/session-foundation/session-app", {"private": True})): + with self.subTest(method=method, url=url), self.assertRaises(PermissionError): + guarded.request(method, url, json=payload) + self.assertEqual(len(api.calls), 3) + + def test_a_token_is_never_minted_for_an_unlisted_repo(self): + with self.assertRaises(PermissionError): + github.publish_token("session-foundation", ["session-ios", "session-pro-backend"]) + + class TestSnodeListFetch(unittest.TestCase): def test_the_list_is_sorted_by_key_with_the_clients_fields_only(self): nodes = [dict(n, extra=1) for n in NODES] + [node(99, public_ip="0.0.0.0")]