feat: per-platform usage stats: Desktop downloads and Android MAU

README.md

@@ -14,7 +14,7 @@ share for translations. One package, `session_ops`, deployed to one self-hosted
 | `crowdin-duplicates` | Crowdin string slots holding more than one translation, as they open and close | [crowdin-duplicates](docs/jobs/crowdin-duplicates.md) |
 | `crowdin-sync` | Weekday: Crowdin translations into iOS, Android and the localization module, and its submodule bumped in each client | [crowdin-sync](docs/jobs/crowdin-sync.md) |
 | `snode-list` | Weekday: the fallback service node list from the seed nodes into dynamic-assets, Desktop and iOS | [snode-list](docs/jobs/snode-list.md) |
-| `release-stats` | On demand: download counts of the latest releases | [release-stats](docs/jobs/release-stats.md) |
+| `mau` | Monthly: active users per platform and in total, Android's from the Play Console export dropped in its inbox | [mau](docs/jobs/mau.md) |
 | `session-ops-silence` | Discord alerts for a job that failed, or stopped running | [session-ops-silence](docs/jobs/session-ops-silence.md) |
 | `token-expiry` | Discord alerts 14 days, 7 days and 24 hours before a token expires | [token-expiry](docs/jobs/token-expiry.md) |
 

deploy/README.md

@@ -7,6 +7,7 @@ does the work: accounts, venv, env files, units, timers, and migrating an older
 | Unit | What it is |
 | --- | --- |
 | `session-ops@<job>.timer` → `.service` | One per job; a generated drop-in sets its account, env files and schedule. |
+| `session-ops@<job>.path` → `.service` | For a job with a `watch`: starts it when a matching file lands in its state directory. |
 | `session-ops-queue.timer` → `.service` | Starts the jobs in `jobs.toml`'s `[queue]`, which then run one at a time in its order. |
 | `zendesk-relay.service` | Always on, `127.0.0.1:8080`: Zendesk's `claude:` note webhooks. |
 | `session-ops-alert@.service` | Every unit's `OnFailure=` backstop; see [session-ops-silence](../docs/jobs/session-ops-silence.md). |
@@ -85,6 +86,9 @@ To end it, stop the timers, then close the rehearsal pull requests:
 for link in /etc/systemd/system/timers.target.wants/session-ops@*.timer; do
   [ -L "$link" ] && systemctl disable --now "${link##*/}"
 done
+for link in /etc/systemd/system/paths.target.wants/session-ops@*.path; do
+  [ -L "$link" ] && systemctl disable --now "${link##*/}"
+done
 systemctl disable --now session-ops-queue.timer zendesk-relay.service
 for repo in session-android session-ios session-localization session-desktop-dynamic-assets \
     session-desktop session-app session-website session-appium session-playwright; do

deploy/env/alerts.env.example

@@ -1,3 +1,3 @@
-# /etc/session-ops/alerts.env: the OnFailure backstop, the silence checker and
-# release-stats. Empty disables the first two, and install.sh warns until it is set.
+# /etc/session-ops/alerts.env: the OnFailure backstop and the silence checker.
+# Empty disables both, and install.sh warns until it is set.
 ALERT_DISCORD_WEBHOOK_URL=

deploy/env/mau.env.example

@@ -0,0 +1,5 @@
+# /etc/session-ops/mau.env: the monthly active users post.
+# Where the figures, the reminders and the job's own failures are posted.
+MAU_DISCORD_WEBHOOK_URL=
+# The user@host the reminder's rsync command copies to; defaults to the host's own name.
+MAU_INBOX_HOST=

deploy/install.sh

@@ -68,7 +68,7 @@ move /etc/github-prs/env "$ETC/github-prs.env" 600 root
 move "$ETC/env" "$ETC/alerts.env" 600 root
 # systemd reads EnvironmentFile= as root before dropping privileges, so these need
 # no group: a job's account cannot read another job's secrets.
-for name in zendesk github-prs crowdin publish alerts; do
+for name in zendesk github-prs crowdin publish alerts mau; do
     [ -e "$ETC/$name.env" ] || install -m 600 /dev/null "$ETC/$name.env"
 done
 # What each file takes, commented; the env files stay empty until filled, since a job
@@ -92,6 +92,12 @@ if [ -e "$NEW_HOUSE" ] && grep -qx "ZENDESK_HOUSE_ANSWERS=$OLD_HOUSE" "$ETC/zend
     echo "pointed ZENDESK_HOUSE_ANSWERS in $ETC/zendesk.env at $NEW_HOUSE"
 fi
 
+# A watched job's inbox: root drops files in, and the job's account moves them out.
+"$OPS" list --watched | while read -r job user dir; do
+    install -d -o "$user" -g "$user" -m 711 "$STATE/$job"
+    install -d -o "$user" -g "$user" -m 700 "$dir"
+done
+
 install -m 644 "$ROOT/deploy/session-ops.tmpfiles" /etc/tmpfiles.d/session-ops.conf
 systemd-tmpfiles --create session-ops.conf
 
@@ -104,14 +110,15 @@ rm -f "$UNITS/zendesk-alert@.service" "$UNITS/github-prs-alert@.service"
 systemctl disable --now crowdin-relay.service 2>/dev/null || true
 rm -f "$UNITS/crowdin-relay.service"
 
-install -m 644 "$ROOT"/deploy/*.service "$ROOT"/deploy/*.timer "$UNITS/"
+install -m 644 "$ROOT"/deploy/*.service "$ROOT"/deploy/*.timer "$ROOT"/deploy/*.path "$UNITS/"
 # Only the generated files go, so a drop-in added by hand survives.
 rm -f "$UNITS"/session-ops@*.service.d/job.conf "$UNITS"/session-ops@*.timer.d/schedule.conf \
-    "$UNITS"/session-ops-queue.timer.d/schedule.conf
+    "$UNITS"/session-ops@*.path.d/watch.conf "$UNITS"/session-ops-queue.timer.d/schedule.conf
 "$OPS" units --out "$UNITS" >/dev/null
 
 READY=$("$OPS" list --ready)
 QUEUED=$("$OPS" list --queued)
+WATCHED=$("$OPS" list --watched | cut -d' ' -f1)
 listed() { printf '%s\n' $2 | grep -qxF "$1"; }
 # What the queue's timer starts: its ready jobs, rebuilt from scratch each install.
 WANTS="$UNITS/session-ops-queue.service.wants"
@@ -142,6 +149,21 @@ for job in $READY; do
         echo "enabled session-ops@$job.timer"
     fi
 done
+for link in "$UNITS"/paths.target.wants/session-ops@*.path; do
+    [ -L "$link" ] || continue
+    job=${link##*/session-ops@}
+    job=${job%.path}
+    if ! listed "$job" "$READY" || ! listed "$job" "$WATCHED"; then
+        systemctl disable --now "session-ops@$job.path" >/dev/null
+        echo "disabled session-ops@$job.path (no longer a ready job with a watch)"
+    fi
+done
+for job in $WATCHED; do
+    if listed "$job" "$READY"; then
+        systemctl enable --now "session-ops@$job.path" >/dev/null
+        echo "enabled session-ops@$job.path"
+    fi
+done
 if [ -d "$WANTS" ]; then
     systemctl enable --now session-ops-queue.timer >/dev/null
     echo "enabled session-ops-queue.timer"

deploy/session-ops@.path

@@ -0,0 +1,12 @@
+# Starts a job when a file matching its watch lands. Its drop-in, written by
+# `session-ops units`, sets PathExistsGlob= from jobs.toml's watch.
+[Unit]
+Description=Inbox watch for session-ops job %i
+Documentation=https://github.com/session-foundation/session-shared-scripts
+
+[Path]
+# Re-triggers for as long as a file matches, so the job must move each one out.
+Unit=session-ops@%i.service
+
+[Install]
+WantedBy=paths.target

docs/jobs/mau.md

@@ -0,0 +1,66 @@
+# Monthly Active Users
+
+Posts last month's monthly active users once a month, per platform and in total.
+
+- Android: Play's MAU on the month's last day, with the change on the month before.
+  Google has no API for it, so the figures come from the Play Console export someone
+  drops in the job's inbox.
+- Desktop, which has no active-user count: the latest stable release's downloads per
+  platform since its release, read from GitHub and Flathub when the post goes out.
+- The total adds the two.
+
+| | |
+| --- | --- |
+| Runs | `session-ops@mau.timer` on the 10th at 11:00 Melbourne, and `session-ops@mau.path` whenever a `.csv` lands in the inbox |
+| Secrets | `/etc/session-ops/mau.env`: `MAU_DISCORD_WEBHOOK_URL`, and `MAU_INBOX_HOST` for the reminder's `rsync` |
+| Inbox | `/var/lib/session-ops/mau/inbox/` |
+| Dry run | `session-ops run mau --dry-run` prints what it would post, and moves and writes nothing |
+| Logs | `journalctl -u session-ops@mau -n 50 --no-pager` |
+
+## The export
+
+In Play Console, Statistics, a report saved once:
+
+- Metric: Monthly active users (MAU), Unique users, Per interval, Daily
+- All countries / regions, no breakdown
+- A date range ending today, such as Last 30 days, with the Console in English
+
+Export report → CSV, then:
+
+```sh
+rsync "All countries _ regions.csv" root@<host>:/var/lib/session-ops/mau/inbox/
+```
+
+`rsync`, not `scp`: it writes to a hidden temporary name and renames it once complete,
+and the job only reads `*.csv`.
+
+## What a run does
+
+1. Merges every export in `inbox/` into `history.json`, one figure per day, then moves
+   it to `done/`. An export may cover any range; where two give a day different
+   figures, the later one wins and the post lists the revision.
+2. Moves a file it cannot read to `rejected/` and fails the run naming it, after the
+   rest of the run.
+3. Posts last month once `history.json` holds its last day. Play's figures trail by
+   about eight days, so that is around the 9th. Until then, from the 10th, each run
+   posts a reminder instead.
+4. Records the month as posted, so neither the timer nor a later export posts it again.
+
+`history.json` cannot be rebuilt by a re-run: an unreadable one stops the job. If it is
+lost, drop an export covering the last 365 days.
+
+## Desktop downloads
+
+The latest Desktop release that is neither a draft nor a pre-release. Updates count
+where the updater fetches an installer.
+
+| Platform | Counted |
+| --- | --- |
+| Linux | `.deb`, `.AppImage`, `.rpm`, `.freebsd`, and Flathub's installs of `network.loki.Session` since the release day |
+| macOS | `.dmg`, `.zip` |
+| Windows | `.exe` |
+
+`.blockmap`, `latest*.yml` and `signature.asc` are left out. Flathub builds from the
+GitHub `.deb` once, so its installs are not in GitHub's counts; Homebrew's `session`
+cask downloads the GitHub `.dmg`, so it already is. Flathub keeps 180 days of daily
+installs, so the run fails on a release older than that rather than undercount.

src/session_ops/jobs.toml

@@ -10,6 +10,8 @@
 #   channel_env    where it posts, and where its own failures are reported; a job
 #                  without one reports to ALERT_DISCORD_WEBHOOK_URL
 #   unit           further [Service] lines, over the template's hardening
+#   watch          a glob under {state}: a file matching it starts the job, which must
+#                  move it out, or the path unit starts it again
 #
 #   max_age_hours  for a queued job, its queue's: the weekend's 72 h (73 h across
 #                  April's DST change), plus every job ahead of it running to its timeout
@@ -127,10 +129,17 @@ timeout = "10min"
 unit = ["LoadCredential=github-app.pem:/etc/session-ops/github-app.pem"]
 
 [[job]]
-name = "release-stats"
-description = "Download counts of the latest Desktop and Android releases"
-entry = "session_ops.platforms.release_stats:main"
-args = ["--out", "{state}/runs"]
+name = "mau"
+description = "Monthly active users per platform, posted once a month"
+entry = "session_ops.platforms.mau:main"
+args = ["--state", "{state}"]
 user = "sessionops"
-env_files = ["/etc/session-ops/alerts.env"]
+env_files = ["/etc/session-ops/mau.env"]
+env = ["MAU_DISCORD_WEBHOOK_URL"]
+channel_env = "MAU_DISCORD_WEBHOOK_URL"
+# An hour clear of the queue's posts at 10:00.
+schedule = "*-*-10 11:00 Australia/Melbourne"
+watch = "inbox/*.csv"
+# The longest month, plus the hour a DST change adds and slack for the run.
+max_age_hours = 750
 timeout = "5min"

src/session_ops/ops/registry.py

@@ -30,6 +30,7 @@ class Job:
     unit: tuple = field(default=())
     queued: bool = False
     after: tuple = ()
+    watch: str = None
 
     @property
     def scheduled(self):
@@ -43,6 +44,10 @@ def timer(self):
     def state_dir(self):
         return os.path.join(STATE_ROOT, self.name)
 
+    @property
+    def watch_glob(self):
+        return os.path.join(self.state_dir, self.watch) if self.watch else None
+
     def argv(self, dry_run=False, state_dir=None):
         """The job's arguments, with {state} standing for its state directory."""
         state = state_dir or self.state_dir
@@ -84,6 +89,8 @@ def load(path=REGISTRY):
                            for k, v in row.items()}))
     jobs = _apply_queue(path, jobs, data.get("queue"))
     for job in jobs:
+        if job.watch and (os.path.isabs(job.watch) or ".." in job.watch.split("/")):
+            raise ValueError(f"{path}: {job.name}'s watch must stay inside its state directory")
         if job.scheduled and not isinstance(job.max_age_hours, (int, float)):
             raise ValueError(f"{path}: scheduled job {job.name} needs a numeric max_age_hours")
     return jobs

src/session_ops/ops/runner.py

@@ -235,6 +235,8 @@ def main(argv=None):
                            help="Only the names of scheduled jobs with an empty env file.")
     readiness.add_argument("--queued", action="store_true",
                            help="Only the names of the queued jobs, in the order they run.")
+    readiness.add_argument("--watched", action="store_true",
+                           help="Name, account and watched directory of each job with a watch.")
     run_parser = sub.add_parser("run", help="Run a job as its timer does.")
     run_parser.add_argument("job")
     run_parser.add_argument("--dry-run", action="store_true",
@@ -250,6 +252,11 @@ def main(argv=None):
         if args.queued:
             print("\n".join(queue.jobs))
             return
+        if args.watched:
+            for job in registry.load():
+                if job.watch:
+                    print(job.name, job.user, os.path.dirname(job.watch_glob))
+            return
         for job in registry.load():
             if args.ready or args.not_ready:
                 if job.scheduled and ready(job) == args.ready:

src/session_ops/ops/units.py

@@ -25,6 +25,10 @@ def timer_dropin(schedule):
     return "\n".join([HEADER, "[Timer]", f"OnCalendar={schedule}"]) + "\n"
 
 
+def path_dropin(pattern):
+    return "\n".join([HEADER, "[Path]", f"PathExistsGlob={pattern}"]) + "\n"
+
+
 def dropins(jobs, queue):
     """{relative path: content} for every job, and the queue's schedule.
 
@@ -38,6 +42,8 @@ def dropins(jobs, queue):
         files[f"session-ops@{job.name}.service.d/job.conf"] = service_dropin(job)
         if job.schedule:
             files[f"session-ops@{job.name}.timer.d/schedule.conf"] = timer_dropin(job.schedule)
+        if job.watch:
+            files[f"session-ops@{job.name}.path.d/watch.conf"] = path_dropin(job.watch_glob)
     return files