From 08cdf125a1f4c75bac2ba876b44e3bdd50c6e072 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 12 Nov 2026 17:31:39 +0100 Subject: [PATCH 1/4] common: take Ubuntu's security pocket from security.ubuntu.com Cherry's image takes every pocket from repo.cherryservers.com, which resolves to a mirror inside each datacenter and is re-signed with Cherry's key. One of those mirrors stopped updating for weeks, and `apt-get update` kept hitting the old index without an error, so that box's unattended upgrades quietly found nothing to install. common now writes ubuntu.sources on the Ubuntu boxes: the release, -updates and -backports stay on the mirror as before, and -security comes from security.ubuntu.com, signed with Ubuntu's own keyring, as on a stock install. When the file changes, the package lists are refreshed straight away. Debian (the indexers) is unchanged. node_apt_security_lists_timestamp_seconds, written to the node-exporter textfile directory by an apt Post-Invoke hook on every box, is when the newest security index was published, so a stale mirror shows up as an age. A timestamp, not an age, so it keeps aging when refreshes stop. Co-Authored-By: Claude Opus 5.5 --- README.md | 35 +++++++++++++ roles/common/defaults/main.yml | 9 ++++ roles/common/tasks/apt_lists_metric.yml | 66 ++++++++++++++++++++++++ roles/common/tasks/apt_sources.yml | 51 ++++++++++++++++++ roles/common/tasks/main.yml | 9 ++++ roles/common/templates/ubuntu.sources.j2 | 18 +++++++ 6 files changed, 188 insertions(+) create mode 100644 roles/common/tasks/apt_lists_metric.yml create mode 100644 roles/common/tasks/apt_sources.yml create mode 100644 roles/common/templates/ubuntu.sources.j2 diff --git a/README.md b/README.md index 1c978a0..e3d54c6 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,41 @@ an origin pattern that matches nothing; for that, look at `/var/log/unattended-upgrades/unattended-upgrades.log` and `apt list --upgradable` on the box. +**Where the IPFS boxes' packages come from.** `common` writes +`/etc/apt/sources.list.d/ubuntu.sources` (`templates/ubuntu.sources.j2`): the +release, `-updates` and `-backports` from Cherry's mirror, signed with +Cherry's key, as the image has them, and `-security` from +`security.ubuntu.com`, signed with Ubuntu's own key (`ubuntu-keyring`), as on a +stock Ubuntu install. `repo.cherryservers.com` resolves to a mirror inside each +datacenter, and one of them stopped updating for weeks. `apt-get update` kept +"hitting" the old index without an error, so unattended-upgrades on that box +quietly found nothing to install while the other boxes kept getting fixes. +Taking the security pocket from upstream means a fix never waits on a mirror. +It costs a little outbound traffic, and an upstream outage only delays one +night's run. If a security update needs a newer version of something from +`-updates` that the mirror does not have yet, apt holds that package back +until it does; nothing breaks. The URIs and keyrings are `common_apt_*` in +`roles/common/defaults/main.yml`. The indexers keep their Debian image's +sources. + +**How old the security lists are.** `node_apt_security_lists_timestamp_seconds` +is when the newest `-security` index on the box was published (the +`Date:` of its `InRelease`; 0 if there is none). An apt hook rewrites it after +every package-list refresh, into +`/var/lib/node_exporter/textfile/apt-security-lists.prom`, on every box. It is +a timestamp rather than an age so that it keeps getting older when the lists +stop refreshing; the age is `time() - node_apt_security_lists_timestamp_seconds`. +The security pocket normally gets a new index every few days at most, and the +IPFS boxes all read the same one, so an alert would fire on either: the lists +more than about three days old, or a day behind another IPFS box's. Like +`node_reboot_required`, it reaches Prometheus only once ipni/infra's +node-exporter reads the textfile directory, and the alert belongs with the +fleet rules there. Until then, on the box: + +```bash +ansible ipfs_nodes -u ipni -m shell -a 'cat /var/lib/node_exporter/textfile/apt-security-lists.prom' +``` + A pending reboot shows in three places: `/var/run/reboot-required`, the `reboot:` line in every `site.yml` summary, and the `node_reboot_required` metric, which an hourly timer writes to `/var/lib/node_exporter/textfile/reboot.prom`. The metric diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index e3cd75c..b8d49fb 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -54,3 +54,12 @@ common_reboot_reporting: false # node-exporter reads *.prom from here once it is given # --collector.textfile.directory (ipni/infra kubernetes/platform/node-exporter). common_textfile_dir: /var/lib/node_exporter/textfile + +# Ubuntu's package sources (tasks/apt_sources.yml): the release, -updates and +# -backports from the provider's mirror, signed with the provider's key; the +# security pocket from Ubuntu, signed with Ubuntu's. +common_apt_mirror_uri: http://repo.cherryservers.com/ubuntu/ +common_apt_mirror_keyring: /usr/share/keyrings/cherryservers-signing-public.gpg +common_apt_security_uri: http://security.ubuntu.com/ubuntu/ +common_apt_security_keyring: /usr/share/keyrings/ubuntu-archive-keyring.gpg +common_apt_components: [main, universe, restricted, multiverse] diff --git a/roles/common/tasks/apt_lists_metric.yml b/roles/common/tasks/apt_lists_metric.yml new file mode 100644 index 0000000..5c63d89 --- /dev/null +++ b/roles/common/tasks/apt_lists_metric.yml @@ -0,0 +1,66 @@ +--- +# node_apt_security_lists_timestamp_seconds: when the newest security index +# this box has was published (the `Date:` of its -security InRelease). +# A mirror that stops updating still answers `apt-get update` without an error, +# so this is how a box that has quietly stopped seeing security fixes shows up. +# +# A timestamp rather than an age: an age is frozen at whatever it was when the +# file was last written, so it would stop growing in exactly the case it is +# for, a box whose lists stop refreshing. Alert on +# `time() - node_apt_security_lists_timestamp_seconds`. 0 means no security +# index at all. +# +# Rewritten after every package-list refresh (an apt hook) and on every run of +# this role. Read by node-exporter's textfile collector, see +# common_textfile_dir. + +- name: Create the node-exporter textfile directory + ansible.builtin.file: + path: "{{ common_textfile_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Install the security-lists age reporter + ansible.builtin.copy: + dest: /usr/local/sbin/ipni-apt-lists-metric + mode: "0755" + owner: root + group: root + content: | + #!/bin/sh + # Managed by Ansible. Writes node_apt_security_lists_timestamp_seconds + # for node-exporter: the Date: of the newest security index apt has. + set -eu + newest=0 + for f in /var/lib/apt/lists/*_dists_{{ ansible_distribution_release }}-security_InRelease; do + [ -e "$f" ] || continue + t=$(stat -c %Y "$f") + d=$(sed -n 's/^Date: //p' "$f" | head -n 1) + if [ -n "$d" ] && dt=$(date -d "$d" +%s 2>/dev/null); then t=$dt; fi + if [ "$t" -gt "$newest" ]; then newest=$t; fi + done + tmp="{{ common_textfile_dir }}/.apt-security-lists.prom.$$" + printf '# HELP node_apt_security_lists_timestamp_seconds When the newest security package index on this box was published (0: none).\n# TYPE node_apt_security_lists_timestamp_seconds gauge\nnode_apt_security_lists_timestamp_seconds %s\n' "$newest" > "$tmp" + chmod 0644 "$tmp" + mv "$tmp" "{{ common_textfile_dir }}/apt-security-lists.prom" + +# Post-Invoke, not Post-Invoke-Success: a refresh that partly fails still +# leaves lists behind worth reporting. Never fails the refresh itself. +- name: Report the security lists' age after every package-list refresh + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/80ipni-apt-lists-metric + mode: "0644" + owner: root + group: root + content: | + // Managed by Ansible. + APT::Update::Post-Invoke { "/usr/local/sbin/ipni-apt-lists-metric || true"; }; + +# So the file exists from the first run, not only after the next refresh. +# Skipped in a dry run, where the reporter may not be installed yet. +- name: Report the security lists' age now + ansible.builtin.command: /usr/local/sbin/ipni-apt-lists-metric + changed_when: false + when: not ansible_check_mode diff --git a/roles/common/tasks/apt_sources.yml b/roles/common/tasks/apt_sources.yml new file mode 100644 index 0000000..54e1621 --- /dev/null +++ b/roles/common/tasks/apt_sources.yml @@ -0,0 +1,51 @@ +--- +# Ubuntu only (main.yml): where the package lists come from. The indexers keep +# their Debian image's own sources. +# +# Cherry's image takes every pocket, -security included, from Cherry's mirror, +# re-signed with Cherry's key. repo.cherryservers.com resolves to a mirror +# inside each datacenter, and one of them stopped updating for weeks: `apt-get +# update` kept "hitting" the old index without an error, so that box quietly +# stopped getting security fixes while the others did not. This takes the +# security pocket from security.ubuntu.com, signed by Ubuntu's own key, as a +# stock Ubuntu install does, and keeps the rest on the mirror. +# +# The whole file is ours, so if a reinstall or cloud-init rewrites it, the next +# run puts it back. Runs before main.yml's "Update apt cache", so everything +# after it installs from the new lists. + +- name: Find the keyrings the package sources are signed with + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ common_apt_mirror_keyring }}" + - "{{ common_apt_security_keyring }}" + register: common_apt_keyrings + +# A source whose keyring is missing makes every `apt-get update` fail for it. +- name: Fail if a keyring is missing + ansible.builtin.assert: + that: common_apt_keyrings.results | rejectattr('stat.exists') | list | length == 0 + fail_msg: >- + {{ inventory_hostname }}: missing + {{ common_apt_keyrings.results | rejectattr('stat.exists') | map(attribute='item') | join(', ') }}; + the security keyring comes from the ubuntu-keyring package. + quiet: true + +- name: Take the security pocket from Ubuntu and the rest from the mirror + ansible.builtin.template: + src: ubuntu.sources.j2 + dest: /etc/apt/sources.list.d/ubuntu.sources + owner: root + group: root + mode: "0644" + register: common_apt_sources + +# "Update apt cache" in main.yml skips a cache less than an hour old, which +# would leave the rest of the play on the old lists. Not reported as a change, +# like that task: the change is the file above, which a dry run shows. +- name: Refresh the package lists from the new sources + ansible.builtin.apt: + update_cache: true + changed_when: false + when: common_apt_sources is changed diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 912b083..714ee91 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -23,6 +23,15 @@ when: current_tz.stdout | trim != 'UTC' changed_when: true +# Both before the first refresh below: the hook reporting the lists' age is in +# place for it, and it reads the sources the box should have. +- name: Report how old the security package lists are + ansible.builtin.import_tasks: apt_lists_metric.yml + +- name: Take Ubuntu's security pocket from Ubuntu itself + ansible.builtin.import_tasks: apt_sources.yml + when: ansible_distribution == 'Ubuntu' + # Refreshing the package lists changes nothing installed. Reported as a change, # it would be one on every run, and in every dry run an approver reads. - name: Update apt cache diff --git a/roles/common/templates/ubuntu.sources.j2 b/roles/common/templates/ubuntu.sources.j2 new file mode 100644 index 0000000..b094e31 --- /dev/null +++ b/roles/common/templates/ubuntu.sources.j2 @@ -0,0 +1,18 @@ +# Managed by Ansible (roles/common). Changes made here are overwritten. +# +# The release, -updates and -backports come from the provider's mirror. The +# security pocket comes from Ubuntu itself, as on a stock Ubuntu install, so a +# mirror that stops updating cannot hold security fixes back. See "Package +# updates and reboots" in the README. +{% set release = ansible_distribution_release %} +Types: deb +URIs: {{ common_apt_mirror_uri }} +Suites: {{ release }} {{ release }}-updates {{ release }}-backports +Components: {{ common_apt_components | join(' ') }} +Signed-By: {{ common_apt_mirror_keyring }} + +Types: deb +URIs: {{ common_apt_security_uri }} +Suites: {{ release }}-security +Components: {{ common_apt_components | join(' ') }} +Signed-By: {{ common_apt_security_keyring }} From 3fc54a6337c5dca4ea773844e62fbe0e1f825906 Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 12 Nov 2026 17:56:06 +0100 Subject: [PATCH 2/4] common: address review on the security-pocket change - Refresh the lists when the upstream security index is missing too, not only when ubuntu.sources changed, so a run that wrote the file and then failed to refresh fixes itself on the next run. Retry that refresh and "Update apt cache" three times. - Add node_apt_lists_refresh_timestamp_seconds, from a stamp our own apt hook touches after a successful refresh (Debian has no update-success-stamp), so a failing refresh can be told from an archive that published nothing new. Both metrics now go to apt-lists.prom. - README: unattended-upgrades never installs from -updates, so a security update that needs a version only there stays pending; an upstream outage stops a site.yml run on that box; the alert should compare boxes first, with a fixed age only once the Date: cadence is known. Co-Authored-By: Claude Opus 5.5 --- README.md | 53 +++++++++++++---------- roles/common/defaults/main.yml | 4 ++ roles/common/tasks/apt_lists_metric.yml | 56 ++++++++++++++++--------- roles/common/tasks/apt_sources.yml | 19 ++++++++- roles/common/tasks/main.yml | 4 ++ 5 files changed, 94 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index e3d54c6..6d12e15 100644 --- a/README.md +++ b/README.md @@ -377,29 +377,40 @@ datacenter, and one of them stopped updating for weeks. `apt-get update` kept "hitting" the old index without an error, so unattended-upgrades on that box quietly found nothing to install while the other boxes kept getting fixes. Taking the security pocket from upstream means a fix never waits on a mirror. -It costs a little outbound traffic, and an upstream outage only delays one -night's run. If a security update needs a newer version of something from -`-updates` that the mirror does not have yet, apt holds that package back -until it does; nothing breaks. The URIs and keyrings are `common_apt_*` in -`roles/common/defaults/main.yml`. The indexers keep their Debian image's -sources. - -**How old the security lists are.** `node_apt_security_lists_timestamp_seconds` -is when the newest `-security` index on the box was published (the -`Date:` of its `InRelease`; 0 if there is none). An apt hook rewrites it after -every package-list refresh, into -`/var/lib/node_exporter/textfile/apt-security-lists.prom`, on every box. It is -a timestamp rather than an age so that it keeps getting older when the lists -stop refreshing; the age is `time() - node_apt_security_lists_timestamp_seconds`. -The security pocket normally gets a new index every few days at most, and the -IPFS boxes all read the same one, so an alert would fire on either: the lists -more than about three days old, or a day behind another IPFS box's. Like -`node_reboot_required`, it reaches Prometheus only once ipni/infra's -node-exporter reads the textfile directory, and the alert belongs with the -fleet rules there. Until then, on the box: +It costs a little outbound traffic. If `security.ubuntu.com` is unreachable, +unattended-upgrades finds nothing new that night and catches up on the next +run, but a `site.yml` run stops on the box where the refresh fails (after +three tries) rather than carrying on without a security index. + +A security update that needs a version only `-updates` has stays pending: +unattended-upgrades does not install from `-updates` (see the table above), +whether or not the mirror has it. Upgrade that one by hand. Ubuntu builds +security updates against the release and security pockets, so this is rare. + +The URIs and keyrings are `common_apt_*` in `roles/common/defaults/main.yml`. +The indexers keep their Debian image's sources. + +**How old the package lists are.** An apt hook writes two timestamps to +`/var/lib/node_exporter/textfile/apt-lists.prom` after every package-list +refresh, on every box: + +| Metric | What it is | Old means | +|---|---|---| +| `node_apt_security_lists_timestamp_seconds` | When the newest `-security` index on the box was published (the `Date:` of its `InRelease`; 0: none). | The box's security source is stale or unreachable, or the archive has simply published nothing new. | +| `node_apt_lists_refresh_timestamp_seconds` | The last refresh apt reported as a success (`common_apt_refresh_stamp`; 0: none yet). | The refresh is not running or is failing. | + +They are timestamps rather than ages, so they keep getting older when the +lists stop refreshing; the age is `time() - `. A quiet archive also +makes the first one grow, and how quiet `noble-security` can get has not been +measured yet. The IPFS boxes all read the same upstream index, though, so the +dependable signal is one IPFS box's security index more than a day behind +another's. Add a fixed age (a week, say) only once the `Date:` has been watched +for a while. Like `node_reboot_required`, these reach Prometheus only once +ipni/infra's node-exporter reads the textfile directory, and the alerts belong +with the fleet rules there. Until then, on the box: ```bash -ansible ipfs_nodes -u ipni -m shell -a 'cat /var/lib/node_exporter/textfile/apt-security-lists.prom' +ansible ipfs_nodes -u ipni -m shell -a 'cat /var/lib/node_exporter/textfile/apt-lists.prom' ``` A pending reboot shows in three places: `/var/run/reboot-required`, the `reboot:` diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index b8d49fb..0cdf556 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -63,3 +63,7 @@ common_apt_mirror_keyring: /usr/share/keyrings/cherryservers-signing-public.gpg common_apt_security_uri: http://security.ubuntu.com/ubuntu/ common_apt_security_keyring: /usr/share/keyrings/ubuntu-archive-keyring.gpg common_apt_components: [main, universe, restricted, multiverse] + +# Touched by an apt hook after each successful package-list refresh, for +# node_apt_lists_refresh_timestamp_seconds (tasks/apt_lists_metric.yml). +common_apt_refresh_stamp: /var/lib/apt/periodic/ipni-update-success-stamp diff --git a/roles/common/tasks/apt_lists_metric.yml b/roles/common/tasks/apt_lists_metric.yml index 5c63d89..217c4eb 100644 --- a/roles/common/tasks/apt_lists_metric.yml +++ b/roles/common/tasks/apt_lists_metric.yml @@ -1,18 +1,23 @@ --- +# Two timestamps, for node-exporter's textfile collector (common_textfile_dir): +# # node_apt_security_lists_timestamp_seconds: when the newest security index -# this box has was published (the `Date:` of its -security InRelease). -# A mirror that stops updating still answers `apt-get update` without an error, -# so this is how a box that has quietly stopped seeing security fixes shows up. +# this box has was published (the `Date:` of its -security +# InRelease; 0: none). A mirror that stops updating still answers `apt-get +# update` without an error, so this is how a box that has quietly stopped +# seeing security fixes shows up. It also grows when the archive simply +# publishes nothing new, so compare it across boxes that read the same +# source rather than only against a fixed age. +# node_apt_lists_refresh_timestamp_seconds: the last package-list refresh apt +# reported as a success (0: none since this was installed). Old means the +# refresh itself is not running or failing, not that the archive is quiet. # -# A timestamp rather than an age: an age is frozen at whatever it was when the +# Timestamps rather than ages: an age is frozen at whatever it was when the # file was last written, so it would stop growing in exactly the case it is -# for, a box whose lists stop refreshing. Alert on -# `time() - node_apt_security_lists_timestamp_seconds`. 0 means no security -# index at all. +# for, a box whose lists stop refreshing. Alert on `time() - `. # # Rewritten after every package-list refresh (an apt hook) and on every run of -# this role. Read by node-exporter's textfile collector, see -# common_textfile_dir. +# this role. - name: Create the node-exporter textfile directory ansible.builtin.file: @@ -22,7 +27,7 @@ group: root mode: "0755" -- name: Install the security-lists age reporter +- name: Install the package-list age reporter ansible.builtin.copy: dest: /usr/local/sbin/ipni-apt-lists-metric mode: "0755" @@ -30,8 +35,8 @@ group: root content: | #!/bin/sh - # Managed by Ansible. Writes node_apt_security_lists_timestamp_seconds - # for node-exporter: the Date: of the newest security index apt has. + # Managed by Ansible. Writes node_apt_security_lists_timestamp_seconds and + # node_apt_lists_refresh_timestamp_seconds for node-exporter. set -eu newest=0 for f in /var/lib/apt/lists/*_dists_{{ ansible_distribution_release }}-security_InRelease; do @@ -41,14 +46,26 @@ if [ -n "$d" ] && dt=$(date -d "$d" +%s 2>/dev/null); then t=$dt; fi if [ "$t" -gt "$newest" ]; then newest=$t; fi done - tmp="{{ common_textfile_dir }}/.apt-security-lists.prom.$$" - printf '# HELP node_apt_security_lists_timestamp_seconds When the newest security package index on this box was published (0: none).\n# TYPE node_apt_security_lists_timestamp_seconds gauge\nnode_apt_security_lists_timestamp_seconds %s\n' "$newest" > "$tmp" + refreshed=0 + stamp={{ common_apt_refresh_stamp }} + [ -e "$stamp" ] && refreshed=$(stat -c %Y "$stamp") + tmp="{{ common_textfile_dir }}/.apt-lists.prom.$$" + { + printf '# HELP node_apt_security_lists_timestamp_seconds When the newest security package index on this box was published (0: none).\n' + printf '# TYPE node_apt_security_lists_timestamp_seconds gauge\n' + printf 'node_apt_security_lists_timestamp_seconds %s\n' "$newest" + printf '# HELP node_apt_lists_refresh_timestamp_seconds When apt last reported a package-list refresh as a success (0: never).\n' + printf '# TYPE node_apt_lists_refresh_timestamp_seconds gauge\n' + printf 'node_apt_lists_refresh_timestamp_seconds %s\n' "$refreshed" + } > "$tmp" chmod 0644 "$tmp" - mv "$tmp" "{{ common_textfile_dir }}/apt-security-lists.prom" + mv "$tmp" "{{ common_textfile_dir }}/apt-lists.prom" -# Post-Invoke, not Post-Invoke-Success: a refresh that partly fails still -# leaves lists behind worth reporting. Never fails the refresh itself. -- name: Report the security lists' age after every package-list refresh +# Our own stamp: Ubuntu's update-success-stamp comes from update-notifier, +# which Debian does not have. Post-Invoke-Success touches it and reports; the +# Post-Invoke entry reports after a refresh apt counts as failed too, when the +# lists left behind are still worth reporting. Neither fails the refresh. +- name: Report the package lists' age after every refresh ansible.builtin.copy: dest: /etc/apt/apt.conf.d/80ipni-apt-lists-metric mode: "0644" @@ -56,11 +73,12 @@ group: root content: | // Managed by Ansible. + APT::Update::Post-Invoke-Success { "touch {{ common_apt_refresh_stamp }}; /usr/local/sbin/ipni-apt-lists-metric || true"; }; APT::Update::Post-Invoke { "/usr/local/sbin/ipni-apt-lists-metric || true"; }; # So the file exists from the first run, not only after the next refresh. # Skipped in a dry run, where the reporter may not be installed yet. -- name: Report the security lists' age now +- name: Report the package lists' age now ansible.builtin.command: /usr/local/sbin/ipni-apt-lists-metric changed_when: false when: not ansible_check_mode diff --git a/roles/common/tasks/apt_sources.yml b/roles/common/tasks/apt_sources.yml index 54e1621..8b3898b 100644 --- a/roles/common/tasks/apt_sources.yml +++ b/roles/common/tasks/apt_sources.yml @@ -41,11 +41,26 @@ mode: "0644" register: common_apt_sources +# apt names the file after the URI: security.ubuntu.com_ubuntu_dists_... +- name: Look for the security index from the configured source + ansible.builtin.stat: + path: >- + /var/lib/apt/lists/{{ common_apt_security_uri | regex_replace('^[a-z]+://', '') | replace('/', '_') + }}dists_{{ ansible_distribution_release }}-security_InRelease + register: common_apt_security_index + # "Update apt cache" in main.yml skips a cache less than an hour old, which -# would leave the rest of the play on the old lists. Not reported as a change, +# would leave the rest of the play on the old lists. Keyed on the index as well +# as the file, so a run that wrote the file and then failed here refreshes on +# the next run too. A failure after the retries stops the run on this box: it +# would otherwise carry on without a security index. Not reported as a change, # like that task: the change is the file above, which a dry run shows. - name: Refresh the package lists from the new sources ansible.builtin.apt: update_cache: true + register: common_apt_refresh + retries: 3 + delay: 10 + until: common_apt_refresh is succeeded changed_when: false - when: common_apt_sources is changed + when: common_apt_sources is changed or not common_apt_security_index.stat.exists diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 714ee91..eeabd2d 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -38,6 +38,10 @@ ansible.builtin.apt: update_cache: true cache_valid_time: 3600 + register: apt_cache + retries: 3 + delay: 10 + until: apt_cache is succeeded changed_when: false # Gated: see common_apt_upgrade in defaults/main.yml for why this does not run From 29cb9d2c41d31a489fb5c821509a1535eafb89eb Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 12 Nov 2026 18:06:00 +0100 Subject: [PATCH 3/4] common: check the security index after the refresh; say what the stamp means apt only warns about a source it could not reach when another one answered, so the forced refresh could "succeed" without the upstream security index. Check the index is there afterwards and stop the run on that box if it is not; the next run refreshes again. Post-Invoke-Success runs when any source answered, so the refresh stamp records that a refresh ran and something answered, not that every source did. Say so, and add a generous fixed-age alert on the security index as the backstop for upstream being unreachable from every box at once. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 ++++++++---- roles/common/tasks/apt_lists_metric.yml | 11 +++++--- roles/common/tasks/apt_sources.yml | 35 ++++++++++++++++++++----- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6d12e15..ff92561 100644 --- a/README.md +++ b/README.md @@ -397,15 +397,22 @@ refresh, on every box: | Metric | What it is | Old means | |---|---|---| | `node_apt_security_lists_timestamp_seconds` | When the newest `-security` index on the box was published (the `Date:` of its `InRelease`; 0: none). | The box's security source is stale or unreachable, or the archive has simply published nothing new. | -| `node_apt_lists_refresh_timestamp_seconds` | The last refresh apt reported as a success (`common_apt_refresh_stamp`; 0: none yet). | The refresh is not running or is failing. | +| `node_apt_lists_refresh_timestamp_seconds` | The last refresh in which at least one source answered (`common_apt_refresh_stamp`, touched by apt's `Post-Invoke-Success`, which runs even when another source failed; 0: none yet). | The refresh is not running at all, or no source answers. One unreachable source does not show here. | They are timestamps rather than ages, so they keep getting older when the lists stop refreshing; the age is `time() - `. A quiet archive also makes the first one grow, and how quiet `noble-security` can get has not been -measured yet. The IPFS boxes all read the same upstream index, though, so the -dependable signal is one IPFS box's security index more than a day behind -another's. Add a fixed age (a week, say) only once the `Date:` has been watched -for a while. Like `node_reboot_required`, these reach Prometheus only once +measured yet. Three alerts between them cover the failures: + +- one IPFS box's security index more than a day behind another's (they all + read the same upstream index): one box's source is stale or unreachable; +- the refresh timestamp more than a day or two old: the refresh has stopped; +- the security index older than a generous fixed age, a week say: the backstop + for `security.ubuntu.com` being unreachable from every box at once, which + the first alert cannot see. Tighten it once the `Date:` has been watched for + a while. + +Like `node_reboot_required`, these reach Prometheus only once ipni/infra's node-exporter reads the textfile directory, and the alerts belong with the fleet rules there. Until then, on the box: diff --git a/roles/common/tasks/apt_lists_metric.yml b/roles/common/tasks/apt_lists_metric.yml index 217c4eb..7f10c5e 100644 --- a/roles/common/tasks/apt_lists_metric.yml +++ b/roles/common/tasks/apt_lists_metric.yml @@ -8,9 +8,12 @@ # seeing security fixes shows up. It also grows when the archive simply # publishes nothing new, so compare it across boxes that read the same # source rather than only against a fixed age. -# node_apt_lists_refresh_timestamp_seconds: the last package-list refresh apt -# reported as a success (0: none since this was installed). Old means the -# refresh itself is not running or failing, not that the archive is quiet. +# node_apt_lists_refresh_timestamp_seconds: the last package-list refresh in +# which at least one source answered (0: none since this was installed). +# That is what apt's Post-Invoke-Success means: it still runs when one +# source fails and another answers. So old means the refresh is not running +# at all, or no source answers; it does not show one source failing (the +# security index above does). # # Timestamps rather than ages: an age is frozen at whatever it was when the # file was last written, so it would stop growing in exactly the case it is @@ -54,7 +57,7 @@ printf '# HELP node_apt_security_lists_timestamp_seconds When the newest security package index on this box was published (0: none).\n' printf '# TYPE node_apt_security_lists_timestamp_seconds gauge\n' printf 'node_apt_security_lists_timestamp_seconds %s\n' "$newest" - printf '# HELP node_apt_lists_refresh_timestamp_seconds When apt last reported a package-list refresh as a success (0: never).\n' + printf '# HELP node_apt_lists_refresh_timestamp_seconds When a package-list refresh last got an answer from at least one source (0: never).\n' printf '# TYPE node_apt_lists_refresh_timestamp_seconds gauge\n' printf 'node_apt_lists_refresh_timestamp_seconds %s\n' "$refreshed" } > "$tmp" diff --git a/roles/common/tasks/apt_sources.yml b/roles/common/tasks/apt_sources.yml index 8b3898b..84a154a 100644 --- a/roles/common/tasks/apt_sources.yml +++ b/roles/common/tasks/apt_sources.yml @@ -42,19 +42,22 @@ register: common_apt_sources # apt names the file after the URI: security.ubuntu.com_ubuntu_dists_... -- name: Look for the security index from the configured source - ansible.builtin.stat: - path: >- +- name: Work out where apt keeps the security index from the configured source + ansible.builtin.set_fact: + common_apt_security_index_path: >- /var/lib/apt/lists/{{ common_apt_security_uri | regex_replace('^[a-z]+://', '') | replace('/', '_') }}dists_{{ ansible_distribution_release }}-security_InRelease + +- name: Look for the security index from the configured source + ansible.builtin.stat: + path: "{{ common_apt_security_index_path }}" register: common_apt_security_index # "Update apt cache" in main.yml skips a cache less than an hour old, which # would leave the rest of the play on the old lists. Keyed on the index as well # as the file, so a run that wrote the file and then failed here refreshes on -# the next run too. A failure after the retries stops the run on this box: it -# would otherwise carry on without a security index. Not reported as a change, -# like that task: the change is the file above, which a dry run shows. +# the next run too. Not reported as a change, like that task: the change is the +# file above, which a dry run shows. - name: Refresh the package lists from the new sources ansible.builtin.apt: update_cache: true @@ -64,3 +67,23 @@ until: common_apt_refresh is succeeded changed_when: false when: common_apt_sources is changed or not common_apt_security_index.stat.exists + +# apt treats a source it could not reach as a warning, not a failure, as long +# as another one answered. So check the index is really there now, and stop +# the run on this box if it is not, rather than carry on without it. The next +# run refreshes again (above). Skipped in a dry run, which refreshes nothing. +- name: Look again for the security index after the refresh + ansible.builtin.stat: + path: "{{ common_apt_security_index_path }}" + register: common_apt_security_index_after + when: not ansible_check_mode + +- name: Fail unless the security index from the configured source is there + ansible.builtin.assert: + that: common_apt_security_index_after.stat.exists + fail_msg: >- + {{ inventory_hostname }}: no {{ common_apt_security_index_path }} after + `apt-get update`; {{ common_apt_security_uri }} was probably unreachable. + The sources file is in place; re-run once it answers. + quiet: true + when: not ansible_check_mode From 645216d76f5c0bc56727f0d7fec6e166c4bca9aa Mon Sep 17 00:00:00 2001 From: Takwa Date: Thu, 12 Nov 2026 18:10:18 +0100 Subject: [PATCH 4/4] common: normalise the security URI's trailing slash for the index path apt adds a missing trailing slash to a source URI before naming its lists files, so an override without one (http://security.ubuntu.com/ubuntu) gave a path apt never writes and the new check would stop the run after a good refresh. Strip trailing slashes and add exactly one. Co-Authored-By: Claude Opus 5.5 --- roles/common/tasks/apt_sources.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/common/tasks/apt_sources.yml b/roles/common/tasks/apt_sources.yml index 84a154a..fbdf4af 100644 --- a/roles/common/tasks/apt_sources.yml +++ b/roles/common/tasks/apt_sources.yml @@ -41,12 +41,13 @@ mode: "0644" register: common_apt_sources -# apt names the file after the URI: security.ubuntu.com_ubuntu_dists_... +# apt names the file after the URI: security.ubuntu.com_ubuntu_dists_... It +# adds a missing trailing slash first, so this does too. - name: Work out where apt keeps the security index from the configured source ansible.builtin.set_fact: common_apt_security_index_path: >- - /var/lib/apt/lists/{{ common_apt_security_uri | regex_replace('^[a-z]+://', '') | replace('/', '_') - }}dists_{{ ansible_distribution_release }}-security_InRelease + /var/lib/apt/lists/{{ (common_apt_security_uri | regex_replace('^[a-z]+://', '') | regex_replace('/+$', '') + ~ '/') | replace('/', '_') }}dists_{{ ansible_distribution_release }}-security_InRelease - name: Look for the security index from the configured source ansible.builtin.stat: