diff --git a/README.md b/README.md index dd28134..3221c49 100644 --- a/README.md +++ b/README.md @@ -336,12 +336,36 @@ would fail until the box is built and pinned. If root takes neither your key nor ### Package updates and reboots -| | Policy on an indexer | -|---|---| -| What updates | `unattended-upgrades`, daily: Debian stable, its security archive and `-updates`. Never backports. | -| Reboots | Never automatic. | -| Services | Packages that restart their own service on upgrade still do (ssh, chrony, fail2ban). Established SSH sessions survive. k3s is not an apt package, so apt never restarts k3s or containers. `needrestart` only lists what is left running on old libraries. | -| Kernels | Installed but inactive until a reboot. | +| | IPFS boxes (Ubuntu) | Indexers (Debian) | +|---|---|---| +| What updates | `unattended-upgrades`, daily, with Ubuntu's default allowlist: `noble`, `noble-security`, and the ESM security origins `noble-apps-security` and `noble-infra-security`. In practice that is security fixes: `noble` is the release pocket, which does not change after release, and the ESM origins only apply on a box attached to Ubuntu Pro, which these are not. Never `noble-updates`. | `unattended-upgrades`, daily: Debian stable, its security archive and `-updates`. Never backports. | +| Reboots | Never automatic. | Never automatic. | +| Services | Packages that restart their own service on upgrade still do (ssh, chrony, fail2ban). Established SSH sessions survive. k3s is not an apt package, so apt never restarts k3s or containers. `needrestart` only lists what is left running on old libraries. | Same. | +| Kernels | Installed but inactive until a reboot. | Same. | + +Two things on Cherry's images silently stopped this, both found on 2026-11-11: + +- **Ubuntu: periodic apt switched off.** The image ships + `/etc/apt/apt.conf.d/docker-disable-periodic-update` with + `APT::Periodic::Enable "0"`. It sorts after everything of ours, so the + timers fired and did nothing. `common` deletes it. +- **Debian: relabelled suites.** Cherry's mirror rewrites `Label:` (to + `debian trixie-security` and so on), so Debian's default + `label=Debian-Security` patterns matched nothing and unattended-upgrades + reported "No packages found" every day. `group_vars/indexers` matches by + origin and codename instead, which hold on Cherry's mirror and on Debian's. + +`common` keeps `apt-daily.timer` (refreshes the lists) and +`apt-daily-upgrade.timer` (installs) enabled and running. It then asks apt what +it settled on (`apt-config shell` for `APT::Periodic::Enable`, +`APT::Periodic::Update-Package-Lists` and `APT::Periodic::Unattended-Upgrade`) +and fails the run if periodic apt is off, the lists are not refreshed daily or +unattended upgrades are not on, the way `hardening` checks `sshd -T`. needrestart's +list-only setting goes in before any of this, so no unattended run on a box ever +uses Ubuntu's restart-everything default. The check is skipped in a dry run. It cannot see +an origin pattern that matches nothing; for that, look at +`/var/log/unattended-upgrades/unattended-upgrades.log` and +`apt list --upgradable` on the box. A pending reboot shows in three places: `/var/run/reboot-required`, the `reboot:` line in every `site.yml` summary, and the `node_reboot_required` metric, which an @@ -349,9 +373,12 @@ hourly timer writes to `/var/lib/node_exporter/textfile/reboot.prom`. The metric reaches Prometheus only once ipni/infra's node-exporter is given that directory (`--collector.textfile.directory`, with a host mount) and an alert is added. -To reboot: one box at a time. Every container restarts. +The metric is written on indexers only (`common_reboot_reporting`); the IPFS +boxes have the marker file and the summary line. -The IPFS boxes keep their existing update policy. They only gain the `reboot:` summary line. +To reboot: one box at a time. Every container restarts, needle included, and +needle stays degraded for tens of minutes after it is Ready, so reboot an IPFS +box only once needle is warm on the other two. ## needle diff --git a/group_vars/indexers/main.yml b/group_vars/indexers/main.yml index 146bfc4..07c76eb 100644 --- a/group_vars/indexers/main.yml +++ b/group_vars/indexers/main.yml @@ -11,15 +11,23 @@ hardening_removed_accounts: - ubuntu - debian -# Point releases as well as security fixes, so packages keep up with Debian -# stable and not only with its advisories. Debian's own defaults already allow -# the release itself and Debian-Security; this adds -updates. Backports are -# never matched. See "Package updates and reboots" in the README. +# The release itself, its security archive and its point releases (-updates), +# so packages keep up with Debian stable and not only with its advisories. +# Backports are never matched. +# +# Matched by origin and codename only. Debian's own defaults also require +# label=Debian / label=Debian-Security, and Cherry's mirror rewrites the labels +# (`Label: debian trixie-security`, `Codename: trixie-security`), so they +# matched nothing: unattended-upgrades logged "No packages found" daily +# (found 2026-11-11). Origin and +# codename hold on Cherry's mirror and on Debian's own. See "Package updates +# and reboots" in the README. common_unattended_extra_origins: + - "origin=Debian,codename=${distro_codename}" + - "origin=Debian,codename=${distro_codename}-security" - "origin=Debian,codename=${distro_codename}-updates" -# needrestart reports and restarts nothing, and the reboot state goes to a -# file node-exporter can read. +# The reboot state goes to a file node-exporter can read. common_reboot_reporting: true # Cherry's Debian image ships 50-cloud-init.conf with PasswordAuthentication diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 1db15a9..e3cd75c 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -46,8 +46,9 @@ common_distro_packages: # Origins allowed on top of the distribution's own 50unattended-upgrades. common_unattended_extra_origins: [] -# needrestart in list-only mode, and a metric saying a reboot is pending. Off on -# the IPFS boxes, whose behaviour this must not change. +# The node_reboot_required metric (an hourly timer writing a textfile for +# node-exporter). needrestart's list-only mode is not part of this: every box +# gets it. Off on the IPFS boxes, whose node-exporter does not read the file. common_reboot_reporting: false # node-exporter reads *.prom from here once it is given diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 5782920..912b083 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -190,6 +190,24 @@ # --------------------------------------------------------------------------- # Automatic security updates # --------------------------------------------------------------------------- +# A package upgrade can restart services through needrestart, and Ubuntu 24.04's +# default in a non-interactive run is to restart them. `l` only lists them: k3s +# and everything it runs restart when a person decides, not at 06:00 on every +# box at once. First in this section, so it is in place before anything below +# turns unattended upgrades on, and stays in place if the play fails later. +- name: Install needrestart + ansible.builtin.apt: + name: needrestart + state: present + +- name: Keep needrestart to listing what needs a restart + ansible.builtin.copy: + dest: /etc/needrestart/conf.d/50-ipni.conf + mode: "0644" + content: | + # Managed by Ansible. + $nrconf{restart} = 'l'; + - name: Enable unattended security upgrades ansible.builtin.copy: dest: /etc/apt/apt.conf.d/20auto-upgrades @@ -199,8 +217,11 @@ APT::Periodic::Unattended-Upgrade "1"; APT::Periodic::AutocleanInterval "7"; -# With the defaults this writes what it always has: the policy on the IPFS boxes -# is unchanged. An indexer widens the origins. +# With the defaults this keeps the distribution's own allowed origins. On Ubuntu +# that is noble, noble-security and the two ESM security origins (see "Package +# updates and reboots" in the README). An indexer adds origin-and-codename +# patterns, because Debian's label-based defaults match nothing on Cherry's +# mirror: see group_vars/indexers. - name: Configure unattended-upgrades to security only, no auto-reboot ansible.builtin.copy: dest: /etc/apt/apt.conf.d/51-ipfs-unattended @@ -220,6 +241,61 @@ }; {% endif %} +# Cherry's Ubuntu image ships this file with `APT::Periodic::Enable "0"`. It +# belongs to no package and is not ours, and it sorts after 20auto-upgrades and +# 99*, so it wins: apt-daily and apt-daily-upgrade start and exit at once, and +# nothing above ever runs. Found 2026-11-11 on the IPFS boxes: no +# /var/log/unattended-upgrades at all and package lists weeks old. Removed +# rather than overridden, so no file of ours has to sort after it. +- name: Remove the provider's switch that turns periodic apt off + ansible.builtin.file: + path: /etc/apt/apt.conf.d/docker-disable-periodic-update + state: absent + +# apt-daily refreshes the package lists, apt-daily-upgrade installs. Both ship +# enabled, but nothing above helps if either is masked or stopped. +- name: Keep the apt timers enabled and running + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - apt-daily.timer + - apt-daily-upgrade.timer + +# apt reads every file under apt.conf.d and the last value wins, so a file +# someone else drops in can quietly switch this off again, as above: all of it, +# or just the list refresh, which would leave upgrades running on stale lists. +# Ask apt what it settled on. Skipped in a dry run, where the removal has not +# happened and the answer would still be the old one. +- name: Read the periodic apt settings apt is actually using + ansible.builtin.command: + argv: + - apt-config + - shell + - enable + - APT::Periodic::Enable + - update + - APT::Periodic::Update-Package-Lists + - unattended + - APT::Periodic::Unattended-Upgrade + register: apt_periodic_effective + changed_when: false + check_mode: false + when: not ansible_check_mode + +- name: Fail unless the package lists refresh and unattended upgrades run + ansible.builtin.assert: + that: + - "\"enable='0'\" not in apt_periodic_effective.stdout_lines" + - "\"update='1'\" in apt_periodic_effective.stdout_lines" + - "\"unattended='1'\" in apt_periodic_effective.stdout_lines" + fail_msg: >- + {{ inventory_hostname }}: another file under /etc/apt/apt.conf.d turns + periodic apt off; apt-config shows {{ apt_periodic_effective.stdout_lines }}. + quiet: true + when: not ansible_check_mode + - name: Report pending reboots and restarts ansible.builtin.import_tasks: reboot_reporting.yml when: common_reboot_reporting | bool diff --git a/roles/common/tasks/reboot_reporting.yml b/roles/common/tasks/reboot_reporting.yml index c6b6854..4392316 100644 --- a/roles/common/tasks/reboot_reporting.yml +++ b/roles/common/tasks/reboot_reporting.yml @@ -1,22 +1,7 @@ --- # Nothing here reboots or restarts anything. It makes "this box needs a reboot" # something a person or an alert can see. - -- name: Install needrestart - ansible.builtin.apt: - name: needrestart - state: present - -# The default in a non-interactive apt run can be to restart services. `l` only -# lists them. Restarting k3s would be survivable, but this box decides nothing -# on its own. -- name: Keep needrestart to listing what needs a restart - ansible.builtin.copy: - dest: /etc/needrestart/conf.d/50-ipni.conf - mode: "0644" - content: | - # Managed by Ansible. - $nrconf{restart} = 'l'; +# needrestart itself, in list-only mode, is installed on every box in main.yml. - name: Create the node-exporter textfile directory ansible.builtin.file: