From 70076297a9dff9605f9e8167073c1460fd60524a Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 17 Sep 2026 15:15:50 +0200
Subject: [PATCH 01/17] feat(coredump): capture PostgreSQL-only core dumps on
 the AMI

- LimitCORE=infinity, scoped to postgresql.service only (not machine-wide)
  so an unrelated crash still produces no core
- CoredumpFilter excludes shared_buffers (an anonymous shared mapping) from
  every core, since it can be many GB - this is what actually bounds core size
- systemd-coredump gets its own conservative storage limits (compression,
  size caps), and is installed explicitly (not assumed to already be present)
- confirm postgres_prestart.sh never resets ulimit -c (regression guard)
- testinfra: LimitCORE, coredump_filter value, storage limits, coredump
  dir permissions, and a real end-to-end crash test (crash produces a
  core, an unrelated process's crash doesn't)
---
 .../postgresql_config/coredump-storage.conf   |   7 +
 ansible/files/postgresql_config/coredump.conf |   2 +
 .../postgresql_config/postgresql.service      |   4 +
 ansible/tasks/setup-postgres.yml              |  37 ++++
 testinfra/test_ami_nix.py                     | 180 ++++++++++++++++++
 5 files changed, 230 insertions(+)
 create mode 100644 ansible/files/postgresql_config/coredump-storage.conf
 create mode 100644 ansible/files/postgresql_config/coredump.conf

diff --git a/ansible/files/postgresql_config/coredump-storage.conf b/ansible/files/postgresql_config/coredump-storage.conf
new file mode 100644
index 0000000000..fcd9c211e8
--- /dev/null
+++ b/ansible/files/postgresql_config/coredump-storage.conf
@@ -0,0 +1,7 @@
+[Coredump]
+Storage=external
+Compress=yes
+ProcessSizeMax=8G
+ExternalSizeMax=8G
+MaxUse=10G
+KeepFree=5G
diff --git a/ansible/files/postgresql_config/coredump.conf b/ansible/files/postgresql_config/coredump.conf
new file mode 100644
index 0000000000..7c45cd49e5
--- /dev/null
+++ b/ansible/files/postgresql_config/coredump.conf
@@ -0,0 +1,2 @@
+[Service]
+LimitCORE=infinity
diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service
index 68c37140bd..c7ed7cad5d 100644
--- a/ansible/files/postgresql_config/postgresql.service
+++ b/ansible/files/postgresql_config/postgresql.service
@@ -11,6 +11,10 @@ Type=notify
 User=postgres
 ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql
 ExecStartPre=+/usr/local/bin/postgres_prestart.sh
+# Excludes shared_buffers (an anonymous *shared* mapping) from core dumps -
+# without this, every core would include the whole (potentially many-GB)
+# buffer pool. Inherited by every process postgres forks.
+CoredumpFilter=private-anonymous elf-headers private-huge
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT
diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index 1bcc0420e4..12bc85a640 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -292,6 +292,43 @@
     state: 'directory'
   become: true
 
+# Required for this (plus CoredumpFilter= in postgresql.service and the
+# storage limits below) to do anything at all: without systemd-coredump,
+# core_pattern stays at the kernel's plain "core" default and none of this
+# gets exercised (confirmed on a local test VM). Unconditional - every
+# Postgres flavor's capture depends on it, not just OrioleDB.
+- name: ensure systemd-coredump is installed
+  ansible.builtin.apt:
+    name: systemd-coredump
+  become: true
+
+- name: copy PostgreSQL coredump systemd drop-in
+  ansible.builtin.copy:
+    dest: '/etc/systemd/system/postgresql.service.d/coredump.conf'
+    group: 'root'
+    mode: '0644'
+    owner: 'root'
+    src: 'files/postgresql_config/coredump.conf'
+  become: true
+
+- name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits
+  ansible.builtin.file:
+    group: 'root'
+    mode: '0755'
+    owner: 'root'
+    path: '/etc/systemd/coredump.conf.d'
+    state: 'directory'
+  become: true
+
+- name: copy systemd-coredump storage limits
+  ansible.builtin.copy:
+    dest: '/etc/systemd/coredump.conf.d/postgres.conf'
+    group: 'root'
+    mode: '0644'
+    owner: 'root'
+    src: 'files/postgresql_config/coredump-storage.conf'
+  become: true
+
 - name: Ensure PostgreSQL starts after tuned
   become: true
   community.general.ini_file:
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index cfd77554d6..869ab3a020 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1407,3 +1407,183 @@ def test_apparmor_denies_access_to_sensitive_paths(host):
             f"to have succeeded.\nstdout: {result['stdout']}\nstderr: {result['stderr']}"
         )
         print(f"Confirmed: access to {test_file} denied by AppArmor")
+
+
+def test_postgresql_service_allows_unlimited_core_dumps(host):
+    """Verify the postgresql.service coredump drop-in sets LimitCORE=infinity.
+
+    Coredumps are intentionally scoped to the postgresql unit only (via a
+    systemd.service.d drop-in), not enabled machine-wide, so other services
+    must keep the default core limit.
+    """
+    result = run_ssh_command(host["ssh"], "systemctl show postgresql -p LimitCORE")
+    assert result["succeeded"], f"systemctl show failed: {result['stderr']}"
+    assert "LimitCORE=infinity" in result["stdout"], (
+        f"Expected postgresql.service to have LimitCORE=infinity, got:\n{result['stdout']}"
+    )
+
+
+def test_coredump_storage_limits_configured(host):
+    """Verify /etc/systemd/coredump.conf.d/postgres.conf sets conservative,
+    bounded storage limits for the systemd-coredump storage that backs
+    Postgres core capture."""
+    result = run_ssh_command(
+        host["ssh"], "cat /etc/systemd/coredump.conf.d/postgres.conf"
+    )
+    assert result["succeeded"], (
+        f"Could not read coredump storage config: {result['stderr']}"
+    )
+    for expected in [
+        "Storage=external",
+        "Compress=yes",
+        "ProcessSizeMax=",
+        "ExternalSizeMax=",
+        "MaxUse=",
+        "KeepFree=",
+    ]:
+        assert expected in result["stdout"], (
+            f"Expected '{expected}' in /etc/systemd/coredump.conf.d/postgres.conf, "
+            f"got:\n{result['stdout']}"
+        )
+
+
+def test_postgres_coredump_filter_excludes_shared_buffers(host):
+    """Verify the running postmaster's /proc/[pid]/coredump_filter is 0x31
+    (49): private mappings + ELF headers, but not anonymous-shared mappings.
+
+    shared_buffers is mmap(MAP_SHARED|MAP_ANONYMOUS) (shared_memory_type
+    defaults to 'mmap' and is not overridden), which the kernel classifies
+    as an anonymous shared mapping - excluding it from every core is what
+    actually keeps core size bounded, since shared_buffers can be many GB.
+    postgresql.service sets this via the native systemd
+    'CoredumpFilter=private-anonymous elf-headers private-huge' directive
+    (systemd >= 246), which coredump_filter (inherited across fork(2) and
+    preserved across execve(2)) then propagates to everything postgres forks.
+    """
+    pid = run_ssh_command(
+        host["ssh"], "systemctl show postgresql -p MainPID --value"
+    )["stdout"].strip()
+    assert pid.isdigit() and pid != "0", f"Could not resolve postgresql.service MainPID: {pid}"
+
+    result = run_ssh_command(host["ssh"], f"cat /proc/{pid}/coredump_filter")
+    assert result["succeeded"], f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
+    assert result["stdout"].strip() == "31", (
+        f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), "
+        f"got '{result['stdout'].strip()}'"
+    )
+
+
+def test_coredump_storage_directory_root_only(host):
+    """Verify /var/lib/systemd/coredump is root-only, since it can hold
+    core files containing customer data."""
+    result = run_ssh_command(
+        host["ssh"], "stat -c '%a %U:%G' /var/lib/systemd/coredump"
+    )
+    assert result["succeeded"], f"stat failed: {result['stderr']}"
+    mode, owner = result["stdout"].strip().split()
+    assert mode in ("700", "750"), (
+        f"Expected /var/lib/systemd/coredump to be root-only, got mode {mode}"
+    )
+    assert owner.startswith("root:"), (
+        f"Expected /var/lib/systemd/coredump to be owned by root, got {owner}"
+    )
+
+
+def test_postgres_prestart_does_not_reset_core_limit(host):
+    """Regression guard: postgres_prestart.sh must never touch 'ulimit -c',
+    or it would silently defeat the postgresql.service LimitCORE=infinity
+    coredump drop-in."""
+    result = run_ssh_command(
+        host["ssh"], "cat /usr/local/bin/postgres_prestart.sh"
+    )
+    assert result["succeeded"], f"Could not read prestart script: {result['stderr']}"
+    assert "ulimit -c" not in result["stdout"], (
+        "postgres_prestart.sh must not set 'ulimit -c' - doing so would silently "
+        "defeat the postgresql.service coredump drop-in"
+    )
+
+
+def _crash_a_backend_and_wait_for_recovery(host):
+    """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's
+    normal crash-recovery to bring the instance back on its own
+    (Restart=always / auto-reinit, same as production). Returns the crashed
+    backend's pid. Used by tests that need a real, fresh core to appear."""
+    backend_pid = run_ssh_command(
+        host["ssh"],
+        "sudo -u postgres psql -U supabase_admin -h localhost -d postgres "
+        "-tAc 'select pg_backend_pid()'",
+    )["stdout"].strip()
+    assert backend_pid.isdigit(), (
+        f"Could not resolve a Postgres backend pid: {backend_pid}"
+    )
+    run_ssh_command(host["ssh"], f"sudo kill -SEGV {backend_pid}")
+
+    recovered = False
+    for _ in range(30):
+        sleep(2)
+        probe = run_ssh_command(
+            host["ssh"],
+            "sudo -u postgres psql -U supabase_admin -h localhost -d postgres "
+            "-tAc 'select 1'",
+        )
+        if probe["succeeded"] and probe["stdout"].strip() == "1":
+            recovered = True
+            break
+    assert recovered, (
+        "PostgreSQL did not come back up within 60s after the induced backend crash"
+    )
+    return backend_pid
+
+
+def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(host):
+    """End-to-end capture check: a segfaulted Postgres backend must produce a
+    coredumpctl-visible core, while an unrelated process crashing the same way
+    must not.
+
+    This intentionally crashes a live backend (mirroring a real SIGSEGV, e.g.
+    the background-writer crash in incident ORI-261) and relies on
+    PostgreSQL's normal crash-recovery to bring the instance back on its own
+    (Restart=always / auto-reinit), the same as in production - it does not
+    reinstall data or otherwise reset the shared test instance.
+    """
+    before = run_ssh_command(
+        host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
+    )
+    before_lines = set(before["stdout"].splitlines())
+
+    # Unrelated process: launch a disposable process outside the
+    # LimitCORE=infinity-scoped postgresql.service and SIGSEGV it - it must
+    # not produce a core, since the default core limit is unchanged for it.
+    run_ssh_command(
+        host["ssh"],
+        "setsid bash -c 'sleep 60 & echo $! > /tmp/unrelated_pid; wait' >/dev/null 2>&1 &",
+    )
+    sleep(1)
+    unrelated_pid = run_ssh_command(host["ssh"], "cat /tmp/unrelated_pid")[
+        "stdout"
+    ].strip()
+    assert unrelated_pid.isdigit(), (
+        f"Could not resolve the disposable unrelated process pid: {unrelated_pid}"
+    )
+    run_ssh_command(host["ssh"], f"kill -SEGV {unrelated_pid}")
+    sleep(2)
+
+    # Postgres backend: grab a real backend pid and crash it the same way.
+    backend_pid = _crash_a_backend_and_wait_for_recovery(host)
+
+    after = run_ssh_command(
+        host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
+    )
+    new_lines = [
+        line for line in after["stdout"].splitlines() if line not in before_lines
+    ]
+    assert any("postgres" in line for line in new_lines), (
+        f"Expected a new postgres core dump after SIGSEGV to backend {backend_pid}, "
+        f"but coredumpctl list shows:\n{after['stdout']}"
+    )
+    assert not any(unrelated_pid in line for line in new_lines), (
+        f"Unrelated process {unrelated_pid} should not have produced a core dump:\n"
+        f"{after['stdout']}"
+    )
+
+

From 748554920c914f75a4f2b5736779057400f78445 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 17 Sep 2026 15:16:06 +0200
Subject: [PATCH 02/17] test(coredump): verify shipped debug symbols actually
 resolve

- postgres/orioledb debug+source packages already ship on every AMI
  (postgres-env bundle) - nothing new to package here, verification only
- check build-ID match: shipped postgres binary and orioledb.so vs the
  installed debug package
- check GDB can read real source content through the source package (not
  just that a filename is known - info sources lists names regardless of
  whether the file is actually reachable on disk)
---
 testinfra/test_ami_nix.py | 107 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 869ab3a020..d629fc49e2 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1587,3 +1587,110 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     )
 
 
+def _build_id_of(host, path):
+    """Return the ELF build-id (hex string) of the binary/library at path, or None."""
+    import re
+
+    result = run_ssh_command(host["ssh"], f"readelf -n {path} 2>/dev/null")
+    if not result["succeeded"]:
+        return None
+    match = re.search(r"Build ID:\s*([0-9a-f]+)", result["stdout"])
+    return match.group(1) if match else None
+
+
+def _debug_file_exists_for_build_id(host, build_id):
+    """Check whether the postgres nix-profile's debug output has a
+    .build-id/xx/yyyy...debug file matching the given build-id."""
+    prefix, rest = build_id[:2], build_id[2:]
+    debug_path = (
+        f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
+    )
+    result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present")
+    return result["succeeded"] and "present" in result["stdout"]
+
+
+def test_postgres_binary_build_id_matches_shipped_debug_symbols(host):
+    """Verify the installed 'postgres' binary's build-id has a matching
+    .build-id/xx/yyyy.debug file in the postgres-env debug output, so a
+    future coredump-processing GDB session can actually resolve symbols."""
+    build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres")
+    assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres"
+    assert _debug_file_exists_for_build_id(host, build_id), (
+        f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
+        f".build-id/ matching postgres build-id {build_id} - the shipped "
+        f"_debug package may be out of sync with the shipped binary"
+    )
+
+
+def test_orioledb_library_build_id_matches_shipped_debug_symbols(host):
+    """Verify orioledb.so's build-id has a matching debug file, same as for
+    the postgres binary - orioledb.so is built by the same derivation
+    (isOrioleDB flavor) so its debug info ships in the same _debug output."""
+    orioledb_so = "/usr/lib/postgresql/lib/orioledb.so"
+    exists = run_ssh_command(host["ssh"], f"test -f {orioledb_so} && echo present")
+    if "present" not in exists["stdout"]:
+        pytest.skip("orioledb.so not present on this AMI (not an OrioleDB build)")
+
+    build_id = _build_id_of(host, orioledb_so)
+    assert build_id, f"Could not read a build-id from {orioledb_so}"
+    assert _debug_file_exists_for_build_id(host, build_id), (
+        f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
+        f".build-id/ matching orioledb.so build-id {build_id}"
+    )
+
+
+def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
+    """Verify GDB can read actual source *content* for the installed postgres
+    binary via the shipped _src package - not just that a filename is known
+    from debug info (which 'info sources' would show regardless of whether
+    the file is reachable on disk).
+
+    The _src package mirrors the exact build-time source tree under the
+    nix-profile root (see nix/postgresql/src.nix), but the debug info records
+    the original nix build sandbox directory (DW_AT_comp_dir, e.g.
+    /build/postgres-<rev>) as each file's location. GDB needs a
+    'substitute-path' from that recorded build directory to the profile root
+    to find the files - this test discovers that build directory dynamically
+    (rather than hardcoding a guess) and confirms 'list main' then prints
+    real source lines instead of falling back to the 'in <path>' placeholder
+    GDB uses when a source file can't be found.
+    """
+    import re
+
+    build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres")
+    assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres"
+    prefix, rest = build_id[:2], build_id[2:]
+    debug_file = f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
+
+    comp_dir = run_ssh_command(
+        host["ssh"],
+        f"readelf --debug-dump=info {debug_file} 2>/dev/null "
+        "| grep -m1 DW_AT_comp_dir | grep -oE '/[^ ]+$'",
+    )["stdout"].strip()
+    assert comp_dir, f"Could not determine DW_AT_comp_dir from {debug_file}"
+
+    result = run_ssh_command(
+        host["ssh"],
+        "sudo -u postgres gdb --batch -quiet "
+        "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' "
+        f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' "
+        "-ex 'file /usr/lib/postgresql/bin/postgres' "
+        "-ex 'list main' "
+        "2>&1",
+    )
+    assert result["succeeded"], f"gdb invocation failed: {result['stderr']}"
+    assert "No debugging symbols found" not in result["stdout"], (
+        f"GDB could not find debug symbols for postgres:\n{result['stdout']}"
+    )
+    assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), (
+        f"GDB fell back to the 'in <path>' placeholder, meaning it could not "
+        f"actually read the source file even with substitute-path set from "
+        f"{comp_dir} to /var/lib/postgresql/.nix-profile:\n{result['stdout']}"
+    )
+    numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE)
+    assert len(numbered_lines) >= 3, (
+        f"Expected 'list main' to print several lines of real source code "
+        f"content via the shipped _src package, got:\n{result['stdout']}"
+    )
+
+

From a20c283560bb6a13a804e45d76b1f1c7fb3d6724 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 17 Sep 2026 15:16:37 +0200
Subject: [PATCH 03/17] feat(coredump): process captured cores into diagnostic
 bundles

- orioledb-coredump.path (triggers on new cores) + .timer (10-min fallback
  sweep) both run a flock-guarded oneshot service
- gdb and binutils are installed explicitly for this (scoped to OrioleDB
  images via is_psql_oriole, since only this processor needs them)
- processor script: keeps only postgres-owned cores, extracts a bundle via
  GDB (cmds.gdb - matches OrioleDB's own CI debugging script: full
  backtrace, argv, shared libraries, registers, lock state), deletes the
  raw core, quarantines metadata-only after repeated failures, and enforces
  its own age/size retention independent of systemd-coredump's own limits
- handles /usr/lib/postgresql/bin/postgres being a Nix wrapper script (not
  the real ELF) throughout: the executable filter, readelf, and gdb all use
  the real per-crash path reported by coredumpctl instead of a fixed guess
- matches/deletes cores by PID, not raw file path (coredumpctl has no "rm"
  verb on this systemd version, and path-based matching was unreliable)
- reads the active postgresql log path from current_logfiles at runtime,
  since this AMI logs via csvlog with no fixed filename
- rolled out to OrioleDB images only for now (is_psql_oriole)
- testinfra: end-to-end check that a real crash produces a diagnostic
  bundle and the raw core gets deleted
---
 ansible/files/coredump/cmds.gdb               |  20 ++
 ansible/files/coredump/orioledb-coredump.path |   9 +
 .../files/coredump/orioledb-coredump.service  |  14 +
 .../files/coredump/orioledb-coredump.timer    |  10 +
 .../coredump/process-orioledb-coredumps.sh    | 281 ++++++++++++++++++
 ansible/playbook.yml                          |   4 +
 ansible/tasks/setup-coredump-processing.yml   |  75 +++++
 testinfra/test_ami_nix.py                     |  54 ++++
 8 files changed, 467 insertions(+)
 create mode 100644 ansible/files/coredump/cmds.gdb
 create mode 100644 ansible/files/coredump/orioledb-coredump.path
 create mode 100644 ansible/files/coredump/orioledb-coredump.service
 create mode 100644 ansible/files/coredump/orioledb-coredump.timer
 create mode 100644 ansible/files/coredump/process-orioledb-coredumps.sh
 create mode 100644 ansible/tasks/setup-coredump-processing.yml

diff --git a/ansible/files/coredump/cmds.gdb b/ansible/files/coredump/cmds.gdb
new file mode 100644
index 0000000000..93b4599453
--- /dev/null
+++ b/ansible/files/coredump/cmds.gdb
@@ -0,0 +1,20 @@
+# Matches orioledb/ci/cmds.gdb (OrioleDB's own CI crash-debugging script).
+# debug-file-directory/file/core-file are set by the caller before this
+# file is sourced (see process-orioledb-coredumps.sh).
+#
+# GDB aborts the rest of a sourced script on the first command error, so
+# order matters here: the reliable, high-value output runs first; the
+# OrioleDB-internal lock-state dumps run last, since they can fail if
+# orioledb.so's own debug symbols aren't available (a known, separate issue)
+thread apply all bt full
+up 99999
+set $i=0
+set $end=argc
+while ($i < $end)
+p argv[$i++]
+end
+info sharedlibrary
+info registers
+eval "p *((LWLockHandle (*) [%u]) held_lwlocks)", num_held_lwlocks
+eval "p *((MyLockedPage (*) [%u]) myLockedPages)", numberOfMyLockedPages
+quit
diff --git a/ansible/files/coredump/orioledb-coredump.path b/ansible/files/coredump/orioledb-coredump.path
new file mode 100644
index 0000000000..c9a5ce83d1
--- /dev/null
+++ b/ansible/files/coredump/orioledb-coredump.path
@@ -0,0 +1,9 @@
+[Unit]
+Description=Watch for PostgreSQL/OrioleDB core dumps
+
+[Path]
+PathChanged=/var/lib/systemd/coredump
+Unit=orioledb-coredump.service
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/files/coredump/orioledb-coredump.service b/ansible/files/coredump/orioledb-coredump.service
new file mode 100644
index 0000000000..49e574438d
--- /dev/null
+++ b/ansible/files/coredump/orioledb-coredump.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Process PostgreSQL/OrioleDB core dumps into a redacted diagnostic bundle
+After=systemd-coredump@.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/process-orioledb-coredumps.sh
+User=root
+Group=root
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/var/lib/systemd/coredump /var/lib/orioledb-coredumps /run
diff --git a/ansible/files/coredump/orioledb-coredump.timer b/ansible/files/coredump/orioledb-coredump.timer
new file mode 100644
index 0000000000..15d332ba66
--- /dev/null
+++ b/ansible/files/coredump/orioledb-coredump.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodic fallback sweep for PostgreSQL/OrioleDB core dumps
+
+[Timer]
+OnBootSec=10min
+OnUnitActiveSec=10min
+Unit=orioledb-coredump.service
+
+[Install]
+WantedBy=timers.target
diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh
new file mode 100644
index 0000000000..896e96a380
--- /dev/null
+++ b/ansible/files/coredump/process-orioledb-coredumps.sh
@@ -0,0 +1,281 @@
+#!/bin/bash
+# Process PostgreSQL/OrioleDB core dumps captured by systemd-coredump into a
+# text summary.
+#
+# Key ideas:
+#  - Only cores of the postgres binary are processed; any other core left
+#    untouched
+#  - No environment variables are ever collected. The GDB extraction (see
+#    cmds.gdb, matching OrioleDB's own CI debugging script) does run
+#    `thread apply all bt full`, which prints local variable values and can
+#    surface fragments of in-memory data (buffer/tuple pointers etc.) - this
+#    is a deliberate, reviewed trade-off in favor of debuggability, not an
+#    oversight.
+#  - Cores are deleted after a successful run or quarantined (metadata only)
+#    after MAX_ATTEMPTS failures.
+
+set -euo pipefail
+
+STATE_DIR=/var/lib/orioledb-coredumps/state
+OUTPUT_DIR=/var/lib/orioledb-coredumps/diagnostics
+QUARANTINE_DIR=/var/lib/orioledb-coredumps/quarantine
+LOCK_FILE=/run/orioledb-coredump.lock
+
+MAX_ATTEMPTS=3
+EXTRACTION_TIMEOUT=120
+MAX_AGE_DAYS=7
+MAX_TOTAL_BYTES=$((5 * 1024 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse
+
+GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug
+GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb
+PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles
+
+log() {
+	echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"
+}
+
+# /usr/lib/postgresql/bin/postgres is a Nix wrapper *script* (sets
+# NIX_PGLIBDIR, then execs the real ELF at .../bin/.postgres-wrapped) - not
+# an executable itself. A crashing backend's real, kernel-recorded
+# Executable: is that wrapped path (basename ".postgres-wrapped"), never the
+# wrapper. Normalize both forms to "postgres" for comparison, and always use
+# the per-crash Executable: path (not a hardcoded one) for readelf/gdb.
+normalize_exe_basename() {
+	local base
+	base=$(basename "$1")
+	base="${base#.}"
+	base="${base%-wrapped}"
+	printf '%s' "$base"
+}
+
+# orioledb.so has no fixed, predictable path either - there is no
+# /usr/lib/postgresql/lib mirror of it. It lives in the same nix store
+# derivation as the resolved postgres executable, just under lib/ instead
+# of bin/, e.g. .../postgresql-and-plugins-17_20/{bin/.postgres-wrapped,
+# lib/orioledb.so} - so derive it from $exe rather than guessing a path.
+orioledb_lib_path() {
+	local exe_dir
+	exe_dir=$(dirname "$(dirname "$1")")
+	printf '%s/lib/orioledb.so' "$exe_dir"
+}
+
+# coredumpctl on this systemd version (255.4) has no "rm"/"delete" verb - the
+# only reliable way to remove a core is to delete its on-disk Storage: path
+# directly. Returns success only if the path is actually gone afterward.
+delete_core() {
+	local path="$1"
+	[ -z "$path" ] && return 1
+	rm -f -- "$path"
+	[ ! -e "$path" ]
+}
+
+# The active postgresql log file has an unpredictable name and can be
+# csvlog, stderr-text, or both depending on config (this AMI defaults to
+# csvlog-only, e.g. /var/log/postgresql/postgresql.csv - the stderr-format
+# postgresql.log stops receiving anything the moment the logging collector
+# switches over at startup). PGDATA/current_logfiles is postgres's own,
+# always-current record of the real path(s); prefer csvlog, fall back to
+# stderr. Either format still starts each line with a literal timestamp, so
+# the grep -F substring match below works unchanged either way.
+current_postgres_log() {
+	[ -f "$PGDATA_CURRENT_LOGFILES" ] || return 0
+	awk '$1 == "csvlog" {p = $2} $1 == "stderr" && !p {p = $2} END {print p}' "$PGDATA_CURRENT_LOGFILES"
+}
+
+enforce_retention() {
+	find "$OUTPUT_DIR" -maxdepth 1 -type f -mtime "+${MAX_AGE_DAYS}" -delete 2>/dev/null || true
+
+	while true; do
+		total=$(du -sb "$OUTPUT_DIR" 2>/dev/null | cut -f1)
+		[ -z "$total" ] && break
+		[ "$total" -le "$MAX_TOTAL_BYTES" ] && break
+		oldest=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | head -1 | cut -d' ' -f2-)
+		[ -z "$oldest" ] && break
+		log "retention: removing oldest bundle $oldest to stay under ${MAX_TOTAL_BYTES} bytes"
+		rm -f "$oldest"
+	done
+}
+
+# Handles one candidate crash, given its PID: decide whether it's ours to
+# process, extract a diagnostic bundle via GDB, then delete the raw core.
+# (1) look up the crash via coredumpctl
+# (2) decide keep/ignore/quarantine based on prior attempts
+# (3) export the core and run GDB against it
+# (4) write the bundle and delete the raw core. Returns 1 only for failures
+#     worth retrying next run (main() logs those); every other outcome is
+#     ignored, quarantined, or successfully processed - returns 0.
+process_one() {
+	local pid="$1"
+
+	# coredumpctl matches reliably by PID; matching by the raw storage path
+	# (which encodes an escaped comm, e.g. "core.\x2epostgres-wrapp....zst"
+	# for the wrapped binary below) was found to fail in practice.
+	local meta
+	if ! meta=$(coredumpctl info "$pid" --no-pager 2>/dev/null); then
+		log "pid ${pid}: coredumpctl info failed"
+		return 1
+	fi
+
+	local exe boot_id storage_path
+	exe=$(awk -F': ' '/^ *Executable:/ {print $2; exit}' <<<"$meta")
+	boot_id=$(awk -F': ' '/^ *Boot ID:/ {print $2; exit}' <<<"$meta")
+	# "Storage: /path/to/core (present)" -> "/path/to/core"
+	storage_path=$(sed -n 's/^ *Storage: \(.*\) (.*)$/\1/p' <<<"$meta" | head -1)
+	# PID alone isn't a safe dedup key long-term (PIDs get reused across
+	# boots), so pair it with boot ID - matches "state keyed by boot ID plus
+	# dump identifier" from the original design.
+	local key="${boot_id}-${pid}"
+	local state_file="${STATE_DIR}/${key}"
+
+	# .done means "final decision made, never look at this dump again"
+	# (processed successfully, ignored as non-postgres, or quarantined).
+	# .attempts only counts *failed* tries, to cap retries before quarantine.
+	[ -f "${state_file}.done" ] && return 0
+
+	local attempts=0
+	[ -f "${state_file}.attempts" ] && attempts=$(cat "${state_file}.attempts")
+	if [ "$attempts" -ge "$MAX_ATTEMPTS" ]; then
+		log "pid ${pid}: exceeded ${MAX_ATTEMPTS} attempts, discarding core (metadata kept in ${QUARANTINE_DIR})"
+		echo "$meta" >"${QUARANTINE_DIR}/${key}.info"
+		if delete_core "$storage_path"; then
+			log "pid ${pid}: raw core deleted"
+		else
+			log "pid ${pid}: WARNING - could not delete raw core at '${storage_path}'"
+		fi
+		touch "${state_file}.done"
+		return 0
+	fi
+
+	# /usr/lib/postgresql/bin/postgres is a wrapper script that execs the
+	# real ELF at .../bin/.postgres-wrapped - the kernel (and therefore
+	# coredumpctl's Executable:) always records the latter. Normalize both
+	# forms before comparing, so we don't silently ignore every real crash.
+	if [[ "$(normalize_exe_basename "$exe")" != "postgres" ]]; then
+		log "pid ${pid}: executable '${exe}' is not postgres, ignoring"
+		touch "${state_file}.done"
+		return 0
+	fi
+
+	# From here on we're committed to actually processing this dump, so
+	# count it as an attempt before doing any of the risky (slow, can fail)
+	# work below - a crash/timeout past this point still gets retried, up
+	# to MAX_ATTEMPTS
+	echo $((attempts + 1)) >"${state_file}.attempts"
+
+	# coredumpctl stores the core compressed; pull a private, working copy
+	# out into a root-only scratch dir before handing it to GDB. The trap
+	# guarantees that scratch dir is removed when this function returns, no
+	# matter which of the several `return`s below fires.
+	local tmpdir
+	tmpdir=$(mktemp -d /tmp/coredump-XXXXXX)
+	chmod 700 "$tmpdir"
+	# suppress shellcheck warning about quoting $tmpdir in the trap command - 
+	# it's correct to quote it, and the trap is evaluated at runtime,
+	# not parse time.
+	# shellcheck disable=SC2064
+	trap "rm -rf '$tmpdir'" RETURN
+
+	if ! timeout "$EXTRACTION_TIMEOUT" coredumpctl dump "$pid" --output "${tmpdir}/core" >/dev/null 2>&1; then
+		log "pid ${pid}: export failed or timed out"
+		return 1
+	fi
+
+	# Already have this in $meta from the coredumpctl info call above -
+	# just pulling out the two fields the bundle header needs.
+	local signal timestamp
+	signal=$(awk -F': ' '/^ *Signal:/ {print $2; exit}' <<<"$meta")
+	timestamp=$(awk -F': ' '/^ *Timestamp:/ {print $2; exit}' <<<"$meta")
+
+	# Everything from here to the closing "}" is the diagnostic bundle
+	# itself, one section at a time, redirected straight to $bundle -
+	# there's no in-memory buffering of it, so a slow/hanging step just
+	# shows up as a truncated file rather than blocking the whole write.
+	local bundle="${OUTPUT_DIR}/${key}.txt"
+	{
+		echo "== OrioleDB/PostgreSQL coredump diagnostic bundle =="
+		echo "generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+		echo "pid: ${pid}"
+		echo "boot_id: ${boot_id}"
+		echo "signal: ${signal}"
+		echo "crash_timestamp: ${timestamp}"
+		echo "executable: ${exe}"
+		echo
+
+		echo "== build ids =="
+		echo "postgres (${exe}):"
+		readelf -n "$exe" 2>/dev/null | grep 'Build ID' || echo "  (could not read build id)"
+		orioledb_lib=$(orioledb_lib_path "$exe")
+		if [ -f "$orioledb_lib" ]; then
+			echo "orioledb.so (${orioledb_lib}):"
+			readelf -n "$orioledb_lib" 2>/dev/null | grep 'Build ID' || echo "  (could not read build id)"
+		fi
+		echo
+
+		echo "== gdb backtrace (full), lwlocks, locked pages, argv, shared libraries, registers =="
+		timeout "$EXTRACTION_TIMEOUT" gdb --batch -quiet \
+			-ex "set debug-file-directory ${GDB_DEBUG_DIR}" \
+			-ex "file ${exe}" \
+			-ex "core-file ${tmpdir}/core" \
+			-x "$GDB_CMDS_FILE" \
+			2>&1 || echo "(gdb extraction failed or timed out)"
+		echo
+
+		echo "== postgresql.log excerpt around crash =="
+		# $timestamp looks like "Thu 2026-09-17 11:25:49 UTC (1s ago)" - pull
+		# out just the "YYYY-MM-DD HH:MM:SS" portion to match against
+		# postgres's own log line prefix (a fixed offset previously grabbed
+		# the leading weekday name instead and never matched anything).
+		log_ts=$(grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' <<<"$timestamp" | head -1)
+		postgres_log=$(current_postgres_log)
+		if [ -n "$log_ts" ] && [ -n "$postgres_log" ] && [ -f "$postgres_log" ]; then
+			grep -F "$log_ts" -A 5 -B 20 "$postgres_log" 2>/dev/null | tail -200 ||
+				echo "(no log lines found matching ${log_ts} in ${postgres_log})"
+		else
+			echo "(no crash timestamp or log file available)"
+		fi
+	} >"$bundle"
+	chmod 600 "$bundle"
+
+	# Bundle is written either way at this point, even if the raw-core
+	# delete below fails - we don't want a delete failure to make us
+	# reprocess (and re-append to) an already-complete bundle next run.
+	touch "${state_file}.done"
+	if delete_core "$storage_path"; then
+		log "pid ${pid}: wrote ${bundle}, deleted raw core"
+	else
+		log "pid ${pid}: wrote ${bundle}, but WARNING - could not delete raw core at '${storage_path}'"
+	fi
+}
+
+main() {
+	mkdir -p "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR"
+	chmod 700 "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR"
+
+	enforce_retention
+
+	# Enumerate via coredumpctl (per INSTRUCTIONS.md's original design),
+	# restricted to entries whose raw core is still on disk ("present") -
+	# already-removed/historical journal entries are skipped without
+	# needing a coredumpctl info round-trip. Field positions match the
+	# observed `coredumpctl list` table layout (systemd 255):
+	#   TIME(4 tokens) PID UID GID SIG COREFILE EXE SIZE
+	local rc=0
+	local pid
+	while read -r pid; do
+		[ -z "$pid" ] && continue
+		process_one "$pid" || {
+			log "pid ${pid}: processing failed, will retry on next run"
+			rc=1
+		}
+	done < <(coredumpctl --no-legend --no-pager list 2>/dev/null | awk '$9 == "present" {print $5}')
+
+	return "$rc"
+}
+
+exec 9>"$LOCK_FILE"
+if ! flock -n 9; then
+	log "another processor run is already in progress, exiting"
+	exit 0
+fi
+
+main
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index 93bc178060..104eacebe9 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -39,6 +39,10 @@
     - name: Install Postgres from source
       import_tasks: tasks/setup-postgres.yml
 
+    - name: Install PostgreSQL/OrioleDB coredump processing
+      when: stage2 and is_psql_oriole
+      import_tasks: tasks/setup-coredump-processing.yml
+
     - name: Install PgBouncer
       import_tasks: tasks/setup-pgbouncer.yml
       tags:
diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml
new file mode 100644
index 0000000000..6696b44d74
--- /dev/null
+++ b/ansible/tasks/setup-coredump-processing.yml
@@ -0,0 +1,75 @@
+---
+# gdb and binutils (readelf) are only used by the processor script below,
+# which is only installed for OrioleDB images due to is_psql_oriole
+- name: ensure gdb and binutils are installed
+  ansible.builtin.apt:
+    name:
+      - gdb
+      - binutils
+  become: true
+
+- name: Create orioledb-coredumps state/output directories
+  ansible.builtin.file:
+    group: 'root'
+    mode: '0700'
+    owner: 'root'
+    path: "{{ coredump_processing_item }}"
+    state: 'directory'
+  become: true
+  loop:
+    - '/var/lib/orioledb-coredumps'
+    - '/var/lib/orioledb-coredumps/state'
+    - '/var/lib/orioledb-coredumps/diagnostics'
+    - '/var/lib/orioledb-coredumps/quarantine'
+  loop_control:
+    loop_var: 'coredump_processing_item'
+
+- name: copy PostgreSQL/OrioleDB coredump processor script
+  ansible.builtin.copy:
+    dest: '/usr/local/sbin/process-orioledb-coredumps.sh'
+    group: 'root'
+    mode: '0700'
+    owner: 'root'
+    src: 'coredump/process-orioledb-coredumps.sh'
+  become: true
+
+- name: copy GDB extraction commands (matches orioledb/ci/cmds.gdb)
+  ansible.builtin.copy:
+    dest: '/usr/local/sbin/orioledb-coredump-cmds.gdb'
+    group: 'root'
+    mode: '0600'
+    owner: 'root'
+    src: 'coredump/cmds.gdb'
+  become: true
+
+- name: copy PostgreSQL/OrioleDB coredump systemd units
+  ansible.builtin.copy:
+    dest: "/etc/systemd/system/{{ coredump_unit_item }}"
+    group: 'root'
+    mode: '0644'
+    owner: 'root'
+    src: "coredump/{{ coredump_unit_item }}"
+  become: true
+  loop:
+    - 'orioledb-coredump.path'
+    - 'orioledb-coredump.timer'
+    - 'orioledb-coredump.service'
+  loop_control:
+    loop_var: 'coredump_unit_item'
+
+- name: reload systemd for coredump processing units
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+  become: true
+
+- name: enable coredump processing path and timer units
+  ansible.builtin.systemd_service:
+    name: "{{ coredump_enable_item }}"
+    enabled: true
+    state: 'started'
+  become: true
+  loop:
+    - 'orioledb-coredump.path'
+    - 'orioledb-coredump.timer'
+  loop_control:
+    loop_var: 'coredump_enable_item'
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index d629fc49e2..ba1b27e4b5 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1694,3 +1694,57 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     )
 
 
+def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host):
+    """End-to-end processor check: after a real Postgres backend crash, the
+    orioledb-coredump.path-triggered processor must turn the raw core into a
+    small diagnostic bundle and delete the raw core - not leave it sitting in
+    /var/lib/systemd/coredump indefinitely.
+
+    Only runs on images where the coredump processor is installed (OrioleDB
+    builds, per the gated rollout); skipped otherwise.
+    """
+    unit_check = run_ssh_command(
+        host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend"
+    )
+    if "orioledb-coredump.path" not in unit_check["stdout"]:
+        pytest.skip("coredump processor not installed on this AMI (not an OrioleDB build)")
+
+    before_bundles = set(
+        run_ssh_command(
+            host["ssh"], "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f"
+        )["stdout"].splitlines()
+    )
+
+    _crash_a_backend_and_wait_for_recovery(host)
+
+    new_bundle = None
+    for _ in range(30):
+        sleep(2)
+        current = set(
+            run_ssh_command(
+                host["ssh"],
+                "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f",
+            )["stdout"].splitlines()
+        )
+        new = current - before_bundles
+        if new:
+            new_bundle = sorted(new)[0]
+            break
+    assert new_bundle, (
+        "Expected a new diagnostic bundle under /var/lib/orioledb-coredumps/diagnostics "
+        "after the induced backend crash, but none appeared within 60s"
+    )
+
+    bundle_contents = run_ssh_command(host["ssh"], f"sudo cat {new_bundle}")["stdout"]
+    assert "gdb backtrace" in bundle_contents, (
+        f"Expected the diagnostic bundle to contain a gdb backtrace section, got:\n"
+        f"{bundle_contents[:500]}"
+    )
+
+    remaining_cores = run_ssh_command(
+        host["ssh"], "sudo find /var/lib/systemd/coredump -maxdepth 1 -type f"
+    )["stdout"].strip()
+    assert remaining_cores == "", (
+        f"Expected the raw core to be deleted after successful processing, but "
+        f"/var/lib/systemd/coredump still has:\n{remaining_cores}"
+    )

From 239250df10d77d808be1e847b8150634fbe555d9 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 17 Sep 2026 17:12:13 +0200
Subject: [PATCH 04/17] Attempt to fix the gid collision

---
 ansible/tasks/setup-postgres.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index 12bc85a640..7c7bcd2f92 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -292,6 +292,26 @@
     state: 'directory'
   become: true
 
+# Pin the systemd-coredump group/user before installing the package: its
+# postinst creates them with a dynamically-assigned system gid/uid (whatever
+# is next free at that point), which is not reproducible across AMI variants
+# that install a different set of packages before this point - it has
+# collided with the vector group's own pinned gid (989) on some builds.
+- name: add systemd-coredump system group
+  ansible.builtin.group:
+    name: systemd-coredump
+    gid: 986
+    system: yes
+
+- name: add systemd-coredump system user
+  ansible.builtin.user:
+    name: systemd-coredump
+    uid: 986
+    group: systemd-coredump
+    system: yes
+    create_home: false
+    shell: /usr/sbin/nologin
+
 # Required for this (plus CoredumpFilter= in postgresql.service and the
 # storage limits below) to do anything at all: without systemd-coredump,
 # core_pattern stays at the kernel's plain "core" default and none of this

From e3655f2b110fe28b8901bcce1f19c1c2e968c7a5 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Tue, 22 Sep 2026 10:42:17 +0200
Subject: [PATCH 05/17] Fix default value for DefaultLimitCORE

---
 .../coredump/process-orioledb-coredumps.sh    |   2 +-
 .../disable-coredumps-by-default.conf         |   2 +
 ansible/tasks/setup-postgres.yml              |  27 ++++
 nix/packages/postgres-env.nix                 |   8 +-
 testinfra/test_ami_nix.py                     | 116 ++++++++++++++----
 5 files changed, 130 insertions(+), 25 deletions(-)
 create mode 100644 ansible/files/postgresql_config/disable-coredumps-by-default.conf

diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh
index 896e96a380..89a0e650e0 100644
--- a/ansible/files/coredump/process-orioledb-coredumps.sh
+++ b/ansible/files/coredump/process-orioledb-coredumps.sh
@@ -169,7 +169,7 @@ process_one() {
 	local tmpdir
 	tmpdir=$(mktemp -d /tmp/coredump-XXXXXX)
 	chmod 700 "$tmpdir"
-	# suppress shellcheck warning about quoting $tmpdir in the trap command - 
+	# suppress shellcheck warning about quoting $tmpdir in the trap command -
 	# it's correct to quote it, and the trap is evaluated at runtime,
 	# not parse time.
 	# shellcheck disable=SC2064
diff --git a/ansible/files/postgresql_config/disable-coredumps-by-default.conf b/ansible/files/postgresql_config/disable-coredumps-by-default.conf
new file mode 100644
index 0000000000..9f1d52b50a
--- /dev/null
+++ b/ansible/files/postgresql_config/disable-coredumps-by-default.conf
@@ -0,0 +1,2 @@
+[Manager]
+DefaultLimitCORE=0
diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index 7c7bcd2f92..af231c8e59 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -331,6 +331,33 @@
     src: 'files/postgresql_config/coredump.conf'
   become: true
 
+# This AMI's base image ships systemd's DefaultLimitCORE=infinity - without
+# an explicit override, *every* service that doesn't set its own LimitCORE=
+# already gets unlimited-size core dumps, not just postgres (confirmed via
+# `systemctl show -p DefaultLimitCORE` on a real instance). This is the
+# other half of "only PostgreSQL permitted to generate cores" - postgres
+# gets LimitCORE=infinity above, everyone else gets 0 here. [Manager]
+# section changes only take effect on the next full boot (or a manual
+# `systemctl daemon-reexec`), which is fine for a freshly-imaged AMI but
+# means re-testing this on an already-running instance needs a reexec.
+- name: Create a systemd system.conf.d dir for the machine-wide coredump default
+  ansible.builtin.file:
+    group: 'root'
+    mode: '0755'
+    owner: 'root'
+    path: '/etc/systemd/system.conf.d'
+    state: 'directory'
+  become: true
+
+- name: copy machine-wide default core limit (postgres is the only exception)
+  ansible.builtin.copy:
+    dest: '/etc/systemd/system.conf.d/disable-coredumps-by-default.conf'
+    group: 'root'
+    mode: '0644'
+    owner: 'root'
+    src: 'files/postgresql_config/disable-coredumps-by-default.conf'
+  become: true
+
 - name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits
   ansible.builtin.file:
     group: 'root'
diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix
index 7c4153f2b2..0ae009fff7 100644
--- a/nix/packages/postgres-env.nix
+++ b/nix/packages/postgres-env.nix
@@ -20,7 +20,13 @@
             self'.packages."postgresql_${version}_src"
           ]
           ++ lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ]
-          ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ];
+          ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ]
+          # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not
+          # part of the postgresql derivation itself, so its debug output isn't covered by
+          # postgresql_${version}_debug above and has to be pulled in explicitly.
+          ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [
+            self'.legacyPackages."psql_${version}".exts.orioledb.debug
+          ];
         };
     in
     {
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index ba1b27e4b5..8b0f7f3bfc 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1473,20 +1473,46 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
     )
 
 
+def test_default_core_limit_is_disabled_machine_wide(host):
+    """Verify DefaultLimitCORE=0 is configured, so postgresql.service's own
+    LimitCORE=infinity is the *only* exception, not one of many.
+
+    This AMI's base image ships systemd's own DefaultLimitCORE=infinity -
+    without an explicit override here, every other service that doesn't set
+    its own LimitCORE= would also get unlimited-size core dumps, not just
+    postgres. This is the other half of "only PostgreSQL permitted to
+    generate cores" (postgres gets LimitCORE=infinity, everyone else gets 0
+    via this machine-wide default).
+    """
+    result = run_ssh_command(host["ssh"], "systemctl show -p DefaultLimitCORE")
+    assert result["succeeded"], f"systemctl show failed: {result['stderr']}"
+    assert "DefaultLimitCORE=0" in result["stdout"], (
+        f"Expected DefaultLimitCORE=0 machine-wide, got:\n{result['stdout']}"
+    )
+
+
 def test_coredump_storage_directory_root_only(host):
-    """Verify /var/lib/systemd/coredump is root-only, since it can hold
-    core files containing customer data."""
+    """Verify /var/lib/systemd/coredump is owned by root and not writable by
+    anyone else. Systemd's own default for this directory is 0755 (world
+    *readable*, confirmed on a real AMI - only 700/750 were checked here
+    originally, which was an unverified guess, not systemd's actual
+    behavior) - a world-readable directory only leaks core *filenames*, not
+    content, since the actual core files get their own restrictive
+    permissions from systemd-coredump. What actually matters is that no one
+    but root can create/replace/delete files in it."""
     result = run_ssh_command(
         host["ssh"], "stat -c '%a %U:%G' /var/lib/systemd/coredump"
     )
     assert result["succeeded"], f"stat failed: {result['stderr']}"
     mode, owner = result["stdout"].strip().split()
-    assert mode in ("700", "750"), (
-        f"Expected /var/lib/systemd/coredump to be root-only, got mode {mode}"
-    )
     assert owner.startswith("root:"), (
         f"Expected /var/lib/systemd/coredump to be owned by root, got {owner}"
     )
+    group_other_bits = mode[-2:]
+    assert all(int(bit) & 0o2 == 0 for bit in group_other_bits), (
+        f"Expected /var/lib/systemd/coredump to not be group/other-writable, "
+        f"got mode {mode}"
+    )
 
 
 def test_postgres_prestart_does_not_reset_core_limit(host):
@@ -1507,11 +1533,28 @@ def _crash_a_backend_and_wait_for_recovery(host):
     """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's
     normal crash-recovery to bring the instance back on its own
     (Restart=always / auto-reinit, same as production). Returns the crashed
-    backend's pid. Used by tests that need a real, fresh core to appear."""
+    backend's pid. Used by tests that need a real, fresh core to appear.
+
+    A single one-shot `select pg_backend_pid()` query is not safe here: the
+    psql client disconnects the instant it gets its answer, which makes the
+    server-side backend exit normally right after - so by the time `kill`
+    runs, that pid is usually already gone (confirmed in practice: this used
+    to silently signal a dead/nonexistent pid, producing no crash and no
+    core at all). Instead, start a backend that's still busy (via
+    pg_sleep), and look its pid up out-of-band via pg_stat_activity so it's
+    guaranteed to still be alive when we signal it.
+    """
+    run_ssh_command(
+        host["ssh"],
+        "sudo -u postgres psql -U supabase_admin -h localhost -d postgres "
+        "-c 'select pg_sleep(30);' >/dev/null 2>&1 &",
+    )
+    sleep(1)
     backend_pid = run_ssh_command(
         host["ssh"],
         "sudo -u postgres psql -U supabase_admin -h localhost -d postgres "
-        "-tAc 'select pg_backend_pid()'",
+        "-tAc \"select pid from pg_stat_activity where query = 'select pg_sleep(30);' "
+        "and state = 'active' limit 1;\"",
     )["stdout"].strip()
     assert backend_pid.isdigit(), (
         f"Could not resolve a Postgres backend pid: {backend_pid}"
@@ -1551,21 +1594,20 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     )
     before_lines = set(before["stdout"].splitlines())
 
-    # Unrelated process: launch a disposable process outside the
-    # LimitCORE=infinity-scoped postgresql.service and SIGSEGV it - it must
-    # not produce a core, since the default core limit is unchanged for it.
+    # run a systemd process that isn't part of  postgresql.service and check
+    # that it keeps the default core limit (thus not producing a coredump)
     run_ssh_command(
         host["ssh"],
-        "setsid bash -c 'sleep 60 & echo $! > /tmp/unrelated_pid; wait' >/dev/null 2>&1 &",
+        "sudo systemd-run --unit=testinfra-unrelated-crash --collect /bin/sleep 60",
     )
     sleep(1)
-    unrelated_pid = run_ssh_command(host["ssh"], "cat /tmp/unrelated_pid")[
-        "stdout"
-    ].strip()
-    assert unrelated_pid.isdigit(), (
-        f"Could not resolve the disposable unrelated process pid: {unrelated_pid}"
+    unrelated_pid = run_ssh_command(
+        host["ssh"], "systemctl show testinfra-unrelated-crash -p MainPID --value"
+    )["stdout"].strip()
+    assert unrelated_pid.isdigit() and unrelated_pid != "0", (
+        f"Could not resolve the disposable unrelated unit's pid: {unrelated_pid}"
     )
-    run_ssh_command(host["ssh"], f"kill -SEGV {unrelated_pid}")
+    run_ssh_command(host["ssh"], f"sudo kill -SEGV {unrelated_pid}")
     sleep(2)
 
     # Postgres backend: grab a real backend pid and crash it the same way.
@@ -1587,6 +1629,32 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     )
 
 
+def _resolve_postgres_binary(host):
+    """/usr/lib/postgresql/bin/postgres is a Nix wrapper *script* (sets
+    NIX_PGLIBDIR, then execs the real ELF elsewhere in the nix store) - not
+    an executable itself, so readelf/gdb can't be pointed at it directly.
+    Resolve the real binary via the live postmaster's /proc/<pid>/exe,
+    which always shows the actual running ELF regardless of the wrapper -
+    the same real path a crash would report via coredumpctl."""
+    pid = run_ssh_command(
+        host["ssh"], "systemctl show postgresql -p MainPID --value"
+    )["stdout"].strip()
+    return run_ssh_command(host["ssh"], f"sudo readlink -f /proc/{pid}/exe")[
+        "stdout"
+    ].strip()
+
+
+def _resolve_orioledb_lib(host, postgres_binary):
+    """orioledb.so has no fixed path either - it lives in the same nix store
+    derivation as the resolved postgres binary, just under lib/ instead of
+    bin/ (see orioledb_lib_path() in process-orioledb-coredumps.sh, which
+    this mirrors)."""
+    exe_dir = run_ssh_command(
+        host["ssh"], f"dirname \"$(dirname '{postgres_binary}')\""
+    )["stdout"].strip()
+    return f"{exe_dir}/lib/orioledb.so"
+
+
 def _build_id_of(host, path):
     """Return the ELF build-id (hex string) of the binary/library at path, or None."""
     import re
@@ -1613,8 +1681,9 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host):
     """Verify the installed 'postgres' binary's build-id has a matching
     .build-id/xx/yyyy.debug file in the postgres-env debug output, so a
     future coredump-processing GDB session can actually resolve symbols."""
-    build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres")
-    assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres"
+    postgres_binary = _resolve_postgres_binary(host)
+    build_id = _build_id_of(host, postgres_binary)
+    assert build_id, f"Could not read a build-id from {postgres_binary}"
     assert _debug_file_exists_for_build_id(host, build_id), (
         f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
         f".build-id/ matching postgres build-id {build_id} - the shipped "
@@ -1626,7 +1695,7 @@ def test_orioledb_library_build_id_matches_shipped_debug_symbols(host):
     """Verify orioledb.so's build-id has a matching debug file, same as for
     the postgres binary - orioledb.so is built by the same derivation
     (isOrioleDB flavor) so its debug info ships in the same _debug output."""
-    orioledb_so = "/usr/lib/postgresql/lib/orioledb.so"
+    orioledb_so = _resolve_orioledb_lib(host, _resolve_postgres_binary(host))
     exists = run_ssh_command(host["ssh"], f"test -f {orioledb_so} && echo present")
     if "present" not in exists["stdout"]:
         pytest.skip("orioledb.so not present on this AMI (not an OrioleDB build)")
@@ -1657,8 +1726,9 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     """
     import re
 
-    build_id = _build_id_of(host, "/usr/lib/postgresql/bin/postgres")
-    assert build_id, "Could not read a build-id from /usr/lib/postgresql/bin/postgres"
+    postgres_binary = _resolve_postgres_binary(host)
+    build_id = _build_id_of(host, postgres_binary)
+    assert build_id, f"Could not read a build-id from {postgres_binary}"
     prefix, rest = build_id[:2], build_id[2:]
     debug_file = f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
 
@@ -1674,7 +1744,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
         "sudo -u postgres gdb --batch -quiet "
         "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' "
         f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' "
-        "-ex 'file /usr/lib/postgresql/bin/postgres' "
+        f"-ex 'file {postgres_binary}' "
         "-ex 'list main' "
         "2>&1",
     )

From 09cb9cc3d55b0a5232204a14ba98ce14a7414c55 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 09:51:42 +0200
Subject: [PATCH 06/17] Address CI

Still need to address review comments, thus:

[SKIP CI]
---
 .../postgresql_config/postgresql.service      |  9 +++++
 testinfra/test_ami_nix.py                     | 35 +++++++++++--------
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service
index c7ed7cad5d..a4cc3c05a0 100644
--- a/ansible/files/postgresql_config/postgresql.service
+++ b/ansible/files/postgresql_config/postgresql.service
@@ -14,6 +14,8 @@ ExecStartPre=+/usr/local/bin/postgres_prestart.sh
 # Excludes shared_buffers (an anonymous *shared* mapping) from core dumps -
 # without this, every core would include the whole (potentially many-GB)
 # buffer pool. Inherited by every process postgres forks.
+# Even though we do this again below, it's worth to keep this in case
+# we have a non supabase_internal instance.
 CoredumpFilter=private-anonymous elf-headers private-huge
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
@@ -29,6 +31,13 @@ LimitNOFILE=16384
 ReadOnlyPaths=/etc
 InaccessiblePaths=/root -/var/lib/supabase -/var/lib/supabase-admin-agent -/var/cache/supabase-admin-agent -/opt/saltstack -/etc/salt
 AppArmorProfile=-sbpostgres
+# AppArmorProfile= above makes ExecStart's exec an unconfined->confined
+# transition, which the kernel treats as a "secure exec" and resets
+# coredump_filter back to the system default - silently discarding the
+# CoredumpFilter= setting above in the same fork. Re-apply it here from
+# outside the confined process (a plain file write, not an exec, so it
+# can't be reset the same way) once the process is actually running.
+ExecStartPost=+/bin/sh -c 'echo 0x31 > /proc/$MAINPID/coredump_filter'
 {% endif %}
 [Install]
 WantedBy=multi-user.target
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 8b0f7f3bfc..11f010682b 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1460,13 +1460,17 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
     (systemd >= 246), which coredump_filter (inherited across fork(2) and
     preserved across execve(2)) then propagates to everything postgres forks.
     """
-    pid = run_ssh_command(
-        host["ssh"], "systemctl show postgresql -p MainPID --value"
-    )["stdout"].strip()
-    assert pid.isdigit() and pid != "0", f"Could not resolve postgresql.service MainPID: {pid}"
+    pid = run_ssh_command(host["ssh"], "systemctl show postgresql -p MainPID --value")[
+        "stdout"
+    ].strip()
+    assert pid.isdigit() and pid != "0", (
+        f"Could not resolve postgresql.service MainPID: {pid}"
+    )
 
     result = run_ssh_command(host["ssh"], f"cat /proc/{pid}/coredump_filter")
-    assert result["succeeded"], f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
+    assert result["succeeded"], (
+        f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
+    )
     assert result["stdout"].strip() == "31", (
         f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), "
         f"got '{result['stdout'].strip()}'"
@@ -1519,9 +1523,7 @@ def test_postgres_prestart_does_not_reset_core_limit(host):
     """Regression guard: postgres_prestart.sh must never touch 'ulimit -c',
     or it would silently defeat the postgresql.service LimitCORE=infinity
     coredump drop-in."""
-    result = run_ssh_command(
-        host["ssh"], "cat /usr/local/bin/postgres_prestart.sh"
-    )
+    result = run_ssh_command(host["ssh"], "cat /usr/local/bin/postgres_prestart.sh")
     assert result["succeeded"], f"Could not read prestart script: {result['stderr']}"
     assert "ulimit -c" not in result["stdout"], (
         "postgres_prestart.sh must not set 'ulimit -c' - doing so would silently "
@@ -1636,9 +1638,9 @@ def _resolve_postgres_binary(host):
     Resolve the real binary via the live postmaster's /proc/<pid>/exe,
     which always shows the actual running ELF regardless of the wrapper -
     the same real path a crash would report via coredumpctl."""
-    pid = run_ssh_command(
-        host["ssh"], "systemctl show postgresql -p MainPID --value"
-    )["stdout"].strip()
+    pid = run_ssh_command(host["ssh"], "systemctl show postgresql -p MainPID --value")[
+        "stdout"
+    ].strip()
     return run_ssh_command(host["ssh"], f"sudo readlink -f /proc/{pid}/exe")[
         "stdout"
     ].strip()
@@ -1730,7 +1732,9 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     build_id = _build_id_of(host, postgres_binary)
     assert build_id, f"Could not read a build-id from {postgres_binary}"
     prefix, rest = build_id[:2], build_id[2:]
-    debug_file = f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
+    debug_file = (
+        f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
+    )
 
     comp_dir = run_ssh_command(
         host["ssh"],
@@ -1777,11 +1781,14 @@ def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host
         host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend"
     )
     if "orioledb-coredump.path" not in unit_check["stdout"]:
-        pytest.skip("coredump processor not installed on this AMI (not an OrioleDB build)")
+        pytest.skip(
+            "coredump processor not installed on this AMI (not an OrioleDB build)"
+        )
 
     before_bundles = set(
         run_ssh_command(
-            host["ssh"], "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f"
+            host["ssh"],
+            "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f",
         )["stdout"].splitlines()
     )
 

From aa818407eefcc0467fe50a0b18a9c73f43472ae4 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 10:01:56 +0200
Subject: [PATCH 07/17] Address review comments

- Adjust limits both in coredumpctl and in our script
- Clean up some inconsistencies in python code
- Tighten ulimit check for postgres
- Remove `sudo` from setup-coredump-processing
---
 .../coredump/process-orioledb-coredumps.sh     |  2 +-
 .../postgresql_config/coredump-storage.conf    |  6 +++---
 ansible/tasks/setup-coredump-processing.yml    |  7 -------
 testinfra/test_ami_nix.py                      | 18 +++++++++---------
 4 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh
index 89a0e650e0..313fd5fae3 100644
--- a/ansible/files/coredump/process-orioledb-coredumps.sh
+++ b/ansible/files/coredump/process-orioledb-coredumps.sh
@@ -24,7 +24,7 @@ LOCK_FILE=/run/orioledb-coredump.lock
 MAX_ATTEMPTS=3
 EXTRACTION_TIMEOUT=120
 MAX_AGE_DAYS=7
-MAX_TOTAL_BYTES=$((5 * 1024 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse
+MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse
 
 GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug
 GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb
diff --git a/ansible/files/postgresql_config/coredump-storage.conf b/ansible/files/postgresql_config/coredump-storage.conf
index fcd9c211e8..a715ad7c11 100644
--- a/ansible/files/postgresql_config/coredump-storage.conf
+++ b/ansible/files/postgresql_config/coredump-storage.conf
@@ -1,7 +1,7 @@
 [Coredump]
 Storage=external
 Compress=yes
-ProcessSizeMax=8G
-ExternalSizeMax=8G
-MaxUse=10G
+ProcessSizeMax=256M
+ExternalSizeMax=256M
+MaxUse=512M
 KeepFree=5G
diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml
index 6696b44d74..2c33afef86 100644
--- a/ansible/tasks/setup-coredump-processing.yml
+++ b/ansible/tasks/setup-coredump-processing.yml
@@ -6,7 +6,6 @@
     name:
       - gdb
       - binutils
-  become: true
 
 - name: Create orioledb-coredumps state/output directories
   ansible.builtin.file:
@@ -15,7 +14,6 @@
     owner: 'root'
     path: "{{ coredump_processing_item }}"
     state: 'directory'
-  become: true
   loop:
     - '/var/lib/orioledb-coredumps'
     - '/var/lib/orioledb-coredumps/state'
@@ -31,7 +29,6 @@
     mode: '0700'
     owner: 'root'
     src: 'coredump/process-orioledb-coredumps.sh'
-  become: true
 
 - name: copy GDB extraction commands (matches orioledb/ci/cmds.gdb)
   ansible.builtin.copy:
@@ -40,7 +37,6 @@
     mode: '0600'
     owner: 'root'
     src: 'coredump/cmds.gdb'
-  become: true
 
 - name: copy PostgreSQL/OrioleDB coredump systemd units
   ansible.builtin.copy:
@@ -49,7 +45,6 @@
     mode: '0644'
     owner: 'root'
     src: "coredump/{{ coredump_unit_item }}"
-  become: true
   loop:
     - 'orioledb-coredump.path'
     - 'orioledb-coredump.timer'
@@ -60,14 +55,12 @@
 - name: reload systemd for coredump processing units
   ansible.builtin.systemd_service:
     daemon_reload: true
-  become: true
 
 - name: enable coredump processing path and timer units
   ansible.builtin.systemd_service:
     name: "{{ coredump_enable_item }}"
     enabled: true
     state: 'started'
-  become: true
   loop:
     - 'orioledb-coredump.path'
     - 'orioledb-coredump.timer'
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 11f010682b..2b15c6fc8a 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1520,14 +1520,16 @@ def test_coredump_storage_directory_root_only(host):
 
 
 def test_postgres_prestart_does_not_reset_core_limit(host):
-    """Regression guard: postgres_prestart.sh must never touch 'ulimit -c',
-    or it would silently defeat the postgresql.service LimitCORE=infinity
-    coredump drop-in."""
+    """Regression guard: postgres_prestart.sh must never touch 'ulimit' at
+    all, or it would risk silently defeating the postgresql.service
+    LimitCORE=infinity coredump drop-in. Blocking 'ulimit' outright (rather
+    than just 'ulimit -c') is a deliberately conservative stance - if a
+    legitimate future need for ulimit shows up here, revisit this test then."""
     result = run_ssh_command(host["ssh"], "cat /usr/local/bin/postgres_prestart.sh")
     assert result["succeeded"], f"Could not read prestart script: {result['stderr']}"
-    assert "ulimit -c" not in result["stdout"], (
-        "postgres_prestart.sh must not set 'ulimit -c' - doing so would silently "
-        "defeat the postgresql.service coredump drop-in"
+    assert "ulimit" not in result["stdout"], (
+        "postgres_prestart.sh must not use 'ulimit' - doing so risks silently "
+        "defeating the postgresql.service coredump drop-in"
     )
 
 
@@ -1618,9 +1620,7 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     after = run_ssh_command(
         host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
     )
-    new_lines = [
-        line for line in after["stdout"].splitlines() if line not in before_lines
-    ]
+    new_lines = set(after["stdout"].splitlines()) - before_lines
     assert any("postgres" in line for line in new_lines), (
         f"Expected a new postgres core dump after SIGSEGV to backend {backend_pid}, "
         f"but coredumpctl list shows:\n{after['stdout']}"

From 3eb2033c334bcd12085c7b342dee5c7c19ac64cc Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 10:35:32 +0200
Subject: [PATCH 08/17] Make formatter happy

---
 ansible/tasks/setup-coredump-processing.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml
index 2c33afef86..3587b40eb4 100644
--- a/ansible/tasks/setup-coredump-processing.yml
+++ b/ansible/tasks/setup-coredump-processing.yml
@@ -1,7 +1,7 @@
 ---
 # gdb and binutils (readelf) are only used by the processor script below,
 # which is only installed for OrioleDB images due to is_psql_oriole
-- name: ensure gdb and binutils are installed
+- name: Ensure gdb and binutils are installed
   ansible.builtin.apt:
     name:
       - gdb
@@ -22,7 +22,7 @@
   loop_control:
     loop_var: 'coredump_processing_item'
 
-- name: copy PostgreSQL/OrioleDB coredump processor script
+- name: Copy PostgreSQL/OrioleDB coredump processor script
   ansible.builtin.copy:
     dest: '/usr/local/sbin/process-orioledb-coredumps.sh'
     group: 'root'
@@ -30,7 +30,7 @@
     owner: 'root'
     src: 'coredump/process-orioledb-coredumps.sh'
 
-- name: copy GDB extraction commands (matches orioledb/ci/cmds.gdb)
+- name: Copy GDB extraction commands (matches orioledb/ci/cmds.gdb)
   ansible.builtin.copy:
     dest: '/usr/local/sbin/orioledb-coredump-cmds.gdb'
     group: 'root'
@@ -38,7 +38,7 @@
     owner: 'root'
     src: 'coredump/cmds.gdb'
 
-- name: copy PostgreSQL/OrioleDB coredump systemd units
+- name: Copy PostgreSQL/OrioleDB coredump systemd units
   ansible.builtin.copy:
     dest: "/etc/systemd/system/{{ coredump_unit_item }}"
     group: 'root'
@@ -52,11 +52,11 @@
   loop_control:
     loop_var: 'coredump_unit_item'
 
-- name: reload systemd for coredump processing units
+- name: Reload systemd for coredump processing units
   ansible.builtin.systemd_service:
     daemon_reload: true
 
-- name: enable coredump processing path and timer units
+- name: Enable coredump processing path and timer units
   ansible.builtin.systemd_service:
     name: "{{ coredump_enable_item }}"
     enabled: true

From 80d90d475eba358c83f8ef781ddf3a0e82f0254b Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 13:42:11 +0200
Subject: [PATCH 09/17] Adjust coredump timer and attempt to fix
 coredump_filter

---
 .../files/coredump/orioledb-coredump.timer    |  5 +-
 .../postgresql_config/sbpostgres_apparmor     | 11 +++
 testinfra/test_ami_nix.py                     | 79 ++++++++++++++-----
 3 files changed, 75 insertions(+), 20 deletions(-)

diff --git a/ansible/files/coredump/orioledb-coredump.timer b/ansible/files/coredump/orioledb-coredump.timer
index 15d332ba66..393bb70fd7 100644
--- a/ansible/files/coredump/orioledb-coredump.timer
+++ b/ansible/files/coredump/orioledb-coredump.timer
@@ -2,8 +2,9 @@
 Description=Periodic fallback sweep for PostgreSQL/OrioleDB core dumps
 
 [Timer]
-OnBootSec=10min
-OnUnitActiveSec=10min
+OnBootSec=60min
+RandomizedDelaySec=15m
+OnUnitActiveSec=15min
 Unit=orioledb-coredump.service
 
 [Install]
diff --git a/ansible/files/postgresql_config/sbpostgres_apparmor b/ansible/files/postgresql_config/sbpostgres_apparmor
index ea59999849..b2f5330c9a 100644
--- a/ansible/files/postgresql_config/sbpostgres_apparmor
+++ b/ansible/files/postgresql_config/sbpostgres_apparmor
@@ -39,6 +39,17 @@ profile sbpostgres flags=(attach_disconnected) {
     # All capabilities
     capability,
 
+    # postgresql.service's ExecStartPost re-applies CoredumpFilter= by writing
+    # to /proc/$MAINPID/coredump_filter from a separate (also sbpostgres-
+    # confined) process, after the ExecStart exec's unconfined->confined
+    # transition resets it. Writing another process's coredump_filter goes
+    # through ptrace_may_access() in the kernel, which AppArmor mediates as
+    # its own "ptrace" rule class - separate from `capability` (all caps,
+    # above) and from the plain file rules below. Without this, that write is
+    # denied and coredump_filter silently stays at the OS default (0x33,
+    # includes shared_buffers) instead of 0x31.
+    ptrace (trace, read) peer=sbpostgres,
+
     # Libraries
     /lib/aarch64-linux-gnu/*.so* mr,
     /usr/lib/aarch64-linux-gnu/*.so* mr,
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 2b15c6fc8a..fa193930b8 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1471,10 +1471,28 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
     assert result["succeeded"], (
         f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
     )
-    assert result["stdout"].strip() == "31", (
-        f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), "
-        f"got '{result['stdout'].strip()}'"
-    )
+    if result["stdout"].strip() != "31":
+        # Most likely cause: the ExecStartPost that re-applies 0x31 after the
+        # ExecStart exec's unconfined->confined AppArmor transition resets it
+        # writes to *another* process's coredump_filter, which the kernel
+        # gates via ptrace_may_access() - denied by AppArmor's separate
+        # "ptrace" mediation class if the profile lacks a matching rule.
+        # Surface that denial directly instead of leaving the next person to
+        # rediscover it by hand.
+        denials = run_ssh_command(
+            host["ssh"],
+            "sudo dmesg | grep -i apparmor | grep -iE 'denied|ptrace' | tail -20",
+        )["stdout"]
+        journal = run_ssh_command(
+            host["ssh"],
+            "sudo journalctl -u postgresql --no-pager | grep -i coredump_filter -B2 -A2",
+        )["stdout"]
+        raise AssertionError(
+            f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), "
+            f"got '{result['stdout'].strip()}'.\n"
+            f"--- dmesg apparmor denied/ptrace lines ---\n{denials}\n"
+            f"--- journalctl -u postgresql (coredump_filter context) ---\n{journal}"
+        )
 
 
 def test_default_core_limit_is_disabled_machine_wide(host):
@@ -1625,9 +1643,21 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
         f"Expected a new postgres core dump after SIGSEGV to backend {backend_pid}, "
         f"but coredumpctl list shows:\n{after['stdout']}"
     )
-    assert not any(unrelated_pid in line for line in new_lines), (
-        f"Unrelated process {unrelated_pid} should not have produced a core dump:\n"
-        f"{after['stdout']}"
+
+    # coredumpctl always logs a crash *entry* for any SIGSEGV it observes,
+    # regardless of whether a core file was actually captured - the COREFILE
+    # column is what distinguishes "captured" (a path/size) from "missing"
+    # (recorded, but no core saved). A new line existing for unrelated_pid is
+    # therefore not itself a failure; only a captured core is.
+    unrelated_corefile = None
+    for line in new_lines:
+        fields = line.split()
+        if len(fields) >= 9 and fields[4] == unrelated_pid:
+            unrelated_corefile = fields[8]
+            break
+    assert unrelated_corefile in (None, "missing"), (
+        f"Unrelated process {unrelated_pid} should not have produced a captured "
+        f"core dump (COREFILE={unrelated_corefile}):\n{after['stdout']}"
     )
 
 
@@ -1658,14 +1688,18 @@ def _resolve_orioledb_lib(host, postgres_binary):
 
 
 def _build_id_of(host, path):
-    """Return the ELF build-id (hex string) of the binary/library at path, or None."""
+    """Return (build_id, readelf_output) for the binary/library at path.
+    build_id is None if readelf failed or found no build-id note - the raw
+    output is returned alongside so callers can report *why* on failure
+    instead of a bare 'no build-id' with no evidence."""
     import re
 
-    result = run_ssh_command(host["ssh"], f"readelf -n {path} 2>/dev/null")
+    result = run_ssh_command(host["ssh"], f"readelf -n {path}")
+    output = result["stdout"] + result["stderr"]
     if not result["succeeded"]:
-        return None
-    match = re.search(r"Build ID:\s*([0-9a-f]+)", result["stdout"])
-    return match.group(1) if match else None
+        return None, output
+    match = re.search(r"Build ID:\s*([0-9a-f]+)", output)
+    return (match.group(1) if match else None), output
 
 
 def _debug_file_exists_for_build_id(host, build_id):
@@ -1684,8 +1718,11 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host):
     .build-id/xx/yyyy.debug file in the postgres-env debug output, so a
     future coredump-processing GDB session can actually resolve symbols."""
     postgres_binary = _resolve_postgres_binary(host)
-    build_id = _build_id_of(host, postgres_binary)
-    assert build_id, f"Could not read a build-id from {postgres_binary}"
+    build_id, readelf_output = _build_id_of(host, postgres_binary)
+    assert build_id, (
+        f"Could not read a build-id from {postgres_binary}, readelf -n output:\n"
+        f"{readelf_output}"
+    )
     assert _debug_file_exists_for_build_id(host, build_id), (
         f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
         f".build-id/ matching postgres build-id {build_id} - the shipped "
@@ -1702,8 +1739,11 @@ def test_orioledb_library_build_id_matches_shipped_debug_symbols(host):
     if "present" not in exists["stdout"]:
         pytest.skip("orioledb.so not present on this AMI (not an OrioleDB build)")
 
-    build_id = _build_id_of(host, orioledb_so)
-    assert build_id, f"Could not read a build-id from {orioledb_so}"
+    build_id, readelf_output = _build_id_of(host, orioledb_so)
+    assert build_id, (
+        f"Could not read a build-id from {orioledb_so}, readelf -n output:\n"
+        f"{readelf_output}"
+    )
     assert _debug_file_exists_for_build_id(host, build_id), (
         f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
         f".build-id/ matching orioledb.so build-id {build_id}"
@@ -1729,8 +1769,11 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     import re
 
     postgres_binary = _resolve_postgres_binary(host)
-    build_id = _build_id_of(host, postgres_binary)
-    assert build_id, f"Could not read a build-id from {postgres_binary}"
+    build_id, readelf_output = _build_id_of(host, postgres_binary)
+    assert build_id, (
+        f"Could not read a build-id from {postgres_binary}, readelf -n output:\n"
+        f"{readelf_output}"
+    )
     prefix, rest = build_id[:2], build_id[2:]
     debug_file = (
         f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"

From 63e34bfdbfa95866292012ece80f6c8b541db17f Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 14:44:59 +0200
Subject: [PATCH 10/17] Another attempt to fix coredump filters; limit to
 orioledb

---
 .../files/coredump/orioledb-coredump.timer    |   2 +-
 .../postgresql_config/postgresql.service      |   4 +
 .../postgresql_config/sbpostgres_apparmor     |  11 --
 ansible/tasks/setup-postgres.yml              | 159 +++++++++---------
 ansible/tasks/setup-tuned.yml                 |  19 +++
 testinfra/test_ami_nix.py                     |  36 +++-
 6 files changed, 134 insertions(+), 97 deletions(-)

diff --git a/ansible/files/coredump/orioledb-coredump.timer b/ansible/files/coredump/orioledb-coredump.timer
index 393bb70fd7..7b1a25f419 100644
--- a/ansible/files/coredump/orioledb-coredump.timer
+++ b/ansible/files/coredump/orioledb-coredump.timer
@@ -3,7 +3,7 @@ Description=Periodic fallback sweep for PostgreSQL/OrioleDB core dumps
 
 [Timer]
 OnBootSec=60min
-RandomizedDelaySec=15m
+RandomizedDelaySec=15min
 OnUnitActiveSec=15min
 Unit=orioledb-coredump.service
 
diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service
index a4cc3c05a0..b822fa2ffb 100644
--- a/ansible/files/postgresql_config/postgresql.service
+++ b/ansible/files/postgresql_config/postgresql.service
@@ -11,12 +11,14 @@ Type=notify
 User=postgres
 ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql
 ExecStartPre=+/usr/local/bin/postgres_prestart.sh
+{% if is_psql_oriole is defined and is_psql_oriole %}
 # Excludes shared_buffers (an anonymous *shared* mapping) from core dumps -
 # without this, every core would include the whole (potentially many-GB)
 # buffer pool. Inherited by every process postgres forks.
 # Even though we do this again below, it's worth to keep this in case
 # we have a non supabase_internal instance.
 CoredumpFilter=private-anonymous elf-headers private-huge
+{% endif %}
 ExecReload=/bin/kill -HUP $MAINPID
 KillMode=mixed
 KillSignal=SIGINT
@@ -31,6 +33,7 @@ LimitNOFILE=16384
 ReadOnlyPaths=/etc
 InaccessiblePaths=/root -/var/lib/supabase -/var/lib/supabase-admin-agent -/var/cache/supabase-admin-agent -/opt/saltstack -/etc/salt
 AppArmorProfile=-sbpostgres
+{% if is_psql_oriole is defined and is_psql_oriole %}
 # AppArmorProfile= above makes ExecStart's exec an unconfined->confined
 # transition, which the kernel treats as a "secure exec" and resets
 # coredump_filter back to the system default - silently discarding the
@@ -39,5 +42,6 @@ AppArmorProfile=-sbpostgres
 # can't be reset the same way) once the process is actually running.
 ExecStartPost=+/bin/sh -c 'echo 0x31 > /proc/$MAINPID/coredump_filter'
 {% endif %}
+{% endif %}
 [Install]
 WantedBy=multi-user.target
diff --git a/ansible/files/postgresql_config/sbpostgres_apparmor b/ansible/files/postgresql_config/sbpostgres_apparmor
index b2f5330c9a..ea59999849 100644
--- a/ansible/files/postgresql_config/sbpostgres_apparmor
+++ b/ansible/files/postgresql_config/sbpostgres_apparmor
@@ -39,17 +39,6 @@ profile sbpostgres flags=(attach_disconnected) {
     # All capabilities
     capability,
 
-    # postgresql.service's ExecStartPost re-applies CoredumpFilter= by writing
-    # to /proc/$MAINPID/coredump_filter from a separate (also sbpostgres-
-    # confined) process, after the ExecStart exec's unconfined->confined
-    # transition resets it. Writing another process's coredump_filter goes
-    # through ptrace_may_access() in the kernel, which AppArmor mediates as
-    # its own "ptrace" rule class - separate from `capability` (all caps,
-    # above) and from the plain file rules below. Without this, that write is
-    # denied and coredump_filter silently stays at the OS default (0x33,
-    # includes shared_buffers) instead of 0x31.
-    ptrace (trace, read) peer=sbpostgres,
-
     # Libraries
     /lib/aarch64-linux-gnu/*.so* mr,
     /usr/lib/aarch64-linux-gnu/*.so* mr,
diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index af231c8e59..6022a74e73 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -292,89 +292,92 @@
     state: 'directory'
   become: true
 
-# Pin the systemd-coredump group/user before installing the package: its
-# postinst creates them with a dynamically-assigned system gid/uid (whatever
-# is next free at that point), which is not reproducible across AMI variants
-# that install a different set of packages before this point - it has
-# collided with the vector group's own pinned gid (989) on some builds.
-- name: add systemd-coredump system group
-  ansible.builtin.group:
-    name: systemd-coredump
-    gid: 986
-    system: yes
-
-- name: add systemd-coredump system user
-  ansible.builtin.user:
-    name: systemd-coredump
-    uid: 986
-    group: systemd-coredump
-    system: yes
-    create_home: false
-    shell: /usr/sbin/nologin
-
-# Required for this (plus CoredumpFilter= in postgresql.service and the
-# storage limits below) to do anything at all: without systemd-coredump,
-# core_pattern stays at the kernel's plain "core" default and none of this
-# gets exercised (confirmed on a local test VM). Unconditional - every
-# Postgres flavor's capture depends on it, not just OrioleDB.
-- name: ensure systemd-coredump is installed
-  ansible.builtin.apt:
-    name: systemd-coredump
-  become: true
+# Coredump capture is OrioleDB-only - see docs/plans/2026-09-07-orioledb-coredump-capture.md.
+- name: Set up PostgreSQL coredump capture (OrioleDB only)
+  when: is_psql_oriole
+  block:
+    # Pin the systemd-coredump group/user before installing the package: its
+    # postinst creates them with a dynamically-assigned system gid/uid (whatever
+    # is next free at that point), which is not reproducible across AMI variants
+    # that install a different set of packages before this point - it has
+    # collided with the vector group's own pinned gid (989) on some builds.
+    - name: add systemd-coredump system group
+      ansible.builtin.group:
+        name: systemd-coredump
+        gid: 986
+        system: yes
 
-- name: copy PostgreSQL coredump systemd drop-in
-  ansible.builtin.copy:
-    dest: '/etc/systemd/system/postgresql.service.d/coredump.conf'
-    group: 'root'
-    mode: '0644'
-    owner: 'root'
-    src: 'files/postgresql_config/coredump.conf'
-  become: true
+    - name: add systemd-coredump system user
+      ansible.builtin.user:
+        name: systemd-coredump
+        uid: 986
+        group: systemd-coredump
+        system: yes
+        create_home: false
+        shell: /usr/sbin/nologin
 
-# This AMI's base image ships systemd's DefaultLimitCORE=infinity - without
-# an explicit override, *every* service that doesn't set its own LimitCORE=
-# already gets unlimited-size core dumps, not just postgres (confirmed via
-# `systemctl show -p DefaultLimitCORE` on a real instance). This is the
-# other half of "only PostgreSQL permitted to generate cores" - postgres
-# gets LimitCORE=infinity above, everyone else gets 0 here. [Manager]
-# section changes only take effect on the next full boot (or a manual
-# `systemctl daemon-reexec`), which is fine for a freshly-imaged AMI but
-# means re-testing this on an already-running instance needs a reexec.
-- name: Create a systemd system.conf.d dir for the machine-wide coredump default
-  ansible.builtin.file:
-    group: 'root'
-    mode: '0755'
-    owner: 'root'
-    path: '/etc/systemd/system.conf.d'
-    state: 'directory'
-  become: true
+    # Required for this (plus CoredumpFilter= in postgresql.service and the
+    # storage limits below) to do anything at all: without systemd-coredump,
+    # core_pattern stays at the kernel's plain "core" default and none of this
+    # gets exercised (confirmed on a local test VM).
+    - name: ensure systemd-coredump is installed
+      ansible.builtin.apt:
+        name: systemd-coredump
+      become: true
 
-- name: copy machine-wide default core limit (postgres is the only exception)
-  ansible.builtin.copy:
-    dest: '/etc/systemd/system.conf.d/disable-coredumps-by-default.conf'
-    group: 'root'
-    mode: '0644'
-    owner: 'root'
-    src: 'files/postgresql_config/disable-coredumps-by-default.conf'
-  become: true
+    - name: copy PostgreSQL coredump systemd drop-in
+      ansible.builtin.copy:
+        dest: '/etc/systemd/system/postgresql.service.d/coredump.conf'
+        group: 'root'
+        mode: '0644'
+        owner: 'root'
+        src: 'files/postgresql_config/coredump.conf'
+      become: true
 
-- name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits
-  ansible.builtin.file:
-    group: 'root'
-    mode: '0755'
-    owner: 'root'
-    path: '/etc/systemd/coredump.conf.d'
-    state: 'directory'
-  become: true
+    # This AMI's base image ships systemd's DefaultLimitCORE=infinity - without
+    # an explicit override, *every* service that doesn't set its own LimitCORE=
+    # already gets unlimited-size core dumps, not just postgres (confirmed via
+    # `systemctl show -p DefaultLimitCORE` on a real instance). This is the
+    # other half of "only PostgreSQL permitted to generate cores" - postgres
+    # gets LimitCORE=infinity above, everyone else gets 0 here. [Manager]
+    # section changes only take effect on the next full boot (or a manual
+    # `systemctl daemon-reexec`), which is fine for a freshly-imaged AMI but
+    # means re-testing this on an already-running instance needs a reexec.
+    - name: Create a systemd system.conf.d dir for the machine-wide coredump default
+      ansible.builtin.file:
+        group: 'root'
+        mode: '0755'
+        owner: 'root'
+        path: '/etc/systemd/system.conf.d'
+        state: 'directory'
+      become: true
 
-- name: copy systemd-coredump storage limits
-  ansible.builtin.copy:
-    dest: '/etc/systemd/coredump.conf.d/postgres.conf'
-    group: 'root'
-    mode: '0644'
-    owner: 'root'
-    src: 'files/postgresql_config/coredump-storage.conf'
-  become: true
+    - name: copy machine-wide default core limit (postgres is the only exception)
+      ansible.builtin.copy:
+        dest: '/etc/systemd/system.conf.d/disable-coredumps-by-default.conf'
+        group: 'root'
+        mode: '0644'
+        owner: 'root'
+        src: 'files/postgresql_config/disable-coredumps-by-default.conf'
+      become: true
+
+    - name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits
+      ansible.builtin.file:
+        group: 'root'
+        mode: '0755'
+        owner: 'root'
+        path: '/etc/systemd/coredump.conf.d'
+        state: 'directory'
+      become: true
+
+    - name: copy systemd-coredump storage limits
+      ansible.builtin.copy:
+        dest: '/etc/systemd/coredump.conf.d/postgres.conf'
+        group: 'root'
+        mode: '0644'
+        owner: 'root'
+        src: 'files/postgresql_config/coredump-storage.conf'
+      become: true
 
 - name: Ensure PostgreSQL starts after tuned
   become: true
diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml
index 1d4fb16ce7..ccd90174f3 100644
--- a/ansible/tasks/setup-tuned.yml
+++ b/ansible/tasks/setup-tuned.yml
@@ -107,6 +107,25 @@
           value: '1048576'
         - option: 'fs.file-max'
           value: '312139770'
+        - option: 'fs.suid_dumpable'
+          # postgresql.service's AppArmorProfile= makes ExecStart's exec an
+          # unconfined->confined transition, which the kernel treats as a
+          # privilege-changing ("secure") exec. With the kernel default (0,
+          # SUID_DUMP_DISABLE), such processes are made entirely non-dumpable
+          # - no core is ever produced on crash, and a non-dumpable process's
+          # coredump_filter can't be rewritten afterwards either (silently
+          # discarding postgresql.service's own re-apply of it), which is why
+          # postgres crashes were intermittently producing no core at all, or
+          # a core with the un-filtered default (0x33, includes
+          # shared_buffers) instead of the requested 0x31. 2 (suidsafe) only
+          # restores dumpability for privilege-changed processes when
+          # core_pattern is a pipe or fully-qualified path (true here once
+          # systemd-coredump is installed) - it does not affect any process
+          # on a machine where core_pattern is left at its plain default.
+          # This AMI is single-purpose (postgres only); there's no other
+          # privilege-transitioning process here with sensitive memory to
+          # protect from being dumped.
+          value: '2'
         - option: 'kernel.panic_on_oops'
           value: '1'
         - option: 'kernel.sched_autogroup_enabled'
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index fa193930b8..91557a7116 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1409,6 +1409,19 @@ def test_apparmor_denies_access_to_sensitive_paths(host):
         print(f"Confirmed: access to {test_file} denied by AppArmor")
 
 
+def _skip_if_not_orioledb(host):
+    """Coredump capture/processing is gated to OrioleDB builds only; skip
+    capture-layer tests on vanilla 15/17 AMIs where none of this is
+    installed."""
+    unit_check = run_ssh_command(
+        host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend"
+    )
+    if "orioledb-coredump.path" not in unit_check["stdout"]:
+        pytest.skip(
+            "coredump capture not installed on this AMI (not an OrioleDB build)"
+        )
+
+
 def test_postgresql_service_allows_unlimited_core_dumps(host):
     """Verify the postgresql.service coredump drop-in sets LimitCORE=infinity.
 
@@ -1416,6 +1429,7 @@ def test_postgresql_service_allows_unlimited_core_dumps(host):
     systemd.service.d drop-in), not enabled machine-wide, so other services
     must keep the default core limit.
     """
+    _skip_if_not_orioledb(host)
     result = run_ssh_command(host["ssh"], "systemctl show postgresql -p LimitCORE")
     assert result["succeeded"], f"systemctl show failed: {result['stderr']}"
     assert "LimitCORE=infinity" in result["stdout"], (
@@ -1427,6 +1441,7 @@ def test_coredump_storage_limits_configured(host):
     """Verify /etc/systemd/coredump.conf.d/postgres.conf sets conservative,
     bounded storage limits for the systemd-coredump storage that backs
     Postgres core capture."""
+    _skip_if_not_orioledb(host)
     result = run_ssh_command(
         host["ssh"], "cat /etc/systemd/coredump.conf.d/postgres.conf"
     )
@@ -1460,6 +1475,7 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
     (systemd >= 246), which coredump_filter (inherited across fork(2) and
     preserved across execve(2)) then propagates to everything postgres forks.
     """
+    _skip_if_not_orioledb(host)
     pid = run_ssh_command(host["ssh"], "systemctl show postgresql -p MainPID --value")[
         "stdout"
     ].strip()
@@ -1472,13 +1488,16 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
         f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
     )
     if result["stdout"].strip() != "31":
-        # Most likely cause: the ExecStartPost that re-applies 0x31 after the
-        # ExecStart exec's unconfined->confined AppArmor transition resets it
-        # writes to *another* process's coredump_filter, which the kernel
-        # gates via ptrace_may_access() - denied by AppArmor's separate
-        # "ptrace" mediation class if the profile lacks a matching rule.
-        # Surface that denial directly instead of leaving the next person to
-        # rediscover it by hand.
+        # Confirmed root cause (empty dmesg output here on a real run ruled
+        # out AppArmor/ptrace as the culprit): fs.suid_dumpable=0 (the kernel
+        # default) makes the ExecStart exec's unconfined->confined AppArmor
+        # transition ("secure exec") mark the process fully non-dumpable, and
+        # a non-dumpable process's coredump_filter can't be rewritten
+        # afterwards either - silently discarding postgresql.service's own
+        # ExecStartPost re-apply of it. See fs.suid_dumpable=2 in
+        # ansible/tasks/setup-tuned.yml. Kept capturing dmesg/journalctl here
+        # so a *different* regression shows real evidence instead of a bare
+        # "got 33" again.
         denials = run_ssh_command(
             host["ssh"],
             "sudo dmesg | grep -i apparmor | grep -iE 'denied|ptrace' | tail -20",
@@ -1506,6 +1525,7 @@ def test_default_core_limit_is_disabled_machine_wide(host):
     generate cores" (postgres gets LimitCORE=infinity, everyone else gets 0
     via this machine-wide default).
     """
+    _skip_if_not_orioledb(host)
     result = run_ssh_command(host["ssh"], "systemctl show -p DefaultLimitCORE")
     assert result["succeeded"], f"systemctl show failed: {result['stderr']}"
     assert "DefaultLimitCORE=0" in result["stdout"], (
@@ -1522,6 +1542,7 @@ def test_coredump_storage_directory_root_only(host):
     content, since the actual core files get their own restrictive
     permissions from systemd-coredump. What actually matters is that no one
     but root can create/replace/delete files in it."""
+    _skip_if_not_orioledb(host)
     result = run_ssh_command(
         host["ssh"], "stat -c '%a %U:%G' /var/lib/systemd/coredump"
     )
@@ -1611,6 +1632,7 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     (Restart=always / auto-reinit), the same as in production - it does not
     reinstall data or otherwise reset the shared test instance.
     """
+    _skip_if_not_orioledb(host)
     before = run_ssh_command(
         host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
     )

From ac2451cdd72fafa316d3d330ca9a31e65c51cf43 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 15:16:28 +0200
Subject: [PATCH 11/17] Fix is_psql_orioledb

---
 ansible/tasks/setup-postgres.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml
index 6022a74e73..adc0b9d747 100644
--- a/ansible/tasks/setup-postgres.yml
+++ b/ansible/tasks/setup-postgres.yml
@@ -1,3 +1,10 @@
+# is_psql_oriole is also (re)computed in stage2-setup-postgres.yml, but this
+# file's coredump block below runs in both stage1 and stage2, so the fact
+# needs to exist before stage2-only tasks have had a chance to set it.
+- name: Determine postgres flavor from psql_version
+  ansible.builtin.set_fact:
+    is_psql_oriole: "{{ psql_version is defined and psql_version == 'psql_orioledb-17' }}"
+
 - name: execute stage1 tasks
   when: not stage2
   block:

From 9b5554a6abb77cce7fc623af83b61bf461d000ac Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 17:21:21 +0200
Subject: [PATCH 12/17] another attempt at fixing this

---
 .../postgresql_config/postgresql.service      |  2 +-
 .../postgresql_config/sbpostgres_apparmor     |  8 +++
 ansible/tasks/setup-coredump-processing.yml   | 22 +++++++
 testinfra/test_ami_nix.py                     | 64 +++++++++++++++++++
 4 files changed, 95 insertions(+), 1 deletion(-)

diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service
index b822fa2ffb..7a50b4956f 100644
--- a/ansible/files/postgresql_config/postgresql.service
+++ b/ansible/files/postgresql_config/postgresql.service
@@ -40,7 +40,7 @@ AppArmorProfile=-sbpostgres
 # CoredumpFilter= setting above in the same fork. Re-apply it here from
 # outside the confined process (a plain file write, not an exec, so it
 # can't be reset the same way) once the process is actually running.
-ExecStartPost=+/bin/sh -c 'echo 0x31 > /proc/$MAINPID/coredump_filter'
+ExecStartPost=+/bin/sh -c 'echo 0x31 > /proc/$MAINPID/coredump_filter 2>&1; logger -t coredump-filter-post "postmaster $MAINPID coredump_filter now $(cat /proc/$MAINPID/coredump_filter 2>&1)"'
 {% endif %}
 {% endif %}
 [Install]
diff --git a/ansible/files/postgresql_config/sbpostgres_apparmor b/ansible/files/postgresql_config/sbpostgres_apparmor
index ea59999849..9f0527dd91 100644
--- a/ansible/files/postgresql_config/sbpostgres_apparmor
+++ b/ansible/files/postgresql_config/sbpostgres_apparmor
@@ -13,6 +13,14 @@ profile sbpostgres flags=(attach_disconnected) {
     /nix/store/*/bin/.postgres-wrapped mr,
     /nix/store/*/bin/.postgres-wrapped ix,
 
+    # Without an explicit ptrace rule, AppArmor's default-deny applies to
+    # this confined process being the *target* of ptrace-class accesses from
+    # anyone else - including unconfined root. That access class also covers
+    # plain writes to /proc/<pid>/coredump_filter (kernel routes it through
+    # ptrace_may_access()), which is why postgresql.service's ExecStartPost
+    # re-apply of CoredumpFilter= was silently failing to stick. Allow it.
+    ptrace (read, trace, readby, tracedby),
+
     # Wide open permissions for parent - to be tuned
     # some of this is managed via systemd sandboxing already
     /** rw,
diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml
index 3587b40eb4..1bb3a3020a 100644
--- a/ansible/tasks/setup-coredump-processing.yml
+++ b/ansible/tasks/setup-coredump-processing.yml
@@ -1,4 +1,26 @@
 ---
+# Sets the kernel's *default* coredump_filter (normally 0x33) to 0x31,
+# excluding shared_buffers (an anonymous shared mapping, can be many GB)
+# from every core on the machine. This is a stronger mechanism than
+# postgresql.service's own CoredumpFilter=/ExecStartPost, both of which only
+# affect the postmaster after it's already running and get silently undone
+# by the AppArmorProfile= unconfined->confined "secure exec" transition,
+# which resets coredump_filter back to whatever the kernel considers
+# "default" - so changing what "default" means here fixes it regardless of
+# that reset, for every process on the machine, from the very first fork of
+# PID 1 onward. Machine is single-purpose (postgres only), so a global
+# default is fine.
+- name: Set kernel default coredump_filter via GRUB
+  ansible.builtin.copy:
+    content: |
+      GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} coredump_filter=0x31"
+    dest: '/etc/default/grub.d/60-coredump-filter.cfg'
+    group: 'root'
+    mode: '0644'
+    owner: 'root'
+  notify:
+    - Regenerate grub configuration
+
 # gdb and binutils (readelf) are only used by the processor script below,
 # which is only installed for OrioleDB images due to is_psql_oriole
 - name: Ensure gdb and binutils are installed
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 91557a7116..4612634506 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1422,6 +1422,18 @@ def _skip_if_not_orioledb(host):
         )
 
 
+def _skip_if_orioledb(host):
+    """Inverse of _skip_if_not_orioledb: some tests assert the *absence* of
+    coredump capture and should only run on vanilla 15/17 AMIs."""
+    unit_check = run_ssh_command(
+        host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend"
+    )
+    if "orioledb-coredump.path" in unit_check["stdout"]:
+        pytest.skip(
+            "coredump capture installed on this AMI (this is an OrioleDB build)"
+        )
+
+
 def test_postgresql_service_allows_unlimited_core_dumps(host):
     """Verify the postgresql.service coredump drop-in sets LimitCORE=infinity.
 
@@ -1683,6 +1695,56 @@ def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(hos
     )
 
 
+def test_vanilla_postgres_crash_does_not_capture_core(host):
+    """Mirror image of
+    test_postgres_backend_crash_produces_core_but_unrelated_process_does_not:
+    on vanilla 15/17 AMIs (no coredump capture installed), a segfaulted
+    Postgres backend must NOT produce a captured core, same as any other
+    process - this branch's coredump capture is OrioleDB-only, so vanilla
+    AMIs must come out of a crash exactly as they did before this work.
+    """
+    _skip_if_orioledb(host)
+    before = run_ssh_command(
+        host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
+    )
+    before_lines = set(before["stdout"].splitlines())
+
+    run_ssh_command(
+        host["ssh"],
+        "sudo systemd-run --unit=testinfra-unrelated-crash --collect /bin/sleep 60",
+    )
+    sleep(1)
+    unrelated_pid = run_ssh_command(
+        host["ssh"], "systemctl show testinfra-unrelated-crash -p MainPID --value"
+    )["stdout"].strip()
+    assert unrelated_pid.isdigit() and unrelated_pid != "0", (
+        f"Could not resolve the disposable unrelated unit's pid: {unrelated_pid}"
+    )
+    run_ssh_command(host["ssh"], f"sudo kill -SEGV {unrelated_pid}")
+    sleep(2)
+
+    backend_pid = _crash_a_backend_and_wait_for_recovery(host)
+
+    after = run_ssh_command(
+        host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true"
+    )
+    new_lines = set(after["stdout"].splitlines()) - before_lines
+
+    crashed = [(backend_pid, "postgres backend"), (unrelated_pid, "unrelated process")]
+    for pid, label in crashed:
+        corefile = None
+        for line in new_lines:
+            fields = line.split()
+            if len(fields) >= 9 and fields[4] == pid:
+                corefile = fields[8]
+                break
+        assert corefile in (None, "missing"), (
+            f"On a vanilla (non-OrioleDB) AMI, the {label} (pid {pid}) should not "
+            f"have produced a captured core dump (COREFILE={corefile}):\n"
+            f"{after['stdout']}"
+        )
+
+
 def _resolve_postgres_binary(host):
     """/usr/lib/postgresql/bin/postgres is a Nix wrapper *script* (sets
     NIX_PGLIBDIR, then execs the real ELF elsewhere in the nix store) - not
@@ -1739,6 +1801,7 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host):
     """Verify the installed 'postgres' binary's build-id has a matching
     .build-id/xx/yyyy.debug file in the postgres-env debug output, so a
     future coredump-processing GDB session can actually resolve symbols."""
+    _skip_if_not_orioledb(host)
     postgres_binary = _resolve_postgres_binary(host)
     build_id, readelf_output = _build_id_of(host, postgres_binary)
     assert build_id, (
@@ -1790,6 +1853,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     """
     import re
 
+    _skip_if_not_orioledb(host)
     postgres_binary = _resolve_postgres_binary(host)
     build_id, readelf_output = _build_id_of(host, postgres_binary)
     assert build_id, (

From 68bdfc42a66a036b9520e6743064d9986a119530 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 20:56:25 +0200
Subject: [PATCH 13/17] Strip the number properly

---
 testinfra/test_ami_nix.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 4612634506..f2bc18fdd8 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1499,15 +1499,18 @@ def test_postgres_coredump_filter_excludes_shared_buffers(host):
     assert result["succeeded"], (
         f"Could not read coredump_filter for pid {pid}: {result['stderr']}"
     )
-    if result["stdout"].strip() != "31":
-        # Confirmed root cause (empty dmesg output here on a real run ruled
-        # out AppArmor/ptrace as the culprit): fs.suid_dumpable=0 (the kernel
-        # default) makes the ExecStart exec's unconfined->confined AppArmor
-        # transition ("secure exec") mark the process fully non-dumpable, and
-        # a non-dumpable process's coredump_filter can't be rewritten
-        # afterwards either - silently discarding postgresql.service's own
-        # ExecStartPost re-apply of it. See fs.suid_dumpable=2 in
-        # ansible/tasks/setup-tuned.yml. Kept capturing dmesg/journalctl here
+    # /proc/[pid]/coredump_filter reads back as zero-padded hex (e.g.
+    # '00000031'), not the bare '31' written to it.
+    if int(result["stdout"].strip(), 16) != 0x31:
+        # Confirmed root cause: postgresql.service's own CoredumpFilter=/
+        # ExecStartPost only affect the postmaster after it's already
+        # running, and get silently undone by the AppArmorProfile=
+        # unconfined->confined "secure exec" transition, which resets
+        # coredump_filter back to whatever the kernel considers "default"
+        # (0x33). Fixed at the source instead: the kernel's own default is
+        # now set to 0x31 via the `coredump_filter=` boot parameter (see
+        # ansible/tasks/setup-coredump-processing.yml), so the reset lands on
+        # the value we want regardless. Kept capturing dmesg/journalctl here
         # so a *different* regression shows real evidence instead of a bare
         # "got 33" again.
         denials = run_ssh_command(

From 5696a585dc802ec7124c068b62b5966ac3930118 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 21:45:04 +0200
Subject: [PATCH 14/17] Fix source file permissions

---
 nix/postgresql/src.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/nix/postgresql/src.nix b/nix/postgresql/src.nix
index 65ba8b1cb3..f670b0a929 100644
--- a/nix/postgresql/src.nix
+++ b/nix/postgresql/src.nix
@@ -20,6 +20,11 @@ stdenv.mkDerivation {
   installPhase = ''
     mkdir -p $out
     cp -r . $out
+    # This tree is read later by a different, unrelated user (e.g. `postgres`
+    # via /var/lib/postgresql/.nix-profile, for GDB source resolution) - make
+    # sure it's readable by everyone rather than relying on whatever mode
+    # bits happened to survive the tarball extraction + copy.
+    chmod -R u+rwX,go+rX $out
   '';
 
   meta = with lib; {

From 637a2621bf3652deabc3f6926225f8252223893a Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 22:55:41 +0200
Subject: [PATCH 15/17] Fix the real issue with permission denied

---
 nix/postgresql/src.nix    | 5 -----
 testinfra/test_ami_nix.py | 8 +++++++-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nix/postgresql/src.nix b/nix/postgresql/src.nix
index f670b0a929..65ba8b1cb3 100644
--- a/nix/postgresql/src.nix
+++ b/nix/postgresql/src.nix
@@ -20,11 +20,6 @@ stdenv.mkDerivation {
   installPhase = ''
     mkdir -p $out
     cp -r . $out
-    # This tree is read later by a different, unrelated user (e.g. `postgres`
-    # via /var/lib/postgresql/.nix-profile, for GDB source resolution) - make
-    # sure it's readable by everyone rather than relying on whatever mode
-    # bits happened to survive the tarball extraction + copy.
-    chmod -R u+rwX,go+rX $out
   '';
 
   meta = with lib; {
diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index f2bc18fdd8..20a7b7bb30 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1877,7 +1877,13 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
 
     result = run_ssh_command(
         host["ssh"],
-        "sudo -u postgres gdb --batch -quiet "
+        # cd into a directory the postgres user can actually read first: GDB's
+        # source search path includes '$cwd', and the SSH session's default
+        # cwd is the login user's home directory (e.g. /home/ubuntu), which
+        # postgres can't traverse - that alone is enough to make GDB report
+        # "Permission denied" on the bare filename before it ever tries the
+        # substitute-path'd absolute path.
+        "cd /var/lib/postgresql && sudo -u postgres gdb --batch -quiet "
         "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' "
         f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' "
         f"-ex 'file {postgres_binary}' "

From d3089d5bef0d06a4baa3afe8c85f9dee39f02072 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Thu, 24 Sep 2026 23:50:01 +0200
Subject: [PATCH 16/17] Another attempt

---
 testinfra/test_ami_nix.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index 20a7b7bb30..dd87934c9c 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1868,12 +1868,22 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
         f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
     )
 
+    # DW_AT_comp_dir is recorded per compilation unit, not once globally -
+    # PostgreSQL's recursive-Makefile build compiles each .c file from its
+    # own subdirectory (e.g. main.c from .../src/backend/main, a different
+    # object from a different subdirectory entirely), so grabbing the first
+    # DW_AT_comp_dir in the whole dump grabs whichever unrelated CU happens
+    # to appear first - not main.c's own. Find main.c's CU specifically and
+    # take its comp_dir.
     comp_dir = run_ssh_command(
         host["ssh"],
-        f"readelf --debug-dump=info {debug_file} 2>/dev/null "
-        "| grep -m1 DW_AT_comp_dir | grep -oE '/[^ ]+$'",
+        f"readelf --debug-dump=info {debug_file} 2>/dev/null | awk '"
+        "/DW_TAG_compile_unit/ { want=0 } "
+        "/DW_AT_name/ && /: main\\.c$/ { want=1 } "
+        "want && /DW_AT_comp_dir/ { print; exit }"
+        "' | grep -oE '/[^ ]+$'",
     )["stdout"].strip()
-    assert comp_dir, f"Could not determine DW_AT_comp_dir from {debug_file}"
+    assert comp_dir, f"Could not determine main.c's DW_AT_comp_dir from {debug_file}"
 
     result = run_ssh_command(
         host["ssh"],

From 083518b28ba9214043d9ad2c7682b3969b1e5798 Mon Sep 17 00:00:00 2001
From: pgnickb <pgnickb@github.com>
Date: Fri, 25 Sep 2026 08:50:46 +0200
Subject: [PATCH 17/17] Fix path substitution

---
 testinfra/test_ami_nix.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py
index dd87934c9c..5781e492cf 100644
--- a/testinfra/test_ami_nix.py
+++ b/testinfra/test_ami_nix.py
@@ -1875,7 +1875,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     # DW_AT_comp_dir in the whole dump grabs whichever unrelated CU happens
     # to appear first - not main.c's own. Find main.c's CU specifically and
     # take its comp_dir.
-    comp_dir = run_ssh_command(
+    main_c_comp_dir = run_ssh_command(
         host["ssh"],
         f"readelf --debug-dump=info {debug_file} 2>/dev/null | awk '"
         "/DW_TAG_compile_unit/ { want=0 } "
@@ -1883,7 +1883,19 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
         "want && /DW_AT_comp_dir/ { print; exit }"
         "' | grep -oE '/[^ ]+$'",
     )["stdout"].strip()
-    assert comp_dir, f"Could not determine main.c's DW_AT_comp_dir from {debug_file}"
+    assert main_c_comp_dir, (
+        f"Could not determine main.c's DW_AT_comp_dir from {debug_file}"
+    )
+    # main_c_comp_dir is main.c's own subdirectory (.../src/backend/main), not
+    # the shared build root - substituting that whole path would map main.c
+    # to '<profile>/main.c' instead of '<profile>/src/backend/main/main.c'.
+    # main.c's location in the postgres tree is fixed, so strip that known
+    # suffix to recover the actual build root shared by every CU.
+    suffix = "/src/backend/main"
+    assert main_c_comp_dir.endswith(suffix), (
+        f"Expected main.c's comp_dir to end with {suffix!r}, got {main_c_comp_dir!r}"
+    )
+    comp_dir = main_c_comp_dir[: -len(suffix)]
 
     result = run_ssh_command(
         host["ssh"],