From 5480cbba809dd446991cf6f70e09ab66571e3028 Mon Sep 17 00:00:00 2001
From: Hermes Agent <gromyko.ss83@gmail.com>
Date: Mon, 12 Oct 2026 03:04:29 +0000
Subject: [PATCH] Add SQLite health & WAL workflows + deleted-but-open files
 finder

- Check SQLite database integrity (integrity_check)
- Find deleted-but-open SQLite WAL/SHM sidecars (lock-drop detection)
- Checkpoint SQLite WAL file into the main database
- Find deleted-but-open files holding disk space (lsof +L1)

Battle-tested on a live multi-agent deployment after debugging a
SQLite permission-helper lock-drop (upstream hermes-lcm #588).
---
 ...eleted_but_open_files_holding_disk_space.yaml | 15 +++++++++++++++
 specs/sql/check_sqlite_database_integrity.yaml   | 15 +++++++++++++++
 specs/sql/checkpoint_sqlite_wal_file.yaml        | 15 +++++++++++++++
 ...deleted_but_open_sqlite_wal_shm_sidecars.yaml | 16 ++++++++++++++++
 4 files changed, 61 insertions(+)
 create mode 100644 specs/shell/find_deleted_but_open_files_holding_disk_space.yaml
 create mode 100644 specs/sql/check_sqlite_database_integrity.yaml
 create mode 100644 specs/sql/checkpoint_sqlite_wal_file.yaml
 create mode 100644 specs/sql/find_deleted_but_open_sqlite_wal_shm_sidecars.yaml

diff --git a/specs/shell/find_deleted_but_open_files_holding_disk_space.yaml b/specs/shell/find_deleted_but_open_files_holding_disk_space.yaml
new file mode 100644
index 00000000..1cb97bd5
--- /dev/null
+++ b/specs/shell/find_deleted_but_open_files_holding_disk_space.yaml
@@ -0,0 +1,15 @@
+---
+name: Find deleted-but-open files holding disk space
+command: |-
+    lsof +L1 | head -30
+tags:
+  - lsof
+  - disk
+  - cleanup
+description: "Lists open files whose directory entries were deleted (shown as '(deleted)'). Classic hidden disk-space eater: long-lived daemons keep the FD open after a log rotation or a database file swap, so `df` stays full even though nothing is visible on disk. Restarting the offending service releases the space."
+source_url: "https://man7.org/linux/man-pages/man8/lsof.8.html"
+author: Gromykoss
+author_url: "https://github.com/Gromykoss"
+shells:
+  - bash
+  - zsh
diff --git a/specs/sql/check_sqlite_database_integrity.yaml b/specs/sql/check_sqlite_database_integrity.yaml
new file mode 100644
index 00000000..24183b8a
--- /dev/null
+++ b/specs/sql/check_sqlite_database_integrity.yaml
@@ -0,0 +1,15 @@
+---
+name: Check SQLite database integrity
+command: |-
+    sqlite3 "{{database_path}}" "PRAGMA integrity_check;"
+tags:
+  - sqlite
+  - database
+  - health
+description: "Runs a full integrity check on a SQLite database file. Reports 'ok' or a list of corrupted pages. Use quick_check (via the related workflow) for a faster, lighter pass on large databases."
+source_url: "https://www.sqlite.org/pragma.html#pragma_integrity_check"
+author: Gromykoss
+author_url: "https://github.com/Gromykoss"
+shells:
+  - bash
+  - zsh
diff --git a/specs/sql/checkpoint_sqlite_wal_file.yaml b/specs/sql/checkpoint_sqlite_wal_file.yaml
new file mode 100644
index 00000000..8c0687bc
--- /dev/null
+++ b/specs/sql/checkpoint_sqlite_wal_file.yaml
@@ -0,0 +1,15 @@
+---
+name: Checkpoint SQLite WAL file into the main database
+command: |-
+    sqlite3 "{{database_path}}" "PRAGMA wal_checkpoint(TRUNCATE);"
+tags:
+  - sqlite
+  - database
+  - wal
+description: "Runs a TRUNCATE checkpoint: moves WAL content into the main database and truncates the -wal file to zero bytes. Useful when a -wal file has grown large because a long-lived connection never releases its read snapshot. Requires no other connections to be writing; busy connections simply block the checkpoint and it reports busy."
+source_url: "https://www.sqlite.org/pragma.html#pragma_wal_checkpoint"
+author: Gromykoss
+author_url: "https://github.com/Gromykoss"
+shells:
+  - bash
+  - zsh
diff --git a/specs/sql/find_deleted_but_open_sqlite_wal_shm_sidecars.yaml b/specs/sql/find_deleted_but_open_sqlite_wal_shm_sidecars.yaml
new file mode 100644
index 00000000..43b9db75
--- /dev/null
+++ b/specs/sql/find_deleted_but_open_sqlite_wal_shm_sidecars.yaml
@@ -0,0 +1,16 @@
+---
+name: Find deleted-but-open SQLite WAL/SHM sidecars
+command: |-
+    lsof +L1 | grep -E "\.(db|sqlite)(-wal|-shm)?" | head -20
+tags:
+  - sqlite
+  - database
+  - health
+  - wal
+description: "Detects live processes holding file descriptors to deleted SQLite WAL/SHM sidecars. This happens when a helper opens and closes an ordinary descriptor on the same path: the open/close cycle drops the process-owned POSIX record locks, causing SQLite to recreate the sidecars while the old connection keeps stale handles - later checkpointing fails with SQLITE_IOERR / disk I/O errors. Healthy state: zero lines."
+source_url: "https://github.com/stephenschoettler/hermes-lcm/issues/588"
+author: Gromykoss
+author_url: "https://github.com/Gromykoss"
+shells:
+  - bash
+  - zsh