From 317e1c3020ffb0e9dd92e447e64788344683f55e Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Tue, 6 Oct 2026 13:18:00 -0400 Subject: [PATCH 01/11] feat(dgw): keep agent tunnel data in a shared gateway.db Gateway features that need to store a little data now share one database, gateway.db, with tables named after the feature, instead of one file per feature. The job queue keeps its own file. The new gateway-db crate opens the file, runs one migration list and hands out a connection. The agent tunnel tables get an agent_tunnel_ prefix. gateway.db is opened on every start, and the agent tunnel CA is loaded, not generated, once the database is bound to a CA. On the first start after 2026.3, agent_tunnel.db is copied to gateway.db and then removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 15 + crates/agent-tunnel-libsql/Cargo.toml | 1 + crates/agent-tunnel-libsql/src/lib.rs | 311 +++++++----------- crates/gateway-db/Cargo.toml | 19 ++ .../migrations/01_agent_tunnel.sql} | 0 .../02_prefix_agent_tunnel_tables.sql | 4 + crates/gateway-db/src/lib.rs | 275 ++++++++++++++++ devolutions-gateway/Cargo.toml | 1 + devolutions-gateway/src/service.rs | 13 +- testsuite/Cargo.toml | 1 + testsuite/tests/agent_tunnel/common.rs | 5 +- 11 files changed, 443 insertions(+), 202 deletions(-) create mode 100644 crates/gateway-db/Cargo.toml rename crates/{agent-tunnel-libsql/migrations/01_agent_authorization.sql => gateway-db/migrations/01_agent_tunnel.sql} (100%) create mode 100644 crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql create mode 100644 crates/gateway-db/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c796c7439..e06a36e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ dependencies = [ "agent-tunnel", "anyhow", "async-trait", + "gateway-db", "libsql", "tempfile", "tokio 1.52.3", @@ -1853,6 +1854,7 @@ dependencies = [ "etherparse", "expect-test", "futures", + "gateway-db", "hex", "hostname 0.4.2", "http-body-util", @@ -2666,6 +2668,18 @@ dependencies = [ "slab", ] +[[package]] +name = "gateway-db" +version = "0.0.0" +dependencies = [ + "anyhow", + "camino", + "libsql", + "tempfile", + "tokio 1.52.3", + "tracing", +] + [[package]] name = "generate-openapi" version = "0.0.0" @@ -7656,6 +7670,7 @@ dependencies = [ "expect-test", "fastrand", "futures-util", + "gateway-db", "ipnetwork", "jmux-proto", "libsql", diff --git a/crates/agent-tunnel-libsql/Cargo.toml b/crates/agent-tunnel-libsql/Cargo.toml index 627a04e64..2a27d8225 100644 --- a/crates/agent-tunnel-libsql/Cargo.toml +++ b/crates/agent-tunnel-libsql/Cargo.toml @@ -17,5 +17,6 @@ tokio = { version = "1.52", features = ["sync"] } uuid = "1.23" [dev-dependencies] +gateway-db.path = "../gateway-db" tempfile = "3.24" tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/agent-tunnel-libsql/src/lib.rs b/crates/agent-tunnel-libsql/src/lib.rs index e23d9d3e7..21c09ecfa 100644 --- a/crates/agent-tunnel-libsql/src/lib.rs +++ b/crates/agent-tunnel-libsql/src/lib.rs @@ -7,119 +7,36 @@ use libsql::{Connection, TransactionBehavior, params}; use tokio::sync::Mutex; use uuid::Uuid; -const MIGRATIONS: &[&str] = &[include_str!("../migrations/01_agent_authorization.sql")]; const CA_SPKI_SHA256_KEY: &str = "ca_spki_sha256"; +/// Agent authorization kept in the `agent_tunnel_*` tables of `gateway.db`. pub struct LibSqlAgentAuthorizationStore { conn: Mutex, } impl LibSqlAgentAuthorizationStore { - pub async fn open(path: &str, ca_spki_sha256: SpkiSha256) -> anyhow::Result { - let database = libsql::Builder::new_local(path) - .build() - .await - .context("build Agent authorization database")?; - let conn = database.connect().context("open Agent authorization database")?; + /// Opens the store on a `gateway.db` connection from the `gateway-db` crate. + /// + /// The stored Agents only make sense with the CA that signed their certificates, so the first open ties the + /// store to `ca_spki_sha256`, and later opens refuse any other CA. + pub async fn open(conn: Connection, ca_spki_sha256: SpkiSha256) -> anyhow::Result { let store = Self { conn: Mutex::new(conn) }; - store.apply_pragmas().await?; - store.migrate_and_bind_ca(ca_spki_sha256).await?; store.validate_integrity_and_schema().await?; + store.bind_ca(ca_spki_sha256).await?; Ok(store) } - async fn apply_pragmas(&self) -> anyhow::Result<()> { - const PRAGMAS: &str = " - PRAGMA journal_mode = WAL; - PRAGMA synchronous = FULL; - PRAGMA busy_timeout = 15000; - PRAGMA foreign_keys = ON; - PRAGMA temp_store = MEMORY; - "; - - self.conn - .lock() - .await - .execute_batch(PRAGMAS) - .await - .context("apply Agent authorization database PRAGMAs")?; - Ok(()) - } - - async fn migrate_and_bind_ca(&self, ca_spki_sha256: SpkiSha256) -> anyhow::Result<()> { - let conn = self.conn.lock().await; - let user_version = Self::schema_version(&conn).await?; - - if MIGRATIONS.len() < user_version { - bail!( - "Agent authorization schema version {user_version} is newer than supported version {}", - MIGRATIONS.len() - ); - } - - if user_version == 0 { - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .context("begin Agent authorization initialization")?; - for (migration_id, migration) in MIGRATIONS.iter().enumerate() { - tx.execute_batch(migration) - .await - .with_context(|| format!("apply Agent authorization migration {}", migration_id + 1))?; - if migration_id == 0 { - tx.execute( - "INSERT INTO metadata (key, value) VALUES (?1, ?2)", - params![CA_SPKI_SHA256_KEY, ca_spki_sha256.to_vec()], - ) - .await - .context("bind Agent authorization database to CA")?; - } - tx.execute_batch(&format!("PRAGMA user_version = {}", migration_id + 1)) - .await - .with_context(|| format!("record Agent authorization migration {}", migration_id + 1))?; - } - tx.commit().await.context("commit Agent authorization initialization")?; - return Ok(()); - } - - Self::validate_ca_binding(&conn, ca_spki_sha256).await?; - for (migration_id, migration) in MIGRATIONS.iter().enumerate().skip(user_version) { - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .with_context(|| format!("begin Agent authorization migration {}", migration_id + 1))?; - tx.execute_batch(migration) - .await - .with_context(|| format!("apply Agent authorization migration {}", migration_id + 1))?; - tx.execute_batch(&format!("PRAGMA user_version = {}", migration_id + 1)) - .await - .with_context(|| format!("record Agent authorization migration {}", migration_id + 1))?; - tx.commit() - .await - .with_context(|| format!("commit Agent authorization migration {}", migration_id + 1))?; - } - - Ok(()) - } - - async fn schema_version(conn: &Connection) -> anyhow::Result { - let row = conn - .query("PRAGMA user_version", ()) - .await - .context("query Agent authorization schema version")? - .next() - .await - .context("read Agent authorization schema version")? - .context("Agent authorization schema version query returned no row")?; - let user_version = row.get::(0).context("decode Agent authorization schema version")?; - usize::try_from(user_version).context("Agent authorization schema version is too large") - } - - async fn validate_ca_binding(conn: &Connection, ca_spki_sha256: SpkiSha256) -> anyhow::Result<()> { + /// Returns the CA the stored Agents belong to, or `None` if the agent tunnel never ran. + /// + /// Once Agents are tied to a CA, Gateway must load that CA and never generate a new one. + pub async fn bound_ca(conn: &Connection) -> anyhow::Result> { let stored = conn - .query("SELECT value FROM metadata WHERE key = ?1", params![CA_SPKI_SHA256_KEY]) + .query( + "SELECT value FROM agent_tunnel_metadata WHERE key = ?1", + params![CA_SPKI_SHA256_KEY], + ) .await .context("query Agent authorization CA identity")? .next() @@ -129,12 +46,30 @@ impl LibSqlAgentAuthorizationStore { .transpose() .context("decode Agent authorization CA identity")?; - let stored = stored.context("Agent authorization database is missing CA metadata")?; - if stored.as_slice() != ca_spki_sha256 { - bail!("Agent authorization database belongs to a different CA"); - } + stored + .map(|stored| { + SpkiSha256::try_from(stored) + .map_err(|_| anyhow::anyhow!("Agent authorization CA identity has an invalid length")) + }) + .transpose() + } - Ok(()) + async fn bind_ca(&self, ca_spki_sha256: SpkiSha256) -> anyhow::Result<()> { + let conn = self.conn.lock().await; + + match Self::bound_ca(&conn).await? { + Some(stored) if stored == ca_spki_sha256 => Ok(()), + Some(_) => bail!("Agent authorization database belongs to a different CA"), + None => { + conn.execute( + "INSERT INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", + params![CA_SPKI_SHA256_KEY, ca_spki_sha256.to_vec()], + ) + .await + .context("bind Agent authorization database to CA")?; + Ok(()) + } + } } async fn validate_integrity_and_schema(&self) -> anyhow::Result<()> { @@ -162,18 +97,21 @@ impl LibSqlAgentAuthorizationStore { } const TABLE_PROBES: &[(&str, &str)] = &[ - ("metadata", "SELECT key, value FROM metadata LIMIT 0"), ( - "accepted_agents", - "SELECT agent_id, name, client_spki_sha256, enrollment_jti FROM accepted_agents LIMIT 0", + "agent_tunnel_metadata", + "SELECT key, value FROM agent_tunnel_metadata LIMIT 0", + ), + ( + "agent_tunnel_accepted_agents", + "SELECT agent_id, name, client_spki_sha256, enrollment_jti FROM agent_tunnel_accepted_agents LIMIT 0", ), ( - "enrollment_attempts", - "SELECT jti, agent_id, request_sha256, expires_at, deleted FROM enrollment_attempts LIMIT 0", + "agent_tunnel_enrollment_attempts", + "SELECT jti, agent_id, request_sha256, expires_at, deleted FROM agent_tunnel_enrollment_attempts LIMIT 0", ), ( - "deleted_agent_keys", - "SELECT agent_id, client_spki_sha256 FROM deleted_agent_keys LIMIT 0", + "agent_tunnel_deleted_agent_keys", + "SELECT agent_id, client_spki_sha256 FROM agent_tunnel_deleted_agent_keys LIMIT 0", ), ]; for (table, probe) in TABLE_PROBES { @@ -182,12 +120,12 @@ impl LibSqlAgentAuthorizationStore { .with_context(|| format!("validate Agent authorization table {table}"))?; } - Self::validate_unique_index(&conn, "metadata", &["key"]).await?; - Self::validate_unique_index(&conn, "accepted_agents", &["agent_id"]).await?; - Self::validate_unique_index(&conn, "accepted_agents", &["name"]).await?; - Self::validate_unique_index(&conn, "accepted_agents", &["enrollment_jti"]).await?; - Self::validate_unique_index(&conn, "enrollment_attempts", &["jti"]).await?; - Self::validate_unique_index(&conn, "deleted_agent_keys", &["client_spki_sha256"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_metadata", &["key"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_accepted_agents", &["agent_id"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_accepted_agents", &["name"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_accepted_agents", &["enrollment_jti"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_enrollment_attempts", &["jti"]).await?; + Self::validate_unique_index(&conn, "agent_tunnel_deleted_agent_keys", &["client_spki_sha256"]).await?; Ok(()) } @@ -278,7 +216,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { }; let conn = self.conn.lock().await; conn.execute( - "DELETE FROM enrollment_attempts + "DELETE FROM agent_tunnel_enrollment_attempts WHERE expires_at < unixepoch() - 86400", (), ) @@ -288,7 +226,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let existing_attempt = conn .query( "SELECT agent_id, request_sha256, deleted - FROM enrollment_attempts + FROM agent_tunnel_enrollment_attempts WHERE jti = ?1", params![attempt.token_id.to_string()], ) @@ -315,7 +253,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let existing_agent = conn .query( "SELECT agent_id, name, client_spki_sha256 - FROM accepted_agents + FROM agent_tunnel_accepted_agents WHERE enrollment_jti = ?1", params![attempt.token_id.to_string()], ) @@ -336,7 +274,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let agent_id_exists = conn .query( - "SELECT 1 FROM accepted_agents WHERE agent_id = ?1", + "SELECT 1 FROM agent_tunnel_accepted_agents WHERE agent_id = ?1", params![attempt.agent_id.to_string()], ) .await @@ -351,7 +289,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let agent_name_exists = conn .query( - "SELECT 1 FROM accepted_agents WHERE name = ?1", + "SELECT 1 FROM agent_tunnel_accepted_agents WHERE name = ?1", params![attempt.name.clone()], ) .await @@ -367,7 +305,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let deleted_key_exists = conn .query( "SELECT 1 - FROM deleted_agent_keys + FROM agent_tunnel_deleted_agent_keys WHERE client_spki_sha256 = ?1", params![attempt.client_spki_sha256.to_vec()], ) @@ -387,7 +325,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { .context("begin Agent enrollment transaction")?; tx.execute( - "INSERT INTO enrollment_attempts (jti, agent_id, request_sha256, expires_at) + "INSERT INTO agent_tunnel_enrollment_attempts (jti, agent_id, request_sha256, expires_at) VALUES (?1, ?2, ?3, ?4)", params![ attempt.token_id.to_string(), @@ -399,7 +337,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { .await .context("persist Agent enrollment attempt")?; tx.execute( - "INSERT INTO accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) + "INSERT INTO agent_tunnel_accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) VALUES (?1, ?2, ?3, ?4)", params![ attempt.agent_id.to_string(), @@ -420,7 +358,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let row = conn .query( "SELECT agent_id, name, client_spki_sha256 - FROM accepted_agents + FROM agent_tunnel_accepted_agents WHERE agent_id = ?1 AND client_spki_sha256 = ?2", params![agent_id.to_string(), client_spki_sha256.to_vec()], ) @@ -438,7 +376,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let row = conn .query( "SELECT agent_id, name, client_spki_sha256 - FROM accepted_agents + FROM agent_tunnel_accepted_agents WHERE agent_id = ?1", params![agent_id.to_string()], ) @@ -456,7 +394,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let mut rows = conn .query( "SELECT agent_id, name, client_spki_sha256 - FROM accepted_agents + FROM agent_tunnel_accepted_agents ORDER BY name COLLATE NOCASE, agent_id", (), ) @@ -480,7 +418,7 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let row = tx .query( "SELECT agent_id, name, client_spki_sha256, enrollment_jti - FROM accepted_agents + FROM agent_tunnel_accepted_agents WHERE agent_id = ?1", params![agent_id.to_string()], ) @@ -498,20 +436,20 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { let enrollment_jti = row.get::(3).context("decode deleted Agent enrollment JTI")?; tx.execute( - "INSERT OR IGNORE INTO deleted_agent_keys (client_spki_sha256, agent_id) + "INSERT OR IGNORE INTO agent_tunnel_deleted_agent_keys (client_spki_sha256, agent_id) VALUES (?1, ?2)", params![accepted.client_spki_sha256.to_vec(), agent_id.to_string()], ) .await .context("remember deleted Agent key")?; tx.execute( - "DELETE FROM accepted_agents WHERE agent_id = ?1", + "DELETE FROM agent_tunnel_accepted_agents WHERE agent_id = ?1", params![agent_id.to_string()], ) .await .context("delete accepted Agent")?; tx.execute( - "UPDATE enrollment_attempts SET deleted = 1 WHERE jti = ?1", + "UPDATE agent_tunnel_enrollment_attempts SET deleted = 1 WHERE jti = ?1", params![enrollment_jti], ) .await @@ -526,6 +464,11 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { mod tests { use super::*; + async fn open_store(path: &str, ca_spki_sha256: SpkiSha256) -> anyhow::Result { + let conn = gateway_db::open_path(path).await?; + LibSqlAgentAuthorizationStore::open(conn, ca_spki_sha256).await + } + fn enrollment(agent_id: Uuid) -> EnrollmentAttempt { EnrollmentAttempt { token_id: Uuid::new_v4(), @@ -540,19 +483,19 @@ mod tests { #[tokio::test] async fn accepted_agent_survives_reopen() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); let ca_spki_sha256 = [0xCA; 32]; let agent_id = Uuid::new_v4(); - let store = LibSqlAgentAuthorizationStore::open(database_path, ca_spki_sha256) + let store = open_store(database_path, ca_spki_sha256) .await .expect("open Agent authorization store"); let outcome = store.enroll(enrollment(agent_id)).await.expect("enroll Agent"); assert!(matches!(outcome, EnrollmentOutcome::Created(_))); drop(store); - let reopened = LibSqlAgentAuthorizationStore::open(database_path, ca_spki_sha256) + let reopened = open_store(database_path, ca_spki_sha256) .await .expect("reopen Agent authorization store"); let accepted = reopened @@ -567,7 +510,7 @@ mod tests { #[tokio::test] async fn identical_enrollment_retry_is_idempotent() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let attempt = enrollment(Uuid::new_v4()); @@ -581,7 +524,7 @@ mod tests { #[tokio::test] async fn delete_prevents_original_enrollment_retry() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let attempt = enrollment(Uuid::new_v4()); @@ -607,7 +550,7 @@ mod tests { #[tokio::test] async fn accepted_agent_id_and_name_are_unique() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -626,7 +569,7 @@ mod tests { #[tokio::test] async fn list_and_get_include_offline_accepted_agents() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -654,7 +597,7 @@ mod tests { #[tokio::test] async fn enrollment_token_rejects_different_material() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -669,7 +612,7 @@ mod tests { #[tokio::test] async fn new_token_and_key_can_reenroll_deleted_agent() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -708,9 +651,9 @@ mod tests { #[tokio::test] async fn new_token_cannot_reenroll_deleted_agent_key() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let store = open_store(database_path, [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -722,7 +665,7 @@ mod tests { .expect("Agent was accepted"); drop(store); - let reopened = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let reopened = open_store(database_path, [0xCA; 32]) .await .expect("reopen Agent authorization store"); let replay = enrollment(first.agent_id); @@ -733,7 +676,7 @@ mod tests { #[tokio::test] async fn new_agent_id_cannot_reuse_deleted_agent_key() { - let store = LibSqlAgentAuthorizationStore::open(":memory:", [0xCA; 32]) + let store = open_store(":memory:", [0xCA; 32]) .await .expect("open Agent authorization store"); let first = enrollment(Uuid::new_v4()); @@ -754,14 +697,14 @@ mod tests { #[tokio::test] async fn database_rejects_another_ca() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let store = open_store(database_path, [0xCA; 32]) .await .expect("open Agent authorization store"); drop(store); - let error = LibSqlAgentAuthorizationStore::open(database_path, [0xBB; 32]) + let error = open_store(database_path, [0xBB; 32]) .await .err() .expect("database must reject another CA"); @@ -769,67 +712,39 @@ mod tests { } #[tokio::test] - async fn database_rejects_a_newer_schema() { + async fn first_open_binds_the_ca() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) - .await - .expect("open Agent authorization store"); - drop(store); - let database = libsql::Builder::new_local(database_path) - .build() - .await - .expect("open Agent authorization database"); - let conn = database.connect().expect("connect to Agent authorization database"); - conn.execute_batch("PRAGMA user_version = 99") + let conn = gateway_db::open_path(database_path) .await - .expect("set unsupported schema version"); - drop(conn); - drop(database); - - let error = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + .expect("open gateway database"); + let bound = LibSqlAgentAuthorizationStore::bound_ca(&conn) .await - .err() - .expect("database must reject a newer schema"); - assert!(error.to_string().contains("newer than supported")); - } + .expect("read bound CA"); + assert_eq!(bound, None); - #[tokio::test] - async fn initialized_database_rejects_missing_ca_metadata() { - let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); - let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let store = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) .await .expect("open Agent authorization store"); drop(store); - let database = libsql::Builder::new_local(database_path) - .build() + let conn = gateway_db::open_path(database_path) .await - .expect("open Agent authorization database"); - let conn = database.connect().expect("connect to Agent authorization database"); - conn.execute("DELETE FROM metadata WHERE key = ?1", params![CA_SPKI_SHA256_KEY]) + .expect("reopen gateway database"); + let bound = LibSqlAgentAuthorizationStore::bound_ca(&conn) .await - .expect("delete CA metadata"); - drop(conn); - drop(database); - - let error = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) - .await - .err() - .expect("initialized database must require CA metadata"); - assert!(error.to_string().contains("missing CA")); + .expect("read bound CA"); + assert_eq!(bound, Some([0xCA; 32])); } #[tokio::test] async fn database_rejects_missing_required_table() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let store = open_store(database_path, [0xCA; 32]) .await .expect("open Agent authorization store"); drop(store); @@ -839,13 +754,13 @@ mod tests { .await .expect("open Agent authorization database"); let conn = database.connect().expect("connect to Agent authorization database"); - conn.execute("DROP TABLE accepted_agents", ()) + conn.execute("DROP TABLE agent_tunnel_accepted_agents", ()) .await .expect("drop required table"); drop(conn); drop(database); - let error = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let error = open_store(database_path, [0xCA; 32]) .await .err() .expect("database must reject missing required table"); @@ -855,9 +770,9 @@ mod tests { #[tokio::test] async fn database_rejects_missing_required_unique_index() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let database_path = temp_dir.path().join("agent_tunnel.db"); + let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let store = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let store = open_store(database_path, [0xCA; 32]) .await .expect("open Agent authorization store"); drop(store); @@ -869,8 +784,8 @@ mod tests { let conn = database.connect().expect("connect to Agent authorization database"); conn.execute_batch( " - ALTER TABLE accepted_agents RENAME TO accepted_agents_with_name_index; - CREATE TABLE accepted_agents ( + ALTER TABLE agent_tunnel_accepted_agents RENAME TO agent_tunnel_accepted_agents_with_name_index; + CREATE TABLE agent_tunnel_accepted_agents ( agent_id TEXT PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE, client_spki_sha256 BLOB NOT NULL, @@ -879,7 +794,7 @@ mod tests { CHECK (name = trim(name)), CHECK (length(client_spki_sha256) = 32) ); - DROP TABLE accepted_agents_with_name_index; + DROP TABLE agent_tunnel_accepted_agents_with_name_index; ", ) .await @@ -887,7 +802,7 @@ mod tests { drop(conn); drop(database); - let error = LibSqlAgentAuthorizationStore::open(database_path, [0xCA; 32]) + let error = open_store(database_path, [0xCA; 32]) .await .err() .expect("database must reject missing Agent name index"); diff --git a/crates/gateway-db/Cargo.toml b/crates/gateway-db/Cargo.toml new file mode 100644 index 000000000..ad6dc3255 --- /dev/null +++ b/crates/gateway-db/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "gateway-db" +version = "0.0.0" +edition = "2024" +authors = ["Devolutions Inc. "] +publish = false + +[lints] +workspace = true + +[dependencies] +anyhow = "1" +camino = "1.1" +libsql = { version = "0.9", default-features = false, features = ["core"] } +tracing = "0.1" + +[dev-dependencies] +tempfile = "3.24" +tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/agent-tunnel-libsql/migrations/01_agent_authorization.sql b/crates/gateway-db/migrations/01_agent_tunnel.sql similarity index 100% rename from crates/agent-tunnel-libsql/migrations/01_agent_authorization.sql rename to crates/gateway-db/migrations/01_agent_tunnel.sql diff --git a/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql b/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql new file mode 100644 index 000000000..8ce15eb72 --- /dev/null +++ b/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql @@ -0,0 +1,4 @@ +ALTER TABLE metadata RENAME TO agent_tunnel_metadata; +ALTER TABLE accepted_agents RENAME TO agent_tunnel_accepted_agents; +ALTER TABLE enrollment_attempts RENAME TO agent_tunnel_enrollment_attempts; +ALTER TABLE deleted_agent_keys RENAME TO agent_tunnel_deleted_agent_keys; diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs new file mode 100644 index 000000000..20c94fb2d --- /dev/null +++ b/crates/gateway-db/src/lib.rs @@ -0,0 +1,275 @@ +//! Gateway's own database, `gateway.db`. +//! +//! When a feature needs to remember a little data, it keeps it here in tables named after the feature +//! (`agent_tunnel_*`, ...), so we don't end up with one database file per feature. The job queue is the +//! exception: it tunes its database for queue work, so it keeps its own file. + +#[macro_use] +extern crate tracing; + +use anyhow::{Context as _, bail}; +use camino::{Utf8Path, Utf8PathBuf}; +use libsql::{Connection, TransactionBehavior, params}; + +pub const FILE_NAME: &str = "gateway.db"; + +/// Gateway 2026.3 kept the agent tunnel tables in a file of their own. +const LEGACY_AGENT_TUNNEL_FILE_NAME: &str = "agent_tunnel.db"; + +const PRAGMAS: &str = " + PRAGMA journal_mode = WAL; + PRAGMA synchronous = FULL; + PRAGMA busy_timeout = 15000; + PRAGMA foreign_keys = ON; + PRAGMA temp_store = MEMORY; +"; + +/// Every schema change ever made to `gateway.db`, oldest first. Only ever append to it. +const MIGRATIONS: &[&str] = &[ + include_str!("../migrations/01_agent_tunnel.sql"), + include_str!("../migrations/02_prefix_agent_tunnel_tables.sql"), +]; + +/// Opens `gateway.db` in `data_dir`, creating it or bringing its tables up to date. +/// +/// The first time a Gateway that ran 2026.3 starts, its `agent_tunnel.db` becomes `gateway.db`. +pub async fn open(data_dir: &Utf8Path) -> anyhow::Result { + let path = data_dir.join(FILE_NAME); + adopt_legacy_agent_tunnel_database(&data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME), &path).await?; + open_path(path.as_str()).await +} + +/// Opens the database at `path` (`:memory:` works too), creating it or bringing its tables up to date. +pub async fn open_path(path: &str) -> anyhow::Result { + let conn = connect(path).await?; + + conn.execute_batch(PRAGMAS) + .await + .context("apply gateway database PRAGMAs")?; + migrate(&conn).await?; + + Ok(conn) +} + +async fn connect(path: &str) -> anyhow::Result { + libsql::Builder::new_local(path) + .build() + .await + .with_context(|| format!("build database {path}"))? + .connect() + .with_context(|| format!("connect to database {path}")) +} + +async fn migrate(conn: &Connection) -> anyhow::Result<()> { + let user_version = schema_version(conn).await?; + + if MIGRATIONS.len() < user_version { + bail!( + "gateway database schema version {user_version} is newer than supported version {}", + MIGRATIONS.len() + ); + } + + for (index, migration) in MIGRATIONS.iter().enumerate().skip(user_version) { + let version = index + 1; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .with_context(|| format!("begin gateway database migration {version}"))?; + tx.execute_batch(migration) + .await + .with_context(|| format!("apply gateway database migration {version}"))?; + tx.execute_batch(&format!("PRAGMA user_version = {version}")) + .await + .with_context(|| format!("record gateway database migration {version}"))?; + tx.commit() + .await + .with_context(|| format!("commit gateway database migration {version}"))?; + } + + Ok(()) +} + +async fn schema_version(conn: &Connection) -> anyhow::Result { + let row = conn + .query("PRAGMA user_version", ()) + .await + .context("query gateway database schema version")? + .next() + .await + .context("read gateway database schema version")? + .context("gateway database schema version query returned no row")?; + let user_version = row.get::(0).context("decode gateway database schema version")?; + usize::try_from(user_version).context("gateway database schema version is too large") +} + +/// Turns a 2026.3 `agent_tunnel.db` into `gateway.db`. +/// +/// We write a complete copy first and only then rename it into place, so `gateway.db` either doesn't exist yet +/// or is whole. If Gateway stops halfway, the next start simply does it again. +async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) -> anyhow::Result<()> { + if path.exists() || !legacy.exists() { + return Ok(()); + } + + let copy = Utf8PathBuf::from(format!("{path}.tmp")); + remove_file_if_exists(©).with_context(|| format!("remove the incomplete copy {copy}"))?; + + let legacy_conn = connect(legacy.as_str()).await?; + legacy_conn + .execute("VACUUM INTO ?1", params![copy.as_str()]) + .await + .with_context(|| format!("copy {legacy} to {copy}"))?; + drop(legacy_conn); + + std::fs::rename(©, path).with_context(|| format!("rename {copy} to {path}"))?; + + for suffix in ["", "-wal", "-shm"] { + let legacy_file = Utf8PathBuf::from(format!("{legacy}{suffix}")); + + if let Err(error) = remove_file_if_exists(&legacy_file) { + warn!(%error, path = %legacy_file, "Failed to remove the old agent tunnel database file"); + } + } + + info!(from = %legacy, to = %path, "Moved the agent tunnel database into the gateway database"); + + Ok(()) +} + +fn remove_file_if_exists(path: &Utf8Path) -> std::io::Result<()> { + match std::fs::remove_file(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + result => result, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_data_dir() -> (tempfile::TempDir, Utf8PathBuf) { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let data_dir = Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("temporary path is UTF-8"); + (temp_dir, data_dir) + } + + async fn user_version(conn: &Connection) -> usize { + schema_version(conn).await.expect("read schema version") + } + + async fn table_exists(conn: &Connection, table: &str) -> bool { + conn.query( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1", + params![table], + ) + .await + .expect("query tables") + .next() + .await + .expect("read tables") + .is_some() + } + + /// Builds the `agent_tunnel.db` that Gateway 2026.3 leaves behind. + async fn write_legacy_database(path: &Utf8Path) { + let conn = connect(path.as_str()).await.expect("open legacy database"); + conn.execute_batch(PRAGMAS).await.expect("apply legacy PRAGMAs"); + conn.execute_batch(MIGRATIONS[0]).await.expect("create legacy tables"); + conn.execute( + "INSERT INTO metadata (key, value) VALUES ('ca_spki_sha256', ?1)", + params![vec![0xCAu8; 32]], + ) + .await + .expect("bind legacy CA"); + conn.execute_batch("PRAGMA user_version = 1") + .await + .expect("record legacy schema version"); + } + + #[tokio::test] + async fn new_database_has_prefixed_agent_tunnel_tables() { + let conn = open_path(":memory:").await.expect("open gateway database"); + + assert_eq!(user_version(&conn).await, MIGRATIONS.len()); + assert!(table_exists(&conn, "agent_tunnel_metadata").await); + assert!(table_exists(&conn, "agent_tunnel_accepted_agents").await); + assert!(table_exists(&conn, "agent_tunnel_enrollment_attempts").await); + assert!(table_exists(&conn, "agent_tunnel_deleted_agent_keys").await); + assert!(!table_exists(&conn, "metadata").await); + } + + #[tokio::test] + async fn legacy_agent_tunnel_database_becomes_gateway_database() { + let (_temp_dir, data_dir) = temp_data_dir(); + let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); + write_legacy_database(&legacy).await; + + let conn = open(&data_dir).await.expect("open gateway database"); + + assert!(!legacy.exists()); + assert!(data_dir.join(FILE_NAME).exists()); + assert_eq!(user_version(&conn).await, MIGRATIONS.len()); + let ca = conn + .query( + "SELECT value FROM agent_tunnel_metadata WHERE key = 'ca_spki_sha256'", + (), + ) + .await + .expect("query CA") + .next() + .await + .expect("read CA") + .expect("CA survives the move") + .get::>(0) + .expect("decode CA"); + assert_eq!(ca, vec![0xCAu8; 32]); + } + + #[tokio::test] + async fn interrupted_move_starts_over() { + let (_temp_dir, data_dir) = temp_data_dir(); + let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); + write_legacy_database(&legacy).await; + std::fs::write(data_dir.join(format!("{FILE_NAME}.tmp")), b"half written").expect("leave an incomplete copy"); + + let conn = open(&data_dir).await.expect("open gateway database"); + + assert!(table_exists(&conn, "agent_tunnel_metadata").await); + assert!(!data_dir.join(format!("{FILE_NAME}.tmp")).exists()); + } + + #[tokio::test] + async fn existing_gateway_database_is_kept() { + let (_temp_dir, data_dir) = temp_data_dir(); + drop(open(&data_dir).await.expect("create gateway database")); + let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); + write_legacy_database(&legacy).await; + + let conn = open(&data_dir).await.expect("reopen gateway database"); + + assert!(legacy.exists()); + let rows = conn + .query("SELECT 1 FROM agent_tunnel_metadata", ()) + .await + .expect("query CA") + .next() + .await + .expect("read CA"); + assert!(rows.is_none()); + } + + #[tokio::test] + async fn newer_schema_is_rejected() { + let (_temp_dir, data_dir) = temp_data_dir(); + let conn = open(&data_dir).await.expect("create gateway database"); + conn.execute_batch("PRAGMA user_version = 99") + .await + .expect("set unsupported schema version"); + drop(conn); + + let error = open(&data_dir).await.expect_err("newer schema must be rejected"); + + assert!(error.to_string().contains("newer than supported")); + } +} diff --git a/devolutions-gateway/Cargo.toml b/devolutions-gateway/Cargo.toml index 6d68ec6e8..908f5beaf 100644 --- a/devolutions-gateway/Cargo.toml +++ b/devolutions-gateway/Cargo.toml @@ -37,6 +37,7 @@ sysevent.path = "../crates/sysevent" sysevent-codes.path = "../crates/sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel" } agent-tunnel-libsql = { path = "../crates/agent-tunnel-libsql" } +gateway-db.path = "../crates/gateway-db" ironrdp-pdu = { version = "0.9", features = ["std"] } ironrdp-core = { version = "0.2", features = ["std"] } ironrdp-rdcleanpath = "0.2" diff --git a/devolutions-gateway/src/service.rs b/devolutions-gateway/src/service.rs index 97b733f17..00dab064b 100644 --- a/devolutions-gateway/src/service.rs +++ b/devolutions-gateway/src/service.rs @@ -275,13 +275,20 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { ); let monitoring_state = Arc::new(network_monitor::State::new(Arc::new(filesystem_monitor_config_cache))?); + let gateway_db = gateway_db::open(&config::get_data_dir()) + .await + .context("failed to open the gateway database")?; + // Initialize the agent tunnel when enabled. let agent_tunnel_handle = if conf.agent_tunnel.enabled { let data_dir = config::get_data_dir(); let hostname = &conf.hostname; - let authorization_database = data_dir.join("agent_tunnel.db"); - let ca_manager = if authorization_database.exists() { + let agents_have_ca = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::bound_ca(&gateway_db) + .await + .context("failed to read the agent tunnel CA identity")? + .is_some(); + let ca_manager = if agents_have_ca { agent_tunnel::cert::CaManager::load(&data_dir) } else { agent_tunnel::cert::CaManager::load_or_generate(&data_dir) @@ -291,7 +298,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .ca_spki_sha256() .context("failed to identify agent tunnel CA")?; let authorization_store = - agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(authorization_database.as_str(), ca_spki_sha256) + agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db.clone(), ca_spki_sha256) .await .context("failed to initialize Agent authorization database")?; let authorization_store: agent_tunnel::authorization::DynAgentAuthorizationStore = diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index a40691073..5ab347bbe 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -34,6 +34,7 @@ tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } agent-sysevent-codes.path = "../crates/agent-sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel", features = ["test-utils"] } agent-tunnel-libsql = { path = "../crates/agent-tunnel-libsql" } +gateway-db.path = "../crates/gateway-db" agent-tunnel-proto = { path = "../crates/agent-tunnel-proto", features = ["serde"] } base64 = "0.23" camino = "1" diff --git a/testsuite/tests/agent_tunnel/common.rs b/testsuite/tests/agent_tunnel/common.rs index ca4af024e..4c697e418 100644 --- a/testsuite/tests/agent_tunnel/common.rs +++ b/testsuite/tests/agent_tunnel/common.rs @@ -175,7 +175,10 @@ pub(super) async fn bind_test_listener() -> TestListener { let data_dir = Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("use utf-8 temporary path"); let ca_manager = CaManager::load_or_generate(&data_dir).expect("generate test ca"); let ca_spki_sha256 = ca_manager.ca_spki_sha256().expect("hash test CA public key"); - let authorization_store = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(":memory:", ca_spki_sha256) + let gateway_db = gateway_db::open_path(":memory:") + .await + .expect("open test gateway database"); + let authorization_store = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) .await .expect("open test Agent authorization store"); let listen_addr: SocketAddr = "127.0.0.1:0".parse().expect("parse listener address"); From 3abb65b8ba1e2fec559c0789ae11ea14440c5d9f Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Tue, 6 Oct 2026 13:35:41 -0400 Subject: [PATCH 02/11] fix(dgw): give each feature its own gateway.db connection A cloned libsql connection shares one SQLite handle, so two features using clones would end up inside each other's transactions. GatewayDb now hands each feature a connection of its own, and migrations re-read the schema version under the write lock so two connections can open at once. The CA is bound in a transaction, and a database that has Agents but no CA row is refused instead of being tied to a new CA. Gateway warns when an old agent_tunnel.db is left next to gateway.db. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + crates/agent-tunnel-libsql/Cargo.toml | 1 + crates/agent-tunnel-libsql/src/lib.rs | 155 +++++++++++++++++++---- crates/gateway-db/src/lib.rs | 165 ++++++++++++++++++------- devolutions-gateway/src/service.rs | 12 +- testsuite/Cargo.toml | 2 +- testsuite/tests/agent_tunnel/common.rs | 7 +- 7 files changed, 267 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e06a36e62..bbe5f3ab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,6 +140,7 @@ dependencies = [ "agent-tunnel", "anyhow", "async-trait", + "camino", "gateway-db", "libsql", "tempfile", diff --git a/crates/agent-tunnel-libsql/Cargo.toml b/crates/agent-tunnel-libsql/Cargo.toml index 2a27d8225..2667c251a 100644 --- a/crates/agent-tunnel-libsql/Cargo.toml +++ b/crates/agent-tunnel-libsql/Cargo.toml @@ -17,6 +17,7 @@ tokio = { version = "1.52", features = ["sync"] } uuid = "1.23" [dev-dependencies] +camino = "1.1" gateway-db.path = "../gateway-db" tempfile = "3.24" tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/agent-tunnel-libsql/src/lib.rs b/crates/agent-tunnel-libsql/src/lib.rs index 21c09ecfa..b7e30fcc3 100644 --- a/crates/agent-tunnel-libsql/src/lib.rs +++ b/crates/agent-tunnel-libsql/src/lib.rs @@ -15,10 +15,11 @@ pub struct LibSqlAgentAuthorizationStore { } impl LibSqlAgentAuthorizationStore { - /// Opens the store on a `gateway.db` connection from the `gateway-db` crate. + /// Opens the store on a `gateway.db` connection of its own, from `GatewayDb::connect` in the `gateway-db` crate. /// - /// The stored Agents only make sense with the CA that signed their certificates, so the first open ties the - /// store to `ca_spki_sha256`, and later opens refuse any other CA. + /// The connection must already have up-to-date `agent_tunnel_*` tables; opening checks them and fails if they + /// don't match. The stored Agents only make sense with the CA that signed their certificates, so the first open + /// ties the store to `ca_spki_sha256`, and later opens refuse any other CA. pub async fn open(conn: Connection, ca_spki_sha256: SpkiSha256) -> anyhow::Result { let store = Self { conn: Mutex::new(conn) }; @@ -56,20 +57,44 @@ impl LibSqlAgentAuthorizationStore { async fn bind_ca(&self, ca_spki_sha256: SpkiSha256) -> anyhow::Result<()> { let conn = self.conn.lock().await; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .context("begin Agent authorization CA binding")?; - match Self::bound_ca(&conn).await? { - Some(stored) if stored == ca_spki_sha256 => Ok(()), + match Self::bound_ca(&tx).await? { + Some(stored) if stored == ca_spki_sha256 => return Ok(()), Some(_) => bail!("Agent authorization database belongs to a different CA"), - None => { - conn.execute( - "INSERT INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", - params![CA_SPKI_SHA256_KEY, ca_spki_sha256.to_vec()], - ) - .await - .context("bind Agent authorization database to CA")?; - Ok(()) - } + None => {} } + + let has_agent_data = tx + .query( + "SELECT 1 FROM agent_tunnel_accepted_agents + UNION ALL SELECT 1 FROM agent_tunnel_deleted_agent_keys + UNION ALL SELECT 1 FROM agent_tunnel_enrollment_attempts + LIMIT 1", + (), + ) + .await + .context("query Agent data before binding a CA")? + .next() + .await + .context("read Agent data before binding a CA")? + .is_some(); + if has_agent_data { + bail!("Agent authorization database has Agent data but is missing CA metadata"); + } + + tx.execute( + "INSERT INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", + params![CA_SPKI_SHA256_KEY, ca_spki_sha256.to_vec()], + ) + .await + .context("bind Agent authorization database to CA")?; + tx.commit().await.context("commit Agent authorization CA binding")?; + + Ok(()) } async fn validate_integrity_and_schema(&self) -> anyhow::Result<()> { @@ -464,9 +489,12 @@ impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { mod tests { use super::*; + async fn connect(path: &str) -> anyhow::Result { + gateway_db::GatewayDb::open_path(path).await?.connect().await + } + async fn open_store(path: &str, ca_spki_sha256: SpkiSha256) -> anyhow::Result { - let conn = gateway_db::open_path(path).await?; - LibSqlAgentAuthorizationStore::open(conn, ca_spki_sha256).await + LibSqlAgentAuthorizationStore::open(connect(path).await?, ca_spki_sha256).await } fn enrollment(agent_id: Uuid) -> EnrollmentAttempt { @@ -717,9 +745,7 @@ mod tests { let database_path = temp_dir.path().join("gateway.db"); let database_path = database_path.to_str().expect("temporary database path is UTF-8"); - let conn = gateway_db::open_path(database_path) - .await - .expect("open gateway database"); + let conn = connect(database_path).await.expect("open gateway database"); let bound = LibSqlAgentAuthorizationStore::bound_ca(&conn) .await .expect("read bound CA"); @@ -730,15 +756,100 @@ mod tests { .expect("open Agent authorization store"); drop(store); - let conn = gateway_db::open_path(database_path) - .await - .expect("reopen gateway database"); + let conn = connect(database_path).await.expect("reopen gateway database"); let bound = LibSqlAgentAuthorizationStore::bound_ca(&conn) .await .expect("read bound CA"); assert_eq!(bound, Some([0xCA; 32])); } + #[tokio::test] + async fn database_with_agents_but_no_ca_is_rejected() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let database_path = temp_dir.path().join("gateway.db"); + let database_path = database_path.to_str().expect("temporary database path is UTF-8"); + let store = open_store(database_path, [0xCA; 32]) + .await + .expect("open Agent authorization store"); + store.enroll(enrollment(Uuid::new_v4())).await.expect("enroll Agent"); + drop(store); + + let conn = connect(database_path).await.expect("reopen gateway database"); + conn.execute( + "DELETE FROM agent_tunnel_metadata WHERE key = ?1", + params![CA_SPKI_SHA256_KEY], + ) + .await + .expect("delete CA metadata"); + + let error = LibSqlAgentAuthorizationStore::open(conn, [0xBB; 32]) + .await + .err() + .expect("Agents without a CA must not be bound to a new CA"); + assert!(error.to_string().contains("missing CA")); + } + + #[tokio::test] + async fn agents_enrolled_on_2026_3_still_authorize_after_the_move() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let data_dir = + camino::Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("temporary path is UTF-8"); + let agent_id = Uuid::new_v4(); + + let legacy = libsql::Builder::new_local(data_dir.join("agent_tunnel.db").as_str()) + .build() + .await + .expect("build 2026.3 database") + .connect() + .expect("open 2026.3 database"); + legacy + .execute_batch(include_str!("../../gateway-db/migrations/01_agent_tunnel.sql")) + .await + .expect("create 2026.3 tables"); + legacy + .execute( + "INSERT INTO metadata (key, value) VALUES (?1, ?2)", + params![CA_SPKI_SHA256_KEY, vec![0xCAu8; 32]], + ) + .await + .expect("bind 2026.3 CA"); + legacy + .execute( + "INSERT INTO accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) VALUES (?1, ?2, ?3, ?4)", + params![agent_id.to_string(), "montreal-office", vec![0x11u8; 32], Uuid::new_v4().to_string()], + ) + .await + .expect("store 2026.3 Agent"); + legacy + .execute_batch("PRAGMA user_version = 1") + .await + .expect("record 2026.3 schema version"); + drop(legacy); + + let conn = gateway_db::GatewayDb::open(&data_dir) + .await + .expect("open gateway database") + .connect() + .await + .expect("connect to gateway database"); + assert_eq!( + LibSqlAgentAuthorizationStore::bound_ca(&conn) + .await + .expect("read bound CA"), + Some([0xCA; 32]) + ); + let store = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) + .await + .expect("open Agent authorization store"); + + let accepted = store + .authorize(agent_id, [0x11; 32]) + .await + .expect("authorize moved Agent") + .expect("Agent is still accepted"); + assert_eq!(accepted.name, "montreal-office"); + } + #[tokio::test] async fn database_rejects_missing_required_table() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs index 20c94fb2d..d4a96152e 100644 --- a/crates/gateway-db/src/lib.rs +++ b/crates/gateway-db/src/lib.rs @@ -9,9 +9,9 @@ extern crate tracing; use anyhow::{Context as _, bail}; use camino::{Utf8Path, Utf8PathBuf}; -use libsql::{Connection, TransactionBehavior, params}; +use libsql::{Connection, Database, TransactionBehavior, params}; -pub const FILE_NAME: &str = "gateway.db"; +const FILE_NAME: &str = "gateway.db"; /// Gateway 2026.3 kept the agent tunnel tables in a file of their own. const LEGACY_AGENT_TUNNEL_FILE_NAME: &str = "agent_tunnel.db"; @@ -30,52 +30,78 @@ const MIGRATIONS: &[&str] = &[ include_str!("../migrations/02_prefix_agent_tunnel_tables.sql"), ]; -/// Opens `gateway.db` in `data_dir`, creating it or bringing its tables up to date. -/// -/// The first time a Gateway that ran 2026.3 starts, its `agent_tunnel.db` becomes `gateway.db`. -pub async fn open(data_dir: &Utf8Path) -> anyhow::Result { - let path = data_dir.join(FILE_NAME); - adopt_legacy_agent_tunnel_database(&data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME), &path).await?; - open_path(path.as_str()).await +/// The open `gateway.db`, ready to hand a connection to each feature that keeps tables in it. +#[derive(Debug)] +pub struct GatewayDb { + database: Database, } -/// Opens the database at `path` (`:memory:` works too), creating it or bringing its tables up to date. -pub async fn open_path(path: &str) -> anyhow::Result { - let conn = connect(path).await?; +impl GatewayDb { + /// Opens `gateway.db` in `data_dir`, creating it or bringing its tables up to date. + /// + /// The first time a Gateway that ran 2026.3 starts, its `agent_tunnel.db` becomes `gateway.db`. + pub async fn open(data_dir: &Utf8Path) -> anyhow::Result { + let path = data_dir.join(FILE_NAME); + adopt_legacy_agent_tunnel_database(&data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME), &path).await?; + Self::open_path(path.as_str()).await + } - conn.execute_batch(PRAGMAS) - .await - .context("apply gateway database PRAGMAs")?; - migrate(&conn).await?; + /// Opens the database at `path`, creating it or bringing its tables up to date. + /// + /// With `:memory:`, every connection gets its own empty database with up-to-date tables, which is handy in tests. + pub async fn open_path(path: &str) -> anyhow::Result { + let database = libsql::Builder::new_local(path) + .build() + .await + .with_context(|| format!("build database {path}"))?; + let db = Self { database }; - Ok(conn) -} + db.connect().await?; -async fn connect(path: &str) -> anyhow::Result { - libsql::Builder::new_local(path) - .build() - .await - .with_context(|| format!("build database {path}"))? - .connect() - .with_context(|| format!("connect to database {path}")) + Ok(db) + } + + /// Gives a feature a connection of its own. + /// + /// SQLite tracks transactions per connection, so features sharing one connection would end up inside each + /// other's transactions. Each feature should call this once and keep its connection. + pub async fn connect(&self) -> anyhow::Result { + let conn = self.database.connect().context("connect to the gateway database")?; + + conn.execute_batch(PRAGMAS) + .await + .context("apply gateway database PRAGMAs")?; + migrate(&conn).await?; + + Ok(conn) + } } async fn migrate(conn: &Connection) -> anyhow::Result<()> { - let user_version = schema_version(conn).await?; - - if MIGRATIONS.len() < user_version { - bail!( - "gateway database schema version {user_version} is newer than supported version {}", - MIGRATIONS.len() - ); + if schema_version(conn).await? == MIGRATIONS.len() { + return Ok(()); } - for (index, migration) in MIGRATIONS.iter().enumerate().skip(user_version) { - let version = index + 1; + loop { + // Another connection may migrate at the same time, so the version is read again under the write lock. let tx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .await - .with_context(|| format!("begin gateway database migration {version}"))?; + .context("begin gateway database migration")?; + let user_version = schema_version(&tx).await?; + + let Some(migration) = MIGRATIONS.get(user_version) else { + if MIGRATIONS.len() < user_version { + bail!( + "gateway database schema version {user_version} is newer than supported version {}", + MIGRATIONS.len() + ); + } + + return Ok(()); + }; + + let version = user_version + 1; tx.execute_batch(migration) .await .with_context(|| format!("apply gateway database migration {version}"))?; @@ -86,8 +112,6 @@ async fn migrate(conn: &Connection) -> anyhow::Result<()> { .await .with_context(|| format!("commit gateway database migration {version}"))?; } - - Ok(()) } async fn schema_version(conn: &Connection) -> anyhow::Result { @@ -108,14 +132,27 @@ async fn schema_version(conn: &Connection) -> anyhow::Result { /// We write a complete copy first and only then rename it into place, so `gateway.db` either doesn't exist yet /// or is whole. If Gateway stops halfway, the next start simply does it again. async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) -> anyhow::Result<()> { - if path.exists() || !legacy.exists() { + if !legacy.exists() { + return Ok(()); + } + + if path.exists() { + warn!( + %legacy, + "Ignoring an old agent tunnel database because gateway.db already exists; its Agents are not imported. Delete the file to silence this warning" + ); return Ok(()); } let copy = Utf8PathBuf::from(format!("{path}.tmp")); remove_file_if_exists(©).with_context(|| format!("remove the incomplete copy {copy}"))?; - let legacy_conn = connect(legacy.as_str()).await?; + let legacy_conn = libsql::Builder::new_local(legacy.as_str()) + .build() + .await + .with_context(|| format!("build database {legacy}"))? + .connect() + .with_context(|| format!("connect to database {legacy}"))?; legacy_conn .execute("VACUUM INTO ?1", params![copy.as_str()]) .await @@ -154,6 +191,10 @@ mod tests { (temp_dir, data_dir) } + async fn open_dir(data_dir: &Utf8Path) -> anyhow::Result { + GatewayDb::open(data_dir).await?.connect().await + } + async fn user_version(conn: &Connection) -> usize { schema_version(conn).await.expect("read schema version") } @@ -173,7 +214,12 @@ mod tests { /// Builds the `agent_tunnel.db` that Gateway 2026.3 leaves behind. async fn write_legacy_database(path: &Utf8Path) { - let conn = connect(path.as_str()).await.expect("open legacy database"); + let conn = libsql::Builder::new_local(path.as_str()) + .build() + .await + .expect("build legacy database") + .connect() + .expect("open legacy database"); conn.execute_batch(PRAGMAS).await.expect("apply legacy PRAGMAs"); conn.execute_batch(MIGRATIONS[0]).await.expect("create legacy tables"); conn.execute( @@ -189,7 +235,12 @@ mod tests { #[tokio::test] async fn new_database_has_prefixed_agent_tunnel_tables() { - let conn = open_path(":memory:").await.expect("open gateway database"); + let conn = GatewayDb::open_path(":memory:") + .await + .expect("open gateway database") + .connect() + .await + .expect("connect to gateway database"); assert_eq!(user_version(&conn).await, MIGRATIONS.len()); assert!(table_exists(&conn, "agent_tunnel_metadata").await); @@ -205,7 +256,7 @@ mod tests { let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); write_legacy_database(&legacy).await; - let conn = open(&data_dir).await.expect("open gateway database"); + let conn = open_dir(&data_dir).await.expect("open gateway database"); assert!(!legacy.exists()); assert!(data_dir.join(FILE_NAME).exists()); @@ -233,7 +284,7 @@ mod tests { write_legacy_database(&legacy).await; std::fs::write(data_dir.join(format!("{FILE_NAME}.tmp")), b"half written").expect("leave an incomplete copy"); - let conn = open(&data_dir).await.expect("open gateway database"); + let conn = open_dir(&data_dir).await.expect("open gateway database"); assert!(table_exists(&conn, "agent_tunnel_metadata").await); assert!(!data_dir.join(format!("{FILE_NAME}.tmp")).exists()); @@ -242,11 +293,11 @@ mod tests { #[tokio::test] async fn existing_gateway_database_is_kept() { let (_temp_dir, data_dir) = temp_data_dir(); - drop(open(&data_dir).await.expect("create gateway database")); + drop(open_dir(&data_dir).await.expect("create gateway database")); let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); write_legacy_database(&legacy).await; - let conn = open(&data_dir).await.expect("reopen gateway database"); + let conn = open_dir(&data_dir).await.expect("reopen gateway database"); assert!(legacy.exists()); let rows = conn @@ -259,16 +310,36 @@ mod tests { assert!(rows.is_none()); } + #[tokio::test] + async fn each_connection_has_its_own_transactions() { + let (_temp_dir, data_dir) = temp_data_dir(); + let db = GatewayDb::open(&data_dir).await.expect("open gateway database"); + let first = db.connect().await.expect("connect first feature"); + let second = db.connect().await.expect("connect second feature"); + + let tx = first + .transaction_with_behavior(TransactionBehavior::Deferred) + .await + .expect("begin a transaction on the first connection"); + let second_tx = second + .transaction_with_behavior(TransactionBehavior::Deferred) + .await + .expect("the second connection is not inside the first one's transaction"); + + drop(second_tx); + drop(tx); + } + #[tokio::test] async fn newer_schema_is_rejected() { let (_temp_dir, data_dir) = temp_data_dir(); - let conn = open(&data_dir).await.expect("create gateway database"); + let conn = open_dir(&data_dir).await.expect("create gateway database"); conn.execute_batch("PRAGMA user_version = 99") .await .expect("set unsupported schema version"); drop(conn); - let error = open(&data_dir).await.expect_err("newer schema must be rejected"); + let error = open_dir(&data_dir).await.expect_err("newer schema must be rejected"); assert!(error.to_string().contains("newer than supported")); } diff --git a/devolutions-gateway/src/service.rs b/devolutions-gateway/src/service.rs index 00dab064b..0b68ade04 100644 --- a/devolutions-gateway/src/service.rs +++ b/devolutions-gateway/src/service.rs @@ -275,16 +275,20 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { ); let monitoring_state = Arc::new(network_monitor::State::new(Arc::new(filesystem_monitor_config_cache))?); - let gateway_db = gateway_db::open(&config::get_data_dir()) + let data_dir = config::get_data_dir(); + let gateway_db = gateway_db::GatewayDb::open(&data_dir) .await .context("failed to open the gateway database")?; // Initialize the agent tunnel when enabled. let agent_tunnel_handle = if conf.agent_tunnel.enabled { - let data_dir = config::get_data_dir(); let hostname = &conf.hostname; - let agents_have_ca = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::bound_ca(&gateway_db) + let agent_tunnel_conn = gateway_db + .connect() + .await + .context("failed to connect the agent tunnel to the gateway database")?; + let agents_have_ca = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::bound_ca(&agent_tunnel_conn) .await .context("failed to read the agent tunnel CA identity")? .is_some(); @@ -298,7 +302,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .ca_spki_sha256() .context("failed to identify agent tunnel CA")?; let authorization_store = - agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db.clone(), ca_spki_sha256) + agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(agent_tunnel_conn, ca_spki_sha256) .await .context("failed to initialize Agent authorization database")?; let authorization_store: agent_tunnel::authorization::DynAgentAuthorizationStore = diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 5ab347bbe..e6791ac08 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -34,13 +34,13 @@ tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } agent-sysevent-codes.path = "../crates/agent-sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel", features = ["test-utils"] } agent-tunnel-libsql = { path = "../crates/agent-tunnel-libsql" } -gateway-db.path = "../crates/gateway-db" agent-tunnel-proto = { path = "../crates/agent-tunnel-proto", features = ["serde"] } base64 = "0.23" camino = "1" devolutions-gateway-task = { path = "../crates/devolutions-gateway-task" } devolutions-gateway = { path = "../devolutions-gateway" } futures-util = "0.3" +gateway-db.path = "../crates/gateway-db" ipnetwork = "0.20" jmux-proto = { path = "../crates/jmux-proto" } libsql = { version = "0.9", default-features = false, features = ["core"] } diff --git a/testsuite/tests/agent_tunnel/common.rs b/testsuite/tests/agent_tunnel/common.rs index 4c697e418..e87b76d33 100644 --- a/testsuite/tests/agent_tunnel/common.rs +++ b/testsuite/tests/agent_tunnel/common.rs @@ -175,9 +175,12 @@ pub(super) async fn bind_test_listener() -> TestListener { let data_dir = Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("use utf-8 temporary path"); let ca_manager = CaManager::load_or_generate(&data_dir).expect("generate test ca"); let ca_spki_sha256 = ca_manager.ca_spki_sha256().expect("hash test CA public key"); - let gateway_db = gateway_db::open_path(":memory:") + let gateway_db = gateway_db::GatewayDb::open_path(":memory:") .await - .expect("open test gateway database"); + .expect("open test gateway database") + .connect() + .await + .expect("connect to test gateway database"); let authorization_store = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) .await .expect("open test Agent authorization store"); From 01d41dca9bbc04ae4c19de4b8e7d087e527d8876 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Wed, 7 Oct 2026 12:57:41 -0400 Subject: [PATCH 03/11] refactor(dgw): move the agent tunnel store into gateway-db The agent_tunnel_* tables are created by gateway-db's migrations, so the code reading them now lives next to them as gateway_db::agent_tunnel. The agent-tunnel-libsql crate is removed. --- Cargo.lock | 20 +++------------- crates/agent-tunnel-libsql/Cargo.toml | 23 ------------------- crates/gateway-db/Cargo.toml | 6 ++++- .../lib.rs => gateway-db/src/agent_tunnel.rs} | 8 ++++--- crates/gateway-db/src/lib.rs | 7 ++++-- devolutions-gateway/Cargo.toml | 1 - devolutions-gateway/src/service.rs | 4 ++-- testsuite/Cargo.toml | 1 - testsuite/tests/agent_tunnel/common.rs | 2 +- 9 files changed, 21 insertions(+), 51 deletions(-) delete mode 100644 crates/agent-tunnel-libsql/Cargo.toml rename crates/{agent-tunnel-libsql/src/lib.rs => gateway-db/src/agent_tunnel.rs} (99%) diff --git a/Cargo.lock b/Cargo.lock index bbe5f3ab1..4b9d8b151 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,21 +133,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "agent-tunnel-libsql" -version = "0.0.0" -dependencies = [ - "agent-tunnel", - "anyhow", - "async-trait", - "camino", - "gateway-db", - "libsql", - "tempfile", - "tokio 1.52.3", - "uuid", -] - [[package]] name = "agent-tunnel-proto" version = "0.0.0" @@ -1830,7 +1815,6 @@ name = "devolutions-gateway" version = "2026.3.0" dependencies = [ "agent-tunnel", - "agent-tunnel-libsql", "anyhow", "argon2", "async-trait", @@ -2673,12 +2657,15 @@ dependencies = [ name = "gateway-db" version = "0.0.0" dependencies = [ + "agent-tunnel", "anyhow", + "async-trait", "camino", "libsql", "tempfile", "tokio 1.52.3", "tracing", + "uuid", ] [[package]] @@ -7658,7 +7645,6 @@ version = "0.0.0" dependencies = [ "agent-sysevent-codes", "agent-tunnel", - "agent-tunnel-libsql", "agent-tunnel-proto", "anyhow", "assert_cmd", diff --git a/crates/agent-tunnel-libsql/Cargo.toml b/crates/agent-tunnel-libsql/Cargo.toml deleted file mode 100644 index 2667c251a..000000000 --- a/crates/agent-tunnel-libsql/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "agent-tunnel-libsql" -version = "0.0.0" -edition = "2024" -authors = ["Devolutions Inc. "] -publish = false - -[lints] -workspace = true - -[dependencies] -agent-tunnel.path = "../agent-tunnel" -anyhow = "1" -async-trait = "0.1" -libsql = { version = "0.9", default-features = false, features = ["core", "serde"] } -tokio = { version = "1.52", features = ["sync"] } -uuid = "1.23" - -[dev-dependencies] -camino = "1.1" -gateway-db.path = "../gateway-db" -tempfile = "3.24" -tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/gateway-db/Cargo.toml b/crates/gateway-db/Cargo.toml index ad6dc3255..eb930b1ba 100644 --- a/crates/gateway-db/Cargo.toml +++ b/crates/gateway-db/Cargo.toml @@ -9,10 +9,14 @@ publish = false workspace = true [dependencies] +agent-tunnel.path = "../agent-tunnel" anyhow = "1" +async-trait = "0.1" camino = "1.1" -libsql = { version = "0.9", default-features = false, features = ["core"] } +libsql = { version = "0.9", default-features = false, features = ["core", "serde"] } +tokio = { version = "1.52", features = ["sync"] } tracing = "0.1" +uuid = "1.23" [dev-dependencies] tempfile = "3.24" diff --git a/crates/agent-tunnel-libsql/src/lib.rs b/crates/gateway-db/src/agent_tunnel.rs similarity index 99% rename from crates/agent-tunnel-libsql/src/lib.rs rename to crates/gateway-db/src/agent_tunnel.rs index b7e30fcc3..7c492311b 100644 --- a/crates/agent-tunnel-libsql/src/lib.rs +++ b/crates/gateway-db/src/agent_tunnel.rs @@ -1,3 +1,5 @@ +//! The agent tunnel's tables: accepted Agents, enrollment attempts and the CA they belong to. + use agent_tunnel::authorization::{ AcceptedAgent, AgentAuthorizationStore, EnrollmentAttempt, EnrollmentConflict, EnrollmentOutcome, SpkiSha256, }; @@ -15,7 +17,7 @@ pub struct LibSqlAgentAuthorizationStore { } impl LibSqlAgentAuthorizationStore { - /// Opens the store on a `gateway.db` connection of its own, from `GatewayDb::connect` in the `gateway-db` crate. + /// Opens the store on a `gateway.db` connection of its own, from [`GatewayDb::connect`](crate::GatewayDb::connect). /// /// The connection must already have up-to-date `agent_tunnel_*` tables; opening checks them and fails if they /// don't match. The stored Agents only make sense with the CA that signed their certificates, so the first open @@ -490,7 +492,7 @@ mod tests { use super::*; async fn connect(path: &str) -> anyhow::Result { - gateway_db::GatewayDb::open_path(path).await?.connect().await + crate::GatewayDb::open_path(path).await?.connect().await } async fn open_store(path: &str, ca_spki_sha256: SpkiSha256) -> anyhow::Result { @@ -826,7 +828,7 @@ mod tests { .expect("record 2026.3 schema version"); drop(legacy); - let conn = gateway_db::GatewayDb::open(&data_dir) + let conn = crate::GatewayDb::open(&data_dir) .await .expect("open gateway database") .connect() diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs index d4a96152e..12c82ce00 100644 --- a/crates/gateway-db/src/lib.rs +++ b/crates/gateway-db/src/lib.rs @@ -1,12 +1,15 @@ //! Gateway's own database, `gateway.db`. //! //! When a feature needs to remember a little data, it keeps it here in tables named after the feature -//! (`agent_tunnel_*`, ...), so we don't end up with one database file per feature. The job queue is the -//! exception: it tunes its database for queue work, so it keeps its own file. +//! (`agent_tunnel_*`, ...), with the code reading them in a module of the same name, so we don't end up with one +//! database file per feature. Queues are the exception: the job queue and the traffic audit tune their databases +//! for queue work, so they keep their own files. #[macro_use] extern crate tracing; +pub mod agent_tunnel; + use anyhow::{Context as _, bail}; use camino::{Utf8Path, Utf8PathBuf}; use libsql::{Connection, Database, TransactionBehavior, params}; diff --git a/devolutions-gateway/Cargo.toml b/devolutions-gateway/Cargo.toml index 908f5beaf..24acd386c 100644 --- a/devolutions-gateway/Cargo.toml +++ b/devolutions-gateway/Cargo.toml @@ -36,7 +36,6 @@ network-monitor.path = "../crates/network-monitor" sysevent.path = "../crates/sysevent" sysevent-codes.path = "../crates/sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel" } -agent-tunnel-libsql = { path = "../crates/agent-tunnel-libsql" } gateway-db.path = "../crates/gateway-db" ironrdp-pdu = { version = "0.9", features = ["std"] } ironrdp-core = { version = "0.2", features = ["std"] } diff --git a/devolutions-gateway/src/service.rs b/devolutions-gateway/src/service.rs index 0b68ade04..b07b96c14 100644 --- a/devolutions-gateway/src/service.rs +++ b/devolutions-gateway/src/service.rs @@ -288,7 +288,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .connect() .await .context("failed to connect the agent tunnel to the gateway database")?; - let agents_have_ca = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::bound_ca(&agent_tunnel_conn) + let agents_have_ca = gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::bound_ca(&agent_tunnel_conn) .await .context("failed to read the agent tunnel CA identity")? .is_some(); @@ -302,7 +302,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .ca_spki_sha256() .context("failed to identify agent tunnel CA")?; let authorization_store = - agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(agent_tunnel_conn, ca_spki_sha256) + gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::open(agent_tunnel_conn, ca_spki_sha256) .await .context("failed to initialize Agent authorization database")?; let authorization_store: agent_tunnel::authorization::DynAgentAuthorizationStore = diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index e6791ac08..4e7968d30 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -33,7 +33,6 @@ tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } [dev-dependencies] agent-sysevent-codes.path = "../crates/agent-sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel", features = ["test-utils"] } -agent-tunnel-libsql = { path = "../crates/agent-tunnel-libsql" } agent-tunnel-proto = { path = "../crates/agent-tunnel-proto", features = ["serde"] } base64 = "0.23" camino = "1" diff --git a/testsuite/tests/agent_tunnel/common.rs b/testsuite/tests/agent_tunnel/common.rs index e87b76d33..6021625f5 100644 --- a/testsuite/tests/agent_tunnel/common.rs +++ b/testsuite/tests/agent_tunnel/common.rs @@ -181,7 +181,7 @@ pub(super) async fn bind_test_listener() -> TestListener { .connect() .await .expect("connect to test gateway database"); - let authorization_store = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) + let authorization_store = gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) .await .expect("open test Agent authorization store"); let listen_addr: SocketAddr = "127.0.0.1:0".parse().expect("parse listener address"); From 5ba367d1fb9ae65e2b08985465632f9c8b978235 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Wed, 7 Oct 2026 13:33:23 -0400 Subject: [PATCH 04/11] docs(dgw): add the gateway-db intent --- crates/gateway-db/INTENT.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 crates/gateway-db/INTENT.md diff --git a/crates/gateway-db/INTENT.md b/crates/gateway-db/INTENT.md new file mode 100644 index 000000000..df988dc3e --- /dev/null +++ b/crates/gateway-db/INTENT.md @@ -0,0 +1,15 @@ +# Background +Gateway needs a database to store information for certain features. + +# Invariants +For each domain operations, the `gateway-db` crate should have its domain mod, for example `gateway_db::agent_tunnel::{functions}`. +For each domain table, the table should have its prefix in table's name, for example, `agent_tunnel_*` for agent tunnel domain tables. +The database migration files must live at `crates/gateway-db/migrations`. Any new migration file in `lib.rs` `MIGRATIONS` constant array, and it's append-only. NEVER remove, reorder any migration file from the list. + + +# Goals +All persistent state except the job queue and traffic audit should be stored in this database. + + +# Upgrade from 2026.3 +If for 2026.3, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. \ No newline at end of file From 7bfed87bd8d12b092804c67e6e6b3d1a41d12312 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Wed, 7 Oct 2026 13:36:14 -0400 Subject: [PATCH 05/11] fix(dgw): flush the moved agent tunnel database before removing the old one VACUUM INTO does not flush the copy it writes, so a power loss after the old files were removed could leave only a partial gateway.db. The copy is now flushed, and the rename waits until it is on disk (directory fsync on Unix, MOVEFILE_WRITE_THROUGH on Windows) before the old files go. --- Cargo.lock | 1 + crates/gateway-db/Cargo.toml | 3 ++ crates/gateway-db/src/lib.rs | 54 ++++++++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b9d8b151..6120d2340 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2666,6 +2666,7 @@ dependencies = [ "tokio 1.52.3", "tracing", "uuid", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/gateway-db/Cargo.toml b/crates/gateway-db/Cargo.toml index eb930b1ba..a6b5eb584 100644 --- a/crates/gateway-db/Cargo.toml +++ b/crates/gateway-db/Cargo.toml @@ -18,6 +18,9 @@ tokio = { version = "1.52", features = ["sync"] } tracing = "0.1" uuid = "1.23" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } + [dev-dependencies] tempfile = "3.24" tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs index 12c82ce00..a760975ca 100644 --- a/crates/gateway-db/src/lib.rs +++ b/crates/gateway-db/src/lib.rs @@ -132,8 +132,9 @@ async fn schema_version(conn: &Connection) -> anyhow::Result { /// Turns a 2026.3 `agent_tunnel.db` into `gateway.db`. /// -/// We write a complete copy first and only then rename it into place, so `gateway.db` either doesn't exist yet -/// or is whole. If Gateway stops halfway, the next start simply does it again. +/// We write a complete copy, flush it to disk, and only then rename it into place, so `gateway.db` either doesn't +/// exist yet or is whole. The rename reaches the disk before the old files are removed, so a power loss never leaves +/// only a partial copy behind. If Gateway stops halfway, the next start simply does it again. async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) -> anyhow::Result<()> { if !legacy.exists() { return Ok(()); @@ -162,7 +163,14 @@ async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) .with_context(|| format!("copy {legacy} to {copy}"))?; drop(legacy_conn); - std::fs::rename(©, path).with_context(|| format!("rename {copy} to {path}"))?; + // VACUUM INTO does not flush the file it writes. + std::fs::OpenOptions::new() + .write(true) + .open(©) + .and_then(|file| file.sync_all()) + .with_context(|| format!("flush {copy} to disk"))?; + + rename_durably(©, path).with_context(|| format!("rename {copy} to {path}"))?; for suffix in ["", "-wal", "-shm"] { let legacy_file = Utf8PathBuf::from(format!("{legacy}{suffix}")); @@ -177,6 +185,46 @@ async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) Ok(()) } +/// Renames `from` to `to` and waits until the rename is on disk. +#[cfg(unix)] +fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { + std::fs::rename(from, to)?; + + // On Unix, the new name is only durable once its directory is flushed. + let dir = to + .parent() + .filter(|dir| !dir.as_str().is_empty()) + .unwrap_or(Utf8Path::new(".")); + std::fs::File::open(dir)?.sync_all() +} + +/// Renames `from` to `to` and waits until the rename is on disk. +#[cfg(windows)] +fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt as _; + + use windows_sys::Win32::Storage::FileSystem::{MOVEFILE_WRITE_THROUGH, MoveFileExW}; + + let wide = |path: &Utf8Path| -> Vec { + path.as_std_path() + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect() + }; + let from = wide(from); + let to = wide(to); + + // SAFETY: Both paths are null-terminated UTF-16 strings that outlive the call. + let succeeded = unsafe { MoveFileExW(from.as_ptr(), to.as_ptr(), MOVEFILE_WRITE_THROUGH) }; + + if succeeded == 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(()) +} + fn remove_file_if_exists(path: &Utf8Path) -> std::io::Result<()> { match std::fs::remove_file(path) { Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), From 7370bd3dca841cfd3b9f0a94649ef732908c421b Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Wed, 7 Oct 2026 13:49:48 -0400 Subject: [PATCH 06/11] fix(dgw): avoid a function call in unwrap_or for the database directory --- crates/gateway-db/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs index a760975ca..3b00a85c2 100644 --- a/crates/gateway-db/src/lib.rs +++ b/crates/gateway-db/src/lib.rs @@ -191,10 +191,10 @@ fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { std::fs::rename(from, to)?; // On Unix, the new name is only durable once its directory is flushed. - let dir = to - .parent() - .filter(|dir| !dir.as_str().is_empty()) - .unwrap_or(Utf8Path::new(".")); + let dir = match to.parent() { + Some(dir) if !dir.as_str().is_empty() => dir, + _ => Utf8Path::new("."), + }; std::fs::File::open(dir)?.sync_all() } From c9df5f318ed2b1792bafbb41576ff5fb0ce40034 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Thu, 8 Oct 2026 10:12:42 -0400 Subject: [PATCH 07/11] refactor(dgw): move the agent tunnel store back into agent-tunnel-libsql gateway-db now only owns the shared file: it opens gateway.db, applies the PRAGMAs and hands a connection to each feature. It no longer depends on agent-tunnel, and no longer reads or writes PRAGMA user_version, since no single crate owns every table. agent-tunnel-libsql runs its own migrations again, creates the prefixed agent_tunnel_* tables directly, and records its schema version as the schema_version row of agent_tunnel_metadata. The 2026.3 upgrade moves to agent-tunnel-libsql and runs from service.rs before gateway.db is opened. It checks the old file's user_version (1 converts, 0 removes the empty file, anything else is refused), converts a copy with journal_mode = DELETE, adds the table prefixes and the version row, clears user_version, then flushes and renames it into place. Redoing an interrupted move also removes the copy's -journal, -wal and -shm files. A local AGENTS.md tells agents to keep gateway-db free of feature crates. --- Cargo.lock | 22 +- crates/agent-tunnel-libsql/Cargo.toml | 24 ++ .../migrations/01_agent_tunnel.sql | 8 +- .../src/lib.rs} | 194 +++++++--- crates/agent-tunnel-libsql/src/upgrade.rs | 361 ++++++++++++++++++ crates/gateway-db/AGENTS.md | 9 + crates/gateway-db/Cargo.toml | 7 +- .../02_prefix_agent_tunnel_tables.sql | 4 - crates/gateway-db/src/lib.rs | 301 +++------------ devolutions-gateway/Cargo.toml | 1 + devolutions-gateway/src/service.rs | 7 +- testsuite/Cargo.toml | 1 + testsuite/tests/agent_tunnel/common.rs | 2 +- 13 files changed, 615 insertions(+), 326 deletions(-) create mode 100644 crates/agent-tunnel-libsql/Cargo.toml rename crates/{gateway-db => agent-tunnel-libsql}/migrations/01_agent_tunnel.sql (79%) rename crates/{gateway-db/src/agent_tunnel.rs => agent-tunnel-libsql/src/lib.rs} (86%) create mode 100644 crates/agent-tunnel-libsql/src/upgrade.rs create mode 100644 crates/gateway-db/AGENTS.md delete mode 100644 crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql diff --git a/Cargo.lock b/Cargo.lock index 6120d2340..78fef39a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,22 @@ dependencies = [ "uuid", ] +[[package]] +name = "agent-tunnel-libsql" +version = "0.0.0" +dependencies = [ + "agent-tunnel", + "anyhow", + "async-trait", + "camino", + "gateway-db", + "libsql", + "tempfile", + "tokio 1.52.3", + "tracing", + "uuid", +] + [[package]] name = "agent-tunnel-proto" version = "0.0.0" @@ -1815,6 +1831,7 @@ name = "devolutions-gateway" version = "2026.3.0" dependencies = [ "agent-tunnel", + "agent-tunnel-libsql", "anyhow", "argon2", "async-trait", @@ -2657,15 +2674,11 @@ dependencies = [ name = "gateway-db" version = "0.0.0" dependencies = [ - "agent-tunnel", "anyhow", - "async-trait", "camino", "libsql", "tempfile", "tokio 1.52.3", - "tracing", - "uuid", "windows-sys 0.61.2", ] @@ -7646,6 +7659,7 @@ version = "0.0.0" dependencies = [ "agent-sysevent-codes", "agent-tunnel", + "agent-tunnel-libsql", "agent-tunnel-proto", "anyhow", "assert_cmd", diff --git a/crates/agent-tunnel-libsql/Cargo.toml b/crates/agent-tunnel-libsql/Cargo.toml new file mode 100644 index 000000000..a3612b6ad --- /dev/null +++ b/crates/agent-tunnel-libsql/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "agent-tunnel-libsql" +version = "0.0.0" +edition = "2024" +authors = ["Devolutions Inc. "] +publish = false + +[lints] +workspace = true + +[dependencies] +agent-tunnel.path = "../agent-tunnel" +anyhow = "1" +async-trait = "0.1" +camino = "1.1" +gateway-db.path = "../gateway-db" +libsql = { version = "0.9", default-features = false, features = ["core", "serde"] } +tokio = { version = "1.52", features = ["sync"] } +tracing = "0.1" +uuid = "1.23" + +[dev-dependencies] +tempfile = "3.24" +tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] } diff --git a/crates/gateway-db/migrations/01_agent_tunnel.sql b/crates/agent-tunnel-libsql/migrations/01_agent_tunnel.sql similarity index 79% rename from crates/gateway-db/migrations/01_agent_tunnel.sql rename to crates/agent-tunnel-libsql/migrations/01_agent_tunnel.sql index eaa5fb462..bd3159927 100644 --- a/crates/gateway-db/migrations/01_agent_tunnel.sql +++ b/crates/agent-tunnel-libsql/migrations/01_agent_tunnel.sql @@ -1,9 +1,9 @@ -CREATE TABLE metadata ( +CREATE TABLE agent_tunnel_metadata ( key TEXT PRIMARY KEY, value BLOB NOT NULL ); -CREATE TABLE accepted_agents ( +CREATE TABLE agent_tunnel_accepted_agents ( agent_id TEXT PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE UNIQUE, client_spki_sha256 BLOB NOT NULL, @@ -13,7 +13,7 @@ CREATE TABLE accepted_agents ( CHECK (length(client_spki_sha256) = 32) ); -CREATE TABLE enrollment_attempts ( +CREATE TABLE agent_tunnel_enrollment_attempts ( jti TEXT PRIMARY KEY, agent_id TEXT NOT NULL, request_sha256 BLOB NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE enrollment_attempts ( CHECK (deleted IN (0, 1)) ); -CREATE TABLE deleted_agent_keys ( +CREATE TABLE agent_tunnel_deleted_agent_keys ( client_spki_sha256 BLOB PRIMARY KEY, agent_id TEXT NOT NULL, CHECK (length(client_spki_sha256) = 32) diff --git a/crates/gateway-db/src/agent_tunnel.rs b/crates/agent-tunnel-libsql/src/lib.rs similarity index 86% rename from crates/gateway-db/src/agent_tunnel.rs rename to crates/agent-tunnel-libsql/src/lib.rs index 7c492311b..8d61d5005 100644 --- a/crates/gateway-db/src/agent_tunnel.rs +++ b/crates/agent-tunnel-libsql/src/lib.rs @@ -1,4 +1,13 @@ -//! The agent tunnel's tables: accepted Agents, enrollment attempts and the CA they belong to. +//! The agent tunnel's tables in `gateway.db`: accepted Agents, enrollment attempts and the CA they belong to. +//! +//! The tables are named `agent_tunnel_*`, and this crate alone creates and migrates them. Their schema version is the +//! `schema_version` row of `agent_tunnel_metadata`, not the file's `PRAGMA user_version`, because other features keep +//! their own tables in the same file. + +#[macro_use] +extern crate tracing; + +mod upgrade; use agent_tunnel::authorization::{ AcceptedAgent, AgentAuthorizationStore, EnrollmentAttempt, EnrollmentConflict, EnrollmentOutcome, SpkiSha256, @@ -9,7 +18,13 @@ use libsql::{Connection, TransactionBehavior, params}; use tokio::sync::Mutex; use uuid::Uuid; +pub use self::upgrade::import_2026_3_database; + +/// Every schema change ever made to the `agent_tunnel_*` tables, oldest first. Only ever append to it. +const MIGRATIONS: &[&str] = &[include_str!("../migrations/01_agent_tunnel.sql")]; + const CA_SPKI_SHA256_KEY: &str = "ca_spki_sha256"; +const SCHEMA_VERSION_KEY: &str = "schema_version"; /// Agent authorization kept in the `agent_tunnel_*` tables of `gateway.db`. pub struct LibSqlAgentAuthorizationStore { @@ -17,12 +32,14 @@ pub struct LibSqlAgentAuthorizationStore { } impl LibSqlAgentAuthorizationStore { - /// Opens the store on a `gateway.db` connection of its own, from [`GatewayDb::connect`](crate::GatewayDb::connect). + /// Opens the store on a `gateway.db` connection of its own, from `GatewayDb::connect`. /// - /// The connection must already have up-to-date `agent_tunnel_*` tables; opening checks them and fails if they + /// Opening creates the `agent_tunnel_*` tables or brings them up to date, then checks them and fails if they /// don't match. The stored Agents only make sense with the CA that signed their certificates, so the first open /// ties the store to `ca_spki_sha256`, and later opens refuse any other CA. pub async fn open(conn: Connection, ca_spki_sha256: SpkiSha256) -> anyhow::Result { + migrate(&conn).await?; + let store = Self { conn: Mutex::new(conn) }; store.validate_integrity_and_schema().await?; @@ -35,6 +52,10 @@ impl LibSqlAgentAuthorizationStore { /// /// Once Agents are tied to a CA, Gateway must load that CA and never generate a new one. pub async fn bound_ca(conn: &Connection) -> anyhow::Result> { + if !metadata_table_exists(conn).await? { + return Ok(None); + } + let stored = conn .query( "SELECT value FROM agent_tunnel_metadata WHERE key = ?1", @@ -233,6 +254,81 @@ impl LibSqlAgentAuthorizationStore { } } +/// Creates the `agent_tunnel_*` tables or brings them up to date. +async fn migrate(conn: &Connection) -> anyhow::Result<()> { + loop { + // Another connection may migrate at the same time, so the version is read under the write lock. + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .context("begin agent tunnel migration")?; + let version = schema_version(&tx).await?; + + let Some(migration) = MIGRATIONS.get(version) else { + if MIGRATIONS.len() < version { + bail!( + "agent tunnel schema version {version} is newer than supported version {}", + MIGRATIONS.len() + ); + } + + return Ok(()); + }; + + let version = version + 1; + tx.execute_batch(migration) + .await + .with_context(|| format!("apply agent tunnel migration {version}"))?; + tx.execute( + "INSERT OR REPLACE INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", + params![SCHEMA_VERSION_KEY, i64::try_from(version)?], + ) + .await + .with_context(|| format!("record agent tunnel migration {version}"))?; + tx.commit() + .await + .with_context(|| format!("commit agent tunnel migration {version}"))?; + } +} + +/// Reads the version of the `agent_tunnel_*` tables; 0 when there are none yet. +async fn schema_version(conn: &Connection) -> anyhow::Result { + if !metadata_table_exists(conn).await? { + return Ok(0); + } + + let version = conn + .query( + "SELECT value FROM agent_tunnel_metadata WHERE key = ?1", + params![SCHEMA_VERSION_KEY], + ) + .await + .context("query agent tunnel schema version")? + .next() + .await + .context("read agent tunnel schema version")? + .context("agent tunnel tables have no schema version")? + .get::(0) + .context("decode agent tunnel schema version")?; + + usize::try_from(version).context("agent tunnel schema version is out of range") +} + +async fn metadata_table_exists(conn: &Connection) -> anyhow::Result { + let row = conn + .query( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'agent_tunnel_metadata'", + (), + ) + .await + .context("look for the agent tunnel tables")? + .next() + .await + .context("read the agent tunnel tables")?; + + Ok(row.is_some()) +} + #[async_trait] impl AgentAuthorizationStore for LibSqlAgentAuthorizationStore { async fn enroll(&self, attempt: EnrollmentAttempt) -> anyhow::Result { @@ -492,7 +588,7 @@ mod tests { use super::*; async fn connect(path: &str) -> anyhow::Result { - crate::GatewayDb::open_path(path).await?.connect().await + gateway_db::GatewayDb::open_path(path).await?.connect().await } async fn open_store(path: &str, ca_spki_sha256: SpkiSha256) -> anyhow::Result { @@ -792,66 +888,56 @@ mod tests { } #[tokio::test] - async fn agents_enrolled_on_2026_3_still_authorize_after_the_move() { - let temp_dir = tempfile::tempdir().expect("create temporary directory"); - let data_dir = - camino::Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("temporary path is UTF-8"); - let agent_id = Uuid::new_v4(); + async fn new_tables_record_their_schema_version_in_metadata() { + let conn = connect(":memory:").await.expect("open gateway database"); - let legacy = libsql::Builder::new_local(data_dir.join("agent_tunnel.db").as_str()) - .build() - .await - .expect("build 2026.3 database") - .connect() - .expect("open 2026.3 database"); - legacy - .execute_batch(include_str!("../../gateway-db/migrations/01_agent_tunnel.sql")) - .await - .expect("create 2026.3 tables"); - legacy - .execute( - "INSERT INTO metadata (key, value) VALUES (?1, ?2)", - params![CA_SPKI_SHA256_KEY, vec![0xCAu8; 32]], - ) - .await - .expect("bind 2026.3 CA"); - legacy - .execute( - "INSERT INTO accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) VALUES (?1, ?2, ?3, ?4)", - params![agent_id.to_string(), "montreal-office", vec![0x11u8; 32], Uuid::new_v4().to_string()], - ) - .await - .expect("store 2026.3 Agent"); - legacy - .execute_batch("PRAGMA user_version = 1") + let store = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) .await - .expect("record 2026.3 schema version"); - drop(legacy); + .expect("open Agent authorization store"); + let conn = store.conn.lock().await; - let conn = crate::GatewayDb::open(&data_dir) + assert_eq!( + schema_version(&conn).await.expect("read schema version"), + MIGRATIONS.len() + ); + let user_version = conn + .query("PRAGMA user_version", ()) .await - .expect("open gateway database") - .connect() + .expect("query user_version") + .next() .await - .expect("connect to gateway database"); - assert_eq!( - LibSqlAgentAuthorizationStore::bound_ca(&conn) + .expect("read user_version") + .expect("user_version row") + .get::(0) + .expect("decode user_version"); + assert_eq!(user_version, 0, "the file's user_version belongs to no feature"); + } + + #[tokio::test] + async fn newer_schema_is_rejected() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let database_path = temp_dir.path().join("gateway.db"); + let database_path = database_path.to_str().expect("temporary database path is UTF-8"); + drop( + open_store(database_path, [0xCA; 32]) .await - .expect("read bound CA"), - Some([0xCA; 32]) + .expect("open Agent authorization store"), ); - let store = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) - .await - .expect("open Agent authorization store"); - let accepted = store - .authorize(agent_id, [0x11; 32]) + let conn = connect(database_path).await.expect("reopen gateway database"); + conn.execute( + "UPDATE agent_tunnel_metadata SET value = 99 WHERE key = ?1", + params![SCHEMA_VERSION_KEY], + ) + .await + .expect("set unsupported schema version"); + + let error = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) .await - .expect("authorize moved Agent") - .expect("Agent is still accepted"); - assert_eq!(accepted.name, "montreal-office"); + .err() + .expect("newer schema must be rejected"); + assert!(error.to_string().contains("newer than supported")); } - #[tokio::test] async fn database_rejects_missing_required_table() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); diff --git a/crates/agent-tunnel-libsql/src/upgrade.rs b/crates/agent-tunnel-libsql/src/upgrade.rs new file mode 100644 index 000000000..d9c7cf649 --- /dev/null +++ b/crates/agent-tunnel-libsql/src/upgrade.rs @@ -0,0 +1,361 @@ +//! The upgrade from Gateway 2026.3, which kept the agent tunnel tables in an `agent_tunnel.db` of their own. + +use anyhow::{Context as _, bail}; +use camino::{Utf8Path, Utf8PathBuf}; +use libsql::{Connection, params}; + +use crate::SCHEMA_VERSION_KEY; + +/// Only Gateway 2026.3 wrote this file. +pub(crate) const LEGACY_FILE_NAME: &str = "agent_tunnel.db"; + +/// The only schema 2026.3 ever had, as its `PRAGMA user_version`. +const LEGACY_SCHEMA_VERSION: i64 = 1; + +/// The conversion yields the tables of the first migration. +const CONVERTED_SCHEMA_VERSION: i64 = 1; + +/// Turns the 2026.3 tables into the first version of the `agent_tunnel_*` tables. +const CONVERSION: &str = " + ALTER TABLE metadata RENAME TO agent_tunnel_metadata; + ALTER TABLE accepted_agents RENAME TO agent_tunnel_accepted_agents; + ALTER TABLE enrollment_attempts RENAME TO agent_tunnel_enrollment_attempts; + ALTER TABLE deleted_agent_keys RENAME TO agent_tunnel_deleted_agent_keys; +"; + +/// Turns the `agent_tunnel.db` left by Gateway 2026.3 in `data_dir` into `gateway.db`. +/// +/// Call it before opening `gateway.db`: once `gateway.db` exists, the old file is ignored. It does nothing when there +/// is no old file. +/// +/// We convert a copy, flush it to disk, and only then rename it into place, so `gateway.db` either doesn't exist yet +/// or is whole and in the new layout. The rename reaches the disk before the old files are removed, so a power loss +/// never leaves only a partial copy behind. If Gateway stops halfway, the next start simply does it again. +pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { + let legacy = data_dir.join(LEGACY_FILE_NAME); + let path = data_dir.join(gateway_db::FILE_NAME); + + if !legacy.exists() { + return Ok(()); + } + + if path.exists() { + warn!( + %legacy, + "Ignoring an old agent tunnel database because gateway.db already exists; its Agents are not imported. Delete the file to silence this warning" + ); + return Ok(()); + } + + let legacy_conn = connect(&legacy).await?; + + match legacy_schema_version(&legacy_conn).await? { + // A first start that failed before creating the tables: there is nothing to import. + 0 => { + drop(legacy_conn); + remove_with_sidecars(&legacy); + info!(%legacy, "Removed an empty agent tunnel database"); + return Ok(()); + } + LEGACY_SCHEMA_VERSION => {} + version => bail!("unsupported schema version {version} in {legacy}"), + } + + let copy = Utf8PathBuf::from(format!("{path}.tmp")); + + // A leftover journal of an interrupted attempt would be replayed into the new copy. + for file in with_sidecars(©) { + gateway_db::remove_file_if_exists(&file).with_context(|| format!("remove the incomplete copy {file}"))?; + } + + legacy_conn + .execute("VACUUM INTO ?1", params![copy.as_str()]) + .await + .with_context(|| format!("copy {legacy} to {copy}"))?; + drop(legacy_conn); + + convert(©).await?; + + gateway_db::flush_file(©).with_context(|| format!("flush {copy} to disk"))?; + gateway_db::rename_durably(©, &path).with_context(|| format!("rename {copy} to {path}"))?; + + remove_with_sidecars(&legacy); + + info!(from = %legacy, to = %path, "Moved the agent tunnel database into the gateway database"); + + Ok(()) +} + +async fn convert(copy: &Utf8Path) -> anyhow::Result<()> { + let conn = connect(copy).await?; + + // The changes must be in the file itself before the rename, not in a `-wal` file left behind. + conn.execute_batch("PRAGMA journal_mode = DELETE") + .await + .with_context(|| format!("use a rollback journal for {copy}"))?; + + let tx = conn.transaction().await.context("begin agent tunnel conversion")?; + tx.execute_batch(CONVERSION) + .await + .context("add the agent tunnel table prefixes")?; + tx.execute( + "INSERT INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", + params![SCHEMA_VERSION_KEY, CONVERTED_SCHEMA_VERSION], + ) + .await + .context("record the agent tunnel schema version")?; + tx.execute_batch("PRAGMA user_version = 0") + .await + .context("clear the 2026.3 schema version")?; + tx.commit().await.context("commit agent tunnel conversion")?; + + Ok(()) +} + +async fn connect(path: &Utf8Path) -> anyhow::Result { + libsql::Builder::new_local(path.as_str()) + .build() + .await + .with_context(|| format!("build database {path}"))? + .connect() + .with_context(|| format!("connect to database {path}")) +} + +async fn legacy_schema_version(conn: &Connection) -> anyhow::Result { + conn.query("PRAGMA user_version", ()) + .await + .context("query the 2026.3 agent tunnel schema version")? + .next() + .await + .context("read the 2026.3 agent tunnel schema version")? + .context("schema version query returned no row")? + .get::(0) + .context("decode the 2026.3 agent tunnel schema version") +} + +fn with_sidecars(path: &Utf8Path) -> [Utf8PathBuf; 4] { + ["", "-journal", "-wal", "-shm"].map(|suffix| Utf8PathBuf::from(format!("{path}{suffix}"))) +} + +fn remove_with_sidecars(path: &Utf8Path) { + for file in with_sidecars(path) { + if let Err(error) = gateway_db::remove_file_if_exists(&file) { + warn!(%error, path = %file, "Failed to remove an old agent tunnel database file"); + } + } +} + +#[cfg(test)] +mod tests { + use agent_tunnel::authorization::AgentAuthorizationStore as _; + use gateway_db::GatewayDb; + use uuid::Uuid; + + use super::*; + use crate::LibSqlAgentAuthorizationStore; + + /// The tables Gateway 2026.3 created in `agent_tunnel.db`. + const LEGACY_SCHEMA: &str = " + CREATE TABLE metadata (key TEXT PRIMARY KEY, value BLOB NOT NULL); + CREATE TABLE accepted_agents ( + agent_id TEXT PRIMARY KEY, + name TEXT NOT NULL COLLATE NOCASE UNIQUE, + client_spki_sha256 BLOB NOT NULL, + enrollment_jti TEXT NOT NULL UNIQUE, + CHECK (length(name) BETWEEN 1 AND 255), + CHECK (name = trim(name)), + CHECK (length(client_spki_sha256) = 32) + ); + CREATE TABLE enrollment_attempts ( + jti TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + request_sha256 BLOB NOT NULL, + expires_at INTEGER NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0, + CHECK (length(request_sha256) = 32), + CHECK (deleted IN (0, 1)) + ); + CREATE TABLE deleted_agent_keys ( + client_spki_sha256 BLOB PRIMARY KEY, + agent_id TEXT NOT NULL, + CHECK (length(client_spki_sha256) = 32) + ); + "; + + fn temp_data_dir() -> (tempfile::TempDir, Utf8PathBuf) { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let data_dir = Utf8PathBuf::from_path_buf(temp_dir.path().to_path_buf()).expect("temporary path is UTF-8"); + (temp_dir, data_dir) + } + + /// Writes the `agent_tunnel.db` that Gateway 2026.3 leaves behind, with one Agent, and returns its ID. + async fn write_legacy_database(data_dir: &Utf8Path) -> Uuid { + let agent_id = Uuid::new_v4(); + let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) + .await + .expect("open 2026.3 database"); + conn.execute_batch("PRAGMA journal_mode = WAL") + .await + .expect("use WAL like 2026.3"); + conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3 tables"); + conn.execute( + "INSERT INTO metadata (key, value) VALUES ('ca_spki_sha256', ?1)", + params![vec![0xCAu8; 32]], + ) + .await + .expect("bind 2026.3 CA"); + conn.execute( + "INSERT INTO accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) VALUES (?1, ?2, ?3, ?4)", + params![ + agent_id.to_string(), + "montreal-office", + vec![0x11u8; 32], + Uuid::new_v4().to_string() + ], + ) + .await + .expect("store 2026.3 Agent"); + conn.execute_batch("PRAGMA user_version = 1") + .await + .expect("record 2026.3 schema version"); + agent_id + } + + async fn open_gateway_db(data_dir: &Utf8Path) -> Connection { + GatewayDb::open(data_dir) + .await + .expect("open gateway database") + .connect() + .await + .expect("connect to gateway database") + } + + async fn query_i64(conn: &Connection, sql: &str) -> i64 { + conn.query(sql, ()) + .await + .expect("run query") + .next() + .await + .expect("read row") + .expect("query returns a row") + .get::(0) + .expect("decode integer") + } + + #[tokio::test] + async fn agents_enrolled_on_2026_3_still_authorize_after_the_move() { + let (_temp_dir, data_dir) = temp_data_dir(); + let agent_id = write_legacy_database(&data_dir).await; + + import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + + for file in with_sidecars(&data_dir.join(LEGACY_FILE_NAME)) { + assert!(!file.exists(), "{file} is removed"); + } + assert!(!data_dir.join(format!("{}.tmp", gateway_db::FILE_NAME)).exists()); + + let conn = open_gateway_db(&data_dir).await; + assert_eq!(query_i64(&conn, "PRAGMA user_version").await, 0); + assert_eq!( + query_i64( + &conn, + "SELECT value FROM agent_tunnel_metadata WHERE key = 'schema_version'" + ) + .await, + CONVERTED_SCHEMA_VERSION + ); + assert_eq!( + LibSqlAgentAuthorizationStore::bound_ca(&conn) + .await + .expect("read bound CA"), + Some([0xCA; 32]) + ); + + let store = LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) + .await + .expect("open Agent authorization store"); + let accepted = store + .authorize(agent_id, [0x11; 32]) + .await + .expect("authorize moved Agent") + .expect("Agent is still accepted"); + assert_eq!(accepted.name, "montreal-office"); + } + + #[tokio::test] + async fn interrupted_move_starts_over() { + let (_temp_dir, data_dir) = temp_data_dir(); + write_legacy_database(&data_dir).await; + let copy = Utf8PathBuf::from(format!("{}.tmp", data_dir.join(gateway_db::FILE_NAME))); + std::fs::write(©, b"half written").expect("leave an incomplete copy"); + std::fs::write(format!("{copy}-journal"), b"stale journal").expect("leave its journal"); + + import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + + for file in with_sidecars(©) { + assert!(!file.exists(), "{file} is removed"); + } + let conn = open_gateway_db(&data_dir).await; + LibSqlAgentAuthorizationStore::open(conn, [0xCA; 32]) + .await + .expect("open Agent authorization store"); + } + + #[tokio::test] + async fn existing_gateway_database_is_kept() { + let (_temp_dir, data_dir) = temp_data_dir(); + drop(open_gateway_db(&data_dir).await); + write_legacy_database(&data_dir).await; + + import_2026_3_database(&data_dir).await.expect("ignore 2026.3 database"); + + assert!(data_dir.join(LEGACY_FILE_NAME).exists()); + let conn = open_gateway_db(&data_dir).await; + assert_eq!( + LibSqlAgentAuthorizationStore::bound_ca(&conn) + .await + .expect("read bound CA"), + None + ); + } + + #[tokio::test] + async fn empty_2026_3_database_is_removed() { + let (_temp_dir, data_dir) = temp_data_dir(); + let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) + .await + .expect("create empty 2026.3 database"); + conn.execute_batch("PRAGMA journal_mode = WAL") + .await + .expect("write the database header"); + drop(conn); + assert!(data_dir.join(LEGACY_FILE_NAME).exists()); + + import_2026_3_database(&data_dir) + .await + .expect("import empty 2026.3 database"); + + assert!(!data_dir.join(LEGACY_FILE_NAME).exists()); + assert!(!data_dir.join(gateway_db::FILE_NAME).exists()); + } + + #[tokio::test] + async fn unknown_2026_3_schema_is_refused() { + let (_temp_dir, data_dir) = temp_data_dir(); + let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) + .await + .expect("create 2026.3 database"); + conn.execute_batch("PRAGMA user_version = 7") + .await + .expect("set unknown schema version"); + drop(conn); + + let error = import_2026_3_database(&data_dir) + .await + .expect_err("unknown schema must be refused"); + + assert!(error.to_string().contains("unsupported schema version 7")); + assert!(data_dir.join(LEGACY_FILE_NAME).exists()); + assert!(!data_dir.join(gateway_db::FILE_NAME).exists()); + } +} diff --git a/crates/gateway-db/AGENTS.md b/crates/gateway-db/AGENTS.md new file mode 100644 index 000000000..24b31d13d --- /dev/null +++ b/crates/gateway-db/AGENTS.md @@ -0,0 +1,9 @@ +# gateway-db + +This crate owns the shared `gateway.db` file and nothing else. +Read `INTENT.md` in this directory first. + +- Never add a dependency on a crate that implements feature behavior, such as `agent-tunnel`, `agent-tunnel-libsql` or `provisioner-task`. + A feature's tables, migrations and schema version belong in its own adapter crate, which may depend on `gateway-db`. +- Never read or write `PRAGMA user_version` here: no single crate owns every table in the file. +- After changing `Cargo.toml`, run `cargo tree -p gateway-db --edges normal --depth 1` and check that only generic crates are listed. diff --git a/crates/gateway-db/Cargo.toml b/crates/gateway-db/Cargo.toml index a6b5eb584..1da7eb4c6 100644 --- a/crates/gateway-db/Cargo.toml +++ b/crates/gateway-db/Cargo.toml @@ -9,14 +9,9 @@ publish = false workspace = true [dependencies] -agent-tunnel.path = "../agent-tunnel" anyhow = "1" -async-trait = "0.1" camino = "1.1" -libsql = { version = "0.9", default-features = false, features = ["core", "serde"] } -tokio = { version = "1.52", features = ["sync"] } -tracing = "0.1" -uuid = "1.23" +libsql = { version = "0.9", default-features = false, features = ["core"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql b/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql deleted file mode 100644 index 8ce15eb72..000000000 --- a/crates/gateway-db/migrations/02_prefix_agent_tunnel_tables.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE metadata RENAME TO agent_tunnel_metadata; -ALTER TABLE accepted_agents RENAME TO agent_tunnel_accepted_agents; -ALTER TABLE enrollment_attempts RENAME TO agent_tunnel_enrollment_attempts; -ALTER TABLE deleted_agent_keys RENAME TO agent_tunnel_deleted_agent_keys; diff --git a/crates/gateway-db/src/lib.rs b/crates/gateway-db/src/lib.rs index 3b00a85c2..8f96df4f7 100644 --- a/crates/gateway-db/src/lib.rs +++ b/crates/gateway-db/src/lib.rs @@ -1,23 +1,19 @@ //! Gateway's own database, `gateway.db`. //! -//! When a feature needs to remember a little data, it keeps it here in tables named after the feature -//! (`agent_tunnel_*`, ...), with the code reading them in a module of the same name, so we don't end up with one -//! database file per feature. Queues are the exception: the job queue and the traffic audit tune their databases -//! for queue work, so they keep their own files. - -#[macro_use] -extern crate tracing; - -pub mod agent_tunnel; - -use anyhow::{Context as _, bail}; -use camino::{Utf8Path, Utf8PathBuf}; -use libsql::{Connection, Database, TransactionBehavior, params}; +//! Features that need to remember a little data keep it here, in tables named after the feature (`agent_tunnel_*`, +//! ...), so we don't end up with one database file per feature. This crate only owns the file: it opens it, applies +//! the PRAGMAs and hands a connection to each feature. Each feature's tables, migrations and schema version live in +//! the feature's own adapter crate, such as `agent-tunnel-libsql`; the table prefix is the compatibility boundary. +//! +//! Queues are the exception: the job queue and the traffic audit tune their databases for queue work, so they keep +//! their own files. -const FILE_NAME: &str = "gateway.db"; +use anyhow::Context as _; +use camino::Utf8Path; +use libsql::{Connection, Database}; -/// Gateway 2026.3 kept the agent tunnel tables in a file of their own. -const LEGACY_AGENT_TUNNEL_FILE_NAME: &str = "agent_tunnel.db"; +/// The name of the database file in Gateway's data directory. +pub const FILE_NAME: &str = "gateway.db"; const PRAGMAS: &str = " PRAGMA journal_mode = WAL; @@ -27,12 +23,6 @@ const PRAGMAS: &str = " PRAGMA temp_store = MEMORY; "; -/// Every schema change ever made to `gateway.db`, oldest first. Only ever append to it. -const MIGRATIONS: &[&str] = &[ - include_str!("../migrations/01_agent_tunnel.sql"), - include_str!("../migrations/02_prefix_agent_tunnel_tables.sql"), -]; - /// The open `gateway.db`, ready to hand a connection to each feature that keeps tables in it. #[derive(Debug)] pub struct GatewayDb { @@ -40,18 +30,14 @@ pub struct GatewayDb { } impl GatewayDb { - /// Opens `gateway.db` in `data_dir`, creating it or bringing its tables up to date. - /// - /// The first time a Gateway that ran 2026.3 starts, its `agent_tunnel.db` becomes `gateway.db`. + /// Opens `gateway.db` in `data_dir`, creating it if needed. pub async fn open(data_dir: &Utf8Path) -> anyhow::Result { - let path = data_dir.join(FILE_NAME); - adopt_legacy_agent_tunnel_database(&data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME), &path).await?; - Self::open_path(path.as_str()).await + Self::open_path(data_dir.join(FILE_NAME).as_str()).await } - /// Opens the database at `path`, creating it or bringing its tables up to date. + /// Opens the database at `path`, creating it if needed. /// - /// With `:memory:`, every connection gets its own empty database with up-to-date tables, which is handy in tests. + /// With `:memory:`, every connection gets its own empty database, which is handy in tests. pub async fn open_path(path: &str) -> anyhow::Result { let database = libsql::Builder::new_local(path) .build() @@ -74,120 +60,19 @@ impl GatewayDb { conn.execute_batch(PRAGMAS) .await .context("apply gateway database PRAGMAs")?; - migrate(&conn).await?; Ok(conn) } } -async fn migrate(conn: &Connection) -> anyhow::Result<()> { - if schema_version(conn).await? == MIGRATIONS.len() { - return Ok(()); - } - - loop { - // Another connection may migrate at the same time, so the version is read again under the write lock. - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .context("begin gateway database migration")?; - let user_version = schema_version(&tx).await?; - - let Some(migration) = MIGRATIONS.get(user_version) else { - if MIGRATIONS.len() < user_version { - bail!( - "gateway database schema version {user_version} is newer than supported version {}", - MIGRATIONS.len() - ); - } - - return Ok(()); - }; - - let version = user_version + 1; - tx.execute_batch(migration) - .await - .with_context(|| format!("apply gateway database migration {version}"))?; - tx.execute_batch(&format!("PRAGMA user_version = {version}")) - .await - .with_context(|| format!("record gateway database migration {version}"))?; - tx.commit() - .await - .with_context(|| format!("commit gateway database migration {version}"))?; - } -} - -async fn schema_version(conn: &Connection) -> anyhow::Result { - let row = conn - .query("PRAGMA user_version", ()) - .await - .context("query gateway database schema version")? - .next() - .await - .context("read gateway database schema version")? - .context("gateway database schema version query returned no row")?; - let user_version = row.get::(0).context("decode gateway database schema version")?; - usize::try_from(user_version).context("gateway database schema version is too large") -} - -/// Turns a 2026.3 `agent_tunnel.db` into `gateway.db`. -/// -/// We write a complete copy, flush it to disk, and only then rename it into place, so `gateway.db` either doesn't -/// exist yet or is whole. The rename reaches the disk before the old files are removed, so a power loss never leaves -/// only a partial copy behind. If Gateway stops halfway, the next start simply does it again. -async fn adopt_legacy_agent_tunnel_database(legacy: &Utf8Path, path: &Utf8Path) -> anyhow::Result<()> { - if !legacy.exists() { - return Ok(()); - } - - if path.exists() { - warn!( - %legacy, - "Ignoring an old agent tunnel database because gateway.db already exists; its Agents are not imported. Delete the file to silence this warning" - ); - return Ok(()); - } - - let copy = Utf8PathBuf::from(format!("{path}.tmp")); - remove_file_if_exists(©).with_context(|| format!("remove the incomplete copy {copy}"))?; - - let legacy_conn = libsql::Builder::new_local(legacy.as_str()) - .build() - .await - .with_context(|| format!("build database {legacy}"))? - .connect() - .with_context(|| format!("connect to database {legacy}"))?; - legacy_conn - .execute("VACUUM INTO ?1", params![copy.as_str()]) - .await - .with_context(|| format!("copy {legacy} to {copy}"))?; - drop(legacy_conn); - - // VACUUM INTO does not flush the file it writes. - std::fs::OpenOptions::new() - .write(true) - .open(©) - .and_then(|file| file.sync_all()) - .with_context(|| format!("flush {copy} to disk"))?; - - rename_durably(©, path).with_context(|| format!("rename {copy} to {path}"))?; - - for suffix in ["", "-wal", "-shm"] { - let legacy_file = Utf8PathBuf::from(format!("{legacy}{suffix}")); - - if let Err(error) = remove_file_if_exists(&legacy_file) { - warn!(%error, path = %legacy_file, "Failed to remove the old agent tunnel database file"); - } - } - - info!(from = %legacy, to = %path, "Moved the agent tunnel database into the gateway database"); - - Ok(()) +/// Waits until the content of the file at `path` is on disk. +pub fn flush_file(path: &Utf8Path) -> std::io::Result<()> { + std::fs::OpenOptions::new().write(true).open(path)?.sync_all() } /// Renames `from` to `to` and waits until the rename is on disk. #[cfg(unix)] -fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { +pub fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { std::fs::rename(from, to)?; // On Unix, the new name is only durable once its directory is flushed. @@ -200,7 +85,7 @@ fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { /// Renames `from` to `to` and waits until the rename is on disk. #[cfg(windows)] -fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { +pub fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { use std::os::windows::ffi::OsStrExt as _; use windows_sys::Win32::Storage::FileSystem::{MOVEFILE_WRITE_THROUGH, MoveFileExW}; @@ -225,7 +110,8 @@ fn rename_durably(from: &Utf8Path, to: &Utf8Path) -> std::io::Result<()> { Ok(()) } -fn remove_file_if_exists(path: &Utf8Path) -> std::io::Result<()> { +/// Removes the file at `path`, if there is one. +pub fn remove_file_if_exists(path: &Utf8Path) -> std::io::Result<()> { match std::fs::remove_file(path) { Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), result => result, @@ -234,6 +120,9 @@ fn remove_file_if_exists(path: &Utf8Path) -> std::io::Result<()> { #[cfg(test)] mod tests { + use camino::Utf8PathBuf; + use libsql::TransactionBehavior; + use super::*; fn temp_data_dir() -> (tempfile::TempDir, Utf8PathBuf) { @@ -242,123 +131,32 @@ mod tests { (temp_dir, data_dir) } - async fn open_dir(data_dir: &Utf8Path) -> anyhow::Result { - GatewayDb::open(data_dir).await?.connect().await - } - - async fn user_version(conn: &Connection) -> usize { - schema_version(conn).await.expect("read schema version") - } - - async fn table_exists(conn: &Connection, table: &str) -> bool { - conn.query( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1", - params![table], - ) - .await - .expect("query tables") - .next() - .await - .expect("read tables") - .is_some() - } - - /// Builds the `agent_tunnel.db` that Gateway 2026.3 leaves behind. - async fn write_legacy_database(path: &Utf8Path) { - let conn = libsql::Builder::new_local(path.as_str()) - .build() + async fn query_i64(conn: &Connection, sql: &str) -> i64 { + conn.query(sql, ()) .await - .expect("build legacy database") - .connect() - .expect("open legacy database"); - conn.execute_batch(PRAGMAS).await.expect("apply legacy PRAGMAs"); - conn.execute_batch(MIGRATIONS[0]).await.expect("create legacy tables"); - conn.execute( - "INSERT INTO metadata (key, value) VALUES ('ca_spki_sha256', ?1)", - params![vec![0xCAu8; 32]], - ) - .await - .expect("bind legacy CA"); - conn.execute_batch("PRAGMA user_version = 1") + .expect("run query") + .next() .await - .expect("record legacy schema version"); + .expect("read row") + .expect("query returns a row") + .get::(0) + .expect("decode integer") } #[tokio::test] - async fn new_database_has_prefixed_agent_tunnel_tables() { - let conn = GatewayDb::open_path(":memory:") + async fn open_creates_an_empty_database_without_a_schema_version() { + let (_temp_dir, data_dir) = temp_data_dir(); + + let conn = GatewayDb::open(&data_dir) .await .expect("open gateway database") .connect() .await .expect("connect to gateway database"); - assert_eq!(user_version(&conn).await, MIGRATIONS.len()); - assert!(table_exists(&conn, "agent_tunnel_metadata").await); - assert!(table_exists(&conn, "agent_tunnel_accepted_agents").await); - assert!(table_exists(&conn, "agent_tunnel_enrollment_attempts").await); - assert!(table_exists(&conn, "agent_tunnel_deleted_agent_keys").await); - assert!(!table_exists(&conn, "metadata").await); - } - - #[tokio::test] - async fn legacy_agent_tunnel_database_becomes_gateway_database() { - let (_temp_dir, data_dir) = temp_data_dir(); - let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); - write_legacy_database(&legacy).await; - - let conn = open_dir(&data_dir).await.expect("open gateway database"); - - assert!(!legacy.exists()); assert!(data_dir.join(FILE_NAME).exists()); - assert_eq!(user_version(&conn).await, MIGRATIONS.len()); - let ca = conn - .query( - "SELECT value FROM agent_tunnel_metadata WHERE key = 'ca_spki_sha256'", - (), - ) - .await - .expect("query CA") - .next() - .await - .expect("read CA") - .expect("CA survives the move") - .get::>(0) - .expect("decode CA"); - assert_eq!(ca, vec![0xCAu8; 32]); - } - - #[tokio::test] - async fn interrupted_move_starts_over() { - let (_temp_dir, data_dir) = temp_data_dir(); - let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); - write_legacy_database(&legacy).await; - std::fs::write(data_dir.join(format!("{FILE_NAME}.tmp")), b"half written").expect("leave an incomplete copy"); - - let conn = open_dir(&data_dir).await.expect("open gateway database"); - - assert!(table_exists(&conn, "agent_tunnel_metadata").await); - assert!(!data_dir.join(format!("{FILE_NAME}.tmp")).exists()); - } - - #[tokio::test] - async fn existing_gateway_database_is_kept() { - let (_temp_dir, data_dir) = temp_data_dir(); - drop(open_dir(&data_dir).await.expect("create gateway database")); - let legacy = data_dir.join(LEGACY_AGENT_TUNNEL_FILE_NAME); - write_legacy_database(&legacy).await; - - let conn = open_dir(&data_dir).await.expect("reopen gateway database"); - - assert!(legacy.exists()); - let rows = conn - .query("SELECT 1 FROM agent_tunnel_metadata", ()) - .await - .expect("query CA") - .next() - .await - .expect("read CA"); - assert!(rows.is_none()); + assert_eq!(query_i64(&conn, "PRAGMA user_version").await, 0); + assert_eq!(query_i64(&conn, "SELECT count(*) FROM sqlite_master").await, 0); } #[tokio::test] @@ -381,17 +179,18 @@ mod tests { drop(tx); } - #[tokio::test] - async fn newer_schema_is_rejected() { + #[test] + fn copy_is_flushed_and_renamed_into_place() { let (_temp_dir, data_dir) = temp_data_dir(); - let conn = open_dir(&data_dir).await.expect("create gateway database"); - conn.execute_batch("PRAGMA user_version = 99") - .await - .expect("set unsupported schema version"); - drop(conn); + let from = data_dir.join("gateway.db.tmp"); + let to = data_dir.join(FILE_NAME); + std::fs::write(&from, b"copy").expect("write the copy"); - let error = open_dir(&data_dir).await.expect_err("newer schema must be rejected"); + flush_file(&from).expect("flush the copy"); + rename_durably(&from, &to).expect("rename the copy"); - assert!(error.to_string().contains("newer than supported")); + assert!(!from.exists()); + assert_eq!(std::fs::read(&to).expect("read the target"), b"copy"); + remove_file_if_exists(&from).expect("a missing file is not an error"); } } diff --git a/devolutions-gateway/Cargo.toml b/devolutions-gateway/Cargo.toml index 24acd386c..73db5a544 100644 --- a/devolutions-gateway/Cargo.toml +++ b/devolutions-gateway/Cargo.toml @@ -36,6 +36,7 @@ network-monitor.path = "../crates/network-monitor" sysevent.path = "../crates/sysevent" sysevent-codes.path = "../crates/sysevent-codes" agent-tunnel = { path = "../crates/agent-tunnel" } +agent-tunnel-libsql.path = "../crates/agent-tunnel-libsql" gateway-db.path = "../crates/gateway-db" ironrdp-pdu = { version = "0.9", features = ["std"] } ironrdp-core = { version = "0.2", features = ["std"] } diff --git a/devolutions-gateway/src/service.rs b/devolutions-gateway/src/service.rs index b07b96c14..a69e9c117 100644 --- a/devolutions-gateway/src/service.rs +++ b/devolutions-gateway/src/service.rs @@ -276,6 +276,9 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { let monitoring_state = Arc::new(network_monitor::State::new(Arc::new(filesystem_monitor_config_cache))?); let data_dir = config::get_data_dir(); + agent_tunnel_libsql::import_2026_3_database(&data_dir) + .await + .context("failed to import the 2026.3 agent tunnel database")?; let gateway_db = gateway_db::GatewayDb::open(&data_dir) .await .context("failed to open the gateway database")?; @@ -288,7 +291,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .connect() .await .context("failed to connect the agent tunnel to the gateway database")?; - let agents_have_ca = gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::bound_ca(&agent_tunnel_conn) + let agents_have_ca = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::bound_ca(&agent_tunnel_conn) .await .context("failed to read the agent tunnel CA identity")? .is_some(); @@ -302,7 +305,7 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { .ca_spki_sha256() .context("failed to identify agent tunnel CA")?; let authorization_store = - gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::open(agent_tunnel_conn, ca_spki_sha256) + agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(agent_tunnel_conn, ca_spki_sha256) .await .context("failed to initialize Agent authorization database")?; let authorization_store: agent_tunnel::authorization::DynAgentAuthorizationStore = diff --git a/testsuite/Cargo.toml b/testsuite/Cargo.toml index 4e7968d30..b6c687fc1 100644 --- a/testsuite/Cargo.toml +++ b/testsuite/Cargo.toml @@ -39,6 +39,7 @@ camino = "1" devolutions-gateway-task = { path = "../crates/devolutions-gateway-task" } devolutions-gateway = { path = "../devolutions-gateway" } futures-util = "0.3" +agent-tunnel-libsql.path = "../crates/agent-tunnel-libsql" gateway-db.path = "../crates/gateway-db" ipnetwork = "0.20" jmux-proto = { path = "../crates/jmux-proto" } diff --git a/testsuite/tests/agent_tunnel/common.rs b/testsuite/tests/agent_tunnel/common.rs index 6021625f5..e87b76d33 100644 --- a/testsuite/tests/agent_tunnel/common.rs +++ b/testsuite/tests/agent_tunnel/common.rs @@ -181,7 +181,7 @@ pub(super) async fn bind_test_listener() -> TestListener { .connect() .await .expect("connect to test gateway database"); - let authorization_store = gateway_db::agent_tunnel::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) + let authorization_store = agent_tunnel_libsql::LibSqlAgentAuthorizationStore::open(gateway_db, ca_spki_sha256) .await .expect("open test Agent authorization store"); let listen_addr: SocketAddr = "127.0.0.1:0".parse().expect("parse listener address"); From 3b3f7ba8d18a56d97986b0f346de59142791775f Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Thu, 8 Oct 2026 11:42:04 -0400 Subject: [PATCH 08/11] docs(dgw): keep feature stores out of gateway-db in its intent --- crates/gateway-db/INTENT.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/gateway-db/INTENT.md b/crates/gateway-db/INTENT.md index df988dc3e..9fafbe260 100644 --- a/crates/gateway-db/INTENT.md +++ b/crates/gateway-db/INTENT.md @@ -2,9 +2,13 @@ Gateway needs a database to store information for certain features. # Invariants -For each domain operations, the `gateway-db` crate should have its domain mod, for example `gateway_db::agent_tunnel::{functions}`. -For each domain table, the table should have its prefix in table's name, for example, `agent_tunnel_*` for agent tunnel domain tables. -The database migration files must live at `crates/gateway-db/migrations`. Any new migration file in `lib.rs` `MIGRATIONS` constant array, and it's append-only. NEVER remove, reorder any migration file from the list. +`gateway-db` only owns the shared `gateway.db` file: it opens the file, applies the PRAGMAs, and hands out a connection per feature. + +`gateway-db` must never depend on crates that implement feature behavior, such as `agent-tunnel`. Adapters (for example `agent-tunnel-libsql`) may depend on `gateway-db`, never the other way around. + +For each domain table, the table should have its prefix in table's name, for example, `agent_tunnel_*` for agent tunnel domain tables. The prefix is the compatibility boundary. + +`gateway-db` does not read or write `PRAGMA user_version`. Each adapter owns its migrations and tracks its own schema version within its prefixed tables, for example as a row in `agent_tunnel_metadata`. Migrations are append-only. NEVER remove or reorder a migration. # Goals @@ -12,4 +16,8 @@ All persistent state except the job queue and traffic audit should be stored in # Upgrade from 2026.3 -If for 2026.3, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. \ No newline at end of file +If for 2026.3, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. + +# Future +Later, `gateway-db` could take migrations from each adapter. +An adapter would pass its prefix and its list of migrations, and `gateway-db` would run them and track a version per prefix \ No newline at end of file From 4c66f2e9e4463c85779b6d549110de52bd75a84b Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Thu, 8 Oct 2026 11:48:46 -0400 Subject: [PATCH 09/11] fix(dgw): harden the 2026.3 agent tunnel import The conversion now checks that the copy really switched to a rollback journal before renaming it, a 2026.3 file with tables but no schema version is refused instead of removed, and missing-file checks report I/O errors. The warning for a leftover agent_tunnel.db no longer claims its Agents were not imported, since a crash right after the rename leaves the file behind with its Agents already moved. The migration and the conversion record the schema version the same way, and an up-to-date migration ends its transaction explicitly. New tests cover Agents still in the 2026.3 WAL and the renamed file being in rollback journal mode with no journals left behind. --- crates/agent-tunnel-libsql/src/lib.rs | 22 +++- crates/agent-tunnel-libsql/src/upgrade.rs | 150 ++++++++++++++++++---- crates/gateway-db/AGENTS.md | 2 +- 3 files changed, 145 insertions(+), 29 deletions(-) diff --git a/crates/agent-tunnel-libsql/src/lib.rs b/crates/agent-tunnel-libsql/src/lib.rs index 8d61d5005..43c9db6ea 100644 --- a/crates/agent-tunnel-libsql/src/lib.rs +++ b/crates/agent-tunnel-libsql/src/lib.rs @@ -265,6 +265,8 @@ async fn migrate(conn: &Connection) -> anyhow::Result<()> { let version = schema_version(&tx).await?; let Some(migration) = MIGRATIONS.get(version) else { + tx.rollback().await.context("end agent tunnel migration")?; + if MIGRATIONS.len() < version { bail!( "agent tunnel schema version {version} is newer than supported version {}", @@ -279,18 +281,25 @@ async fn migrate(conn: &Connection) -> anyhow::Result<()> { tx.execute_batch(migration) .await .with_context(|| format!("apply agent tunnel migration {version}"))?; - tx.execute( - "INSERT OR REPLACE INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", - params![SCHEMA_VERSION_KEY, i64::try_from(version)?], - ) - .await - .with_context(|| format!("record agent tunnel migration {version}"))?; + record_schema_version(&tx, i64::try_from(version)?).await?; tx.commit() .await .with_context(|| format!("commit agent tunnel migration {version}"))?; } } +/// Records the version of the `agent_tunnel_*` tables, inside the transaction that changes them. +async fn record_schema_version(tx: &Connection, version: i64) -> anyhow::Result<()> { + tx.execute( + "INSERT OR REPLACE INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", + params![SCHEMA_VERSION_KEY, version], + ) + .await + .with_context(|| format!("record agent tunnel schema version {version}"))?; + + Ok(()) +} + /// Reads the version of the `agent_tunnel_*` tables; 0 when there are none yet. async fn schema_version(conn: &Connection) -> anyhow::Result { if !metadata_table_exists(conn).await? { @@ -938,6 +947,7 @@ mod tests { .expect("newer schema must be rejected"); assert!(error.to_string().contains("newer than supported")); } + #[tokio::test] async fn database_rejects_missing_required_table() { let temp_dir = tempfile::tempdir().expect("create temporary directory"); diff --git a/crates/agent-tunnel-libsql/src/upgrade.rs b/crates/agent-tunnel-libsql/src/upgrade.rs index d9c7cf649..b6747e076 100644 --- a/crates/agent-tunnel-libsql/src/upgrade.rs +++ b/crates/agent-tunnel-libsql/src/upgrade.rs @@ -4,7 +4,7 @@ use anyhow::{Context as _, bail}; use camino::{Utf8Path, Utf8PathBuf}; use libsql::{Connection, params}; -use crate::SCHEMA_VERSION_KEY; +use crate::record_schema_version; /// Only Gateway 2026.3 wrote this file. pub(crate) const LEGACY_FILE_NAME: &str = "agent_tunnel.db"; @@ -35,15 +35,12 @@ pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { let legacy = data_dir.join(LEGACY_FILE_NAME); let path = data_dir.join(gateway_db::FILE_NAME); - if !legacy.exists() { + if !exists(&legacy)? { return Ok(()); } - if path.exists() { - warn!( - %legacy, - "Ignoring an old agent tunnel database because gateway.db already exists; its Agents are not imported. Delete the file to silence this warning" - ); + if exists(&path)? { + warn!(%legacy, "Ignoring agent_tunnel.db because gateway.db already exists"); return Ok(()); } @@ -52,9 +49,16 @@ pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { match legacy_schema_version(&legacy_conn).await? { // A first start that failed before creating the tables: there is nothing to import. 0 => { + if has_tables(&legacy_conn).await? { + bail!("{legacy} has tables but no schema version"); + } + drop(legacy_conn); - remove_with_sidecars(&legacy); - info!(%legacy, "Removed an empty agent tunnel database"); + + if remove_with_sidecars(&legacy) { + info!(%legacy, "Removed an empty agent tunnel database"); + } + return Ok(()); } LEGACY_SCHEMA_VERSION => {} @@ -79,7 +83,8 @@ pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { gateway_db::flush_file(©).with_context(|| format!("flush {copy} to disk"))?; gateway_db::rename_durably(©, &path).with_context(|| format!("rename {copy} to {path}"))?; - remove_with_sidecars(&legacy); + // A failed removal is only noise: gateway.db exists now, so the old files are ignored. + let _ = remove_with_sidecars(&legacy); info!(from = %legacy, to = %path, "Moved the agent tunnel database into the gateway database"); @@ -89,21 +94,27 @@ pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { async fn convert(copy: &Utf8Path) -> anyhow::Result<()> { let conn = connect(copy).await?; - // The changes must be in the file itself before the rename, not in a `-wal` file left behind. - conn.execute_batch("PRAGMA journal_mode = DELETE") + // The changes must be in the file itself before the rename, not in a `-wal` file left behind. SQLite keeps the old + // mode without an error when it can't switch, so we check the mode it reports. + let journal_mode = conn + .query("PRAGMA journal_mode = DELETE", ()) .await - .with_context(|| format!("use a rollback journal for {copy}"))?; + .with_context(|| format!("use a rollback journal for {copy}"))? + .next() + .await + .with_context(|| format!("read the journal mode of {copy}"))? + .context("journal mode query returned no row")? + .get::(0) + .context("decode the journal mode")?; + if !journal_mode.eq_ignore_ascii_case("delete") { + bail!("journal mode of {copy} is {journal_mode}, not delete"); + } let tx = conn.transaction().await.context("begin agent tunnel conversion")?; tx.execute_batch(CONVERSION) .await .context("add the agent tunnel table prefixes")?; - tx.execute( - "INSERT INTO agent_tunnel_metadata (key, value) VALUES (?1, ?2)", - params![SCHEMA_VERSION_KEY, CONVERTED_SCHEMA_VERSION], - ) - .await - .context("record the agent tunnel schema version")?; + record_schema_version(&tx, CONVERTED_SCHEMA_VERSION).await?; tx.execute_batch("PRAGMA user_version = 0") .await .context("clear the 2026.3 schema version")?; @@ -121,6 +132,22 @@ async fn connect(path: &Utf8Path) -> anyhow::Result { .with_context(|| format!("connect to database {path}")) } +fn exists(path: &Utf8Path) -> anyhow::Result { + path.try_exists().with_context(|| format!("look for {path}")) +} + +async fn has_tables(conn: &Connection) -> anyhow::Result { + let row = conn + .query("SELECT 1 FROM sqlite_master LIMIT 1", ()) + .await + .context("query the 2026.3 agent tunnel tables")? + .next() + .await + .context("read the 2026.3 agent tunnel tables")?; + + Ok(row.is_some()) +} + async fn legacy_schema_version(conn: &Connection) -> anyhow::Result { conn.query("PRAGMA user_version", ()) .await @@ -137,12 +164,18 @@ fn with_sidecars(path: &Utf8Path) -> [Utf8PathBuf; 4] { ["", "-journal", "-wal", "-shm"].map(|suffix| Utf8PathBuf::from(format!("{path}{suffix}"))) } -fn remove_with_sidecars(path: &Utf8Path) { +/// Removes the database at `path` with its journals; returns whether every file is gone. +fn remove_with_sidecars(path: &Utf8Path) -> bool { + let mut removed = true; + for file in with_sidecars(path) { if let Err(error) = gateway_db::remove_file_if_exists(&file) { warn!(%error, path = %file, "Failed to remove an old agent tunnel database file"); + removed = false; } } + + removed } #[cfg(test)] @@ -190,13 +223,18 @@ mod tests { /// Writes the `agent_tunnel.db` that Gateway 2026.3 leaves behind, with one Agent, and returns its ID. async fn write_legacy_database(data_dir: &Utf8Path) -> Uuid { - let agent_id = Uuid::new_v4(); let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) .await .expect("open 2026.3 database"); conn.execute_batch("PRAGMA journal_mode = WAL") .await .expect("use WAL like 2026.3"); + fill_legacy_database(&conn).await + } + + /// Creates the 2026.3 tables with one Agent, and returns its ID. + async fn fill_legacy_database(conn: &Connection) -> Uuid { + let agent_id = Uuid::new_v4(); conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3 tables"); conn.execute( "INSERT INTO metadata (key, value) VALUES ('ca_spki_sha256', ?1)", @@ -252,7 +290,17 @@ mod tests { for file in with_sidecars(&data_dir.join(LEGACY_FILE_NAME)) { assert!(!file.exists(), "{file} is removed"); } - assert!(!data_dir.join(format!("{}.tmp", gateway_db::FILE_NAME)).exists()); + let path = data_dir.join(gateway_db::FILE_NAME); + let copy = Utf8PathBuf::from(format!("{path}.tmp")); + for file in with_sidecars(©) + .into_iter() + .chain(with_sidecars(&path).into_iter().skip(1)) + { + assert!(!file.exists(), "{file} is not left behind"); + } + // Bytes 18 and 19 of the header are 1 for a rollback journal and 2 for WAL. + let header = std::fs::read(&path).expect("read gateway.db"); + assert_eq!(header[18..20], [1, 1], "gateway.db is renamed in rollback journal mode"); let conn = open_gateway_db(&data_dir).await; assert_eq!(query_i64(&conn, "PRAGMA user_version").await, 0); @@ -282,6 +330,47 @@ mod tests { assert_eq!(accepted.name, "montreal-office"); } + #[tokio::test] + async fn agents_still_in_the_2026_3_wal_are_imported() { + let (_source_dir, source) = temp_data_dir(); + let (_temp_dir, data_dir) = temp_data_dir(); + let legacy = source.join(LEGACY_FILE_NAME); + let conn = connect(&legacy).await.expect("open 2026.3 database"); + conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0") + .await + .expect("keep changes in the WAL"); + let agent_id = fill_legacy_database(&conn).await; + + // Copied while open, as if 2026.3 had crashed before writing its WAL into the database. + for suffix in ["", "-wal"] { + std::fs::copy( + format!("{legacy}{suffix}"), + format!("{}{suffix}", data_dir.join(LEGACY_FILE_NAME)), + ) + .expect("copy the 2026.3 files"); + } + drop(conn); + assert!( + std::fs::metadata(data_dir.join(format!("{LEGACY_FILE_NAME}-wal"))) + .expect("WAL copied") + .len() + > 0 + ); + + import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + + let store = LibSqlAgentAuthorizationStore::open(open_gateway_db(&data_dir).await, [0xCA; 32]) + .await + .expect("open Agent authorization store"); + assert!( + store + .authorize(agent_id, [0x11; 32]) + .await + .expect("authorize moved Agent") + .is_some() + ); + } + #[tokio::test] async fn interrupted_move_starts_over() { let (_temp_dir, data_dir) = temp_data_dir(); @@ -339,6 +428,23 @@ mod tests { assert!(!data_dir.join(gateway_db::FILE_NAME).exists()); } + #[tokio::test] + async fn version_0_with_tables_is_refused() { + let (_temp_dir, data_dir) = temp_data_dir(); + let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) + .await + .expect("create 2026.3 database"); + conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3 tables"); + drop(conn); + + let error = import_2026_3_database(&data_dir) + .await + .expect_err("tables without a schema version must be refused"); + + assert!(error.to_string().contains("no schema version")); + assert!(data_dir.join(LEGACY_FILE_NAME).exists()); + } + #[tokio::test] async fn unknown_2026_3_schema_is_refused() { let (_temp_dir, data_dir) = temp_data_dir(); diff --git a/crates/gateway-db/AGENTS.md b/crates/gateway-db/AGENTS.md index 24b31d13d..6f0c39a62 100644 --- a/crates/gateway-db/AGENTS.md +++ b/crates/gateway-db/AGENTS.md @@ -3,7 +3,7 @@ This crate owns the shared `gateway.db` file and nothing else. Read `INTENT.md` in this directory first. -- Never add a dependency on a crate that implements feature behavior, such as `agent-tunnel`, `agent-tunnel-libsql` or `provisioner-task`. +- Never add a dependency on a crate that implements feature behavior, such as `agent-tunnel` or `agent-tunnel-libsql`. A feature's tables, migrations and schema version belong in its own adapter crate, which may depend on `gateway-db`. - Never read or write `PRAGMA user_version` here: no single crate owns every table in the file. - After changing `Cargo.toml`, run `cargo tree -p gateway-db --edges normal --depth 1` and check that only generic crates are listed. From 5c1bbef4959edbbfc62fb57e61b08c4ebeb7810a Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Thu, 8 Oct 2026 12:14:50 -0400 Subject: [PATCH 10/11] updated intent.md --- crates/gateway-db/INTENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gateway-db/INTENT.md b/crates/gateway-db/INTENT.md index 9fafbe260..d35308dee 100644 --- a/crates/gateway-db/INTENT.md +++ b/crates/gateway-db/INTENT.md @@ -16,7 +16,7 @@ All persistent state except the job queue and traffic audit should be stored in # Upgrade from 2026.3 -If for 2026.3, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. +If for 2026.3.0, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. # Future Later, `gateway-db` could take migrations from each adapter. From 934b26dfd1bbf94a750cf0a347c420797bbd03b8 Mon Sep 17 00:00:00 2001 From: Junyi Ou Date: Thu, 8 Oct 2026 12:31:06 -0400 Subject: [PATCH 11/11] refactor(dgw): name the agent tunnel upgrade after 2026.3.0 Only 2026.3.0 wrote agent_tunnel.db; 2026.3.1 will use a different format. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/agent-tunnel-libsql/src/lib.rs | 2 +- crates/agent-tunnel-libsql/src/upgrade.rs | 88 ++++++++++++----------- crates/gateway-db/INTENT.md | 2 +- devolutions-gateway/src/service.rs | 4 +- 4 files changed, 52 insertions(+), 44 deletions(-) diff --git a/crates/agent-tunnel-libsql/src/lib.rs b/crates/agent-tunnel-libsql/src/lib.rs index 43c9db6ea..56ea02620 100644 --- a/crates/agent-tunnel-libsql/src/lib.rs +++ b/crates/agent-tunnel-libsql/src/lib.rs @@ -18,7 +18,7 @@ use libsql::{Connection, TransactionBehavior, params}; use tokio::sync::Mutex; use uuid::Uuid; -pub use self::upgrade::import_2026_3_database; +pub use self::upgrade::import_2026_3_0_database; /// Every schema change ever made to the `agent_tunnel_*` tables, oldest first. Only ever append to it. const MIGRATIONS: &[&str] = &[include_str!("../migrations/01_agent_tunnel.sql")]; diff --git a/crates/agent-tunnel-libsql/src/upgrade.rs b/crates/agent-tunnel-libsql/src/upgrade.rs index b6747e076..6f72293cf 100644 --- a/crates/agent-tunnel-libsql/src/upgrade.rs +++ b/crates/agent-tunnel-libsql/src/upgrade.rs @@ -1,4 +1,4 @@ -//! The upgrade from Gateway 2026.3, which kept the agent tunnel tables in an `agent_tunnel.db` of their own. +//! The upgrade from Gateway 2026.3.0, which kept the agent tunnel tables in an `agent_tunnel.db` of their own. use anyhow::{Context as _, bail}; use camino::{Utf8Path, Utf8PathBuf}; @@ -6,16 +6,16 @@ use libsql::{Connection, params}; use crate::record_schema_version; -/// Only Gateway 2026.3 wrote this file. +/// Only Gateway 2026.3.0 wrote this file. pub(crate) const LEGACY_FILE_NAME: &str = "agent_tunnel.db"; -/// The only schema 2026.3 ever had, as its `PRAGMA user_version`. +/// The only schema 2026.3.0 ever had, as its `PRAGMA user_version`. const LEGACY_SCHEMA_VERSION: i64 = 1; /// The conversion yields the tables of the first migration. const CONVERTED_SCHEMA_VERSION: i64 = 1; -/// Turns the 2026.3 tables into the first version of the `agent_tunnel_*` tables. +/// Turns the 2026.3.0 tables into the first version of the `agent_tunnel_*` tables. const CONVERSION: &str = " ALTER TABLE metadata RENAME TO agent_tunnel_metadata; ALTER TABLE accepted_agents RENAME TO agent_tunnel_accepted_agents; @@ -23,7 +23,7 @@ const CONVERSION: &str = " ALTER TABLE deleted_agent_keys RENAME TO agent_tunnel_deleted_agent_keys; "; -/// Turns the `agent_tunnel.db` left by Gateway 2026.3 in `data_dir` into `gateway.db`. +/// Turns the `agent_tunnel.db` left by Gateway 2026.3.0 in `data_dir` into `gateway.db`. /// /// Call it before opening `gateway.db`: once `gateway.db` exists, the old file is ignored. It does nothing when there /// is no old file. @@ -31,7 +31,7 @@ const CONVERSION: &str = " /// We convert a copy, flush it to disk, and only then rename it into place, so `gateway.db` either doesn't exist yet /// or is whole and in the new layout. The rename reaches the disk before the old files are removed, so a power loss /// never leaves only a partial copy behind. If Gateway stops halfway, the next start simply does it again. -pub async fn import_2026_3_database(data_dir: &Utf8Path) -> anyhow::Result<()> { +pub async fn import_2026_3_0_database(data_dir: &Utf8Path) -> anyhow::Result<()> { let legacy = data_dir.join(LEGACY_FILE_NAME); let path = data_dir.join(gateway_db::FILE_NAME); @@ -117,7 +117,7 @@ async fn convert(copy: &Utf8Path) -> anyhow::Result<()> { record_schema_version(&tx, CONVERTED_SCHEMA_VERSION).await?; tx.execute_batch("PRAGMA user_version = 0") .await - .context("clear the 2026.3 schema version")?; + .context("clear the 2026.3.0 schema version")?; tx.commit().await.context("commit agent tunnel conversion")?; Ok(()) @@ -140,10 +140,10 @@ async fn has_tables(conn: &Connection) -> anyhow::Result { let row = conn .query("SELECT 1 FROM sqlite_master LIMIT 1", ()) .await - .context("query the 2026.3 agent tunnel tables")? + .context("query the 2026.3.0 agent tunnel tables")? .next() .await - .context("read the 2026.3 agent tunnel tables")?; + .context("read the 2026.3.0 agent tunnel tables")?; Ok(row.is_some()) } @@ -151,13 +151,13 @@ async fn has_tables(conn: &Connection) -> anyhow::Result { async fn legacy_schema_version(conn: &Connection) -> anyhow::Result { conn.query("PRAGMA user_version", ()) .await - .context("query the 2026.3 agent tunnel schema version")? + .context("query the 2026.3.0 agent tunnel schema version")? .next() .await - .context("read the 2026.3 agent tunnel schema version")? + .context("read the 2026.3.0 agent tunnel schema version")? .context("schema version query returned no row")? .get::(0) - .context("decode the 2026.3 agent tunnel schema version") + .context("decode the 2026.3.0 agent tunnel schema version") } fn with_sidecars(path: &Utf8Path) -> [Utf8PathBuf; 4] { @@ -187,7 +187,7 @@ mod tests { use super::*; use crate::LibSqlAgentAuthorizationStore; - /// The tables Gateway 2026.3 created in `agent_tunnel.db`. + /// The tables Gateway 2026.3.0 created in `agent_tunnel.db`. const LEGACY_SCHEMA: &str = " CREATE TABLE metadata (key TEXT PRIMARY KEY, value BLOB NOT NULL); CREATE TABLE accepted_agents ( @@ -221,27 +221,27 @@ mod tests { (temp_dir, data_dir) } - /// Writes the `agent_tunnel.db` that Gateway 2026.3 leaves behind, with one Agent, and returns its ID. + /// Writes the `agent_tunnel.db` that Gateway 2026.3.0 leaves behind, with one Agent, and returns its ID. async fn write_legacy_database(data_dir: &Utf8Path) -> Uuid { let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) .await - .expect("open 2026.3 database"); + .expect("open 2026.3.0 database"); conn.execute_batch("PRAGMA journal_mode = WAL") .await - .expect("use WAL like 2026.3"); + .expect("use WAL like 2026.3.0"); fill_legacy_database(&conn).await } - /// Creates the 2026.3 tables with one Agent, and returns its ID. + /// Creates the 2026.3.0 tables with one Agent, and returns its ID. async fn fill_legacy_database(conn: &Connection) -> Uuid { let agent_id = Uuid::new_v4(); - conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3 tables"); + conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3.0 tables"); conn.execute( "INSERT INTO metadata (key, value) VALUES ('ca_spki_sha256', ?1)", params![vec![0xCAu8; 32]], ) .await - .expect("bind 2026.3 CA"); + .expect("bind 2026.3.0 CA"); conn.execute( "INSERT INTO accepted_agents (agent_id, name, client_spki_sha256, enrollment_jti) VALUES (?1, ?2, ?3, ?4)", params![ @@ -252,10 +252,10 @@ mod tests { ], ) .await - .expect("store 2026.3 Agent"); + .expect("store 2026.3.0 Agent"); conn.execute_batch("PRAGMA user_version = 1") .await - .expect("record 2026.3 schema version"); + .expect("record 2026.3.0 schema version"); agent_id } @@ -281,11 +281,13 @@ mod tests { } #[tokio::test] - async fn agents_enrolled_on_2026_3_still_authorize_after_the_move() { + async fn agents_enrolled_on_2026_3_0_still_authorize_after_the_move() { let (_temp_dir, data_dir) = temp_data_dir(); let agent_id = write_legacy_database(&data_dir).await; - import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + import_2026_3_0_database(&data_dir) + .await + .expect("import 2026.3.0 database"); for file in with_sidecars(&data_dir.join(LEGACY_FILE_NAME)) { assert!(!file.exists(), "{file} is removed"); @@ -331,23 +333,23 @@ mod tests { } #[tokio::test] - async fn agents_still_in_the_2026_3_wal_are_imported() { + async fn agents_still_in_the_2026_3_0_wal_are_imported() { let (_source_dir, source) = temp_data_dir(); let (_temp_dir, data_dir) = temp_data_dir(); let legacy = source.join(LEGACY_FILE_NAME); - let conn = connect(&legacy).await.expect("open 2026.3 database"); + let conn = connect(&legacy).await.expect("open 2026.3.0 database"); conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0") .await .expect("keep changes in the WAL"); let agent_id = fill_legacy_database(&conn).await; - // Copied while open, as if 2026.3 had crashed before writing its WAL into the database. + // Copied while open, as if 2026.3.0 had crashed before writing its WAL into the database. for suffix in ["", "-wal"] { std::fs::copy( format!("{legacy}{suffix}"), format!("{}{suffix}", data_dir.join(LEGACY_FILE_NAME)), ) - .expect("copy the 2026.3 files"); + .expect("copy the 2026.3.0 files"); } drop(conn); assert!( @@ -357,7 +359,9 @@ mod tests { > 0 ); - import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + import_2026_3_0_database(&data_dir) + .await + .expect("import 2026.3.0 database"); let store = LibSqlAgentAuthorizationStore::open(open_gateway_db(&data_dir).await, [0xCA; 32]) .await @@ -379,7 +383,9 @@ mod tests { std::fs::write(©, b"half written").expect("leave an incomplete copy"); std::fs::write(format!("{copy}-journal"), b"stale journal").expect("leave its journal"); - import_2026_3_database(&data_dir).await.expect("import 2026.3 database"); + import_2026_3_0_database(&data_dir) + .await + .expect("import 2026.3.0 database"); for file in with_sidecars(©) { assert!(!file.exists(), "{file} is removed"); @@ -396,7 +402,9 @@ mod tests { drop(open_gateway_db(&data_dir).await); write_legacy_database(&data_dir).await; - import_2026_3_database(&data_dir).await.expect("ignore 2026.3 database"); + import_2026_3_0_database(&data_dir) + .await + .expect("ignore 2026.3.0 database"); assert!(data_dir.join(LEGACY_FILE_NAME).exists()); let conn = open_gateway_db(&data_dir).await; @@ -409,20 +417,20 @@ mod tests { } #[tokio::test] - async fn empty_2026_3_database_is_removed() { + async fn empty_2026_3_0_database_is_removed() { let (_temp_dir, data_dir) = temp_data_dir(); let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) .await - .expect("create empty 2026.3 database"); + .expect("create empty 2026.3.0 database"); conn.execute_batch("PRAGMA journal_mode = WAL") .await .expect("write the database header"); drop(conn); assert!(data_dir.join(LEGACY_FILE_NAME).exists()); - import_2026_3_database(&data_dir) + import_2026_3_0_database(&data_dir) .await - .expect("import empty 2026.3 database"); + .expect("import empty 2026.3.0 database"); assert!(!data_dir.join(LEGACY_FILE_NAME).exists()); assert!(!data_dir.join(gateway_db::FILE_NAME).exists()); @@ -433,11 +441,11 @@ mod tests { let (_temp_dir, data_dir) = temp_data_dir(); let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) .await - .expect("create 2026.3 database"); - conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3 tables"); + .expect("create 2026.3.0 database"); + conn.execute_batch(LEGACY_SCHEMA).await.expect("create 2026.3.0 tables"); drop(conn); - let error = import_2026_3_database(&data_dir) + let error = import_2026_3_0_database(&data_dir) .await .expect_err("tables without a schema version must be refused"); @@ -446,17 +454,17 @@ mod tests { } #[tokio::test] - async fn unknown_2026_3_schema_is_refused() { + async fn unknown_2026_3_0_schema_is_refused() { let (_temp_dir, data_dir) = temp_data_dir(); let conn = connect(&data_dir.join(LEGACY_FILE_NAME)) .await - .expect("create 2026.3 database"); + .expect("create 2026.3.0 database"); conn.execute_batch("PRAGMA user_version = 7") .await .expect("set unknown schema version"); drop(conn); - let error = import_2026_3_database(&data_dir) + let error = import_2026_3_0_database(&data_dir) .await .expect_err("unknown schema must be refused"); diff --git a/crates/gateway-db/INTENT.md b/crates/gateway-db/INTENT.md index d35308dee..100c2c464 100644 --- a/crates/gateway-db/INTENT.md +++ b/crates/gateway-db/INTENT.md @@ -15,7 +15,7 @@ For each domain table, the table should have its prefix in table's name, for exa All persistent state except the job queue and traffic audit should be stored in this database. -# Upgrade from 2026.3 +# Upgrade from 2026.3.0 If for 2026.3.0, user turned on the feature of agent-tunnel, then the database will be migrated from `agent_tunnel.db` to `gateway.db`. The migration will be done automatically on Gateway start up. This is one-off, atomic operation. # Future diff --git a/devolutions-gateway/src/service.rs b/devolutions-gateway/src/service.rs index a69e9c117..258e15d23 100644 --- a/devolutions-gateway/src/service.rs +++ b/devolutions-gateway/src/service.rs @@ -276,9 +276,9 @@ async fn spawn_tasks(conf_handle: ConfHandle) -> anyhow::Result { let monitoring_state = Arc::new(network_monitor::State::new(Arc::new(filesystem_monitor_config_cache))?); let data_dir = config::get_data_dir(); - agent_tunnel_libsql::import_2026_3_database(&data_dir) + agent_tunnel_libsql::import_2026_3_0_database(&data_dir) .await - .context("failed to import the 2026.3 agent tunnel database")?; + .context("failed to import the 2026.3.0 agent tunnel database")?; let gateway_db = gateway_db::GatewayDb::open(&data_dir) .await .context("failed to open the gateway database")?;