From eab5ce16699537420cc8e70ebcb7dbbaadfe7486 Mon Sep 17 00:00:00 2001 From: devmaxxx Date: Mon, 19 Oct 2026 13:24:49 +0200 Subject: [PATCH 1/2] feat(index): redact secrets written inside indexed files The walk already refuses credential files by name, but a token pasted into a source file, a README or a shell script reached graph.json, the lexical and dense indexes and rerank prompts. Every text read for extraction now passes through secrets_scanner's bundled rules first; each secret becomes [REDACTED_SECRET], keeping its newlines so line numbers hold, and inline gitleaks:allow markers are not honoured. Compiling all ~1,000 rules costs 1.8 s per process, which a reader's inline update would pay on every edit. The scanner already skips a rule whose keywords a text lacks, so a run builds one scanner from only the rules its texts' keywords name: a one-file update takes 0.14 s in all. A test pins that the subset finds what the full ruleset finds on every .rs and .md file in this repository. The regex crates build optimised in the dev profile, since every test that builds a store compiles the subset. --- Cargo.lock | 210 +++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 16 ++++ docs/configuration.md | 9 ++ src/lib.rs | 29 ++++++ src/secrets.rs | 194 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 443 insertions(+), 15 deletions(-) create mode 100644 src/secrets.rs diff --git a/Cargo.lock b/Cargo.lock index 67a2143b..21d88ba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ checksum = "2af0513e0a025ce1ebbbdbf00e81ca84c29f26b33a6e1e0dd877945ceef75042" dependencies = [ "bit-set", "regex", - "thiserror", + "thiserror 2.0.20", "tree-sitter", ] @@ -215,7 +215,16 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", ] [[package]] @@ -347,7 +356,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -444,6 +453,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -493,6 +511,16 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.20.11" @@ -584,6 +612,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "dirs" version = "6.0.0" @@ -793,6 +832,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -894,6 +943,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hf-hub" version = "0.5.0" @@ -910,11 +965,20 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 2.0.20", "ureq", "windows-sys 0.61.2", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -1644,7 +1708,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -1751,7 +1815,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -1943,6 +2007,24 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "secrets_scanner" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d3e9b1732051e9d5b17c071414ffe702231a6af3df74492ea0bb06119cbc5" +dependencies = [ + "aho-corasick", + "hex", + "hmac", + "log", + "memchr", + "regex", + "serde", + "sha2", + "thiserror 1.0.69", + "toml 0.8.23", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2016,6 +2098,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -2050,6 +2141,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2222,13 +2324,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -2309,7 +2431,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror", + "thiserror 2.0.20", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -2363,6 +2485,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "1.1.4+spec-1.1.0" @@ -2371,11 +2505,20 @@ checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] @@ -2387,6 +2530,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.15+spec-1.1.0" @@ -2394,9 +2551,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.4", ] [[package]] @@ -2405,9 +2562,15 @@ version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -2723,6 +2886,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3140,6 +3309,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" @@ -3229,6 +3407,7 @@ dependencies = [ name = "zerograph" version = "0.7.0" dependencies = [ + "aho-corasick", "anyhow", "ast-grep-language", "blake3", @@ -3242,6 +3421,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", + "secrets_scanner", "serde", "serde_json", "serde_yaml", @@ -3249,7 +3429,7 @@ dependencies = [ "streaming-iterator", "tempfile", "tokenizers", - "toml", + "toml 1.1.4+spec-1.1.0", "tree-sitter", "tree-sitter-bash", "tree-sitter-bicep", diff --git a/Cargo.toml b/Cargo.toml index 8e615030..e799f80b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,6 +82,11 @@ ast-grep-language = { version = "=0.45.3", default-features = false, features = "tree-sitter-ruby", "tree-sitter-elixir", "tree-sitter-scala", ] } postcard = { version = "=1.1.3", features = ["use-std"] } +# Credentials inside files that are read (src/secrets.rs). Without `native`: its CLI, walker, logger and +# thread pool are not needed to scan a string, and the walk here is `ignore`'s. +secrets_scanner = { version = "=0.2.3", default-features = false } +# Already in the tree through `regex`; named here to find which rules a text's keywords call for. +aho-corasick = "=1.1.5" [dev-dependencies] tempfile = "=3.27.0" @@ -102,3 +107,14 @@ windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Stor [profile.release] lto = "thin" + +# The secret scanner compiles hundreds of regexes, which an unoptimised regex-automata does seven +# times slower: every test that builds a store would pay that, not only the scanner's own. +[profile.dev.package.regex-automata] +opt-level = 3 +[profile.dev.package.regex-syntax] +opt-level = 3 +[profile.dev.package.aho-corasick] +opt-level = 3 +[profile.dev.package.secrets_scanner] +opt-level = 3 diff --git a/docs/configuration.md b/docs/configuration.md index 6e888d7a..3947e579 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,15 @@ keys and keystores, `.npmrc`, `.pypirc`, `.netrc`, `.git-credentials`, cloud cre `*.tfvars`, `*.tfstate` and `secrets.{yaml,json,toml,…}` — their content would otherwise reach the store, a `--rerank` prompt sent to a model, and the answers. +A credential written inside a file that is read — a token in a source file, a key exported in a +shell script, a private key pasted into a README — is replaced with `[REDACTED_SECRET]` before the +file is extracted, by the bundled rules of +[secrets_scanner](https://crates.io/crates/secrets_scanner) (gitleaks-compatible). The rest of the +file is still read and keeps its line numbers; `update` logs how many were redacted in which file. +An inline `gitleaks:allow` does not exempt one. A run compiles only the rules whose keywords the +files it re-reads contain — a few dozen for a one-file update, tens of milliseconds, where all ~1,000 +would cost 1.8 s. + The store writes its own `.gitignore` holding `*`, so `.zerograph/` stays out of a commit in a repository that never listed it. diff --git a/src/lib.rs b/src/lib.rs index 690961dd..e3164dbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ mod prime; mod priority; mod query; mod refresh; +mod secrets; mod serve; mod status; mod store; @@ -345,6 +346,7 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m work.sort_by(|a, b| a.rel.cmp(&b.rel)); let reread = co_declared.iter().map(|rel| by_rel[rel]); let mut unread = false; + let mut read = Vec::new(); for e in work.into_iter().chain(reread) { let text = match std::fs::read(repo.join(&e.rel)) { // NUL is legal inside a TypeScript string literal; only invalid UTF-8 marks a binary. @@ -356,6 +358,14 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m // that would not open is one this pass has no reading of at all. Err(err) => { eprintln!("{}", log_line(&format!("read {}: {err}", e.rel))); unread = true; continue; } }; + read.push((e, text)); + } + // Every text is read before any is redacted: the scanner compiles only the rules whose + // keywords these texts hold, so a one-file update pays for a few dozen regexes, not ~1,000. + let redactor = secrets::Redactor::for_texts(read.iter().map(|(_, text)| text.as_str())); + for (e, text) in read { + let (text, redacted) = redactor.redact(&e.rel, &text); + if redacted > 0 { eprintln!("{}", log_line(&format!("{}: {redacted} secret(s) redacted", e.rel))); } let extractor = match e.kind { walk::FileKind::Doc => &ex.doc, walk::FileKind::Code => &ex.code, @@ -1435,6 +1445,25 @@ mod tests { run_update(repo, cfg, true).unwrap(); } + #[test] + fn a_secret_written_in_a_document_reaches_no_file_of_the_store() { + let token = format!("ghp_{}", "R7xK2mQ9vT4wL8nB3cF6hJ1pS5dG0aZyEuXo"); + let dir = doc_repo(&format!("# A\n\n**FR-PAY-22 · MUST · cancellation window**\n\nCall with `{token}`.\n")); + let (repo, cfg) = (dir.path(), config::Config::default()); + built(repo, &cfg); + let (graph, _) = store::Store::new(repo).load().unwrap(); + assert!(graph.nodes.contains_key("FR-PAY-22"), "the rest of the document is still read"); + let leaked: Vec<_> = walkdir_files(&repo.join(".zerograph")).into_iter() + .filter(|p| std::fs::read(p).unwrap().windows(token.len()).any(|w| w == token.as_bytes())) + .collect(); + assert!(leaked.is_empty(), "{leaked:?}"); + } + + fn walkdir_files(dir: &std::path::Path) -> Vec { + ignore::WalkBuilder::new(dir).standard_filters(false).build().flatten() + .filter(|e| e.file_type().is_some_and(|t| t.is_file())).map(|e| e.into_path()).collect() + } + /// The report exists to say the one thing a configuration-shaped answer would hide: a store /// keeps answering with the model that wrote it, whatever the file says today. So where the /// store's name comes from, and what is printed when the two differ, are the report. diff --git a/src/secrets.rs b/src/secrets.rs new file mode 100644 index 00000000..bde969f0 --- /dev/null +++ b/src/secrets.rs @@ -0,0 +1,194 @@ +//! Credentials written inside a file that is read. `walk` refuses the files that exist to hold +//! them (`.env`, `*.pem`); this catches the key pasted into a source file, a README or a shell +//! script, before its text reaches `graph.json`, the lexical and dense indexes, or a rerank prompt. + +use aho_corasick::AhoCorasick; +use secrets_scanner::{RedactionMode, ScanConfig, Scanner}; +use std::sync::OnceLock; + +pub const MARKER: &str = "[REDACTED_SECRET]"; + +/// The bundled rules, split so a scanner can be built from any subset of them. Compiling all of +/// them costs 1.8 s, and the scanner already skips a rule whose keywords a text does not hold — +/// so a scanner of only the rules a text's keywords name finds exactly what the full one does. +struct Catalog { + /// Everything but the rules: the global allowlist, the minimum version. + base: toml::Table, + rules: Vec, + keywords: AhoCorasick, + rule_of_keyword: Vec, + unkeyworded: Vec, +} + +fn catalog() -> &'static Catalog { + static C: OnceLock = OnceLock::new(); + C.get_or_init(|| { + // The bundled rules only: `Scanner::new` would also read `$SECRETS_SCANNER_RULES` and a + // cache in the user's data directory, so what is redacted would vary by machine. + let mut base: toml::Table = toml::from_str(secrets_scanner::rules::BUNDLED_RULES).expect("the bundled rules parse"); + let rules = match base.remove("rules") { Some(toml::Value::Array(r)) => r, _ => Vec::new() }; + let (mut words, mut rule_of_keyword, mut unkeyworded) = (Vec::new(), Vec::new(), Vec::new()); + for (i, rule) in rules.iter().enumerate() { + let listed: Vec<&str> = rule.get("keywords").and_then(|k| k.as_array()).into_iter().flatten().filter_map(|k| k.as_str()).collect(); + if listed.is_empty() { unkeyworded.push(i); } + for word in listed { words.push(word.to_lowercase()); rule_of_keyword.push(i); } + } + let keywords = AhoCorasick::builder().ascii_case_insensitive(true).build(&words).expect("the rule keywords build an automaton"); + Catalog { base, rules, keywords, rule_of_keyword, unkeyworded } + }) +} + +/// A scanner for one pass's texts, holding only the rules their keywords name. +pub struct Redactor(Option); + +impl Redactor { + pub fn for_texts<'a>(texts: impl IntoIterator) -> Redactor { + let c = catalog(); + let mut wanted = vec![false; c.rules.len()]; + for &i in &c.unkeyworded { wanted[i] = true; } + for text in texts { + for m in c.keywords.find_overlapping_iter(text) { wanted[c.rule_of_keyword[m.pattern().as_usize()]] = true; } + } + let rules: Vec = c.rules.iter().zip(&wanted).filter(|(_, w)| **w).map(|(r, _)| r.clone()).collect(); + if rules.is_empty() { + return Redactor(None); + } + let mut subset = c.base.clone(); + subset.insert("rules".into(), toml::Value::Array(rules)); + let toml = toml::to_string(&subset).expect("a subset of the bundled rules serializes"); + let scanner = Scanner::from_toml(&toml).expect("a subset of the bundled rules compiles"); + Redactor(Some(scanner.with_config(ScanConfig { + redaction_mode: RedactionMode::Full, + // A repository is content someone else wrote: a `gitleaks:allow` beside a key must not + // be what lets it into the graph. + honor_allow_markers: false, + capture_context: false, + ..ScanConfig::default() + }))) + } + + /// `text` with every detected secret replaced by [`MARKER`], and how many were found. A secret + /// spanning lines keeps its newlines after the marker, so every line below it keeps its number. + pub fn redact(&self, rel: &str, text: &str) -> (String, usize) { + let Some(scanner) = &self.0 else { return (text.to_string(), 0) }; + redact_found(text, &scanner.scan_content_detailed(rel, text).findings) + } +} + +fn redact_found(text: &str, found: &[secrets_scanner::Finding]) -> (String, usize) { + let mut spans: Vec<(usize, usize)> = found.iter() + .map(|f| (f.secret_start_offset, f.secret_end_offset)) + .filter(|(s, e)| s < e && *e <= text.len()) + .map(|(s, e)| (floor_boundary(text, s), ceil_boundary(text, e))) + .collect(); + if spans.is_empty() { + return (text.to_string(), 0); + } + spans.sort_unstable(); + // Two rules often match one key: overlapping or touching spans become one marker. + let mut merged: Vec<(usize, usize)> = Vec::with_capacity(spans.len()); + for (start, end) in spans { + match merged.last_mut() { + Some(last) if start <= last.1 => last.1 = last.1.max(end), + _ => merged.push((start, end)), + } + } + let mut out = String::with_capacity(text.len()); + let mut cursor = 0; + for (start, end) in merged { + out.push_str(&text[cursor..start]); + out.push_str(MARKER); + out.extend(text[start..end].chars().filter(|&c| c == '\n')); + cursor = end; + } + out.push_str(&text[cursor..]); + (out, found.len()) +} + +fn floor_boundary(text: &str, mut i: usize) -> usize { + while !text.is_char_boundary(i) { i -= 1; } + i +} + +fn ceil_boundary(text: &str, mut i: usize) -> usize { + while !text.is_char_boundary(i) { i += 1; } + i +} + +#[cfg(test)] +mod tests { + use super::{Redactor, MARKER}; + + fn redact(rel: &str, text: &str) -> (String, usize) { + Redactor::for_texts([text]).redact(rel, text) + } + + // Assembled at run time so this file holds no string a scanner of this repository flags. + fn github_token() -> String { format!("ghp_{}", "R7xK2mQ9vT4wL8nB3cF6hJ1pS5dG0aZyEuXo") } + fn aws_key() -> String { format!("AKIA{}", "QX4TZ7WMB2KD6RLN") } + + #[test] + fn a_token_in_source_is_replaced_and_the_rest_of_the_line_stays() { + let text = format!("const client = new Octokit({{ auth: \"{}\" }});\nexport default client;\n", github_token()); + let (out, n) = redact("src/github.ts", &text); + assert_eq!(n, 1); + assert!(!out.contains(&github_token()), "{out}"); + assert!(out.contains(MARKER) && out.contains("new Octokit") && out.contains("export default client;"), "{out}"); + } + + #[test] + fn a_key_exported_in_a_shell_script_is_replaced() { + let text = format!("#!/bin/sh\nexport AWS_ACCESS_KEY_ID={}\naws s3 ls\n", aws_key()); + let (out, n) = redact("scripts/deploy.sh", &text); + assert!(n >= 1); + assert!(!out.contains(&aws_key()), "{out}"); + assert!(out.contains("aws s3 ls")); + } + + #[test] + fn a_private_key_block_is_replaced_and_the_lines_below_keep_their_numbers() { + let body = "MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn/ygWyF8PbnGy0AHB7MhgHcTz6sE2I2yPB\naFDrBz9vFqU4yQmpQ6fFoXyZ0YkRh4AXKzk7b1HgGzK7v1mN3rGv5WqL8XKjR2Ys\n"; + let text = format!("# Ops\n\n-----BEGIN RSA {k}-----\n{body}-----END RSA {k}-----\n\nnext line\n", k = "PRIVATE KEY"); + let (out, n) = redact("docs/ops.md", &text); + assert!(n >= 1); + assert!(!out.contains("MIIEowIBAAKCAQEA"), "{out}"); + assert_eq!(out.lines().count(), text.lines().count(), "{out}"); + assert_eq!(out.lines().position(|l| l == "next line"), text.lines().position(|l| l == "next line")); + } + + #[test] + fn an_allow_marker_in_the_file_does_not_let_a_secret_through() { + let text = format!("token = \"{}\" # gitleaks:allow\n", github_token()); + let (out, _) = redact("config.py", &text); + assert!(!out.contains(&github_token()), "{out}"); + } + + #[test] + fn text_without_a_secret_comes_back_unchanged() { + let text = "fn main() {\n let password = read_password();\n println!(\"héllo\");\n}\n"; + assert_eq!(redact("src/main.rs", text), (text.to_string(), 0)); + } + + /// The subset is only an optimisation if it finds what the whole ruleset finds, on real text + /// that names many providers' keywords: this repository's own sources and documents. + #[test] + fn a_scanner_of_the_rules_a_text_names_finds_what_the_whole_ruleset_finds() { + let full = secrets_scanner::Scanner::from_toml(secrets_scanner::rules::BUNDLED_RULES).expect("the whole ruleset passes the strict gate"); + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let spans = |f: Vec| { let mut s: Vec<_> = f.into_iter().map(|f| (f.rule_id, f.secret_start_offset, f.secret_end_offset)).collect(); s.sort(); s }; + let mut paths: Vec<_> = ignore::Walk::new(root).flatten().map(|e| e.into_path()) + .filter(|p| p.extension().is_some_and(|x| x == "rs" || x == "md")).collect(); + paths.sort(); + let (mut compared, mut flagged) = (0, 0); + for path in &paths { + let Ok(text) = std::fs::read_to_string(path) else { continue }; + let rel = path.strip_prefix(root).unwrap().to_string_lossy(); + let expected = spans(full.scan_content_detailed(&rel, &text).findings); + let Redactor(Some(subset)) = Redactor::for_texts([text.as_str()]) else { panic!("no rule at all for {rel}") }; + assert_eq!(spans(subset.scan_content_detailed(&rel, &text).findings), expected, "{rel}"); + compared += 1; + flagged += usize::from(!expected.is_empty()); + } + assert!(compared > 100 && flagged > 0, "compared {compared} files, {flagged} flagged"); + } +} From f31b1e201fa2afc34be879a1000ddba1c5eaa248 Mon Sep 17 00:00:00 2001 From: devmaxxx Date: Mon, 19 Oct 2026 13:33:09 +0200 Subject: [PATCH 2/2] fix(index): bound redaction memory and re-read stores once Texts are scanned and extracted in 64 MiB batches instead of all at once, and an empty batch builds no scanner, so a no-op update stays free of the rules. GRAMMAR moves to 7 so a store written before redaction is re-read once rather than keeping the secrets of files that never change. The redaction count is the markers written, not the findings, and the docs name the paths the rules' allowlist skips. --- docs/configuration.md | 3 ++- src/lib.rs | 47 ++++++++++++++++++++++++++++--------------- src/secrets.rs | 3 ++- src/walk.rs | 4 +++- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3947e579..5777f41d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,7 +39,8 @@ shell script, a private key pasted into a README — is replaced with `[REDACTED file is extracted, by the bundled rules of [secrets_scanner](https://crates.io/crates/secrets_scanner) (gitleaks-compatible). The rest of the file is still read and keeps its line numbers; `update` logs how many were redacted in which file. -An inline `gitleaks:allow` does not exempt one. A run compiles only the rules whose keywords the +An inline `gitleaks:allow` does not exempt one; the rules' own path allowlist does — images, fonts, +lockfiles, `vendor/` and `node_modules/` trees are not scanned. A run compiles only the rules whose keywords the files it re-reads contain — a few dozen for a one-file update, tens of milliseconds, where all ~1,000 would cost 1.8 s. diff --git a/src/lib.rs b/src/lib.rs index e3164dbe..dbac57f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -277,6 +277,9 @@ pub(crate) struct UpdateReport { pub graph_at: Option, } +/// How much file text is held before it is scanned for secrets and extracted. +const REDACT_BATCH_BYTES: usize = 64 << 20; + /// Re-extracts what the diff names, drops what is gone, and writes the store back. Nothing else /// is read: every citation a file holds was extracted when the file was read, whatever prefix it /// names, and which of them a reader may follow is decided on the graph by `settle` — so a family @@ -346,7 +349,27 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m work.sort_by(|a, b| a.rel.cmp(&b.rel)); let reread = co_declared.iter().map(|rel| by_rel[rel]); let mut unread = false; - let mut read = Vec::new(); + // The scanner compiles only the rules whose keywords a batch of texts holds, so a one-file + // update pays for a few dozen regexes, not ~1,000. Texts are held a batch at a time: every + // text of the tree at once would be the largest allocation of a build. An empty batch builds + // no scanner, which keeps the no-op update a commit hook fires free of the rules. + let mut batch: Vec<(&walk::Entry, String)> = Vec::new(); + let mut batch_bytes = 0; + let mut extract_batch = |batch: &mut Vec<(&walk::Entry, String)>| { + if batch.is_empty() { return; } + let redactor = secrets::Redactor::for_texts(batch.iter().map(|(_, text)| text.as_str())); + for (e, text) in batch.drain(..) { + let (text, redacted) = redactor.redact(&e.rel, &text); + if redacted > 0 { eprintln!("{}", log_line(&format!("{}: {redacted} secret(s) redacted", e.rel))); } + let extractor = match e.kind { + walk::FileKind::Doc => &ex.doc, + walk::FileKind::Code => &ex.code, + walk::FileKind::Registry => &ex.registry, + walk::FileKind::Text => &ex.text, + }; + graph.apply(extractor.extract(&e.rel, &text)); + } + }; for e in work.into_iter().chain(reread) { let text = match std::fs::read(repo.join(&e.rel)) { // NUL is legal inside a TypeScript string literal; only invalid UTF-8 marks a binary. @@ -358,22 +381,14 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m // that would not open is one this pass has no reading of at all. Err(err) => { eprintln!("{}", log_line(&format!("read {}: {err}", e.rel))); unread = true; continue; } }; - read.push((e, text)); - } - // Every text is read before any is redacted: the scanner compiles only the rules whose - // keywords these texts hold, so a one-file update pays for a few dozen regexes, not ~1,000. - let redactor = secrets::Redactor::for_texts(read.iter().map(|(_, text)| text.as_str())); - for (e, text) in read { - let (text, redacted) = redactor.redact(&e.rel, &text); - if redacted > 0 { eprintln!("{}", log_line(&format!("{}: {redacted} secret(s) redacted", e.rel))); } - let extractor = match e.kind { - walk::FileKind::Doc => &ex.doc, - walk::FileKind::Code => &ex.code, - walk::FileKind::Registry => &ex.registry, - walk::FileKind::Text => &ex.text, - }; - graph.apply(extractor.extract(&e.rel, &text)); + batch_bytes += text.len(); + batch.push((e, text)); + if batch_bytes >= REDACT_BATCH_BYTES { + extract_batch(&mut batch); + batch_bytes = 0; + } } + extract_batch(&mut batch); // Whether anything was re-extracted or dropped. Nothing else can move a family, so on a // no-op update — the one a commit hook fires — the settle below would read every node and // rebuild the whole edge set to arrive at what is already there. diff --git a/src/secrets.rs b/src/secrets.rs index bde969f0..f752b6ec 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -95,6 +95,7 @@ fn redact_found(text: &str, found: &[secrets_scanner::Finding]) -> (String, usiz } let mut out = String::with_capacity(text.len()); let mut cursor = 0; + let count = merged.len(); for (start, end) in merged { out.push_str(&text[cursor..start]); out.push_str(MARKER); @@ -102,7 +103,7 @@ fn redact_found(text: &str, found: &[secrets_scanner::Finding]) -> (String, usiz cursor = end; } out.push_str(&text[cursor..]); - (out, found.len()) + (out, count) } fn floor_boundary(text: &str, mut i: usize) -> usize { diff --git a/src/walk.rs b/src/walk.rs index 76b8e364..f4b834c7 100644 --- a/src/walk.rs +++ b/src/walk.rs @@ -37,7 +37,9 @@ pub struct Entry { pub rel: String, pub kind: FileKind, pub hash: String, pub st /// 6: every language read by one query engine. Some nodes' spans and bodies differ from what 5's /// per-language readers wrote, such as a decorated field's line or the comment kept above a /// declaration, so a store stamped 5 is re-read once. -pub const GRAMMAR: u32 = 6; +/// 7: a credential written inside a file is replaced before the file is read, so a store stamped 6 +/// still holds the ones its unmoved files carry until it is re-read once. +pub const GRAMMAR: u32 = 7; /// Refused as text whatever `text_globs` says: machine output nobody asks a question of, any one of /// which would outweigh the hand-written files in the lexical index. Text entries only — `skip`