From 7198c1c4dc571ac55cbeb839936bfe4f19e53438 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 12 Oct 2026 21:13:41 +1100 Subject: [PATCH 1/3] feat(eql-bindings): produce EQL v4 TextMatch from Stack Encrypt's match terms TextMatch needs no ordering term, and the engine already derives match terms. Under ADR-0002 the terms only have to be ones EQL's SQL can compare, not cipherstash-client's bytes: both the stored value and the query come from Stack Encrypt, and cipherstash-client never writes an eql_v4 column. EQL's bloom filter is a smallint[] of set bit positions, and @@ tests containment. The new BloomFilter adapter writes the engine's positions under its default match configuration (3-grams, lower-cased, k = 3, a 256-bit filter), so every position fits a smallint, and refuses one that does not rather than wrapping it. It names the match term MatchTerm, the spelling that compiles against both the tree and the crates.io stack-encrypt 0.2.0 the published crate builds against. Proven in PostgreSQL on an eql_v4 install: @@ returns every row for a shared substring, one row for a unique one and none for a miss. Text too short to make a 3-gram is refused rather than stored with a filter no query could match. Refs #1131 --- .changeset/eql-v4-bindings.md | 2 +- languages/golang/cmd/stashgen/main_test.go | 4 +- languages/golang/encrypt/eql/eql_gen.go | 4 +- languages/golang/encrypt/eql/eql_test.go | 2 +- packages/eql/crates/eql-bindings/README.md | 7 +- .../eql/crates/eql-bindings/src/encryption.rs | 63 +++++++++++- .../eql/crates/eql-bindings/src/v4/targets.rs | 18 +++- .../eql/crates/eql-bindings/src/v4/text.rs | 35 +++++++ .../eql/crates/eql-codegen/src/bindings.rs | 3 +- .../eql/crates/eql-codegen/src/targets.rs | 4 +- .../eql/tests/encryption/tests/targets.rs | 12 ++- .../eql/tests/encryption/tests/v4_types.rs | 95 +++++++++++++++++-- 12 files changed, 222 insertions(+), 27 deletions(-) diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md index 546a460d7..6b2bb66e2 100644 --- a/.changeset/eql-v4-bindings.md +++ b/.changeset/eql-v4-bindings.md @@ -2,6 +2,6 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. +**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch`, from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. Nothing a released version shipped has changed. The v3 types, the v3 SQL and the TypeScript package are as they were; the `stack-encrypt` derives on the v3 text types and the `v3::targets` module were added after the last release and are moved to v4 here, not removed from a release. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index b832fcb4f..a9558ab1a 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -142,7 +142,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { t.Fatalf("the generated file does not name the EQL type:\n%s", written) } // A type the engine cannot produce yet: refused by name, nothing written. - unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextMatch", 1) + unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextOrdOre", 1) dir = writeModule(t, map[string]string{"model.go": unproducible}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { @@ -150,7 +150,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // Refused by the generator with the engine's reason, before the engine // is asked: se_targets lists the type as not producible. - if !strings.Contains(stderr.String(), "cannot produce the EQL type TextMatch yet") { + if !strings.Contains(stderr.String(), "cannot produce the EQL type TextOrdOre yet") { t.Fatalf("stderr = %q", stderr.String()) } if _, err := os.Stat(filepath.Join(dir, "user_stash.go")); !os.IsNotExist(err) { diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index d7b185e15..77eee6fe3 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -41,7 +41,7 @@ var Types = []Type{ {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v4_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ope", Producible: false, Reason: "vitaminc 0.5's value enum has no kind for this family; the kind and its encodings arrive in vitaminc 0.6 (ADR-0002)"}, {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v4_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v4_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, Reason: ""}, - {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: false, Reason: "the engine derives match terms, but no EQL type is built from them yet"}, + {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: true, Reason: ""}, {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, @@ -1406,7 +1406,7 @@ func (v *TextEqQuery) UnmarshalJSON(b []byte) error { } // TextMatch is the EQL type stored in public.eql_v4_text_match: the text family with the match index. -// The engine cannot produce it yet: the engine derives match terms, but no EQL type is built from them yet. +// The engine produces it from a string. type TextMatch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 92ac9ea5a..efc4c4319 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -19,7 +19,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { t.Errorf("%s: an unproducible type has a reason", typ.Name) } } - want := []string{"Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"} + want := []string{"Integer", "Bigint", "Text", "TextEq", "TextMatch", "Boolean", "Real", "Double"} if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) } diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 86c2a9a8c..2c33f6e29 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -112,7 +112,7 @@ encryption for the scalar types: | v4 type | Plaintext | | --- | --- | -| `Text` | `String` | +| `Text`, `TextMatch` | `String` | | `Integer` | `Int32` | | `Bigint` | `Int64` | | `Boolean` | `Bool` | @@ -124,7 +124,10 @@ seals for the same value, so values written here and by the Go SDK open each other. `IntegerEq` and `BigintEq` wait for vitaminc 0.6, whose integer equality term uses one PRF domain for every width; terms derived under vitaminc 0.5's per-width domains would stop matching after that upgrade. -Ordering types wait for vitaminc 0.6's order encodings, float +`TextMatch` uses Stack Encrypt's default match configuration +(3-grams, lower-cased, `k = 3`, a 256-bit filter) for stored values and +queries alike, and refuses text too short to make a 3-gram. Ordering types +wait for vitaminc 0.6's order encodings, float equality for its canonical float form, and `Json` for a decision on sealing a document as one leaf. diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index 0134178a3..e174b3d01 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -82,7 +82,9 @@ pub mod plaintext; pub mod targets; use base64::{engine::general_purpose::STANDARD, Engine as _}; -use stack_encrypt::sem::EqualityTerm; +#[allow(deprecated)] +use stack_encrypt::sem::MatchTerm; +use stack_encrypt::sem::{DefaultMatch, EqualityTerm, MatchConfig}; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext}; use stack_encrypt::{ @@ -93,7 +95,7 @@ use stack_encrypt::{ use vitaminc_prf::PrfValue; use crate::{ - v3::terms::{Ciphertext, Hmac256}, + v3::terms::{BloomFilter, Ciphertext, Hmac256}, Identifier, }; @@ -252,6 +254,63 @@ impl DecryptField for Hmac256 { } } +/// Builds EQL's bloom filter, the set bit positions as `smallint`s, from the +/// native match terms. +/// +/// The configuration is Stack Encrypt's default (3-grams, lower-cased, +/// `k = 3`, a 256-bit filter): every position is below 256, so it fits a +/// `smallint` unchanged. Stored values and queries take it from the same type, +/// so they always agree; matching cipherstash-client's parameters is not a +/// goal, because cipherstash-client never writes an `eql_v4` column. +#[doc(hidden)] +pub struct MatchVisitor; +impl Visitor for MatchVisitor { + type Value = BloomFilter; + // `MatchTerm` is the type's name in the crates.io stack-encrypt this + // crate publishes against, and a deprecated alias of `MatchTerms` in the + // tree: the one spelling that compiles against both. + #[allow(deprecated)] + fn matching(self, terms: MatchTerm) -> Result { + terms + .positions() + .iter() + .map(|&position| { + i16::try_from(position).map_err(|_| { + Error::Other( + "a match position does not fit a smallint; EQL's bloom filter stores \ + filters of at most 32768 bits" + .into(), + ) + }) + }) + .collect::>() + .map(BloomFilter) + } +} +impl Transcode for BloomFilter { + type Visitor = MatchVisitor; + fn visitor() -> Self::Visitor { + MatchVisitor + } +} +impl> EncryptFrom for BloomFilter { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + target::matching::<_, _, _, DefaultMatch>().transcode() + } +} +impl Decryptable for BloomFilter { + const DECRYPTABLE: bool = false; +} +impl DecryptField for BloomFilter { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + #[cfg(test)] mod tests { use stack_encrypt::{nonempty, Descriptor, Label}; diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index e89ab754b..1fb65935f 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -465,10 +465,8 @@ pub const TARGETS: &[Target] = &[ indexes: &["match"], query: Some("TextMatchQuery"), query_sql_domain: Some("eql_v4.query_text_match"), - producible: false, - reason: Some( - "the engine derives match terms, but no EQL type is built from them yet", - ), + producible: true, + reason: None, }, Target { name: "TextOrdOre", @@ -735,6 +733,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "TextEq" => { run_target::("TextEq", keyset, column, plaintext) } + "TextMatch" => { + run_target::("TextMatch", keyset, column, plaintext) + } "Boolean" => { run_target::("Boolean", keyset, column, plaintext) } @@ -762,6 +763,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( } "Text" => open_target::("Text", opener, column, stored), "TextEq" => open_target::("TextEq", opener, column, stored), + "TextMatch" => { + open_target::("TextMatch", opener, column, stored) + } "Boolean" => { open_target::("Boolean", opener, column, stored) } @@ -785,6 +789,12 @@ pub(crate) fn query_named<'a, K: 'static>( "TextEq" => { run_target::("TextEq", keyset, column, plaintext) } + "TextMatch" => run_target::( + "TextMatch", + keyset, + column, + plaintext, + ), _ => Err(refuse_query(name)), } } diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 28ae34d1b..89c9e1cde 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -141,10 +141,28 @@ impl DomainType for TextEq { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +#[cfg_attr(all(doc, feature = "stack-encrypt"), doc = "")] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Plaintext input and decrypted output: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` and `DecryptInto`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " See the [complete encryption example](crate::encryption#example)." +)] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::EncryptFrom))] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::DecryptInto))] +#[cfg_attr(feature = "stack-encrypt", stash(plaintext = String))] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextMatch { + #[cfg_attr(feature = "stack-encrypt", stash(default))] pub v: SchemaVersion, + #[cfg_attr(feature = "stack-encrypt", stash(context_field))] pub i: Identifier, pub c: Ciphertext, pub bf: BloomFilter, @@ -462,10 +480,27 @@ impl DomainType for TextEqQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +#[cfg_attr(all(doc, feature = "stack-encrypt"), doc = "")] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Plaintext input: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` to build equality query terms. Query operands do not decrypt." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " See the [complete encryption example](crate::encryption#example)." +)] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::EncryptFrom))] +#[cfg_attr(feature = "stack-encrypt", stash(plaintext = String))] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextMatchQuery { + #[cfg_attr(feature = "stack-encrypt", stash(default))] pub v: SchemaVersion, + #[cfg_attr(feature = "stack-encrypt", stash(context_field))] pub i: Identifier, pub bf: BloomFilter, } diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index e0a100094..08afcb21a 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -344,6 +344,7 @@ pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("bigint", ""), ("text", ""), ("text", "eq"), + ("text", "match"), ("boolean", ""), ("real", ""), ("double", ""), @@ -1732,8 +1733,6 @@ mod tests { .any(|t| matches!(t, Term::Ore | Term::Ope)) { "ordering terms" - } else if domain.terms.contains(&Term::Bloom) { - "match terms" } else if matches!(family.name, "integer" | "bigint") { "integer-orderable/v1" } else { diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index d9e32fd1d..d13ae65dc 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -390,7 +390,7 @@ mod tests { target_gap(family, family.domain_by_name(name).expect("domain")) }; let text = eql_domains::TEXT; - for producible in ["", "eq"] { + for producible in ["", "eq", "match"] { assert!(gap(&text, producible).is_none(), "text {producible:?}"); } for ordered in ["ord", "ord_ore", "ord_ope", "search", "search_ore"] { @@ -399,7 +399,7 @@ mod tests { "{ordered}" ); } - assert!(gap(&text, "match").unwrap().contains("match terms")); + assert!(gap(&text, "match").is_none(), "TextMatch is producible"); let family = |name: &str| { eql_domains::CATALOG .iter() diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 97c7b1189..d4089d024 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -565,7 +565,16 @@ mod the_table { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], ); for row in targets::targets().iter().filter(|t| !t.producible) { let reason = row.reason.expect("an unproducible type has a reason"); @@ -574,7 +583,6 @@ mod the_table { ("json", "") => "tree of leaves", ("smallint" | "date" | "timestamp" | "numeric", _) => "has no kind", (_, "Ord" | "OrdOre" | "OrdOpe" | "Search" | "SearchOre") => "ordering terms", - ("text", "Match") => "match terms", ("real" | "double", "Eq") => "float", ("integer" | "bigint", "Eq") => "integer-orderable/v1", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 126df4ac5..2dce6de1c 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -1,9 +1,9 @@ //! The EQL v4 types Stack Encrypt produces besides `TextEq`: the -//! storage-only integer, bigint, boolean, float and text types. `Json` is not -//! producible yet: a JSON value seals as a tree of leaves and EQL's `c` holds -//! one. `IntegerEq` and `BigintEq` are held back: their equality term moves to -//! vitaminc 0.6's shared integer PRF domain, and terms stored before that -//! would stop matching. +//! storage-only integer, bigint, boolean, float and text types, and +//! `TextMatch`. `Json` is not producible yet: a JSON value seals as a tree of +//! leaves and EQL's `c` holds one. `IntegerEq` and `BigintEq` are held back: +//! their equality term moves to vitaminc 0.6's shared integer PRF domain, and +//! terms stored before that would stop matching. //! //! Each type must round-trip through both entry points (the typed //! `encrypt_as` and the by-name dispatch a Go plan reaches), seal exactly the @@ -365,6 +365,63 @@ async fn storage_only_text_round_trips_through_both_entry_points() { ); } +/// A match query's filter positions, which EQL's `@@` tests for containment. +fn positions(filter: &eql_bindings::v3::terms::BloomFilter) -> std::collections::BTreeSet { + filter.0.iter().copied().collect() +} + +#[tokio::test] +async fn text_match_filters_contain_their_substrings() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let stored: text::TextMatch = keyset + .encrypt_as(&"Alice@Example.com".to_owned(), column()) + .await + .unwrap(); + assert_v4_envelope(&stored, &["v", "i", "c", "bf"]); + assert!( + stored.bf.0.iter().all(|&p| (0..256).contains(&p)), + "a 256-bit filter" + ); + let opened: String = cipher + .decrypt_as(stored.clone(), column().into()) + .await + .unwrap(); + assert_eq!(opened, "Alice@Example.com"); + + let all = positions(&stored.bf); + for (needle, contained) in [ + ("alice", true), + ("EXAMPLE", true), + ("ice@ex", true), + ("bob", false), + ] { + let query: text::TextMatchQuery = keyset + .encrypt_as(&needle.to_owned(), column()) + .await + .unwrap(); + assert_eq!( + positions(&query.bf).is_subset(&all), + contained, + "{needle:?} in Alice@Example.com" + ); + } + + // The dispatch derives the same filter. + let named: text::TextMatch = by_name( + &keyset, + "TextMatch", + FfiValue::String("Alice@Example.com".into()), + ) + .await; + assert_eq!(named.bf, stored.bf); + + // Text shorter than a 3-gram has no tokens: refused, not stored with an + // empty filter that no query could ever match. + let short: Result = keyset.encrypt_as(&"ab".to_owned(), column()).await; + assert!(short.is_err()); +} + #[tokio::test] async fn a_value_of_another_kind_is_refused_not_converted() { let (cipher, _) = common::cipher().await; @@ -419,7 +476,8 @@ fn postgres_stores_and_queries_every_producible_v4_type() { active public.eql_v4_boolean, ratio public.eql_v4_real, score public.eql_v4_double, - note public.eql_v4_text + note public.eql_v4_text, + name public.eql_v4_text_match );", ) .unwrap(); @@ -452,6 +510,10 @@ fn postgres_stores_and_queries_every_producible_v4_type() { .encrypt_as::<_, text::Text>(&"note".to_owned(), column()) .await .unwrap(), + keyset + .encrypt_as::<_, text::TextMatch>(&format!("user{id}@example.com"), column()) + .await + .unwrap(), ) }); db.execute( @@ -461,7 +523,8 @@ fn postgres_stores_and_queries_every_producible_v4_type() { $4::text::jsonb::public.eql_v4_boolean, $5::text::jsonb::public.eql_v4_real, $6::text::jsonb::public.eql_v4_double, - $7::text::jsonb::public.eql_v4_text)", + $7::text::jsonb::public.eql_v4_text, + $8::text::jsonb::public.eql_v4_text_match)", &[ &id, &json(&row.0), @@ -470,11 +533,29 @@ fn postgres_stores_and_queries_every_producible_v4_type() { &json(&row.3), &json(&row.4), &json(&row.5), + &json(&row.6), ], ) .unwrap(); } + let mut matching = |needle: &str| -> Vec { + let probe: text::TextMatchQuery = runtime + .block_on(async { keyset.encrypt_as(&needle.to_owned(), column()).await }) + .unwrap(); + db.query( + "SELECT id FROM v4_types WHERE name @@ $1::text::jsonb::eql_v4.query_text_match ORDER BY id", + &[&json(&probe)], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect() + }; + assert_eq!(matching("example.com"), [1, 2, 3], "TextMatch: every row"); + assert_eq!(matching("user2"), [2], "TextMatch: one row"); + assert_eq!(matching("nobody"), Vec::::new(), "TextMatch: none"); + // Every stored value reads back from its domain and opens. let row = db .query_one( From 92e7d3316556c9ba9e61c88042d08e83e5b86973 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 12 Oct 2026 23:49:03 +1100 Subject: [PATCH 2/3] fix(eql-bindings): pin TextMatch's filter at k = 6, m = 2048 and package against crates.io The adapter's turbofish named four type parameters for target::matching, and stack-encrypt 0.2.0 on crates.io, which eql-bindings publishes against, takes three, so cargo publish --dry-run --all-features failed. Naming the Encryption type instead lets inference fill the rest under either version. The match configuration was Stack Encrypt's default (k = 3, 256 bits), taken from another crate and recorded nowhere. A 256-bit filter saturates on a few hundred characters, so @@ returned most of a table, and a change to that default would have left stored filters disagreeing with new queries. eql-bindings now pins its own EqlMatch (3-grams, lower-cased, k = 6, 2048 bits); a unit test pins the values and an encryption test pins the filter positions for one query, so a change has to be deliberate. Also: TextMatchQuery's generated docs say match query terms, the Postgres test pins the reverse operand order (query @@ stored) as the known SQL gap it is, the Go guest lists TextMatch as producible, and the READMEs and changeset say that short text cannot be stored yet. Storing an empty filter for it needs a stack-encrypt release first: 0.2.0 refuses token-less text inside the term derivation, before anything here can see it. --- .changeset/eql-v4-bindings.md | 2 +- languages/golang/cmd/stashgen/README.md | 3 +- languages/golang/encrypt/guest/src/targets.rs | 11 +++- .../golang/encrypt/guest/tests/native_ops.rs | 11 +++- packages/eql/crates/eql-bindings/README.md | 11 +++- .../eql/crates/eql-bindings/src/encryption.rs | 64 ++++++++++++++++--- .../eql/crates/eql-bindings/src/v4/text.rs | 2 +- .../eql/crates/eql-codegen/src/bindings.rs | 7 +- .../eql/tests/encryption/tests/v4_types.rs | 51 +++++++++++++-- 9 files changed, 140 insertions(+), 22 deletions(-) diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md index 6b2bb66e2..9059b355a 100644 --- a/.changeset/eql-v4-bindings.md +++ b/.changeset/eql-v4-bindings.md @@ -2,6 +2,6 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch`, from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. +**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused), from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. Nothing a released version shipped has changed. The v3 types, the v3 SQL and the TypeScript package are as they were; the `stack-encrypt` derives on the v3 text types and the `v3::targets` module were added after the last release and are moved to v4 here, not removed from a release. diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 227e1c855..79fd5f867 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -153,7 +153,7 @@ The engine produces these EQL types today, each from one Go field type: | `encrypt_into` | Go field | |---|---| -| `TextEq`, `Text` | `string` | +| `TextEq`, `Text`, `TextMatch` | `string` | | `Integer` | `int32` | | `Bigint` | `int64` | | `Boolean` | `bool` | @@ -161,6 +161,7 @@ The engine produces these EQL types today, each from one Go field type: | `Double` | `float64` | The width must match: an `int` or `int64` field is not an `Integer`, and a `float64` field is not a `Real`. +A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index. `encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 83df35d40..3451b9176 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -181,7 +181,16 @@ mod tests { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], "the engine's producible types, in catalog order" ); assert!(matches!( diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 53e0a42f4..0e653f452 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1368,7 +1368,16 @@ fn targets_lists_what_the_build_holds() { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], "the engine's producible types, in catalog order" ); } diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 2c33f6e29..8a5db0d99 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -124,9 +124,14 @@ seals for the same value, so values written here and by the Go SDK open each other. `IntegerEq` and `BigintEq` wait for vitaminc 0.6, whose integer equality term uses one PRF domain for every width; terms derived under vitaminc 0.5's per-width domains would stop matching after that upgrade. -`TextMatch` uses Stack Encrypt's default match configuration -(3-grams, lower-cased, `k = 3`, a 256-bit filter) for stored values and -queries alike, and refuses text too short to make a 3-gram. Ordering types +`TextMatch` builds its filter under a configuration pinned in this crate +(`EqlMatch`: 3-grams of the lower-cased text, `k = 6`, a 2048-bit filter) for +stored values and queries alike. The configuration is part of the stored +format, so changing it means a new domain or payload version. A filter can +match a row that does not contain the query, so re-check after decrypting +where that matters. Text too short to make a 3-gram (empty strings, two-letter +codes) cannot be stored in a `TextMatch` column yet: Stack Encrypt 0.2 refuses +text with no tokens. Ordering types wait for vitaminc 0.6's order encodings, float equality for its canonical float form, and `Json` for a decision on sealing a document as one leaf. diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index e174b3d01..aeae19653 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -84,7 +84,7 @@ pub mod targets; use base64::{engine::general_purpose::STANDARD, Engine as _}; #[allow(deprecated)] use stack_encrypt::sem::MatchTerm; -use stack_encrypt::sem::{DefaultMatch, EqualityTerm, MatchConfig}; +use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, Tokenizer}; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext}; use stack_encrypt::{ @@ -254,14 +254,38 @@ impl DecryptField for Hmac256 { } } -/// Builds EQL's bloom filter, the set bit positions as `smallint`s, from the -/// native match terms. +/// The match configuration of EQL v4's `TextMatch`: 3-grams of the +/// lower-cased text, `k = 6` positions per token in a 2048-bit filter. +/// +/// Pinned here rather than taken from Stack Encrypt's default, because it is +/// part of the stored format: a stored filter and a query filter agree only +/// when both were built under the same values, and nothing in the payload +/// records them. Changing any of them is a new domain or payload version, +/// never an edit in place. Every position is below 2048, so it fits a +/// `smallint`. /// -/// The configuration is Stack Encrypt's default (3-grams, lower-cased, -/// `k = 3`, a 256-bit filter): every position is below 256, so it fits a -/// `smallint` unchanged. Stored values and queries take it from the same type, -/// so they always agree; matching cipherstash-client's parameters is not a -/// goal, because cipherstash-client never writes an `eql_v4` column. +/// A filter is a set of bits, so `@@` can return rows that do not contain the +/// query (false positives) and never misses one that does. 2048 bits keeps +/// that rate low for text of a few hundred characters; re-check matches +/// after decrypting where it matters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct EqlMatch; +impl MatchConfig for EqlMatch { + fn options() -> MatchOptions { + MatchOptions { + tokenizer: Tokenizer::Ngram { length: 3 }, + downcase: true, + k: 6, + m: 2048, + } + } +} + +/// Builds EQL's bloom filter, the set bit positions as `smallint`s, from the +/// native match terms built under [`EqlMatch`]. Stored values and queries +/// take it from the same type, so they always agree; matching +/// cipherstash-client's parameters is not a goal, because cipherstash-client +/// never writes an `eql_v4` column. #[doc(hidden)] pub struct MatchVisitor; impl Visitor for MatchVisitor { @@ -299,7 +323,13 @@ impl> EncryptFrom for BloomFilter { where S: 's, { - target::matching::<_, _, _, DefaultMatch>().transcode() + // The type names the configuration and lets inference fill the rest: + // `matching` takes three type parameters in the crates.io + // stack-encrypt this crate publishes against and four in the tree, + // so a turbofish compiles against only one of them. + #[allow(deprecated)] + let terms: Encryption<'s, S, MatchTerm, K, CallerContext> = target::matching(); + terms.transcode() } } impl Decryptable for BloomFilter { @@ -317,6 +347,22 @@ mod tests { use super::*; + /// The stored format: a change here strands every filter already stored + /// under the old values, so it must arrive as a new domain or payload + /// version, and this test makes that a deliberate edit. + #[test] + fn the_match_configuration_is_pinned() { + let options = EqlMatch::options(); + assert!(matches!(options.tokenizer, Tokenizer::Ngram { length: 3 })); + assert!(options.downcase); + assert_eq!(options.k, 6); + assert_eq!(options.m, 2048); + assert!( + options.m <= 1 << 15, + "every position must fit EQL's smallint filter" + ); + } + fn id(t: &str, c: &str) -> Identifier { Identifier { t: t.into(), diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 89c9e1cde..d4212dc88 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -483,7 +483,7 @@ impl DomainType for TextEqQuery { #[cfg_attr(all(doc, feature = "stack-encrypt"), doc = "")] #[cfg_attr( all(doc, feature = "stack-encrypt"), - doc = " Plaintext input: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` to build equality query terms. Query operands do not decrypt." + doc = " Plaintext input: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` to build match query terms. Query operands do not decrypt." )] #[cfg_attr( all(doc, feature = "stack-encrypt"), diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 08afcb21a..855e18690 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -441,9 +441,14 @@ fn encryption_attrs( // not exist without it and the link would not resolve. let plaintext_doc = |named: &str| { if query { + let terms = if domain.terms.contains(&Term::Bloom) { + "match" + } else { + "equality" + }; format!( " Plaintext input: {named}. With the `stack-encrypt` feature, \ - implements `EncryptFrom<{shown}>` to build equality query terms. Query operands \ + implements `EncryptFrom<{shown}>` to build {terms} query terms. Query operands \ do not decrypt." ) } else { diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 2dce6de1c..ebff78f11 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -380,8 +380,8 @@ async fn text_match_filters_contain_their_substrings() { .unwrap(); assert_v4_envelope(&stored, &["v", "i", "c", "bf"]); assert!( - stored.bf.0.iter().all(|&p| (0..256).contains(&p)), - "a 256-bit filter" + stored.bf.0.iter().all(|&p| (0..2048).contains(&p)), + "a 2048-bit filter" ); let opened: String = cipher .decrypt_as(stored.clone(), column().into()) @@ -416,12 +416,34 @@ async fn text_match_filters_contain_their_substrings() { .await; assert_eq!(named.bf, stored.bf); - // Text shorter than a 3-gram has no tokens: refused, not stored with an - // empty filter that no query could ever match. + // Text shorter than a 3-gram has no tokens. Stack Encrypt 0.2 refuses it + // for a stored value as well as a query, so a short value cannot be + // written to a TextMatch column yet. let short: Result = keyset.encrypt_as(&"ab".to_owned(), column()).await; assert!(short.is_err()); } +/// The filter bits for one query under the test key source. The +/// configuration is part of the stored format (`EqlMatch`): if these move, +/// every stored filter stops agreeing with new queries, so a change here must +/// arrive as a new domain or payload version. +#[tokio::test] +async fn text_match_filter_positions_are_pinned() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let query: text::TextMatchQuery = keyset + .encrypt_as(&"alice".to_owned(), column()) + .await + .unwrap(); + // 3 tokens ("ali", "lic", "ice") at 6 positions each, de-duplicated. + assert_eq!(query.bf.0, PINNED_ALICE); +} + +const PINNED_ALICE: [i16; 18] = [ + 52, 99, 224, 311, 899, 1190, 1228, 1232, 1290, 1378, 1486, 1505, 1529, 1613, 1763, 1848, 1880, + 2027, +]; + #[tokio::test] async fn a_value_of_another_kind_is_refused_not_converted() { let (cipher, _) = common::cipher().await; @@ -556,6 +578,27 @@ fn postgres_stores_and_queries_every_producible_v4_type() { assert_eq!(matching("user2"), [2], "TextMatch: one row"); assert_eq!(matching("nobody"), Vec::::new(), "TextMatch: none"); + // Known gap in the SQL, pinned so its fix shows here: with the query on + // the left, `@@` asks whether the query's filter contains the stored + // value's, so the operand order users reach for finds nothing. + let probe: text::TextMatchQuery = runtime + .block_on(async { keyset.encrypt_as(&"user2".to_owned(), column()).await }) + .unwrap(); + let reversed: Vec = db + .query( + "SELECT id FROM v4_types WHERE $1::text::jsonb::eql_v4.query_text_match @@ name", + &[&json(&probe)], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!( + reversed, + Vec::::new(), + "query @@ stored still finds nothing; when the SQL is fixed this becomes [2]" + ); + // Every stored value reads back from its domain and opens. let row = db .query_one( From be7e1e3262df0c57194f11b89e603e81a0aa8855 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 15:03:26 +1100 Subject: [PATCH 3/3] fix(eql-bindings, stashgen): document TextMatch's limits, and teach the test engine TextMatch - The eql-bindings README, the stashgen README and the changeset say that only `stored @@ query` matches until #1171 is fixed, and that a TextMatchQuery for text shorter than three characters is refused too. - The PostgreSQL test's reversed-order assertion names #1171, and the filter test says its negative case is a property of one key source and configuration, not of Bloom filters. - enginetest.Static produces TextMatch; the refusal case for an EQL type the engine cannot produce now names IntegerEq; the users golden case stores Name as TextMatch, so generation of its field methods is exercised. - encryption_gap drops its Bloom branch, which no catalog domain reached once text_match became producible. - The generated TextMatchQuery doc keeps the plaintext line without the stack-encrypt feature. Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- .changeset/eql-v4-bindings.md | 2 +- languages/golang/cmd/stashgen/README.md | 3 ++- .../golang/stashgen/enginetest/enginetest.go | 1 + languages/golang/stashgen/refusal_test.go | 2 +- .../stashgen/testdata/cases/users/model.go | 2 +- .../testdata/cases/users/user_stash.go.golden | 18 +++++++++--------- .../testdata/stubsdk/encrypt/eql/eql.go | 10 ++++++++-- packages/eql/crates/eql-bindings/README.md | 8 +++++--- .../eql/crates/eql-bindings/src/v4/text.rs | 12 ++++++++++-- .../eql/crates/eql-codegen/src/bindings.rs | 2 -- .../eql/tests/encryption/tests/v4_types.rs | 14 ++++++++++---- 11 files changed, 48 insertions(+), 26 deletions(-) diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md index 9059b355a..1ccfcdd3a 100644 --- a/.changeset/eql-v4-bindings.md +++ b/.changeset/eql-v4-bindings.md @@ -2,6 +2,6 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused), from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. +**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused, stored or queried, and only `stored @@ query` matches until #1171 is fixed), from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. Nothing a released version shipped has changed. The v3 types, the v3 SQL and the TypeScript package are as they were; the `stack-encrypt` derives on the v3 text types and the `v3::targets` module were added after the last release and are moved to v4 here, not removed from a release. diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 79fd5f867..054659d1f 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -161,7 +161,8 @@ The engine produces these EQL types today, each from one Go field type: | `Double` | `float64` | The width must match: an `int` or `int64` field is not an `Integer`, and a `float64` field is not a `Real`. -A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index. +A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index; its query refuses such text too. +Put the column on the left of `@@` (`WHERE name @@ $1::eql_v4.query_text_match`): the other operand order finds nothing until #1171 is fixed. `encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 31e8b8419..bae3a6b38 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -32,6 +32,7 @@ var eqlTypes = []struct { {stashgen.EQLType{Name: "Bigint", Plaintext: stashgen.KindInt, Producible: true}, "int64"}, {stashgen.EQLType{Name: "Text", Plaintext: stashgen.KindString, Producible: true}, ""}, {stashgen.EQLType{Name: "TextEq", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality}, Query: "TextEqQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextMatch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexMatch}, Query: "TextMatchQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "Boolean", Plaintext: stashgen.KindBool, Producible: true}, ""}, {stashgen.EQLType{Name: "Real", Plaintext: stashgen.KindFloat, Producible: true}, "float32"}, {stashgen.EQLType{Name: "Double", Plaintext: stashgen.KindFloat, Producible: true}, "float64"}, diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 2ab440c56..4e44f7ebb 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -91,7 +91,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, - {"an EQL type the engine cannot produce yet", user(ctx + "\tEmail string `stash:\"email,encrypt_into=TextMatch\"`"), stashgen.Request{Type: "User"}, "Email", "has no EQL type TextMatch"}, + {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int32 `stash:\"age,encrypt_into=IntegerEq\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type IntegerEq"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, diff --git a/languages/golang/stashgen/testdata/cases/users/model.go b/languages/golang/stashgen/testdata/cases/users/model.go index 2c1d23961..8fd94e5eb 100644 --- a/languages/golang/stashgen/testdata/cases/users/model.go +++ b/languages/golang/stashgen/testdata/cases/users/model.go @@ -8,6 +8,6 @@ type User struct { _ struct{} `stash:"context=users"` ID int64 `stash:"id,passthrough" db:"id" gorm:"primaryKey"` Email string `stash:"email,encrypt_into=TextEq" db:"email"` - Name string `stash:"name,encrypt_into=TextEq" db:"name"` + Name string `stash:"name,encrypt_into=TextMatch" db:"name"` Internal string `stash:"-"` } diff --git a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden index 9a82606d3..a03ae9544 100644 --- a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden @@ -18,9 +18,9 @@ const _ = gensupport.GeneratedVersion1 // method. Write them, or run stashgen with -redact. type EncryptedUser struct { - ID int64 `db:"id" gorm:"primaryKey"` - Email eql.TextEq `db:"email"` - Name eql.TextEq `db:"name"` + ID int64 `db:"id" gorm:"primaryKey"` + Email eql.TextEq `db:"email"` + Name eql.TextMatch `db:"name"` } func (e EncryptedUser) String() string { @@ -45,7 +45,7 @@ type userShape struct { var declaration = gensupport.Declare("users"). Passthrough("id"). EncryptInto("email", gensupport.String, "TextEq"). - EncryptInto("name", gensupport.String, "TextEq"). + EncryptInto("name", gensupport.String, "TextMatch"). Omit("internal") var codec = gensupport.New(gensupport.Generated[User, EncryptedUser]{ @@ -66,7 +66,7 @@ var codec = gensupport.New(gensupport.Generated[User, EncryptedUser]{ return EncryptedUser{}, err } e.Email = eql.TextEq(rec["email"].EQL) - e.Name = eql.TextEq(rec["name"].EQL) + e.Name = eql.TextMatch(rec["name"].EQL) return e, nil }, Open: func(e EncryptedUser) gensupport.Record { @@ -131,12 +131,12 @@ type NameField struct { field gensupport.Field[string] } -func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextEq, error) { +func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.TextEq(out.EQL), err + return eql.TextMatch(out.EQL), err } -func (f NameField) Query(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextEqQuery, error) { +func (f NameField) Query(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatchQuery, error) { out, err := f.field.Query(ctx, c, v) - return eql.TextEqQuery(out.EQL), err + return eql.TextMatchQuery(out.EQL), err } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go index 09f4934a7..17fac8ef2 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go @@ -1,5 +1,5 @@ -// Package eql is a TEST STUB of the generated EQL types: the one type the -// engine produces today, and its query type. +// Package eql is a TEST STUB of the generated EQL types: the types the +// golden cases name, and their query types. package eql // TextEq is an EQL value with an equality term. @@ -7,3 +7,9 @@ type TextEq []byte // TextEqQuery is the query value for TextEq. type TextEqQuery []byte + +// TextMatch is an EQL value with a match term. +type TextMatch []byte + +// TextMatchQuery is the query value for TextMatch. +type TextMatchQuery []byte diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 8a5db0d99..5a16a25fb 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -129,9 +129,11 @@ vitaminc 0.5's per-width domains would stop matching after that upgrade. stored values and queries alike. The configuration is part of the stored format, so changing it means a new domain or payload version. A filter can match a row that does not contain the query, so re-check after decrypting -where that matters. Text too short to make a 3-gram (empty strings, two-letter -codes) cannot be stored in a `TextMatch` column yet: Stack Encrypt 0.2 refuses -text with no tokens. Ordering types +where that matters. Write the stored column on the left, `col @@ query`: the +other order finds nothing until #1171 is fixed. Text too short to make a +3-gram (empty strings, two-letter codes) cannot be stored in a `TextMatch` +column yet, and a `TextMatchQuery` for it is refused too: Stack Encrypt 0.2 +refuses text with no tokens, on both sides. Ordering types wait for vitaminc 0.6's order encodings, float equality for its canonical float form, and `Json` for a decision on sealing a document as one leaf. diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index d4212dc88..0270227ed 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -141,11 +141,15 @@ impl DomainType for TextEq { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] -#[cfg_attr(all(doc, feature = "stack-encrypt"), doc = "")] +#[cfg_attr(doc, doc = "")] #[cfg_attr( all(doc, feature = "stack-encrypt"), doc = " Plaintext input and decrypted output: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` and `DecryptInto`." )] +#[cfg_attr( + all(doc, not(feature = "stack-encrypt")), + doc = " Plaintext input and decrypted output: `String`. With the `stack-encrypt` feature, implements `EncryptFrom` and `DecryptInto`." +)] #[cfg_attr( all(doc, feature = "stack-encrypt"), doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." @@ -480,11 +484,15 @@ impl DomainType for TextEqQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] -#[cfg_attr(all(doc, feature = "stack-encrypt"), doc = "")] +#[cfg_attr(doc, doc = "")] #[cfg_attr( all(doc, feature = "stack-encrypt"), doc = " Plaintext input: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` to build match query terms. Query operands do not decrypt." )] +#[cfg_attr( + all(doc, not(feature = "stack-encrypt")), + doc = " Plaintext input: `String`. With the `stack-encrypt` feature, implements `EncryptFrom` to build match query terms. Query operands do not decrypt." +)] #[cfg_attr( all(doc, feature = "stack-encrypt"), doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index 855e18690..d6396a642 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -400,8 +400,6 @@ pub fn encryption_gap( .any(|t| matches!(t, Term::Ore | Term::Ope)) { "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)" - } else if domain.terms.contains(&Term::Bloom) { - "the engine derives match terms, but no EQL type is built from them yet" } else if matches!(family.name, "integer" | "bigint") && domain.terms == [Term::Hm] { "the integer equality term moves to vitaminc 0.6's shared integer-orderable/v1 PRF \ domain; producing it before that release would leave stored terms unmatchable after \ diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index ebff78f11..9ad6e4613 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -389,6 +389,11 @@ async fn text_match_filters_contain_their_substrings() { .unwrap(); assert_eq!(opened, "Alice@Example.com"); + // `true` is a property of the filter: every bit a contained needle sets + // is set in the stored value's. `false` is not: a Bloom filter can cover + // an absent needle's bits by chance. "bob" is clear of it under the + // test key source and the pinned configuration, so a change to either + // can flip that case with no bug behind it. let all = positions(&stored.bf); for (needle, contained) in [ ("alice", true), @@ -578,9 +583,10 @@ fn postgres_stores_and_queries_every_producible_v4_type() { assert_eq!(matching("user2"), [2], "TextMatch: one row"); assert_eq!(matching("nobody"), Vec::::new(), "TextMatch: none"); - // Known gap in the SQL, pinned so its fix shows here: with the query on - // the left, `@@` asks whether the query's filter contains the stored - // value's, so the operand order users reach for finds nothing. + // Known gap in the SQL (#1171), pinned so its fix shows here: with the + // query on the left, `@@` asks whether the query's filter contains the + // stored value's, so that operand order finds nothing. Only + // `stored @@ query` works until then; the README says so. let probe: text::TextMatchQuery = runtime .block_on(async { keyset.encrypt_as(&"user2".to_owned(), column()).await }) .unwrap(); @@ -596,7 +602,7 @@ fn postgres_stores_and_queries_every_producible_v4_type() { assert_eq!( reversed, Vec::::new(), - "query @@ stored still finds nothing; when the SQL is fixed this becomes [2]" + "query @@ stored still finds nothing (#1171); when the SQL is fixed this becomes [2]" ); // Every stored value reads back from its domain and opens.