diff --git a/.changeset/eql-v4-bindings.md b/.changeset/eql-v4-bindings.md index 546a460d7..1ccfcdd3a 100644 --- a/.changeset/eql-v4-bindings.md +++ b/.changeset/eql-v4-bindings.md @@ -2,6 +2,6 @@ '@cipherstash/eql': minor --- -**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. +**The `eql-bindings` crate has an `eql_bindings::v4` module: payload types for the EQL v4 domains.** EQL v4 is the v3 SQL installed under a second name (ADR-0002), for values written by Stack Encrypt. The module is generated from the same catalog as `v3`, with `eql_v4_*` domain names and `"v": 4`. `SchemaVersion` is now generic over the version it admits: `v3::SchemaVersion` is `SchemaVersion<3>` and `v4::SchemaVersion` is `SchemaVersion<4>`, each refusing the other's payloads, and plain `SchemaVersion` still means v3. Only v4 types derive Stack Encrypt's `EncryptFrom` and `DecryptInto`, and the Stack Encrypt target table (now `eql_bindings::v4::targets`) names v4 domains. Besides `TextEq`, Stack Encrypt now produces the storage-only types `Text`, `Integer`, `Bigint`, `Boolean`, `Real` and `Double`, and `TextMatch` (3-grams, `k = 6`, a 2048-bit filter, pinned as `eql_bindings::encryption::EqlMatch`; text shorter than three characters is refused, stored or queried, and only `stored @@ query` matches until #1171 is fixed), from the plaintext types in `eql_bindings::encryption::plaintext`; the Go SDK's `encrypt/eql` table marks them producible. `IntegerEq` and `BigintEq` stay unproducible until vitaminc 0.6 settles the integer equality term. Nothing a released version shipped has changed. The v3 types, the v3 SQL and the TypeScript package are as they were; the `stack-encrypt` derives on the v3 text types and the `v3::targets` module were added after the last release and are moved to v4 here, not removed from a release. diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 227e1c855..054659d1f 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -153,7 +153,7 @@ The engine produces these EQL types today, each from one Go field type: | `encrypt_into` | Go field | |---|---| -| `TextEq`, `Text` | `string` | +| `TextEq`, `Text`, `TextMatch` | `string` | | `Integer` | `int32` | | `Bigint` | `int64` | | `Boolean` | `bool` | @@ -161,6 +161,8 @@ The engine produces these EQL types today, each from one Go field type: | `Double` | `float64` | The width must match: an `int` or `int64` field is not an `Integer`, and a `float64` field is not a `Real`. +A `TextMatch` field refuses text shorter than three characters, such as an empty string or a two-letter code, because it has no 3-grams to index; its query refuses such text too. +Put the column on the left of `@@` (`WHERE name @@ $1::eql_v4.query_text_match`): the other operand order finds nothing until #1171 is fixed. `encrypt_into` with any other type is refused with the type's name and the reason it is not produced yet. Separate columns work today for four indexes: `equality`, `match`, `ore` and `ope`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index b832fcb4f..a9558ab1a 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -142,7 +142,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { t.Fatalf("the generated file does not name the EQL type:\n%s", written) } // A type the engine cannot produce yet: refused by name, nothing written. - unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextMatch", 1) + unproducible := strings.Replace(userSource, "encrypt_into=TextEq", "encrypt_into=TextOrdOre", 1) dir = writeModule(t, map[string]string{"model.go": unproducible}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { @@ -150,7 +150,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // Refused by the generator with the engine's reason, before the engine // is asked: se_targets lists the type as not producible. - if !strings.Contains(stderr.String(), "cannot produce the EQL type TextMatch yet") { + if !strings.Contains(stderr.String(), "cannot produce the EQL type TextOrdOre yet") { t.Fatalf("stderr = %q", stderr.String()) } if _, err := os.Stat(filepath.Join(dir, "user_stash.go")); !os.IsNotExist(err) { diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index d7b185e15..77eee6fe3 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -41,7 +41,7 @@ var Types = []Type{ {Name: "NumericOrdOpe", GoName: "NumericOrdOpe", Family: "numeric", Suffix: "OrdOpe", Plaintext: "", SQLDomain: "public.eql_v4_numeric_ord_ope", Indexes: []string{"ope"}, Query: "NumericOrdOpeQuery", QuerySQLDomain: "eql_v4.query_numeric_ord_ope", Producible: false, Reason: "vitaminc 0.5's value enum has no kind for this family; the kind and its encodings arrive in vitaminc 0.6 (ADR-0002)"}, {Name: "Text", GoName: "Text", Family: "text", Suffix: "", Plaintext: "string", SQLDomain: "public.eql_v4_text", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: true, Reason: ""}, {Name: "TextEq", GoName: "TextEq", Family: "text", Suffix: "Eq", Plaintext: "string", SQLDomain: "public.eql_v4_text_eq", Indexes: []string{"eq"}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, Reason: ""}, - {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: false, Reason: "the engine derives match terms, but no EQL type is built from them yet"}, + {Name: "TextMatch", GoName: "TextMatch", Family: "text", Suffix: "Match", Plaintext: "string", SQLDomain: "public.eql_v4_text_match", Indexes: []string{"match"}, Query: "TextMatchQuery", QuerySQLDomain: "eql_v4.query_text_match", Producible: true, Reason: ""}, {Name: "TextOrdOre", GoName: "TextOrdOre", Family: "text", Suffix: "OrdOre", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ore", Indexes: []string{"eq", "ore"}, Query: "TextOrdOreQuery", QuerySQLDomain: "eql_v4.query_text_ord_ore", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, {Name: "TextOrd", GoName: "TextOrd", Family: "text", Suffix: "Ord", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord", Indexes: []string{"eq", "ope"}, Query: "TextOrdQuery", QuerySQLDomain: "eql_v4.query_text_ord", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, {Name: "TextOrdOpe", GoName: "TextOrdOpe", Family: "text", Suffix: "OrdOpe", Plaintext: "string", SQLDomain: "public.eql_v4_text_ord_ope", Indexes: []string{"eq", "ope"}, Query: "TextOrdOpeQuery", QuerySQLDomain: "eql_v4.query_text_ord_ope", Producible: false, Reason: "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)"}, @@ -1406,7 +1406,7 @@ func (v *TextEqQuery) UnmarshalJSON(b []byte) error { } // TextMatch is the EQL type stored in public.eql_v4_text_match: the text family with the match index. -// The engine cannot produce it yet: the engine derives match terms, but no EQL type is built from them yet. +// The engine produces it from a string. type TextMatch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 92ac9ea5a..efc4c4319 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -19,7 +19,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { t.Errorf("%s: an unproducible type has a reason", typ.Name) } } - want := []string{"Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"} + want := []string{"Integer", "Bigint", "Text", "TextEq", "TextMatch", "Boolean", "Real", "Double"} if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) } diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 83df35d40..3451b9176 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -181,7 +181,16 @@ mod tests { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], "the engine's producible types, in catalog order" ); assert!(matches!( diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 53e0a42f4..0e653f452 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1368,7 +1368,16 @@ fn targets_lists_what_the_build_holds() { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], "the engine's producible types, in catalog order" ); } diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 31e8b8419..bae3a6b38 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -32,6 +32,7 @@ var eqlTypes = []struct { {stashgen.EQLType{Name: "Bigint", Plaintext: stashgen.KindInt, Producible: true}, "int64"}, {stashgen.EQLType{Name: "Text", Plaintext: stashgen.KindString, Producible: true}, ""}, {stashgen.EQLType{Name: "TextEq", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexEquality}, Query: "TextEqQuery", Producible: true}, ""}, + {stashgen.EQLType{Name: "TextMatch", Plaintext: stashgen.KindString, Indexes: []stashgen.IndexName{stashgen.IndexMatch}, Query: "TextMatchQuery", Producible: true}, ""}, {stashgen.EQLType{Name: "Boolean", Plaintext: stashgen.KindBool, Producible: true}, ""}, {stashgen.EQLType{Name: "Real", Plaintext: stashgen.KindFloat, Producible: true}, "float32"}, {stashgen.EQLType{Name: "Double", Plaintext: stashgen.KindFloat, Producible: true}, "float64"}, diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 2ab440c56..4e44f7ebb 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -91,7 +91,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with an interface", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tAny any"), stashgen.Request{Type: "User"}, "Any", "is not a type JSON carries"}, {"an opaque struct with a struct JSON would truncate", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tTx hidden") + "\ntype hidden struct {\n\tA int\n\tb int\n}\n", stashgen.Request{Type: "User"}, "Tx", "JSON would drop"}, {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, - {"an EQL type the engine cannot produce yet", user(ctx + "\tEmail string `stash:\"email,encrypt_into=TextMatch\"`"), stashgen.Request{Type: "User"}, "Email", "has no EQL type TextMatch"}, + {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int32 `stash:\"age,encrypt_into=IntegerEq\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type IntegerEq"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, diff --git a/languages/golang/stashgen/testdata/cases/users/model.go b/languages/golang/stashgen/testdata/cases/users/model.go index 2c1d23961..8fd94e5eb 100644 --- a/languages/golang/stashgen/testdata/cases/users/model.go +++ b/languages/golang/stashgen/testdata/cases/users/model.go @@ -8,6 +8,6 @@ type User struct { _ struct{} `stash:"context=users"` ID int64 `stash:"id,passthrough" db:"id" gorm:"primaryKey"` Email string `stash:"email,encrypt_into=TextEq" db:"email"` - Name string `stash:"name,encrypt_into=TextEq" db:"name"` + Name string `stash:"name,encrypt_into=TextMatch" db:"name"` Internal string `stash:"-"` } diff --git a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden index 9a82606d3..a03ae9544 100644 --- a/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden +++ b/languages/golang/stashgen/testdata/cases/users/user_stash.go.golden @@ -18,9 +18,9 @@ const _ = gensupport.GeneratedVersion1 // method. Write them, or run stashgen with -redact. type EncryptedUser struct { - ID int64 `db:"id" gorm:"primaryKey"` - Email eql.TextEq `db:"email"` - Name eql.TextEq `db:"name"` + ID int64 `db:"id" gorm:"primaryKey"` + Email eql.TextEq `db:"email"` + Name eql.TextMatch `db:"name"` } func (e EncryptedUser) String() string { @@ -45,7 +45,7 @@ type userShape struct { var declaration = gensupport.Declare("users"). Passthrough("id"). EncryptInto("email", gensupport.String, "TextEq"). - EncryptInto("name", gensupport.String, "TextEq"). + EncryptInto("name", gensupport.String, "TextMatch"). Omit("internal") var codec = gensupport.New(gensupport.Generated[User, EncryptedUser]{ @@ -66,7 +66,7 @@ var codec = gensupport.New(gensupport.Generated[User, EncryptedUser]{ return EncryptedUser{}, err } e.Email = eql.TextEq(rec["email"].EQL) - e.Name = eql.TextEq(rec["name"].EQL) + e.Name = eql.TextMatch(rec["name"].EQL) return e, nil }, Open: func(e EncryptedUser) gensupport.Record { @@ -131,12 +131,12 @@ type NameField struct { field gensupport.Field[string] } -func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextEq, error) { +func (f NameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.TextEq(out.EQL), err + return eql.TextMatch(out.EQL), err } -func (f NameField) Query(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextEqQuery, error) { +func (f NameField) Query(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatchQuery, error) { out, err := f.field.Query(ctx, c, v) - return eql.TextEqQuery(out.EQL), err + return eql.TextMatchQuery(out.EQL), err } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go index 09f4934a7..17fac8ef2 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/eql/eql.go @@ -1,5 +1,5 @@ -// Package eql is a TEST STUB of the generated EQL types: the one type the -// engine produces today, and its query type. +// Package eql is a TEST STUB of the generated EQL types: the types the +// golden cases name, and their query types. package eql // TextEq is an EQL value with an equality term. @@ -7,3 +7,9 @@ type TextEq []byte // TextEqQuery is the query value for TextEq. type TextEqQuery []byte + +// TextMatch is an EQL value with a match term. +type TextMatch []byte + +// TextMatchQuery is the query value for TextMatch. +type TextMatchQuery []byte diff --git a/packages/eql/crates/eql-bindings/README.md b/packages/eql/crates/eql-bindings/README.md index 86c2a9a8c..5a16a25fb 100644 --- a/packages/eql/crates/eql-bindings/README.md +++ b/packages/eql/crates/eql-bindings/README.md @@ -112,7 +112,7 @@ encryption for the scalar types: | v4 type | Plaintext | | --- | --- | -| `Text` | `String` | +| `Text`, `TextMatch` | `String` | | `Integer` | `Int32` | | `Bigint` | `Int64` | | `Boolean` | `Bool` | @@ -124,7 +124,17 @@ seals for the same value, so values written here and by the Go SDK open each other. `IntegerEq` and `BigintEq` wait for vitaminc 0.6, whose integer equality term uses one PRF domain for every width; terms derived under vitaminc 0.5's per-width domains would stop matching after that upgrade. -Ordering types wait for vitaminc 0.6's order encodings, float +`TextMatch` builds its filter under a configuration pinned in this crate +(`EqlMatch`: 3-grams of the lower-cased text, `k = 6`, a 2048-bit filter) for +stored values and queries alike. The configuration is part of the stored +format, so changing it means a new domain or payload version. A filter can +match a row that does not contain the query, so re-check after decrypting +where that matters. Write the stored column on the left, `col @@ query`: the +other order finds nothing until #1171 is fixed. Text too short to make a +3-gram (empty strings, two-letter codes) cannot be stored in a `TextMatch` +column yet, and a `TextMatchQuery` for it is refused too: Stack Encrypt 0.2 +refuses text with no tokens, on both sides. Ordering types +wait for vitaminc 0.6's order encodings, float equality for its canonical float form, and `Json` for a decision on sealing a document as one leaf. diff --git a/packages/eql/crates/eql-bindings/src/encryption.rs b/packages/eql/crates/eql-bindings/src/encryption.rs index 0134178a3..aeae19653 100644 --- a/packages/eql/crates/eql-bindings/src/encryption.rs +++ b/packages/eql/crates/eql-bindings/src/encryption.rs @@ -82,7 +82,9 @@ pub mod plaintext; pub mod targets; use base64::{engine::general_purpose::STANDARD, Engine as _}; -use stack_encrypt::sem::EqualityTerm; +#[allow(deprecated)] +use stack_encrypt::sem::MatchTerm; +use stack_encrypt::sem::{EqualityTerm, MatchConfig, MatchOptions, Tokenizer}; use stack_encrypt::target::transcode::{Reader, Transcode, Visitor}; use stack_encrypt::target::{self, AeadContext, CallerContext}; use stack_encrypt::{ @@ -93,7 +95,7 @@ use stack_encrypt::{ use vitaminc_prf::PrfValue; use crate::{ - v3::terms::{Ciphertext, Hmac256}, + v3::terms::{BloomFilter, Ciphertext, Hmac256}, Identifier, }; @@ -252,12 +254,115 @@ impl DecryptField for Hmac256 { } } +/// The match configuration of EQL v4's `TextMatch`: 3-grams of the +/// lower-cased text, `k = 6` positions per token in a 2048-bit filter. +/// +/// Pinned here rather than taken from Stack Encrypt's default, because it is +/// part of the stored format: a stored filter and a query filter agree only +/// when both were built under the same values, and nothing in the payload +/// records them. Changing any of them is a new domain or payload version, +/// never an edit in place. Every position is below 2048, so it fits a +/// `smallint`. +/// +/// A filter is a set of bits, so `@@` can return rows that do not contain the +/// query (false positives) and never misses one that does. 2048 bits keeps +/// that rate low for text of a few hundred characters; re-check matches +/// after decrypting where it matters. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct EqlMatch; +impl MatchConfig for EqlMatch { + fn options() -> MatchOptions { + MatchOptions { + tokenizer: Tokenizer::Ngram { length: 3 }, + downcase: true, + k: 6, + m: 2048, + } + } +} + +/// Builds EQL's bloom filter, the set bit positions as `smallint`s, from the +/// native match terms built under [`EqlMatch`]. Stored values and queries +/// take it from the same type, so they always agree; matching +/// cipherstash-client's parameters is not a goal, because cipherstash-client +/// never writes an `eql_v4` column. +#[doc(hidden)] +pub struct MatchVisitor; +impl Visitor for MatchVisitor { + type Value = BloomFilter; + // `MatchTerm` is the type's name in the crates.io stack-encrypt this + // crate publishes against, and a deprecated alias of `MatchTerms` in the + // tree: the one spelling that compiles against both. + #[allow(deprecated)] + fn matching(self, terms: MatchTerm) -> Result { + terms + .positions() + .iter() + .map(|&position| { + i16::try_from(position).map_err(|_| { + Error::Other( + "a match position does not fit a smallint; EQL's bloom filter stores \ + filters of at most 32768 bits" + .into(), + ) + }) + }) + .collect::>() + .map(BloomFilter) + } +} +impl Transcode for BloomFilter { + type Visitor = MatchVisitor; + fn visitor() -> Self::Visitor { + MatchVisitor + } +} +impl> EncryptFrom for BloomFilter { + type Context = CallerContext; + fn encryption<'s, K: 'static>() -> Encryption<'s, S, Self, K, Self::Context> + where + S: 's, + { + // The type names the configuration and lets inference fill the rest: + // `matching` takes three type parameters in the crates.io + // stack-encrypt this crate publishes against and four in the tree, + // so a turbofish compiles against only one of them. + #[allow(deprecated)] + let terms: Encryption<'s, S, MatchTerm, K, CallerContext> = target::matching(); + terms.transcode() + } +} +impl Decryptable for BloomFilter { + const DECRYPTABLE: bool = false; +} +impl DecryptField for BloomFilter { + fn decryption_field(self, _: Ctx) -> Option> { + None + } +} + #[cfg(test)] mod tests { use stack_encrypt::{nonempty, Descriptor, Label}; use super::*; + /// The stored format: a change here strands every filter already stored + /// under the old values, so it must arrive as a new domain or payload + /// version, and this test makes that a deliberate edit. + #[test] + fn the_match_configuration_is_pinned() { + let options = EqlMatch::options(); + assert!(matches!(options.tokenizer, Tokenizer::Ngram { length: 3 })); + assert!(options.downcase); + assert_eq!(options.k, 6); + assert_eq!(options.m, 2048); + assert!( + options.m <= 1 << 15, + "every position must fit EQL's smallint filter" + ); + } + fn id(t: &str, c: &str) -> Identifier { Identifier { t: t.into(), diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index e89ab754b..1fb65935f 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -465,10 +465,8 @@ pub const TARGETS: &[Target] = &[ indexes: &["match"], query: Some("TextMatchQuery"), query_sql_domain: Some("eql_v4.query_text_match"), - producible: false, - reason: Some( - "the engine derives match terms, but no EQL type is built from them yet", - ), + producible: true, + reason: None, }, Target { name: "TextOrdOre", @@ -735,6 +733,9 @@ pub(crate) fn encrypt_named<'a, K: 'static>( "TextEq" => { run_target::("TextEq", keyset, column, plaintext) } + "TextMatch" => { + run_target::("TextMatch", keyset, column, plaintext) + } "Boolean" => { run_target::("Boolean", keyset, column, plaintext) } @@ -762,6 +763,9 @@ pub(crate) fn decrypt_named<'a, K: 'static>( } "Text" => open_target::("Text", opener, column, stored), "TextEq" => open_target::("TextEq", opener, column, stored), + "TextMatch" => { + open_target::("TextMatch", opener, column, stored) + } "Boolean" => { open_target::("Boolean", opener, column, stored) } @@ -785,6 +789,12 @@ pub(crate) fn query_named<'a, K: 'static>( "TextEq" => { run_target::("TextEq", keyset, column, plaintext) } + "TextMatch" => run_target::( + "TextMatch", + keyset, + column, + plaintext, + ), _ => Err(refuse_query(name)), } } diff --git a/packages/eql/crates/eql-bindings/src/v4/text.rs b/packages/eql/crates/eql-bindings/src/v4/text.rs index 28ae34d1b..0270227ed 100644 --- a/packages/eql/crates/eql-bindings/src/v4/text.rs +++ b/packages/eql/crates/eql-bindings/src/v4/text.rs @@ -141,10 +141,32 @@ impl DomainType for TextEq { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +#[cfg_attr(doc, doc = "")] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Plaintext input and decrypted output: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` and `DecryptInto`." +)] +#[cfg_attr( + all(doc, not(feature = "stack-encrypt")), + doc = " Plaintext input and decrypted output: `String`. With the `stack-encrypt` feature, implements `EncryptFrom` and `DecryptInto`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " See the [complete encryption example](crate::encryption#example)." +)] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::EncryptFrom))] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::DecryptInto))] +#[cfg_attr(feature = "stack-encrypt", stash(plaintext = String))] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextMatch { + #[cfg_attr(feature = "stack-encrypt", stash(default))] pub v: SchemaVersion, + #[cfg_attr(feature = "stack-encrypt", stash(context_field))] pub i: Identifier, pub c: Ciphertext, pub bf: BloomFilter, @@ -462,10 +484,31 @@ impl DomainType for TextEqQuery { doc, doc = " See the [EQL text reference](https://cipherstash.com/docs/reference/eql/text) for SQL domain variants, operators, and query examples." )] +#[cfg_attr(doc, doc = "")] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Plaintext input: [`String`](String). With the `stack-encrypt` feature, implements `EncryptFrom` to build match query terms. Query operands do not decrypt." +)] +#[cfg_attr( + all(doc, not(feature = "stack-encrypt")), + doc = " Plaintext input: `String`. With the `stack-encrypt` feature, implements `EncryptFrom` to build match query terms. Query operands do not decrypt." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " Encryption context: `NonEmpty`, constructed with `Identifier::for_column(table, column)` and stored in `i`." +)] +#[cfg_attr( + all(doc, feature = "stack-encrypt"), + doc = " See the [complete encryption example](crate::encryption#example)." +)] +#[cfg_attr(feature = "stack-encrypt", derive(stack_encrypt::EncryptFrom))] +#[cfg_attr(feature = "stack-encrypt", stash(plaintext = String))] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] #[serde(deny_unknown_fields)] pub struct TextMatchQuery { + #[cfg_attr(feature = "stack-encrypt", stash(default))] pub v: SchemaVersion, + #[cfg_attr(feature = "stack-encrypt", stash(context_field))] pub i: Identifier, pub bf: BloomFilter, } diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index e0a100094..d6396a642 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -344,6 +344,7 @@ pub const ENCRYPTION_DOMAINS: &[(&str, &str)] = &[ ("bigint", ""), ("text", ""), ("text", "eq"), + ("text", "match"), ("boolean", ""), ("real", ""), ("double", ""), @@ -399,8 +400,6 @@ pub fn encryption_gap( .any(|t| matches!(t, Term::Ore | Term::Ope)) { "ordering terms wait for vitaminc-ore, the order encodings in vitaminc 0.6 (ADR-0002)" - } else if domain.terms.contains(&Term::Bloom) { - "the engine derives match terms, but no EQL type is built from them yet" } else if matches!(family.name, "integer" | "bigint") && domain.terms == [Term::Hm] { "the integer equality term moves to vitaminc 0.6's shared integer-orderable/v1 PRF \ domain; producing it before that release would leave stored terms unmatchable after \ @@ -440,9 +439,14 @@ fn encryption_attrs( // not exist without it and the link would not resolve. let plaintext_doc = |named: &str| { if query { + let terms = if domain.terms.contains(&Term::Bloom) { + "match" + } else { + "equality" + }; format!( " Plaintext input: {named}. With the `stack-encrypt` feature, \ - implements `EncryptFrom<{shown}>` to build equality query terms. Query operands \ + implements `EncryptFrom<{shown}>` to build {terms} query terms. Query operands \ do not decrypt." ) } else { @@ -1732,8 +1736,6 @@ mod tests { .any(|t| matches!(t, Term::Ore | Term::Ope)) { "ordering terms" - } else if domain.terms.contains(&Term::Bloom) { - "match terms" } else if matches!(family.name, "integer" | "bigint") { "integer-orderable/v1" } else { diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index d9e32fd1d..d13ae65dc 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -390,7 +390,7 @@ mod tests { target_gap(family, family.domain_by_name(name).expect("domain")) }; let text = eql_domains::TEXT; - for producible in ["", "eq"] { + for producible in ["", "eq", "match"] { assert!(gap(&text, producible).is_none(), "text {producible:?}"); } for ordered in ["ord", "ord_ore", "ord_ope", "search", "search_ore"] { @@ -399,7 +399,7 @@ mod tests { "{ordered}" ); } - assert!(gap(&text, "match").unwrap().contains("match terms")); + assert!(gap(&text, "match").is_none(), "TextMatch is producible"); let family = |name: &str| { eql_domains::CATALOG .iter() diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 97c7b1189..d4089d024 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -565,7 +565,16 @@ mod the_table { .collect(); assert_eq!( producible, - ["Integer", "Bigint", "Text", "TextEq", "Boolean", "Real", "Double"], + [ + "Integer", + "Bigint", + "Text", + "TextEq", + "TextMatch", + "Boolean", + "Real", + "Double" + ], ); for row in targets::targets().iter().filter(|t| !t.producible) { let reason = row.reason.expect("an unproducible type has a reason"); @@ -574,7 +583,6 @@ mod the_table { ("json", "") => "tree of leaves", ("smallint" | "date" | "timestamp" | "numeric", _) => "has no kind", (_, "Ord" | "OrdOre" | "OrdOpe" | "Search" | "SearchOre") => "ordering terms", - ("text", "Match") => "match terms", ("real" | "double", "Eq") => "float", ("integer" | "bigint", "Eq") => "integer-orderable/v1", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 126df4ac5..9ad6e4613 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -1,9 +1,9 @@ //! The EQL v4 types Stack Encrypt produces besides `TextEq`: the -//! storage-only integer, bigint, boolean, float and text types. `Json` is not -//! producible yet: a JSON value seals as a tree of leaves and EQL's `c` holds -//! one. `IntegerEq` and `BigintEq` are held back: their equality term moves to -//! vitaminc 0.6's shared integer PRF domain, and terms stored before that -//! would stop matching. +//! storage-only integer, bigint, boolean, float and text types, and +//! `TextMatch`. `Json` is not producible yet: a JSON value seals as a tree of +//! leaves and EQL's `c` holds one. `IntegerEq` and `BigintEq` are held back: +//! their equality term moves to vitaminc 0.6's shared integer PRF domain, and +//! terms stored before that would stop matching. //! //! Each type must round-trip through both entry points (the typed //! `encrypt_as` and the by-name dispatch a Go plan reaches), seal exactly the @@ -365,6 +365,90 @@ async fn storage_only_text_round_trips_through_both_entry_points() { ); } +/// A match query's filter positions, which EQL's `@@` tests for containment. +fn positions(filter: &eql_bindings::v3::terms::BloomFilter) -> std::collections::BTreeSet { + filter.0.iter().copied().collect() +} + +#[tokio::test] +async fn text_match_filters_contain_their_substrings() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let stored: text::TextMatch = keyset + .encrypt_as(&"Alice@Example.com".to_owned(), column()) + .await + .unwrap(); + assert_v4_envelope(&stored, &["v", "i", "c", "bf"]); + assert!( + stored.bf.0.iter().all(|&p| (0..2048).contains(&p)), + "a 2048-bit filter" + ); + let opened: String = cipher + .decrypt_as(stored.clone(), column().into()) + .await + .unwrap(); + assert_eq!(opened, "Alice@Example.com"); + + // `true` is a property of the filter: every bit a contained needle sets + // is set in the stored value's. `false` is not: a Bloom filter can cover + // an absent needle's bits by chance. "bob" is clear of it under the + // test key source and the pinned configuration, so a change to either + // can flip that case with no bug behind it. + let all = positions(&stored.bf); + for (needle, contained) in [ + ("alice", true), + ("EXAMPLE", true), + ("ice@ex", true), + ("bob", false), + ] { + let query: text::TextMatchQuery = keyset + .encrypt_as(&needle.to_owned(), column()) + .await + .unwrap(); + assert_eq!( + positions(&query.bf).is_subset(&all), + contained, + "{needle:?} in Alice@Example.com" + ); + } + + // The dispatch derives the same filter. + let named: text::TextMatch = by_name( + &keyset, + "TextMatch", + FfiValue::String("Alice@Example.com".into()), + ) + .await; + assert_eq!(named.bf, stored.bf); + + // Text shorter than a 3-gram has no tokens. Stack Encrypt 0.2 refuses it + // for a stored value as well as a query, so a short value cannot be + // written to a TextMatch column yet. + let short: Result = keyset.encrypt_as(&"ab".to_owned(), column()).await; + assert!(short.is_err()); +} + +/// The filter bits for one query under the test key source. The +/// configuration is part of the stored format (`EqlMatch`): if these move, +/// every stored filter stops agreeing with new queries, so a change here must +/// arrive as a new domain or payload version. +#[tokio::test] +async fn text_match_filter_positions_are_pinned() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let query: text::TextMatchQuery = keyset + .encrypt_as(&"alice".to_owned(), column()) + .await + .unwrap(); + // 3 tokens ("ali", "lic", "ice") at 6 positions each, de-duplicated. + assert_eq!(query.bf.0, PINNED_ALICE); +} + +const PINNED_ALICE: [i16; 18] = [ + 52, 99, 224, 311, 899, 1190, 1228, 1232, 1290, 1378, 1486, 1505, 1529, 1613, 1763, 1848, 1880, + 2027, +]; + #[tokio::test] async fn a_value_of_another_kind_is_refused_not_converted() { let (cipher, _) = common::cipher().await; @@ -419,7 +503,8 @@ fn postgres_stores_and_queries_every_producible_v4_type() { active public.eql_v4_boolean, ratio public.eql_v4_real, score public.eql_v4_double, - note public.eql_v4_text + note public.eql_v4_text, + name public.eql_v4_text_match );", ) .unwrap(); @@ -452,6 +537,10 @@ fn postgres_stores_and_queries_every_producible_v4_type() { .encrypt_as::<_, text::Text>(&"note".to_owned(), column()) .await .unwrap(), + keyset + .encrypt_as::<_, text::TextMatch>(&format!("user{id}@example.com"), column()) + .await + .unwrap(), ) }); db.execute( @@ -461,7 +550,8 @@ fn postgres_stores_and_queries_every_producible_v4_type() { $4::text::jsonb::public.eql_v4_boolean, $5::text::jsonb::public.eql_v4_real, $6::text::jsonb::public.eql_v4_double, - $7::text::jsonb::public.eql_v4_text)", + $7::text::jsonb::public.eql_v4_text, + $8::text::jsonb::public.eql_v4_text_match)", &[ &id, &json(&row.0), @@ -470,11 +560,51 @@ fn postgres_stores_and_queries_every_producible_v4_type() { &json(&row.3), &json(&row.4), &json(&row.5), + &json(&row.6), ], ) .unwrap(); } + let mut matching = |needle: &str| -> Vec { + let probe: text::TextMatchQuery = runtime + .block_on(async { keyset.encrypt_as(&needle.to_owned(), column()).await }) + .unwrap(); + db.query( + "SELECT id FROM v4_types WHERE name @@ $1::text::jsonb::eql_v4.query_text_match ORDER BY id", + &[&json(&probe)], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect() + }; + assert_eq!(matching("example.com"), [1, 2, 3], "TextMatch: every row"); + assert_eq!(matching("user2"), [2], "TextMatch: one row"); + assert_eq!(matching("nobody"), Vec::::new(), "TextMatch: none"); + + // Known gap in the SQL (#1171), pinned so its fix shows here: with the + // query on the left, `@@` asks whether the query's filter contains the + // stored value's, so that operand order finds nothing. Only + // `stored @@ query` works until then; the README says so. + let probe: text::TextMatchQuery = runtime + .block_on(async { keyset.encrypt_as(&"user2".to_owned(), column()).await }) + .unwrap(); + let reversed: Vec = db + .query( + "SELECT id FROM v4_types WHERE $1::text::jsonb::eql_v4.query_text_match @@ name", + &[&json(&probe)], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + assert_eq!( + reversed, + Vec::::new(), + "query @@ stored still finds nothing (#1171); when the SQL is fixed this becomes [2]" + ); + // Every stored value reads back from its domain and opens. let row = db .query_one(