From ed73a1716c8d724764b29d509b0d8c5e62a05816 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 12 Oct 2026 21:16:32 +1100 Subject: [PATCH 1/4] test(golang): prove the EQL v4 types from Go, against Rust and PostgreSQL stashgen needed no change: it maps int32, int64, bool, float32, float64 and string to their kinds and accepts any type the engine's se_targets table marks producible, so every type the previous commits made producible already generates typed Encrypt and Query methods. A Profile fixture stores one field in each new type. The hermetic test seals it through generated code over the deterministic EQL guest, checks every value is a v4 envelope with the type's keys, and opens it back. A cross-language fixture, v4_queries.json, holds the IntegerEq, BigintEq and TextMatch query operands the Rust dispatch derives under the shared test index key; the Go fields derive them byte for byte. Checked by hand against PostgreSQL: rows sealed by this Go code, loaded into an eql_v4 install, answer =, = and @@ with exactly the expected ids. The Go module has no PostgreSQL driver, and adding one for a test would change every consumer's dependency graph, so that check is not committed. The encrypt and stashgen READMEs name eql_v4 and list the producible types with the Go field type each takes. Refs #1144 --- languages/golang/cmd/stashgen/README.md | 2 +- languages/golang/encrypt/README.md | 20 +- languages/golang/encrypt/eql_v4_test.go | 155 ++++++++++ .../internal/testusers/profile_stash.go | 265 ++++++++++++++++++ .../encrypt/internal/testusers/profiles.go | 20 ++ .../tests/encryption/fixtures/v4_queries.json | 16 ++ .../eql/tests/encryption/tests/v4_types.rs | 50 ++++ 7 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 languages/golang/encrypt/eql_v4_test.go create mode 100644 languages/golang/encrypt/internal/testusers/profile_stash.go create mode 100644 languages/golang/encrypt/internal/testusers/profiles.go create mode 100644 packages/eql/tests/encryption/fixtures/v4_queries.json diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 054659d1f..a281d0a4c 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -44,7 +44,7 @@ Your program calls those functions, and it never builds or names a plan. ``` 6. Store the encrypted type. - Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v4_text_eq` column holds. Stack Encrypt writes EQL v4; see the `encrypt` package README. + Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v4_text_eq` column holds. Stack Encrypt writes EQL v4; see the `encrypt` package README for the EQL types it produces and the Go field type each takes. 7. Run the generator again after each change to the struct or to a tag. A change to the fields of the struct stops the build until you do. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index a5cb8237c..bbf14e14d 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -282,7 +282,25 @@ of EQL's SQL; the `eql_v3` columns the TypeScript SDK writes hold cipherstash-client's terms, which these never match. Install EQL v4 into PostgreSQL first, with the `eql-v4-install.sql` attached to each Go release (`psql -f eql-v4-install.sql`); `stash eql install` installs EQL v3 only, and -an `eql_v4_*` column needs the v4 domains. The guest builds the value: the generated code +an `eql_v4_*` column needs the v4 domains. + +The EQL types the engine produces, and the Go field types each takes: + +| EQL type | Go field | Queries | +|---|---|---| +| `TextEq` | `string` | `=`, `<>` | +| `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | +| `Text` | `string` | none: stored and read only | +| `IntegerEq` | `int32` | `=`, `<>` | +| `Integer` | `int32` | none | +| `BigintEq` | `int64` | `=`, `<>` | +| `Boolean` | `bool` | none | +| `Real` | `float32` | none | +| `Double` | `float64` | none | + +The rest of the catalog (ordering, dates, decimals, JSON) is listed in +`eql.Types` with the reason it is not produced yet, and `stashgen` refuses it +with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the EQL types and registers it on import, so the program runs that build and stores what it returns. `Fields.Email.Query` returns the `eql.TextEqQuery` diff --git a/languages/golang/encrypt/eql_v4_test.go b/languages/golang/encrypt/eql_v4_test.go new file mode 100644 index 000000000..4faa5c520 --- /dev/null +++ b/languages/golang/encrypt/eql_v4_test.go @@ -0,0 +1,155 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// The EQL v4 types the engine produces besides TextEq, sealed through +// generated code over the deterministic guest build with the EQL types: +// each Go field type reaches its v4 type's own plan, and the values open +// back to what went in. + +var profiles = []testusers.Profile{ + {ID: 1, Age: 42, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice"}, + {ID: 2, Age: -7, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith"}, +} + +// keysOf is the sorted top-level keys of an EQL value. +func keysOf(t *testing.T, value []byte) string { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal(value, &raw); err != nil { + t.Fatalf("the EQL value is not JSON: %v: %s", err, value) + } + if string(raw["v"]) != "4" { + t.Errorf("v = %s, want 4: %s", raw["v"], value) + } + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles) + if err != nil { + t.Fatal(err) + } + for i, e := range encrypted { + for name, tc := range map[string]struct { + value []byte + keys string + }{ + "age": {e.Age, "c,hm,i,v"}, + "visits": {e.Visits, "c,i,v"}, + "balance": {e.Balance, "c,hm,i,v"}, + "active": {e.Active, "c,i,v"}, + "ratio": {e.Ratio, "c,i,v"}, + "score": {e.Score, "c,i,v"}, + "note": {e.Note, "c,i,v"}, + "name": {e.Name, "bf,c,i,v"}, + } { + if got := keysOf(t, tc.value); got != tc.keys { + t.Errorf("row %d %s: keys %s, want %s", i, name, got, tc.keys) + } + } + } + + decrypted, err := testusers.DecryptProfile(ctx, cipher, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range profiles { + if decrypted[i] != profiles[i] { + t.Errorf("row %d: %+v, want %+v", i, decrypted[i], profiles[i]) + } + } +} + +func TestV4QueriesMatchTheRustFixture(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", "v4_queries.json")) + if err != nil { + t.Fatal(err) + } + var fixture struct { + Table string + Age struct { + Plaintext int32 + Query string + } + Balance struct { + Plaintext int64 + Query string + } + Name struct { + Plaintext string + Query string + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + if fixture.Table != "profiles" { + t.Fatalf("the fixture's table changed: %q", fixture.Table) + } + + // Byte for byte what eql-bindings' own dispatch derived under the same + // index key: each Go field runs its v4 type's plan and no other. + age, err := testusers.ProfileFields.Age.Query(ctx, cipher, fixture.Age.Plaintext) + if err != nil { + t.Fatal(err) + } + if string(age) != fixture.Age.Query { + t.Errorf("Age.Query = %s\nwant %s", age, fixture.Age.Query) + } + balance, err := testusers.ProfileFields.Balance.Query(ctx, cipher, fixture.Balance.Plaintext) + if err != nil { + t.Fatal(err) + } + if string(balance) != fixture.Balance.Query { + t.Errorf("Balance.Query = %s\nwant %s", balance, fixture.Balance.Query) + } + name, err := testusers.ProfileFields.Name.Query(ctx, cipher, fixture.Name.Plaintext) + if err != nil { + t.Fatal(err) + } + if string(name) != fixture.Name.Query { + t.Errorf("Name.Query = %s\nwant %s", name, fixture.Name.Query) + } + + // And a stored value carries the term its query probes for. + stored, err := testusers.ProfileFields.Age.Encrypt(ctx, cipher, fixture.Age.Plaintext) + if err != nil { + t.Fatal(err) + } + var s, q struct { + HM string `json:"hm"` + } + if err := json.Unmarshal(stored, &s); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(age, &q); err != nil { + t.Fatal(err) + } + if s.HM != q.HM { + t.Errorf("stored hm %s, query hm %s", s.HM, q.HM) + } +} diff --git a/languages/golang/encrypt/internal/testusers/profile_stash.go b/languages/golang/encrypt/internal/testusers/profile_stash.go new file mode 100644 index 000000000..77da29ecc --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/profile_stash.go @@ -0,0 +1,265 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Profile prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedProfile struct { + ID int64 + Age eql.IntegerEq + Visits eql.Integer + Balance eql.BigintEq + Active eql.Boolean + Ratio eql.Real + Score eql.Double + Note eql.Text + Name eql.TextMatch +} + +func (e EncryptedProfile) String() string { + return gensupport.Redacted("EncryptedProfile", map[string]any{"ID": e.ID}, "Age", "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") +} + +func (e EncryptedProfile) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Age", "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") +} + +// Stops compiling when Profile gains, loses, reorders or retypes a field. +var _ = profileShape(Profile{}) + +type profileShape struct { + _ struct{} + ID int64 + Age int32 + Visits int32 + Balance int64 + Active bool + Ratio float32 + Score float64 + Note string + Name string +} + +var profileDeclaration = gensupport.Declare("profiles"). + Passthrough("id"). + EncryptInto("age", gensupport.Int32, "IntegerEq"). + EncryptInto("visits", gensupport.Int32, "Integer"). + EncryptInto("balance", gensupport.Int64, "BigintEq"). + EncryptInto("active", gensupport.Bool, "Boolean"). + EncryptInto("ratio", gensupport.Float32, "Real"). + EncryptInto("score", gensupport.Float64, "Double"). + EncryptInto("note", gensupport.String, "Text"). + EncryptInto("name", gensupport.String, "TextMatch") + +var profileCodec = gensupport.New(gensupport.Generated[Profile, EncryptedProfile]{ + TypeName: "Profile", + Declaration: profileDeclaration, + PrintsPlaintext: true, + Source: func(v Profile) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "age": v.Age, + "visits": v.Visits, + "balance": v.Balance, + "active": v.Active, + "ratio": v.Ratio, + "score": v.Score, + "note": v.Note, + "name": v.Name, + } + }, + Seal: func(rec gensupport.Record) (EncryptedProfile, error) { + var e EncryptedProfile + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedProfile{}, err + } + e.Age = eql.IntegerEq(rec["age"].EQL) + e.Visits = eql.Integer(rec["visits"].EQL) + e.Balance = eql.BigintEq(rec["balance"].EQL) + e.Active = eql.Boolean(rec["active"].EQL) + e.Ratio = eql.Real(rec["ratio"].EQL) + e.Score = eql.Double(rec["score"].EQL) + e.Note = eql.Text(rec["note"].EQL) + e.Name = eql.TextMatch(rec["name"].EQL) + return e, nil + }, + Open: func(e EncryptedProfile) gensupport.Record { + return gensupport.Record{ + "id": {Value: e.ID}, + "age": {EQL: e.Age}, + "visits": {EQL: e.Visits}, + "balance": {EQL: e.Balance}, + "active": {EQL: e.Active}, + "ratio": {EQL: e.Ratio}, + "score": {EQL: e.Score}, + "note": {EQL: e.Note}, + "name": {EQL: e.Name}, + } + }, + Value: func(e EncryptedProfile, vals gensupport.Values) (Profile, error) { + var v Profile + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Profile{}, err + } + if v.Age, err = gensupport.Get[int32](vals, "age"); err != nil { + return Profile{}, err + } + if v.Visits, err = gensupport.Get[int32](vals, "visits"); err != nil { + return Profile{}, err + } + if v.Balance, err = gensupport.Get[int64](vals, "balance"); err != nil { + return Profile{}, err + } + if v.Active, err = gensupport.Get[bool](vals, "active"); err != nil { + return Profile{}, err + } + if v.Ratio, err = gensupport.Get[float32](vals, "ratio"); err != nil { + return Profile{}, err + } + if v.Score, err = gensupport.Get[float64](vals, "score"); err != nil { + return Profile{}, err + } + if v.Note, err = gensupport.Get[string](vals, "note"); err != nil { + return Profile{}, err + } + if v.Name, err = gensupport.Get[string](vals, "name"); err != nil { + return Profile{}, err + } + return v, nil + }, +}) + +// EncryptProfile seals each Profile, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. The result has one +// element for each input, in the same order. +func EncryptProfile(ctx context.Context, cipher *encrypt.Cipher, values []Profile) ([]EncryptedProfile, error) { + return profileCodec.Encrypt(ctx, cipher, values) +} + +// DecryptProfile opens each EncryptedProfile, with one ZeroKMS request for each +// 500 sealed values, plus one the first time a keyset is used. +func DecryptProfile(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedProfile) ([]Profile, error) { + return profileCodec.Decrypt(ctx, d, encrypted) +} + +var ProfileFields = struct { + Age ProfileAgeField + Visits ProfileVisitsField + Balance ProfileBalanceField + Active ProfileActiveField + Ratio ProfileRatioField + Score ProfileScoreField + Note ProfileNoteField + Name ProfileNameField +}{ + Age: ProfileAgeField{gensupport.NewField[int32](profileDeclaration, "age")}, + Visits: ProfileVisitsField{gensupport.NewField[int32](profileDeclaration, "visits")}, + Balance: ProfileBalanceField{gensupport.NewField[int64](profileDeclaration, "balance")}, + Active: ProfileActiveField{gensupport.NewField[bool](profileDeclaration, "active")}, + Ratio: ProfileRatioField{gensupport.NewField[float32](profileDeclaration, "ratio")}, + Score: ProfileScoreField{gensupport.NewField[float64](profileDeclaration, "score")}, + Note: ProfileNoteField{gensupport.NewField[string](profileDeclaration, "note")}, + Name: ProfileNameField{gensupport.NewField[string](profileDeclaration, "name")}, +} + +type ProfileAgeField struct { + field gensupport.Field[int32] +} + +func (f ProfileAgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) (eql.IntegerEq, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.IntegerEq(out.EQL), err +} + +func (f ProfileAgeField) Query(ctx context.Context, c *encrypt.Cipher, v int32) (eql.IntegerEqQuery, error) { + out, err := f.field.Query(ctx, c, v) + return eql.IntegerEqQuery(out.EQL), err +} + +type ProfileVisitsField struct { + field gensupport.Field[int32] +} + +func (f ProfileVisitsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) (eql.Integer, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Integer(out.EQL), err +} + +type ProfileBalanceField struct { + field gensupport.Field[int64] +} + +func (f ProfileBalanceField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int64) (eql.BigintEq, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.BigintEq(out.EQL), err +} + +func (f ProfileBalanceField) Query(ctx context.Context, c *encrypt.Cipher, v int64) (eql.BigintEqQuery, error) { + out, err := f.field.Query(ctx, c, v) + return eql.BigintEqQuery(out.EQL), err +} + +type ProfileActiveField struct { + field gensupport.Field[bool] +} + +func (f ProfileActiveField) Encrypt(ctx context.Context, c *encrypt.Cipher, v bool) (eql.Boolean, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Boolean(out.EQL), err +} + +type ProfileRatioField struct { + field gensupport.Field[float32] +} + +func (f ProfileRatioField) Encrypt(ctx context.Context, c *encrypt.Cipher, v float32) (eql.Real, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Real(out.EQL), err +} + +type ProfileScoreField struct { + field gensupport.Field[float64] +} + +func (f ProfileScoreField) Encrypt(ctx context.Context, c *encrypt.Cipher, v float64) (eql.Double, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Double(out.EQL), err +} + +type ProfileNoteField struct { + field gensupport.Field[string] +} + +func (f ProfileNoteField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.Text, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.Text(out.EQL), err +} + +type ProfileNameField struct { + field gensupport.Field[string] +} + +func (f ProfileNameField) Encrypt(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatch, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.TextMatch(out.EQL), err +} + +func (f ProfileNameField) Query(ctx context.Context, c *encrypt.Cipher, v string) (eql.TextMatchQuery, error) { + out, err := f.field.Query(ctx, c, v) + return eql.TextMatchQuery(out.EQL), err +} diff --git a/languages/golang/encrypt/internal/testusers/profiles.go b/languages/golang/encrypt/internal/testusers/profiles.go new file mode 100644 index 000000000..e86aa6c19 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/profiles.go @@ -0,0 +1,20 @@ +package testusers + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Profile -name Profile + +// Profile stores one field in each EQL v4 type the engine produces besides +// TextEq, the shape the cross-language fixture +// packages/eql/tests/encryption/fixtures/v4_queries.json is derived under +// (table profiles). +type Profile struct { + _ struct{} `stash:"context=profiles"` + ID int64 `stash:"id,passthrough"` + Age int32 `stash:"age,encrypt_into=IntegerEq"` + Visits int32 `stash:"visits,encrypt_into=Integer"` + Balance int64 `stash:"balance,encrypt_into=BigintEq"` + Active bool `stash:"active,encrypt_into=Boolean"` + Ratio float32 `stash:"ratio,encrypt_into=Real"` + Score float64 `stash:"score,encrypt_into=Double"` + Note string `stash:"note,encrypt_into=Text"` + Name string `stash:"name,encrypt_into=TextMatch"` +} diff --git a/packages/eql/tests/encryption/fixtures/v4_queries.json b/packages/eql/tests/encryption/fixtures/v4_queries.json new file mode 100644 index 000000000..b7ff038ce --- /dev/null +++ b/packages/eql/tests/encryption/fixtures/v4_queries.json @@ -0,0 +1,16 @@ +{ + "_comment": "The v4 query operands eql-bindings derives for each plaintext under FakeDataKeySource's index key (keyset nil), read by the Go SDK's hermetic test; regenerate with EQL_UPDATE_FIXTURES=1, do not edit.", + "age": { + "plaintext": 42, + "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"age\"},\"hm\":\"0fe024ff2c58b7840dafb754980c4c587ccd4c47edf35ae4c0f157183c5c6f7f\"}" + }, + "balance": { + "plaintext": 1234567890123, + "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"balance\"},\"hm\":\"b3d4bbae7880b2a05d71e3b810357e8a4eefc50924db4387fec94435811b908a\"}" + }, + "name": { + "plaintext": "alice", + "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"name\"},\"bf\":[37,38,59,139,167,199,214,242,245]}" + }, + "table": "profiles" +} diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 9ad6e4613..2c8f00a4d 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -688,3 +688,53 @@ mod erased { } } } + +/// `fixtures/v4_queries.json`: the query operands the Rust dispatch derives +/// for one value of each v4 type that answers a query, under the key source +/// every test in the repository derives terms under (`FakeDataKeySource`'s +/// index key, keyset nil). The Go SDK's hermetic test derives the same +/// queries through generated code over the deterministic guest build, whose +/// index key is the same, and asserts the bytes are equal: a Go field typed +/// `encrypt_into=IntegerEq` runs this plan and no other. Regenerate with +/// `EQL_UPDATE_FIXTURES=1`; the bytes must not change otherwise. +#[tokio::test] +async fn the_v4_query_fixture_is_what_the_dispatch_derives() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let query = |name: &'static str, column: &'static str, value: FfiValue| { + let keyset = &keyset; + async move { + let label = Label::new(["profiles", column]).unwrap(); + let bytes = targets::query(name, keyset, &label, value) + .unwrap() + .await + .unwrap(); + String::from_utf8(bytes).unwrap() + } + }; + let fixture = serde_json::json!({ + "_comment": "The v4 query operands eql-bindings derives for each plaintext under FakeDataKeySource's index key (keyset nil), read by the Go SDK's hermetic test; regenerate with EQL_UPDATE_FIXTURES=1, do not edit.", + "table": "profiles", + "age": { "plaintext": 42, "query": query("IntegerEq", "age", FfiValue::Int32(42)).await }, + "balance": { + "plaintext": 1_234_567_890_123i64, + "query": query("BigintEq", "balance", FfiValue::Int64(1_234_567_890_123)).await, + }, + "name": { + "plaintext": "alice", + "query": query("TextMatch", "name", FfiValue::String("alice".into())).await, + }, + }); + let path = + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/v4_queries.json"); + let rendered = serde_json::to_string_pretty(&fixture).unwrap() + "\n"; + if std::env::var_os("EQL_UPDATE_FIXTURES").is_some() { + std::fs::write(&path, &rendered).unwrap(); + } + let committed = std::fs::read_to_string(&path) + .expect("fixtures/v4_queries.json is committed; EQL_UPDATE_FIXTURES=1 writes it"); + assert_eq!( + committed, rendered, + "the committed fixture is what the dispatch derives today" + ); +} From 5d420871a5b20caa64e987de2ed52d4719489761 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 12 Oct 2026 23:54:47 +1100 Subject: [PATCH 2/4] test(golang): send Go-sealed v4 rows through PostgreSQL, and drop the withdrawn integer equality types Nothing in the repository put a value the generated Go code sealed into an eql_v4 domain or operator; the PostgreSQL check was manual. The Go test now writes v4_rows.json, the Profile rows the generated code seals over the deterministic EQL guest. On every later run it checks that file against today's output, envelopes and match terms byte for byte and ciphertexts by opening them (their AEAD nonce is random). The EQL suite opens those rows with stack-kms's DeterministicSource over the same seed, and its PostgreSQL test, which test:encryption:postgres runs in CI, inserts them into eql_v4_* columns and queries name @@ with the Go query operand from v4_queries.json. With IntegerEq and BigintEq held back, Profile drops Age and stores Balance as Bigint, and v4_queries.json carries TextMatch only. The one stored-versus-query check had used Age, so it is replaced by a TextMatch check that a contained needle's filter bits are a subset of the stored filter and an absent one's are not. Every stored field's i is checked against its table and column. The Go workflow now runs when the fixtures or EQL's lockfile change, and biome leaves the generated rows file alone. --- .github/workflows/tests-golang.yml | 8 + biome.json | 1 + languages/golang/encrypt/README.md | 8 +- languages/golang/encrypt/eql_v4_test.go | 259 ++++++++++++++---- .../internal/testusers/profile_stash.go | 44 +-- .../encrypt/internal/testusers/profiles.go | 9 +- .../tests/encryption/fixtures/v4_queries.json | 10 +- .../tests/encryption/fixtures/v4_rows.json | 211 ++++++++++++++ .../eql/tests/encryption/tests/v4_types.rs | 240 +++++++++++++++- 9 files changed, 681 insertions(+), 109 deletions(-) create mode 100644 packages/eql/tests/encryption/fixtures/v4_rows.json diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 9bfff334a..4a31490e7 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -36,6 +36,10 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # The Go tests read the cross-language fixtures the EQL suite writes, + # which are derived under EQL's lockfile. + - packages/eql/tests/encryption/fixtures/** + - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py - scripts/go-binding-test.sh @@ -61,6 +65,10 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # The Go tests read the cross-language fixtures the EQL suite writes, + # which are derived under EQL's lockfile. + - packages/eql/tests/encryption/fixtures/** + - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py - scripts/go-binding-test.sh diff --git a/biome.json b/biome.json index 099fd0918..7b0c4d09a 100644 --- a/biome.json +++ b/biome.json @@ -11,6 +11,7 @@ "!languages/typescript/packages/protect-ffi/target", "!languages/typescript/packages/protect-ffi/src/eql-v3-types", "!packages/stack-encrypt/tests/fixtures/record_lowering.json", + "!packages/eql/tests/encryption/fixtures/v4_rows.json", "!packages/eql/crates/eql-bindings/bindings", "!packages/eql/crates/eql-bindings/schema", "!packages/eql/packages/eql/src/generated", diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index bbf14e14d..49daffc6c 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -291,14 +291,16 @@ The EQL types the engine produces, and the Go field types each takes: | `TextEq` | `string` | `=`, `<>` | | `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) | | `Text` | `string` | none: stored and read only | -| `IntegerEq` | `int32` | `=`, `<>` | | `Integer` | `int32` | none | -| `BigintEq` | `int64` | `=`, `<>` | +| `Bigint` | `int64` | none | | `Boolean` | `bool` | none | | `Real` | `float32` | none | | `Double` | `float64` | none | -The rest of the catalog (ordering, dates, decimals, JSON) is listed in +A `TextMatch` value must be at least three characters long: shorter text has +no 3-grams to index and is refused. `IntegerEq` and `BigintEq` are held back +until the engine's integer equality term settles in vitaminc 0.6. The rest of +the catalog (ordering, dates, decimals, JSON) is listed in `eql.Types` with the reason it is not produced yet, and `stashgen` refuses it with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the diff --git a/languages/golang/encrypt/eql_v4_test.go b/languages/golang/encrypt/eql_v4_test.go index 4faa5c520..da6a65c0e 100644 --- a/languages/golang/encrypt/eql_v4_test.go +++ b/languages/golang/encrypt/eql_v4_test.go @@ -1,6 +1,7 @@ package encrypt_test import ( + "bytes" "context" "encoding/json" "os" @@ -15,15 +16,23 @@ import ( // The EQL v4 types the engine produces besides TextEq, sealed through // generated code over the deterministic guest build with the EQL types: // each Go field type reaches its v4 type's own plan, and the values open -// back to what went in. +// back to what went in. PostgreSQL sees these values through the Rust +// suite: TestV4RowsMatchTheCommittedFixture commits them to +// v4_rows.json, and packages/eql/tests/encryption inserts that file into +// eql_v4 columns and queries it. var profiles = []testusers.Profile{ - {ID: 1, Age: 42, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice"}, - {ID: 2, Age: -7, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith"}, + {ID: 1, Visits: 3, Balance: 1234567890123, Active: true, Ratio: 0.5, Score: 2.25, Note: "a private note", Name: "alice"}, + {ID: 2, Visits: 0, Balance: -1, Active: false, Ratio: -1.5, Score: 0, Note: "", Name: "bob smith"}, } -// keysOf is the sorted top-level keys of an EQL value. -func keysOf(t *testing.T, value []byte) string { +func fixturePath(name string) string { + return filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", name) +} + +// keysOf is the sorted top-level keys of an EQL value. It also checks the +// envelope: version 4, and the table and column the value was stored under. +func keysOf(t *testing.T, column string, value []byte) string { t.Helper() var raw map[string]json.RawMessage if err := json.Unmarshal(value, &raw); err != nil { @@ -32,6 +41,10 @@ func keysOf(t *testing.T, value []byte) string { if string(raw["v"]) != "4" { t.Errorf("v = %s, want 4: %s", raw["v"], value) } + var i struct{ T, C string } + if err := json.Unmarshal(raw["i"], &i); err != nil || i.T != "profiles" || i.C != column { + t.Errorf("i = %s, want {\"t\":\"profiles\",\"c\":%q}", raw["i"], column) + } keys := make([]string, 0, len(raw)) for k := range raw { keys = append(keys, k) @@ -40,6 +53,19 @@ func keysOf(t *testing.T, value []byte) string { return strings.Join(keys, ",") } +// stored is each sealed field of an encrypted profile by column. +func stored(e testusers.EncryptedProfile) map[string][]byte { + return map[string][]byte{ + "visits": e.Visits, + "balance": e.Balance, + "active": e.Active, + "ratio": e.Ratio, + "score": e.Score, + "note": e.Note, + "name": e.Name, + } +} + func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { c := deterministicEQLClient(t) ctx := context.Background() @@ -49,22 +75,19 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { if err != nil { t.Fatal(err) } + want := map[string]string{ + "visits": "c,i,v", + "balance": "c,i,v", + "active": "c,i,v", + "ratio": "c,i,v", + "score": "c,i,v", + "note": "c,i,v", + "name": "bf,c,i,v", + } for i, e := range encrypted { - for name, tc := range map[string]struct { - value []byte - keys string - }{ - "age": {e.Age, "c,hm,i,v"}, - "visits": {e.Visits, "c,i,v"}, - "balance": {e.Balance, "c,hm,i,v"}, - "active": {e.Active, "c,i,v"}, - "ratio": {e.Ratio, "c,i,v"}, - "score": {e.Score, "c,i,v"}, - "note": {e.Note, "c,i,v"}, - "name": {e.Name, "bf,c,i,v"}, - } { - if got := keysOf(t, tc.value); got != tc.keys { - t.Errorf("row %d %s: keys %s, want %s", i, name, got, tc.keys) + for column, value := range stored(e) { + if got := keysOf(t, column, value); got != want[column] { + t.Errorf("row %d %s: keys %s, want %s", i, column, got, want[column]) } } } @@ -80,26 +103,43 @@ func TestV4FieldsSealToTheirEQLTypesAndOpenBack(t *testing.T) { } } +// bloom is the set of filter positions in a TextMatch value or query. +func bloom(t *testing.T, value []byte) map[int]bool { + t.Helper() + var v struct { + BF []int `json:"bf"` + } + if err := json.Unmarshal(value, &v); err != nil { + t.Fatal(err) + } + set := make(map[int]bool, len(v.BF)) + for _, p := range v.BF { + set[p] = true + } + return set +} + +func subset(a, b map[int]bool) bool { + for p := range a { + if !b[p] { + return false + } + } + return true +} + func TestV4QueriesMatchTheRustFixture(t *testing.T) { c := deterministicEQLClient(t) ctx := context.Background() cipher := c.DefaultKeyset() - raw, err := os.ReadFile(filepath.Join("..", "..", "..", "packages", "eql", "tests", "encryption", "fixtures", "v4_queries.json")) + raw, err := os.ReadFile(fixturePath("v4_queries.json")) if err != nil { t.Fatal(err) } var fixture struct { Table string - Age struct { - Plaintext int32 - Query string - } - Balance struct { - Plaintext int64 - Query string - } - Name struct { + Name struct { Plaintext string Query string } @@ -112,44 +152,165 @@ func TestV4QueriesMatchTheRustFixture(t *testing.T) { } // Byte for byte what eql-bindings' own dispatch derived under the same - // index key: each Go field runs its v4 type's plan and no other. - age, err := testusers.ProfileFields.Age.Query(ctx, cipher, fixture.Age.Plaintext) + // index key: the Go field runs its v4 type's plan and no other. + name, err := testusers.ProfileFields.Name.Query(ctx, cipher, fixture.Name.Plaintext) if err != nil { t.Fatal(err) } - if string(age) != fixture.Age.Query { - t.Errorf("Age.Query = %s\nwant %s", age, fixture.Age.Query) + if string(name) != fixture.Name.Query { + t.Errorf("Name.Query = %s\nwant %s", name, fixture.Name.Query) } - balance, err := testusers.ProfileFields.Balance.Query(ctx, cipher, fixture.Balance.Plaintext) + + // And a stored value holds every filter bit of a query for text it + // contains, which is what `@@` tests, and not those of text it lacks. + encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles[:1]) if err != nil { t.Fatal(err) } - if string(balance) != fixture.Balance.Query { - t.Errorf("Balance.Query = %s\nwant %s", balance, fixture.Balance.Query) + all := bloom(t, encrypted[0].Name) + for needle, contained := range map[string]bool{"alice": true, "LIC": true, "zzz": false, "bob": false} { + q, err := testusers.ProfileFields.Name.Query(ctx, cipher, needle) + if err != nil { + t.Fatal(err) + } + if got := subset(bloom(t, q), all); got != contained { + t.Errorf("%q in %q: %v, want %v", needle, profiles[0].Name, got, contained) + } } - name, err := testusers.ProfileFields.Name.Query(ctx, cipher, fixture.Name.Plaintext) +} + +// rowsFixture is v4_rows.json: the profiles above as the generated code seals +// them under the deterministic guest and testSeed. The Rust suite opens them +// with stack-kms's DeterministicSource over the same seed, and inserts and +// queries them in PostgreSQL. +type rowsFixture struct { + Comment string `json:"_comment"` + Seed string `json:"seed"` + Table string `json:"table"` + Rows []fixtureRow `json:"rows"` +} + +type fixtureRow struct { + ID int64 `json:"id"` + Plaintext fixturePlain `json:"plaintext"` + Stored map[string]json.RawMessage `json:"stored"` +} + +type fixturePlain struct { + Visits int32 `json:"visits"` + Balance int64 `json:"balance"` + Active bool `json:"active"` + Ratio float32 `json:"ratio"` + Score float64 `json:"score"` + Note string `json:"note"` + Name string `json:"name"` +} + +// TestV4RowsMatchTheCommittedFixture writes v4_rows.json when +// EQL_UPDATE_FIXTURES is set, and otherwise checks the committed file +// against what the generated code seals today. The deterministic guest +// derives every data key from the seed, so the envelopes and the terms +// (`bf`) are the same every run; the AEAD nonce inside each `c` is random, +// so `c` is not. The test therefore compares everything but `c`, and opens +// the committed `c` values with today's code. +func TestV4RowsMatchTheCommittedFixture(t *testing.T) { + c := deterministicEQLClient(t) + encrypted, err := testusers.EncryptProfile(context.Background(), c.DefaultKeyset(), profiles) if err != nil { t.Fatal(err) } - if string(name) != fixture.Name.Query { - t.Errorf("Name.Query = %s\nwant %s", name, fixture.Name.Query) + fixture := rowsFixture{ + Comment: "Profile rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4Rows, do not edit.", + Seed: string(testSeed[:]), + Table: "profiles", } - - // And a stored value carries the term its query probes for. - stored, err := testusers.ProfileFields.Age.Encrypt(ctx, cipher, fixture.Age.Plaintext) + for i, e := range encrypted { + p := profiles[i] + row := fixtureRow{ + ID: p.ID, + Plaintext: fixturePlain{p.Visits, p.Balance, p.Active, p.Ratio, p.Score, p.Note, p.Name}, + Stored: map[string]json.RawMessage{}, + } + for column, value := range stored(e) { + row.Stored[column] = json.RawMessage(value) + } + fixture.Rows = append(fixture.Rows, row) + } + rendered, err := json.MarshalIndent(fixture, "", " ") if err != nil { t.Fatal(err) } - var s, q struct { - HM string `json:"hm"` + rendered = append(rendered, '\n') + path := fixturePath("v4_rows.json") + if os.Getenv("EQL_UPDATE_FIXTURES") != "" { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + } + committedBytes, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) } - if err := json.Unmarshal(stored, &s); err != nil { + var committed rowsFixture + if err := json.Unmarshal(committedBytes, &committed); err != nil { t.Fatal(err) } - if err := json.Unmarshal(age, &q); err != nil { + if !bytes.Equal(withoutCiphertexts(t, committed), withoutCiphertexts(t, fixture)) { + t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) + } + + // The committed ciphertexts still open, to the committed plaintexts. + sealed := make([]testusers.EncryptedProfile, len(committed.Rows)) + for i, row := range committed.Rows { + sealed[i] = testusers.EncryptedProfile{ + ID: row.ID, + Visits: []byte(row.Stored["visits"]), + Balance: []byte(row.Stored["balance"]), + Active: []byte(row.Stored["active"]), + Ratio: []byte(row.Stored["ratio"]), + Score: []byte(row.Stored["score"]), + Note: []byte(row.Stored["note"]), + Name: []byte(row.Stored["name"]), + } + } + opened, err := testusers.DecryptProfile(context.Background(), c.DefaultKeyset(), sealed) + if err != nil { t.Fatal(err) } - if s.HM != q.HM { - t.Errorf("stored hm %s, query hm %s", s.HM, q.HM) + for i, row := range committed.Rows { + p := opened[i] + got := fixturePlain{p.Visits, p.Balance, p.Active, p.Ratio, p.Score, p.Note, p.Name} + if got != row.Plaintext { + t.Errorf("row %d opens to %+v, want %+v", row.ID, got, row.Plaintext) + } + } +} + +// withoutCiphertexts renders a fixture with every stored value's `c` +// removed: the part of the file that is the same on every run. +func withoutCiphertexts(t *testing.T, f rowsFixture) []byte { + t.Helper() + out := f + out.Rows = nil + for _, row := range f.Rows { + stripped := fixtureRow{ID: row.ID, Plaintext: row.Plaintext, Stored: map[string]json.RawMessage{}} + for column, value := range row.Stored { + var fields map[string]json.RawMessage + if err := json.Unmarshal(value, &fields); err != nil { + t.Fatal(err) + } + delete(fields, "c") + b, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + stripped.Stored[column] = b + } + out.Rows = append(out.Rows, stripped) + } + b, err := json.Marshal(out) + if err != nil { + t.Fatal(err) } + return b } diff --git a/languages/golang/encrypt/internal/testusers/profile_stash.go b/languages/golang/encrypt/internal/testusers/profile_stash.go index 77da29ecc..c3c782e60 100644 --- a/languages/golang/encrypt/internal/testusers/profile_stash.go +++ b/languages/golang/encrypt/internal/testusers/profile_stash.go @@ -19,9 +19,8 @@ const _ = gensupport.GeneratedVersion1 type EncryptedProfile struct { ID int64 - Age eql.IntegerEq Visits eql.Integer - Balance eql.BigintEq + Balance eql.Bigint Active eql.Boolean Ratio eql.Real Score eql.Double @@ -30,11 +29,11 @@ type EncryptedProfile struct { } func (e EncryptedProfile) String() string { - return gensupport.Redacted("EncryptedProfile", map[string]any{"ID": e.ID}, "Age", "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") + return gensupport.Redacted("EncryptedProfile", map[string]any{"ID": e.ID}, "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") } func (e EncryptedProfile) LogValue() slog.Value { - return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Age", "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Visits", "Balance", "Active", "Ratio", "Score", "Note", "Name") } // Stops compiling when Profile gains, loses, reorders or retypes a field. @@ -43,7 +42,6 @@ var _ = profileShape(Profile{}) type profileShape struct { _ struct{} ID int64 - Age int32 Visits int32 Balance int64 Active bool @@ -55,9 +53,8 @@ type profileShape struct { var profileDeclaration = gensupport.Declare("profiles"). Passthrough("id"). - EncryptInto("age", gensupport.Int32, "IntegerEq"). EncryptInto("visits", gensupport.Int32, "Integer"). - EncryptInto("balance", gensupport.Int64, "BigintEq"). + EncryptInto("balance", gensupport.Int64, "Bigint"). EncryptInto("active", gensupport.Bool, "Boolean"). EncryptInto("ratio", gensupport.Float32, "Real"). EncryptInto("score", gensupport.Float64, "Double"). @@ -71,7 +68,6 @@ var profileCodec = gensupport.New(gensupport.Generated[Profile, EncryptedProfile Source: func(v Profile) gensupport.Values { return gensupport.Values{ "id": v.ID, - "age": v.Age, "visits": v.Visits, "balance": v.Balance, "active": v.Active, @@ -87,9 +83,8 @@ var profileCodec = gensupport.New(gensupport.Generated[Profile, EncryptedProfile if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { return EncryptedProfile{}, err } - e.Age = eql.IntegerEq(rec["age"].EQL) e.Visits = eql.Integer(rec["visits"].EQL) - e.Balance = eql.BigintEq(rec["balance"].EQL) + e.Balance = eql.Bigint(rec["balance"].EQL) e.Active = eql.Boolean(rec["active"].EQL) e.Ratio = eql.Real(rec["ratio"].EQL) e.Score = eql.Double(rec["score"].EQL) @@ -100,7 +95,6 @@ var profileCodec = gensupport.New(gensupport.Generated[Profile, EncryptedProfile Open: func(e EncryptedProfile) gensupport.Record { return gensupport.Record{ "id": {Value: e.ID}, - "age": {EQL: e.Age}, "visits": {EQL: e.Visits}, "balance": {EQL: e.Balance}, "active": {EQL: e.Active}, @@ -116,9 +110,6 @@ var profileCodec = gensupport.New(gensupport.Generated[Profile, EncryptedProfile if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { return Profile{}, err } - if v.Age, err = gensupport.Get[int32](vals, "age"); err != nil { - return Profile{}, err - } if v.Visits, err = gensupport.Get[int32](vals, "visits"); err != nil { return Profile{}, err } @@ -158,7 +149,6 @@ func DecryptProfile(ctx context.Context, d encrypt.Decrypter, encrypted []Encryp } var ProfileFields = struct { - Age ProfileAgeField Visits ProfileVisitsField Balance ProfileBalanceField Active ProfileActiveField @@ -167,7 +157,6 @@ var ProfileFields = struct { Note ProfileNoteField Name ProfileNameField }{ - Age: ProfileAgeField{gensupport.NewField[int32](profileDeclaration, "age")}, Visits: ProfileVisitsField{gensupport.NewField[int32](profileDeclaration, "visits")}, Balance: ProfileBalanceField{gensupport.NewField[int64](profileDeclaration, "balance")}, Active: ProfileActiveField{gensupport.NewField[bool](profileDeclaration, "active")}, @@ -177,20 +166,6 @@ var ProfileFields = struct { Name: ProfileNameField{gensupport.NewField[string](profileDeclaration, "name")}, } -type ProfileAgeField struct { - field gensupport.Field[int32] -} - -func (f ProfileAgeField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int32) (eql.IntegerEq, error) { - out, err := f.field.Encrypt(ctx, c, v) - return eql.IntegerEq(out.EQL), err -} - -func (f ProfileAgeField) Query(ctx context.Context, c *encrypt.Cipher, v int32) (eql.IntegerEqQuery, error) { - out, err := f.field.Query(ctx, c, v) - return eql.IntegerEqQuery(out.EQL), err -} - type ProfileVisitsField struct { field gensupport.Field[int32] } @@ -204,14 +179,9 @@ type ProfileBalanceField struct { field gensupport.Field[int64] } -func (f ProfileBalanceField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int64) (eql.BigintEq, error) { +func (f ProfileBalanceField) Encrypt(ctx context.Context, c *encrypt.Cipher, v int64) (eql.Bigint, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.BigintEq(out.EQL), err -} - -func (f ProfileBalanceField) Query(ctx context.Context, c *encrypt.Cipher, v int64) (eql.BigintEqQuery, error) { - out, err := f.field.Query(ctx, c, v) - return eql.BigintEqQuery(out.EQL), err + return eql.Bigint(out.EQL), err } type ProfileActiveField struct { diff --git a/languages/golang/encrypt/internal/testusers/profiles.go b/languages/golang/encrypt/internal/testusers/profiles.go index e86aa6c19..ab5a07c1d 100644 --- a/languages/golang/encrypt/internal/testusers/profiles.go +++ b/languages/golang/encrypt/internal/testusers/profiles.go @@ -3,15 +3,14 @@ package testusers //go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Profile -name Profile // Profile stores one field in each EQL v4 type the engine produces besides -// TextEq, the shape the cross-language fixture -// packages/eql/tests/encryption/fixtures/v4_queries.json is derived under -// (table profiles). +// TextEq, the shape the cross-language fixtures under +// packages/eql/tests/encryption/fixtures (v4_queries.json, v4_rows.json) are +// derived under (table profiles). type Profile struct { _ struct{} `stash:"context=profiles"` ID int64 `stash:"id,passthrough"` - Age int32 `stash:"age,encrypt_into=IntegerEq"` Visits int32 `stash:"visits,encrypt_into=Integer"` - Balance int64 `stash:"balance,encrypt_into=BigintEq"` + Balance int64 `stash:"balance,encrypt_into=Bigint"` Active bool `stash:"active,encrypt_into=Boolean"` Ratio float32 `stash:"ratio,encrypt_into=Real"` Score float64 `stash:"score,encrypt_into=Double"` diff --git a/packages/eql/tests/encryption/fixtures/v4_queries.json b/packages/eql/tests/encryption/fixtures/v4_queries.json index b7ff038ce..4e1ce409f 100644 --- a/packages/eql/tests/encryption/fixtures/v4_queries.json +++ b/packages/eql/tests/encryption/fixtures/v4_queries.json @@ -1,16 +1,8 @@ { "_comment": "The v4 query operands eql-bindings derives for each plaintext under FakeDataKeySource's index key (keyset nil), read by the Go SDK's hermetic test; regenerate with EQL_UPDATE_FIXTURES=1, do not edit.", - "age": { - "plaintext": 42, - "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"age\"},\"hm\":\"0fe024ff2c58b7840dafb754980c4c587ccd4c47edf35ae4c0f157183c5c6f7f\"}" - }, - "balance": { - "plaintext": 1234567890123, - "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"balance\"},\"hm\":\"b3d4bbae7880b2a05d71e3b810357e8a4eefc50924db4387fec94435811b908a\"}" - }, "name": { "plaintext": "alice", - "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"name\"},\"bf\":[37,38,59,139,167,199,214,242,245]}" + "query": "{\"v\":4,\"i\":{\"t\":\"profiles\",\"c\":\"name\"},\"bf\":[59,96,167,214,554,603,889,1061,1127,1148,1467,1479,1574,1675,1778,1781,1925,2008]}" }, "table": "profiles" } diff --git a/packages/eql/tests/encryption/fixtures/v4_rows.json b/packages/eql/tests/encryption/fixtures/v4_rows.json new file mode 100644 index 000000000..7f233b031 --- /dev/null +++ b/packages/eql/tests/encryption/fixtures/v4_rows.json @@ -0,0 +1,211 @@ +{ + "_comment": "Profile rows sealed by the Go SDK's generated code over the deterministic EQL guest with the seed below; the Rust suite opens and queries them. Regenerate with EQL_UPDATE_FIXTURES=1 go test ./encrypt -run TestV4Rows, do not edit.", + "seed": "encrypt round-trip tests seed v1", + "table": "profiles", + "rows": [ + { + "id": 1, + "plaintext": { + "visits": 3, + "balance": 1234567890123, + "active": true, + "ratio": 0.5, + "score": 2.25, + "note": "a private note", + "name": "alice" + }, + "stored": { + "active": { + "v": 4, + "i": { + "t": "profiles", + "c": "active" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAA7z71FqWkCCWSkyPiBK7IQIAD0ug/nDu5R6UBEM9vJjtvDMBTuommRH359HgY7qUe5cAGqLPvdkpm6yw99+ZyvZ/D0GwxwF7jiS0n79tWSkg==" + }, + "balance": { + "v": 4, + "i": { + "t": "profiles", + "c": "balance" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAAZJDEBcfVqv/o1B2CASJrAIAA9a/qD4wfv0t6fGVxDhKQOlQDIuYj5si13wnlmGENABAF7hMm8/oFuh1IxWGO3alDB86saK1iUeiWeC+NH10iDJzaWG8JP" + }, + "name": { + "v": 4, + "i": { + "t": "profiles", + "c": "name" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAASxNA10KeHfMqSIglat8/GIADzC2+uswJkz9HL6DqUBBNDtGODBX5yNHLKryZ0Lgc24AHLdWoZqUFokOA4X/RHwY8HLX/T8ltWVZz/heIAOGxpuGY=", + "bf": [ + 59, + 96, + 167, + 214, + 554, + 603, + 889, + 1061, + 1127, + 1148, + 1467, + 1479, + 1574, + 1675, + 1778, + 1781, + 1925, + 2008 + ] + }, + "note": { + "v": 4, + "i": { + "t": "profiles", + "c": "note" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAACaWtPMkbUZL3WUSHDD8aD7IADB6CfVElii+gglc1gys6JmpNlLoIFn47AvStPnpJb63wEGdkbtSd+MIQURFj5j9R0iQpEA9zXGHSnKKTDGDBmveJjSwGn2r0NA2cM=" + }, + "ratio": { + "v": 4, + "i": { + "t": "profiles", + "c": "ratio" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAAuHszuxtkGGxdJ+yeeEgrtIACJ8iRTZnq+2wD3LmxdAqmDDHzDzzH5th5lG8EQq4qujQEsYLgPsjMiNaL8ePzWTx0zmDmQKF7VtItuQqEFLJUguIE=" + }, + "score": { + "v": 4, + "i": { + "t": "profiles", + "c": "score" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAAbQulFdQ7itXP4biDOHDhxIAA4jkCscZEzyZymEuOdqFk34IZ2lMPnDiYFZgGjl+Qh0gEDR8u18kphPu/+f4858qnbijVEzqzfHPJGCXD5w1l5bXR99M8f" + }, + "visits": { + "v": 4, + "i": { + "t": "profiles", + "c": "visits" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAAFDIc6s4iXDM+dzEA4ntTLIACXLoQBRqkNqM7yC8syykzCq7A4M32xA4/pWvh7C8atkgGTGrJ9dKvtm/a2fJWaHbrmXpEgNIWno718QRpijgZ/hYA=" + } + } + }, + { + "id": 2, + "plaintext": { + "visits": 0, + "balance": -1, + "active": false, + "ratio": -1.5, + "score": 0, + "note": "", + "name": "bob smith" + }, + "stored": { + "active": { + "v": 4, + "i": { + "t": "profiles", + "c": "active" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAABGXg/rWzFqWgcCw0KVmlV7IADK5ojyhDsQsCNphHB+3xjIrvP/87lbuOrEITfr0CumXgHI4YSxjlcPwyOXHLayGQAGvwWuAFlpYrH5X+d9Vg==" + }, + "balance": { + "v": 4, + "i": { + "t": "profiles", + "c": "balance" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAACHOuVBG9X5O4D9ApUC1DpIADAVYvyCPRFFVrNaWA5vGxA5PF75NkyEagzyfXSd1UIKwHr8BV896CyFbYSClca7yamexnn0aL1nH9V0jpHrTkTj1DFD5MC" + }, + "name": { + "v": 4, + "i": { + "t": "profiles", + "c": "name" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAAAlsDmtaCz7PVYXLYbtkdYiIADMhjWHFWFQffW6FoF5Kd0/vSO5UX+T+vZyisdueEEtugFjeI9b2b8JAmPZVA1YwGnWAQvTk/NzycoZCSO93CEGlPqUfS99", + "bf": [ + 3, + 15, + 23, + 50, + 91, + 140, + 162, + 308, + 382, + 441, + 455, + 505, + 532, + 575, + 760, + 770, + 863, + 957, + 979, + 1083, + 1109, + 1126, + 1166, + 1187, + 1196, + 1252, + 1266, + 1293, + 1328, + 1331, + 1337, + 1344, + 1364, + 1371, + 1426, + 1702, + 1730, + 1738, + 1881, + 1890, + 1925, + 1981 + ] + }, + "note": { + "v": 4, + "i": { + "t": "profiles", + "c": "note" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAABtkZKQt8AupdgAkSYhBWuLIABushFe05tB4gQmcP9j+7VPX95cAW65PgvtgFajKobc6gFRQvqrq/jE0/A6z/h1wkJaprHCR2rzks2dQO0T" + }, + "ratio": { + "v": 4, + "i": { + "t": "profiles", + "c": "ratio" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAADJpOTT8kFqPHwznhWnYNgQIAC3GPCNDOMGBnh6gPSa5KTDr+VBQDNZAZr7kqKF16sx0QGRhJu9MKyPnOHlPvMXoeAJaiwDz/4FFzW39/K6QYHakms=" + }, + "score": { + "v": 4, + "i": { + "t": "profiles", + "c": "score" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAABAfBAgu3xTGBg6oVcxVdCnIADQvBhh5sMdxmdosyrWjFQngkmQ+312sbVAFpgs2UBpcQFVAT9+lg/ey6n6zeWVQydtF2UVswD18AhJnGMs3DGhMVnxU7Ow" + }, + "visits": { + "v": 4, + "i": { + "t": "profiles", + "c": "visits" + }, + "c": "stack-encrypt:1:AQAAAAAAAAAAAAAAAAAAAABi/myUaRsBCFbuVTRBJLPiIADSnhrChMGg0gQaL8/2ghirKHXXC41P2aq5DghRI2FtWgHdF0NP0QUsZgASmJF0gwN1v4RdsYU6LW96F5S3NrNBBe0=" + } + } + } + ] +} diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index 2c8f00a4d..d7be852da 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -695,7 +695,7 @@ mod erased { /// index key, keyset nil). The Go SDK's hermetic test derives the same /// queries through generated code over the deterministic guest build, whose /// index key is the same, and asserts the bytes are equal: a Go field typed -/// `encrypt_into=IntegerEq` runs this plan and no other. Regenerate with +/// `encrypt_into=TextMatch` runs this plan and no other. Regenerate with /// `EQL_UPDATE_FIXTURES=1`; the bytes must not change otherwise. #[tokio::test] async fn the_v4_query_fixture_is_what_the_dispatch_derives() { @@ -715,11 +715,6 @@ async fn the_v4_query_fixture_is_what_the_dispatch_derives() { let fixture = serde_json::json!({ "_comment": "The v4 query operands eql-bindings derives for each plaintext under FakeDataKeySource's index key (keyset nil), read by the Go SDK's hermetic test; regenerate with EQL_UPDATE_FIXTURES=1, do not edit.", "table": "profiles", - "age": { "plaintext": 42, "query": query("IntegerEq", "age", FfiValue::Int32(42)).await }, - "balance": { - "plaintext": 1_234_567_890_123i64, - "query": query("BigintEq", "balance", FfiValue::Int64(1_234_567_890_123)).await, - }, "name": { "plaintext": "alice", "query": query("TextMatch", "name", FfiValue::String("alice".into())).await, @@ -738,3 +733,236 @@ async fn the_v4_query_fixture_is_what_the_dispatch_derives() { "the committed fixture is what the dispatch derives today" ); } + +/// `fixtures/v4_rows.json`: `Profile` rows the Go SDK's generated code sealed +/// over the deterministic EQL guest (`TestV4RowsMatchTheCommittedFixture` in +/// languages/golang/encrypt writes it). The guest derives its keys with +/// stack-kms's `DeterministicSource`, so a cipher over the same seed opens +/// what Go sealed. +mod go_rows { + use super::*; + use std::collections::BTreeMap; + + #[derive(serde::Deserialize)] + pub struct Fixture { + pub seed: String, + pub table: String, + pub rows: Vec, + } + #[derive(serde::Deserialize)] + pub struct Row { + pub id: i32, + pub plaintext: Plain, + pub stored: BTreeMap, + } + #[derive(serde::Deserialize, Clone)] + pub struct Plain { + pub visits: i32, + pub balance: i64, + pub active: bool, + pub ratio: f32, + pub score: f64, + pub note: String, + pub name: String, + } + + pub fn fixture() -> Fixture { + let path = + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/v4_rows.json"); + let raw = std::fs::read_to_string(path).expect( + "fixtures/v4_rows.json is written by the Go SDK's TestV4RowsMatchTheCommittedFixture", + ); + serde_json::from_str(&raw).unwrap() + } + + pub async fn cipher(seed: &str) -> stack_encrypt::StackCipher { + let seed: [u8; 32] = seed.as_bytes().try_into().expect("a 32-byte seed"); + stack_encrypt::StackCipher::builder() + .kms(stack_kms::DeterministicSource::new(seed)) + .init() + .await + .unwrap() + } + + pub fn column(name: &str) -> NonEmpty { + Identifier::for_column("profiles", name).unwrap() + } + + /// The stored value under `column`, parsed as its v4 type: the shape, the + /// version and the terms the Go code wrote. + pub fn parsed(row: &Row, column: &str) -> T { + serde_json::from_value(row.stored[column].clone()) + .unwrap_or_else(|e| panic!("row {} {column}: {e}", row.id)) + } + + /// Open every field of `row` and check it against the plaintext Go + /// sealed. + pub async fn open_row( + cipher: &stack_encrypt::StackCipher, + row: &Row, + ) { + let p = &row.plaintext; + let visits: Int32 = cipher + .decrypt_as( + parsed::(row, "visits"), + column("visits").into(), + ) + .await + .unwrap(); + assert_eq!(visits.0, p.visits); + let balance: Int64 = cipher + .decrypt_as( + parsed::(row, "balance"), + column("balance").into(), + ) + .await + .unwrap(); + assert_eq!(balance.0, p.balance); + let active: Bool = cipher + .decrypt_as( + parsed::(row, "active"), + column("active").into(), + ) + .await + .unwrap(); + assert_eq!(active.0, p.active); + let ratio: Float32 = cipher + .decrypt_as(parsed::(row, "ratio"), column("ratio").into()) + .await + .unwrap(); + assert_eq!(ratio.0.to_bits(), p.ratio.to_bits()); + let score: Float64 = cipher + .decrypt_as( + parsed::(row, "score"), + column("score").into(), + ) + .await + .unwrap(); + assert_eq!(score.0.to_bits(), p.score.to_bits()); + let note: String = cipher + .decrypt_as(parsed::(row, "note"), column("note").into()) + .await + .unwrap(); + assert_eq!(note, p.note); + let name: String = cipher + .decrypt_as( + parsed::(row, "name"), + column("name").into(), + ) + .await + .unwrap(); + assert_eq!(name, p.name); + } +} + +/// What the Go SDK sealed parses as each v4 type and opens in Rust, field by +/// field, under the column it was stored under. +#[tokio::test] +async fn rows_the_go_sdk_sealed_open_in_rust() { + let fixture = go_rows::fixture(); + assert_eq!(fixture.table, "profiles"); + assert!(!fixture.rows.is_empty()); + let cipher = go_rows::cipher(&fixture.seed).await; + for row in &fixture.rows { + go_rows::open_row(&cipher, row).await; + } +} + +/// The Go-sealed rows in PostgreSQL: each value passes its `eql_v4_*` domain +/// check, `@@` finds the row with the fixture's Go-derived query operand, +/// and what comes back out still opens. +#[test] +#[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] +fn postgres_stores_and_queries_rows_the_go_sdk_sealed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let fixture = go_rows::fixture(); + let cipher = runtime.block_on(go_rows::cipher(&fixture.seed)); + let mut db = postgres::Client::connect( + &std::env::var("EQL_TEST_DATABASE_URL").expect("EQL_TEST_DATABASE_URL"), + postgres::NoTls, + ) + .unwrap(); + db.batch_execute( + "CREATE TEMP TABLE go_profiles ( + id int PRIMARY KEY, + visits public.eql_v4_integer, + balance public.eql_v4_bigint, + active public.eql_v4_boolean, + ratio public.eql_v4_real, + score public.eql_v4_double, + note public.eql_v4_text, + name public.eql_v4_text_match + );", + ) + .unwrap(); + for row in &fixture.rows { + let v = |column: &str| row.stored[column].to_string(); + db.execute( + "INSERT INTO go_profiles VALUES ($1, + $2::text::jsonb::public.eql_v4_integer, + $3::text::jsonb::public.eql_v4_bigint, + $4::text::jsonb::public.eql_v4_boolean, + $5::text::jsonb::public.eql_v4_real, + $6::text::jsonb::public.eql_v4_double, + $7::text::jsonb::public.eql_v4_text, + $8::text::jsonb::public.eql_v4_text_match)", + &[ + &row.id, + &v("visits"), + &v("balance"), + &v("active"), + &v("ratio"), + &v("score"), + &v("note"), + &v("name"), + ], + ) + .unwrap(); + } + + let queries: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/v4_queries.json"), + ) + .unwrap(), + ) + .unwrap(); + let needle = queries["name"]["plaintext"].as_str().unwrap(); + let ids: Vec = db + .query( + "SELECT id FROM go_profiles WHERE name @@ $1::text::jsonb::eql_v4.query_text_match ORDER BY id", + &[&queries["name"]["query"].as_str().unwrap()], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect(); + let expected: Vec = fixture + .rows + .iter() + .filter(|r| r.plaintext.name.contains(needle)) + .map(|r| r.id) + .collect(); + assert!(!expected.is_empty()); + assert_eq!(ids, expected, "@@ {needle:?} over Go-sealed names"); + + // Read back from the domains and open again. + for row in &fixture.rows { + let back = db + .query_one( + "SELECT (to_jsonb(p) - 'id')::text FROM go_profiles p WHERE id = $1", + &[&row.id], + ) + .unwrap(); + let stored: String = back.get(0); + let round_tripped = go_rows::Row { + id: row.id, + plaintext: row.plaintext.clone(), + stored: serde_json::from_str(&stored).unwrap(), + }; + runtime.block_on(go_rows::open_row(&cipher, &round_tripped)); + } +} From 25606d271f2fe3e4e883d1b5fdebe5bb8e88abfe Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 15:10:09 +1100 Subject: [PATCH 3/4] test(golang): hand today's Go-sealed rows to Rust, and keep the fixture stable - TestV4RowsMatchTheCommittedFixture writes the rows it seals to EQL_V4_ROWS_OUT, and the Rust suite opens the file EQL_V4_ROWS names. The Go job runs both, so the ciphertexts the Go SDK seals in that run are opened by native Rust, not only the committed ones. - EQL_UPDATE_FIXTURES rewrites v4_rows.json only when something besides the ciphertexts moved, so a regeneration that changes nothing leaves no diff of fresh nonces. - The Go and Rust TextMatch checks say that their negative cases hold for one seed and configuration, the PostgreSQL expectation folds case as `@@` does, and a needle shorter than three characters is asserted to be refused. - tests-golang.yml runs on any change under packages/eql/tests/encryption. Claude-Session: https://claude.ai/code/session_01VYKAWbd361xDLmqBdLoUcB --- .github/workflows/tests-golang.yml | 17 ++++++- languages/golang/encrypt/eql_v4_test.go | 51 +++++++++++++++---- .../eql/tests/encryption/tests/v4_types.rs | 16 ++++-- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 4a31490e7..9cd9258e4 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -38,7 +38,7 @@ on: - packages/eql/crates/eql-domains/** # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - - packages/eql/tests/encryption/fixtures/** + - packages/eql/tests/encryption/** - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py @@ -67,7 +67,7 @@ on: - packages/eql/crates/eql-domains/** # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - - packages/eql/tests/encryption/fixtures/** + - packages/eql/tests/encryption/** - packages/eql/Cargo.lock - languages/golang/** - scripts/check-wasm-imports.py @@ -141,6 +141,7 @@ jobs: . languages/golang/encrypt/guest languages/golang/auth/guest + packages/eql # The HTTP-free core compiles for wasm32-wasip1 with no JS-host backend # and no native HTTP/TLS stack in its graph: the invariant the wazero @@ -228,6 +229,18 @@ jobs: echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB" mise run go:test + # The Rust suite opens the committed v4_rows.json, whose ciphertexts the + # Go SDK sealed when the file was last regenerated. This also hands it + # the rows the Go SDK seals in this run, so a Go guest and a native + # Rust build that stop agreeing on today's ciphertexts fail here rather + # than when someone next regenerates the file. The toolchain is the + # root's pin, as for the guests. + - name: Rows the Go SDK seals today open in Rust + run: | + rows="$RUNNER_TEMP/v4_rows.json" + (cd languages/golang && CGO_ENABLED=0 EQL_V4_ROWS_OUT="$rows" go test -count=1 -run TestV4RowsMatchTheCommittedFixture ./encrypt) + EQL_V4_ROWS="$rows" cargo test --locked --manifest-path packages/eql/Cargo.toml -p eql-encryption-tests --test v4_types rows_the_go_sdk_sealed_open_in_rust + # What is encrypted is fixed before the program ships: every generated # file is committed, and this fails when `go generate` would change # one. It runs the real stashgen against the guest just built, so it diff --git a/languages/golang/encrypt/eql_v4_test.go b/languages/golang/encrypt/eql_v4_test.go index da6a65c0e..afbcc2df0 100644 --- a/languages/golang/encrypt/eql_v4_test.go +++ b/languages/golang/encrypt/eql_v4_test.go @@ -163,6 +163,10 @@ func TestV4QueriesMatchTheRustFixture(t *testing.T) { // And a stored value holds every filter bit of a query for text it // contains, which is what `@@` tests, and not those of text it lacks. + // The first is a property of the filter. The second is not: a Bloom + // filter can cover an absent needle's bits by chance, and "zzz" and + // "bob" are clear of it under testSeed and the pinned configuration + // only, so a change to either can flip them with no bug behind it. encrypted, err := testusers.EncryptProfile(ctx, cipher, profiles[:1]) if err != nil { t.Fatal(err) @@ -177,6 +181,14 @@ func TestV4QueriesMatchTheRustFixture(t *testing.T) { t.Errorf("%q in %q: %v, want %v", needle, profiles[0].Name, got, contained) } } + + // A needle too short for a 3-gram has no filter bits, and an empty + // filter would be a subset of every row's: the engine refuses it. + for _, short := range []string{"", "al"} { + if q, err := testusers.ProfileFields.Name.Query(ctx, cipher, short); err == nil { + t.Errorf("Name.Query(%q) = %s, want a refusal", short, q) + } + } } // rowsFixture is v4_rows.json: the profiles above as the generated code seals @@ -206,13 +218,18 @@ type fixturePlain struct { Name string `json:"name"` } -// TestV4RowsMatchTheCommittedFixture writes v4_rows.json when -// EQL_UPDATE_FIXTURES is set, and otherwise checks the committed file +// TestV4RowsMatchTheCommittedFixture checks the committed v4_rows.json // against what the generated code seals today. The deterministic guest // derives every data key from the seed, so the envelopes and the terms // (`bf`) are the same every run; the AEAD nonce inside each `c` is random, // so `c` is not. The test therefore compares everything but `c`, and opens // the committed `c` values with today's code. +// +// EQL_UPDATE_FIXTURES rewrites the file only when something other than the +// ciphertexts moved, so a regeneration that changes nothing leaves no diff. +// EQL_V4_ROWS_OUT names a file to write today's rows to, ciphertexts and +// all: CI hands it to the Rust suite (EQL_V4_ROWS), so the ciphertexts the +// Go SDK seals now are opened by Rust, not only the committed ones. func TestV4RowsMatchTheCommittedFixture(t *testing.T) { c := deterministicEQLClient(t) encrypted, err := testusers.EncryptProfile(context.Background(), c.DefaultKeyset(), profiles) @@ -241,21 +258,33 @@ func TestV4RowsMatchTheCommittedFixture(t *testing.T) { t.Fatal(err) } rendered = append(rendered, '\n') - path := fixturePath("v4_rows.json") - if os.Getenv("EQL_UPDATE_FIXTURES") != "" { - if err := os.WriteFile(path, rendered, 0o644); err != nil { + if out := os.Getenv("EQL_V4_ROWS_OUT"); out != "" { + if err := os.WriteFile(out, rendered, 0o644); err != nil { t.Fatal(err) } } + path := fixturePath("v4_rows.json") + var committed rowsFixture committedBytes, err := os.ReadFile(path) - if err != nil { - t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) + if err == nil { + err = json.Unmarshal(committedBytes, &committed) } - var committed rowsFixture - if err := json.Unmarshal(committedBytes, &committed); err != nil { - t.Fatal(err) + same := err == nil && bytes.Equal(withoutCiphertexts(t, committed), withoutCiphertexts(t, fixture)) + if os.Getenv("EQL_UPDATE_FIXTURES") != "" { + if same { + t.Logf("%s: unchanged beyond the ciphertexts; left as it is", path) + } else { + if err := os.WriteFile(path, rendered, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%s: rewritten", path) + committed, same = fixture, true + } + } + if err != nil && !same { + t.Fatalf("%v: run with EQL_UPDATE_FIXTURES=1 to write it", err) } - if !bytes.Equal(withoutCiphertexts(t, committed), withoutCiphertexts(t, fixture)) { + if !same { t.Errorf("%s differs from what the generated code seals today beyond the ciphertexts; regenerate it with EQL_UPDATE_FIXTURES=1", path) } diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index d7be852da..a2e673ae0 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -766,9 +766,14 @@ mod go_rows { pub name: String, } + /// The committed rows, or the file `EQL_V4_ROWS` names: CI points it at + /// rows the Go SDK sealed in the same job, so today's Go ciphertexts + /// reach Rust, not only the committed ones. pub fn fixture() -> Fixture { - let path = - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/v4_rows.json"); + let path = std::env::var_os("EQL_V4_ROWS").map_or_else( + || std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/v4_rows.json"), + std::path::PathBuf::from, + ); let raw = std::fs::read_to_string(path).expect( "fixtures/v4_rows.json is written by the Go SDK's TestV4RowsMatchTheCommittedFixture", ); @@ -940,10 +945,15 @@ fn postgres_stores_and_queries_rows_the_go_sdk_sealed() { .iter() .map(|r| r.get(0)) .collect(); + // `@@` folds case, so the expectation does too. It is exact only because + // no committed row's filter covers the needle's bits by chance: a Bloom + // filter may, and a regenerated fixture that turns one up fails here + // against a correct database. + let needle_folded = needle.to_lowercase(); let expected: Vec = fixture .rows .iter() - .filter(|r| r.plaintext.name.contains(needle)) + .filter(|r| r.plaintext.name.to_lowercase().contains(&needle_folded)) .map(|r| r.id) .collect(); assert!(!expected.is_empty()); From 53e5cca3a7dcc0744df398deffad90e655329f97 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 13 Oct 2026 12:52:01 +1100 Subject: [PATCH 4/4] docs(golang): lead the encrypt quickstart with EQL v4 columns The encrypt README and package doc opened with the index= grammar (Ciphertext plus term byte columns, Fields.Email.Equality), while the stashgen README and the EQL v4 alpha setup guide use encrypt_into= and Fields.Email.Query. A reader starting from go doc wrote tags the guide does not describe, for a storage model it does not cover. Both quickstarts now declare encrypt_into fields, build the query with Query, and store each field as one eql_v4 column, with the install step and the query cast. The index= form stays documented and is named as the alternative; the example program still uses it, and the README now says so rather than calling it the eight steps. --- languages/golang/encrypt/README.md | 32 ++++++++++++++++++++---------- languages/golang/encrypt/doc.go | 26 +++++++++++++++++------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 49daffc6c..26a3546ab 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -31,11 +31,15 @@ The package reference is on [pkg.go.dev]; the generator's reference is type User struct { _ struct{} `stash:"context=users"` ID int64 `stash:"id,passthrough"` - Email string `stash:"email,encrypt,index=equality;match"` - Age uint32 `stash:"age,encrypt,index=equality;ore"` + Email string `stash:"email,encrypt_into=TextEq"` + Name string `stash:"name,encrypt_into=TextMatch"` + Age int32 `stash:"age,encrypt_into=Integer"` } ``` + `encrypt_into=` names the EQL type the field is stored as. The Go field + type must match it; see [EQL columns](#eql-columns). + 3. Run the generator. It writes `user_stash.go` beside the struct. ```sh @@ -56,17 +60,25 @@ The package reference is on [pkg.go.dev]; the generator's reference is encrypted, err := users.Encrypt(ctx, cipher, people) opened, err := users.Decrypt(ctx, cipher, encrypted) - term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com") + query, err := users.Fields.Email.Query(ctx, cipher, "bob@example.com") ``` `Encrypt` and `Decrypt` send one ZeroKMS request for each 500 sealed values in the batch, plus one request the first time a keyset is used. -6. Store the encrypted type. Each sealed field is one or more byte columns: - `e.Email.Ciphertext`, `e.Email.Equality`, `e.Email.Match`. `Ciphertext` - and each term type implement `driver.Valuer` and `sql.Scanner`, so a - database library binds and scans each column as bytes; map each one to - its own column. +6. Store the encrypted type. Each `encrypt_into` field is one EQL v4 column: + `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database + library binds and scans it as the JSON a `public.eql_v4_text_eq` column + holds. Install EQL v4 into PostgreSQL first, with the + `eql-v4-install.sql` attached to the release. A query casts its parameter + to the column's query type: + + ```sql + SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq + ``` + + A field tagged `encrypt` or `index=` stores its ciphertext and each term + as separate byte columns instead; see [What is stored](#what-is-stored). 7. Run the generator again after each change to the struct or to a tag. A change to the fields of the struct stops the build until you do. @@ -80,8 +92,8 @@ The package reference is on [pkg.go.dev]; the generator's reference is `git diff` sees only files git already tracks; the `git status` check also fails on a generated file that was never committed. -The rest of this file is the reference. [`example/`](example/) is the eight -steps as a program. +The rest of this file is the reference. [`example/`](example/) is a program +that stores separate byte columns (`index=`) against real ZeroKMS. ## Connect diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 7b114f0e3..f29c861bf 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -15,10 +15,15 @@ // type User struct { // _ struct{} `stash:"context=users"` // ID int64 `stash:"id,passthrough"` -// Email string `stash:"email,encrypt,index=equality;match"` -// Age uint32 `stash:"age,encrypt,index=equality;ore"` +// Email string `stash:"email,encrypt_into=TextEq"` +// Name string `stash:"name,encrypt_into=TextMatch"` +// Age int32 `stash:"age,encrypt_into=Integer"` // } // +// encrypt_into names the EQL type the field is stored as. The Go field type +// must match it: TextEq, TextMatch and Text take a string, Integer an int32, +// Bigint an int64, Boolean a bool, Real a float32 and Double a float64. +// // 3. Run the generator. It writes user_stash.go beside the struct: // // go generate ./... @@ -35,15 +40,22 @@ // cipher := client.Keyset(encrypt.KeysetName("tenant-42")) // encrypted, err := users.Encrypt(ctx, cipher, people) // []users.EncryptedUser // opened, err := users.Decrypt(ctx, cipher, encrypted) // []users.User -// term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com") +// query, err := users.Fields.Email.Query(ctx, cipher, "bob@example.com") // // Encrypt and Decrypt send one ZeroKMS request for each 500 sealed values in // the batch, plus one the first time a keyset is used. // -// 6. Store the encrypted type. Each sealed field is one or more columns of -// bytes ([Ciphertext] and the term types), each implementing driver.Valuer -// and sql.Scanner, so a database library binds and scans each one as bytes; -// map each one to its own column. +// 6. Store the encrypted type. Each encrypt_into field is one EQL v4 column +// (an eql.TextEq goes in a public.eql_v4_text_eq column), implementing +// driver.Valuer and sql.Scanner, so a database library binds and scans it +// like any other value. Install EQL v4 into PostgreSQL first, with the +// eql-v4-install.sql attached to the release. A query casts its parameter to +// the column's query type: +// +// SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq +// +// A field tagged encrypt or index= stores its ciphertext ([Ciphertext]) and +// each term as separate byte columns instead; see Terms below. // // 7. Run the generator again after each change to the struct or to a tag. A // change to the fields of the struct stops the build until you do.