test(golang): prove the EQL v4 types from Go, against Rust and PostgreSQL

.github/workflows/tests-golang.yml

@@ -36,6 +36,10 @@ on:
       # The eql guest build links eql-bindings.
       - packages/eql/crates/eql-bindings/**
       - packages/eql/crates/eql-domains/**
+      # The Go tests read the cross-language fixtures the EQL suite writes,
+      # which are derived under EQL's lockfile.
+      - packages/eql/tests/encryption/**
+      - packages/eql/Cargo.lock
       - languages/golang/**
       - scripts/check-wasm-imports.py
       - scripts/go-binding-test.sh
@@ -61,6 +65,10 @@ on:
       # The eql guest build links eql-bindings.
       - packages/eql/crates/eql-bindings/**
       - packages/eql/crates/eql-domains/**
+      # The Go tests read the cross-language fixtures the EQL suite writes,
+      # which are derived under EQL's lockfile.
+      - packages/eql/tests/encryption/**
+      - packages/eql/Cargo.lock
       - languages/golang/**
       - scripts/check-wasm-imports.py
       - scripts/go-binding-test.sh
@@ -133,6 +141,7 @@ jobs:
             .
             languages/golang/encrypt/guest
             languages/golang/auth/guest
+            packages/eql
 
       # The HTTP-free core compiles for wasm32-wasip1 with no JS-host backend
       # and no native HTTP/TLS stack in its graph: the invariant the wazero
@@ -220,6 +229,18 @@ jobs:
           echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB"
           mise run go:test
 
+      # The Rust suite opens the committed v4_rows.json, whose ciphertexts the
+      # Go SDK sealed when the file was last regenerated. This also hands it
+      # the rows the Go SDK seals in this run, so a Go guest and a native
+      # Rust build that stop agreeing on today's ciphertexts fail here rather
+      # than when someone next regenerates the file. The toolchain is the
+      # root's pin, as for the guests.
+      - name: Rows the Go SDK seals today open in Rust
+        run: |
+          rows="$RUNNER_TEMP/v4_rows.json"
+          (cd languages/golang && CGO_ENABLED=0 EQL_V4_ROWS_OUT="$rows" go test -count=1 -run TestV4RowsMatchTheCommittedFixture ./encrypt)
+          EQL_V4_ROWS="$rows" cargo test --locked --manifest-path packages/eql/Cargo.toml -p eql-encryption-tests --test v4_types rows_the_go_sdk_sealed_open_in_rust
+
       # What is encrypted is fixed before the program ships: every generated
       # file is committed, and this fails when `go generate` would change
       # one. It runs the real stashgen against the guest just built, so it

biome.json

@@ -11,6 +11,7 @@
       "!languages/typescript/packages/protect-ffi/target",
       "!languages/typescript/packages/protect-ffi/src/eql-v3-types",
       "!packages/stack-encrypt/tests/fixtures/record_lowering.json",
+      "!packages/eql/tests/encryption/fixtures/v4_rows.json",
       "!packages/eql/crates/eql-bindings/bindings",
       "!packages/eql/crates/eql-bindings/schema",
       "!packages/eql/packages/eql/src/generated",

languages/golang/cmd/stashgen/README.md

@@ -44,7 +44,7 @@ Your program calls those functions, and it never builds or names a plan.
    ```
 
 6. Store the encrypted type.
-   Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v4_text_eq` column holds. Stack Encrypt writes EQL v4; see the `encrypt` package README.
+   Each `encrypt_into` field is one EQL column: `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database library binds and scans it as the JSON a `public.eql_v4_text_eq` column holds. Stack Encrypt writes EQL v4; see the `encrypt` package README for the EQL types it produces and the Go field type each takes.
 
 7. Run the generator again after each change to the struct or to a tag.
    A change to the fields of the struct stops the build until you do.

languages/golang/encrypt/README.md

@@ -31,11 +31,15 @@ The package reference is on [pkg.go.dev]; the generator's reference is
    type User struct {
    	_     struct{} `stash:"context=users"`
    	ID    int64    `stash:"id,passthrough"`
-   	Email string   `stash:"email,encrypt,index=equality;match"`
-   	Age   uint32   `stash:"age,encrypt,index=equality;ore"`
+   	Email string   `stash:"email,encrypt_into=TextEq"`
+   	Name  string   `stash:"name,encrypt_into=TextMatch"`
+   	Age   int32    `stash:"age,encrypt_into=Integer"`
    }
    ```
 
+   `encrypt_into=` names the EQL type the field is stored as. The Go field
+   type must match it; see [EQL columns](#eql-columns).
+
 3. Run the generator. It writes `user_stash.go` beside the struct.
 
    ```sh
@@ -56,17 +60,25 @@ The package reference is on [pkg.go.dev]; the generator's reference is
 
    encrypted, err := users.Encrypt(ctx, cipher, people)
    opened, err := users.Decrypt(ctx, cipher, encrypted)
-   term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com")
+   query, err := users.Fields.Email.Query(ctx, cipher, "bob@example.com")
    ```
 
    `Encrypt` and `Decrypt` send one ZeroKMS request for each 500 sealed
    values in the batch, plus one request the first time a keyset is used.
 
-6. Store the encrypted type. Each sealed field is one or more byte columns:
-   `e.Email.Ciphertext`, `e.Email.Equality`, `e.Email.Match`. `Ciphertext`
-   and each term type implement `driver.Valuer` and `sql.Scanner`, so a
-   database library binds and scans each column as bytes; map each one to
-   its own column.
+6. Store the encrypted type. Each `encrypt_into` field is one EQL v4 column:
+   `eql.TextEq` implements `driver.Valuer` and `sql.Scanner`, so a database
+   library binds and scans it as the JSON a `public.eql_v4_text_eq` column
+   holds. Install EQL v4 into PostgreSQL first, with the
+   `eql-v4-install.sql` attached to the release. A query casts its parameter
+   to the column's query type:
+
+   ```sql
+   SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq
+   ```
+
+   A field tagged `encrypt` or `index=` stores its ciphertext and each term
+   as separate byte columns instead; see [What is stored](#what-is-stored).
 
 7. Run the generator again after each change to the struct or to a tag. A
    change to the fields of the struct stops the build until you do.
@@ -80,8 +92,8 @@ The package reference is on [pkg.go.dev]; the generator's reference is
    `git diff` sees only files git already tracks; the `git status` check
    also fails on a generated file that was never committed.
 
-The rest of this file is the reference. [`example/`](example/) is the eight
-steps as a program.
+The rest of this file is the reference. [`example/`](example/) is a program
+that stores separate byte columns (`index=`) against real ZeroKMS.
 
 ## Connect
 
@@ -282,7 +294,27 @@ of EQL's SQL; the `eql_v3` columns the TypeScript SDK writes hold
 cipherstash-client's terms, which these never match. Install EQL v4 into
 PostgreSQL first, with the `eql-v4-install.sql` attached to each Go release
 (`psql -f eql-v4-install.sql`); `stash eql install` installs EQL v3 only, and
-an `eql_v4_*` column needs the v4 domains. The guest builds the value: the generated code
+an `eql_v4_*` column needs the v4 domains.
+
+The EQL types the engine produces, and the Go field types each takes:
+
+| EQL type | Go field | Queries |
+|---|---|---|
+| `TextEq` | `string` | `=`, `<>` |
+| `TextMatch` | `string` | `@@` (substring match, 3-grams, case-insensitive) |
+| `Text` | `string` | none: stored and read only |
+| `Integer` | `int32` | none |
+| `Bigint` | `int64` | none |
+| `Boolean` | `bool` | none |
+| `Real` | `float32` | none |
+| `Double` | `float64` | none |
+
+A `TextMatch` value must be at least three characters long: shorter text has
+no 3-grams to index and is refused. `IntegerEq` and `BigintEq` are held back
+until the engine's integer equality term settles in vitaminc 0.6. The rest of
+the catalog (ordering, dates, decimals, JSON) is listed in
+`eql.Types` with the reason it is not produced yet, and `stashgen` refuses it
+with that reason. The guest builds the value: the generated code
 imports `encrypt/eql`, which embeds the build of the engine that holds the
 EQL types and registers it on import, so the program runs that build and
 stores what it returns. `Fields.Email.Query` returns the `eql.TextEqQuery`

languages/golang/encrypt/doc.go

@@ -15,10 +15,15 @@
 //	type User struct {
 //		_     struct{} `stash:"context=users"`
 //		ID    int64    `stash:"id,passthrough"`
-//		Email string   `stash:"email,encrypt,index=equality;match"`
-//		Age   uint32   `stash:"age,encrypt,index=equality;ore"`
+//		Email string   `stash:"email,encrypt_into=TextEq"`
+//		Name  string   `stash:"name,encrypt_into=TextMatch"`
+//		Age   int32    `stash:"age,encrypt_into=Integer"`
 //	}
 //
+// encrypt_into names the EQL type the field is stored as. The Go field type
+// must match it: TextEq, TextMatch and Text take a string, Integer an int32,
+// Bigint an int64, Boolean a bool, Real a float32 and Double a float64.
+//
 // 3. Run the generator. It writes user_stash.go beside the struct:
 //
 //	go generate ./...
@@ -35,15 +40,22 @@
 //	cipher := client.Keyset(encrypt.KeysetName("tenant-42"))
 //	encrypted, err := users.Encrypt(ctx, cipher, people) // []users.EncryptedUser
 //	opened, err := users.Decrypt(ctx, cipher, encrypted) // []users.User
-//	term, err := users.Fields.Email.Equality(ctx, cipher, "bob@example.com")
+//	query, err := users.Fields.Email.Query(ctx, cipher, "bob@example.com")
 //
 // Encrypt and Decrypt send one ZeroKMS request for each 500 sealed values in
 // the batch, plus one the first time a keyset is used.
 //
-// 6. Store the encrypted type. Each sealed field is one or more columns of
-// bytes ([Ciphertext] and the term types), each implementing driver.Valuer
-// and sql.Scanner, so a database library binds and scans each one as bytes;
-// map each one to its own column.
+// 6. Store the encrypted type. Each encrypt_into field is one EQL v4 column
+// (an eql.TextEq goes in a public.eql_v4_text_eq column), implementing
+// driver.Valuer and sql.Scanner, so a database library binds and scans it
+// like any other value. Install EQL v4 into PostgreSQL first, with the
+// eql-v4-install.sql attached to the release. A query casts its parameter to
+// the column's query type:
+//
+//	SELECT id, email, name, age FROM users WHERE email = $1::eql_v4.query_text_eq
+//
+// A field tagged encrypt or index= stores its ciphertext ([Ciphertext]) and
+// each term as separate byte columns instead; see Terms below.
 //
 // 7. Run the generator again after each change to the struct or to a tag. A
 // change to the fields of the struct stops the build until you do.