diff --git a/internal/duckdb/policy.go b/internal/duckdb/policy.go index 6986223e..3e9e6a20 100644 --- a/internal/duckdb/policy.go +++ b/internal/duckdb/policy.go @@ -26,9 +26,12 @@ func ConfigurePolicy(ctx context.Context, db *sql.DB, directories ...string) err "SET allow_community_extensions=false", "SET allowed_directories=[" + strings.Join(paths, ",") + "]", "SET enable_external_access=false", - // UTC is set on each newly pooled connection. Public SQL cannot SET - // options; memory and worker limits remain available to the operator. - "SET allowed_configs=['memory_limit','threads','TimeZone']", + // UTC and parquet_metadata_cache are set on each newly pooled connection, + // so both must stay settable after the lock. Public SQL cannot change any + // other option; memory and worker limits remain available to the operator. + // parquet_metadata_cache only caches immutable Parquet footers, so allowing + // it does not widen file access or the configuration lockdown. + "SET allowed_configs=['memory_limit','threads','TimeZone','parquet_metadata_cache']", "SET lock_configuration=true", } for _, statement := range statements { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9fcaed39..68090723 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -502,6 +502,17 @@ func openDuckDB(dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, * if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil { return fmt.Errorf("set timezone: %w", err) } + // parquet_metadata_cache is a GLOBAL-scoped option, but duckdb-go applies a + // SET to the issuing connection only — it does not reach the other pooled + // connections, so it must be set on each one here rather than once at boot. + // Every read statement binds an explicit per-file Parquet list and a panel + // issues several, so caching footers by path+mtime+size avoids re-reading + // them per statement. Our batch files are immutable with unique paths, so a + // cached footer is never stale. The policy keeps it in allowed_configs so + // this SET still succeeds on connections opened after lock_configuration. + if _, err := execer.ExecContext(ctx, "SET parquet_metadata_cache=true", nil); err != nil { + return fmt.Errorf("set parquet_metadata_cache: %w", err) + } tempDirOnce.Do(func() { _, tempDirErr = execer.ExecContext(ctx, "SET temp_directory="+sqlLiteral(tempDir), nil) }) diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index a20592bc..f3ff9f21 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -147,6 +147,47 @@ func TestNewDuckReconnectsInUTCAfterStartupContextCancelled(t *testing.T) { } } +// Every pooled read connection must enable parquet_metadata_cache so repeated +// statements over the same immutable files reuse their footers instead of +// re-reading them. The setting is applied per connection and kept in the policy +// allowed_configs, so it must also hold on a connection opened after the +// configuration lock, and the lock must still refuse other configuration. +func TestReadConnectionsEnableParquetMetadataCache(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB", DuckDBThreads: 2, DuckDBMaxConns: 4} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck() error = %v", err) + } + defer d.Close() + + for name, db := range map[string]*sql.DB{"read": d.DB, "write": d.writeDB, "memory": d.memoryDB, "parse": d.parseDB} { + t.Run(name, func(t *testing.T) { + // Retire idle connections so this query runs on a freshly opened one, + // whose init hook runs after the configuration was locked. + db.SetMaxIdleConns(0) + var enabled bool + if err := db.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&enabled); err != nil { + t.Fatal(err) + } + if !enabled { + t.Fatalf("parquet_metadata_cache = false on a %s connection, want true", name) + } + }) + } + + // The lock still refuses configuration outside allowed_configs, so public SQL + // cannot re-open file or external access. + if _, err := d.DB.ExecContext(ctx, "SET enable_external_access=true"); err == nil { + t.Fatal("expected locked configuration to refuse SET enable_external_access") + } +} + // A cancelled statement's interrupt can reach the ROLLBACK that ends its read // transaction, leaving the transaction open. The read must roll back itself // after cancellation, and a connection whose rollback failed must leave the diff --git a/internal/engine/metadata_cache_bench_test.go b/internal/engine/metadata_cache_bench_test.go new file mode 100644 index 00000000..56dfedcf --- /dev/null +++ b/internal/engine/metadata_cache_bench_test.go @@ -0,0 +1,254 @@ +//go:build filebench + +package engine + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sort" + "testing" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/labstack/pitot/internal/config" + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +// A/B for parquet_metadata_cache (#484). The engine passes an explicit per-file +// Parquet list on every statement, and a panel issues several. With the footer +// cache OFF, each statement re-reads every file footer; with it ON, footers are +// cached by path+mtime+size. Both arms read the SAME immutable fixtures and run +// the same explicit-list statement; the only difference is the per-connection +// cache SET. Both engines are opened and warmed up front, then sampled with +// AB/BA alternation so OS page-cache and time drift affect both arms equally. +// This is an engine-internal control, not a product code path and not capacity +// evidence. +// +// go test -tags=filebench ./internal/engine -run '^TestParquetMetadataCacheComparisonBenchmark$' -count=1 -v +func TestParquetMetadataCacheComparisonBenchmark(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + + cfg := config.Config{DataDir: t.TempDir()} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + + base := time.Date(2026, 11, 1, 12, 0, 0, 123, time.UTC) + w := queryrows.Window{Start: base, End: base.Add(time.Minute)} + signal, _ := telemetry.LookupSignal("spans") + + published := 0 + for _, files := range []int{128, 512, 1024} { + for ; published < files; published++ { + batch := telemetrystore.Batch{ID: fmt.Sprintf("mdc_%05d", published)} + for row := range 32 { + service := []string{"api", "worker", "db"}[row%3] + n := base.Add(time.Duration(row) * time.Millisecond).UnixNano() + if row == 0 { + n = w.Start.UnixNano() + } + if row == 31 { + n = w.End.UnixNano() - 1 + } + status := "STATUS_CODE_OK" + if row%7 == 0 { + status = "STATUS_CODE_ERROR" + } + span := telemetry.Span{Namespace: "prod", ServiceName: service, TraceID: fmt.Sprintf("mdc_trace_%05d_%02d", published, row/2), SpanID: fmt.Sprint(row), StartUnixNanos: n, EndUnixNanos: n + int64(row+1)*int64(time.Millisecond), DurationMS: float64(row + 1), StatusCode: status, IngestedAt: base.UnixNano() + int64(published)} + batch.Spans = append(batch.Spans, span) + } + if err := repo.Commit(ctx, batch); err != nil { + t.Fatal(err) + } + } + batches := repo.Parquet.BatchMetadata() + if len(batches) != files { + t.Fatalf("captured files=%d expected=%d", len(batches), files) + } + + sort.Slice(batches, func(i, j int) bool { return batches[i].ID < batches[j].ID }) + paths := make([]string, 0, len(batches)) + for _, b := range batches { + paths = append(paths, sqlLiteral(filepath.ToSlash(filepath.Join(repo.Parquet.Dir(), "batches", b.ID+".batch", "spans.parquet")))) + } + physical := "SELECT " + rawProjection(signal) + " FROM read_parquet([" + joinCSV(paths) + "], union_by_name=false, hive_partitioning=false)" + statement := `SELECT service,count(*)::BIGINT,count(DISTINCT trace_id)::BIGINT, + sum(end_unix_nano-start_unix_nano)::BIGINT, + sum(CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END)::BIGINT + FROM (` + cleanSource("spans", physical) + `) WHERE start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time 0 { + out += "," + } + out += s + } + return out +} + +type mdcConnector struct { + *duckdb.Connector + cache bool +} + +func (c *mdcConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + if err != nil { + return nil, err + } + ex := conn.(driver.ExecerContext) + stmts := []string{"LOAD parquet", "SET TimeZone='UTC'"} + if c.cache { + stmts = append(stmts, "SET parquet_metadata_cache=true") + } + for _, stmt := range stmts { + if _, err := ex.ExecContext(ctx, stmt, nil); err != nil { + return nil, errors.Join(err, conn.Close()) + } + } + return conn, nil +} + +type metadataCacheArm struct { + t *testing.T + ctx context.Context + pool *sql.DB + conn *sql.Conn + tx *sql.Tx + statement string + w queryrows.Window + cache bool + timings []float64 +} + +func newMetadataCacheArm(t *testing.T, ctx context.Context, statement string, w queryrows.Window, cache bool) *metadataCacheArm { + t.Helper() + dsn, err := duckDSN(filepath.Join(t.TempDir(), "arm.duckdb"), "4GB", 4) + if err != nil { + t.Fatal(err) + } + native, err := duckdb.NewConnector(dsn, nil) + if err != nil { + t.Fatal(err) + } + pool := sql.OpenDB(&mdcConnector{Connector: native, cache: cache}) + pool.SetMaxOpenConns(1) + pool.SetMaxIdleConns(1) + conn, err := pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + var got bool + if err := conn.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&got); err != nil { + t.Fatal(err) + } + if got != cache { + t.Fatalf("arm cache=%v but connection reports %v", cache, got) + } + tx, err := conn.BeginTx(context.WithoutCancel(ctx), nil) + if err != nil { + t.Fatal(err) + } + return &metadataCacheArm{t: t, ctx: ctx, pool: pool, conn: conn, tx: tx, statement: statement, w: w, cache: cache} +} + +func (a *metadataCacheArm) exec() { + rows, err := a.tx.QueryContext(a.ctx, a.statement, a.w.Start, a.w.End) + if err != nil { + a.t.Fatal(err) + } + for rows.Next() { + var s string + var x, y, z, q int64 + if err := rows.Scan(&s, &x, &y, &z, &q); err != nil { + a.t.Fatal(err) + } + } + if err := rows.Err(); err != nil { + a.t.Fatal(err) + } + rows.Close() +} + +func (a *metadataCacheArm) sample() { + started := time.Now() + a.exec() + a.timings = append(a.timings, float64(time.Since(started))/float64(time.Millisecond)) +} + +func (a *metadataCacheArm) report() map[string]any { + sort.Float64s(a.timings) + var sum float64 + for _, v := range a.timings { + sum += v + } + var objectCacheBytes int64 + _ = a.tx.QueryRowContext(a.ctx, "SELECT coalesce(sum(memory_usage_bytes),0)::BIGINT FROM duckdb_memory() WHERE tag='OBJECT_CACHE'").Scan(&objectCacheBytes) + n := len(a.timings) + return map[string]any{ + "arm": map[bool]string{false: "cache_off", true: "cache_on"}[a.cache], + "cache_enabled": a.cache, + "samples": n, + "mean_ms": sum / float64(n), + "median_ms": (a.timings[n/2-1] + a.timings[n/2]) / 2, + "p95_ms": a.timings[int(float64(n)*0.95)-1], + "p99_ms": a.timings[int(float64(n)*0.99)-1], + "min_ms": a.timings[0], + "max_ms": a.timings[n-1], + "object_cache_bytes": objectCacheBytes, + "unit": "native_execute_scan_same_tx_ab_ba", + } +} + +func (a *metadataCacheArm) close() { + _ = a.tx.Rollback() + _ = a.conn.Close() + _ = a.pool.Close() +}