Retriever parquet

README.md

@@ -78,7 +78,8 @@ comparison mode yet. The command can emit JSONL records plus Markdown and JSON s
 against a previous JSONL baseline.
 
 `go run ./cmd/retriever` dumps and loads live Dawgs graph databases as
-manifest-based collections of compressed JSONL fragments. It supports
+manifest-based collections of compressed JSONL fragments, with an optional
+Parquet sidecar for dumps. It supports
 PostgreSQL and Neo4j, gzip and zstd compression, checksum validation before
 load, optional deterministic property scrubbing, and a read-throughput benchmark
 mode. It can also package dumps as single HPKE/ML-KEM encrypted TAR archives.
@@ -87,10 +88,11 @@ archive, load, scrubbed dump, metrics verification, and benchmark examples.
 The same import/export functionality is available to library consumers from
 `github.com/specterops/dawgs/retriever`; callers provide an already-open
 `graph.Database`, and archive helpers support both path-based and stream-based
-APIs. The package exposes CLI-matching default option constructors, structured
-progress callbacks, manifest/metrics helpers, HPKE key envelope reader/writer
-helpers, and typed errors for validation, compatibility, checksum, metrics, and
-count mismatches.
+APIs. Library dumps enable the optional sidecar with `DumpOptions.Parquet` and
+receive its manifest and success-marker paths in `DumpResult`. The package
+exposes CLI-matching default option constructors, structured progress callbacks,
+manifest/metrics helpers, HPKE key envelope reader/writer helpers, and typed
+errors for validation, compatibility, checksum, metrics, and count mismatches.
 
 PostgreSQL translates exact string property equality with a JSON string type guard and `properties ->>` extraction, so
 indexes created on expressions such as `properties ->> 'objectid'` and `properties ->> 'name'` can be used for selective

cmd/retriever/README.md

@@ -25,6 +25,7 @@ retriever dump \
   -scrub none \
   -compression zstd \
   -zstd-level 11 \
+  -parquet \
   -shard-size 100000
 ```
 
@@ -34,9 +35,10 @@ validates that expected node and edge partitions exist. For Neo4j, `-all-graphs`
 means the selected Neo4j database only.
 
 Existing non-empty output directories are refused unless `-force` is supplied.
-The manifest is written last as `manifest.json`; if a dump fails before that
-point, the directory is intentionally left for inspection without a success
-manifest.
+For JSONL-only dumps, the manifest is written last as `manifest.json`. With a
+Parquet sidecar, its metadata precedes `manifest.json` and `parquet/_SUCCESS`
+is written last. Failed dumps intentionally leave completed fragments for
+inspection without the final applicable success boundary.
 
 New dumps include a `retriever-metrics-v1` manifest section with graph metrics
 computed from the same node and relationship streams written to the fragments.
@@ -54,6 +56,28 @@ Dump progress is emitted with `log/slog` on stderr. Notices mark output
 directory preparation, graph counting, scrub pre-pass work, node and relationship
 phase boundaries, periodic entity progress, manifest writing, and completion.
 
+### Parquet sidecar
+
+Pass `-parquet` to write a strict Parquet sidecar in addition to the normal
+compressed JSONL collection. JSONL remains mandatory and keeps its existing
+paths, bytes, manifest, load behavior, and checksums.
+
+Parquet fragments use matching logical shard boundaries under
+`parquet/graphs/<graph>/`. Node rows contain `id`, `kinds`, and JSON-annotated
+`properties`. Relationship rows contain `id`, `start_id`, `end_id`, `kind`, and
+JSON-annotated `properties`; unlike JSONL v1, the Parquet sidecar retains the
+source relationship ID. Columns use Zstandard compression.
+
+The sidecar metadata is `parquet/manifest.json` with format
+`retriever-parquet-export-v1`. Publication writes that metadata first, the
+existing JSONL `manifest.json` second, and `parquet/_SUCCESS` last. A requested
+Parquet failure fails the dump, and `_SUCCESS` is absent unless both outputs
+and their metadata completed.
+
+Load and verify continue to consume the JSONL collection. Encrypted archive
+creation also retains its existing behavior and packages the JSONL manifest
+and files; it does not include the optional Parquet sidecar.
+
 ## Encrypted Archives
 
 Generate a recipient key pair before creating encrypted archives:

cmd/retriever/main.go

@@ -89,6 +89,7 @@ func (s commandRuntime) runDump(ctx context.Context, args []string) error {
 	allGraphs := flags.Bool("all-graphs", false, "Dump every graph discoverable by the selected driver.")
 	flags.StringVar(&cfg.OutputDir, "out", "", "Output collection directory.")
 	flags.BoolVar(&cfg.Force, "force", false, "Replace an existing non-empty output directory.")
+	flags.BoolVar(&cfg.Parquet, "parquet", false, "Write a Parquet sidecar export in addition to JSONL.")
 	flags.StringVar(&archiveOut, "archive-out", "", "Optional encrypted archive output path.")
 	flags.StringVar(&recipientPath, "recipient", "", "Recipient public key for -archive-out.")
 	flags.StringVar(&scrubValue, "scrub", string(cfg.Scrub), "Scrub mode: none or full.")
@@ -171,8 +172,12 @@ func (s commandRuntime) runDump(ctx context.Context, args []string) error {
 
 		archiveLine = fmt.Sprintf("archive: %s\n", archiveOut)
 	}
+	var parquetLines string
+	if result.ParquetManifestPath != "" {
+		parquetLines = fmt.Sprintf("parquet manifest: %s\nparquet success: %s\n", result.ParquetManifestPath, result.ParquetSuccessPath)
+	}
 
-	fmt.Fprintf(s.stdout, "dumped %d graph(s)\nmanifest: %s\n%snodes: %d\nrelationships: %d\n", len(result.Manifest.Graphs), result.ManifestPath, archiveLine, result.NodeCount, result.EdgeCount)
+	fmt.Fprintf(s.stdout, "dumped %d graph(s)\nmanifest: %s\n%s%snodes: %d\nrelationships: %d\n", len(result.Manifest.Graphs), result.ManifestPath, parquetLines, archiveLine, result.NodeCount, result.EdgeCount)
 	return nil
 }
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/klauspost/compress v1.19.0
 	github.com/neo4j/neo4j-go-driver/v5 v5.28.4
+	github.com/parquet-go/parquet-go v0.30.1
 	github.com/pashagolub/pgxmock/v5 v5.1.0
 	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/stretchr/testify v1.11.1
@@ -59,6 +60,7 @@ require (
 	github.com/alfatraining/structtag v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/alingse/nilnesserr v0.2.0 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect
 	github.com/ashanbrown/makezero/v2 v2.1.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
@@ -120,6 +122,7 @@ require (
 	github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect
 	github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gordonklaus/ineffassign v0.2.0 // indirect
 	github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
 	github.com/gostaticanalysis/comment v1.5.0 // indirect
@@ -172,6 +175,9 @@ require (
 	github.com/nishanths/exhaustive v0.12.0 // indirect
 	github.com/nishanths/predeclared v0.2.2 // indirect
 	github.com/nunnatsa/ginkgolinter v0.23.0 // indirect
+	github.com/parquet-go/bitpack v1.0.0 // indirect
+	github.com/parquet-go/jsonlite v1.0.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
@@ -212,6 +218,7 @@ require (
 	github.com/timonwong/loggercheck v0.11.0 // indirect
 	github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect
 	github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
+	github.com/twpayne/go-geom v1.6.1 // indirect
 	github.com/ultraware/funlen v0.2.0 // indirect
 	github.com/ultraware/whitespace v0.2.0 // indirect
 	github.com/uudashr/gocognit v1.2.1 // indirect

go.sum

@@ -31,6 +31,8 @@ github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
 github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
+github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g=
 github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -60,6 +62,8 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ
 github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I=
 github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w=
 github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
 github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo=
@@ -443,10 +447,18 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
+github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
+github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
+github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
 github.com/pashagolub/pgxmock/v5 v5.1.0 h1:NZ4pl82b335sEGIbD/+tk2fVIgVs3yNWr1R42ukpUvU=
 github.com/pashagolub/pgxmock/v5 v5.1.0/go.mod h1:8IJct22b7+EuqecVmYb9aKiENJLLqTsbjFHXH/znAEg=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
 github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -568,6 +580,8 @@ github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVF
 github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo=
 github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw=
 github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
+github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4=
+github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
 github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI=
 github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA=
 github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g=
@@ -580,6 +594,8 @@ github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pH
 github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
 github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
 github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs=

retriever/archive_tar_test.go

@@ -54,6 +54,25 @@ func TestCollectionTarDeterministicAndSorted(t *testing.T) {
 	}
 }
 
+func TestCollectionTarIgnoresFilesAbsentFromManifest(t *testing.T) {
+	dir := writeArchiveFixture(t)
+	orphanPath := filepath.Join(dir, "graphs", "secret-graph", "orphan.jsonl.gz")
+	if err := os.WriteFile(orphanPath, []byte("orphan"), 0o600); err != nil {
+		t.Fatalf("write orphan fragment: %v", err)
+	}
+
+	var buffer bytes.Buffer
+	if err := writeCollectionTar(&buffer, dir); err != nil {
+		t.Fatalf("write tar: %v", err)
+	}
+
+	for _, name := range tarEntryNames(t, buffer.Bytes()) {
+		if name == "graphs/secret-graph/orphan.jsonl.gz" {
+			t.Fatalf("archive included unmanifested file %q", name)
+		}
+	}
+}
+
 func TestCollectionTarStableMetadata(t *testing.T) {
 	dir := writeArchiveFixture(t)
 	var buffer bytes.Buffer
@@ -194,23 +213,27 @@ func writeArchiveFixture(t *testing.T) string {
 		Compression: CompressionGzip,
 		ZstdLevel:   DefaultZstdLevel,
 	}
-	nodeEntry, err := writeNodeFragment(dir, "secret-graph", 1, options, []FragmentNode{{
+	nodeSummary := shardSummary{
+		ID:   shardID{Graph: "secret-graph", Phase: PhaseNodes, Number: 1},
+		Rows: 1,
+	}
+	nodeMetadata := writeTestJSONLShard(t, newJSONLNodeSink(options), nodeSummary, []normalizedNode{{
 		ID:         "1",
 		Kinds:      []string{"User"},
 		Properties: map[string]any{"name": "alice"},
-	}}, nil)
-	if err != nil {
-		t.Fatalf("write node fragment: %v", err)
-	}
+	}})
+	nodeEntry := newJSONLFileManifest(nodeSummary, nodeMetadata)
 
-	edgeEntry, err := writeEdgeFragment(dir, "secret-graph", 1, options, []FragmentEdge{{
+	edgeSummary := shardSummary{
+		ID:   shardID{Graph: "secret-graph", Phase: PhaseEdges, Number: 1},
+		Rows: 1,
+	}
+	edgeMetadata := writeTestJSONLShard(t, newJSONLEdgeSink(options), edgeSummary, []normalizedEdge{{
 		StartID: "1",
 		EndID:   "1",
 		Kind:    "MemberOf",
-	}}, nil)
-	if err != nil {
-		t.Fatalf("write edge fragment: %v", err)
-	}
+	}})
+	edgeEntry := newJSONLFileManifest(edgeSummary, edgeMetadata)
 
 	value := newValidTestManifest(1)
 	value.Schema.Graphs = []GraphSchemaMetadata{{