From 203732ad7c31474f27a5102df158e085923ad7ce Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 30 Oct 2026 15:00:31 -0400 Subject: [PATCH 1/2] Fix `list_flatten` from schema, add `page_to_disk` and `list_flatten` to `PerspectiveWidget` Signed-off-by: Andrew Stein --- .../test/js/constructors/arrow_nested.spec.ts | 46 +++++++++++++++++ .../test/js/constructors/json_nested.spec.ts | 38 ++++++++++++++ .../test/js/derived/backing.spec.ts | 35 +++++++++++++ .../tests/table/test_table_arrow_nested.py | 48 +++++++++++++++++ .../tests/widget/test_anywidget_smoke.py | 25 +++++++++ .../perspective/widget/__init__.py | 15 ++++-- .../perspective/src/cpp/derived_source.cpp | 2 +- .../cpp/perspective/src/cpp/json_loader.cpp | 51 ++++++++++++++----- .../cpp/perspective/src/cpp/server.cpp | 13 +++-- .../cpp/perspective/src/cpp/table.cpp | 5 +- .../src/include/perspective/json_loader.h | 1 + .../src/include/perspective/server.h | 1 + .../src/include/perspective/table.h | 3 +- 13 files changed, 258 insertions(+), 25 deletions(-) diff --git a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts index b163ec5661..83e00cd865 100644 --- a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts @@ -341,6 +341,52 @@ test.describe("Arrow nested columns", function () { ).rejects.toThrow(/`batch`/); }); + test("Stringify keeps a sibling index usable on a schema table", async function () { + const table = await perspective.table( + { id: "integer", y: "string" }, + { index: "id", list_flatten: "stringify" }, + ); + + await table.update( + ipc({ + id: arrow.vectorFromArray([1, 2], int32()), + y: arrow.vectorFromArray([[10, 20], [30]], list(int32())), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + y: ["[10,20]", "[30]"], + }); + + await view.delete(); + await table.delete(); + }); + + test("Cartesian persists across update on a schema table", async function () { + const table = await perspective.table( + { a: "integer", b: "integer" }, + { list_flatten: "cartesian" }, + ); + + await table.update( + ipc({ + a: arrow.vectorFromArray([[1, 2]], list(int32())), + b: arrow.vectorFromArray([[3, 4, 5]], list(int32())), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + a: [1, 1, 1, 2, 2, 2], + b: [3, 4, 5, 3, 4, 5], + }); + + await view.delete(); + await table.delete(); + }); + test("Rejects any index under a multi-list cartesian", async function () { await expect( perspective.table( diff --git a/rust/perspective-js/test/js/constructors/json_nested.spec.ts b/rust/perspective-js/test/js/constructors/json_nested.spec.ts index 875f77fd9e..f4e72a6f77 100644 --- a/rust/perspective-js/test/js/constructors/json_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/json_nested.spec.ts @@ -353,6 +353,44 @@ test.describe("JSON nested columns", function () { ).rejects.toThrow(/`id`/); }); + test("cartesian indexes on a column from one array of objects", async function () { + const table = await perspective.table( + orders([1, 2], [1.5, 2.5]) as any, + { index: "orders.id", list_flatten: "cartesian" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1], + "orders.id": [1, 2], + "orders.price": [1.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("stringify keeps a sibling index usable on a schema table", async function () { + const table = await perspective.table( + { id: "integer", y: "string" }, + { index: "id", list_flatten: "stringify" }, + ); + + await table.update([ + { id: 1, y: [10, 20] }, + { id: 2, y: [30] }, + ] as any); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + y: ["[10,20]", "[30]"], + }); + + await view.delete(); + await table.delete(); + }); + test("stringify keeps a sibling index usable", async function () { const table = await perspective.table( [ diff --git a/rust/perspective-js/test/js/derived/backing.spec.ts b/rust/perspective-js/test/js/derived/backing.spec.ts index 5827eebe8e..68dd170be6 100644 --- a/rust/perspective-js/test/js/derived/backing.spec.ts +++ b/rust/perspective-js/test/js/derived/backing.spec.ts @@ -63,3 +63,38 @@ test.describe("Derived tables over a disk-backed source", function () { await source.delete(); }); }); + +test.describe("Disk-backed derived tables", function () { + test("group_by parent", async function () { + const source = await perspective.table(SCHEMA, { index: "id" }); + await source.update(rows(24)); + const view = await source.view(TREE); + const derived = await perspective.table(view, { page_to_disk: true }); + await expect_tree_parity(source, TREE, derived, view); + await source.update(rows(12, 18)); + await source.remove([0, 1, 2]); + await expect_tree_parity(source, TREE, derived, view); + await derived.delete(); + await view.delete(); + await source.delete(); + }); + + test("flat parent", async function () { + const source = await perspective.table(SCHEMA, { index: "id" }); + await source.update(rows(24)); + const view = await source.view({ + expressions: { double: '"x" * 2' }, + filter: [["y", ">", 3]], + }); + + const derived = await perspective.table(view, { page_to_disk: true }); + await expect_flat_parity(view, derived); + await source.update(rows(12, 18)); + await source.update([{ id: 0, y: 11 }]); + await source.remove([5]); + await expect_flat_parity(view, derived); + await derived.delete(); + await view.delete(); + await source.delete(); + }); +}); diff --git a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py index 8ab0d19662..13bafebded 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py +++ b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py @@ -423,6 +423,54 @@ def test_stringify_keeps_a_sibling_index_usable(self): } +class TestTableArrowSchemaListFlatten(object): + def test_schema_table_stringify_keeps_a_sibling_index_usable(self): + tbl = Table( + {"id": "integer", "y": "string"}, + index="id", + list_flatten="stringify", + ) + + tbl.update( + arrow_bytes( + pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "y": pa.array( + [[10, 20], [30]], type=pa.list_(pa.int64()) + ), + } + ) + ) + ) + + assert tbl.view().to_columns() == { + "id": [1, 2], + "y": ["[10,20]", "[30]"], + } + + def test_schema_table_cartesian_persists_across_update(self): + tbl = Table( + {"a": "integer", "b": "integer"}, list_flatten="cartesian" + ) + + tbl.update( + arrow_bytes( + pa.table( + { + "a": pa.array([[1, 2]], type=pa.list_(pa.int64())), + "b": pa.array([[3, 4, 5]], type=pa.list_(pa.int64())), + } + ) + ) + ) + + assert tbl.view().to_columns() == { + "a": [1, 1, 1, 2, 2, 2], + "b": [3, 4, 5, 3, 4, 5], + } + + class TestTableArrowGather(object): """The expansion is deferred into the `t_column` write rather than materialized in Arrow, so these stress the gather paths specifically.""" diff --git a/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py b/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py index fd0bf29501..790a85cec7 100644 --- a/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py +++ b/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py @@ -107,3 +107,28 @@ def test_proxy_session_lifecycle(self): def test_deferred_load_without_data(self): widget = PerspectiveWidget(None, group_by=["a"]) assert widget.group_by == ["a"] + + def test_forwards_list_flatten_with_index(self): + widget = PerspectiveWidget( + {"id": [1, 2], "y": [[10, 20], [30]]}, + index="id", + list_flatten="stringify", + ) + + assert widget.table.get_index() == "id" + assert widget.table.view().to_columns() == { + "id": [1, 2], + "y": ["[10,20]", "[30]"], + } + + def test_forwards_page_to_disk_and_name(self): + widget = PerspectiveWidget( + {"a": [1, 2, 3]}, page_to_disk=True, name="forwarded" + ) + + assert widget.table.get_name() == "forwarded" + assert widget.table.view().to_columns() == {"a": [1, 2, 3]} + + def test_table_options_without_data(self): + with pytest.raises(TypeError): + PerspectiveWidget(None, list_flatten="stringify") diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index 3a0545c1c4..4c8de83c1d 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -30,6 +30,8 @@ _STATIC = pathlib.Path(__file__).parent / "static" +_TABLE_OPTIONS = ("name", "format", "page_to_disk", "list_flatten") + __doc__ = """ `PerspectiveWidget` is a JupyterLab widget that implements the same API as ``, allowing for fast, intuitive @@ -158,8 +160,9 @@ def __init__( - `limit` (`int`): A upper limit on the number of rows in the Table. Cannot be set at the same time as `index`, ignored if `server` is True. - - `kwargs` (`dict`): configuration options for the `PerspectiveViewer`, - and `Table` constructor if `data` is a dataset. + - `name`, `format`, `page_to_disk`, `list_flatten`: `Table` + constructor options, used when `data` is a dataset. + - `kwargs` (`dict`): configuration options for the `PerspectiveViewer`. # Examples @@ -180,6 +183,9 @@ def __init__( if index is not None and limit is not None: raise TypeError("Index and Limit cannot be set at the same time!") + table_options = {key: kwargs.pop(key, None) for key in _TABLE_OPTIONS} + table_options = {k: v for k, v in table_options.items() if v is not None} + # Parse the dataset we pass in - if it's Pandas, preserve pivots # if isinstance(data, pandas.DataFrame) or isinstance(data, pandas.Series): # data, config = deconstruct_pandas(data) @@ -203,11 +209,12 @@ def __init__( # If an empty dataset is provided, don't call `load()` and wait # for the user to call `load()`. if data is None: - if index is not None or limit is not None: + if index is not None or limit is not None or table_options: raise TypeError( - "Cannot initialize PerspectiveWidget `index` or `limit` without a Table, data, or schema!" + "Cannot initialize PerspectiveWidget `index`, `limit` or `Table` options without a Table, data, or schema!" ) else: + self._options.update(table_options) if index is not None: self._options.update({"index": index}) diff --git a/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp b/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp index 8aaf361b3c..2b97a0f32d 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/derived_source.cpp @@ -298,7 +298,7 @@ t_derived_source::step(bool notify) { std::shared_ptr master = gnode->get_table_sptr(); const t_schema& schema = master->get_schema(); t_uindex num_columns = m_columns.size(); - if (full) { + if (full && m_child->get_backing_store() == BACKING_STORE_MEMORY) { for (t_uindex cidx = 0; cidx < num_columns; ++cidx) { auto column = alias_column(cidx); if (column diff --git a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp index aca6f8eb01..bc6040b7f3 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp @@ -416,6 +416,11 @@ fill_column_json( */ static const char FLATTEN_SEPARATOR = '.'; +/** + * The `array_path` of a leaf not reached through any array. + */ +static constexpr std::size_t NO_ARRAY = std::string::npos; + /** * Visit every scalar beneath `value`, naming it by its dotted path from * `prefix`. An object contributes its leaves rather than itself, because @@ -430,20 +435,20 @@ for_each_leaf( const rapidjson::Value& value, t_list_flatten mode, F&& fn, - bool through_array = false + std::size_t array_path = NO_ARRAY ) { if (mode != LIST_FLATTEN_STRINGIFY && value.IsArray()) { // Inference only needs one element to learn the leaves' types; any // path a later element introduces is grown into by `resolve_column`. if (!value.Empty()) { - for_each_leaf(prefix, value[0], mode, fn, true); + for_each_leaf(prefix, value[0], mode, fn, prefix.size()); } return; } if (!value.IsObject()) { - fn(static_cast(prefix), value, through_array); + fn(static_cast(prefix), value, array_path); return; } @@ -451,11 +456,28 @@ for_each_leaf( for (const auto& child : value.GetObj()) { prefix += FLATTEN_SEPARATOR; prefix += child.name.GetString(); - for_each_leaf(prefix, child.value, mode, fn, through_array); + for_each_leaf(prefix, child.value, mode, fn, array_path); prefix.resize(len); } } +/** + * Record a leaf reached through an array as per-element, and the innermost + * array it was reached through as an expansion point. + */ +static void +note_expansion( + const std::string& name, + std::size_t array_path, + std::set& per_element, + std::set& expansions +) { + if (array_path != NO_ARRAY) { + per_element.insert(name); + expansions.insert(name.substr(0, array_path)); + } +} + /** * The value written when an expansion has no element for a slot -- an empty or * absent array yields one row rather than none, matching the Arrow path. @@ -779,7 +801,7 @@ JsonLoader::init( } m_expands = !m_per_element.empty(); - if (mode == LIST_FLATTEN_CARTESIAN && m_per_element.size() > 1) { + if (mode == LIST_FLATTEN_CARTESIAN && m_expansions.size() > 1) { // A product repeats every factor against the others' dimensions, so // with more than one expansion point nothing varies per row. m_per_element.clear(); @@ -851,6 +873,7 @@ infer_record( std::set& literal, std::set& descended, std::set& per_element, + std::set& expansions, t_list_flatten mode ) { std::string path; @@ -861,12 +884,10 @@ infer_record( path, col.value, mode, - [&](const auto& name, const auto&, bool through_array) { + [&](const auto& name, const auto&, std::size_t array_path) { seen.insert(name); (name.size() == top ? literal : descended).insert(name); - if (through_array) { - per_element.insert(name); - } + note_expansion(name, array_path, per_element, expansions); } ); } @@ -874,7 +895,7 @@ infer_record( // https://github.com/Tencent/rapidjson/issues/1994 for (const auto& col : record.GetObj()) { path.assign(col.name.GetString()); - for_each_leaf(path, col.value, mode, [&](const auto& name, const auto& leaf, bool) { + for_each_leaf(path, col.value, mode, [&](const auto& name, const auto& leaf, std::size_t) { if (std::find(index.begin(), index.end(), name) != index.end()) { is_implicit = false; } @@ -959,6 +980,7 @@ JsonLoader::infer_rows(const std::vector& index) { literal, descended, m_per_element, + m_expansions, m_mode ); @@ -992,6 +1014,7 @@ JsonLoader::infer_ndjson(const std::vector& index) { literal, descended, m_per_element, + m_expansions, m_mode ); @@ -1031,10 +1054,10 @@ JsonLoader::infer_cols(const std::vector& index) { path, cell, m_mode, - [&](const auto& name, const auto& v, bool through_array) { - if (through_array) { - m_per_element.insert(name); - } + [&](const auto& name, const auto& v, std::size_t array_path) { + note_expansion( + name, array_path, m_per_element, m_expansions + ); if (known.count(name) > 0) { return; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index d315d97e58..40b4c2dc35 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -2598,7 +2598,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { switch (r.data().data_case()) { case proto::MakeTableData::kFromView: { make_error = make_derived_table( - r, entity_id, !index.empty() || has_limit, proto_resp, table + r, + entity_id, + !index.empty() || has_limit, + backing_store, + proto_resp, + table ); break; } @@ -2678,7 +2683,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { t_schema table_schema(columns, types); table = Table::from_schema( - index, table_schema, limit, backing_store + index, table_schema, limit, backing_store, list_flatten ); break; } @@ -4064,6 +4069,7 @@ ProtoServer::make_derived_table( const proto::MakeTableReq& req, const ServerResources::t_id& table_id, bool has_identity_options, + t_backing_store backing_store, std::vector>& outs, std::shared_ptr& table ) { @@ -4117,7 +4123,8 @@ ProtoServer::make_derived_table( source->pkey_dtype(), source->index(), source->limit(), - source->pkey_codec() + source->pkey_codec(), + backing_store ); #ifdef PSP_PARALLEL_FOR diff --git a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp index 0773dfbdf6..85acd9fce8 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp @@ -999,7 +999,8 @@ Table::make_derived( t_dtype pkey_dtype, const std::vector& index, std::uint32_t limit, - std::shared_ptr pkey_codec + std::shared_ptr pkey_codec, + t_backing_store backing_store ) { auto pool = std::make_shared(); pool->init(); @@ -1011,7 +1012,7 @@ Table::make_derived( data_table.add_column("psp_op", DTYPE_UINT8, false); auto tbl = std::make_shared
( - pool, schema.columns(), schema.types(), limit, index + pool, schema.columns(), schema.types(), limit, index, backing_store ); tbl->m_derived = true; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h index 6719a560ef..362ee4ee19 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h @@ -138,6 +138,7 @@ namespace json { t_list_flatten m_mode{LIST_FLATTEN_ZIP}; std::set m_per_element; + std::set m_expansions; std::vector m_child_widths; bool m_expands{false}; bool m_is_implicit{true}; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h index f11e5c5e79..7d28b27103 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h @@ -756,6 +756,7 @@ namespace server { const proto::MakeTableReq& req, const ServerResources::t_id& table_id, bool has_identity_options, + t_backing_store backing_store, std::vector>& outs, std::shared_ptr
& table ); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h index 989bbcad03..63f7f9e32b 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h @@ -299,7 +299,8 @@ class PERSPECTIVE_EXPORT Table { t_dtype pkey_dtype, const std::vector& index, std::uint32_t limit, - std::shared_ptr pkey_codec + std::shared_ptr pkey_codec, + t_backing_store backing_store = BACKING_STORE_MEMORY ); bool is_derived() const; From 4fef8f656caf2ae00867a250e68cd10ae45e28c8 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 3 Nov 2026 16:43:22 -0500 Subject: [PATCH 2/2] Arrow `Map` support + `struct_flatten` mode Signed-off-by: Andrew Stein --- docs/md/how_to/javascript/loading_data.md | 32 ++ docs/md/how_to/python/table_data.md | 49 ++- rust/perspective-client/build.rs | 4 + rust/perspective-client/perspective.proto | 7 + rust/perspective-client/src/rust/client.rs | 2 + rust/perspective-client/src/rust/table.rs | 15 +- .../test/js/constructors/arrow_nested.spec.ts | 236 +++++++++++ .../test/js/constructors/json_nested.spec.ts | 26 ++ .../js/constructors/nested_parity.spec.ts | 46 +++ .../tests/table/test_table_arrow_nested.py | 372 ++++++++++++++++++ .../perspective/widget/__init__.py | 7 +- .../src/client/client_async.rs | 17 +- .../src/client/client_sync.rs | 4 +- .../cpp/perspective/src/cpp/arrow_loader.cpp | 345 ++++++++-------- .../perspective/src/cpp/arrow_normalize.cpp | 318 +++++++++++++-- .../cpp/perspective/src/cpp/json_loader.cpp | 53 ++- .../cpp/perspective/src/cpp/server.cpp | 33 +- .../cpp/perspective/src/cpp/table.cpp | 121 ++++-- .../src/include/perspective/arrow_loader.h | 4 +- .../src/include/perspective/arrow_normalize.h | 21 +- .../src/include/perspective/flatten_mode.h | 10 + .../src/include/perspective/json_loader.h | 5 +- .../src/include/perspective/table.h | 41 +- 23 files changed, 1495 insertions(+), 273 deletions(-) diff --git a/docs/md/how_to/javascript/loading_data.md b/docs/md/how_to/javascript/loading_data.md index 08cf7c4d5e..a6acdc5f6c 100644 --- a/docs/md/how_to/javascript/loading_data.md +++ b/docs/md/how_to/javascript/loading_data.md @@ -74,3 +74,35 @@ than ``s which rely on a Web Worker, and can even share a host application with Web Worker-bound `table()`s. The same `promise`-based API is used to communicate with the server-instantiated `view()`, only in this case it is over a websocket. + +## Nested columns + +Perspective's data model is flat, so nested Arrow and JSON values are +normalized on ingest, controlled by two `Table` options. + +`struct_flatten` applies to Arrow `struct` and `map` columns and to JSON +objects. `"pivot"` (the default) hoists each field or key into its own dotted +column, so `{ s: { a: 1 } }` becomes a column `s.a` and a map `m` becomes one +column `m.` per distinct key in the batch, with a null where a row lacks +the key. `"stringify"` keeps the whole value as JSON text in a single string +column. + +`list_flatten` applies to Arrow `list` columns and JSON arrays. `"zip"` (the +default) expands a row into one row per element, repeating its siblings, and +requires sibling lists in a row to agree in length. `"cartesian"` expands to +the product of the lists' lengths. `"stringify"` keeps each list as a JSON +array in a string column. + +The two options compose from the outside in: a list inside a stringified +struct stays inside its JSON text, while a struct inside an expanded list is +stringified once per element row. + +```javascript +const table = await worker.table( + [{ id: 1, m: { a: 1, b: 2 } }, { id: 2, m: { b: 3 } }], + { struct_flatten: "pivot" }, +); + +// { id: [1, 2], "m.a": [1, null], "m.b": [2, 3] } +await (await table.view()).to_columns(); +``` diff --git a/docs/md/how_to/python/table_data.md b/docs/md/how_to/python/table_data.md index 5eb78aa2f9..582e4706b9 100644 --- a/docs/md/how_to/python/table_data.md +++ b/docs/md/how_to/python/table_data.md @@ -38,8 +38,8 @@ with open("data.arrow", "rb") as f: ### Nested columns -Perspective's data model is flat, so Arrow `struct` and `list` columns are -normalized on ingest. +Perspective's data model is flat, so Arrow `struct`, `map` and `list` columns +are normalized on ingest. A `struct` column is hoisted into one dotted column per leaf, recursively. A null parent nulls every descendant leaf: @@ -86,6 +86,51 @@ perspective.table(arrow_table) perspective.table(arrow_table, list_flatten="stringify") ``` +A `map` column is pivoted into one column per distinct key, `m.`, of the +map's item type, so an Arrow map and a JSON object produce the same columns. A +key absent from a row yields a null in that row. A key absent from the whole +batch yields no column, which on an indexed update leaves that cell unchanged, +so declare the keys in a schema when updates must be stable. Only string keys +are supported, and pivoting suits maps with a bounded key set since each key +becomes a column: + +```python +arrow_table = pa.table({ + "id": pa.array([1, 2], type=pa.int64()), + "m": pa.array( + [[("a", 1), ("b", 2)], [("b", 3)]], + type=pa.map_(pa.string(), pa.int64()), + ), +}) + +# `{"id": [1, 2], "m.a": [1, None], "m.b": [2, 3]}` +perspective.table(arrow_table) +``` + +Both `struct` and `map` columns are controlled by the `struct_flatten` +argument: + +- `"pivot"` (default) hoists each field or key into its own dotted column, + as above. +- `"stringify"` keeps the whole value as JSON text in a single string + column. + +The two arguments compose from the outside in. A list inside a stringified +struct is an array within its JSON text and is never expanded, while a struct +inside an expanded list is stringified once per element row: + +```python +arrow_table = pa.table({ + "s": pa.array( + [{"a": 1, "tags": ["x", "y"]}], + type=pa.struct([("a", pa.int64()), ("tags", pa.list_(pa.string()))]), + ), +}) + +# `{"s": ['{"a":1,"tags":["x","y"]}']}` +perspective.table(arrow_table, struct_flatten="stringify") +``` + ## Polars ```python diff --git a/rust/perspective-client/build.rs b/rust/perspective-client/build.rs index 2ca76f905c..b3a3cada69 100644 --- a/rust/perspective-client/build.rs +++ b/rust/perspective-client/build.rs @@ -76,6 +76,10 @@ fn prost_build() -> Result<()> { "ListFlatten", "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", ) + .type_attribute( + "StructFlatten", + "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", + ) .field_attribute("ViewToArrowResp.arrow", "#[serde(skip)]") .field_attribute("from_arrow", "#[serde(skip)]") .type_attribute(".", "#[derive(serde::Serialize)]") diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 153f622192..ee67110654 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -405,6 +405,8 @@ message MakeTableReq { optional ListFlatten list_flatten = 4; + optional StructFlatten struct_flatten = 7; + // The columns of a `Table` made from a `View`, in place of the ones // inferred from it. Only valid with `MakeTableData.from_view`. optional Schema view_schema = 5; @@ -416,6 +418,11 @@ enum ListFlatten { LIST_FLATTEN_CARTESIAN = 1; LIST_FLATTEN_STRINGIFY = 2; } + +enum StructFlatten { + STRUCT_FLATTEN_PIVOT = 0; + STRUCT_FLATTEN_STRINGIFY = 1; +} message MakeTableResp {} enum JoinType { diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index ff40fdac67..27bf9ec474 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -726,6 +726,7 @@ impl Client { limit: None, page_to_disk: None, list_flatten: None, + struct_flatten: None, schema: None, })), resp => Err(resp.into()), @@ -773,6 +774,7 @@ impl Client { limit: info.limit, page_to_disk: None, list_flatten: None, + struct_flatten: None, schema: None, }; diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index 5008645649..b76973ce03 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -166,11 +166,20 @@ pub struct TableInitOptions { /// default) and `cartesian` expand a row into one row per list element, /// and are incompatible with `index`, as the rows of an expansion /// repeat their index. `stringify` encodes each list as a JSON array in - /// a single string column instead. + /// a single string column instead. `STRUCT` and `MAP` columns are + /// governed by `struct_flatten`. #[serde(default)] #[ts(optional)] pub list_flatten: Option, + /// How Arrow `STRUCT` and `MAP` columns and JSON objects are ingested. + /// `pivot` (the default) hoists each field or key into its own dotted + /// column, and `stringify` keeps the whole value as JSON text in a single + /// string column. + #[serde(default)] + #[ts(optional)] + pub struct_flatten: Option, + /// The columns of a [`Table`] made from a [`View`], in place of the ones /// inferred from it. #[serde(default)] @@ -190,6 +199,7 @@ impl TryFrom for MakeTableOptions { fn try_from(value: TableOptions) -> Result { let page_to_disk = value.page_to_disk; let list_flatten = value.list_flatten.map(|x| x as i32); + let struct_flatten = value.struct_flatten.map(|x| x as i32); let view_schema = value.schema.clone().map(|schema| crate::proto::Schema { schema: schema .into_iter() @@ -203,6 +213,7 @@ impl TryFrom for MakeTableOptions { Ok(MakeTableOptions { page_to_disk, list_flatten, + struct_flatten, view_schema, make_table_type: match value { TableOptions { @@ -235,6 +246,7 @@ pub(crate) struct TableOptions { pub limit: Option, pub page_to_disk: Option, pub list_flatten: Option, + pub struct_flatten: Option, pub schema: Option>, } @@ -254,6 +266,7 @@ impl From for TableOptions { limit: value.limit, page_to_disk: value.page_to_disk, list_flatten: value.list_flatten, + struct_flatten: value.struct_flatten, schema: value.schema, } } diff --git a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts index 83e00cd865..dc7a7f92e1 100644 --- a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts @@ -26,6 +26,17 @@ const list = (type: arrow.DataType) => new arrow.List(field("item", type)); const struct = (fields: [string, arrow.DataType][]) => new arrow.Struct(fields.map(([n, t]) => field(n, t))); +const map = (value: arrow.DataType) => + new arrow.Map_( + field( + "entries", + struct([ + ["key", utf8()], + ["value", value], + ]), + ) as any, + ); + const ipc = (columns: Record) => arrow.tableToIPC(arrow.tableFromArrays(columns as any)); @@ -298,6 +309,20 @@ test.describe("Arrow nested columns", function () { ), }); + const empty_orders = () => + ipc({ + batch: arrow.vectorFromArray([], int32()), + orders: arrow.vectorFromArray( + [], + list( + struct([ + ["id", int32()], + ["price", float64()], + ]), + ), + ), + }); + test("Indexes on a column drawn from the list", async function () { const table = await perspective.table( orders([1, 2, 3], [1.5, 2.5, 3.5]), @@ -341,6 +366,30 @@ test.describe("Arrow nested columns", function () { ).rejects.toThrow(/`batch`/); }); + test("Rejects an index on a repeated sibling from a zero-row arrow", async function () { + await expect( + perspective.table(empty_orders(), { index: "batch" }), + ).rejects.toThrow(/`batch`/); + }); + + test("Accepts an index drawn from the list of a zero-row arrow", async function () { + const table = await perspective.table(empty_orders(), { + index: "orders.id", + }); + + expect(await table.size()).toEqual(0); + await table.update(orders([1, 2], [1.5, 2.5])); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + batch: [1, 1], + "orders.id": [1, 2], + "orders.price": [1.5, 2.5], + }); + + await view.delete(); + await table.delete(); + }); + test("Stringify keeps a sibling index usable on a schema table", async function () { const table = await perspective.table( { id: "integer", y: "string" }, @@ -399,4 +448,191 @@ test.describe("Arrow nested columns", function () { ).rejects.toThrow(/`id`/); }); }); + + test.describe("Map", function () { + test("Pivots a map into dotted columns", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1, 2], int32()), + m: arrow.vectorFromArray( + [{ a: 1, b: 2 }, { b: 3 }], + map(int32()), + ), + }), + ); + + expect(await table.schema()).toEqual({ + id: "integer", + "m.a": "integer", + "m.b": "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + "m.a": [1, null], + "m.b": [2, 3], + }); + + await view.delete(); + await table.delete(); + }); + + test("Empty and null maps yield nulls", async function () { + const table = await perspective.table( + ipc({ + m: arrow.vectorFromArray( + [{ a: 1 }, {}, null], + map(int32()), + ), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "m.a": [1, null, null], + }); + + await view.delete(); + await table.delete(); + }); + + test("Pivots a map of structs", async function () { + const table = await perspective.table( + ipc({ + m: arrow.vectorFromArray( + [{ x: { a: 1, b: 1.5 } }], + map( + struct([ + ["a", int32()], + ["b", float64()], + ]), + ), + ), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "m.x.a": [1], + "m.x.b": [1.5], + }); + + await view.delete(); + await table.delete(); + }); + + test("Explodes a list of maps then pivots", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1], int32()), + m: arrow.vectorFromArray( + [[{ a: 1 }, { b: 2 }]], + list(map(int32())), + ), + }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1], + "m.a": [1, null], + "m.b": [null, 2], + }); + + await view.delete(); + await table.delete(); + }); + }); + + test.describe("Struct stringify", function () { + test("Encodes a struct as JSON text when configured", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ a: 1, b: "x" }, null], + struct([ + ["a", int32()], + ["b", utf8()], + ]), + ), + }), + { struct_flatten: "stringify" }, + ); + + expect(await table.schema()).toEqual({ s: "string" }); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ['{"a":1,"b":"x"}', null], + }); + + await view.delete(); + await table.delete(); + }); + + test("Encodes a map as JSON text when configured", async function () { + const table = await perspective.table( + ipc({ + m: arrow.vectorFromArray( + [{ a: 1, b: 2 }, {}], + map(int32()), + ), + }), + { struct_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + m: ['{"a":1,"b":2}', "{}"], + }); + + await view.delete(); + await table.delete(); + }); + + test("Keeps a list inside a stringified struct", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ a: 1, tags: ["x", "y"] }], + struct([ + ["a", int32()], + ["tags", list(utf8())], + ]), + ), + }), + { struct_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ['{"a":1,"tags":["x","y"]}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Explodes a list of structs then stringifies each", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1], int32()), + l: arrow.vectorFromArray( + [[{ a: 1 }, { a: 2 }]], + list(struct([["a", int32()]])), + ), + }), + { struct_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1], + l: ['{"a":1}', '{"a":2}'], + }); + + await view.delete(); + await table.delete(); + }); + }); }); diff --git a/rust/perspective-js/test/js/constructors/json_nested.spec.ts b/rust/perspective-js/test/js/constructors/json_nested.spec.ts index f4e72a6f77..f7f43e241c 100644 --- a/rust/perspective-js/test/js/constructors/json_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/json_nested.spec.ts @@ -265,6 +265,32 @@ test.describe("JSON nested columns", function () { await table.delete(); }); + test("encodes objects as JSON when configured", async function () { + const { data, options } = shape([ + { id: 1, s: { a: 1, b: "x" } }, + { id: 2, s: null }, + ]); + + const table = await perspective.table(data, { + ...options, + struct_flatten: "stringify", + }); + + expect(await table.schema()).toEqual({ + id: "integer", + s: "string", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 2], + s: ['{"a":1,"b":"x"}', null], + }); + + await view.delete(); + await table.delete(); + }); + test("encodes arrays as JSON when configured", async function () { const { data, options } = shape([ { x: 1, y: [10, 20] }, diff --git a/rust/perspective-js/test/js/constructors/nested_parity.spec.ts b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts index a08ff91f4b..2630a5faf5 100644 --- a/rust/perspective-js/test/js/constructors/nested_parity.spec.ts +++ b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts @@ -23,6 +23,18 @@ const list = (type: arrow.DataType) => new arrow.List(field("item", type)); const struct = (fields: [string, arrow.DataType][]) => new arrow.Struct(fields.map(([n, t]) => field(n, t))); +const utf8 = () => new arrow.Utf8(); +const map = (value: arrow.DataType) => + new arrow.Map_( + field( + "entries", + struct([ + ["key", utf8()], + ["value", value], + ]), + ) as any, + ); + const ipc = (columns: Record) => arrow.tableToIPC(arrow.tableFromArrays(columns as any)); @@ -212,4 +224,38 @@ test.describe("Nested ingest parity", function () { /[Cc]annot zip/, ); }); + + test("map pivot", async function () { + await parity( + "map", + [ + { id: 1, m: { a: 1, b: 2 } }, + { id: 2, m: { b: 3 } }, + ], + { + id: arrow.vectorFromArray([1, 2], int32()), + m: arrow.vectorFromArray( + [{ a: 1, b: 2 }, { b: 3 }], + map(int32()), + ), + }, + ); + }); + + test("struct stringify", async function () { + await parity( + "struct_stringify", + [{ s: { a: 1, b: "x" } }], + { + s: arrow.vectorFromArray( + [{ a: 1, b: "x" }], + struct([ + ["a", int32()], + ["b", utf8()], + ]), + ), + }, + { struct_flatten: "stringify" }, + ); + }); }); diff --git a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py index 13bafebded..f01d55c382 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py +++ b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py @@ -373,6 +373,20 @@ def test_index_on_a_sibling_is_rejected(self): with pytest.raises(psp.PerspectiveError, match=r"`batch`"): Table(self.orders([1, 2], [1.5, 2.5]), index="batch") + def test_index_on_a_sibling_is_rejected_from_a_zero_row_arrow(self): + with pytest.raises(psp.PerspectiveError, match=r"`batch`"): + Table(self.orders([], []).slice(0, 0), index="batch") + + def test_index_from_the_list_is_accepted_from_a_zero_row_arrow(self): + tbl = Table(self.orders([], []).slice(0, 0), index="orders.id") + assert tbl.size() == 0 + tbl.update(arrow_bytes(self.orders([1, 2], [1.5, 2.5]))) + assert tbl.view().to_columns() == { + "batch": [1, 1], + "orders.id": [1, 2], + "orders.price": [1.5, 2.5], + } + def test_index_on_a_list_column_itself_is_allowed(self): data = pa.table( { @@ -581,3 +595,361 @@ def test_flat_arrow_indexed_update(self): ) assert tbl.view().to_columns() == {"a": [1, 2], "b": [10, 99]} + + +class TestTableArrowMap(object): + map_type = pa.map_(pa.string(), pa.int64()) + + def test_map_pivots_to_dotted_columns(self): + data = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "m": pa.array([[("a", 1), ("b", 2)], [("b", 3)]], type=self.map_type), + } + ) + + tbl = Table(data) + assert tbl.schema() == { + "id": "integer", + "m.a": "integer", + "m.b": "integer", + } + + assert tbl.view().to_columns() == { + "id": [1, 2], + "m.a": [1, None], + "m.b": [2, 3], + } + + def test_empty_and_null_maps_yield_nulls(self): + data = pa.table( + {"m": pa.array([[("a", 1)], [], None], type=self.map_type)} + ) + + assert Table(data).view().to_columns() == {"m.a": [1, None, None]} + + def test_null_item_is_a_null_cell(self): + data = pa.table( + {"m": pa.array([[("a", None)], [("a", 2)]], type=self.map_type)} + ) + + assert Table(data).view().to_columns() == {"m.a": [None, 2]} + + def test_duplicate_key_in_a_row_last_wins(self): + data = pa.table( + {"m": pa.array([[("a", 1), ("a", 2)]], type=self.map_type)} + ) + + assert Table(data).view().to_columns() == {"m.a": [2]} + + def test_map_of_struct_pivots_both(self): + item = pa.struct([("a", pa.int64()), ("b", pa.float64())]) + data = pa.table( + { + "m": pa.array( + [[("x", {"a": 1, "b": 1.5})]], + type=pa.map_(pa.string(), item), + ) + } + ) + + assert Table(data).view().to_columns() == {"m.x.a": [1], "m.x.b": [1.5]} + + def test_map_of_list_follows_list_flatten(self): + data = pa.table( + { + "id": pa.array([1], type=pa.int64()), + "m": pa.array( + [[("x", [1, 2])]], + type=pa.map_(pa.string(), pa.list_(pa.int64())), + ), + } + ) + + assert Table(data).view().to_columns() == {"id": [1, 1], "m.x": [1, 2]} + assert Table(data, list_flatten="stringify").view().to_columns() == { + "id": [1], + "m.x": ["[1,2]"], + } + + def test_map_of_lists_zips_as_siblings(self): + item = pa.list_(pa.int64()) + data = pa.table( + { + "m": pa.array( + [[("x", [1, 2]), ("y", [3, 4])], [("x", [5])]], + type=pa.map_(pa.string(), item), + ) + } + ) + + assert Table(data).view().to_columns() == { + "m.x": [1, 2, 5], + "m.y": [3, 4, None], + } + + ragged = pa.table( + { + "m": pa.array( + [[("x", [1, 2]), ("y", [3])]], + type=pa.map_(pa.string(), item), + ) + } + ) + + with pytest.raises(psp.PerspectiveError, match=r"`m\.x`"): + Table(ragged) + + assert Table(ragged, list_flatten="cartesian").view().to_columns() == { + "m.x": [1, 2], + "m.y": [3, 3], + } + + def test_map_of_map_pivots_twice(self): + data = pa.table( + { + "m": pa.array( + [[("x", [("y", 1)])]], + type=pa.map_(pa.string(), self.map_type), + ) + } + ) + + assert Table(data).view().to_columns() == {"m.x.y": [1]} + + def test_map_under_struct(self): + data = pa.table( + { + "s": pa.array( + [{"m": [("a", 1)]}], + type=pa.struct([("m", self.map_type)]), + ) + } + ) + + assert Table(data).view().to_columns() == {"s.m.a": [1]} + + def test_list_of_maps_explodes_then_pivots(self): + data = pa.table( + { + "id": pa.array([1], type=pa.int64()), + "m": pa.array( + [[[("a", 1)], [("b", 2)]]], type=pa.list_(self.map_type) + ), + } + ) + + assert Table(data).view().to_columns() == { + "id": [1, 1], + "m.a": [1, None], + "m.b": [None, 2], + } + + def test_multi_chunk_unions_keys(self): + data = pa.concat_tables( + [ + pa.table({"m": pa.array([[("a", 1)]], type=self.map_type)}), + pa.table({"m": pa.array([[("b", 2)]], type=self.map_type)}), + ] + ) + + assert data.column("m").num_chunks == 2 + assert Table(data).view().to_columns() == { + "m.a": [1, None], + "m.b": [None, 2], + } + + def test_sliced_input(self): + data = pa.table( + {"m": pa.array([[("a", 1)], [("b", 2)], None], type=self.map_type)} + ).slice(1) + + assert Table(data).view().to_columns() == {"m.b": [2, None]} + + def test_dictionary_keys_pivot_like_strings(self): + keys = pa.array(["a", "b", "a"]).dictionary_encode() + items = pa.array([1, 2, 3], type=pa.int64()) + data = pa.table({"m": pa.MapArray.from_arrays([0, 2, 3], keys, items)}) + assert Table(data).view().to_columns() == { + "m.a": [1, 3], + "m.b": [2, None], + } + + def test_non_string_keys_are_rejected(self): + data = pa.table( + {"m": pa.array([[(1, "x")]], type=pa.map_(pa.int32(), pa.string()))} + ) + + with pytest.raises(psp.PerspectiveError, match=r"`m`"): + Table(data) + + def test_schema_table_update_matches_keys(self): + tbl = Table( + {"id": "integer", "m.a": "integer", "m.b": "integer"}, index="id" + ) + + def batch(entries): + return arrow_bytes( + pa.table( + { + "id": pa.array([1], type=pa.int64()), + "m": pa.array([entries], type=self.map_type), + } + ) + ) + + tbl.update(batch([("a", 1), ("b", 2)])) + tbl.update(batch([("a", 10)])) + assert tbl.view().to_columns() == {"id": [1], "m.a": [10], "m.b": [2]} + tbl.update(batch([("c", 5)])) + assert tbl.view().to_columns() == {"id": [1], "m.a": [10], "m.b": [2]} + + def test_columns_follow_first_appearance(self): + data = pa.table( + {"m": pa.array([[("b", 1), ("a", 2)], [("c", 3)]], type=self.map_type)} + ) + + assert Table(data).columns() == ["m.b", "m.a", "m.c"] + + def test_key_colliding_with_a_literal_column_is_rejected(self): + data = pa.table( + { + "m.a": pa.array([1], type=pa.int64()), + "m": pa.array([[("a", 2)]], type=self.map_type), + } + ) + + with pytest.raises(psp.PerspectiveError, match=r"`m\.a`"): + Table(data) + + def test_index_on_a_pivoted_key(self): + data = pa.table( + { + "m": pa.array( + [[("id", 1), ("v", 10)], [("id", 2), ("v", 20)]], + type=self.map_type, + ) + } + ) + + tbl = Table(data, index="m.id") + tbl.update( + arrow_bytes( + pa.table( + {"m": pa.array([[("id", 2), ("v", 99)]], type=self.map_type)} + ) + ) + ) + + assert tbl.view().to_columns() == {"m.id": [1, 2], "m.v": [10, 99]} + + +class TestTableArrowStructStringify(object): + def test_struct_is_json_text(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1, "b": "x"}, None, {"a": None, "b": "y"}], + type=pa.struct([("a", pa.int64()), ("b", pa.string())]), + ) + } + ) + + tbl = Table(data, struct_flatten="stringify") + assert tbl.schema() == {"s": "string"} + assert tbl.view().to_columns() == { + "s": ['{"a":1,"b":"x"}', None, '{"a":null,"b":"y"}'] + } + + def test_map_is_json_text(self): + data = pa.table( + { + "m": pa.array( + [[("a", 1), ("b", 2)], [], None], + type=pa.map_(pa.string(), pa.int64()), + ) + } + ) + + assert Table(data, struct_flatten="stringify").view().to_columns() == { + "m": ['{"a":1,"b":2}', "{}", None] + } + + def test_map_keys_are_coerced_to_text(self): + data = pa.table( + {"m": pa.array([[(1, "x")]], type=pa.map_(pa.int32(), pa.string()))} + ) + + assert Table(data, struct_flatten="stringify").view().to_columns() == { + "m": ['{"1":"x"}'] + } + + def test_outer_struct_wins_over_list_flatten(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1, "tags": ["x", "y"]}], + type=pa.struct( + [("a", pa.int64()), ("tags", pa.list_(pa.string()))] + ), + ) + } + ) + + assert Table(data, struct_flatten="stringify").view().to_columns() == { + "s": ['{"a":1,"tags":["x","y"]}'] + } + + def test_list_of_struct_explodes_then_stringifies(self): + data = pa.table( + { + "id": pa.array([1], type=pa.int64()), + "l": pa.array( + [[{"a": 1}, {"a": 2}]], + type=pa.list_(pa.struct([("a", pa.int64())])), + ), + } + ) + + assert Table(data, struct_flatten="stringify").view().to_columns() == { + "id": [1, 1], + "l": ['{"a":1}', '{"a":2}'], + } + + tbl = Table(data, struct_flatten="stringify", list_flatten="stringify") + assert tbl.view().to_columns() == {"id": [1], "l": ['[{"a":1},{"a":2}]']} + + def test_json_object_is_json_text(self): + tbl = Table([{"s": {"a": 1, "b": "x"}}], struct_flatten="stringify") + assert tbl.schema() == {"s": "string"} + assert tbl.view().to_columns() == {"s": ['{"a":1,"b":"x"}']} + + def test_mode_persists_across_update_on_a_schema_table(self): + tbl = Table({"s": "string"}, struct_flatten="stringify") + tbl.update( + arrow_bytes( + pa.table( + { + "s": pa.array( + [{"a": 1}], type=pa.struct([("a", pa.int64())]) + ) + } + ) + ) + ) + + tbl.update([{"s": {"a": 2}}]) + assert tbl.view().to_columns() == {"s": ['{"a":1}', '{"a":2}']} + + def test_default_pivots(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1}], type=pa.struct([("a", pa.int64())]) + ) + } + ) + + assert Table(data, struct_flatten="pivot").view().to_columns() == { + "s.a": [1] + } diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index 4c8de83c1d..447ddd579d 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -30,7 +30,7 @@ _STATIC = pathlib.Path(__file__).parent / "static" -_TABLE_OPTIONS = ("name", "format", "page_to_disk", "list_flatten") +_TABLE_OPTIONS = ("name", "format", "page_to_disk", "list_flatten", "struct_flatten") __doc__ = """ `PerspectiveWidget` is a JupyterLab widget that implements the same API as @@ -160,8 +160,9 @@ def __init__( - `limit` (`int`): A upper limit on the number of rows in the Table. Cannot be set at the same time as `index`, ignored if `server` is True. - - `name`, `format`, `page_to_disk`, `list_flatten`: `Table` - constructor options, used when `data` is a dataset. + - `name`, `format`, `page_to_disk`, `list_flatten`, + `struct_flatten`: `Table` constructor options, used when `data` + is a dataset. - `kwargs` (`dict`): configuration options for the `PerspectiveViewer`. # Examples diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index 4f409f8201..9dd5e84a78 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use futures::FutureExt; use indexmap::IndexMap; use perspective_client::config::{ColumnType, ViewConfigUpdate}; -use perspective_client::proto::ListFlatten; +use perspective_client::proto::{ListFlatten, StructFlatten}; use perspective_client::{ Client, ColumnWindow, DeleteOptions, DescribeVerdict, OnRemoveData, OnUpdateData, OnUpdateMode, OnUpdateOptions, Table, TableData, TableIndex, TableInitOptions, TableReadFormat, TableRef, @@ -76,6 +76,17 @@ fn parse_list_flatten(value: Option) -> PyResult> { } } +fn parse_struct_flatten(value: Option) -> PyResult> { + match value.as_deref() { + None => Ok(None), + Some("pivot") => Ok(Some(StructFlatten::Pivot)), + Some("stringify") => Ok(Some(StructFlatten::Stringify)), + Some(_x) => Err(PyValueError::new_err( + "Unknown `struct_flatten`".to_string(), + )), + } +} + /// `perspective_server::Server`, whether locally in-memory or remote over some /// transport like a WebSocket. /// @@ -204,7 +215,7 @@ impl AsyncClient { /// table = await client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, schema=None))] + #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, schema=None))] pub async fn table( &self, input: Py, @@ -214,6 +225,7 @@ impl AsyncClient { format: Option>, page_to_disk: Option, list_flatten: Option>, + struct_flatten: Option>, schema: Option>, ) -> PyResult { let client = self.client.clone(); @@ -223,6 +235,7 @@ impl AsyncClient { name: name.map(|x| x.extract::(py)).transpose()?, page_to_disk, list_flatten: parse_list_flatten(list_flatten.map(|x| x.to_string()))?, + struct_flatten: parse_struct_flatten(struct_flatten.map(|x| x.to_string()))?, schema: schema.map(|x| parse_schema(py, x.bind(py))).transpose()?, ..TableInitOptions::default() }; diff --git a/rust/perspective-python/src/client/client_sync.rs b/rust/perspective-python/src/client/client_sync.rs index a17d6b4a83..84ad79085a 100644 --- a/rust/perspective-python/src/client/client_sync.rs +++ b/rust/perspective-python/src/client/client_sync.rs @@ -168,7 +168,7 @@ impl Client { /// table = client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, schema=None))] + #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, schema=None))] pub fn table( &self, py: Python<'_>, @@ -179,6 +179,7 @@ impl Client { format: Option>, page_to_disk: Option, list_flatten: Option>, + struct_flatten: Option>, schema: Option>, ) -> PyResult
{ Ok(Table( @@ -191,6 +192,7 @@ impl Client { format, page_to_disk, list_flatten, + struct_flatten, schema, ) .py_block_on(py)?, diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp index 612a82e2d3..3974b8e7dd 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp @@ -29,6 +29,8 @@ #include #include #include +#include +#include namespace perspective::apachearrow { @@ -197,7 +199,8 @@ convert_type(const std::string& src) { if (src == "null") { return DTYPE_STR; } - if (src == "list") { + if (src == "list" || src == "large_list" || src == "struct" + || src == "map") { return DTYPE_STR; } std::stringstream ss; @@ -208,7 +211,10 @@ convert_type(const std::string& src) { void ArrowLoader::initialize( - const std::uint8_t* ptr, const uint32_t length, t_list_flatten mode + const std::uint8_t* ptr, + const uint32_t length, + t_list_flatten mode, + t_struct_flatten struct_mode ) { if (std::memcmp("ARROW1", (const void*)ptr, 6) == 0) { load_file(ptr, length, m_table); @@ -221,10 +227,10 @@ ArrowLoader::initialize( PSP_COMPLAIN_AND_ABORT(validation.ToString()); } - if (!normalize_table_is_noop(*m_table, mode)) { - m_expanded = normalize_table_expands(*m_table, mode); + if (!normalize_table_is_noop(*m_table, mode, struct_mode)) { + m_expanded = normalize_table_expands(*m_table, mode, struct_mode); m_normalized = std::make_unique( - normalize_table(m_table, mode) + normalize_table(m_table, mode, struct_mode) ); } @@ -385,116 +391,171 @@ iter_col_copy( } } -void -copy_string_list( - std::shared_ptr& list, - const std::shared_ptr& dest, - const int64_t num_rows +/** + * Write element `i` of `array` as JSON, recursing into nested values. + */ +static void +write_json( + rapidjson::Writer& writer, + const arrow::Array& array, + std::int64_t i ) { - const auto typed_data = - std::static_pointer_cast<::arrow::StringArray>(list->values()); - - // the raw bytes of the strings. - const uint8_t* raw_data = typed_data->value_data()->data(); - // col_offsets are the column level offsets, so each element is the boundary - // of a row. - const auto* col_offsets = list->raw_value_offsets(); - // these are the offset locations of the strings within the raw data. - // each element is the boundary of a string. - const auto* string_offsets = typed_data->raw_value_offsets(); - - for (uint32_t i = 0; i < num_rows; i++) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartArray(); - auto row_array_length = col_offsets[i + 1] - col_offsets[i]; - for (uint32_t j = 0; j < row_array_length; j++) { - const auto elem_location = col_offsets[i] + j; - const auto start_loc = string_offsets[elem_location]; - const auto string_len = - string_offsets[elem_location + 1] - start_loc; - writer.String((const char*)raw_data + start_loc, string_len); - } - writer.EndArray(); - dest->set_nth(i, s.GetString()); + if (!array.IsValid(i)) { + writer.Null(); + return; } -} -template -void -copy_integer_list( - std::shared_ptr& list, - const std::shared_ptr& dest, - const int64_t num_rows -) { - const auto typed_data = std::static_pointer_cast(list->values()); - const auto* array_offsets = list->raw_value_offsets(); - const auto* raw_values = typed_data->raw_values(); - for (uint32_t i = 0; i < num_rows; i++) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartArray(); - const auto row_array_length = array_offsets[i + 1] - array_offsets[i]; - for (uint32_t j = 0; j < row_array_length; j++) { - const auto elem_location = array_offsets[i] + j; - const auto elem = raw_values[elem_location]; - if constexpr (std::is_unsigned_v) { - writer.Uint64(elem); + switch (array.type_id()) { + case arrow::Type::BOOL: { + writer.Bool(static_cast(array).Value(i)); + } break; + case arrow::Type::INT8: { + writer.Int64(static_cast(array).Value(i)); + } break; + case arrow::Type::INT16: { + writer.Int64(static_cast(array).Value(i)); + } break; + case arrow::Type::INT32: { + writer.Int64(static_cast(array).Value(i)); + } break; + case arrow::Type::INT64: { + writer.Int64(static_cast(array).Value(i)); + } break; + case arrow::Type::UINT8: { + writer.Uint64(static_cast(array).Value(i)); + } break; + case arrow::Type::UINT16: { + writer.Uint64(static_cast(array).Value(i) + ); + } break; + case arrow::Type::UINT32: { + writer.Uint64(static_cast(array).Value(i) + ); + } break; + case arrow::Type::UINT64: { + writer.Uint64(static_cast(array).Value(i) + ); + } break; + case arrow::Type::HALF_FLOAT: { + const auto bits = + static_cast(array).Value(i); + writer.Double(arrow::util::Float16::FromBits(bits).ToFloat()); + } break; + case arrow::Type::FLOAT: { + const double v = + static_cast(array).Value(i); + if (std::isfinite(v)) { + writer.Double(v); } else { - writer.Int64(elem); + writer.Null(); + } + } break; + case arrow::Type::DOUBLE: { + const double v = + static_cast(array).Value(i); + if (std::isfinite(v)) { + writer.Double(v); + } else { + writer.Null(); + } + } break; + case arrow::Type::STRING: { + const auto view = + static_cast(array).GetView(i); + writer.String( + view.data(), static_cast(view.size()) + ); + } break; + case arrow::Type::LARGE_STRING: { + const auto view = + static_cast(array).GetView(i); + writer.String( + view.data(), static_cast(view.size()) + ); + } break; + case arrow::Type::DICTIONARY: { + const auto& dict = static_cast(array); + write_json(writer, *dict.dictionary(), dict.GetValueIndex(i)); + } break; + case arrow::Type::LIST: { + const auto& list = static_cast(array); + const auto& values = *list.values(); + writer.StartArray(); + for (auto e = list.value_offset(i); e < list.value_offset(i + 1); + ++e) { + write_json(writer, values, e); } - } - writer.EndArray(); - dest->set_nth(i, s.GetString()); - } -} -void -copy_bool_list( - std::shared_ptr& list, - const std::shared_ptr& dest, - const int64_t num_rows -) { - const auto typed_data = - std::static_pointer_cast(list->values()); - const auto* array_offsets = list->raw_value_offsets(); - for (uint32_t i = 0; i < num_rows; i++) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartArray(); - const auto row_array_length = array_offsets[i + 1] - array_offsets[i]; - for (uint32_t j = 0; j < row_array_length; j++) { - const auto elem_location = array_offsets[i] + j; - const auto elem = typed_data->Value(elem_location); - writer.Bool(elem); - } - writer.EndArray(); - dest->set_nth(i, s.GetString()); - } -} + writer.EndArray(); + } break; + case arrow::Type::LARGE_LIST: { + const auto& list = static_cast(array); + const auto& values = *list.values(); + writer.StartArray(); + for (auto e = list.value_offset(i); e < list.value_offset(i + 1); + ++e) { + write_json(writer, values, e); + } -template -void -copy_float_list( - std::shared_ptr& list, - const std::shared_ptr& dest, - const int64_t num_rows -) { - const auto typed_data = std::static_pointer_cast(list->values()); - const auto* array_offsets = list->raw_value_offsets(); - const auto* raw_values = typed_data->raw_values(); - for (uint32_t i = 0; i < num_rows; i++) { - rapidjson::StringBuffer s; - rapidjson::Writer writer(s); - writer.StartArray(); - const auto row_array_length = array_offsets[i + 1] - array_offsets[i]; - for (uint32_t j = 0; j < row_array_length; j++) { - const auto elem_location = array_offsets[i] + j; - const auto elem = raw_values[elem_location]; - writer.Double(elem); + writer.EndArray(); + } break; + case arrow::Type::STRUCT: { + const auto& strct = static_cast(array); + const auto& fields = strct.struct_type()->fields(); + writer.StartObject(); + for (int c = 0; c < strct.num_fields(); ++c) { + const auto& name = fields[c]->name(); + writer.Key( + name.data(), static_cast(name.size()) + ); + + write_json(writer, *strct.field(c), i); + } + + writer.EndObject(); + } break; + case arrow::Type::MAP: { + const auto& map = static_cast(array); + const auto& entries = + static_cast(*map.values()); + const auto& keys = *entries.field(0); + const auto& items = *entries.field(1); + writer.StartObject(); + for (auto e = map.value_offset(i); e < map.value_offset(i + 1); + ++e) { + if (keys.type_id() == arrow::Type::STRING) { + const auto view = + static_cast(keys).GetView(e); + writer.Key( + view.data(), + static_cast(view.size()) + ); + } else if (keys.type_id() == arrow::Type::LARGE_STRING) { + const auto view = + static_cast(keys) + .GetView(e); + writer.Key( + view.data(), + static_cast(view.size()) + ); + } else { + rapidjson::StringBuffer key; + rapidjson::Writer key_writer(key); + write_json(key_writer, keys, e); + writer.Key(key.GetString(), key.GetSize()); + } + + write_json(writer, items, e); + } + + writer.EndObject(); + } break; + default: { + std::stringstream ss; + ss << "Cannot stringify Arrow value of type `" + << array.type()->ToString() << "`\n"; + PSP_COMPLAIN_AND_ABORT(ss.str()); } - writer.EndArray(); - dest->set_nth(i, s.GetString()); } } @@ -508,61 +569,25 @@ copy_array_impl( const GATHER& gather ) { switch (src->type()->id()) { - case arrow::ListType::type_id: { - if constexpr (!GATHER::is_identity) { - PSP_COMPLAIN_AND_ABORT( - "Cannot expand rows of a stringified list column\n" - ); - } + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: + case arrow::Type::STRUCT: + case arrow::Type::MAP: { + rapidjson::StringBuffer buffer; + for (std::int64_t i = 0; i < len; ++i) { + if (gather.is_null(i)) { + continue; + } - auto list = std::static_pointer_cast<::arrow::ListArray>(src); + const auto row = gather[i]; + if (!src->IsValid(row)) { + continue; + } - switch (list->value_type()->id()) { - case arrow::BooleanType::type_id: { - copy_bool_list(list, dest, len); - } break; - case arrow::HalfFloatType::type_id: { - copy_float_list(list, dest, len); - } break; - case arrow::FloatType::type_id: { - copy_float_list(list, dest, len); - } break; - case arrow::DoubleType::type_id: { - copy_float_list(list, dest, len); - } break; - case ::arrow::Int8Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::Int16Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::Int32Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::Int64Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::UInt8Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::UInt16Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::UInt32Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::UInt64Type::type_id: { - copy_integer_list(list, dest, len); - } break; - case ::arrow::StringType::type_id: { - copy_string_list(list, dest, len); - } break; - default: - auto val = list->value_type(); - std::stringstream ss; - ss << "Given Unsupported type '" << *val << "'.\n"; - auto cc = ss.str(); - PSP_COMPLAIN_AND_ABORT(cc); + buffer.Clear(); + rapidjson::Writer writer(buffer); + write_json(writer, *src, row); + dest->set_nth(offset + i, buffer.GetString()); } } break; case arrow::DictionaryType::type_id: { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp index fb9cf11611..325134ff5a 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp @@ -20,11 +20,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #include namespace perspective::apachearrow { @@ -39,19 +42,35 @@ namespace { } bool - type_expands(const arrow::DataType& type) { + is_pivot_id(arrow::Type::type id) { + return id == arrow::Type::STRUCT || id == arrow::Type::MAP; + } + + bool + type_expands(const arrow::DataType& type, t_struct_flatten struct_mode) { if (is_list_id(type.id())) { return true; } + if (struct_mode == STRUCT_FLATTEN_STRINGIFY) { + return false; + } + if (type.id() == arrow::Type::STRUCT) { for (const auto& child : type.fields()) { - if (type_expands(*child->type())) { + if (type_expands(*child->type(), struct_mode)) { return true; } } } + if (type.id() == arrow::Type::MAP) { + return type_expands( + *static_cast(type).item_type(), + struct_mode + ); + } + return false; } @@ -190,6 +209,241 @@ namespace { return true; } + bool + is_string_id(arrow::Type::type id) { + return id == arrow::Type::STRING || id == arrow::Type::LARGE_STRING; + } + + std::string_view + string_at(const arrow::Array& array, std::int64_t i) { + if (array.type_id() == arrow::Type::LARGE_STRING) { + const auto view = + static_cast(array).GetView(i); + return {view.data(), view.size()}; + } + + const auto view = + static_cast(array).GetView(i); + return {view.data(), view.size()}; + } + + /** + * The distinct keys of a map column in first-appearance order. + */ + struct t_key_table { + std::unordered_map ordinals; + std::deque names; + + std::size_t + ordinal(std::string_view key) { + const auto it = ordinals.find(key); + if (it != ordinals.end()) { + return it->second; + } + + names.emplace_back(key); + const auto ord = names.size() - 1; + ordinals.emplace(std::string_view{names.back()}, ord); + return ord; + } + }; + + /** + * One chunk of a map column, addressed by global row and global entry. + */ + struct t_map_chunk { + std::shared_ptr map; + std::shared_ptr keys; + std::int64_t first; + std::int64_t item_base; + std::vector dict_ordinals; + + std::size_t + ordinal(t_key_table& table, std::int64_t e) { + if (keys->type_id() != arrow::Type::DICTIONARY) { + return table.ordinal(string_at(*keys, e)); + } + + const auto& dict = static_cast(*keys); + const auto di = dict.GetValueIndex(e); + auto& cached = dict_ordinals[di]; + if (cached < 0) { + cached = static_cast( + table.ordinal(string_at(*dict.dictionary(), di)) + ); + } + + return static_cast(cached); + } + }; + + void + check_map_key_type( + const std::string& name, const arrow::DataType& key_type + ) { + auto id = key_type.id(); + if (id == arrow::Type::DICTIONARY) { + id = static_cast(key_type) + .value_type() + ->id(); + } + + if (!is_string_id(id)) { + std::stringstream ss; + ss << "Cannot pivot map column `" << name << "` with key type `" + << key_type.ToString() << "`; only string keys are supported\n"; + abort_with(ss.str()); + } + } + + /** + * Collapse to a single chunk, so a gather index needs no chunk resolution. + */ + std::shared_ptr + combine_chunks(const std::shared_ptr& column) { + if (column->num_chunks() <= 1) { + return column; + } + + auto combined = unwrap( + arrow::Concatenate(column->chunks(), arrow::default_memory_pool()), + "Could not combine chunks of an expanded column" + ); + + return std::make_shared( + arrow::ArrayVector{std::move(combined)}, column->type() + ); + } + + /** + * Pivot every top-level map column into one column per distinct key, each + * a gather into the map's items. Returns whether anything changed. + */ + bool + pivot_maps( + std::vector>& fields, + std::vector>& columns, + std::vector>& gathers, + std::vector& per_element, + std::int64_t num_rows + ) { + bool has_map = false; + for (const auto& field : fields) { + if (field->type()->id() == arrow::Type::MAP) { + has_map = true; + break; + } + } + + if (!has_map) { + return false; + } + + std::vector> out_fields; + std::vector> out_columns; + std::vector> out_gathers; + std::vector out_per_element; + for (std::size_t i = 0; i < fields.size(); ++i) { + if (fields[i]->type()->id() != arrow::Type::MAP) { + out_fields.push_back(fields[i]); + out_columns.push_back(columns[i]); + out_gathers.push_back(std::move(gathers[i])); + out_per_element.push_back(per_element[i]); + continue; + } + + const auto& map_type = + static_cast(*fields[i]->type()); + check_map_key_type(fields[i]->name(), *map_type.key_type()); + + std::vector chunks; + std::vector row_base; + arrow::ArrayVector item_slices; + std::int64_t rows = 0; + std::int64_t items_total = 0; + for (const auto& chunk : columns[i]->chunks()) { + auto map = std::static_pointer_cast(chunk); + auto entries = + std::static_pointer_cast(map->values() + ); + + const auto* offsets = map->raw_value_offsets(); + const auto len = map->length(); + const auto first = len > 0 ? offsets[0] : 0; + const auto last = len > 0 ? offsets[len] : 0; + t_map_chunk mc{map, entries->field(0), first, items_total, {}}; + if (mc.keys->type_id() == arrow::Type::DICTIONARY) { + mc.dict_ordinals.assign( + static_cast(*mc.keys) + .dictionary() + ->length(), + -1 + ); + } + + item_slices.push_back(entries->field(1)->Slice(first, last - first) + ); + + row_base.push_back(rows); + rows += len; + items_total += last - first; + chunks.push_back(std::move(mc)); + } + + t_key_table keys; + std::vector> cells; + const auto& parent = gathers[i]; + for (std::int64_t r = 0; r < num_rows; ++r) { + const auto src = parent.empty() ? r : parent[r]; + if (src < 0) { + continue; + } + + const auto c = static_cast( + std::upper_bound(row_base.begin(), row_base.end(), src) + - row_base.begin() - 1 + ); + + auto& chunk = chunks[c]; + const auto local = src - row_base[c]; + if (chunk.map->IsNull(local)) { + continue; + } + + const auto* offsets = chunk.map->raw_value_offsets(); + for (auto e = offsets[local]; e < offsets[local + 1]; ++e) { + const auto ord = chunk.ordinal(keys, e); + if (ord == cells.size()) { + cells.emplace_back(num_rows, -1); + } + + cells[ord][r] = chunk.item_base + (e - chunk.first); + } + } + + auto items = combine_chunks(std::make_shared( + std::move(item_slices), map_type.item_type() + )); + + for (std::size_t ord = 0; ord < keys.names.size(); ++ord) { + out_fields.push_back(arrow::field( + fields[i]->name() + FLATTEN_SEPARATOR + keys.names[ord], + map_type.item_type() + )); + + out_columns.push_back(items); + out_gathers.push_back(std::move(cells[ord])); + out_per_element.push_back(per_element[i]); + } + } + + fields = std::move(out_fields); + columns = std::move(out_columns); + gathers = std::move(out_gathers); + per_element = std::move(out_per_element); + return true; + } + /** * The per-row geometry of one list column, flattened across chunks so that * rows can be addressed globally. Chunk layout is per-column in an @@ -303,25 +557,6 @@ namespace { } } - /** - * Collapse to a single chunk, so a gather index needs no chunk resolution. - */ - std::shared_ptr - combine_chunks(const std::shared_ptr& column) { - if (column->num_chunks() <= 1) { - return column; - } - - auto combined = unwrap( - arrow::Concatenate(column->chunks(), arrow::default_memory_pool()), - "Could not combine chunks of an expanded column" - ); - - return std::make_shared( - arrow::ArrayVector{std::move(combined)}, column->type() - ); - } - /** * Expand every top-level list column into rows. Returns whether anything * changed. @@ -515,10 +750,14 @@ namespace { } // namespace bool -normalize_table_is_noop(const arrow::Table& input, t_list_flatten mode) { +normalize_table_is_noop( + const arrow::Table& input, + t_list_flatten mode, + t_struct_flatten struct_mode +) { for (const auto& field : input.schema()->fields()) { const auto id = field->type()->id(); - if (id == arrow::Type::STRUCT) { + if (struct_mode != STRUCT_FLATTEN_STRINGIFY && is_pivot_id(id)) { return false; } @@ -531,13 +770,17 @@ normalize_table_is_noop(const arrow::Table& input, t_list_flatten mode) { } bool -normalize_table_expands(const arrow::Table& input, t_list_flatten mode) { +normalize_table_expands( + const arrow::Table& input, + t_list_flatten mode, + t_struct_flatten struct_mode +) { if (mode == LIST_FLATTEN_STRINGIFY) { return false; } for (const auto& field : input.schema()->fields()) { - if (type_expands(*field->type())) { + if (type_expands(*field->type(), struct_mode)) { return true; } } @@ -546,7 +789,11 @@ normalize_table_expands(const arrow::Table& input, t_list_flatten mode) { } t_normalized_table -normalize_table(std::shared_ptr input, t_list_flatten mode) { +normalize_table( + std::shared_ptr input, + t_list_flatten mode, + t_struct_flatten struct_mode +) { t_normalized_table out; out.fields = input->schema()->fields(); out.columns = input->columns(); @@ -554,9 +801,22 @@ normalize_table(std::shared_ptr input, t_list_flatten mode) { out.per_element.assign(out.fields.size(), false); out.num_rows = input->num_rows(); while (true) { - bool changed = flatten_structs( - out.fields, out.columns, out.gathers, out.per_element - ); + bool changed = false; + if (struct_mode != STRUCT_FLATTEN_STRINGIFY) { + changed = flatten_structs( + out.fields, out.columns, out.gathers, out.per_element + ); + + if (pivot_maps( + out.fields, + out.columns, + out.gathers, + out.per_element, + out.num_rows + )) { + changed = true; + } + } if (mode != LIST_FLATTEN_STRINGIFY && explode_lists( diff --git a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp index bc6040b7f3..2d14309435 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp @@ -110,11 +110,9 @@ rapidjson_type_to_dtype(const rapidjson::Value& value) { case rapidjson::kNullType: return t_dtype::DTYPE_NONE; case rapidjson::kArrayType: + case rapidjson::kObjectType: // Only reachable under `stringify`; the expanding modes descend. return t_dtype::DTYPE_STR; - case rapidjson::kObjectType: - PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); - return t_dtype::DTYPE_NONE; default: PSP_COMPLAIN_AND_ABORT("Unknown JSON type"); return t_dtype::DTYPE_NONE; @@ -179,7 +177,6 @@ json_into(const rapidjson::Value& value) { case rapidjson::kTrueType: return "true"; case rapidjson::kObjectType: - PSP_COMPLAIN_AND_ABORT("Cannot coerce object to string"); case rapidjson::kArrayType: { // `stringify` keeps the array as its JSON text. rapidjson::StringBuffer buffer; @@ -433,11 +430,11 @@ static void for_each_leaf( std::string& prefix, const rapidjson::Value& value, - t_list_flatten mode, + const t_flatten_mode& mode, F&& fn, std::size_t array_path = NO_ARRAY ) { - if (mode != LIST_FLATTEN_STRINGIFY && value.IsArray()) { + if (mode.list != LIST_FLATTEN_STRINGIFY && value.IsArray()) { // Inference only needs one element to learn the leaves' types; any // path a later element introduces is grown into by `resolve_column`. if (!value.Empty()) { @@ -447,7 +444,7 @@ for_each_leaf( return; } - if (!value.IsObject()) { + if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { fn(static_cast(prefix), value, array_path); return; } @@ -507,8 +504,8 @@ check_row_count(std::uint64_t rows) { * fixpoint the Arrow path reaches by re-running its pass. */ static t_uindex -leaf_width(const rapidjson::Value& value, t_list_flatten mode) { - if (mode == LIST_FLATTEN_STRINGIFY || value.IsNull()) { +leaf_width(const rapidjson::Value& value, const t_flatten_mode& mode) { + if (mode.list == LIST_FLATTEN_STRINGIFY || value.IsNull()) { return 1; } @@ -522,7 +519,7 @@ leaf_width(const rapidjson::Value& value, t_list_flatten mode) { return std::max(total, 1); } - if (!value.IsObject()) { + if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { return 1; } @@ -530,7 +527,7 @@ leaf_width(const rapidjson::Value& value, t_list_flatten mode) { const char* witness = nullptr; for (const auto& child : value.GetObj()) { const auto child_width = leaf_width(child.value, mode); - if (mode == LIST_FLATTEN_CARTESIAN) { + if (mode.list == LIST_FLATTEN_CARTESIAN) { width *= child_width; continue; } @@ -561,9 +558,9 @@ leaf_width(const rapidjson::Value& value, t_list_flatten mode) { * named by its key, with no path string and no width arithmetic. */ static bool -is_nested(const rapidjson::Value& value, t_list_flatten mode) { - return value.IsObject() - || (mode != LIST_FLATTEN_STRINGIFY && value.IsArray()); +is_nested(const rapidjson::Value& value, const t_flatten_mode& mode) { + return (value.IsObject() && mode.structs != STRUCT_FLATTEN_STRINGIFY) + || (mode.list != LIST_FLATTEN_STRINGIFY && value.IsArray()); } /** @@ -616,10 +613,10 @@ emit_leaves( const rapidjson::Value& value, t_uindex k, t_uindex width, - t_list_flatten mode, + const t_flatten_mode& mode, F&& fn ) { - if (mode == LIST_FLATTEN_STRINGIFY || value.IsNull()) { + if (mode.list == LIST_FLATTEN_STRINGIFY || value.IsNull()) { fn(std::string_view{prefix}, value); return; } @@ -651,7 +648,7 @@ emit_leaves( return; } - if (!value.IsObject()) { + if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { fn(std::string_view{prefix}, value); return; } @@ -660,7 +657,7 @@ emit_leaves( const auto len = prefix.size(); for (const auto& child : value.GetObj()) { const auto child_width = leaf_width(child.value, mode); - const auto child_k = child_slot(k, child_width, stride, mode); + const auto child_k = child_slot(k, child_width, stride, mode.list); prefix += FLATTEN_SEPARATOR; prefix += child.name.GetString(); emit_leaves(prefix, child.value, child_k, child_width, mode, fn); @@ -724,10 +721,11 @@ JsonLoader::init( t_json_format format, const std::vector& index, const t_schema* existing, - t_list_flatten mode + t_list_flatten mode, + t_struct_flatten struct_mode ) { m_format = format; - m_mode = mode; + m_mode = t_flatten_mode{mode, struct_mode}; m_index = index; if (format == JSON_FORMAT_NDJSON) { m_stream = rapidjson::StringStream(data.data()); @@ -874,7 +872,7 @@ infer_record( std::set& descended, std::set& per_element, std::set& expansions, - t_list_flatten mode + const t_flatten_mode& mode ) { std::string path; for (const auto& col : record.GetObj()) { @@ -1045,8 +1043,9 @@ JsonLoader::infer_cols(const std::vector& index) { for (const auto& cell : it.value.GetArray()) { // Whether this cell contributes the column itself rather than // paths beneath it -- the same test `for_each_leaf` applies. - const bool is_leaf = !cell.IsObject() - && (m_mode == LIST_FLATTEN_STRINGIFY || !cell.IsArray()); + const bool is_leaf = + (!cell.IsObject() || m_mode.structs == STRUCT_FLATTEN_STRINGIFY) + && (m_mode.list == LIST_FLATTEN_STRINGIFY || !cell.IsArray()); saw_leaf = saw_leaf || is_leaf; path.assign(it.name.GetString()); @@ -1350,7 +1349,7 @@ JsonLoader::fill_rows( std::size_t ci = 0; for (const auto& it : row.GetObj()) { const auto child_width = m_child_widths[ci++]; - const auto sub = child_slot(k, child_width, stride, m_mode); + const auto sub = child_slot(k, child_width, stride, m_mode.list); if (is_update && std::string_view{it.name.GetString()} == "__INDEX__") { fill_pkey_cell(tbl, psp_pkey_col, ii, it.value, is_update); @@ -1501,7 +1500,7 @@ JsonLoader::fill_cols( const std::string* witness = nullptr; for (std::size_t c = 0; c < cells.size(); ++c) { const auto w = widths[c][r]; - if (m_mode == LIST_FLATTEN_CARTESIAN) { + if (m_mode.list == LIST_FLATTEN_CARTESIAN) { width *= w; continue; } @@ -1550,7 +1549,7 @@ JsonLoader::fill_cols( } t_uindex sub = 0; - if (m_mode == LIST_FLATTEN_CARTESIAN) { + if (m_mode.list == LIST_FLATTEN_CARTESIAN) { t_uindex stride = 1; for (std::size_t d = c + 1; d < cells.size(); ++d) { stride *= widths[d][r]; @@ -1687,7 +1686,7 @@ JsonLoader::fill_ndjson( std::size_t ci = 0; for (const auto& it : m_document.GetObj()) { const auto child_width = m_child_widths[ci++]; - const auto sub = child_slot(k, child_width, stride, m_mode); + const auto sub = child_slot(k, child_width, stride, m_mode.list); if (is_update && std::string_view{it.name.GetString()} == "__INDEX__") { fill_pkey_cell(tbl, psp_pkey_col, ii, it.value, is_update); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index 40b4c2dc35..b22e937d0f 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -2586,6 +2586,17 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { break; } + apachearrow::t_struct_flatten struct_flatten; + switch (r.options().struct_flatten()) { + case proto::STRUCT_FLATTEN_STRINGIFY: + struct_flatten = apachearrow::STRUCT_FLATTEN_STRINGIFY; + break; + case proto::STRUCT_FLATTEN_PIVOT: + default: + struct_flatten = apachearrow::STRUCT_FLATTEN_PIVOT; + break; + } + if (r.options().has_view_schema() && r.data().data_case() != proto::MakeTableData::kFromView) { proto::Response resp; @@ -2616,7 +2627,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten + list_flatten, + struct_flatten ); break; } @@ -2629,7 +2641,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten + list_flatten, + struct_flatten ); break; } @@ -2642,7 +2655,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten + list_flatten, + struct_flatten ); break; } @@ -2655,7 +2669,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten + list_flatten, + struct_flatten ); break; } @@ -2668,7 +2683,8 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten + list_flatten, + struct_flatten ); break; } @@ -2683,7 +2699,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { t_schema table_schema(columns, types); table = Table::from_schema( - index, table_schema, limit, backing_store, list_flatten + index, + table_schema, + limit, + backing_store, + list_flatten, + struct_flatten ); break; } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp index 85acd9fce8..4b76550e44 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp @@ -44,7 +44,8 @@ Table::Table( std::uint32_t limit, std::vector index, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) : m_init(false), m_id(GLOBAL_TABLE_ID++), @@ -56,7 +57,8 @@ Table::Table( m_index(std::move(index)), m_gnode_set(false), m_backing_store(backing_store), - m_list_flatten(list_flatten) { + m_list_flatten(list_flatten), + m_struct_flatten(struct_flatten) { validate_columns(m_column_names); if (is_composite_index()) { @@ -548,7 +550,8 @@ Table::from_csv( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { auto map = std::unordered_map>(); @@ -600,7 +603,14 @@ Table::from_csv( auto pool = std::make_shared(); pool->init(); auto tbl = std::make_shared
( - pool, column_names, data_types, limit, index, backing_store, list_flatten + pool, + column_names, + data_types, + limit, + index, + backing_store, + list_flatten, + struct_flatten ); // `psp_pkey` is guaranteed unique only when the index is implicit (a @@ -746,7 +756,8 @@ Table::remove_arrow(const std::string_view& data) { arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - m_list_flatten + m_list_flatten, + m_struct_flatten ); const auto names = arrow_loader.names(); @@ -780,7 +791,8 @@ Table::from_json_loader( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { if (const auto repeated = loader.repeated_index(index)) { std::stringstream ss; @@ -831,7 +843,8 @@ Table::from_json_loader( limit, index, backing_store, - list_flatten + list_flatten, + struct_flatten ); tbl->init(*data_table, nrows, t_op::OP_INSERT, 0); @@ -848,7 +861,9 @@ Table::update_json( ) { t_schema table_schema = get_schema(); json::JsonLoader loader; - loader.init(data, format, m_index, &table_schema, m_list_flatten); + loader.init( + data, format, m_index, &table_schema, m_list_flatten, m_struct_flatten + ); if (loader.empty()) { return; } @@ -883,14 +898,26 @@ Table::from_cols( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { json::JsonLoader loader; loader.init( - data, json::JSON_FORMAT_COLUMNS, index, nullptr, list_flatten + data, + json::JSON_FORMAT_COLUMNS, + index, + nullptr, + list_flatten, + struct_flatten ); return from_json_loader( - loader, index, std::move(data), limit, backing_store, list_flatten + loader, + index, + std::move(data), + limit, + backing_store, + list_flatten, + struct_flatten ); } @@ -911,14 +938,26 @@ Table::from_rows( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { json::JsonLoader loader; loader.init( - data, json::JSON_FORMAT_ROWS, index, nullptr, list_flatten + data, + json::JSON_FORMAT_ROWS, + index, + nullptr, + list_flatten, + struct_flatten ); return from_json_loader( - loader, index, std::move(data), limit, backing_store, list_flatten + loader, + index, + std::move(data), + limit, + backing_store, + list_flatten, + struct_flatten ); } @@ -933,14 +972,26 @@ Table::from_ndjson( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { json::JsonLoader loader; loader.init( - data, json::JSON_FORMAT_NDJSON, index, nullptr, list_flatten + data, + json::JSON_FORMAT_NDJSON, + index, + nullptr, + list_flatten, + struct_flatten ); return from_json_loader( - loader, index, std::move(data), limit, backing_store, list_flatten + loader, + index, + std::move(data), + limit, + backing_store, + list_flatten, + struct_flatten ); } @@ -950,7 +1001,8 @@ Table::from_schema( const t_schema& schema, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { auto pool = std::make_shared(); pool->init(); @@ -985,7 +1037,8 @@ Table::from_schema( limit, index, backing_store, - list_flatten + list_flatten, + struct_flatten ); tbl->init(data_table, 0, t_op::OP_INSERT, 0); @@ -1038,7 +1091,8 @@ Table::update_arrow(const std::string_view& data, std::uint32_t port_id) { arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - m_list_flatten + m_list_flatten, + m_struct_flatten ); if (const auto repeated = arrow_loader.repeated_index(m_index)) { @@ -1096,7 +1150,8 @@ Table::from_arrow( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { apachearrow::ArrowLoader arrow_loader; @@ -1104,7 +1159,8 @@ Table::from_arrow( arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - list_flatten + list_flatten, + struct_flatten ); if (const auto repeated = arrow_loader.repeated_index(index)) { @@ -1160,7 +1216,14 @@ Table::from_arrow( auto pool = std::make_shared(); pool->init(); auto table = std::make_shared
( - pool, columns, types, limit, index, backing_store, list_flatten + pool, + columns, + types, + limit, + index, + backing_store, + list_flatten, + struct_flatten ); table->init(*data_table, data_table->num_rows(), t_op::OP_INSERT, 0); @@ -1177,7 +1240,8 @@ Table::make_table( const std::vector& index, const std::string_view& data, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ) { auto pool = std::make_shared(); pool->init(); @@ -1202,7 +1266,14 @@ Table::make_table( auto columns = data_table.get_schema().columns(); auto dtypes = data_table.get_schema().types(); auto table = std::make_shared
( - pool, columns, dtypes, limit, index, backing_store, list_flatten + pool, + columns, + dtypes, + limit, + index, + backing_store, + list_flatten, + struct_flatten ); table->init(data_table, data_table.num_rows(), t_op::OP_INSERT, 0); pool->_process(); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h index 5845e17665..93f67c9653 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h @@ -45,11 +45,13 @@ namespace apachearrow { * * @param ptr * @param mode - how `LIST` columns are ingested. + * @param struct_mode - how `STRUCT` and `MAP` columns are ingested. */ void initialize( const std::uint8_t* ptr, std::uint32_t, - t_list_flatten mode = LIST_FLATTEN_ZIP + t_list_flatten mode = LIST_FLATTEN_ZIP, + t_struct_flatten struct_mode = STRUCT_FLATTEN_PIVOT ); /** diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h index 4532c90505..f3dc6a49e5 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h @@ -24,9 +24,12 @@ namespace perspective { namespace apachearrow { using perspective::t_list_flatten; + using perspective::t_struct_flatten; using perspective::LIST_FLATTEN_CARTESIAN; using perspective::LIST_FLATTEN_STRINGIFY; using perspective::LIST_FLATTEN_ZIP; + using perspective::STRUCT_FLATTEN_PIVOT; + using perspective::STRUCT_FLATTEN_STRINGIFY; extern const char* const FLATTEN_SEPARATOR; /** @@ -53,14 +56,22 @@ namespace apachearrow { }; PERSPECTIVE_EXPORT t_normalized_table normalize_table( - std::shared_ptr input, t_list_flatten mode + std::shared_ptr input, + t_list_flatten mode, + t_struct_flatten struct_mode ); - PERSPECTIVE_EXPORT bool - normalize_table_is_noop(const arrow::Table& input, t_list_flatten mode); + PERSPECTIVE_EXPORT bool normalize_table_is_noop( + const arrow::Table& input, + t_list_flatten mode, + t_struct_flatten struct_mode + ); - PERSPECTIVE_EXPORT bool - normalize_table_expands(const arrow::Table& input, t_list_flatten mode); + PERSPECTIVE_EXPORT bool normalize_table_expands( + const arrow::Table& input, + t_list_flatten mode, + t_struct_flatten struct_mode + ); } // namespace apachearrow } // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h index d3b2452e5f..7a0e264e52 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h @@ -20,4 +20,14 @@ enum t_list_flatten { LIST_FLATTEN_STRINGIFY = 2 }; +enum t_struct_flatten { + STRUCT_FLATTEN_PIVOT = 0, + STRUCT_FLATTEN_STRINGIFY = 1 +}; + +struct t_flatten_mode { + t_list_flatten list{LIST_FLATTEN_ZIP}; + t_struct_flatten structs{STRUCT_FLATTEN_PIVOT}; +}; + } // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h index 362ee4ee19..78ae8da9a4 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h @@ -63,7 +63,8 @@ namespace json { t_json_format format, const std::vector& index, const t_schema* existing, - t_list_flatten mode + t_list_flatten mode, + t_struct_flatten struct_mode = STRUCT_FLATTEN_PIVOT ); const std::vector& names() const; @@ -135,7 +136,7 @@ namespace json { t_json_format m_format{JSON_FORMAT_ROWS}; std::vector m_names; std::vector m_types; - t_list_flatten m_mode{LIST_FLATTEN_ZIP}; + t_flatten_mode m_mode; std::set m_per_element; std::set m_expansions; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h index 63f7f9e32b..a0daba13f1 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h @@ -60,7 +60,9 @@ class PERSPECTIVE_EXPORT Table { std::vector index, t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); /** @@ -252,7 +254,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); static std::shared_ptr
from_cols( @@ -261,7 +265,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); static std::shared_ptr
from_rows( @@ -270,7 +276,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); static std::shared_ptr
from_ndjson( @@ -279,7 +287,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); static std::shared_ptr
from_schema( @@ -288,7 +298,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); /** @@ -311,7 +323,9 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); static std::shared_ptr
make_table( @@ -322,7 +336,9 @@ class PERSPECTIVE_EXPORT Table { const std::string_view& data, t_backing_store backing_store = BACKING_STORE_MEMORY, apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP + apachearrow::LIST_FLATTEN_ZIP, + apachearrow::t_struct_flatten struct_flatten = + apachearrow::STRUCT_FLATTEN_PIVOT ); private: @@ -336,7 +352,8 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten + apachearrow::t_list_flatten list_flatten, + apachearrow::t_struct_flatten struct_flatten ); /** @@ -406,6 +423,12 @@ class PERSPECTIVE_EXPORT Table { * different column shape than the schema it is written against. */ const apachearrow::t_list_flatten m_list_flatten; + + /** + * @brief How `arrow::Type::STRUCT` and `arrow::Type::MAP` columns are + * ingested, held for the same reason as `m_list_flatten`. + */ + const apachearrow::t_struct_flatten m_struct_flatten; }; } // namespace perspective \ No newline at end of file