From 866a1c4f1f16043b10e7793fb1ba2d704672533b Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 4 Nov 2026 12:17:16 -0500 Subject: [PATCH 1/2] Add `map_flatten` fallback to `struct_flatten` Signed-off-by: Andrew Stein --- .github/workflows/build.yaml | 2 +- docs/md/how_to/javascript/loading_data.md | 24 +-- docs/md/how_to/python/table_data.md | 30 +++- rust/perspective-client/build.rs | 4 + rust/perspective-client/perspective.proto | 8 + rust/perspective-client/src/rust/client.rs | 2 + rust/perspective-client/src/rust/table.rs | 21 ++- .../test/js/constructors/arrow_nested.spec.ts | 148 ++++++++++++++++++ .../test/js/constructors/json_nested.spec.ts | 15 ++ .../js/constructors/nested_parity.spec.ts | 9 ++ .../tests/table/test_table_arrow_nested.py | 104 ++++++++++++ .../tests/widget/test_anywidget_smoke.py | 13 ++ .../perspective/widget/__init__.py | 13 +- .../src/client/client_async.rs | 15 +- .../src/client/client_sync.rs | 4 +- .../cpp/perspective/src/cpp/arrow_loader.cpp | 15 +- .../perspective/src/cpp/arrow_normalize.cpp | 83 +++++----- .../cpp/perspective/src/cpp/json_loader.cpp | 7 +- .../cpp/perspective/src/cpp/server.cpp | 47 +++--- .../cpp/perspective/src/cpp/table.cpp | 76 +++------ .../src/include/perspective/arrow_loader.h | 8 +- .../src/include/perspective/arrow_normalize.h | 25 ++- .../src/include/perspective/flatten_mode.h | 7 + .../src/include/perspective/json_loader.h | 3 +- .../src/include/perspective/table.h | 53 ++----- 25 files changed, 514 insertions(+), 222 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 75acf571cf..051236bfdd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -721,7 +721,7 @@ jobs: - name: Package if: ${{ steps.config-step.outputs.FULL_RUN }} - run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-migrate -p perspective-python + run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-python - uses: actions/upload-artifact@v4 if: ${{ steps.config-step.outputs.FULL_RUN }} diff --git a/docs/md/how_to/javascript/loading_data.md b/docs/md/how_to/javascript/loading_data.md index a6acdc5f6c..2113407e58 100644 --- a/docs/md/how_to/javascript/loading_data.md +++ b/docs/md/how_to/javascript/loading_data.md @@ -78,14 +78,18 @@ it is over a websocket. ## Nested columns Perspective's data model is flat, so nested Arrow and JSON values are -normalized on ingest, controlled by two `Table` options. +normalized on ingest, controlled by three `Table` options. -`struct_flatten` applies to Arrow `struct` and `map` columns and to JSON -objects. `"pivot"` (the default) hoists each field or key into its own dotted -column, so `{ s: { a: 1 } }` becomes a column `s.a` and a map `m` becomes one +`struct_flatten` applies to Arrow `struct` columns and to JSON objects. +`"pivot"` (the default) hoists each field into its own dotted column, so +`{ s: { a: 1 } }` becomes a column `s.a`. `"stringify"` keeps the whole value +as JSON text in a single string column. + +`map_flatten` applies to Arrow `map` columns. `"pivot"` makes a map `m` one column `m.` per distinct key in the batch, with a null where a row lacks -the key. `"stringify"` keeps the whole value as JSON text in a single string -column. +the key, and `"stringify"` keeps the map as JSON text. When unset it follows +`struct_flatten`. JSON has no map type, so JSON objects always follow +`struct_flatten`. `list_flatten` applies to Arrow `list` columns and JSON arrays. `"zip"` (the default) expands a row into one row per element, repeating its siblings, and @@ -93,9 +97,11 @@ requires sibling lists in a row to agree in length. `"cartesian"` expands to the product of the lists' lengths. `"stringify"` keeps each list as a JSON array in a string column. -The two options compose from the outside in: a list inside a stringified -struct stays inside its JSON text, while a struct inside an expanded list is -stringified once per element row. +The options compose from the outside in: a list inside a stringified struct +stays inside its JSON text, a struct inside an expanded list is stringified +once per element row, and a struct under a pivoted map is stringified once per +key column when `struct_flatten` is `"stringify"` and `map_flatten` is +`"pivot"`. ```javascript const table = await worker.table( diff --git a/docs/md/how_to/python/table_data.md b/docs/md/how_to/python/table_data.md index 582e4706b9..e627c136df 100644 --- a/docs/md/how_to/python/table_data.md +++ b/docs/md/how_to/python/table_data.md @@ -107,17 +107,20 @@ arrow_table = pa.table({ perspective.table(arrow_table) ``` -Both `struct` and `map` columns are controlled by the `struct_flatten` -argument: +A `struct` column is controlled by the `struct_flatten` argument and a `map` +column by the `map_flatten` argument, each taking: - `"pivot"` (default) hoists each field or key into its own dotted column, as above. - `"stringify"` keeps the whole value as JSON text in a single string column. -The two arguments compose from the outside in. A list inside a stringified -struct is an array within its JSON text and is never expanded, while a struct -inside an expanded list is stringified once per element row: +An unset `map_flatten` follows `struct_flatten`, so `struct_flatten="stringify"` +alone stringifies both. JSON objects always follow `struct_flatten`. + +The arguments compose from the outside in. A list inside a stringified struct +is an array within its JSON text and is never expanded, while a struct inside +an expanded list is stringified once per element row: ```python arrow_table = pa.table({ @@ -131,6 +134,23 @@ arrow_table = pa.table({ perspective.table(arrow_table, struct_flatten="stringify") ``` +Setting the two independently keeps a map with an open key set as text while +its sibling structs still pivot, or stringifies each struct under a pivoted +map: + +```python +arrow_table = pa.table({ + "s": pa.array([{"a": 1}], type=pa.struct([("a", pa.int64())])), + "m": pa.array([[("k", 1)]], type=pa.map_(pa.string(), pa.int64())), +}) + +# `{"s.a": [1], "m": ['{"k":1}']}` +perspective.table(arrow_table, map_flatten="stringify") + +# `{"s": ['{"a":1}'], "m.k": [1]}` +perspective.table(arrow_table, struct_flatten="stringify", map_flatten="pivot") +``` + ## Polars ```python diff --git a/rust/perspective-client/build.rs b/rust/perspective-client/build.rs index b3a3cada69..10f71eece4 100644 --- a/rust/perspective-client/build.rs +++ b/rust/perspective-client/build.rs @@ -80,6 +80,10 @@ fn prost_build() -> Result<()> { "StructFlatten", "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", ) + .type_attribute( + "MapFlatten", + "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]", + ) .field_attribute("ViewToArrowResp.arrow", "#[serde(skip)]") .field_attribute("from_arrow", "#[serde(skip)]") .type_attribute(".", "#[derive(serde::Serialize)]") diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index ee67110654..1ef85197a1 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -407,6 +407,9 @@ message MakeTableReq { optional StructFlatten struct_flatten = 7; + // Unset falls back to `struct_flatten`. + optional MapFlatten map_flatten = 8; + // The columns of a `Table` made from a `View`, in place of the ones // inferred from it. Only valid with `MakeTableData.from_view`. optional Schema view_schema = 5; @@ -423,6 +426,11 @@ enum StructFlatten { STRUCT_FLATTEN_PIVOT = 0; STRUCT_FLATTEN_STRINGIFY = 1; } + +enum MapFlatten { + MAP_FLATTEN_PIVOT = 0; + MAP_FLATTEN_STRINGIFY = 1; +} message MakeTableResp {} enum JoinType { diff --git a/rust/perspective-client/src/rust/client.rs b/rust/perspective-client/src/rust/client.rs index 27bf9ec474..8497d13969 100644 --- a/rust/perspective-client/src/rust/client.rs +++ b/rust/perspective-client/src/rust/client.rs @@ -727,6 +727,7 @@ impl Client { page_to_disk: None, list_flatten: None, struct_flatten: None, + map_flatten: None, schema: None, })), resp => Err(resp.into()), @@ -775,6 +776,7 @@ impl Client { page_to_disk: None, list_flatten: None, struct_flatten: None, + map_flatten: None, schema: None, }; diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index b76973ce03..5cec689104 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -167,19 +167,26 @@ pub struct TableInitOptions { /// and are incompatible with `index`, as the rows of an expansion /// repeat their index. `stringify` encodes each list as a JSON array in /// a single string column instead. `STRUCT` and `MAP` columns are - /// governed by `struct_flatten`. + /// governed by `struct_flatten` and `map_flatten`. #[serde(default)] #[ts(optional)] pub list_flatten: Option, - /// How Arrow `STRUCT` and `MAP` columns and JSON objects are ingested. - /// `pivot` (the default) hoists each field or key into its own dotted - /// column, and `stringify` keeps the whole value as JSON text in a single - /// string column. + /// How Arrow `STRUCT` columns and JSON objects are ingested. `pivot` (the + /// default) hoists each field into its own dotted column, and `stringify` + /// keeps the whole value as JSON text in a single string column. #[serde(default)] #[ts(optional)] pub struct_flatten: Option, + /// How Arrow `MAP` columns are ingested. `pivot` hoists each distinct key + /// into its own dotted column, and `stringify` keeps the whole value as + /// JSON text in a single string column. When unset, follows + /// `struct_flatten`. JSON objects always follow `struct_flatten`. + #[serde(default)] + #[ts(optional)] + pub map_flatten: Option, + /// The columns of a [`Table`] made from a [`View`], in place of the ones /// inferred from it. #[serde(default)] @@ -200,6 +207,7 @@ impl TryFrom for MakeTableOptions { let page_to_disk = value.page_to_disk; let list_flatten = value.list_flatten.map(|x| x as i32); let struct_flatten = value.struct_flatten.map(|x| x as i32); + let map_flatten = value.map_flatten.map(|x| x as i32); let view_schema = value.schema.clone().map(|schema| crate::proto::Schema { schema: schema .into_iter() @@ -214,6 +222,7 @@ impl TryFrom for MakeTableOptions { page_to_disk, list_flatten, struct_flatten, + map_flatten, view_schema, make_table_type: match value { TableOptions { @@ -247,6 +256,7 @@ pub(crate) struct TableOptions { pub page_to_disk: Option, pub list_flatten: Option, pub struct_flatten: Option, + pub map_flatten: Option, pub schema: Option>, } @@ -267,6 +277,7 @@ impl From for TableOptions { page_to_disk: value.page_to_disk, list_flatten: value.list_flatten, struct_flatten: value.struct_flatten, + map_flatten: value.map_flatten, schema: value.schema, } } diff --git a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts index dc7a7f92e1..159b411ad3 100644 --- a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts @@ -635,4 +635,152 @@ test.describe("Arrow nested columns", function () { await table.delete(); }); }); + + test.describe("Map flatten", function () { + const mixed = () => + ipc({ + s: arrow.vectorFromArray([{ a: 1 }], struct([["a", int32()]])), + m: arrow.vectorFromArray([{ a: 1, b: 2 }], map(int32())), + }); + + test("Stringifies maps while structs pivot", async function () { + const table = await perspective.table(mixed(), { + map_flatten: "stringify", + }); + + expect(await table.schema()).toEqual({ + "s.a": "integer", + m: "string", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "s.a": [1], + m: ['{"a":1,"b":2}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Pivots maps while structs stringify", async function () { + const table = await perspective.table(mixed(), { + struct_flatten: "stringify", + map_flatten: "pivot", + }); + + expect(await table.schema()).toEqual({ + s: "string", + "m.a": "integer", + "m.b": "integer", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ['{"a":1}'], + "m.a": [1], + "m.b": [2], + }); + + await view.delete(); + await table.delete(); + }); + + test("Follows struct_flatten when unset", async function () { + const table = await perspective.table(mixed(), { + struct_flatten: "stringify", + }); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ['{"a":1}'], + m: ['{"a":1,"b":2}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Stringifies a map inside a pivoted struct", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ a: 1, m: { k: 1 } }], + struct([ + ["a", int32()], + ["m", map(int32())], + ]), + ), + }), + { map_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "s.a": [1], + "s.m": ['{"k":1}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Stringifies each struct under a pivoted map", async function () { + const table = await perspective.table( + ipc({ + n: arrow.vectorFromArray( + [{ k: { x: 1 } }], + map(struct([["x", int32()]])), + ), + }), + { struct_flatten: "stringify", map_flatten: "pivot" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + "n.k": ['{"x":1}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Explodes a list of maps then stringifies each", async function () { + const table = await perspective.table( + ipc({ + id: arrow.vectorFromArray([1], int32()), + l: arrow.vectorFromArray( + [[{ a: 1 }, { a: 2 }]], + list(map(int32())), + ), + }), + { map_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + id: [1, 1], + l: ['{"a":1}', '{"a":2}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Persists across update on a schema table", async function () { + const table = await perspective.table( + { m: "string" }, + { map_flatten: "stringify" }, + ); + + await table.update( + ipc({ m: arrow.vectorFromArray([{ a: 1 }], map(int32())) }), + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ m: ['{"a":1}'] }); + await view.delete(); + await table.delete(); + }); + }); }); diff --git a/rust/perspective-js/test/js/constructors/json_nested.spec.ts b/rust/perspective-js/test/js/constructors/json_nested.spec.ts index f7f43e241c..7c353755ae 100644 --- a/rust/perspective-js/test/js/constructors/json_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/json_nested.spec.ts @@ -291,6 +291,21 @@ test.describe("JSON nested columns", function () { await table.delete(); }); + test("ignores map_flatten for objects", async function () { + const { data, options } = shape([{ id: 1, m: { a: 1 } }]); + const table = await perspective.table(data, { + ...options, + map_flatten: "stringify", + }); + + expect(await table.schema()).toEqual({ + id: "integer", + "m.a": "integer", + }); + + await table.delete(); + }); + test("encodes arrays as JSON when configured", async function () { const { data, options } = shape([ { x: 1, y: [10, 20] }, diff --git a/rust/perspective-js/test/js/constructors/nested_parity.spec.ts b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts index 2630a5faf5..29a0259acb 100644 --- a/rust/perspective-js/test/js/constructors/nested_parity.spec.ts +++ b/rust/perspective-js/test/js/constructors/nested_parity.spec.ts @@ -258,4 +258,13 @@ test.describe("Nested ingest parity", function () { { struct_flatten: "stringify" }, ); }); + + test("map_flatten leaves objects and structs pivoted", async function () { + await parity( + "map_flatten_object", + [{ m: { a: 1 } }], + { m: arrow.vectorFromArray([{ a: 1 }], struct([["a", int32()]])) }, + { map_flatten: "stringify" }, + ); + }); }); diff --git a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py index f01d55c382..9f892d292a 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py +++ b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py @@ -953,3 +953,107 @@ def test_default_pivots(self): assert Table(data, struct_flatten="pivot").view().to_columns() == { "s.a": [1] } + + +class TestTableArrowMapFlatten(object): + map_type = pa.map_(pa.string(), pa.int64()) + + def mixed(self): + return pa.table( + { + "s": pa.array([{"a": 1}], type=pa.struct([("a", pa.int64())])), + "m": pa.array([[("a", 1), ("b", 2)]], type=self.map_type), + } + ) + + def test_map_stringifies_while_struct_pivots(self): + tbl = Table(self.mixed(), map_flatten="stringify") + assert tbl.schema() == {"s.a": "integer", "m": "string"} + assert tbl.view().to_columns() == {"s.a": [1], "m": ['{"a":1,"b":2}']} + + def test_map_pivots_while_struct_stringifies(self): + tbl = Table(self.mixed(), struct_flatten="stringify", map_flatten="pivot") + assert tbl.schema() == {"s": "string", "m.a": "integer", "m.b": "integer"} + assert tbl.view().to_columns() == { + "s": ['{"a":1}'], + "m.a": [1], + "m.b": [2], + } + + def test_unset_map_flatten_follows_struct_flatten(self): + tbl = Table(self.mixed(), struct_flatten="stringify") + assert tbl.view().to_columns() == {"s": ['{"a":1}'], "m": ['{"a":1,"b":2}']} + assert Table(self.mixed()).schema() == { + "s.a": "integer", + "m.a": "integer", + "m.b": "integer", + } + + def test_map_inside_pivoted_struct_stringifies(self): + data = pa.table( + { + "s": pa.array( + [{"a": 1, "m": [("k", 1)]}], + type=pa.struct([("a", pa.int64()), ("m", self.map_type)]), + ) + } + ) + + assert Table(data, map_flatten="stringify").view().to_columns() == { + "s.a": [1], + "s.m": ['{"k":1}'], + } + + def test_struct_under_pivoted_map_stringifies(self): + data = pa.table( + { + "n": pa.array( + [[("k", {"x": 1})]], + type=pa.map_(pa.string(), pa.struct([("x", pa.int64())])), + ) + } + ) + + tbl = Table(data, struct_flatten="stringify", map_flatten="pivot") + assert tbl.view().to_columns() == {"n.k": ['{"x":1}']} + + def test_integer_keys_stringify_but_do_not_pivot(self): + data = pa.table( + {"m": pa.array([[(1, "x")]], type=pa.map_(pa.int32(), pa.string()))} + ) + + assert Table(data, map_flatten="stringify").view().to_columns() == { + "m": ['{"1":"x"}'] + } + + with pytest.raises(psp.PerspectiveError, match=r"`int32`"): + Table(data, map_flatten="pivot") + + def test_list_of_maps_explodes_then_stringifies(self): + data = pa.table( + { + "id": pa.array([1], type=pa.int64()), + "l": pa.array( + [[[("a", 1)], [("a", 2)]]], type=pa.list_(self.map_type) + ), + } + ) + + assert Table(data, map_flatten="stringify").view().to_columns() == { + "id": [1, 1], + "l": ['{"a":1}', '{"a":2}'], + } + + def test_mode_persists_across_update_on_a_schema_table(self): + tbl = Table({"m": "string"}, map_flatten="stringify") + tbl.update( + arrow_bytes( + pa.table({"m": pa.array([[("a", 1)]], type=self.map_type)}) + ) + ) + + assert tbl.view().to_columns() == {"m": ['{"a":1}']} + + def test_unknown_value_is_rejected(self): + with pytest.raises(ValueError, match=r"map_flatten"): + Table(self.mixed(), map_flatten="bogus") diff --git a/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py b/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py index 790a85cec7..4b0264da54 100644 --- a/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py +++ b/rust/perspective-python/perspective/tests/widget/test_anywidget_smoke.py @@ -13,6 +13,7 @@ import pathlib import anywidget +import pyarrow as pa import pytest import perspective.widget as widget_module @@ -129,6 +130,18 @@ def test_forwards_page_to_disk_and_name(self): assert widget.table.get_name() == "forwarded" assert widget.table.view().to_columns() == {"a": [1, 2, 3]} + def test_forwards_map_flatten(self): + data = pa.table( + { + "m": pa.array( + [[("a", 1)]], type=pa.map_(pa.string(), pa.int64()) + ) + } + ) + + widget = PerspectiveWidget(data, map_flatten="stringify") + assert widget.table.view().to_columns() == {"m": ['{"a":1}']} + def test_table_options_without_data(self): with pytest.raises(TypeError): PerspectiveWidget(None, list_flatten="stringify") diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index 717cdf328c..6d63297b79 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -30,7 +30,14 @@ _STATIC = pathlib.Path(__file__).parent / "static" -_TABLE_OPTIONS = ("name", "format", "page_to_disk", "list_flatten", "struct_flatten") +_TABLE_OPTIONS = ( + "name", + "format", + "page_to_disk", + "list_flatten", + "struct_flatten", + "map_flatten", +) __doc__ = """ `PerspectiveWidget` is a JupyterLab widget that implements the same API as @@ -161,8 +168,8 @@ def __init__( Cannot be set at the same time as `index`, ignored if `server` is True. - `name`, `format`, `page_to_disk`, `list_flatten`, - `struct_flatten`: `Table` constructor options, used when `data` - is a dataset. + `struct_flatten`, `map_flatten`: `Table` constructor options, + used when `data` is a dataset. - `kwargs` (`dict`): configuration options for the `PerspectiveViewer`. # Examples diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index 9dd5e84a78..47c8a8f894 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use futures::FutureExt; use indexmap::IndexMap; use perspective_client::config::{ColumnType, ViewConfigUpdate}; -use perspective_client::proto::{ListFlatten, StructFlatten}; +use perspective_client::proto::{ListFlatten, MapFlatten, StructFlatten}; use perspective_client::{ Client, ColumnWindow, DeleteOptions, DescribeVerdict, OnRemoveData, OnUpdateData, OnUpdateMode, OnUpdateOptions, Table, TableData, TableIndex, TableInitOptions, TableReadFormat, TableRef, @@ -87,6 +87,15 @@ fn parse_struct_flatten(value: Option) -> PyResult } } +fn parse_map_flatten(value: Option) -> PyResult> { + match value.as_deref() { + None => Ok(None), + Some("pivot") => Ok(Some(MapFlatten::Pivot)), + Some("stringify") => Ok(Some(MapFlatten::Stringify)), + Some(_x) => Err(PyValueError::new_err("Unknown `map_flatten`".to_string())), + } +} + /// `perspective_server::Server`, whether locally in-memory or remote over some /// transport like a WebSocket. /// @@ -215,7 +224,7 @@ impl AsyncClient { /// table = await client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, schema=None))] + #[pyo3(signature=(input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, map_flatten=None, schema=None))] pub async fn table( &self, input: Py, @@ -226,6 +235,7 @@ impl AsyncClient { page_to_disk: Option, list_flatten: Option>, struct_flatten: Option>, + map_flatten: Option>, schema: Option>, ) -> PyResult { let client = self.client.clone(); @@ -236,6 +246,7 @@ impl AsyncClient { page_to_disk, list_flatten: parse_list_flatten(list_flatten.map(|x| x.to_string()))?, struct_flatten: parse_struct_flatten(struct_flatten.map(|x| x.to_string()))?, + map_flatten: parse_map_flatten(map_flatten.map(|x| x.to_string()))?, schema: schema.map(|x| parse_schema(py, x.bind(py))).transpose()?, ..TableInitOptions::default() }; diff --git a/rust/perspective-python/src/client/client_sync.rs b/rust/perspective-python/src/client/client_sync.rs index 84ad79085a..1d08b7fe71 100644 --- a/rust/perspective-python/src/client/client_sync.rs +++ b/rust/perspective-python/src/client/client_sync.rs @@ -168,7 +168,7 @@ impl Client { /// table = client.table("x,y\n1,2\n3,4") /// ``` #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, schema=None))] + #[pyo3(signature = (input, limit=None, index=None, name=None, format=None, page_to_disk=None, list_flatten=None, struct_flatten=None, map_flatten=None, schema=None))] pub fn table( &self, py: Python<'_>, @@ -180,6 +180,7 @@ impl Client { page_to_disk: Option, list_flatten: Option>, struct_flatten: Option>, + map_flatten: Option>, schema: Option>, ) -> PyResult { Ok(Table( @@ -193,6 +194,7 @@ impl Client { page_to_disk, list_flatten, struct_flatten, + map_flatten, schema, ) .py_block_on(py)?, diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp index 3974b8e7dd..90a629ecb4 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp @@ -211,10 +211,7 @@ convert_type(const std::string& src) { void ArrowLoader::initialize( - const std::uint8_t* ptr, - const uint32_t length, - t_list_flatten mode, - t_struct_flatten struct_mode + const std::uint8_t* ptr, const uint32_t length, t_flatten_mode mode ) { if (std::memcmp("ARROW1", (const void*)ptr, 6) == 0) { load_file(ptr, length, m_table); @@ -227,11 +224,11 @@ ArrowLoader::initialize( PSP_COMPLAIN_AND_ABORT(validation.ToString()); } - if (!normalize_table_is_noop(*m_table, mode, struct_mode)) { - m_expanded = normalize_table_expands(*m_table, mode, struct_mode); - m_normalized = std::make_unique( - normalize_table(m_table, mode, struct_mode) - ); + if (!normalize_table_is_noop(*m_table, mode)) { + m_expanded = normalize_table_expands(*m_table, mode); + m_normalized = + std::make_unique(normalize_table(m_table, mode) + ); } for (const auto& field : fields()) { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp index 325134ff5a..bbbb7dbfe0 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp @@ -42,32 +42,24 @@ namespace { } bool - is_pivot_id(arrow::Type::type id) { - return id == arrow::Type::STRUCT || id == arrow::Type::MAP; - } - - bool - type_expands(const arrow::DataType& type, t_struct_flatten struct_mode) { + type_expands(const arrow::DataType& type, t_flatten_mode mode) { if (is_list_id(type.id())) { return true; } - if (struct_mode == STRUCT_FLATTEN_STRINGIFY) { - return false; - } - - if (type.id() == arrow::Type::STRUCT) { + if (type.id() == arrow::Type::STRUCT + && mode.structs != STRUCT_FLATTEN_STRINGIFY) { for (const auto& child : type.fields()) { - if (type_expands(*child->type(), struct_mode)) { + if (type_expands(*child->type(), mode)) { return true; } } } - if (type.id() == arrow::Type::MAP) { + if (type.id() == arrow::Type::MAP + && mode.maps != MAP_FLATTEN_STRINGIFY) { return type_expands( - *static_cast(type).item_type(), - struct_mode + *static_cast(type).item_type(), mode ); } @@ -750,18 +742,19 @@ namespace { } // namespace bool -normalize_table_is_noop( - const arrow::Table& input, - t_list_flatten mode, - t_struct_flatten struct_mode -) { +normalize_table_is_noop(const arrow::Table& input, t_flatten_mode mode) { for (const auto& field : input.schema()->fields()) { const auto id = field->type()->id(); - if (struct_mode != STRUCT_FLATTEN_STRINGIFY && is_pivot_id(id)) { + if (mode.structs != STRUCT_FLATTEN_STRINGIFY + && id == arrow::Type::STRUCT) { return false; } - if (mode != LIST_FLATTEN_STRINGIFY && is_list_id(id)) { + if (mode.maps != MAP_FLATTEN_STRINGIFY && id == arrow::Type::MAP) { + return false; + } + + if (mode.list != LIST_FLATTEN_STRINGIFY && is_list_id(id)) { return false; } } @@ -770,17 +763,13 @@ normalize_table_is_noop( } bool -normalize_table_expands( - const arrow::Table& input, - t_list_flatten mode, - t_struct_flatten struct_mode -) { - if (mode == LIST_FLATTEN_STRINGIFY) { +normalize_table_expands(const arrow::Table& input, t_flatten_mode mode) { + if (mode.list == LIST_FLATTEN_STRINGIFY) { return false; } for (const auto& field : input.schema()->fields()) { - if (type_expands(*field->type(), struct_mode)) { + if (type_expands(*field->type(), mode)) { return true; } } @@ -789,11 +778,7 @@ normalize_table_expands( } t_normalized_table -normalize_table( - std::shared_ptr input, - t_list_flatten mode, - t_struct_flatten struct_mode -) { +normalize_table(std::shared_ptr input, t_flatten_mode mode) { t_normalized_table out; out.fields = input->schema()->fields(); out.columns = input->columns(); @@ -802,30 +787,32 @@ normalize_table( out.num_rows = input->num_rows(); while (true) { bool changed = false; - if (struct_mode != STRUCT_FLATTEN_STRINGIFY) { - changed = flatten_structs( + if (mode.structs != STRUCT_FLATTEN_STRINGIFY + && flatten_structs( out.fields, out.columns, out.gathers, out.per_element - ); + )) { + changed = true; + } - if (pivot_maps( - out.fields, - out.columns, - out.gathers, - out.per_element, - out.num_rows - )) { - changed = true; - } + if (mode.maps != MAP_FLATTEN_STRINGIFY + && pivot_maps( + out.fields, + out.columns, + out.gathers, + out.per_element, + out.num_rows + )) { + changed = true; } - if (mode != LIST_FLATTEN_STRINGIFY + if (mode.list != LIST_FLATTEN_STRINGIFY && explode_lists( out.fields, out.columns, out.gathers, out.per_element, out.num_rows, - mode + mode.list )) { changed = true; } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp index 2d14309435..5eed845b06 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp @@ -721,11 +721,10 @@ JsonLoader::init( t_json_format format, const std::vector& index, const t_schema* existing, - t_list_flatten mode, - t_struct_flatten struct_mode + t_flatten_mode mode ) { m_format = format; - m_mode = t_flatten_mode{mode, struct_mode}; + m_mode = mode; m_index = index; if (format == JSON_FORMAT_NDJSON) { m_stream = rapidjson::StringStream(data.data()); @@ -799,7 +798,7 @@ JsonLoader::init( } m_expands = !m_per_element.empty(); - if (mode == LIST_FLATTEN_CARTESIAN && m_expansions.size() > 1) { + if (mode.list == LIST_FLATTEN_CARTESIAN && m_expansions.size() > 1) { // A product repeats every factor against the others' dimensions, so // with more than one expansion point nothing varies per row. m_per_element.clear(); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index b22e937d0f..f17a05c9ba 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -2572,31 +2572,46 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { ? BACKING_STORE_DISK : BACKING_STORE_MEMORY; - apachearrow::t_list_flatten list_flatten; + t_flatten_mode flatten; switch (r.options().list_flatten()) { case proto::LIST_FLATTEN_CARTESIAN: - list_flatten = apachearrow::LIST_FLATTEN_CARTESIAN; + flatten.list = LIST_FLATTEN_CARTESIAN; break; case proto::LIST_FLATTEN_STRINGIFY: - list_flatten = apachearrow::LIST_FLATTEN_STRINGIFY; + flatten.list = LIST_FLATTEN_STRINGIFY; break; case proto::LIST_FLATTEN_ZIP: default: - list_flatten = apachearrow::LIST_FLATTEN_ZIP; + flatten.list = LIST_FLATTEN_ZIP; break; } - apachearrow::t_struct_flatten struct_flatten; switch (r.options().struct_flatten()) { case proto::STRUCT_FLATTEN_STRINGIFY: - struct_flatten = apachearrow::STRUCT_FLATTEN_STRINGIFY; + flatten.structs = STRUCT_FLATTEN_STRINGIFY; break; case proto::STRUCT_FLATTEN_PIVOT: default: - struct_flatten = apachearrow::STRUCT_FLATTEN_PIVOT; + flatten.structs = STRUCT_FLATTEN_PIVOT; break; } + if (!r.options().has_map_flatten()) { + flatten.maps = flatten.structs == STRUCT_FLATTEN_STRINGIFY + ? MAP_FLATTEN_STRINGIFY + : MAP_FLATTEN_PIVOT; + } else { + switch (r.options().map_flatten()) { + case proto::MAP_FLATTEN_STRINGIFY: + flatten.maps = MAP_FLATTEN_STRINGIFY; + break; + case proto::MAP_FLATTEN_PIVOT: + default: + flatten.maps = MAP_FLATTEN_PIVOT; + break; + } + } + if (r.options().has_view_schema() && r.data().data_case() != proto::MakeTableData::kFromView) { proto::Response resp; @@ -2627,8 +2642,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } @@ -2641,8 +2655,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } @@ -2655,8 +2668,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } @@ -2669,8 +2681,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } @@ -2683,8 +2694,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } @@ -2703,8 +2713,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { table_schema, limit, backing_store, - list_flatten, - struct_flatten + flatten ); break; } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp index 4b76550e44..32e7206c43 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/table.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/table.cpp @@ -44,8 +44,7 @@ Table::Table( std::uint32_t limit, std::vector index, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) : m_init(false), m_id(GLOBAL_TABLE_ID++), @@ -57,8 +56,7 @@ Table::Table( m_index(std::move(index)), m_gnode_set(false), m_backing_store(backing_store), - m_list_flatten(list_flatten), - m_struct_flatten(struct_flatten) { + m_flatten(flatten) { validate_columns(m_column_names); if (is_composite_index()) { @@ -550,8 +548,7 @@ Table::from_csv( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { auto map = std::unordered_map>(); @@ -609,8 +606,7 @@ Table::from_csv( limit, index, backing_store, - list_flatten, - struct_flatten + flatten ); // `psp_pkey` is guaranteed unique only when the index is implicit (a @@ -756,8 +752,7 @@ Table::remove_arrow(const std::string_view& data) { arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - m_list_flatten, - m_struct_flatten + m_flatten ); const auto names = arrow_loader.names(); @@ -791,8 +786,7 @@ Table::from_json_loader( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { if (const auto repeated = loader.repeated_index(index)) { std::stringstream ss; @@ -843,8 +837,7 @@ Table::from_json_loader( limit, index, backing_store, - list_flatten, - struct_flatten + flatten ); tbl->init(*data_table, nrows, t_op::OP_INSERT, 0); @@ -861,9 +854,7 @@ Table::update_json( ) { t_schema table_schema = get_schema(); json::JsonLoader loader; - loader.init( - data, format, m_index, &table_schema, m_list_flatten, m_struct_flatten - ); + loader.init(data, format, m_index, &table_schema, m_flatten); if (loader.empty()) { return; } @@ -898,8 +889,7 @@ Table::from_cols( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { json::JsonLoader loader; loader.init( @@ -907,8 +897,7 @@ Table::from_cols( json::JSON_FORMAT_COLUMNS, index, nullptr, - list_flatten, - struct_flatten + flatten ); return from_json_loader( loader, @@ -916,8 +905,7 @@ Table::from_cols( std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); } @@ -938,8 +926,7 @@ Table::from_rows( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { json::JsonLoader loader; loader.init( @@ -947,8 +934,7 @@ Table::from_rows( json::JSON_FORMAT_ROWS, index, nullptr, - list_flatten, - struct_flatten + flatten ); return from_json_loader( loader, @@ -956,8 +942,7 @@ Table::from_rows( std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); } @@ -972,8 +957,7 @@ Table::from_ndjson( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { json::JsonLoader loader; loader.init( @@ -981,8 +965,7 @@ Table::from_ndjson( json::JSON_FORMAT_NDJSON, index, nullptr, - list_flatten, - struct_flatten + flatten ); return from_json_loader( loader, @@ -990,8 +973,7 @@ Table::from_ndjson( std::move(data), limit, backing_store, - list_flatten, - struct_flatten + flatten ); } @@ -1001,8 +983,7 @@ Table::from_schema( const t_schema& schema, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { auto pool = std::make_shared(); pool->init(); @@ -1037,8 +1018,7 @@ Table::from_schema( limit, index, backing_store, - list_flatten, - struct_flatten + flatten ); tbl->init(data_table, 0, t_op::OP_INSERT, 0); @@ -1091,8 +1071,7 @@ Table::update_arrow(const std::string_view& data, std::uint32_t port_id) { arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - m_list_flatten, - m_struct_flatten + m_flatten ); if (const auto repeated = arrow_loader.repeated_index(m_index)) { @@ -1150,8 +1129,7 @@ Table::from_arrow( std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { apachearrow::ArrowLoader arrow_loader; @@ -1159,8 +1137,7 @@ Table::from_arrow( arrow_loader.initialize( reinterpret_cast(data.data()), data.size(), - list_flatten, - struct_flatten + flatten ); if (const auto repeated = arrow_loader.repeated_index(index)) { @@ -1222,8 +1199,7 @@ Table::from_arrow( limit, index, backing_store, - list_flatten, - struct_flatten + flatten ); table->init(*data_table, data_table->num_rows(), t_op::OP_INSERT, 0); @@ -1240,8 +1216,7 @@ Table::make_table( const std::vector& index, const std::string_view& data, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ) { auto pool = std::make_shared(); pool->init(); @@ -1272,8 +1247,7 @@ Table::make_table( limit, index, backing_store, - list_flatten, - struct_flatten + flatten ); table->init(data_table, data_table.num_rows(), t_op::OP_INSERT, 0); pool->_process(); diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h index 93f67c9653..6799580b9b 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h @@ -44,14 +44,10 @@ namespace apachearrow { * describe the post-flattening shape. * * @param ptr - * @param mode - how `LIST` columns are ingested. - * @param struct_mode - how `STRUCT` and `MAP` columns are ingested. + * @param mode - how nested columns are ingested. */ void initialize( - const std::uint8_t* ptr, - std::uint32_t, - t_list_flatten mode = LIST_FLATTEN_ZIP, - t_struct_flatten struct_mode = STRUCT_FLATTEN_PIVOT + const std::uint8_t* ptr, std::uint32_t, t_flatten_mode mode = {} ); /** diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h index f3dc6a49e5..21f3bd8a0a 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h @@ -23,11 +23,15 @@ namespace perspective { namespace apachearrow { + using perspective::t_flatten_mode; using perspective::t_list_flatten; + using perspective::t_map_flatten; using perspective::t_struct_flatten; using perspective::LIST_FLATTEN_CARTESIAN; using perspective::LIST_FLATTEN_STRINGIFY; using perspective::LIST_FLATTEN_ZIP; + using perspective::MAP_FLATTEN_PIVOT; + using perspective::MAP_FLATTEN_STRINGIFY; using perspective::STRUCT_FLATTEN_PIVOT; using perspective::STRUCT_FLATTEN_STRINGIFY; extern const char* const FLATTEN_SEPARATOR; @@ -55,23 +59,14 @@ namespace apachearrow { std::int64_t num_rows; }; - PERSPECTIVE_EXPORT t_normalized_table normalize_table( - std::shared_ptr input, - t_list_flatten mode, - t_struct_flatten struct_mode - ); + PERSPECTIVE_EXPORT t_normalized_table + normalize_table(std::shared_ptr input, t_flatten_mode mode); - PERSPECTIVE_EXPORT bool normalize_table_is_noop( - const arrow::Table& input, - t_list_flatten mode, - t_struct_flatten struct_mode - ); + PERSPECTIVE_EXPORT bool + normalize_table_is_noop(const arrow::Table& input, t_flatten_mode mode); - PERSPECTIVE_EXPORT bool normalize_table_expands( - const arrow::Table& input, - t_list_flatten mode, - t_struct_flatten struct_mode - ); + PERSPECTIVE_EXPORT bool + normalize_table_expands(const arrow::Table& input, t_flatten_mode mode); } // namespace apachearrow } // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h index 7a0e264e52..2be2baa371 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h @@ -25,9 +25,16 @@ enum t_struct_flatten { STRUCT_FLATTEN_STRINGIFY = 1 }; +enum t_map_flatten { MAP_FLATTEN_PIVOT = 0, MAP_FLATTEN_STRINGIFY = 1 }; + struct t_flatten_mode { t_list_flatten list{LIST_FLATTEN_ZIP}; t_struct_flatten structs{STRUCT_FLATTEN_PIVOT}; + + /** + * Arrow `MAP` columns only; JSON objects follow `structs`. + */ + t_map_flatten maps{MAP_FLATTEN_PIVOT}; }; } // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h index 78ae8da9a4..1b041ccc6a 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/json_loader.h @@ -63,8 +63,7 @@ namespace json { t_json_format format, const std::vector& index, const t_schema* existing, - t_list_flatten mode, - t_struct_flatten struct_mode = STRUCT_FLATTEN_PIVOT + t_flatten_mode mode ); const std::vector& names() const; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h index a0daba13f1..1eda190442 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/table.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/table.h @@ -59,10 +59,7 @@ class PERSPECTIVE_EXPORT Table { std::uint32_t limit, std::vector index, t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); /** @@ -253,10 +250,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); static std::shared_ptr
from_cols( @@ -264,10 +258,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); static std::shared_ptr
from_rows( @@ -275,10 +266,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); static std::shared_ptr
from_ndjson( @@ -286,10 +274,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); static std::shared_ptr
from_schema( @@ -297,10 +282,7 @@ class PERSPECTIVE_EXPORT Table { const t_schema& schema, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); /** @@ -322,10 +304,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit = std::numeric_limits::max(), t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); static std::shared_ptr
make_table( @@ -335,10 +314,7 @@ class PERSPECTIVE_EXPORT Table { const std::vector& index, const std::string_view& data, t_backing_store backing_store = BACKING_STORE_MEMORY, - apachearrow::t_list_flatten list_flatten = - apachearrow::LIST_FLATTEN_ZIP, - apachearrow::t_struct_flatten struct_flatten = - apachearrow::STRUCT_FLATTEN_PIVOT + t_flatten_mode flatten = {} ); private: @@ -352,8 +328,7 @@ class PERSPECTIVE_EXPORT Table { std::string&& data, std::uint32_t limit, t_backing_store backing_store, - apachearrow::t_list_flatten list_flatten, - apachearrow::t_struct_flatten struct_flatten + t_flatten_mode flatten ); /** @@ -416,19 +391,13 @@ class PERSPECTIVE_EXPORT Table { const t_backing_store m_backing_store; /** - * @brief How `arrow::Type::LIST` columns are ingested. + * @brief How nested columns are ingested. * * INVARIANT: this must be applied to `update_arrow` as well as to the * `from_arrow` which created the Table, or an update would produce a * different column shape than the schema it is written against. */ - const apachearrow::t_list_flatten m_list_flatten; - - /** - * @brief How `arrow::Type::STRUCT` and `arrow::Type::MAP` columns are - * ingested, held for the same reason as `m_list_flatten`. - */ - const apachearrow::t_struct_flatten m_struct_flatten; + const t_flatten_mode m_flatten; }; } // namespace perspective \ No newline at end of file From b21f5f540aa294754e36555a9983efdc8072b723 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 4 Nov 2026 13:19:24 -0500 Subject: [PATCH 2/2] Fix date/datetime stringify, add `stringify_formatted` to all three flatten modes Signed-off-by: Andrew Stein --- docs/md/how_to/javascript/loading_data.md | 12 +- docs/md/how_to/python/table_data.md | 10 +- rust/perspective-client/perspective.proto | 3 + rust/perspective-client/src/rust/table.rs | 17 +- .../test/js/constructors/arrow_nested.spec.ts | 124 ++++++ .../tests/table/test_table_arrow_nested.py | 258 ++++++++++++ .../src/client/client_async.rs | 3 + .../cpp/perspective/src/cpp/arrow_loader.cpp | 377 +++++++++++++----- .../perspective/src/cpp/arrow_normalize.cpp | 21 +- .../cpp/perspective/src/cpp/json_loader.cpp | 20 +- .../cpp/perspective/src/cpp/scalar.cpp | 83 ++++ .../cpp/perspective/src/cpp/server.cpp | 24 +- .../cpp/perspective/src/cpp/view.cpp | 82 ---- .../src/include/perspective/arrow_loader.h | 1 + .../src/include/perspective/arrow_normalize.h | 4 + .../src/include/perspective/flatten_mode.h | 27 +- .../src/include/perspective/scalar_json.h | 33 ++ .../src/include/perspective/view.h | 7 +- 18 files changed, 873 insertions(+), 233 deletions(-) create mode 100644 rust/perspective-server/cpp/perspective/src/include/perspective/scalar_json.h diff --git a/docs/md/how_to/javascript/loading_data.md b/docs/md/how_to/javascript/loading_data.md index 2113407e58..8f23b7cdda 100644 --- a/docs/md/how_to/javascript/loading_data.md +++ b/docs/md/how_to/javascript/loading_data.md @@ -83,19 +83,23 @@ normalized on ingest, controlled by three `Table` options. `struct_flatten` applies to Arrow `struct` columns and to JSON objects. `"pivot"` (the default) hoists each field into its own dotted column, so `{ s: { a: 1 } }` becomes a column `s.a`. `"stringify"` keeps the whole value -as JSON text in a single string column. +as JSON text in a single string column, with datetimes as epoch milliseconds; +`"stringify_formatted"` writes them as the text `to_json` emits under +`formatted`. `map_flatten` applies to Arrow `map` columns. `"pivot"` makes a map `m` one column `m.` per distinct key in the batch, with a null where a row lacks -the key, and `"stringify"` keeps the map as JSON text. When unset it follows -`struct_flatten`. JSON has no map type, so JSON objects always follow +the key, and `"stringify"` or `"stringify_formatted"` keeps the map as JSON +text. When unset it follows `struct_flatten`. JSON has no map type, so JSON objects always follow `struct_flatten`. `list_flatten` applies to Arrow `list` columns and JSON arrays. `"zip"` (the default) expands a row into one row per element, repeating its siblings, and requires sibling lists in a row to agree in length. `"cartesian"` expands to the product of the lists' lengths. `"stringify"` keeps each list as a JSON -array in a string column. +array in a string column, with datetimes and dates as epoch milliseconds, as +in `to_json`; `"stringify_formatted"` writes them as the text `to_json` emits +under `formatted`. The options compose from the outside in: a list inside a stringified struct stays inside its JSON text, a struct inside an expanded list is stringified diff --git a/docs/md/how_to/python/table_data.md b/docs/md/how_to/python/table_data.md index e627c136df..ab6dc76dac 100644 --- a/docs/md/how_to/python/table_data.md +++ b/docs/md/how_to/python/table_data.md @@ -71,7 +71,11 @@ A `list` column is controlled by the `list_flatten` argument: - `"cartesian"` expands a row into the product of its list columns' lengths, with an empty or null list counting as a single null element. - `"stringify"` encodes each list as a JSON array in a single string column, - leaving the row count unchanged. + leaving the row count unchanged. Datetimes and dates inside the text are + epoch milliseconds, as in `to_json`; decimals are strings and binary is + base64. +- `"stringify_formatted"` is `"stringify"` with datetimes and dates written + as the text `to_json` emits under `formatted`. ```python arrow_table = pa.table({ @@ -113,7 +117,9 @@ column by the `map_flatten` argument, each taking: - `"pivot"` (default) hoists each field or key into its own dotted column, as above. - `"stringify"` keeps the whole value as JSON text in a single string - column. + column, with datetimes and dates as epoch milliseconds, as in `to_json`. +- `"stringify_formatted"` is `"stringify"` with datetimes and dates written + as the text `to_json` emits under `formatted`. An unset `map_flatten` follows `struct_flatten`, so `struct_flatten="stringify"` alone stringifies both. JSON objects always follow `struct_flatten`. diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index 1ef85197a1..3538cda304 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -420,16 +420,19 @@ enum ListFlatten { LIST_FLATTEN_ZIP = 0; LIST_FLATTEN_CARTESIAN = 1; LIST_FLATTEN_STRINGIFY = 2; + LIST_FLATTEN_STRINGIFY_FORMATTED = 3; } enum StructFlatten { STRUCT_FLATTEN_PIVOT = 0; STRUCT_FLATTEN_STRINGIFY = 1; + STRUCT_FLATTEN_STRINGIFY_FORMATTED = 2; } enum MapFlatten { MAP_FLATTEN_PIVOT = 0; MAP_FLATTEN_STRINGIFY = 1; + MAP_FLATTEN_STRINGIFY_FORMATTED = 2; } message MakeTableResp {} diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index 5cec689104..64501ca167 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -166,23 +166,28 @@ pub struct TableInitOptions { /// default) and `cartesian` expand a row into one row per list element, /// and are incompatible with `index`, as the rows of an expansion /// repeat their index. `stringify` encodes each list as a JSON array in - /// a single string column instead. `STRUCT` and `MAP` columns are - /// governed by `struct_flatten` and `map_flatten`. + /// a single string column instead, with datetimes as epoch milliseconds, + /// and `stringify_formatted` writes them as the text `to_json` emits + /// under `formatted`. `STRUCT` and `MAP` columns are governed by + /// `struct_flatten` and `map_flatten`. #[serde(default)] #[ts(optional)] pub list_flatten: Option, /// How Arrow `STRUCT` columns and JSON objects are ingested. `pivot` (the /// default) hoists each field into its own dotted column, and `stringify` - /// keeps the whole value as JSON text in a single string column. + /// keeps the whole value as JSON text in a single string column, with + /// datetimes as epoch milliseconds; `stringify_formatted` writes them as + /// the text `to_json` emits under `formatted`. #[serde(default)] #[ts(optional)] pub struct_flatten: Option, /// How Arrow `MAP` columns are ingested. `pivot` hoists each distinct key - /// into its own dotted column, and `stringify` keeps the whole value as - /// JSON text in a single string column. When unset, follows - /// `struct_flatten`. JSON objects always follow `struct_flatten`. + /// into its own dotted column, and `stringify` or `stringify_formatted` + /// keeps the whole value as JSON text in a single string column, as for + /// `struct_flatten`. When unset, follows `struct_flatten`. JSON objects + /// always follow `struct_flatten`. #[serde(default)] #[ts(optional)] pub map_flatten: Option, diff --git a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts index 159b411ad3..8a954c007d 100644 --- a/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts +++ b/rust/perspective-js/test/js/constructors/arrow_nested.spec.ts @@ -292,6 +292,87 @@ test.describe("Arrow nested columns", function () { await view.delete(); await table.delete(); }); + + test("Formats list datetimes when configured", async function () { + const table = await perspective.table( + ipc({ + y: arrow.vectorFromArray( + [[1704164645000]], + list(new arrow.TimestampMillisecond()), + ), + }), + { list_flatten: "stringify_formatted" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + y: ['["2024-01-02 03:04:05.000"]'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Encodes timestamps as epoch milliseconds", async function () { + const table = await perspective.table( + ipc({ + y: arrow.vectorFromArray( + [[1704164645006]], + list(new arrow.TimestampMillisecond()), + ), + }), + { list_flatten: "stringify" }, + ); + + expect(await table.schema()).toEqual({ y: "string" }); + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + y: ["[1704164645006]"], + }); + + await view.delete(); + await table.delete(); + }); + + test("Encodes non-finite floats as null", async function () { + const table = await perspective.table( + ipc({ + y: arrow.vectorFromArray( + [[Infinity, -Infinity, NaN, 1.5]], + list(float64()), + ), + }), + { list_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + y: ["[null,null,null,1.5]"], + }); + + await view.delete(); + await table.delete(); + }); + + test("Encodes dates as epoch milliseconds", async function () { + const table = await perspective.table( + ipc({ + y: arrow.vectorFromArray( + [[new Date(1704153600000)]], + list(new arrow.DateDay()), + ), + }), + { list_flatten: "stringify" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + y: ["[1704153600000]"], + }); + + await view.delete(); + await table.delete(); + }); }); test.describe("Expanded index", function () { @@ -767,6 +848,49 @@ test.describe("Arrow nested columns", function () { await table.delete(); }); + test("Formats struct datetimes when configured", async function () { + const table = await perspective.table( + ipc({ + s: arrow.vectorFromArray( + [{ ts: 1704164645000, x: 1 }], + struct([ + ["ts", new arrow.TimestampMillisecond()], + ["x", int32()], + ]), + ), + }), + { struct_flatten: "stringify_formatted" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + s: ['{"ts":"2024-01-02 03:04:05.000","x":1}'], + }); + + await view.delete(); + await table.delete(); + }); + + test("Formats map datetimes when configured", async function () { + const table = await perspective.table( + ipc({ + m: arrow.vectorFromArray( + [{ k: 1704164645000 }], + map(new arrow.TimestampMillisecond()), + ), + }), + { map_flatten: "stringify_formatted" }, + ); + + const view = await table.view(); + expect(await view.to_columns()).toEqual({ + m: ['{"k":"2024-01-02 03:04:05.000"}'], + }); + + await view.delete(); + await table.delete(); + }); + test("Persists across update on a schema table", async function () { const table = await perspective.table( { m: "string" }, diff --git a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py index 9f892d292a..d74c5c63dd 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py +++ b/rust/perspective-python/perspective/tests/table/test_table_arrow_nested.py @@ -10,6 +10,9 @@ # ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +import json +from decimal import Decimal + import pyarrow as pa import pytest import perspective as psp @@ -1057,3 +1060,258 @@ def test_mode_persists_across_update_on_a_schema_table(self): def test_unknown_value_is_rejected(self): with pytest.raises(ValueError, match=r"map_flatten"): Table(self.mixed(), map_flatten="bogus") + + +class TestTableArrowStringifyTemporal(object): + us_utc = pa.timestamp("us", tz="UTC") + instant_ms = 1704164645000 + instant_us = 1704164645000006 + + def flat_json(self, column, **options): + tbl = Table(pa.table({"v": column}), **options) + return json.loads(tbl.view().to_columns_string())["v"] + + def test_list_of_timestamps_matches_the_flat_column(self): + column = pa.array([[self.instant_us]], pa.list_(self.us_utc)) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.schema() == {"v": "string"} + assert tbl.view().to_columns() == {"v": ["[1704164645000]"]} + flat = pa.array([self.instant_us], self.us_utc) + assert self.flat_json(flat) == [self.instant_ms] + + def test_list_of_structs_with_timestamps(self): + item = pa.struct([("ts", self.us_utc), ("x", pa.int64())]) + column = pa.array([[{"ts": self.instant_us, "x": 1}]], pa.list_(item)) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == { + "v": ['[{"ts":1704164645000,"x":1}]'] + } + + def test_struct_with_timestamp(self): + item = pa.struct([("ts", pa.timestamp("ms"))]) + column = pa.array([{"ts": 1704164645006}], item) + tbl = Table(pa.table({"v": column}), struct_flatten="stringify") + assert tbl.view().to_columns() == {"v": ['{"ts":1704164645006}']} + + def test_every_unit_truncates_like_the_flat_path(self): + units = { + "s": 1704164645, + "ms": 1704164645006, + "us": 1704164645006007, + "ns": 1704164645006007008, + } + + for unit, value in units.items(): + column = pa.array([[value]], pa.list_(pa.timestamp(unit))) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + flat = self.flat_json(pa.array([value], pa.timestamp(unit))) + assert tbl.view().to_columns() == {"v": [json.dumps(flat)]} + + def test_timezone_is_ignored(self): + naive = pa.array([[self.instant_us]], pa.list_(pa.timestamp("us"))) + zoned = pa.array( + [[self.instant_us]], + pa.list_(pa.timestamp("us", tz="America/New_York")), + ) + + for column in (naive, zoned): + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ["[1704164645000]"]} + + def test_before_the_epoch(self): + column = pa.array([[-1]], pa.list_(pa.timestamp("ms"))) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ["[-1]"]} + + def test_dates_match_the_flat_column(self): + for column, flat in ( + (pa.array([[19724]], pa.list_(pa.date32())), pa.array([19724], pa.date32())), + ( + pa.array([[1704153600000]], pa.list_(pa.date64())), + pa.array([1704153600000], pa.date64()), + ), + ): + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ["[1704153600000]"]} + assert self.flat_json(flat) == [1704153600000] + + def test_null_timestamp_element(self): + column = pa.array([[self.instant_us, None]], pa.list_(self.us_utc)) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ["[1704164645000,null]"]} + + def test_map_of_timestamps(self): + column = pa.array( + [[("k", self.instant_us)]], pa.map_(pa.string(), self.us_utc) + ) + + tbl = Table(pa.table({"v": column}), map_flatten="stringify") + assert tbl.view().to_columns() == {"v": ['{"k":1704164645000}']} + + def test_formatted_output_is_unchanged(self): + tbl = Table(pa.table({"v": pa.array([self.instant_us], self.us_utc)})) + assert json.loads(tbl.view().to_columns_string(formatted=True)) == { + "v": ["2024-01-02 03:04:05.000"] + } + + def test_time_and_duration_are_unit_counts(self): + time = pa.array([[11045000006]], pa.list_(pa.time64("us"))) + duration = pa.array([[1500]], pa.list_(pa.duration("ns"))) + for column, text in ((time, "[11045000006]"), (duration, "[1500]")): + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": [text]} + + def test_decimal_is_text(self): + column = pa.array([[Decimal("1.50")]], pa.list_(pa.decimal128(10, 2))) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ['["1.50"]']} + + def test_binary_is_base64(self): + column = pa.array([[b"hi"]], pa.list_(pa.binary())) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ['["aGk="]']} + + def test_non_finite_floats_are_null(self): + values = [float("inf"), float("-inf"), float("nan"), 1.5] + column = pa.array([values], pa.list_(pa.float64())) + tbl = Table(pa.table({"v": column}), list_flatten="stringify") + assert tbl.view().to_columns() == {"v": ["[null,null,null,1.5]"]} + assert self.flat_json(pa.array(values, pa.float64())) == [ + None, + None, + None, + 1.5, + ] + + def test_fixed_size_list_inside_a_struct(self): + item = pa.struct([("p", pa.list_(pa.int32(), 2))]) + column = pa.array([{"p": [1, 2]}], item) + tbl = Table(pa.table({"v": column}), struct_flatten="stringify") + assert tbl.view().to_columns() == {"v": ['{"p":[1,2]}']} + + +class TestTableArrowStringifyFormatted(object): + us_utc = pa.timestamp("us", tz="UTC") + instant_us = 1704164645000006 + text = "2024-01-02 03:04:05.000" + + def struct_column(self): + return pa.array( + [{"ts": self.instant_us, "d": 19724, "x": 1}], + pa.struct([("ts", self.us_utc), ("d", pa.date32()), ("x", pa.int64())]), + ) + + def map_column(self): + return pa.array( + [[("k", self.instant_us)]], pa.map_(pa.string(), self.us_utc) + ) + + def test_struct_formats_datetimes(self): + tbl = Table( + pa.table({"v": self.struct_column()}), + struct_flatten="stringify_formatted", + ) + + assert tbl.schema() == {"v": "string"} + assert tbl.view().to_columns() == { + "v": ['{"ts":"2024-01-02 03:04:05.000","d":"2024-01-02","x":1}'] + } + + def test_matches_formatted_to_json(self): + flat = Table( + pa.table( + { + "ts": pa.array([self.instant_us], self.us_utc), + "d": pa.array([19724], pa.date32()), + } + ) + ) + + assert json.loads(flat.view().to_columns_string(formatted=True)) == { + "ts": [self.text], + "d": ["2024-01-02"], + } + + def test_map_formats_datetimes(self): + tbl = Table( + pa.table({"v": self.map_column()}), map_flatten="stringify_formatted" + ) + + assert tbl.view().to_columns() == { + "v": ['{"k":"2024-01-02 03:04:05.000"}'] + } + + def test_unset_map_flatten_follows_formatted_struct(self): + tbl = Table( + pa.table({"v": self.map_column()}), + struct_flatten="stringify_formatted", + ) + + assert tbl.view().to_columns() == { + "v": ['{"k":"2024-01-02 03:04:05.000"}'] + } + + def test_explicit_map_flatten_overrides_formatted_struct(self): + tbl = Table( + pa.table({"v": self.map_column()}), + struct_flatten="stringify_formatted", + map_flatten="stringify", + ) + + assert tbl.view().to_columns() == {"v": ['{"k":1704164645000}']} + + def test_each_exploded_struct_is_formatted(self): + item = pa.struct([("ts", self.us_utc)]) + column = pa.array( + [[{"ts": self.instant_us}, {"ts": self.instant_us}]], pa.list_(item) + ) + + tbl = Table( + pa.table({"v": column}), struct_flatten="stringify_formatted" + ) + + assert tbl.view().to_columns() == { + "v": ['{"ts":"2024-01-02 03:04:05.000"}'] * 2 + } + + def test_outer_list_stringify_wins(self): + item = pa.struct([("ts", self.us_utc)]) + column = pa.array([[{"ts": self.instant_us}]], pa.list_(item)) + tbl = Table( + pa.table({"v": column}), + list_flatten="stringify", + struct_flatten="stringify_formatted", + ) + + assert tbl.view().to_columns() == {"v": ['[{"ts":1704164645000}]']} + + def test_list_formats_datetimes(self): + column = pa.array([[self.instant_us, None]], pa.list_(self.us_utc)) + tbl = Table(pa.table({"v": column}), list_flatten="stringify_formatted") + assert tbl.schema() == {"v": "string"} + assert tbl.view().to_columns() == { + "v": ['["2024-01-02 03:04:05.000",null]'] + } + + def test_list_of_structs_formats_through_the_outer_layer(self): + item = pa.struct([("ts", self.us_utc)]) + column = pa.array([[{"ts": self.instant_us}]], pa.list_(item)) + tbl = Table(pa.table({"v": column}), list_flatten="stringify_formatted") + assert tbl.view().to_columns() == { + "v": ['[{"ts":"2024-01-02 03:04:05.000"}]'] + } + + def test_json_arrays_are_unchanged(self): + tbl = Table([{"y": [1, 2]}], list_flatten="stringify_formatted") + assert tbl.view().to_columns() == {"y": ["[1,2]"]} + + def test_json_objects_are_unchanged(self): + tbl = Table([{"s": {"a": 1}}], struct_flatten="stringify_formatted") + assert tbl.view().to_columns() == {"s": ['{"a":1}']} + + def test_persists_across_update_on_a_schema_table(self): + tbl = Table({"v": "string"}, struct_flatten="stringify_formatted") + tbl.update(arrow_bytes(pa.table({"v": self.struct_column()}))) + assert tbl.view().to_columns() == { + "v": ['{"ts":"2024-01-02 03:04:05.000","d":"2024-01-02","x":1}'] + } diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index 47c8a8f894..2dad3c0455 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -72,6 +72,7 @@ fn parse_list_flatten(value: Option) -> PyResult> { Some("zip") => Ok(Some(ListFlatten::Zip)), Some("cartesian") => Ok(Some(ListFlatten::Cartesian)), Some("stringify") => Ok(Some(ListFlatten::Stringify)), + Some("stringify_formatted") => Ok(Some(ListFlatten::StringifyFormatted)), Some(_x) => Err(PyValueError::new_err("Unknown `list_flatten`".to_string())), } } @@ -81,6 +82,7 @@ fn parse_struct_flatten(value: Option) -> PyResult None => Ok(None), Some("pivot") => Ok(Some(StructFlatten::Pivot)), Some("stringify") => Ok(Some(StructFlatten::Stringify)), + Some("stringify_formatted") => Ok(Some(StructFlatten::StringifyFormatted)), Some(_x) => Err(PyValueError::new_err( "Unknown `struct_flatten`".to_string(), )), @@ -92,6 +94,7 @@ fn parse_map_flatten(value: Option) -> PyResult> { None => Ok(None), Some("pivot") => Ok(Some(MapFlatten::Pivot)), Some("stringify") => Ok(Some(MapFlatten::Stringify)), + Some("stringify_formatted") => Ok(Some(MapFlatten::StringifyFormatted)), Some(_x) => Err(PyValueError::new_err("Unknown `map_flatten`".to_string())), } } diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp index 90a629ecb4..e121b04fe3 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp @@ -13,8 +13,12 @@ #include "perspective/base.h" #include "perspective/raw_types.h" #include +#include #include #include +#include +#include +#include #include #include #include @@ -30,7 +34,6 @@ #include #include #include -#include namespace perspective::apachearrow { @@ -224,6 +227,7 @@ ArrowLoader::initialize( PSP_COMPLAIN_AND_ABORT(validation.ToString()); } + m_mode = mode; if (!normalize_table_is_noop(*m_table, mode)) { m_expanded = normalize_table_expands(*m_table, mode); m_normalized = @@ -388,6 +392,131 @@ iter_col_copy( } } +static std::int64_t +timestamp_to_ms(std::int64_t value, arrow::TimeUnit::type unit) { + switch (unit) { + case arrow::TimeUnit::SECOND: + return value * 1000; + case arrow::TimeUnit::MICRO: + return value / 1000; + case arrow::TimeUnit::NANO: + return value / 1000000; + case arrow::TimeUnit::MILLI: + default: + return value; + } +} + +/** + * `date::month` is [1-12] where `t_date::month()` is [0-11]. + */ +static t_date +date_from_days(date::sys_days days) { + const auto ymd = date::year_month_day{days}; + return t_date( + static_cast(ymd.year()), + static_cast(ymd.month()) - 1, + static_cast(ymd.day()) + ); +} + +static date::sys_days +date32_days(std::int32_t days) { + return date::sys_days{date::days{days}}; +} + +static date::sys_days +date64_days(std::int64_t ms) { + return date::sys_days{ + date::floor(std::chrono::milliseconds(ms)) + }; +} + +/** + * The `t_tscalar` the flat ingest path stores for this scalar element. + */ +static t_tscalar +leaf_scalar(const arrow::Array& array, std::int64_t i) { + switch (array.type_id()) { + case arrow::Type::BOOL: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::INT8: + return mktscalar(static_cast(array).Value(i) + ); + case arrow::Type::INT16: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::INT32: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::INT64: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::UINT8: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::UINT16: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::UINT32: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::UINT64: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::HALF_FLOAT: + return mktscalar( + arrow::util::Float16::FromBits( + static_cast(array).Value(i) + ) + .ToFloat() + ); + case arrow::Type::FLOAT: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::DOUBLE: + return mktscalar( + static_cast(array).Value(i) + ); + case arrow::Type::TIMESTAMP: { + const auto unit = + static_cast(*array.type()).unit(); + const auto value = + static_cast(array).Value(i); + return mktscalar(t_time(timestamp_to_ms(value, unit))); + } + case arrow::Type::DATE32: + return mktscalar(date_from_days(date32_days( + static_cast(array).Value(i) + ))); + case arrow::Type::DATE64: + return mktscalar(date_from_days(date64_days( + static_cast(array).Value(i) + ))); + default: + PSP_COMPLAIN_AND_ABORT("Not a scalar Arrow type"); + return mktscalar(); + } +} + +static void +write_base64( + rapidjson::Writer& writer, std::string_view bytes +) { + const auto text = arrow::util::base64_encode(bytes); + writer.String(text.data(), static_cast(text.size())); +} + /** * Write element `i` of `array` as JSON, recursing into nested values. */ @@ -395,7 +524,8 @@ static void write_json( rapidjson::Writer& writer, const arrow::Array& array, - std::int64_t i + std::int64_t i, + bool formatted ) { if (!array.IsValid(i)) { writer.Null(); @@ -403,59 +533,6 @@ write_json( } switch (array.type_id()) { - case arrow::Type::BOOL: { - writer.Bool(static_cast(array).Value(i)); - } break; - case arrow::Type::INT8: { - writer.Int64(static_cast(array).Value(i)); - } break; - case arrow::Type::INT16: { - writer.Int64(static_cast(array).Value(i)); - } break; - case arrow::Type::INT32: { - writer.Int64(static_cast(array).Value(i)); - } break; - case arrow::Type::INT64: { - writer.Int64(static_cast(array).Value(i)); - } break; - case arrow::Type::UINT8: { - writer.Uint64(static_cast(array).Value(i)); - } break; - case arrow::Type::UINT16: { - writer.Uint64(static_cast(array).Value(i) - ); - } break; - case arrow::Type::UINT32: { - writer.Uint64(static_cast(array).Value(i) - ); - } break; - case arrow::Type::UINT64: { - writer.Uint64(static_cast(array).Value(i) - ); - } break; - case arrow::Type::HALF_FLOAT: { - const auto bits = - static_cast(array).Value(i); - writer.Double(arrow::util::Float16::FromBits(bits).ToFloat()); - } break; - case arrow::Type::FLOAT: { - const double v = - static_cast(array).Value(i); - if (std::isfinite(v)) { - writer.Double(v); - } else { - writer.Null(); - } - } break; - case arrow::Type::DOUBLE: { - const double v = - static_cast(array).Value(i); - if (std::isfinite(v)) { - writer.Double(v); - } else { - writer.Null(); - } - } break; case arrow::Type::STRING: { const auto view = static_cast(array).GetView(i); @@ -472,7 +549,9 @@ write_json( } break; case arrow::Type::DICTIONARY: { const auto& dict = static_cast(array); - write_json(writer, *dict.dictionary(), dict.GetValueIndex(i)); + write_json( + writer, *dict.dictionary(), dict.GetValueIndex(i), formatted + ); } break; case arrow::Type::LIST: { const auto& list = static_cast(array); @@ -480,7 +559,7 @@ write_json( writer.StartArray(); for (auto e = list.value_offset(i); e < list.value_offset(i + 1); ++e) { - write_json(writer, values, e); + write_json(writer, values, e, formatted); } writer.EndArray(); @@ -491,7 +570,7 @@ write_json( writer.StartArray(); for (auto e = list.value_offset(i); e < list.value_offset(i + 1); ++e) { - write_json(writer, values, e); + write_json(writer, values, e, formatted); } writer.EndArray(); @@ -506,7 +585,7 @@ write_json( name.data(), static_cast(name.size()) ); - write_json(writer, *strct.field(c), i); + write_json(writer, *strct.field(c), i, formatted); } writer.EndObject(); @@ -538,15 +617,98 @@ write_json( } else { rapidjson::StringBuffer key; rapidjson::Writer key_writer(key); - write_json(key_writer, keys, e); + write_json(key_writer, keys, e, formatted); writer.Key(key.GetString(), key.GetSize()); } - write_json(writer, items, e); + write_json(writer, items, e, formatted); } writer.EndObject(); } break; + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + case arrow::Type::UINT8: + case arrow::Type::UINT16: + case arrow::Type::UINT32: + case arrow::Type::UINT64: + case arrow::Type::HALF_FLOAT: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: + case arrow::Type::TIMESTAMP: + case arrow::Type::DATE32: + case arrow::Type::DATE64: { + write_scalar(leaf_scalar(array, i), formatted, writer); + } break; + case arrow::Type::TIME32: { + writer.Int64(static_cast(array).Value(i) + ); + } break; + case arrow::Type::TIME64: { + writer.Int64(static_cast(array).Value(i) + ); + } break; + case arrow::Type::DURATION: { + writer.Int64( + static_cast(array).Value(i) + ); + } break; + case arrow::Type::DECIMAL128: { + const auto scale = + static_cast(*array.type()).scale(); + const auto text = + arrow::Decimal128( + static_cast(array).GetValue(i) + ) + .ToString(scale); + writer.String( + text.data(), static_cast(text.size()) + ); + } break; + case arrow::Type::DECIMAL256: { + const auto scale = + static_cast(*array.type()).scale(); + const auto text = + arrow::Decimal256( + static_cast(array).GetValue(i) + ) + .ToString(scale); + writer.String( + text.data(), static_cast(text.size()) + ); + } break; + case arrow::Type::BINARY: { + write_base64( + writer, static_cast(array).GetView(i) + ); + } break; + case arrow::Type::LARGE_BINARY: { + write_base64( + writer, + static_cast(array).GetView(i) + ); + } break; + case arrow::Type::FIXED_SIZE_BINARY: { + write_base64( + writer, + static_cast(array).GetView(i) + ); + } break; + case arrow::Type::FIXED_SIZE_LIST: { + const auto& list = + static_cast(array); + const auto& values = *list.values(); + const auto start = list.value_offset(i); + writer.StartArray(); + for (auto e = start; e < start + list.value_length(i); ++e) { + write_json(writer, values, e, formatted); + } + + writer.EndArray(); + } break; default: { std::stringstream ss; ss << "Cannot stringify Arrow value of type `" @@ -556,6 +718,25 @@ write_json( } } +/** + * Whether a stringified top-level column of this type writes its datetimes as + * `to_json`'s `formatted` text rather than epoch milliseconds. + */ +static bool +stringify_formatted(t_flatten_mode mode, arrow::Type::type id) { + switch (id) { + case arrow::Type::LIST: + case arrow::Type::LARGE_LIST: + return mode.list == LIST_FLATTEN_STRINGIFY_FORMATTED; + case arrow::Type::STRUCT: + return mode.structs == STRUCT_FLATTEN_STRINGIFY_FORMATTED; + case arrow::Type::MAP: + return mode.maps == MAP_FLATTEN_STRINGIFY_FORMATTED; + default: + return false; + } +} + template void copy_array_impl( @@ -563,13 +744,15 @@ copy_array_impl( const std::shared_ptr& src, const int64_t offset, const int64_t len, - const GATHER& gather + const GATHER& gather, + t_flatten_mode mode ) { switch (src->type()->id()) { case arrow::Type::LIST: case arrow::Type::LARGE_LIST: case arrow::Type::STRUCT: case arrow::Type::MAP: { + const bool formatted = stringify_formatted(mode, src->type_id()); rapidjson::StringBuffer buffer; for (std::int64_t i = 0; i < len; ++i) { if (gather.is_null(i)) { @@ -583,7 +766,7 @@ copy_array_impl( buffer.Clear(); rapidjson::Writer writer(buffer); - write_json(writer, *src, row); + write_json(writer, *src, row, formatted); dest->set_nth(offset + i, buffer.GetString()); } } break; @@ -766,27 +949,12 @@ copy_array_impl( } } } break; - case arrow::TimeUnit::NANO: { - const int64_t* vals = scol->raw_values(); - for (int64_t i = 0; i < len; i++) { - dest->set_nth( - offset + i, vals[gather[i]] / 1000000 - ); - } - } break; - case arrow::TimeUnit::MICRO: { - const int64_t* vals = scol->raw_values(); - for (int64_t i = 0; i < len; i++) { - dest->set_nth( - offset + i, vals[gather[i]] / 1000 - ); - } - } break; - case arrow::TimeUnit::SECOND: { + default: { const int64_t* vals = scol->raw_values(); for (int64_t i = 0; i < len; i++) { dest->set_nth( - offset + i, vals[gather[i]] * 1000 + offset + i, + timestamp_to_ms(vals[gather[i]], tunit->unit()) ); } } break; @@ -798,15 +966,9 @@ copy_array_impl( auto scol = std::static_pointer_cast(src); const int64_t* vals = scol->raw_values(); for (int64_t i = 0; i < len; i++) { - std::chrono::milliseconds timestamp(vals[gather[i]]); - date::sys_days days(date::floor(timestamp)); - auto ymd = date::year_month_day{days}; - std::int32_t year = static_cast(ymd.year()); - std::uint32_t month = static_cast(ymd.month()); - std::uint32_t day = static_cast(ymd.day()); - // Decrement month by 1, as date::month is - // [1-12] but t_date::month() is [0-11] - dest->set_nth(offset + i, t_date(year, month - 1, day)); + dest->set_nth( + offset + i, date_from_days(date64_days(vals[gather[i]])) + ); } } break; case arrow::Date32Type::type_id: { @@ -815,15 +977,9 @@ copy_array_impl( auto scol = std::static_pointer_cast(src); const int32_t* vals = scol->raw_values(); for (int64_t i = 0; i < len; i++) { - date::days days{vals[gather[i]]}; - auto ymd = date::year_month_day{date::sys_days{days}}; - // years are signed, month/day are unsigned - std::int32_t year = static_cast(ymd.year()); - std::uint32_t month = static_cast(ymd.month()); - std::uint32_t day = static_cast(ymd.day()); - // Decrement month by 1, as date::month is - // [1-12] but t_date::month() is [0-11] - dest->set_nth(offset + i, t_date(year, month - 1, day)); + dest->set_nth( + offset + i, date_from_days(date32_days(vals[gather[i]])) + ); } } break; case arrow::FloatType::type_id: { @@ -884,7 +1040,7 @@ copy_array( const int64_t offset, const int64_t len ) { - copy_array_impl(dest, src, offset, len, t_identity_gather{}); + copy_array_impl(dest, src, offset, len, t_identity_gather{}, t_flatten_mode{}); } // Defines the full matrix of type interactions between arrow arrays and @@ -991,7 +1147,8 @@ void fill_column_chunk( std::int64_t offset, std::int64_t len, bool is_update, - const GATHER& gather + const GATHER& gather, + t_flatten_mode mode ); void @@ -1044,7 +1201,8 @@ ArrowLoader::fill_column( 0, static_cast(indices->size()), is_update, - t_index_gather{indices->data()} + t_index_gather{indices->data()}, + m_mode ); return; @@ -1055,7 +1213,15 @@ ArrowLoader::fill_column( std::shared_ptr array = carray->chunk(i); int64_t len = array->length(); fill_column_chunk( - col, array, name, type, offset, len, is_update, t_identity_gather{} + col, + array, + name, + type, + offset, + len, + is_update, + t_identity_gather{}, + m_mode ); offset += len; @@ -1072,7 +1238,8 @@ fill_column_chunk( std::int64_t offset, std::int64_t len, bool is_update, - const GATHER& gather + const GATHER& gather, + t_flatten_mode mode ) { { // A null-typed array carries no values and no validity bitmap: every @@ -1134,7 +1301,7 @@ fill_column_chunk( }; } } else { - copy_array_impl(col, array, offset, len, gather); + copy_array_impl(col, array, offset, len, gather, mode); } fill_validity(col, array, offset, len, is_update, gather); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp index bbbb7dbfe0..2a64fd0442 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/arrow_normalize.cpp @@ -47,8 +47,7 @@ namespace { return true; } - if (type.id() == arrow::Type::STRUCT - && mode.structs != STRUCT_FLATTEN_STRINGIFY) { + if (type.id() == arrow::Type::STRUCT && !stringifies(mode.structs)) { for (const auto& child : type.fields()) { if (type_expands(*child->type(), mode)) { return true; @@ -56,8 +55,7 @@ namespace { } } - if (type.id() == arrow::Type::MAP - && mode.maps != MAP_FLATTEN_STRINGIFY) { + if (type.id() == arrow::Type::MAP && !stringifies(mode.maps)) { return type_expands( *static_cast(type).item_type(), mode ); @@ -745,16 +743,15 @@ bool normalize_table_is_noop(const arrow::Table& input, t_flatten_mode mode) { for (const auto& field : input.schema()->fields()) { const auto id = field->type()->id(); - if (mode.structs != STRUCT_FLATTEN_STRINGIFY - && id == arrow::Type::STRUCT) { + if (!stringifies(mode.structs) && id == arrow::Type::STRUCT) { return false; } - if (mode.maps != MAP_FLATTEN_STRINGIFY && id == arrow::Type::MAP) { + if (!stringifies(mode.maps) && id == arrow::Type::MAP) { return false; } - if (mode.list != LIST_FLATTEN_STRINGIFY && is_list_id(id)) { + if (!stringifies(mode.list) && is_list_id(id)) { return false; } } @@ -764,7 +761,7 @@ normalize_table_is_noop(const arrow::Table& input, t_flatten_mode mode) { bool normalize_table_expands(const arrow::Table& input, t_flatten_mode mode) { - if (mode.list == LIST_FLATTEN_STRINGIFY) { + if (stringifies(mode.list)) { return false; } @@ -787,14 +784,14 @@ normalize_table(std::shared_ptr input, t_flatten_mode mode) { out.num_rows = input->num_rows(); while (true) { bool changed = false; - if (mode.structs != STRUCT_FLATTEN_STRINGIFY + if (!stringifies(mode.structs) && flatten_structs( out.fields, out.columns, out.gathers, out.per_element )) { changed = true; } - if (mode.maps != MAP_FLATTEN_STRINGIFY + if (!stringifies(mode.maps) && pivot_maps( out.fields, out.columns, @@ -805,7 +802,7 @@ normalize_table(std::shared_ptr input, t_flatten_mode mode) { changed = true; } - if (mode.list != LIST_FLATTEN_STRINGIFY + if (!stringifies(mode.list) && explode_lists( out.fields, out.columns, diff --git a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp index 5eed845b06..c39cc4e363 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/json_loader.cpp @@ -434,7 +434,7 @@ for_each_leaf( F&& fn, std::size_t array_path = NO_ARRAY ) { - if (mode.list != LIST_FLATTEN_STRINGIFY && value.IsArray()) { + if (!stringifies(mode.list) && value.IsArray()) { // Inference only needs one element to learn the leaves' types; any // path a later element introduces is grown into by `resolve_column`. if (!value.Empty()) { @@ -444,7 +444,7 @@ for_each_leaf( return; } - if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { + if (!value.IsObject() || stringifies(mode.structs)) { fn(static_cast(prefix), value, array_path); return; } @@ -505,7 +505,7 @@ check_row_count(std::uint64_t rows) { */ static t_uindex leaf_width(const rapidjson::Value& value, const t_flatten_mode& mode) { - if (mode.list == LIST_FLATTEN_STRINGIFY || value.IsNull()) { + if (stringifies(mode.list) || value.IsNull()) { return 1; } @@ -519,7 +519,7 @@ leaf_width(const rapidjson::Value& value, const t_flatten_mode& mode) { return std::max(total, 1); } - if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { + if (!value.IsObject() || stringifies(mode.structs)) { return 1; } @@ -559,8 +559,8 @@ leaf_width(const rapidjson::Value& value, const t_flatten_mode& mode) { */ static bool is_nested(const rapidjson::Value& value, const t_flatten_mode& mode) { - return (value.IsObject() && mode.structs != STRUCT_FLATTEN_STRINGIFY) - || (mode.list != LIST_FLATTEN_STRINGIFY && value.IsArray()); + return (value.IsObject() && !stringifies(mode.structs)) + || (!stringifies(mode.list) && value.IsArray()); } /** @@ -616,7 +616,7 @@ emit_leaves( const t_flatten_mode& mode, F&& fn ) { - if (mode.list == LIST_FLATTEN_STRINGIFY || value.IsNull()) { + if (stringifies(mode.list) || value.IsNull()) { fn(std::string_view{prefix}, value); return; } @@ -648,7 +648,7 @@ emit_leaves( return; } - if (!value.IsObject() || mode.structs == STRUCT_FLATTEN_STRINGIFY) { + if (!value.IsObject() || stringifies(mode.structs)) { fn(std::string_view{prefix}, value); return; } @@ -1043,8 +1043,8 @@ JsonLoader::infer_cols(const std::vector& index) { // Whether this cell contributes the column itself rather than // paths beneath it -- the same test `for_each_leaf` applies. const bool is_leaf = - (!cell.IsObject() || m_mode.structs == STRUCT_FLATTEN_STRINGIFY) - && (m_mode.list == LIST_FLATTEN_STRINGIFY || !cell.IsArray()); + (!cell.IsObject() || stringifies(m_mode.structs)) + && (stringifies(m_mode.list) || !cell.IsArray()); saw_leaf = saw_leaf || is_leaf; path.assign(it.name.GetString()); diff --git a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp index 4bdbf990be..f8193ea716 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/scalar.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1764,6 +1765,88 @@ t_tscalar::coerce_numeric() const { return rv; } +void +write_scalar( + t_tscalar scalar, + bool is_formatted, + rapidjson::Writer& writer +) { + + if (!scalar.is_valid()) { + writer.Null(); + return; + } + + switch (scalar.get_dtype()) { + case DTYPE_NONE: + writer.Null(); + break; + case DTYPE_BOOL: + writer.Bool(scalar.get()); + break; + case DTYPE_INT8: + writer.Int(scalar.get()); + break; + case DTYPE_INT16: + writer.Int(scalar.get()); + break; + case DTYPE_INT32: + writer.Int(scalar.get()); + break; + case DTYPE_INT64: + writer.Int64(scalar.get()); + break; + case DTYPE_UINT8: + writer.Uint(scalar.get()); + break; + case DTYPE_UINT16: + writer.Uint(scalar.get()); + break; + case DTYPE_UINT32: + writer.Uint(scalar.get()); + break; + case DTYPE_UINT64: + writer.Uint64(scalar.get()); + break; + case DTYPE_FLOAT32: + if (scalar.is_nan()) { + writer.Null(); + } else { + writer.Double(scalar.get()); + } + break; + case DTYPE_FLOAT64: + if (scalar.is_nan()) { + writer.Null(); + } else { + writer.Double(scalar.get()); + } + break; + case DTYPE_STR: + writer.String(scalar.get()); + break; + case DTYPE_TIME: + if (is_formatted) { + writer.String(scalar.to_string().c_str()); + } else { + writer.Int64(scalar.get()); + } + + break; + case DTYPE_DATE: { + if (is_formatted) { + writer.String(scalar.to_string().c_str()); + } else { + writer.Int64(scalar.get().as_epoch_ms()); + } + break; + } + + default: + break; + } +} + } // end namespace perspective namespace std { diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index f17a05c9ba..f0fb4229d5 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -2580,6 +2580,9 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { case proto::LIST_FLATTEN_STRINGIFY: flatten.list = LIST_FLATTEN_STRINGIFY; break; + case proto::LIST_FLATTEN_STRINGIFY_FORMATTED: + flatten.list = LIST_FLATTEN_STRINGIFY_FORMATTED; + break; case proto::LIST_FLATTEN_ZIP: default: flatten.list = LIST_FLATTEN_ZIP; @@ -2590,6 +2593,9 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { case proto::STRUCT_FLATTEN_STRINGIFY: flatten.structs = STRUCT_FLATTEN_STRINGIFY; break; + case proto::STRUCT_FLATTEN_STRINGIFY_FORMATTED: + flatten.structs = STRUCT_FLATTEN_STRINGIFY_FORMATTED; + break; case proto::STRUCT_FLATTEN_PIVOT: default: flatten.structs = STRUCT_FLATTEN_PIVOT; @@ -2597,14 +2603,26 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { } if (!r.options().has_map_flatten()) { - flatten.maps = flatten.structs == STRUCT_FLATTEN_STRINGIFY - ? MAP_FLATTEN_STRINGIFY - : MAP_FLATTEN_PIVOT; + switch (flatten.structs) { + case STRUCT_FLATTEN_STRINGIFY: + flatten.maps = MAP_FLATTEN_STRINGIFY; + break; + case STRUCT_FLATTEN_STRINGIFY_FORMATTED: + flatten.maps = MAP_FLATTEN_STRINGIFY_FORMATTED; + break; + case STRUCT_FLATTEN_PIVOT: + default: + flatten.maps = MAP_FLATTEN_PIVOT; + break; + } } else { switch (r.options().map_flatten()) { case proto::MAP_FLATTEN_STRINGIFY: flatten.maps = MAP_FLATTEN_STRINGIFY; break; + case proto::MAP_FLATTEN_STRINGIFY_FORMATTED: + flatten.maps = MAP_FLATTEN_STRINGIFY_FORMATTED; + break; case proto::MAP_FLATTEN_PIVOT: default: flatten.maps = MAP_FLATTEN_PIVOT; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp index ca87ae1d0f..f6267b6279 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp @@ -1756,88 +1756,6 @@ View::_map_aggregate_types( return typestring; } -void -write_scalar( - t_tscalar scalar, - bool is_formatted, - rapidjson::Writer& writer -) { - - if (!scalar.is_valid()) { - writer.Null(); - return; - } - - switch (scalar.get_dtype()) { - case DTYPE_NONE: - writer.Null(); - break; - case DTYPE_BOOL: - writer.Bool(scalar.get()); - break; - case DTYPE_INT8: - writer.Int(scalar.get()); - break; - case DTYPE_INT16: - writer.Int(scalar.get()); - break; - case DTYPE_INT32: - writer.Int(scalar.get()); - break; - case DTYPE_INT64: - writer.Int64(scalar.get()); - break; - case DTYPE_UINT8: - writer.Uint(scalar.get()); - break; - case DTYPE_UINT16: - writer.Uint(scalar.get()); - break; - case DTYPE_UINT32: - writer.Uint(scalar.get()); - break; - case DTYPE_UINT64: - writer.Uint64(scalar.get()); - break; - case DTYPE_FLOAT32: - if (scalar.is_nan()) { - writer.Null(); - } else { - writer.Double(scalar.get()); - } - break; - case DTYPE_FLOAT64: - if (scalar.is_nan()) { - writer.Null(); - } else { - writer.Double(scalar.get()); - } - break; - case DTYPE_STR: - writer.String(scalar.get()); - break; - case DTYPE_TIME: - if (is_formatted) { - writer.String(scalar.to_string().c_str()); - } else { - writer.Int64(scalar.get()); - } - - break; - case DTYPE_DATE: { - if (is_formatted) { - writer.String(scalar.to_string().c_str()); - } else { - writer.Int64(scalar.get().as_epoch_ms()); - } - break; - } - - default: - break; - } -} - template void View::write_row_path( diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h index 6799580b9b..1328f8b3eb 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_loader.h @@ -129,6 +129,7 @@ namespace apachearrow { std::vector m_names; std::vector m_types; bool m_expanded{false}; + t_flatten_mode m_mode; }; template diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h index 21f3bd8a0a..11a2de1f56 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/arrow_normalize.h @@ -29,11 +29,15 @@ namespace apachearrow { using perspective::t_struct_flatten; using perspective::LIST_FLATTEN_CARTESIAN; using perspective::LIST_FLATTEN_STRINGIFY; + using perspective::LIST_FLATTEN_STRINGIFY_FORMATTED; using perspective::LIST_FLATTEN_ZIP; + using perspective::stringifies; using perspective::MAP_FLATTEN_PIVOT; using perspective::MAP_FLATTEN_STRINGIFY; + using perspective::MAP_FLATTEN_STRINGIFY_FORMATTED; using perspective::STRUCT_FLATTEN_PIVOT; using perspective::STRUCT_FLATTEN_STRINGIFY; + using perspective::STRUCT_FLATTEN_STRINGIFY_FORMATTED; extern const char* const FLATTEN_SEPARATOR; /** diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h index 2be2baa371..90b0b659e0 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/flatten_mode.h @@ -17,15 +17,36 @@ namespace perspective { enum t_list_flatten { LIST_FLATTEN_ZIP = 0, LIST_FLATTEN_CARTESIAN = 1, - LIST_FLATTEN_STRINGIFY = 2 + LIST_FLATTEN_STRINGIFY = 2, + LIST_FLATTEN_STRINGIFY_FORMATTED = 3 }; enum t_struct_flatten { STRUCT_FLATTEN_PIVOT = 0, - STRUCT_FLATTEN_STRINGIFY = 1 + STRUCT_FLATTEN_STRINGIFY = 1, + STRUCT_FLATTEN_STRINGIFY_FORMATTED = 2 }; -enum t_map_flatten { MAP_FLATTEN_PIVOT = 0, MAP_FLATTEN_STRINGIFY = 1 }; +enum t_map_flatten { + MAP_FLATTEN_PIVOT = 0, + MAP_FLATTEN_STRINGIFY = 1, + MAP_FLATTEN_STRINGIFY_FORMATTED = 2 +}; + +inline bool +stringifies(t_list_flatten mode) { + return mode >= LIST_FLATTEN_STRINGIFY; +} + +inline bool +stringifies(t_struct_flatten mode) { + return mode != STRUCT_FLATTEN_PIVOT; +} + +inline bool +stringifies(t_map_flatten mode) { + return mode != MAP_FLATTEN_PIVOT; +} struct t_flatten_mode { t_list_flatten list{LIST_FLATTEN_ZIP}; diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/scalar_json.h b/rust/perspective-server/cpp/perspective/src/include/perspective/scalar_json.h new file mode 100644 index 0000000000..3d12bac540 --- /dev/null +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/scalar_json.h @@ -0,0 +1,33 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace perspective { + +/** + * @brief Write `scalar` as the JSON value `View::to_json` emits for it. + */ +PERSPECTIVE_EXPORT void write_scalar( + t_tscalar scalar, + bool is_formatted, + rapidjson::Writer& writer +); + +} // namespace perspective diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h index 01e13d5ba8..55bb0e775f 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -39,12 +40,6 @@ namespace perspective { */ enum class t_arrow_compression : std::uint8_t { NONE, LZ4, ZSTD }; -void write_scalar( - t_tscalar scalar, - bool is_formatted, - rapidjson::Writer& writer -); - template class PERSPECTIVE_EXPORT View { public: