Add struct_flatten, fix list_flatten options

docs/md/how_to/javascript/loading_data.md

@@ -74,3 +74,35 @@ than `<perspective-viewer>`s which rely on a Web Worker, and can even share a
 host application with Web Worker-bound `table()`s. The same `promise`-based API
 is used to communicate with the server-instantiated `view()`, only in this case
 it is over a websocket.
+
+## Nested columns
+
+Perspective's data model is flat, so nested Arrow and JSON values are
+normalized on ingest, controlled by two `Table` options.
+
+`struct_flatten` applies to Arrow `struct` and `map` columns and to JSON
+objects. `"pivot"` (the default) hoists each field or key into its own dotted
+column, so `{ s: { a: 1 } }` becomes a column `s.a` and a map `m` becomes one
+column `m.<key>` per distinct key in the batch, with a null where a row lacks
+the key. `"stringify"` keeps the whole value as JSON text in a single string
+column.
+
+`list_flatten` applies to Arrow `list` columns and JSON arrays. `"zip"` (the
+default) expands a row into one row per element, repeating its siblings, and
+requires sibling lists in a row to agree in length. `"cartesian"` expands to
+the product of the lists' lengths. `"stringify"` keeps each list as a JSON
+array in a string column.
+
+The two options compose from the outside in: a list inside a stringified
+struct stays inside its JSON text, while a struct inside an expanded list is
+stringified once per element row.
+
+```javascript
+const table = await worker.table(
+    [{ id: 1, m: { a: 1, b: 2 } }, { id: 2, m: { b: 3 } }],
+    { struct_flatten: "pivot" },
+);
+
+// { id: [1, 2], "m.a": [1, null], "m.b": [2, 3] }
+await (await table.view()).to_columns();
+```

docs/md/how_to/python/table_data.md

@@ -38,8 +38,8 @@ with open("data.arrow", "rb") as f:
 
 ### Nested columns
 
-Perspective's data model is flat, so Arrow `struct` and `list` columns are
-normalized on ingest.
+Perspective's data model is flat, so Arrow `struct`, `map` and `list` columns
+are normalized on ingest.
 
 A `struct` column is hoisted into one dotted column per leaf, recursively. A
 null parent nulls every descendant leaf:
@@ -86,6 +86,51 @@ perspective.table(arrow_table)
 perspective.table(arrow_table, list_flatten="stringify")
 ```
 
+A `map` column is pivoted into one column per distinct key, `m.<key>`, of the
+map's item type, so an Arrow map and a JSON object produce the same columns. A
+key absent from a row yields a null in that row. A key absent from the whole
+batch yields no column, which on an indexed update leaves that cell unchanged,
+so declare the keys in a schema when updates must be stable. Only string keys
+are supported, and pivoting suits maps with a bounded key set since each key
+becomes a column:
+
+```python
+arrow_table = pa.table({
+    "id": pa.array([1, 2], type=pa.int64()),
+    "m": pa.array(
+        [[("a", 1), ("b", 2)], [("b", 3)]],
+        type=pa.map_(pa.string(), pa.int64()),
+    ),
+})
+
+# `{"id": [1, 2], "m.a": [1, None], "m.b": [2, 3]}`
+perspective.table(arrow_table)
+```
+
+Both `struct` and `map` columns are controlled by the `struct_flatten`
+argument:
+
+-   `"pivot"` (default) hoists each field or key into its own dotted column,
+    as above.
+-   `"stringify"` keeps the whole value as JSON text in a single string
+    column.
+
+The two arguments compose from the outside in. A list inside a stringified
+struct is an array within its JSON text and is never expanded, while a struct
+inside an expanded list is stringified once per element row:
+
+```python
+arrow_table = pa.table({
+    "s": pa.array(
+        [{"a": 1, "tags": ["x", "y"]}],
+        type=pa.struct([("a", pa.int64()), ("tags", pa.list_(pa.string()))]),
+    ),
+})
+
+# `{"s": ['{"a":1,"tags":["x","y"]}']}`
+perspective.table(arrow_table, struct_flatten="stringify")
+```
+
 ## Polars
 
 ```python

rust/perspective-client/build.rs

@@ -76,6 +76,10 @@ fn prost_build() -> Result<()> {
                 "ListFlatten",
                 "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]",
             )
+            .type_attribute(
+                "StructFlatten",
+                "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]",
+            )
             .field_attribute("ViewToArrowResp.arrow", "#[serde(skip)]")
             .field_attribute("from_arrow", "#[serde(skip)]")
             .type_attribute(".", "#[derive(serde::Serialize)]")

rust/perspective-client/perspective.proto

@@ -405,6 +405,8 @@ message MakeTableReq {
 
         optional ListFlatten list_flatten = 4;
 
+        optional StructFlatten struct_flatten = 7;
+
         // The columns of a `Table` made from a `View`, in place of the ones
         // inferred from it. Only valid with `MakeTableData.from_view`.
         optional Schema view_schema = 5;
@@ -416,6 +418,11 @@ enum ListFlatten {
     LIST_FLATTEN_CARTESIAN = 1;
     LIST_FLATTEN_STRINGIFY = 2;
 }
+
+enum StructFlatten {
+    STRUCT_FLATTEN_PIVOT = 0;
+    STRUCT_FLATTEN_STRINGIFY = 1;
+}
 message MakeTableResp {}
 
 enum JoinType {

rust/perspective-client/src/rust/client.rs

@@ -726,6 +726,7 @@ impl Client {
                 limit: None,
                 page_to_disk: None,
                 list_flatten: None,
+                struct_flatten: None,
                 schema: None,
             })),
             resp => Err(resp.into()),
@@ -773,6 +774,7 @@ impl Client {
                 limit: info.limit,
                 page_to_disk: None,
                 list_flatten: None,
+                struct_flatten: None,
                 schema: None,
             };
 

rust/perspective-client/src/rust/table.rs

@@ -166,11 +166,20 @@ pub struct TableInitOptions {
     /// default) and `cartesian` expand a row into one row per list element,
     /// and are incompatible with `index`, as the rows of an expansion
     /// repeat their index. `stringify` encodes each list as a JSON array in
-    /// a single string column instead.
+    /// a single string column instead. `STRUCT` and `MAP` columns are
+    /// governed by `struct_flatten`.
     #[serde(default)]
     #[ts(optional)]
     pub list_flatten: Option<crate::proto::ListFlatten>,
 
+    /// How Arrow `STRUCT` and `MAP` columns and JSON objects are ingested.
+    /// `pivot` (the default) hoists each field or key into its own dotted
+    /// column, and `stringify` keeps the whole value as JSON text in a single
+    /// string column.
+    #[serde(default)]
+    #[ts(optional)]
+    pub struct_flatten: Option<crate::proto::StructFlatten>,
+
     /// The columns of a [`Table`] made from a [`View`], in place of the ones
     /// inferred from it.
     #[serde(default)]
@@ -190,6 +199,7 @@ impl TryFrom<TableOptions> for MakeTableOptions {
     fn try_from(value: TableOptions) -> Result<Self, Self::Error> {
         let page_to_disk = value.page_to_disk;
         let list_flatten = value.list_flatten.map(|x| x as i32);
+        let struct_flatten = value.struct_flatten.map(|x| x as i32);
         let view_schema = value.schema.clone().map(|schema| crate::proto::Schema {
             schema: schema
                 .into_iter()
@@ -203,6 +213,7 @@ impl TryFrom<TableOptions> for MakeTableOptions {
         Ok(MakeTableOptions {
             page_to_disk,
             list_flatten,
+            struct_flatten,
             view_schema,
             make_table_type: match value {
                 TableOptions {
@@ -235,6 +246,7 @@ pub(crate) struct TableOptions {
     pub limit: Option<u32>,
     pub page_to_disk: Option<bool>,
     pub list_flatten: Option<crate::proto::ListFlatten>,
+    pub struct_flatten: Option<crate::proto::StructFlatten>,
     pub schema: Option<IndexMap<String, ColumnType>>,
 }
 
@@ -254,6 +266,7 @@ impl From<TableInitOptions> for TableOptions {
             limit: value.limit,
             page_to_disk: value.page_to_disk,
             list_flatten: value.list_flatten,
+            struct_flatten: value.struct_flatten,
             schema: value.schema,
         }
     }