Add map_flatten fallback to struct_flatten

.github/workflows/build.yaml

@@ -721,7 +721,7 @@ jobs:
 
             - name: Package
               if: ${{ steps.config-step.outputs.FULL_RUN }}
-              run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-migrate -p perspective-python
+              run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-python
 
             - uses: actions/upload-artifact@v4
               if: ${{ steps.config-step.outputs.FULL_RUN }}

docs/md/how_to/javascript/loading_data.md

@@ -78,24 +78,34 @@ it is over a websocket.
 ## Nested columns
 
 Perspective's data model is flat, so nested Arrow and JSON values are
-normalized on ingest, controlled by two `Table` options.
+normalized on ingest, controlled by three `Table` options.
 
-`struct_flatten` applies to Arrow `struct` and `map` columns and to JSON
-objects. `"pivot"` (the default) hoists each field or key into its own dotted
-column, so `{ s: { a: 1 } }` becomes a column `s.a` and a map `m` becomes one
+`struct_flatten` applies to Arrow `struct` columns and to JSON objects.
+`"pivot"` (the default) hoists each field into its own dotted column, so
+`{ s: { a: 1 } }` becomes a column `s.a`. `"stringify"` keeps the whole value
+as JSON text in a single string column, with datetimes as epoch milliseconds;
+`"stringify_formatted"` writes them as the text `to_json` emits under
+`formatted`.
+
+`map_flatten` applies to Arrow `map` columns. `"pivot"` makes a map `m` one
 column `m.<key>` per distinct key in the batch, with a null where a row lacks
-the key. `"stringify"` keeps the whole value as JSON text in a single string
-column.
+the key, and `"stringify"` or `"stringify_formatted"` keeps the map as JSON
+text. When unset it follows `struct_flatten`. JSON has no map type, so JSON objects always follow
+`struct_flatten`.
 
 `list_flatten` applies to Arrow `list` columns and JSON arrays. `"zip"` (the
 default) expands a row into one row per element, repeating its siblings, and
 requires sibling lists in a row to agree in length. `"cartesian"` expands to
 the product of the lists' lengths. `"stringify"` keeps each list as a JSON
-array in a string column.
-
-The two options compose from the outside in: a list inside a stringified
-struct stays inside its JSON text, while a struct inside an expanded list is
-stringified once per element row.
+array in a string column, with datetimes and dates as epoch milliseconds, as
+in `to_json`; `"stringify_formatted"` writes them as the text `to_json` emits
+under `formatted`.
+
+The options compose from the outside in: a list inside a stringified struct
+stays inside its JSON text, a struct inside an expanded list is stringified
+once per element row, and a struct under a pivoted map is stringified once per
+key column when `struct_flatten` is `"stringify"` and `map_flatten` is
+`"pivot"`.
 
 ```javascript
 const table = await worker.table(

docs/md/how_to/python/table_data.md

@@ -71,7 +71,11 @@ A `list` column is controlled by the `list_flatten` argument:
 -   `"cartesian"` expands a row into the product of its list columns' lengths,
     with an empty or null list counting as a single null element.
 -   `"stringify"` encodes each list as a JSON array in a single string column,
-    leaving the row count unchanged.
+    leaving the row count unchanged. Datetimes and dates inside the text are
+    epoch milliseconds, as in `to_json`; decimals are strings and binary is
+    base64.
+-   `"stringify_formatted"` is `"stringify"` with datetimes and dates written
+    as the text `to_json` emits under `formatted`.
 
 ```python
 arrow_table = pa.table({
@@ -107,17 +111,22 @@ arrow_table = pa.table({
 perspective.table(arrow_table)
 ```
 
-Both `struct` and `map` columns are controlled by the `struct_flatten`
-argument:
+A `struct` column is controlled by the `struct_flatten` argument and a `map`
+column by the `map_flatten` argument, each taking:
 
 -   `"pivot"` (default) hoists each field or key into its own dotted column,
     as above.
 -   `"stringify"` keeps the whole value as JSON text in a single string
-    column.
+    column, with datetimes and dates as epoch milliseconds, as in `to_json`.
+-   `"stringify_formatted"` is `"stringify"` with datetimes and dates written
+    as the text `to_json` emits under `formatted`.
 
-The two arguments compose from the outside in. A list inside a stringified
-struct is an array within its JSON text and is never expanded, while a struct
-inside an expanded list is stringified once per element row:
+An unset `map_flatten` follows `struct_flatten`, so `struct_flatten="stringify"`
+alone stringifies both. JSON objects always follow `struct_flatten`.
+
+The arguments compose from the outside in. A list inside a stringified struct
+is an array within its JSON text and is never expanded, while a struct inside
+an expanded list is stringified once per element row:
 
 ```python
 arrow_table = pa.table({
@@ -131,6 +140,23 @@ arrow_table = pa.table({
 perspective.table(arrow_table, struct_flatten="stringify")
 ```
 
+Setting the two independently keeps a map with an open key set as text while
+its sibling structs still pivot, or stringifies each struct under a pivoted
+map:
+
+```python
+arrow_table = pa.table({
+    "s": pa.array([{"a": 1}], type=pa.struct([("a", pa.int64())])),
+    "m": pa.array([[("k", 1)]], type=pa.map_(pa.string(), pa.int64())),
+})
+
+# `{"s.a": [1], "m": ['{"k":1}']}`
+perspective.table(arrow_table, map_flatten="stringify")
+
+# `{"s": ['{"a":1}'], "m.k": [1]}`
+perspective.table(arrow_table, struct_flatten="stringify", map_flatten="pivot")
+```
+
 ## Polars
 
 ```python

rust/perspective-client/build.rs

@@ -80,6 +80,10 @@ fn prost_build() -> Result<()> {
                 "StructFlatten",
                 "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]",
             )
+            .type_attribute(
+                "MapFlatten",
+                "#[derive(serde::Deserialize, ts_rs::TS)] #[serde(rename_all = \"snake_case\")]",
+            )
             .field_attribute("ViewToArrowResp.arrow", "#[serde(skip)]")
             .field_attribute("from_arrow", "#[serde(skip)]")
             .type_attribute(".", "#[derive(serde::Serialize)]")

rust/perspective-client/perspective.proto

@@ -407,6 +407,9 @@ message MakeTableReq {
 
         optional StructFlatten struct_flatten = 7;
 
+        // Unset falls back to `struct_flatten`.
+        optional MapFlatten map_flatten = 8;
+
         // The columns of a `Table` made from a `View`, in place of the ones
         // inferred from it. Only valid with `MakeTableData.from_view`.
         optional Schema view_schema = 5;
@@ -417,11 +420,19 @@ enum ListFlatten {
     LIST_FLATTEN_ZIP = 0;
     LIST_FLATTEN_CARTESIAN = 1;
     LIST_FLATTEN_STRINGIFY = 2;
+    LIST_FLATTEN_STRINGIFY_FORMATTED = 3;
 }
 
 enum StructFlatten {
     STRUCT_FLATTEN_PIVOT = 0;
     STRUCT_FLATTEN_STRINGIFY = 1;
+    STRUCT_FLATTEN_STRINGIFY_FORMATTED = 2;
+}
+
+enum MapFlatten {
+    MAP_FLATTEN_PIVOT = 0;
+    MAP_FLATTEN_STRINGIFY = 1;
+    MAP_FLATTEN_STRINGIFY_FORMATTED = 2;
 }
 message MakeTableResp {}
 

rust/perspective-client/src/rust/client.rs

@@ -727,6 +727,7 @@ impl Client {
                 page_to_disk: None,
                 list_flatten: None,
                 struct_flatten: None,
+                map_flatten: None,
                 schema: None,
             })),
             resp => Err(resp.into()),
@@ -775,6 +776,7 @@ impl Client {
                 page_to_disk: None,
                 list_flatten: None,
                 struct_flatten: None,
+                map_flatten: None,
                 schema: None,
             };
 

rust/perspective-client/src/rust/table.rs

@@ -166,20 +166,32 @@ pub struct TableInitOptions {
     /// default) and `cartesian` expand a row into one row per list element,
     /// and are incompatible with `index`, as the rows of an expansion
     /// repeat their index. `stringify` encodes each list as a JSON array in
-    /// a single string column instead. `STRUCT` and `MAP` columns are
-    /// governed by `struct_flatten`.
+    /// a single string column instead, with datetimes as epoch milliseconds,
+    /// and `stringify_formatted` writes them as the text `to_json` emits
+    /// under `formatted`. `STRUCT` and `MAP` columns are governed by
+    /// `struct_flatten` and `map_flatten`.
     #[serde(default)]
     #[ts(optional)]
     pub list_flatten: Option<crate::proto::ListFlatten>,
 
-    /// How Arrow `STRUCT` and `MAP` columns and JSON objects are ingested.
-    /// `pivot` (the default) hoists each field or key into its own dotted
-    /// column, and `stringify` keeps the whole value as JSON text in a single
-    /// string column.
+    /// How Arrow `STRUCT` columns and JSON objects are ingested. `pivot` (the
+    /// default) hoists each field into its own dotted column, and `stringify`
+    /// keeps the whole value as JSON text in a single string column, with
+    /// datetimes as epoch milliseconds; `stringify_formatted` writes them as
+    /// the text `to_json` emits under `formatted`.
     #[serde(default)]
     #[ts(optional)]
     pub struct_flatten: Option<crate::proto::StructFlatten>,
 
+    /// How Arrow `MAP` columns are ingested. `pivot` hoists each distinct key
+    /// into its own dotted column, and `stringify` or `stringify_formatted`
+    /// keeps the whole value as JSON text in a single string column, as for
+    /// `struct_flatten`. When unset, follows `struct_flatten`. JSON objects
+    /// always follow `struct_flatten`.
+    #[serde(default)]
+    #[ts(optional)]
+    pub map_flatten: Option<crate::proto::MapFlatten>,
+
     /// The columns of a [`Table`] made from a [`View`], in place of the ones
     /// inferred from it.
     #[serde(default)]
@@ -200,6 +212,7 @@ impl TryFrom<TableOptions> for MakeTableOptions {
         let page_to_disk = value.page_to_disk;
         let list_flatten = value.list_flatten.map(|x| x as i32);
         let struct_flatten = value.struct_flatten.map(|x| x as i32);
+        let map_flatten = value.map_flatten.map(|x| x as i32);
         let view_schema = value.schema.clone().map(|schema| crate::proto::Schema {
             schema: schema
                 .into_iter()
@@ -214,6 +227,7 @@ impl TryFrom<TableOptions> for MakeTableOptions {
             page_to_disk,
             list_flatten,
             struct_flatten,
+            map_flatten,
             view_schema,
             make_table_type: match value {
                 TableOptions {
@@ -247,6 +261,7 @@ pub(crate) struct TableOptions {
     pub page_to_disk: Option<bool>,
     pub list_flatten: Option<crate::proto::ListFlatten>,
     pub struct_flatten: Option<crate::proto::StructFlatten>,
+    pub map_flatten: Option<crate::proto::MapFlatten>,
     pub schema: Option<IndexMap<String, ColumnType>>,
 }
 
@@ -267,6 +282,7 @@ impl From<TableInitOptions> for TableOptions {
             page_to_disk: value.page_to_disk,
             list_flatten: value.list_flatten,
             struct_flatten: value.struct_flatten,
+            map_flatten: value.map_flatten,
             schema: value.schema,
         }
     }