From 00502aad6ef4de71559c3ce795842168668b7e48 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 23 Oct 2026 17:31:21 -0400 Subject: [PATCH] Add comprehensive migrations Signed-off-by: Andrew Stein --- .cargo/release.toml | 1 + .github/workflows/build.yaml | 2 +- Cargo.lock | 22 + Cargo.toml | 3 + docs/md/SUMMARY.md | 1 + docs/md/explanation/view/advanced.md | 5 +- .../view/config/grouping_and_pivots.md | 4 - docs/md/how_to/javascript/save_restore.md | 38 +- docs/md/how_to/rust/migrate.md | 136 ++++++ .../src/ts/charts/series/series-type.ts | 4 +- .../viewer-charts/src/ts/plugin/plugin.ts | 4 - packages/viewer-datagrid/src/ts/types.ts | 7 - rust/bundle/main.rs | 15 +- .../src/rust/config/expressions.rs | 63 ++- rust/perspective-js/build.mjs | 10 + .../src/ts/perspective.browser.ts | 80 +++- rust/perspective-js/src/ts/perspective.cdn.ts | 2 + .../perspective-js/src/ts/perspective.node.ts | 38 ++ rust/perspective-js/src/ts/wasm/cdn.ts | 39 +- rust/perspective-js/src/ts/wasm/migrate.ts | 80 ++++ rust/perspective-js/src/ts/wasm/registry.ts | 53 ++- rust/perspective-js/test/js/cdn_url.spec.ts | 33 +- .../test/js/derived/tree_aggregates.spec.ts | 2 +- .../test/js/expressions/functionality.spec.js | 109 +++-- .../test/js/expressions/numeric.spec.js | 84 +++- .../test/js/expressions/string.spec.js | 137 ++++-- rust/perspective-js/test/js/migrate.spec.js | 79 ++++ .../test/js/pivot_nulls.spec.js | 15 +- rust/perspective-js/test/js/registry.spec.js | 33 ++ rust/perspective-migrate-js/Cargo.toml | 33 ++ rust/perspective-migrate-js/src/lib.rs | 41 ++ rust/perspective-migrate/Cargo.toml | 35 ++ .../corpus/view/2.0.0/alias_conflict.json | 24 + .../corpus/view/2.0.0/empty.json | 8 + .../corpus/view/2.0.0/legacy_aggregate.json | 37 ++ .../corpus/view/2.10.1/get_config_tuples.json | 44 ++ .../corpus/view/2.10.1/v2_engine_aliases.json | 90 ++++ .../corpus/view/2.6.0/expression_array.json | 31 ++ .../corpus/view/3.7.0/multi_aggregates.json | 53 +++ .../corpus/view/4.2.0/expression_array.json | 30 ++ .../corpus/view/5.1.0/unused_aggregates.json | 62 +++ .../view/5.5.0/duplicates_and_aliases.json | 83 ++++ .../corpus/viewer/2.0.0/empty.json | 10 + .../corpus/viewer/2.0.0/ignored_keys.json | 66 +++ .../viewer/2.10.0/datagrid_editable.json | 32 ++ .../viewer/2.6.0/xy_scatter_symbol_list.json | 82 ++++ .../corpus/viewer/2.7.1/datagrid.json | 130 ++++++ .../corpus/viewer/2.7.1/xy_scatter.json | 71 +++ .../4.0.0/datagrid_shared_columns_config.json | 60 +++ .../4.0.0/plugin_config_not_an_object.json | 31 ++ .../corpus/viewer/4.4.1/d3fc_y_bar.json | 117 +++++ .../viewer/4.4.1/openlayers_map_scatter.json | 93 ++++ .../viewer/4.5.0/version_overrides_from.json | 31 ++ .../5.2.0/datagrid_sizes_and_colors.json | 148 +++++++ .../corpus/viewer/5.3.0/map_carto_tiles.json | 35 ++ .../corpus/viewer/5.4.0/datagrid_formats.json | 136 ++++++ .../viewer/5.6.0/inline_derived_table.json | 53 +++ .../viewer/5.6.1/date_format_presets.json | 72 +++ .../viewer/5.7.0/date_format_current.json | 28 ++ .../corpus/workspace/2.0.0/empty.json | 10 + .../corpus/workspace/2.0.0/superstore.json | 253 +++++++++++ .../corpus/workspace/3.1.2/master_detail.json | 196 +++++++++ .../corpus/workspace/4.5.2/dataset.json | 265 ++++++++++++ .../workspace/4.5.2/panel_versions.json | 85 ++++ .../corpus/workspace/5.1.0/null_table.json | 27 ++ .../corpus/workspace/5.6.1/ignored_keys.json | 83 ++++ .../src/column_style/mod.rs | 115 +++++ rust/perspective-migrate/src/css.rs | 244 +++++++++++ rust/perspective-migrate/src/error.rs | 82 ++++ rust/perspective-migrate/src/json.rs | 43 ++ rust/perspective-migrate/src/lib.rs | 270 ++++++++++++ rust/perspective-migrate/src/notice.rs | 33 ++ rust/perspective-migrate/src/plugin.rs | 71 +++ .../perspective-migrate/src/plugins/charts.rs | 235 ++++++++++ .../src/plugins/datagrid.rs | 409 ++++++++++++++++++ rust/perspective-migrate/src/plugins/maps.rs | 78 ++++ rust/perspective-migrate/src/plugins/mod.rs | 92 ++++ rust/perspective-migrate/src/step.rs | 235 ++++++++++ rust/perspective-migrate/src/version.rs | 50 +++ rust/perspective-migrate/src/view/mod.rs | 88 ++++ rust/perspective-migrate/src/view/v3_0_0.rs | 153 +++++++ rust/perspective-migrate/src/view/v3_8_0.rs | 36 ++ rust/perspective-migrate/src/view/v5_2_0.rs | 145 +++++++ rust/perspective-migrate/src/view/v5_6_0.rs | 43 ++ rust/perspective-migrate/src/view/v5_7_0.rs | 138 ++++++ rust/perspective-migrate/src/viewer/mod.rs | 129 ++++++ rust/perspective-migrate/src/workspace/mod.rs | 187 ++++++++ .../src/workspace/regular_layout.rs | 247 +++++++++++ rust/perspective-migrate/tests/corpus.rs | 223 ++++++++++ rust/perspective-python/Cargo.toml | 2 + .../perspective/__init__.py | 2 + .../multi_threaded/test_multi_threaded.py | 2 +- .../tests/table/test_table_datetime.py | 62 +-- .../tests/table/test_view_expression.py | 117 +++-- .../perspective/tests/test_migrate.py | 61 +++ .../perspective/widget/__init__.py | 2 +- .../perspective/widget/viewer/viewer.py | 7 +- .../widget/viewer/viewer_traitlets.py | 2 +- rust/perspective-python/src/lib.rs | 2 + rust/perspective-python/src/migrate.rs | 59 +++ .../cpp/perspective/src/cpp/aggspec.cpp | 10 +- .../cpp/perspective/src/cpp/base.cpp | 10 +- .../src/rust/components/viewer/panels.rs | 3 +- .../src/rust/config/datetime_column_style.rs | 2 - .../config/datetime_column_style/migrate.rs | 95 ---- .../rust/config/number_string_format/enums.rs | 2 +- .../src/rust/config/viewer_config.rs | 113 +++-- .../src/rust/config/workspace_config.rs | 18 +- .../src/rust/custom_elements/viewer.rs | 26 +- rust/perspective-viewer/src/rust/derived.rs | 3 +- .../src/rust/queries/validate_expression.rs | 10 +- rust/perspective-viewer/src/rust/renderer.rs | 10 +- .../src/rust/session/op_queue.rs | 3 +- .../src/rust/tasks/derive.rs | 5 +- .../src/rust/tasks/sync_update_panels.rs | 8 +- rust/perspective-viewer/src/rust/workspace.rs | 3 +- rust/perspective-viewer/src/ts/bootstrap.ts | 90 +++- .../src/ts/perspective-viewer.cdn.ts | 10 + .../src/ts/perspective-viewer.ts | 13 +- .../viewer_config/config_strictness.spec.ts | 176 ++++++++ .../date_format_migration.spec.ts | 120 ----- .../js/viewer_config/migrate_corpus.spec.ts | 127 ++++++ tools/scripts/publish.mjs | 5 + ...load-viewer-all-types-small-multi-arrow.js | 2 +- tools/test/load-viewer-superstore.js | 5 +- 125 files changed, 7674 insertions(+), 643 deletions(-) create mode 100644 docs/md/how_to/rust/migrate.md create mode 100644 rust/perspective-js/src/ts/wasm/migrate.ts create mode 100644 rust/perspective-js/test/js/migrate.spec.js create mode 100644 rust/perspective-migrate-js/Cargo.toml create mode 100644 rust/perspective-migrate-js/src/lib.rs create mode 100644 rust/perspective-migrate/Cargo.toml create mode 100644 rust/perspective-migrate/corpus/view/2.0.0/alias_conflict.json create mode 100644 rust/perspective-migrate/corpus/view/2.0.0/empty.json create mode 100644 rust/perspective-migrate/corpus/view/2.0.0/legacy_aggregate.json create mode 100644 rust/perspective-migrate/corpus/view/2.10.1/get_config_tuples.json create mode 100644 rust/perspective-migrate/corpus/view/2.10.1/v2_engine_aliases.json create mode 100644 rust/perspective-migrate/corpus/view/2.6.0/expression_array.json create mode 100644 rust/perspective-migrate/corpus/view/3.7.0/multi_aggregates.json create mode 100644 rust/perspective-migrate/corpus/view/4.2.0/expression_array.json create mode 100644 rust/perspective-migrate/corpus/view/5.1.0/unused_aggregates.json create mode 100644 rust/perspective-migrate/corpus/view/5.5.0/duplicates_and_aliases.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.0.0/empty.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.0.0/ignored_keys.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.10.0/datagrid_editable.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.6.0/xy_scatter_symbol_list.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.7.1/datagrid.json create mode 100644 rust/perspective-migrate/corpus/viewer/2.7.1/xy_scatter.json create mode 100644 rust/perspective-migrate/corpus/viewer/4.0.0/datagrid_shared_columns_config.json create mode 100644 rust/perspective-migrate/corpus/viewer/4.0.0/plugin_config_not_an_object.json create mode 100644 rust/perspective-migrate/corpus/viewer/4.4.1/d3fc_y_bar.json create mode 100644 rust/perspective-migrate/corpus/viewer/4.4.1/openlayers_map_scatter.json create mode 100644 rust/perspective-migrate/corpus/viewer/4.5.0/version_overrides_from.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.2.0/datagrid_sizes_and_colors.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.3.0/map_carto_tiles.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.4.0/datagrid_formats.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.6.0/inline_derived_table.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.6.1/date_format_presets.json create mode 100644 rust/perspective-migrate/corpus/viewer/5.7.0/date_format_current.json create mode 100644 rust/perspective-migrate/corpus/workspace/2.0.0/empty.json create mode 100644 rust/perspective-migrate/corpus/workspace/2.0.0/superstore.json create mode 100644 rust/perspective-migrate/corpus/workspace/3.1.2/master_detail.json create mode 100644 rust/perspective-migrate/corpus/workspace/4.5.2/dataset.json create mode 100644 rust/perspective-migrate/corpus/workspace/4.5.2/panel_versions.json create mode 100644 rust/perspective-migrate/corpus/workspace/5.1.0/null_table.json create mode 100644 rust/perspective-migrate/corpus/workspace/5.6.1/ignored_keys.json create mode 100644 rust/perspective-migrate/src/column_style/mod.rs create mode 100644 rust/perspective-migrate/src/css.rs create mode 100644 rust/perspective-migrate/src/error.rs create mode 100644 rust/perspective-migrate/src/json.rs create mode 100644 rust/perspective-migrate/src/lib.rs create mode 100644 rust/perspective-migrate/src/notice.rs create mode 100644 rust/perspective-migrate/src/plugin.rs create mode 100644 rust/perspective-migrate/src/plugins/charts.rs create mode 100644 rust/perspective-migrate/src/plugins/datagrid.rs create mode 100644 rust/perspective-migrate/src/plugins/maps.rs create mode 100644 rust/perspective-migrate/src/plugins/mod.rs create mode 100644 rust/perspective-migrate/src/step.rs create mode 100644 rust/perspective-migrate/src/version.rs create mode 100644 rust/perspective-migrate/src/view/mod.rs create mode 100644 rust/perspective-migrate/src/view/v3_0_0.rs create mode 100644 rust/perspective-migrate/src/view/v3_8_0.rs create mode 100644 rust/perspective-migrate/src/view/v5_2_0.rs create mode 100644 rust/perspective-migrate/src/view/v5_6_0.rs create mode 100644 rust/perspective-migrate/src/view/v5_7_0.rs create mode 100644 rust/perspective-migrate/src/viewer/mod.rs create mode 100644 rust/perspective-migrate/src/workspace/mod.rs create mode 100644 rust/perspective-migrate/src/workspace/regular_layout.rs create mode 100644 rust/perspective-migrate/tests/corpus.rs create mode 100644 rust/perspective-python/perspective/tests/test_migrate.py create mode 100644 rust/perspective-python/src/migrate.rs delete mode 100644 rust/perspective-viewer/src/rust/config/datetime_column_style/migrate.rs create mode 100644 rust/perspective-viewer/test/js/viewer_config/config_strictness.spec.ts delete mode 100644 rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts create mode 100644 rust/perspective-viewer/test/js/viewer_config/migrate_corpus.spec.ts diff --git a/.cargo/release.toml b/.cargo/release.toml index c231d3fa3b..587d5592d5 100644 --- a/.cargo/release.toml +++ b/.cargo/release.toml @@ -9,6 +9,7 @@ perspective-client = { path = "rust/perspective-client" } perspective-server = { path = "rust/perspective-server" } perspective-js = { path = "rust/perspective-js" } +perspective-migrate = { path = "rust/perspective-migrate" } perspective-python = { path = "rust/perspective-python" } perspective-viewer = { path = "rust/perspective-viewer" } perspective = { path = "rust/perspective" } diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 217803a7da..42575f052d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -721,7 +721,7 @@ jobs: - name: Package if: ${{ steps.config-step.outputs.FULL_RUN }} - run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-python + run: cargo package --config .cargo/release.toml --no-verify --allow-dirty -p perspective -p perspective-viewer -p perspective-js -p perspective-client -p perspective-server -p perspective-migrate -p perspective-python - uses: actions/upload-artifact@v4 if: ${{ steps.config-step.outputs.FULL_RUN }} diff --git a/Cargo.lock b/Cargo.lock index d8faee131c..42e2614fb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2010,6 +2010,26 @@ dependencies = [ "yew-fmt", ] +[[package]] +name = "perspective-migrate" +version = "5.6.1" +dependencies = [ + "perspective-client", + "serde", + "serde_json", +] + +[[package]] +name = "perspective-migrate-js" +version = "5.6.1" +dependencies = [ + "perspective-migrate", + "serde", + "serde-wasm-bindgen", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "perspective-python" version = "5.6.1" @@ -2022,6 +2042,7 @@ dependencies = [ "indexmap 2.14.0", "macro_rules_attribute", "perspective-client", + "perspective-migrate", "perspective-server", "pollster", "pyo3", @@ -2029,6 +2050,7 @@ dependencies = [ "pyo3-build-config", "pythonize", "serde", + "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index c1d88de108..39b8633aa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,8 @@ members = [ "rust/perspective", "rust/perspective-client", "rust/perspective-js", + "rust/perspective-migrate", + "rust/perspective-migrate-js", "rust/perspective-python", "rust/perspective-server", "rust/perspective-viewer", @@ -58,6 +60,7 @@ protobuf-src = { git = "https://github.com/carlocorradini/rust-protobuf-native.g perspective-client = { path = "rust/perspective-client" } perspective-server = { path = "rust/perspective-server" } perspective-js = { path = "rust/perspective-js" } +perspective-migrate = { path = "rust/perspective-migrate" } perspective = { path = "rust/perspective" } perspective-viewer = { path = "rust/perspective-viewer" } perspective-python = { path = "rust/perspective-python" } diff --git a/docs/md/SUMMARY.md b/docs/md/SUMMARY.md index 243005fa1a..e6788185f3 100644 --- a/docs/md/SUMMARY.md +++ b/docs/md/SUMMARY.md @@ -88,6 +88,7 @@ # Rust - [Getting Started](./how_to/rust.md) +- [Migrating saved configs](./how_to/rust/migrate.md) # Integrations diff --git a/docs/md/explanation/view/advanced.md b/docs/md/explanation/view/advanced.md index a0287ae04e..49014787bc 100644 --- a/docs/md/explanation/view/advanced.md +++ b/docs/md/explanation/view/advanced.md @@ -150,7 +150,10 @@ const result = await table.validate_expressions({
```python -result = table.validate_expressions(['"Sales" + "Profit"', 'invalid + 1']) +result = table.validate_expressions({ + "expr1": '"Sales" + "Profit"', + "expr2": "invalid_column + 1", +}) ```
diff --git a/docs/md/explanation/view/config/grouping_and_pivots.md b/docs/md/explanation/view/config/grouping_and_pivots.md index d967a41faf..301841b3ec 100644 --- a/docs/md/explanation/view/config/grouping_and_pivots.md +++ b/docs/md/explanation/view/config/grouping_and_pivots.md @@ -237,8 +237,6 @@ a given column accepts depends on its type — see | `pct sum parent` | The group's `sum` as a percentage of its parent row's, `0`–`100`; `100` at the root, and `null` when the parent's sum is `0` | `float` | | `pct sum total` | The group's `sum` as a percentage of the grand total, `0`–`100` | `float` | -`sum not null` is accepted as a legacy alias for `sum`. - A numeric aggregate widens to the input's numeric class — `integer` columns accumulate as `integer`, `float` columns as `float`. @@ -390,10 +388,8 @@ anywhere an aggregate is, and each group refers to one function: | `last` | `last_value` | | `high` | `high_water_mark` | | `low` | `low_water_mark` | -| `pct sum total` | `pct sum grand total`, `pct_sum_grand_total` | | `var` | `variance` | | `stddev` | `standard deviation` | -| `sum` | `sum not null`, `sum_not_null` (legacy name) | Most multi-word aggregates also answer to a snake_case spelling — `sum_or_zero`, `sum_abs`, `abs_sum`, `weighted_mean`, `distinct_leaf`, diff --git a/docs/md/how_to/javascript/save_restore.md b/docs/md/how_to/javascript/save_restore.md index 665fef6a82..be424d34b2 100644 --- a/docs/md/how_to/javascript/save_restore.md +++ b/docs/md/how_to/javascript/save_restore.md @@ -13,30 +13,25 @@ panel visibility, and more. This overloaded feature covers a range of use cases: ## Serializing and deserializing the viewer state To retrieve the entire state as a JSON-ready JavaScript object, use the `save()` -method. `save()` also supports a few other formats such as `"arraybuffer"` and -`"string"` (base64, not JSON), which you may choose for size at the expense of -easy migration/manual-editing. +method. ```javascript -const json_token = await elem.save(); -const string_token = await elem.save("string"); +const token = await elem.save(); ``` -For any format, the serialized token can be restored to any -`` with a `Table` of identical schema, via the `restore()` -method. Note that while the data for a token returned from `save()` may differ, -generally its schema may not, as many other settings depend on column names and -types. +The token can be restored to any `` with a `Table` of +identical schema, via the `restore()` method. Note that while the data for a +token returned from `save()` may differ, generally its schema may not, as many +other settings depend on column names and types. ```javascript -await elem.restore(json_token); -await elem.restore(string_token); +await elem.restore(token); ``` -As `restore()` dispatches on the token's type, it is important to make sure that -these types match! A common source of error occurs when passing a -JSON-stringified token to `restore()`, which will assume base64-encoded msgpack -when a string token is used. +`restore()` validates its argument, rejecting for example a key the config +format does not define; when a rejected token is stamped with another release's +`version`, the error names that release. Tokens saved by older releases should +be upgraded first — see [Migrating saved configs](../rust/migrate.md). ```javascript // This will error! @@ -117,8 +112,8 @@ await elem.restoreWorkspace(workspace_token); A `saveWorkspace()` token is a `WorkspaceConfig` (`{ version, layout, panels, ... }`), not a `ViewerConfig` — passing it to the -single-panel `restore()` will _not_ restore the layout (its `panels`/`layout` -keys are ignored). +single-panel `restore()` is rejected (its `panels`/`layout` keys are not part +of a `ViewerConfig`). The `layout` field is the tree arranging the panels into a grid of rows and columns or stacks of tabs, naming every panel id exactly once. See @@ -205,9 +200,10 @@ inherits `timeZone` and `hour12` individually. Configs saved before 5.7.0 used a `"disabled"` value and a `format: "custom"` marker, and a missing preset field meant `short` date or `medium` time. -`restore()` rewrites these into the current form with the same rendering. A -hand-written config with no `version` and a missing preset field takes the -current meaning: `{ dateStyle: "long" }` now shows the date alone. +`restore()` rejects the removed forms, and restores any other `date_format` +with its current meaning — `{ dateStyle: "long" }` shows the date alone, +whatever its `version`. [`perspective-migrate`](../rust/migrate.md) rewrites +older tokens into the current form with their original rendering. ## Colors, palettes and gradients diff --git a/docs/md/how_to/rust/migrate.md b/docs/md/how_to/rust/migrate.md new file mode 100644 index 0000000000..748fa52859 --- /dev/null +++ b/docs/md/how_to/rust/migrate.md @@ -0,0 +1,136 @@ +# Migrating saved configs + +``'s `restore()`, `addPanel()` and `restoreWorkspace()` +accept only the current config schema: a config which fails validation (for +example, a key the format does not define) is rejected, and when it is stamped +with another release's `version` the error names that release. Engine +`ViewConfig`s passed to `table.view()` likewise accept only the current form +(for example, `expressions` must be a map of names to expressions). + +Configs persisted by an older release — `save()` and `saveWorkspace()` tokens, +`view.get_config()` output, or hand-written `ViewConfig`s — are upgraded by +`migrate()`, which rewrites any JSON config written by a release since 2.0.0 +into the current schema. It is available from `@perspective-dev/client` (in the +browser and in Node.js), from `@perspective-dev/viewer`, from the `perspective` +Python package, and as the `perspective-migrate` Rust crate. + +## Usage + +The release that wrote a config is read from its `version` field, which `save()` +and `saveWorkspace()` have stamped since 2.7.0. A config without one takes the +optional `from_version` argument, and without either it is treated as written by +2.0.0. Each workspace panel's own `version` decides how that panel is upgraded, +so a 2.x–4.x `` layout, whose envelope was never stamped, +needs no `from_version` when its panels were. + +In the browser, the migration WebAssembly binary is registered alongside the +engine binaries and downloaded only by the first `migrate()` call (the CDN +bundles register it automatically): + +```javascript +import perspective from "@perspective-dev/client"; +import perspective_viewer from "@perspective-dev/viewer"; + +await perspective_viewer.init({ + server32: () => fetch(SERVER_WASM), + server64: () => fetch(SERVER_WASM64), + viewer: () => fetch(VIEWER_WASM), + migrate: () => fetch(MIGRATE_WASM), +}); + +const { value, notices } = await perspective_viewer.migrate("workspace", saved); +await viewer.restoreWorkspace(value); +``` + +`MIGRATE_WASM` is +`@perspective-dev/client/dist/wasm/perspective-migrate-js.wasm`. In Node.js, +`perspective.migrate()` needs no registration. + +```python +import perspective + +result = perspective.migrate("viewer", token) +viewer_config = result["value"] +``` + +```bash +cargo add perspective-migrate +``` + +```rust +use perspective_migrate::{Kind, Migrator}; + +let migrated = Migrator::new().migrate(Kind::Viewer, None, token)?; +for notice in &migrated.notices { + eprintln!("{notice:?}"); +} + +save(migrated.value); +``` + +The first argument names the document being upgraded (`Kind` in Rust): + +| JavaScript / Python | Rust | Document | +| ------------------- | ----------------- | ----------------------------------------------------------------- | +| `"view"` | `Kind::View` | An engine `ViewConfig`, as passed to `table.view()` | +| `"viewer"` | `Kind::Viewer` | A `` `save()` / `restore()` token | +| `"workspace"` | `Kind::Workspace` | A `saveWorkspace()` token, or a 2.x–4.x `` | + +Viewer and workspace output is stamped with the current `version`, so migrating +it again changes nothing; `ViewConfig`s have no `version`, but every +`ViewConfig` upgrade leaves a current config unchanged. Each upgrade also +reports `notices` for information which has no current representation +(`Dropped`, with the removed value) or which could only be approximated +(`Approximated`), and `steps`, the ids of the schema changes that applied. + +A config is rejected with a `MigrateError` only when no valid current form +exists — for example a pre-5.2 workspace panel with no `table`, or a 2.x +`ViewConfig` which names both an alias and its canonical key (`row_pivot` and +`group_by`). + +## Third-party plugins + +A plugin's `plugin_config` and `columns_config` payloads are migrated by a +`PluginMigration`, keyed by the Perspective release that wrote the token: + +```rust +use perspective_migrate::{Migrator, PluginMigration, Step, Version}; + +struct Sparkline; + +impl PluginMigration for Sparkline { + fn plugins(&self) -> &[&'static str] { + &["Sparkline"] + } + + fn steps(&self) -> Vec { + vec![Step { + at: Version::new(3, 0, 0), + id: "sparkline.line_width", + apply: rename_width, + }] + } +} + +let migrator = Migrator::new().with_plugin(Sparkline); +``` + +## What is not migrated + +`perspective-migrate` produces a config which is _valid_ for the current release +and keeps the information the old config expressed. Changes to defaults and +behavior are not reversed, so a migrated config renders as a new config with the +same settings would today: + +- Engine: 2.x ignored filters with a `null` term; date filter terms captured + from 3.0.x – 3.4.2 `get_config()` used seconds and 0-based months; an + all-`null` group's `sum` is `null` rather than `0` since 5.6.0 (use + `sum or zero`). +- Column styles: string `"series"` colors derived from a hue rotation (before + 5.3.0); float fraction-digit defaults merged rather than replaced (5.3.1). +- Charts: the `domain_mode`, `interpolate`, `facet_mode` and `legend_mode` + defaults, and the map `numeric_axes` default. +- Derived tables created from grouped views before 5.6.0, whose key columns were + named `" (Group by N)"`. +- Binary `"string"` / `"arraybuffer"` tokens written by 2.0 – 4.0, which must be + re-saved as JSON by the release that wrote them. diff --git a/packages/viewer-charts/src/ts/charts/series/series-type.ts b/packages/viewer-charts/src/ts/charts/series/series-type.ts index b3b1894c65..1eca8246e1 100644 --- a/packages/viewer-charts/src/ts/charts/series/series-type.ts +++ b/packages/viewer-charts/src/ts/charts/series/series-type.ts @@ -45,9 +45,7 @@ export interface ColumnChartConfig { /** * Interpolation mode for line / area glyphs. See - * {@link InterpolateMode}. Legacy values `true` / `false` are also - * accepted by {@link resolveInterpolate} (mapped to `"solid"` / - * `"skip"`). Default `"skip"`. No effect on bar / scatter. + * {@link InterpolateMode}. Default `"skip"`. No effect on bar / scatter. */ interpolate?: InterpolateMode; diff --git a/packages/viewer-charts/src/ts/plugin/plugin.ts b/packages/viewer-charts/src/ts/plugin/plugin.ts index 128ccb9b42..833d42d5aa 100644 --- a/packages/viewer-charts/src/ts/plugin/plugin.ts +++ b/packages/viewer-charts/src/ts/plugin/plugin.ts @@ -951,10 +951,6 @@ export class HTMLPerspectiveViewerWebGLPluginElement } restore(config: any, columns_config?: Record) { - if (config?.zoom) { - this._renderer?.restoreZoom(config.zoom); - } - // Merge incoming plugin_config on top of the `chart_type` // effective defaults so a partial restore (UI emits only // changed fields) keeps untouched defaults in place — and diff --git a/packages/viewer-datagrid/src/ts/types.ts b/packages/viewer-datagrid/src/ts/types.ts index ca92d925b3..c05d0594c7 100644 --- a/packages/viewer-datagrid/src/ts/types.ts +++ b/packages/viewer-datagrid/src/ts/types.ts @@ -287,8 +287,6 @@ export interface ColumnConfig { */ bg_gradient?: number; - fixed?: number; - /** * Group-by rollup depth override for this column when the view is * pivoted in `Rollup` mode. @@ -391,9 +389,6 @@ export type ResolvedColumnsConfig = Record; * `save()`/`restore()` token). */ export interface DatagridPluginConfig { - /** Legacy alias for `edit_mode: "EDIT"`. */ - editable?: boolean; - /** * When `true`, the Datagrid keeps its scroll position pinned during * data updates instead of following appended rows. @@ -452,8 +447,6 @@ export interface DatagridPluginConfig { * `zebra_rows >= 1`. */ zebra_color?: string; - - column_size_override?: Record; } export type Schema = Record; diff --git a/rust/bundle/main.rs b/rust/bundle/main.rs index d522349d02..d1e8154e78 100644 --- a/rust/bundle/main.rs +++ b/rust/bundle/main.rs @@ -33,6 +33,11 @@ struct BundleArgs { /// Extra features to build with #[arg(long)] features: Option, + + /// Omit the `tracing/release_max_level_warn` feature, for crates which do + /// not depend on `tracing`. + #[arg(long)] + no_tracing: bool, } use wasm_bindgen_cli_support::{Bindgen, EncodeInto}; @@ -69,8 +74,12 @@ impl Profile { /// defined in the `/.cargo/config.toml` because they would define this build /// script's parameters also, and there is no way to reset e.g. the `target` /// field to the host platform. -fn build(pkg: Option<&str>, profile: Profile, features: Vec) { - let features = format!("tracing/release_max_level_warn,{}", features.join(",")); +fn build(pkg: Option<&str>, profile: Profile, features: Vec, tracing: bool) { + let features = if tracing { + format!("tracing/release_max_level_warn,{}", features.join(",")) + } else { + features.join(",") + }; // Build RUSTFLAGS including target-specific flags from config.toml and new // panic flags These are the flags from .cargo/config.toml for @@ -256,7 +265,7 @@ fn main() { .map(|x| x.to_string()) .collect(); - build(Some(package.as_str()), profile, features); + build(Some(package.as_str()), profile, features, !args.no_tracing); bindgen(outdir, args.artifact.as_str(), profile); opt(outpath, profile); } diff --git a/rust/perspective-client/src/rust/config/expressions.rs b/rust/perspective-client/src/rust/config/expressions.rs index 7ba2969a34..60d33ebe91 100644 --- a/rust/perspective-client/src/rust/config/expressions.rs +++ b/rust/perspective-client/src/rust/config/expressions.rs @@ -159,20 +159,38 @@ use std::borrow::Cow; use std::collections::HashMap; +use std::fmt; -use serde::{Deserialize, Serialize}; +use serde::de::value::MapAccessDeserializer; +use serde::de::{MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; use ts_rs::TS; -#[derive(Deserialize, Clone, PartialEq, Debug)] -#[serde(untagged)] -pub enum ExpressionsDeserde { - Array(Vec), - Map(HashMap), +#[derive(Serialize, Clone, PartialEq, Debug, Default, TS)] +pub struct Expressions(pub HashMap); + +impl<'de> Deserialize<'de> for Expressions { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(ExpressionsVisitor) + } } -#[derive(Deserialize, Serialize, Clone, PartialEq, Debug, Default, TS)] -#[serde(from = "ExpressionsDeserde")] -pub struct Expressions(pub HashMap); +struct ExpressionsVisitor; + +impl<'de> Visitor<'de> for ExpressionsVisitor { + type Value = Expressions; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str( + "an object mapping expression names to expressions (upgrade the legacy array format \ + with `perspective-migrate`)", + ) + } + + fn visit_map>(self, map: A) -> Result { + HashMap::deserialize(MapAccessDeserializer::new(map)).map(Expressions) + } +} impl std::ops::Deref for Expressions { type Target = HashMap; @@ -188,33 +206,6 @@ impl std::ops::DerefMut for Expressions { } } -fn upgrade_legacy_format(expressions: &[String]) -> HashMap { - tracing::debug!("Legacy `expressions` format: {:?}", expressions); - expressions - .iter() - .map(|s| { - if let Some((name, expression)) = s.split_once('\n') { - if !expression.is_empty() && name.starts_with("//") { - (name.split_at(2).1.trim().to_owned(), expression.to_owned()) - } else { - (s.to_owned(), s.to_owned()) - } - } else { - (s.to_owned(), s.to_owned()) - } - }) - .collect::>() -} - -impl From for Expressions { - fn from(value: ExpressionsDeserde) -> Self { - match value { - ExpressionsDeserde::Array(arr) => Self(upgrade_legacy_format(&arr)), - ExpressionsDeserde::Map(map) => Self(map), - } - } -} - #[derive(Clone, Debug, PartialEq, TS)] pub struct Expression<'a> { pub name: Cow<'a, str>, diff --git a/rust/perspective-js/build.mjs b/rust/perspective-js/build.mjs index f5cd3b9906..bf696e7d0b 100644 --- a/rust/perspective-js/build.mjs +++ b/rust/perspective-js/build.mjs @@ -138,6 +138,16 @@ async function build_rust() { "dist/wasm/perspective-js.wasm", "dist/wasm/perspective-js.wasm", ); + + execSync( + `PSP_ROOT_DIR=../.. cargo bundle --target=${get_host()} -- perspective_migrate_js ${release_flag} --no-tracing`, + INHERIT, + ); + + await compress( + "dist/wasm/perspective-migrate-js.wasm", + "dist/wasm/perspective-migrate-js.wasm", + ); } async function build_web_assets() { diff --git a/rust/perspective-js/src/ts/perspective.browser.ts b/rust/perspective-js/src/ts/perspective.browser.ts index f1be3d2506..8e75499c5f 100644 --- a/rust/perspective-js/src/ts/perspective.browser.ts +++ b/rust/perspective-js/src/ts/perspective.browser.ts @@ -17,13 +17,17 @@ import * as psp_virtual from "./virtual_server.ts"; import * as wasm_module from "../../dist/wasm/perspective-js.js"; import * as api from "./wasm/browser.ts"; import { host_supports_memory64 } from "./wasm/features.ts"; +import type { MigrateKind, MigrateResult } from "./wasm/migrate.ts"; import { init, + MigrateLoader, provide_client_glue, provide_client_runtime, register, resolve_client, + resolve_migrate, resolve_server, + shared_migrate, } from "./wasm/registry.ts"; import type { WasmSource } from "./wasm/registry.ts"; @@ -36,6 +40,12 @@ export type { WasmSource, } from "./wasm/registry.ts"; +export type { + MigrateKind, + MigrateNotice, + MigrateResult, +} from "./wasm/migrate.ts"; + export { GenericSQLVirtualServerModel, VirtualDataSlice, @@ -62,13 +72,15 @@ export type ServerWasmSource = | (() => Promise); /** - * Engine binaries by bitness. Registering `wasm64` is the opt-in for the - * Memory64 engine (4GB heap ceiling raised to 16GB, at some engine - * performance cost) — it is selected whenever the host supports Memory64. + * Engine binaries by bitness, and the `perspective-migrate` binary behind + * `migrate()`. Registering `wasm64` is the opt-in for the Memory64 engine + * (4GB heap ceiling raised to 16GB, at some engine performance cost) — it is + * selected whenever the host supports Memory64. */ export interface ServerWasmRegistration { wasm32?: ServerWasmSource; wasm64?: ServerWasmSource; + migrate?: ServerWasmSource; } export interface InitServerOptions { @@ -86,6 +98,9 @@ provide_client_runtime(async (wasm) => { return { glue: wasm_module, module }; }); +const MIGRATE = new MigrateLoader(); +MIGRATE.register(resolve_migrate); + function client_glue(): Promise { return resolve_client().then((runtime) => runtime.glue); } @@ -111,8 +126,8 @@ function is_wasm_source( /** * Register the Perspective engine ("server") WebAssembly binary that backs - * `worker()`; a wrapper over `init()`'s `server` / `server32` / `server64` - * roles. + * `worker()`; a wrapper over `init()`'s `server` / `server32` / `server64` / + * `migrate` roles. * * The single-argument form registers one binary of either bitness (`server`). * The object form registers a wasm32 and/or wasm64 (Memory64) binary @@ -124,7 +139,8 @@ function is_wasm_source( * source is also registered, selection falls back to wasm32 with a console * warning. Either form replaces any engine binaries registered earlier. * - * Selection and download are deferred to the first `worker()` call. + * Selection and download are deferred to the first `worker()` call. The + * `migrate` binary is likewise only downloaded by the first `migrate()` call. * * # Examples * @@ -136,6 +152,7 @@ function is_wasm_source( * perspective.init_server({ * wasm32: () => fetch("perspective-server.wasm"), * wasm64: () => fetch("perspective-server.memory64.wasm"), + * migrate: () => fetch("perspective-migrate-js.wasm"), * }); * ``` */ @@ -153,18 +170,52 @@ export function init_server( // `WebAssembly.Module`'s lib type is structurally empty, so TypeScript // can't negatively narrow this branch itself. - const { wasm32, wasm64 } = wasm as ServerWasmRegistration; - if (wasm32 === undefined && wasm64 === undefined) { + const { wasm32, wasm64, migrate } = wasm as ServerWasmRegistration; + if (wasm32 === undefined && wasm64 === undefined && migrate === undefined) { throw new Error( - "init_server requires a wasm source or a {wasm32, wasm64} registration", + "init_server requires a wasm source or a {wasm32, wasm64, migrate} registration", ); } - register( - { server32: wasm32, server64: wasm64 }, - { disable_stage_0 }, - "replace_server", - ); + if (wasm32 !== undefined || wasm64 !== undefined) { + register( + { server32: wasm32, server64: wasm64 }, + { disable_stage_0 }, + "replace_server", + ); + } + + if (migrate !== undefined) { + register({ migrate }, { disable_stage_0 }); + } +} + +/** + * Upgrade a persisted config written by any Perspective release since 2.0.0 + * to the current schema, using the `migrate` binary registered with `init()`. + * + * # Arguments + * + * - `kind` - `"view"` for a `ViewConfig`, `"viewer"` for a + * `` `save()` token, or `"workspace"` for a + * `saveWorkspace()` token. + * - `config` - The config to upgrade. + * - `from_version` - The release which wrote `config`, used only when it has + * no `version` field. + * + * # Examples + * + * ```javascript + * const { value, notices } = await perspective.migrate("viewer", token); + * await viewer.restore(value); + * ``` + */ +export async function migrate>( + kind: MigrateKind, + config: unknown, + from_version?: string, +): Promise> { + return await shared_migrate(MIGRATE).migrate(kind, config, from_version); } export type PerspectiveWasm = @@ -277,6 +328,7 @@ export default { init_client, init_server, make_client, + migrate, host_supports_memory64, createMessageHandler, getCompiledClientWasm, diff --git a/rust/perspective-js/src/ts/perspective.cdn.ts b/rust/perspective-js/src/ts/perspective.cdn.ts index 3ecb2a4be2..d0d7d3851f 100644 --- a/rust/perspective-js/src/ts/perspective.cdn.ts +++ b/rust/perspective-js/src/ts/perspective.cdn.ts @@ -26,6 +26,8 @@ register_default({ "perspective-server.memory64.wasm", ), ), + migrate: () => + fetch(new URL("../wasm/perspective-migrate-js.wasm", import.meta.url)), }); export default perspective; diff --git a/rust/perspective-js/src/ts/perspective.node.ts b/rust/perspective-js/src/ts/perspective.node.ts index e31b86c18c..35b80afde8 100644 --- a/rust/perspective-js/src/ts/perspective.node.ts +++ b/rust/perspective-js/src/ts/perspective.node.ts @@ -37,6 +37,8 @@ import * as perspective_client from "../../dist/wasm/perspective-js.js"; import { load_wasm_stage_0 } from "./wasm/decompress.ts"; import { host_supports_memory64 } from "./wasm/features.ts"; import * as engine from "./wasm/engine.ts"; +import { MigrateLoader } from "./wasm/migrate.ts"; +import type { MigrateKind, MigrateResult } from "./wasm/migrate.ts"; export { host_supports_memory64 } from "./wasm/features.ts"; export type { @@ -45,6 +47,11 @@ export type { WasmRegistration, WasmSource, } from "./wasm/registry.ts"; +export type { + MigrateKind, + MigrateNotice, + MigrateResult, +} from "./wasm/migrate.ts"; import { compile_perspective } from "./wasm/emscripten_api.ts"; import * as psp_websocket from "./websocket.ts"; import * as api from "./wasm/browser.ts"; @@ -76,6 +83,36 @@ const uncompressed_client_wasm = await fs await perspective_client.default({ module_or_path: uncompressed_client_wasm }); +const MIGRATE = new MigrateLoader(); +MIGRATE.register(async () => { + const buffer = await fs.readFile( + path.join(__dirname, "../../dist/wasm/perspective-migrate-js.wasm"), + ); + + return await load_wasm_stage_0(buffer.buffer as ArrayBuffer); +}); + +/** + * Upgrade a persisted config written by any Perspective release since 2.0.0 + * to the current schema. + * + * # Arguments + * + * - `kind` - `"view"` for a `ViewConfig`, `"viewer"` for a + * `` `save()` token, or `"workspace"` for a + * `saveWorkspace()` token. + * - `config` - The config to upgrade. + * - `from_version` - The release which wrote `config`, used only when it has + * no `version` field. + */ +export async function migrate>( + kind: MigrateKind, + config: unknown, + from_version?: string, +): Promise> { + return await MIGRATE.migrate(kind, config, from_version); +} + function make_node_disk_bridge({ heap, toAddr, @@ -549,6 +586,7 @@ export default { join, websocket, worker, + migrate, host_supports_memory64, get_hosted_table_names, on_hosted_tables_update, diff --git a/rust/perspective-js/src/ts/wasm/cdn.ts b/rust/perspective-js/src/ts/wasm/cdn.ts index ddfda4afa1..d356f212aa 100644 --- a/rust/perspective-js/src/ts/wasm/cdn.ts +++ b/rust/perspective-js/src/ts/wasm/cdn.ts @@ -11,22 +11,41 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ /** - * Resolve the URL of one of `@perspective-dev/server`'s engine WASM binaries - * relative to the URL of the `@perspective-dev/client` CDN bundle which loads + * Resolve the URL of a WASM binary shipped in the `wasm_package` Perspective + * package, relative to the URL of the `bundle_package` CDN bundle which loads * it. */ -export function resolve_server_wasm_url( - client_url: string, - filename: string = "perspective-server.wasm", +export function resolve_package_wasm_url( + bundle_url: string, + bundle_package: string, + wasm_package: string, + filename: string, ): string { - const rewritten = client_url.replace( - /\/client(@[^/]+)?\/dist\/cdn\/[^/?#]*([?#].*)?$/, - (_, tag) => `/server${tag ?? ""}/dist/wasm/${filename}`, + const pattern = new RegExp( + `/${bundle_package}(@[^/]+)?/dist/cdn/[^/?#]*([?#].*)?$`, ); - if (rewritten !== client_url) { + const rewritten = bundle_url.replace( + pattern, + (_, tag) => `/${wasm_package}${tag ?? ""}/dist/wasm/${filename}`, + ); + + if (rewritten !== bundle_url) { return rewritten; } - return new URL(`../../../server/dist/wasm/${filename}`, client_url).href; + return new URL(`../../../${wasm_package}/dist/wasm/${filename}`, bundle_url) + .href; +} + +/** + * Resolve the URL of one of `@perspective-dev/server`'s engine WASM binaries + * relative to the URL of the `@perspective-dev/client` CDN bundle which loads + * it. + */ +export function resolve_server_wasm_url( + client_url: string, + filename: string = "perspective-server.wasm", +): string { + return resolve_package_wasm_url(client_url, "client", "server", filename); } diff --git a/rust/perspective-js/src/ts/wasm/migrate.ts b/rust/perspective-js/src/ts/wasm/migrate.ts new file mode 100644 index 0000000000..d108bc3762 --- /dev/null +++ b/rust/perspective-js/src/ts/wasm/migrate.ts @@ -0,0 +1,80 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import * as migrate_wasm from "../../../dist/wasm/perspective-migrate-js.js"; + +/** + * The kind of persisted config `migrate()` upgrades: an engine `ViewConfig`, + * a `` `save()` token, or a `saveWorkspace()` token. + */ +export type MigrateKind = "view" | "viewer" | "workspace"; + +/** + * Information `migrate()` could not carry into the current schema exactly. + */ +export type MigrateNotice = + | { kind: "Dropped"; step: string; pointer: string; value: unknown } + | { kind: "Approximated"; step: string; pointer: string; reason: string }; + +/** + * A config upgraded by `migrate()`, with what the upgrade dropped or + * approximated and the ids of the schema changes it applied. + */ +export interface MigrateResult> { + value: T; + notices: MigrateNotice[]; + steps: string[]; +} + +type MigrateBinary = BufferSource | WebAssembly.Module; + +/** + * Instantiates the `perspective-migrate` WebAssembly module from its + * registered source on the first `migrate()` call. + */ +export class MigrateLoader { + private source?: () => Promise; + private module?: Promise; + + register(source: () => Promise) { + this.source = source; + this.module = undefined; + } + + async migrate>( + kind: MigrateKind, + config: unknown, + from_version?: string, + ): Promise> { + if (this.module === undefined) { + if (this.source === undefined) { + throw new Error("Missing perspective-migrate-js.wasm"); + } + + const source = this.source; + const module = source().then(async (module_or_path) => { + await migrate_wasm.default({ module_or_path }); + return migrate_wasm; + }); + + this.module = module; + module.catch(() => { + if (this.module === module) { + this.module = undefined; + } + }); + } + + const wasm = await this.module; + return wasm.migrate(kind, from_version, config) as MigrateResult; + } +} diff --git a/rust/perspective-js/src/ts/wasm/registry.ts b/rust/perspective-js/src/ts/wasm/registry.ts index 60bc3d07c6..f0409f2c90 100644 --- a/rust/perspective-js/src/ts/wasm/registry.ts +++ b/rust/perspective-js/src/ts/wasm/registry.ts @@ -20,6 +20,11 @@ import type * as psp from "../../../dist/wasm/perspective-js.d.ts"; import { load_wasm_stage_0 } from "./decompress.ts"; import { host_supports_memory64 } from "./features.ts"; +import type { MigrateKind, MigrateResult } from "./migrate.ts"; + +export { MigrateLoader } from "./migrate.ts"; +export type { MigrateKind, MigrateNotice, MigrateResult } from "./migrate.ts"; +export { resolve_package_wasm_url } from "./cdn.ts"; declare const __PSP_VERSION__: string; @@ -50,6 +55,7 @@ export interface WasmRegistration { server?: WasmSource; server32?: WasmSource; server64?: WasmSource; + migrate?: WasmSource; } /** @@ -74,6 +80,17 @@ export type RuntimeProvider = ( wasm: ArrayBuffer | WebAssembly.Module, ) => Promise; +/** + * A loader which runs `perspective-migrate` for `migrate()`. + */ +export interface MigrateHost { + migrate>( + kind: MigrateKind, + config: unknown, + from_version?: string, + ): Promise>; +} + type WasmKey = keyof WasmRegistration; type ServerKey = "server" | "server32" | "server64"; @@ -96,6 +113,8 @@ interface RegistryState { client_runtime?: ClientRuntime; viewer?: Promise; server?: Promise; + migrate_binary?: Promise; + migrate?: MigrateHost; } const STATE_KEY = Symbol.for("@perspective-dev/wasm"); @@ -106,6 +125,7 @@ const WASM_KEYS: WasmKey[] = [ "server", "server32", "server64", + "migrate", ]; const SERVER_KEYS: ServerKey[] = ["server", "server32", "server64"]; @@ -325,7 +345,7 @@ export function register( * - `wasm` - Binaries by role: `client` (`perspective-js.wasm`), `viewer` * (`perspective-viewer.wasm`, which also backs every `Client`), `server` * (one engine build) or `server32` / `server64` (selected by Memory64 - * support). + * support), and `migrate` (`perspective-migrate-js.wasm`). * - `options` - `disable_stage_0` skips decompression for these sources. * * # Examples @@ -469,3 +489,34 @@ export function resolve_server(): Promise { wasm instanceof WebAssembly.Module ? wasm : wasm.slice(0), ); } + +/** + * Resolves the extracted `perspective-migrate` binary. + */ +export function resolve_migrate(): Promise { + const s = state(); + if (s.migrate_binary === undefined) { + const entry = s.sources.migrate; + if (entry === undefined) { + return Promise.reject( + new Error( + "Missing perspective-migrate-js.wasm: register `migrate` with init()", + ), + ); + } + + s.consumed.add("migrate"); + s.migrate_binary = materialize(entry); + } + + return s.migrate_binary; +} + +/** + * The page's one `migrate()` loader, installing `local` if there is none. + */ +export function shared_migrate(local: MigrateHost): MigrateHost { + const s = state(); + s.migrate ??= local; + return s.migrate; +} diff --git a/rust/perspective-js/test/js/cdn_url.spec.ts b/rust/perspective-js/test/js/cdn_url.spec.ts index 274d932691..1afa1ce160 100644 --- a/rust/perspective-js/test/js/cdn_url.spec.ts +++ b/rust/perspective-js/test/js/cdn_url.spec.ts @@ -11,7 +11,10 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ import { test, expect } from "@perspective-dev/test"; -import { resolve_server_wasm_url } from "../../src/ts/wasm/cdn.ts"; +import { + resolve_package_wasm_url, + resolve_server_wasm_url, +} from "../../src/ts/wasm/cdn.ts"; test.describe("resolve_server_wasm_url", function () { test("preserves exact version tags on jsdelivr URLs", function () { @@ -102,3 +105,31 @@ test.describe("resolve_server_wasm_url", function () { ); }); }); + +test.describe("resolve_package_wasm_url", function () { + test("resolves the client's migrate binary from the viewer bundle", function () { + expect( + resolve_package_wasm_url( + "https://cdn.jsdelivr.net/npm/@perspective-dev/viewer@5.7.0/dist/cdn/perspective-viewer.js", + "viewer", + "client", + "perspective-migrate-js.wasm", + ), + ).toEqual( + "https://cdn.jsdelivr.net/npm/@perspective-dev/client@5.7.0/dist/wasm/perspective-migrate-js.wasm", + ); + }); + + test("resolves node_modules layouts to the sibling package", function () { + expect( + resolve_package_wasm_url( + "http://localhost:6598/node_modules/@perspective-dev/viewer/dist/cdn/perspective-viewer.js", + "viewer", + "client", + "perspective-migrate-js.wasm", + ), + ).toEqual( + "http://localhost:6598/node_modules/@perspective-dev/client/dist/wasm/perspective-migrate-js.wasm", + ); + }); +}); diff --git a/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts b/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts index 42154fdc2b..be8de124d7 100644 --- a/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts +++ b/rust/perspective-js/test/js/derived/tree_aggregates.spec.ts @@ -42,7 +42,7 @@ const NUMERIC = [ "abs sum", "sum or zero", "pct sum parent", - "pct sum grand total", + "pct sum total", "var", "stddev", ["weighted mean", ["w"]], diff --git a/rust/perspective-js/test/js/expressions/functionality.spec.js b/rust/perspective-js/test/js/expressions/functionality.spec.js index 74fdfb0ab9..bd372f9bb9 100644 --- a/rust/perspective-js/test/js/expressions/functionality.spec.js +++ b/rust/perspective-js/test/js/expressions/functionality.spec.js @@ -28,10 +28,12 @@ import { joined_column_paths } from "../column_paths.js"; const table = await perspective.table( expressions_common.int_float_data, ); - const validate = await table.validate_expressions([ - 'if ("w" > 1) 5;', - 'if ("w" > 1) 5', - ]); + const validate = await table.validate_expressions( + ['if ("w" > 1) 5;', 'if ("w" > 1) 5'].reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); expect( validate.expression_schema['if ("w" > 1) 5;'], @@ -632,7 +634,12 @@ import { joined_column_paths } from "../column_paths.js"; const expressions = [ `for (var x := 'abc'; x < 10; x += 1) { var y := "w" + 1; y}`, ]; - const validate = await table.validate_expressions(expressions); + const validate = await table.validate_expressions( + expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); for (const expr of expressions) { expect(validate.expression_schema[expr]).toBeUndefined(); @@ -818,7 +825,10 @@ import { joined_column_paths } from "../column_paths.js"; ); const expressions = ["1", "0", "var x := 1; x", "var x := 0; x"]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1262,7 +1272,9 @@ import { joined_column_paths } from "../column_paths.js"; ); const columns = await table.columns(); const view = await table.view({ - expressions: columns.map((col) => `"${col}"`), + expressions: columns + .map((col) => `"${col}"`) + .reduce((x, y) => Object.assign(x, { [y]: y }), {}), }); const schema = await view.schema(); @@ -2101,7 +2113,12 @@ import { joined_column_paths } from "../column_paths.js"; expressions[2], ].reduce((x, y) => Object.assign(x, { [y]: y }), {}), }); - const view4 = await table.view({ expressions: expressions }); + const view4 = await table.view({ + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + }); const schema = await view.schema(); const schema2 = await view2.schema(); @@ -2739,7 +2756,12 @@ import { joined_column_paths } from "../column_paths.js"; `bucket("b", 'M')`, `bucket("b", 's')`, ]; - const results = await table.validate_expressions(expressions); + const results = await table.validate_expressions( + expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); expect(results.expression_schema).toEqual({ '"a" ^ 2': "float", "sqrt(144)": "float", @@ -2860,7 +2882,12 @@ import { joined_column_paths } from "../column_paths.js"; 'min("a", "c")', // invalid, 'max(100, "a")', // valid ]; - const results = await table.validate_expressions(expressions); + const results = await table.validate_expressions( + expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); expect(results.expression_schema).toEqual({ '"a" + "d"': "float", @@ -2894,7 +2921,12 @@ import { joined_column_paths } from "../column_paths.js"; 'upper("c")', // valid 'lower("sdfadsj")', // invalid ]; - const results = await table.validate_expressions(expressions); + const results = await table.validate_expressions( + expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); expect(results.expression_schema).toEqual({ '"a" + "d"': "float", '"c"': "string", @@ -2922,7 +2954,12 @@ import { joined_column_paths } from "../column_paths.js"; "if (\"c\" == 'a') 123; else 0;", // valid "if (", // invalid ]; - const results = await table.validate_expressions(expressions); + const results = await table.validate_expressions( + expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + ); expect(results.expression_schema).toEqual({ '"a" + "d"': "float", "if (\"c\" == 'a') 123; else 0;": "float", @@ -3296,46 +3333,28 @@ import { joined_column_paths } from "../column_paths.js"; table.delete(); }); - test("Validation works with both input formats", async function () { + test("Validation rejects the legacy list input format", async function () { const table = await perspective.table({ a: [1, 2, 3, 4], b: [1, 2, 3, 4], }); - const str_exprs = { - x: '"a"', - y: '"b" * 0.5', - c: "'abcdefg'", - d: "true and false", - e: 'float("a") > 2 ? null : 1', - f: "today()", - g: "now()", - h: "length(123)", - }; - - const dict_exprs = { - x: '"a"', - y: '"b" * 0.5', - c: "'abcdefg'", - d: "true and false", - e: 'float("a") > 2 ? null : 1', - f: "today()", - g: "now()", - h: "length(123)", - }; + const str_exprs = [ + '// x\n"a"', + '// y\n"b" * 0.5', + "// c\n'abcdefg'", + "// d\ntrue and false", + '// e\nfloat("a") > 2 ? null : 1', + "// f\ntoday()", + "// g\nnow()", + "// h\nlength(123)", + ]; - let vstrs = await table.validate_expressions(str_exprs); - let vdicts = await table.validate_expressions(dict_exprs); - expect(vstrs.errors).toEqual(vdicts.errors); - expect(vstrs.expression_schema).toEqual( - vdicts.expression_schema, - ); + await expect( + table.validate_expressions(str_exprs), + ).rejects.toThrow(); - for (let [key, val] of Object.entries(vstrs.expression_alias)) { - let dict_val = vdicts.expression_alias[key]; - // let parsed_val = `${dict_val}`; - expect(val).toEqual(dict_val); - } + await table.delete(); }); }); }); diff --git a/rust/perspective-js/test/js/expressions/numeric.spec.js b/rust/perspective-js/test/js/expressions/numeric.spec.js index 9f4952bd0d..1d28044e51 100644 --- a/rust/perspective-js/test/js/expressions/numeric.spec.js +++ b/rust/perspective-js/test/js/expressions/numeric.spec.js @@ -241,7 +241,10 @@ function validate_binary_operations(output, expressions, operator) { ); const expressions = generate_unary_operations("+"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -264,7 +267,10 @@ function validate_binary_operations(output, expressions, operator) { ); const expressions = generate_binary_operations("+"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -285,7 +291,10 @@ function validate_binary_operations(output, expressions, operator) { ); const expressions = generate_binary_operations("-"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -306,7 +315,10 @@ function validate_binary_operations(output, expressions, operator) { ); const expressions = generate_binary_operations("*"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -328,7 +340,10 @@ function validate_binary_operations(output, expressions, operator) { const expressions = generate_binary_operations("/"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -350,7 +365,10 @@ function validate_binary_operations(output, expressions, operator) { const expressions = generate_binary_operations("%"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -369,7 +387,10 @@ function validate_binary_operations(output, expressions, operator) { const table = await perspective.table(common.arrow.slice()); const expressions = generate_binary_operations("^"); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -390,7 +411,10 @@ function validate_binary_operations(output, expressions, operator) { const expressions = generate_binary_operations(operator); const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const col_names = await joined_column_paths(view); @@ -648,7 +672,10 @@ function validate_binary_operations(output, expressions, operator) { ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); await table.update({ @@ -694,7 +721,10 @@ function validate_binary_operations(output, expressions, operator) { ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); await table.update({ @@ -1226,10 +1256,12 @@ function validate_binary_operations(output, expressions, operator) { b: "string", }); - const validated = await table.validate_expressions([ - 'coalesce("a", "b")', - "coalesce(\"a\", 'fallback')", - ]); + const validated = await table.validate_expressions( + [ + 'coalesce("a", "b")', + "coalesce(\"a\", 'fallback')", + ].reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); expect(validated.errors['coalesce("a", "b")']).toEqual({ @@ -1460,11 +1492,13 @@ function validate_binary_operations(output, expressions, operator) { test("mand should fail validation for non boolean inputs", async function () { const table = await perspective.table(common.comparison_data); - const validated = await table.validate_expressions([ - 'mand("w", "x", "y")', - "mand(100, 200, 300)", - "mand('a', 'b', 'cdef')", - ]); + const validated = await table.validate_expressions( + [ + 'mand("w", "x", "y")', + "mand(100, 200, 300)", + "mand('a', 'b', 'cdef')", + ].reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); expect(validated.errors['mand("w", "x", "y")']).toEqual({ @@ -1577,11 +1611,13 @@ function validate_binary_operations(output, expressions, operator) { test("mor should fail validation for non boolean inputs", async function () { const table = await perspective.table(common.comparison_data); - const validated = await table.validate_expressions([ - 'mor("w", "x", "y")', - "mor(100, 200, 300)", - "mor('a', 'b', 'cdef')", - ]); + const validated = await table.validate_expressions( + [ + 'mor("w", "x", "y")', + "mor(100, 200, 300)", + "mor('a', 'b', 'cdef')", + ].reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); expect(validated.errors['mor("w", "x", "y")']).toEqual({ diff --git a/rust/perspective-js/test/js/expressions/string.spec.js b/rust/perspective-js/test/js/expressions/string.spec.js index 0575f5a3e9..c6f76522d8 100644 --- a/rust/perspective-js/test/js/expressions/string.spec.js +++ b/rust/perspective-js/test/js/expressions/string.spec.js @@ -916,7 +916,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -984,7 +987,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1042,7 +1048,9 @@ const random_string = ( `match("a", '?')`, ]; - const validated = await table.validate_expressions(expressions); + const validated = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); for (const expr of expressions) { @@ -1059,7 +1067,10 @@ const random_string = ( // valid to create a view from them. await expect( table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }), ).rejects.toThrow(); @@ -1080,7 +1091,9 @@ const random_string = ( `match("a", '?')`, ]; - const validated = await table.validate_expressions(expressions); + const validated = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); for (const expr of expressions) { @@ -1096,7 +1109,10 @@ const random_string = ( // Because a bad regex does not raise a parse error, it is still // valid to create a view from them. const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1124,7 +1140,9 @@ const random_string = ( `match_all("a", '+.')`, ]; - const validated = await table.validate_expressions(expressions); + const validated = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validated.expression_schema).toEqual({}); for (const expr of expressions) { @@ -1139,7 +1157,10 @@ const random_string = ( await expect( table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }), ).rejects.toThrow(); @@ -1159,7 +1180,9 @@ const random_string = ( `match(False, '[0-9]{7}')`, ]; - const validated = await table.validate_expressions(expressions); + const validated = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); for (const expr of expressions) { expect(validated.expression_schema[expr]).toBeUndefined(); @@ -1203,7 +1226,9 @@ const random_string = ( `match_all(False, '[0-9]{7}')`, ]; - const validated = await table.validate_expressions(expressions); + const validated = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); for (const expr of expressions) { expect(validated.expression_schema[expr]).toBeUndefined(); @@ -1247,7 +1272,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1290,7 +1318,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1333,7 +1364,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1374,7 +1408,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1414,7 +1451,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1462,7 +1502,10 @@ const random_string = ( ]; const view = await table.view({ - expressions, + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), }); const schema = await view.expression_schema(); @@ -1516,7 +1559,12 @@ const random_string = ( `search("b", '([A-Za-z]{3})')`, ]; - const view = await table.view({ expressions }); + const view = await table.view({ + expressions: expressions.reduce( + (x, y) => Object.assign(x, { [y]: y }), + {}, + ), + }); const schema = await view.expression_schema(); @@ -1594,12 +1642,10 @@ const random_string = ( }; const table = await perspective.table({ a: get_data(100) }); - const expressions = [ - `// address - search("a", '^([a-zA-Z0-9._-]+)@');`, - `// domain - search("a", '@([a-zA-Z.]+)$')`, - ]; + const expressions = { + address: `search("a", '^([a-zA-Z0-9._-]+)@');`, + domain: `search("a", '@([a-zA-Z.]+)$')`, + }; // Make the same regex 10x - make sure it's ok to cache the regex for (let i = 0; i < 10; i++) { @@ -1764,7 +1810,9 @@ const random_string = ( } } - const validate = await table.validate_expressions(expressions); + const validate = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validate.expression_schema).toEqual({}); for (const expr of expressions) { @@ -1826,7 +1874,9 @@ const random_string = ( expressions.push(expr); } - const validate = await table.validate_expressions(expressions); + const validate = await table.validate_expressions( + expressions.reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validate.expression_schema).toEqual(expected_schema); await table.delete(); @@ -2028,28 +2078,24 @@ const random_string = ( y: ["$300", "$123.58", "$0.99", "$1.99"], }); - const expressions = [ - `// year - var year_vec[2]; + const expressions = { + year: `var year_vec[2]; indexof("x", '([0-9]{4})$', year_vec); var start := year_vec[0]; var len := year_vec[1] - start + 1; substring("x", start, len);`, - `// year int - var year_vec[2]; + "year int": `var year_vec[2]; indexof("x", '([0-9]{4})$', year_vec); var start := year_vec[0]; var len := year_vec[1] - start + 1; integer(substring("x", start, len));`, - `// dollars - var out[2]; + dollars: `var out[2]; indexof("y", '^[$]([0-9.]+)', out); substring("y", out[0], out[1] - out[0] + 1)`, - `// dollars float - var out[2]; + "dollars float": `var out[2]; indexof("y", '^[$]([0-9.]+)', out); float(substring("y", out[0], out[1] - out[0] + 1))`, - ]; + }; const view = await table.view({ expressions }); const schema = await view.expression_schema(); @@ -2134,12 +2180,14 @@ const random_string = ( // substring() with invalid indices will pass type checking but // will return null. - const validate = await table.validate_expressions([ - 'substring("x", -1)', - 'substring("x", 1, 10000)', - 'substring("x", 10000)', - 'substring("x", 5, 4)', - ]); + const validate = await table.validate_expressions( + [ + 'substring("x", -1)', + 'substring("x", 1, 10000)', + 'substring("x", 10000)', + 'substring("x", 5, 4)', + ].reduce((x, y) => Object.assign(x, { [y]: y }), {}), + ); expect(validate.expression_schema).toEqual({ 'substring("x", -1)': "string", @@ -2363,16 +2411,15 @@ const random_string = ( }; const table = await perspective.table({ a: get_data(100) }); - const expressions = [ - `// ending - var domain := search(\"a\", '@([a-zA-Z.]+)$'); + const expressions = { + ending: `var domain := search(\"a\", '@([a-zA-Z.]+)$'); var len := length(domain); if (len > 0 and is_not_null(domain)) { search(domain, '[.](.*)$'); } else { 'not found'; }`, - ]; + }; // Make the same regex 10x - make sure it's ok to cache the regex for (let i = 0; i < 10; i++) { diff --git a/rust/perspective-js/test/js/migrate.spec.js b/rust/perspective-js/test/js/migrate.spec.js new file mode 100644 index 0000000000..5e23228b9b --- /dev/null +++ b/rust/perspective-js/test/js/migrate.spec.js @@ -0,0 +1,79 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "./perspective_client"; + +const V2_VIEW = { + row_pivot: ["State"], + expressions: ['//profit\n"Sales" - "Cost"'], +}; + +test.describe("migrate", function () { + test("upgrades a 2.x engine config", async function () { + const result = await perspective.migrate("view", V2_VIEW, "2.6.0"); + expect(result).toEqual({ + value: { + group_by: ["State"], + expressions: { profit: '"Sales" - "Cost"' }, + }, + notices: [], + steps: ["view.v2_aliases", "view.expressions_map"], + }); + }); + + test("an upgraded config creates a view", async function () { + const { value } = await perspective.migrate("view", V2_VIEW, "2.6.0"); + const table = await perspective.table({ + State: ["NY", "NJ"], + Sales: [5, 7], + Cost: [1, 2], + }); + + const view = await table.view(value); + const columns = await view.to_columns(); + expect(columns.profit).toEqual([9, 5, 4]); + await view.delete(); + await table.delete(); + }); + + test("a token's own version overrides from_version", async function () { + const { value, steps } = await perspective.migrate( + "viewer", + { + version: "5.4.0", + plugin: "Datagrid", + columns_config: { Sales: { number_fg_mode: "bar" } }, + }, + "2.0.0", + ); + + expect(value.columns_config).toEqual({ Sales: { fg_mode: "bar" } }); + expect(steps).toEqual(["datagrid.mode_renames"]); + }); + + test("rejects an unknown kind", async function () { + await expect(perspective.migrate("table", {})).rejects.toThrow( + "Unknown config kind", + ); + }); + + test("names the step which rejected a config", async function () { + await expect( + perspective.migrate( + "view", + { row_pivot: ["a"], group_by: ["b"] }, + "2.0.0", + ), + ).rejects.toThrow("view.v2_aliases"); + }); +}); diff --git a/rust/perspective-js/test/js/pivot_nulls.spec.js b/rust/perspective-js/test/js/pivot_nulls.spec.js index 8fb78a56f7..aea1d9417d 100644 --- a/rust/perspective-js/test/js/pivot_nulls.spec.js +++ b/rust/perspective-js/test/js/pivot_nulls.spec.js @@ -367,9 +367,10 @@ import perspective from "./perspective_client"; expect(column).toEqual([4, null, 4]); }); - test("sum not null is a legacy alias for sum", async function () { - const column = await grouped({ a: "sum not null" }, {}, "a"); - expect(column).toEqual([4, null, 4]); + test("sum not null is not an aggregate", async function () { + await expect( + grouped({ a: "sum not null" }, {}, "a"), + ).rejects.toThrow(); }); test("sum or zero still matches sum when a value is present", async function () { @@ -393,12 +394,8 @@ import perspective from "./perspective_client"; expect(column).toEqual([100, null, 100]); }); - test("pct sum grand total reports null for an all-null group", async function () { - const column = await grouped( - { a: "pct sum grand total" }, - {}, - "a", - ); + test("pct sum total reports null for an all-null group", async function () { + const column = await grouped({ a: "pct sum total" }, {}, "a"); expect(column).toEqual([100, null, 100]); }); diff --git a/rust/perspective-js/test/js/registry.spec.js b/rust/perspective-js/test/js/registry.spec.js index be82d2a98e..b938a1acb2 100644 --- a/rust/perspective-js/test/js/registry.spec.js +++ b/rust/perspective-js/test/js/registry.spec.js @@ -22,10 +22,16 @@ const URLS = { cdn: `${CLIENT}/dist/cdn/perspective.js`, viewer_cdn: `${VIEWER}/dist/cdn/perspective-viewer.js`, client_wasm: `${CLIENT}/dist/wasm/perspective-js.wasm`, + migrate_wasm: `${CLIENT}/dist/wasm/perspective-migrate-js.wasm`, wasm32: `${SERVER}/dist/wasm/perspective-server.wasm`, wasm64: `${SERVER}/dist/wasm/perspective-server.memory64.wasm`, }; +const V2_VIEW = { + row_pivot: ["State"], + expressions: ['//profit\n"Sales" - "Cost"'], +}; + async function in_page(page, fn, args = {}) { await page.goto(`${CLIENT}/test/html/test.html`); return await page.evaluate(fn, { urls: URLS, ...args }); @@ -245,6 +251,33 @@ test.describe("wasm registry", () => { test.expect(result).toEqual({ esm: false, registry: true, cdn: true }); }); + test("client and viewer migrate() share one download", async ({ page }) => { + const result = await in_page( + page, + async ({ urls, config }) => { + const perspective = await import(urls.esm); + let downloads = 0; + await perspective.init({ + migrate: () => { + downloads += 1; + return fetch(urls.migrate_wasm); + }, + }); + + const viewer = await import(urls.viewer_cdn); + const a = await perspective.migrate("view", config, "2.6.0"); + const b = await viewer.default.migrate("view", config, "2.6.0"); + return { + downloads, + same: JSON.stringify(a) === JSON.stringify(b), + }; + }, + { config: V2_VIEW }, + ); + + test.expect(result).toEqual({ downloads: 1, same: true }); + }); + test("make_client() Clients load into the viewer", async ({ page }) => { const result = await in_page(page, async ({ urls }) => { await import(urls.viewer_cdn); diff --git a/rust/perspective-migrate-js/Cargo.toml b/rust/perspective-migrate-js/Cargo.toml new file mode 100644 index 0000000000..83bd0ca42d --- /dev/null +++ b/rust/perspective-migrate-js/Cargo.toml @@ -0,0 +1,33 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +[package] +name = "perspective-migrate-js" +version = "5.6.1" +authors = ["Andrew Stein "] +edition = "2024" +description = "WebAssembly bindings for `perspective-migrate`, shipped with `@perspective-dev/client`." +repository = "https://github.com/perspective-dev/perspective" +license = "Apache-2.0" +homepage = "https://perspective-dev.github.io" +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[dependencies] +perspective-migrate = { version = "5.6.1" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.107", features = ["preserve_order"] } +serde-wasm-bindgen = "0.6.0" +wasm-bindgen = { version = "=0.2.108" } diff --git a/rust/perspective-migrate-js/src/lib.rs b/rust/perspective-migrate-js/src/lib.rs new file mode 100644 index 0000000000..3e8b063640 --- /dev/null +++ b/rust/perspective-migrate-js/src/lib.rs @@ -0,0 +1,41 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use perspective_migrate::{Kind, Migrator}; +use serde::Serialize; +use wasm_bindgen::prelude::*; + +/// Upgrade a persisted Perspective config to the current schema, returning +/// `{value, notices, steps}`. +#[wasm_bindgen] +pub fn migrate(kind: &str, from: Option, value: JsValue) -> Result { + let kind = match kind { + "view" => Kind::View, + "viewer" => Kind::Viewer, + "workspace" => Kind::Workspace, + _ => { + return Err(JsError::new(&format!( + "Unknown config kind `{kind}`; expected `view`, `viewer` or `workspace`" + ))); + }, + }; + + let value: serde_json::Value = serde_wasm_bindgen::from_value(value)?; + let migrated = Migrator::new().migrate(kind, from.as_deref(), value)?; + let result = serde_json::json!({ + "value": migrated.value, + "notices": migrated.notices, + "steps": migrated.steps, + }); + + Ok(result.serialize(&serde_wasm_bindgen::Serializer::json_compatible())?) +} diff --git a/rust/perspective-migrate/Cargo.toml b/rust/perspective-migrate/Cargo.toml new file mode 100644 index 0000000000..f49c44cdce --- /dev/null +++ b/rust/perspective-migrate/Cargo.toml @@ -0,0 +1,35 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +[package] +name = "perspective-migrate" +version = "5.6.1" +authors = ["Andrew Stein "] +edition = "2024" +description = "Upgrades persisted Perspective `ViewConfig`, `` and workspace configs from any release since 2.0 to the current schema." +repository = "https://github.com/perspective-dev/perspective" +license = "Apache-2.0" +homepage = "https://perspective-dev.github.io" +keywords = ["perspective", "migration", "config", "datagrid", "visualization"] +categories = ["visualization", "data-structures"] +documentation = "https://docs.rs/perspective-migrate" +include = ["src/**/*", "Cargo.toml"] + +[lib] +path = "src/lib.rs" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0.107", features = ["preserve_order"] } + +[dev-dependencies] +perspective-client = { path = "../perspective-client" } diff --git a/rust/perspective-migrate/corpus/view/2.0.0/alias_conflict.json b/rust/perspective-migrate/corpus/view/2.0.0/alias_conflict.json new file mode 100644 index 0000000000..092f336fdb --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.0.0/alias_conflict.json @@ -0,0 +1,24 @@ +{ + "kind": "view", + "from": "2.0.0", + "source": "hand-written", + "input": { + "row_pivot": [ + "A" + ], + "group_by": [ + "B" + ] + }, + "error": { + "kind": "Conflict", + "step": "view.v2_aliases", + "pointer": "", + "keys": [ + "row_pivot", + "group_by" + ] + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/view/2.0.0/empty.json b/rust/perspective-migrate/corpus/view/2.0.0/empty.json new file mode 100644 index 0000000000..34113d8591 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.0.0/empty.json @@ -0,0 +1,8 @@ +{ + "kind": "view", + "source": "hand-written", + "input": {}, + "output": {}, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/view/2.0.0/legacy_aggregate.json b/rust/perspective-migrate/corpus/view/2.0.0/legacy_aggregate.json new file mode 100644 index 0000000000..a443e3aad0 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.0.0/legacy_aggregate.json @@ -0,0 +1,37 @@ +{ + "kind": "view", + "from": "2.0.0", + "source": "hand-written from v2.10.1:packages/perspective/src/js/perspective.js", + "input": { + "aggregate": [ + { + "column": "Sales", + "op": "sum" + }, + { + "column": "Profit", + "op": "avg" + } + ], + "group_by": [ + "Region" + ] + }, + "output": { + "group_by": [ + "Region" + ], + "aggregates": { + "Sales": "sum", + "Profit": "avg" + }, + "columns": [ + "Sales", + "Profit" + ] + }, + "notices": [], + "steps": [ + "view.v2_aliases" + ] +} diff --git a/rust/perspective-migrate/corpus/view/2.10.1/get_config_tuples.json b/rust/perspective-migrate/corpus/view/2.10.1/get_config_tuples.json new file mode 100644 index 0000000000..c3f25c7e37 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.10.1/get_config_tuples.json @@ -0,0 +1,44 @@ +{ + "kind": "view", + "from": "2.10.1", + "source": "hand-written from v2.10.1:packages/perspective/src/js/perspective.js parse_expression_strings", + "input": { + "group_by": [], + "columns": [ + "profit" + ], + "expressions": [ + [ + "profit", + "//profit\n\"Sales\" - \"Cost\"", + "COLUMN0 - COLUMN1", + { + "COLUMN0": "Sales", + "COLUMN1": "Cost" + } + ], + [ + "doubled", + "\"Sales\" * 2", + "COLUMN0 * 2", + { + "COLUMN0": "Sales" + } + ] + ] + }, + "output": { + "group_by": [], + "columns": [ + "profit" + ], + "expressions": { + "profit": "\"Sales\" - \"Cost\"", + "doubled": "\"Sales\" * 2" + } + }, + "notices": [], + "steps": [ + "view.expression_tuples" + ] +} diff --git a/rust/perspective-migrate/corpus/view/2.10.1/v2_engine_aliases.json b/rust/perspective-migrate/corpus/view/2.10.1/v2_engine_aliases.json new file mode 100644 index 0000000000..e6a42660f3 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.10.1/v2_engine_aliases.json @@ -0,0 +1,90 @@ +{ + "kind": "view", + "source": "hand-written from v2.10.1:packages/perspective/src/js/config/constants.js", + "input": { + "row_pivot": [ + "State" + ], + "col_pivots": [ + "Category" + ], + "filters": [ + [ + "Sales", + ">", + 100 + ] + ], + "sorts": [ + [ + "Sales", + "desc" + ] + ], + "computed_columns": [], + "version": "2.10.1", + "split_by_depth": 1, + "viewport": { + "top": 0, + "height": 10 + }, + "filter_op": "|" + }, + "output": { + "group_by": [ + "State" + ], + "split_by": [ + "Category" + ], + "filter": [ + [ + "Sales", + ">", + 100 + ] + ], + "sort": [ + [ + "Sales", + "desc" + ] + ], + "filter_op": "or" + }, + "notices": [ + { + "kind": "Dropped", + "step": "view.v2_aliases", + "pointer": "/computed_columns", + "value": [] + }, + { + "kind": "Dropped", + "step": "view.v2_aliases", + "pointer": "/version", + "value": "2.10.1" + }, + { + "kind": "Dropped", + "step": "view.drop_split_by_depth", + "pointer": "/split_by_depth", + "value": 1 + }, + { + "kind": "Dropped", + "step": "view.drop_viewport", + "pointer": "/viewport", + "value": { + "top": 0, + "height": 10 + } + } + ], + "steps": [ + "view.v2_aliases", + "view.drop_split_by_depth", + "view.drop_viewport", + "view.filter_op_symbols" + ] +} diff --git a/rust/perspective-migrate/corpus/view/2.6.0/expression_array.json b/rust/perspective-migrate/corpus/view/2.6.0/expression_array.json new file mode 100644 index 0000000000..4b5c4da343 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/2.6.0/expression_array.json @@ -0,0 +1,31 @@ +{ + "kind": "view", + "from": "2.6.0", + "source": "hand-written", + "input": { + "columns": [ + "profit", + "Sales" + ], + "expressions": [ + "//profit\n\"Sales\" - \"Cost\"", + "\"Sales\" * 2", + "// \n\"Cost\"" + ] + }, + "output": { + "columns": [ + "profit", + "Sales" + ], + "expressions": { + "profit": "\"Sales\" - \"Cost\"", + "\"Sales\" * 2": "\"Sales\" * 2", + "// \n\"Cost\"": "// \n\"Cost\"" + } + }, + "notices": [], + "steps": [ + "view.expressions_map" + ] +} diff --git a/rust/perspective-migrate/corpus/view/3.7.0/multi_aggregates.json b/rust/perspective-migrate/corpus/view/3.7.0/multi_aggregates.json new file mode 100644 index 0000000000..074db3492e --- /dev/null +++ b/rust/perspective-migrate/corpus/view/3.7.0/multi_aggregates.json @@ -0,0 +1,53 @@ +{ + "kind": "view", + "from": "3.7.0", + "source": "hand-written from v3.7.0:rust/perspective-js/test/js/aggregates.spec.js", + "input": { + "columns": [ + "Sales", + "Profit" + ], + "group_by": [ + "Region" + ], + "aggregates": { + "Sales": [ + "weighted mean", + "Quantity" + ], + "Profit": [ + "max by", + "Sales" + ], + "Region": "count" + } + }, + "output": { + "columns": [ + "Sales", + "Profit" + ], + "group_by": [ + "Region" + ], + "aggregates": { + "Sales": [ + "weighted mean", + [ + "Quantity" + ] + ], + "Profit": [ + "max by", + [ + "Sales" + ] + ], + "Region": "count" + } + }, + "notices": [], + "steps": [ + "view.multi_aggregate_args" + ] +} diff --git a/rust/perspective-migrate/corpus/view/4.2.0/expression_array.json b/rust/perspective-migrate/corpus/view/4.2.0/expression_array.json new file mode 100644 index 0000000000..450eac3cc4 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/4.2.0/expression_array.json @@ -0,0 +1,30 @@ +{ + "kind": "view", + "from": "4.2.0", + "source": "hand-written", + "input": { + "expressions": [ + "// \n\"Cost\"", + "//x\n\"A\"", + "1 + 1" + ] + }, + "output": { + "expressions": { + "": "\"Cost\"", + "x": "\"A\"", + "1 + 1": "1 + 1" + } + }, + "notices": [ + { + "kind": "Approximated", + "step": "view.expressions_map", + "pointer": "/expressions", + "reason": "expression columns without `columns` are no longer ordered by the array" + } + ], + "steps": [ + "view.expressions_map" + ] +} diff --git a/rust/perspective-migrate/corpus/view/5.1.0/unused_aggregates.json b/rust/perspective-migrate/corpus/view/5.1.0/unused_aggregates.json new file mode 100644 index 0000000000..79a48ad147 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/5.1.0/unused_aggregates.json @@ -0,0 +1,62 @@ +{ + "kind": "view", + "from": "5.1.0", + "source": "hand-written", + "input": { + "columns": [ + "Sales" + ], + "aggregates": { + "Sales": "sum", + "Profit": "py_agg", + "Quantity": "weighted mean", + "Discount": [ + "weighted mean", + [] + ], + "Cost": "not an aggregate", + "Margin": "avg" + } + }, + "output": { + "columns": [ + "Sales" + ], + "aggregates": { + "Sales": "sum", + "Margin": "avg" + } + }, + "notices": [ + { + "kind": "Dropped", + "step": "view.prune_unused_aggregates", + "pointer": "/aggregates/Profit", + "value": "py_agg" + }, + { + "kind": "Dropped", + "step": "view.prune_unused_aggregates", + "pointer": "/aggregates/Quantity", + "value": "weighted mean" + }, + { + "kind": "Dropped", + "step": "view.prune_unused_aggregates", + "pointer": "/aggregates/Discount", + "value": [ + "weighted mean", + [] + ] + }, + { + "kind": "Dropped", + "step": "view.prune_unused_aggregates", + "pointer": "/aggregates/Cost", + "value": "not an aggregate" + } + ], + "steps": [ + "view.prune_unused_aggregates" + ] +} diff --git a/rust/perspective-migrate/corpus/view/5.5.0/duplicates_and_aliases.json b/rust/perspective-migrate/corpus/view/5.5.0/duplicates_and_aliases.json new file mode 100644 index 0000000000..a64eae82f9 --- /dev/null +++ b/rust/perspective-migrate/corpus/view/5.5.0/duplicates_and_aliases.json @@ -0,0 +1,83 @@ +{ + "kind": "view", + "from": "5.5.0", + "source": "hand-written", + "input": { + "group_by": [ + "A", + "B", + "A" + ], + "columns": [ + "x", + "y", + "z", + "v" + ], + "filter": [ + [ + "A", + "is None", + null + ], + [ + "B", + "is not None" + ] + ], + "aggregates": { + "x": "sum not null", + "y": "last value", + "z": "pct_sum_grand_total", + "w": "py agg", + "v": "high minuslow" + } + }, + "output": { + "group_by": [ + "A", + "B" + ], + "columns": [ + "x", + "y", + "z", + "v" + ], + "filter": [ + [ + "A", + "is null", + null + ], + [ + "B", + "is not null" + ] + ], + "aggregates": { + "x": "sum", + "y": "last", + "z": "pct sum total", + "v": "high minus low" + } + }, + "notices": [ + { + "kind": "Dropped", + "step": "view.dedupe_group_by", + "pointer": "/group_by", + "value": "A" + }, + { + "kind": "Dropped", + "step": "view.canonical_names", + "pointer": "/aggregates/w", + "value": "py agg" + } + ], + "steps": [ + "view.dedupe_group_by", + "view.canonical_names" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.0.0/empty.json b/rust/perspective-migrate/corpus/viewer/2.0.0/empty.json new file mode 100644 index 0000000000..1ca1af26e4 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.0.0/empty.json @@ -0,0 +1,10 @@ +{ + "kind": "viewer", + "source": "hand-written", + "input": {}, + "output": { + "version": "CURRENT" + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.0.0/ignored_keys.json b/rust/perspective-migrate/corpus/viewer/2.0.0/ignored_keys.json new file mode 100644 index 0000000000..5fd7fae689 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.0.0/ignored_keys.json @@ -0,0 +1,66 @@ +{ + "kind": "viewer", + "from": "2.0.0", + "source": "8041a712eb^:rust/perspective-viewer/test/js/migrate_viewer.spec.ts \"Bucket by year\"", + "input": { + "plugin": "Datagrid", + "row-pivots": [ + "State" + ], + "computed-columns": [], + "selectable": null, + "editable": null, + "group_by": [], + "columns": [ + "Sales" + ], + "plugin_config": {}, + "name": "Old" + }, + "output": { + "plugin": "Datagrid", + "group_by": [], + "columns": [ + "Sales" + ], + "plugin_config": {}, + "version": "CURRENT" + }, + "notices": [ + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/row-pivots", + "value": [ + "State" + ] + }, + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/computed-columns", + "value": [] + }, + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/selectable", + "value": null + }, + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/editable", + "value": null + }, + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/name", + "value": "Old" + } + ], + "steps": [ + "viewer.unknown_keys" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.10.0/datagrid_editable.json b/rust/perspective-migrate/corpus/viewer/2.10.0/datagrid_editable.json new file mode 100644 index 0000000000..46b518abf4 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.10.0/datagrid_editable.json @@ -0,0 +1,32 @@ +{ + "kind": "viewer", + "source": "hand-written from v2.10.0:packages/perspective-viewer-datagrid/src/js/plugin/save.js", + "input": { + "version": "2.10.0", + "plugin": "Datagrid", + "plugin_config": { + "columns": {}, + "editable": false, + "scroll_lock": true + }, + "columns": [ + "Sales" + ] + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "scroll_lock": true, + "edit_mode": "READ_ONLY" + }, + "columns": [ + "Sales" + ] + }, + "notices": [], + "steps": [ + "datagrid.edit_mode", + "datagrid.size_overrides" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.6.0/xy_scatter_symbol_list.json b/rust/perspective-migrate/corpus/viewer/2.6.0/xy_scatter_symbol_list.json new file mode 100644 index 0000000000..7cd6c48279 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.6.0/xy_scatter_symbol_list.json @@ -0,0 +1,82 @@ +{ + "kind": "viewer", + "from": "2.6.0", + "source": "8041a712eb^:rust/perspective-viewer/test/js/migrate_viewer.spec.ts \"From 0.0.0\"", + "input": { + "plugin": "X/Y Scatter", + "plugin_config": { + "columns": { + "Region": { + "symbols": [ + { + "key": "Central", + "value": "circle" + } + ] + } + } + }, + "title": null, + "group_by": [], + "split_by": [], + "columns": [ + "'hello'", + "expr", + null, + null, + "Region" + ], + "filter": [], + "sort": [], + "expressions": [ + "// expr\n1+1", + "'hello'" + ], + "aggregates": {} + }, + "output": { + "plugin": "X/Y Scatter", + "plugin_config": {}, + "title": null, + "group_by": [], + "split_by": [], + "columns": [ + "'hello'", + "expr", + null, + null + ], + "filter": [], + "sort": [], + "expressions": { + "expr": "1+1", + "'hello'": "'hello'" + }, + "aggregates": {}, + "columns_config": {}, + "version": "CURRENT" + }, + "notices": [ + { + "kind": "Dropped", + "step": "charts.scatter_symbol_slot", + "pointer": "/columns/4", + "value": "Region" + }, + { + "kind": "Dropped", + "step": "charts.scatter_symbol_slot", + "pointer": "/columns_config/Region/symbols", + "value": { + "Central": "circle" + } + } + ], + "steps": [ + "charts.symbols_map", + "charts.symbols_move", + "charts.scatter_symbol_slot", + "charts.d3fc_removed", + "view.expressions_map" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.7.1/datagrid.json b/rust/perspective-migrate/corpus/viewer/2.7.1/datagrid.json new file mode 100644 index 0000000000..a3a93b80c2 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.7.1/datagrid.json @@ -0,0 +1,130 @@ +{ + "kind": "viewer", + "source": "8041a712eb^:rust/perspective-viewer/test/js/migrate_viewer.spec.ts \"From 2.7.1 - Datagrid\"", + "input": { + "version": "2.7.1", + "plugin": "Datagrid", + "plugin_config": { + "columns": { + "datetime": { + "format": "custom", + "fractionalSecondDigits": 3, + "second": "2-digit", + "minute": "disabled", + "hour": "2-digit", + "day": "2-digit", + "weekday": "short", + "month": "narrow", + "year": "numeric", + "hour12": false, + "timeZone": "America/Curacao", + "datetime_color_mode": "foreground" + }, + "Order ID": { + "format": "link", + "string_color_mode": "foreground", + "color": "#ff0000" + }, + "Row ID": { + "number_bg_mode": "gradient", + "fixed": 1, + "pos_fg_color": "#000000", + "neg_fg_color": "#000000", + "bg_gradient": 9 + }, + "Order Date": { + "dateStyle": "disabled", + "datetime_color_mode": "foreground", + "color": "#00ff00" + } + }, + "editable": true, + "scroll_lock": false + }, + "columns": [ + "Row ID", + "Order ID", + "Order Date", + "datetime" + ], + "expressions": { + "datetime": "datetime(100)" + }, + "aggregates": {}, + "filter": [], + "group_by": [], + "sort": [], + "split_by": [], + "title": null + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "scroll_lock": false, + "edit_mode": "EDIT" + }, + "columns": [ + "Row ID", + "Order ID", + "Order Date", + "datetime" + ], + "expressions": { + "datetime": "datetime(100)" + }, + "aggregates": {}, + "filter": [], + "group_by": [], + "sort": [], + "split_by": [], + "title": null, + "columns_config": { + "datetime": { + "fg_mode": "color", + "date_format": { + "fractionalSecondDigits": 3, + "second": "2-digit", + "hour": "2-digit", + "day": "2-digit", + "weekday": "short", + "month": "narrow", + "year": "numeric", + "hour12": false, + "timeZone": "America/Curacao" + } + }, + "Order ID": { + "fg_mode": "color", + "fg_color": "#ff0000", + "link": true + }, + "Row ID": { + "bg_mode": "gradient", + "bg_gradient": 9, + "number_format": { + "minimumFractionDigits": 1, + "maximumFractionDigits": 1 + }, + "fg_color": "linear-gradient(to right, #000000 0%, #000000 100%)" + }, + "Order Date": { + "fg_mode": "color", + "fg_color": "#00ff00", + "date_format": { + "timeStyle": "medium" + } + } + } + }, + "notices": [], + "steps": [ + "datagrid.columns_config", + "datagrid.edit_mode", + "datagrid.size_overrides", + "datagrid.colors", + "datagrid.mode_renames", + "datagrid.format_flags", + "style.date_format" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/2.7.1/xy_scatter.json b/rust/perspective-migrate/corpus/viewer/2.7.1/xy_scatter.json new file mode 100644 index 0000000000..87b8f45416 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/2.7.1/xy_scatter.json @@ -0,0 +1,71 @@ +{ + "kind": "viewer", + "source": "8041a712eb^:rust/perspective-viewer/test/js/migrate_viewer.spec.ts \"From 2.7.1 - X/Y Scatter\"", + "input": { + "version": "2.7.1", + "plugin": "X/Y Scatter", + "plugin_config": { + "columns": { + "Category": { + "symbols": { + "Furniture": "star" + } + } + } + }, + "columns": [ + "Row ID", + "City", + null, + null, + "Category" + ], + "aggregates": {}, + "expressions": {}, + "filter": [], + "group_by": [], + "sort": [], + "split_by": [], + "title": null + }, + "output": { + "version": "CURRENT", + "plugin": "X/Y Scatter", + "plugin_config": {}, + "columns": [ + "Row ID", + "City", + null, + null + ], + "aggregates": {}, + "expressions": {}, + "filter": [], + "group_by": [], + "sort": [], + "split_by": [], + "title": null, + "columns_config": {} + }, + "notices": [ + { + "kind": "Dropped", + "step": "charts.scatter_symbol_slot", + "pointer": "/columns/4", + "value": "Category" + }, + { + "kind": "Dropped", + "step": "charts.scatter_symbol_slot", + "pointer": "/columns_config/Category/symbols", + "value": { + "Furniture": "star" + } + } + ], + "steps": [ + "charts.symbols_move", + "charts.scatter_symbol_slot", + "charts.d3fc_removed" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/4.0.0/datagrid_shared_columns_config.json b/rust/perspective-migrate/corpus/viewer/4.0.0/datagrid_shared_columns_config.json new file mode 100644 index 0000000000..c3f1602d76 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/4.0.0/datagrid_shared_columns_config.json @@ -0,0 +1,60 @@ +{ + "kind": "viewer", + "source": "hand-written", + "input": { + "version": "4.0.0", + "plugin": "Datagrid", + "plugin_config": { + "columns": {}, + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Region": { + "symbols": { + "East": "circle" + } + }, + "Sales": { + "number_fg_mode": "disabled" + } + }, + "table": null, + "columns": [ + "Sales" + ] + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Region": {}, + "Sales": { + "fg_mode": "disabled" + } + }, + "columns": [ + "Sales" + ] + }, + "notices": [ + { + "kind": "Dropped", + "step": "datagrid.chart_keys", + "pointer": "/columns_config/Region/symbols", + "value": { + "East": "circle" + } + } + ], + "steps": [ + "datagrid.chart_keys", + "viewer.null_table", + "datagrid.size_overrides", + "datagrid.mode_renames" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/4.0.0/plugin_config_not_an_object.json b/rust/perspective-migrate/corpus/viewer/4.0.0/plugin_config_not_an_object.json new file mode 100644 index 0000000000..2400873e7c --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/4.0.0/plugin_config_not_an_object.json @@ -0,0 +1,31 @@ +{ + "kind": "viewer", + "source": "hand-written", + "input": { + "version": "4.0.0", + "plugin": "My Plugin", + "plugin_config": [ + 1, + 2 + ] + }, + "output": { + "version": "CURRENT", + "plugin": "My Plugin", + "plugin_config": {} + }, + "notices": [ + { + "kind": "Dropped", + "step": "viewer.plugin_config_object", + "pointer": "/plugin_config", + "value": [ + 1, + 2 + ] + } + ], + "steps": [ + "viewer.plugin_config_object" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/4.4.1/d3fc_y_bar.json b/rust/perspective-migrate/corpus/viewer/4.4.1/d3fc_y_bar.json new file mode 100644 index 0000000000..0fd1018408 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/4.4.1/d3fc_y_bar.json @@ -0,0 +1,117 @@ +{ + "kind": "viewer", + "source": "hand-written from v4.4.1:packages/viewer-d3fc/src/ts/types.ts Settings", + "input": { + "version": "4.4.1", + "plugin": "Y Bar", + "plugin_config": { + "legend": { + "left": "100px", + "top": "20px" + }, + "zoom": { + "k": 2, + "x": 0, + "y": 0 + }, + "splitMainValues": [ + "Profit" + ], + "hideKeys": [] + }, + "columns_config": { + "Sales": { + "number_fg_mode": "bar", + "number_format": { + "style": "currency", + "currency": "USD" + } + } + }, + "columns": [ + "Sales", + "Profit" + ], + "group_by": [ + "Region" + ], + "split_by": [], + "filter": [], + "sort": [], + "expressions": {}, + "aggregates": {}, + "title": "Sales", + "table": "superstore", + "settings": true, + "theme": "Pro Dark" + }, + "output": { + "version": "CURRENT", + "plugin": "Y Bar", + "plugin_config": {}, + "columns_config": { + "Sales": { + "number_format": { + "style": "currency", + "currency": "USD" + } + }, + "Profit": { + "alt_axis": true + } + }, + "columns": [ + "Sales", + "Profit" + ], + "group_by": [ + "Region" + ], + "split_by": [], + "filter": [], + "sort": [], + "expressions": {}, + "aggregates": {}, + "title": "Sales", + "table": "superstore", + "settings": true, + "theme": "Pro Dark" + }, + "notices": [ + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/plugin_config/legend", + "value": { + "left": "100px", + "top": "20px" + } + }, + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/plugin_config/zoom", + "value": { + "k": 2, + "x": 0, + "y": 0 + } + }, + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/plugin_config/hideKeys", + "value": [] + }, + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/columns_config/Sales/number_fg_mode", + "value": "bar" + } + ], + "steps": [ + "charts.alt_axis", + "charts.d3fc_removed" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/4.4.1/openlayers_map_scatter.json b/rust/perspective-migrate/corpus/viewer/4.4.1/openlayers_map_scatter.json new file mode 100644 index 0000000000..6a5ad498f0 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/4.4.1/openlayers_map_scatter.json @@ -0,0 +1,93 @@ +{ + "kind": "viewer", + "source": "hand-written from v4.4.1:packages/viewer-openlayers/src/js/views/base-map.js", + "input": { + "version": "4.4.1", + "plugin": "Map Scatter", + "plugin_config": { + "center": [ + -8226000, + 4970000 + ], + "zoom": 10 + }, + "columns_config": { + "Count": { + "number_format": { + "maximumFractionDigits": 0 + } + }, + "Name": { + "string_color_mode": "foreground" + } + }, + "columns": [ + "Lon", + "Lat", + "Count", + "Size", + "Name" + ], + "group_by": [], + "split_by": [], + "filter": [], + "sort": [], + "expressions": {}, + "aggregates": {}, + "title": null + }, + "output": { + "version": "CURRENT", + "plugin": "Map Scatter", + "plugin_config": {}, + "columns_config": { + "Count": { + "number_format": { + "maximumFractionDigits": 0 + } + } + }, + "columns": [ + "Lon", + "Lat", + "Count", + "Size", + null, + "Name" + ], + "group_by": [], + "split_by": [], + "filter": [], + "sort": [], + "expressions": {}, + "aggregates": {}, + "title": null + }, + "notices": [ + { + "kind": "Dropped", + "step": "maps.openlayers_removed", + "pointer": "/plugin_config/center", + "value": [ + -8226000, + 4970000 + ] + }, + { + "kind": "Dropped", + "step": "maps.openlayers_removed", + "pointer": "/plugin_config/zoom", + "value": 10 + }, + { + "kind": "Dropped", + "step": "maps.openlayers_removed", + "pointer": "/columns_config/Name/string_color_mode", + "value": "foreground" + } + ], + "steps": [ + "maps.label_slot", + "maps.openlayers_removed" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/4.5.0/version_overrides_from.json b/rust/perspective-migrate/corpus/viewer/4.5.0/version_overrides_from.json new file mode 100644 index 0000000000..a367801e0e --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/4.5.0/version_overrides_from.json @@ -0,0 +1,31 @@ +{ + "kind": "viewer", + "from": "2.6.0", + "source": "hand-written", + "input": { + "version": "4.5.0", + "plugin": "X/Y Scatter", + "columns": [ + "X", + "Y", + "Color", + "Size", + "Label", + "Tooltip" + ] + }, + "output": { + "version": "CURRENT", + "plugin": "X/Y Scatter", + "columns": [ + "X", + "Y", + "Color", + "Size", + "Label", + "Tooltip" + ] + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.2.0/datagrid_sizes_and_colors.json b/rust/perspective-migrate/corpus/viewer/5.2.0/datagrid_sizes_and_colors.json new file mode 100644 index 0000000000..3f9041424d --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.2.0/datagrid_sizes_and_colors.json @@ -0,0 +1,148 @@ +{ + "kind": "viewer", + "source": "hand-written from v5.2.0:packages/viewer-datagrid/src/ts/types.ts", + "input": { + "version": "5.2.0", + "plugin": "Datagrid", + "plugin_config": { + "columns": { + "Sales": { + "column_size_override": 120 + }, + "__ROW_PATH__": { + "column_size_override": 200 + }, + "Profit": {} + }, + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Sales": { + "number_bg_mode": "gradient", + "pos_bg_color": "#00ff00", + "neg_bg_color": "#ff0000", + "bg_gradient": 100 + }, + "Profit": { + "number_fg_mode": "bar", + "pos_fg_color": "#123456" + }, + "Ship Mode": { + "string_color_mode": "series", + "color": "#ff0000" + }, + "Order Date": { + "datetime_color_mode": "background", + "color": "#0000ff" + }, + "Region": { + "string_color_mode": "foreground", + "color": "#00ff00" + } + }, + "columns": [ + "Sales", + "Profit", + "Ship Mode", + "Order Date" + ], + "group_by": [ + "Region" + ], + "split_by": [], + "aggregates": { + "Ship Mode": "dominant", + "Order Date": "dominant" + }, + "table": "superstore" + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Sales": { + "bg_gradient": 100, + "column_size_override": 120, + "bg_mode": "gradient", + "bg_color": "linear-gradient(to right, #ff0000 0%, #00ff00 100%)" + }, + "Profit": { + "fg_mode": "bar" + }, + "Ship Mode": { + "bg_mode": "series" + }, + "Order Date": { + "bg_mode": "color", + "bg_color": "#0000ff" + }, + "Region": {} + }, + "columns": [ + "Sales", + "Profit", + "Ship Mode", + "Order Date" + ], + "group_by": [ + "Region" + ], + "split_by": [], + "aggregates": { + "Ship Mode": "dominant", + "Order Date": "dominant" + }, + "table": "superstore" + }, + "notices": [ + { + "kind": "Dropped", + "step": "datagrid.size_overrides", + "pointer": "/plugin_config/columns/__ROW_PATH__", + "value": { + "column_size_override": 200 + } + }, + { + "kind": "Approximated", + "step": "datagrid.colors", + "pointer": "/columns_config/Sales/bg_colors", + "reason": "the theme background middle stop is not reconstructed" + }, + { + "kind": "Dropped", + "step": "datagrid.colors", + "pointer": "/columns_config/Profit/fg_colors", + "value": "#123456" + }, + { + "kind": "Dropped", + "step": "datagrid.mode_renames", + "pointer": "/columns_config/Ship Mode/color", + "value": "#ff0000" + }, + { + "kind": "Dropped", + "step": "datagrid.role_keys", + "pointer": "/columns_config/Region/fg_mode", + "value": "color" + }, + { + "kind": "Dropped", + "step": "datagrid.role_keys", + "pointer": "/columns_config/Region/fg_color", + "value": "#00ff00" + } + ], + "steps": [ + "datagrid.size_overrides", + "datagrid.colors", + "datagrid.mode_renames", + "datagrid.role_keys" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.3.0/map_carto_tiles.json b/rust/perspective-migrate/corpus/viewer/5.3.0/map_carto_tiles.json new file mode 100644 index 0000000000..09bfebd8d7 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.3.0/map_carto_tiles.json @@ -0,0 +1,35 @@ +{ + "kind": "viewer", + "source": "hand-written from 23b5f8f292:packages/viewer-charts/src/ts/map/tile-sources.json", + "input": { + "version": "5.3.0", + "plugin": "Map Scatter", + "plugin_config": { + "map_tile_provider": "carto-dark-matter" + }, + "columns": [ + "Lon", + "Lat" + ] + }, + "output": { + "version": "CURRENT", + "plugin": "Map Scatter", + "plugin_config": {}, + "columns": [ + "Lon", + "Lat" + ] + }, + "notices": [ + { + "kind": "Dropped", + "step": "maps.carto_tiles", + "pointer": "/plugin_config/map_tile_provider", + "value": "carto-dark-matter" + } + ], + "steps": [ + "maps.carto_tiles" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.4.0/datagrid_formats.json b/rust/perspective-migrate/corpus/viewer/5.4.0/datagrid_formats.json new file mode 100644 index 0000000000..9e1e385932 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.4.0/datagrid_formats.json @@ -0,0 +1,136 @@ +{ + "kind": "viewer", + "source": "hand-written from v5.4.0:packages/viewer-datagrid/src/ts/types.ts", + "input": { + "version": "5.4.0", + "plugin": "datagrid", + "plugin_config": { + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Order ID": { + "format": "bold", + "string_color_mode": "none", + "color": "#111111" + }, + "Customer ID": { + "format": "italics" + }, + "Product ID": { + "format": "image" + }, + "Sales": { + "number_format": { + "style": "unit", + "unit": "gallob" + } + }, + "Order Date": { + "date_format": { + "format": "custom", + "year": "2-digit", + "month": "disabled" + } + }, + "Ship Date": { + "date_format": { + "date_style": "short", + "year": "numeric" + } + } + }, + "columns": [ + "Order ID", + "Customer ID", + "Product ID", + "Sales", + "Order Date", + "Ship Date" + ], + "group_by": [], + "split_by": [] + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": { + "Order ID": { + "bold": true + }, + "Customer ID": { + "italic": true + }, + "Product ID": {}, + "Sales": { + "number_format": { + "style": "unit", + "unit": "gallon" + } + }, + "Order Date": { + "date_format": { + "year": "2-digit", + "day": "numeric", + "hour": "numeric", + "minute": "numeric", + "second": "numeric", + "hour12": true + } + }, + "Ship Date": { + "date_format": { + "dateStyle": "short", + "timeStyle": "medium" + } + } + }, + "columns": [ + "Order ID", + "Customer ID", + "Product ID", + "Sales", + "Order Date", + "Ship Date" + ], + "group_by": [], + "split_by": [] + }, + "notices": [ + { + "kind": "Dropped", + "step": "datagrid.mode_renames", + "pointer": "/columns_config/Order ID/color", + "value": "#111111" + }, + { + "kind": "Dropped", + "step": "datagrid.format_flags", + "pointer": "/columns_config/Product ID/format", + "value": "image" + }, + { + "kind": "Dropped", + "step": "style.date_format", + "pointer": "/columns_config/Ship Date/date_format/date_style", + "value": "short" + }, + { + "kind": "Dropped", + "step": "style.date_format", + "pointer": "/columns_config/Ship Date/date_format/year", + "value": "numeric" + } + ], + "steps": [ + "datagrid.mode_renames", + "datagrid.format_flags", + "viewer.plugin_names", + "style.date_format", + "style.unit_gallon" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.6.0/inline_derived_table.json b/rust/perspective-migrate/corpus/viewer/5.6.0/inline_derived_table.json new file mode 100644 index 0000000000..12897bf975 --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.6.0/inline_derived_table.json @@ -0,0 +1,53 @@ +{ + "kind": "viewer", + "source": "hand-written from v5.6.0:rust/perspective-viewer/src/rust/config/workspace_config.rs TableRef", + "input": { + "version": "5.6.0", + "plugin": "Datagrid", + "table": { + "table": { + "table": "superstore", + "group_by": [ + "Region" + ], + "foo": 1 + }, + "expressions": [ + "1+1" + ], + "columns": [ + "Region" + ] + } + }, + "output": { + "version": "CURRENT", + "plugin": "Datagrid", + "table": { + "table": { + "table": "superstore", + "group_by": [ + "Region" + ] + }, + "expressions": { + "1+1": "1+1" + }, + "columns": [ + "Region" + ] + } + }, + "notices": [ + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/table/table/foo", + "value": 1 + } + ], + "steps": [ + "view.expressions_map", + "viewer.unknown_keys" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.6.1/date_format_presets.json b/rust/perspective-migrate/corpus/viewer/5.6.1/date_format_presets.json new file mode 100644 index 0000000000..7a93e2092e --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.6.1/date_format_presets.json @@ -0,0 +1,72 @@ +{ + "kind": "viewer", + "source": "rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts@f423f05698", + "input": { + "version": "5.6.1", + "plugin": "Debug", + "columns_config": { + "A": { + "date_format": { + "format": "custom", + "timeZone": "UTC", + "second": "disabled", + "minute": "disabled", + "hour": "disabled" + } + }, + "B": { + "date_format": { + "timeStyle": "disabled" + } + }, + "C": { + "date_format": { + "dateStyle": "disabled", + "timeStyle": "disabled" + } + }, + "D": { + "date_format": { + "dateStyle": "long" + } + } + } + }, + "output": { + "version": "CURRENT", + "plugin": "Debug", + "columns_config": { + "A": { + "date_format": { + "timeZone": "UTC", + "year": "2-digit", + "month": "numeric", + "day": "numeric", + "hour12": true + } + }, + "B": { + "date_format": { + "dateStyle": "short" + } + }, + "C": { + "date_format": { + "year": "numeric", + "month": "numeric", + "day": "numeric" + } + }, + "D": { + "date_format": { + "dateStyle": "long", + "timeStyle": "medium" + } + } + } + }, + "notices": [], + "steps": [ + "style.date_format" + ] +} diff --git a/rust/perspective-migrate/corpus/viewer/5.7.0/date_format_current.json b/rust/perspective-migrate/corpus/viewer/5.7.0/date_format_current.json new file mode 100644 index 0000000000..d9cdbe306a --- /dev/null +++ b/rust/perspective-migrate/corpus/viewer/5.7.0/date_format_current.json @@ -0,0 +1,28 @@ +{ + "kind": "viewer", + "source": "rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts@f423f05698", + "input": { + "version": "5.7.0", + "plugin": "Debug", + "columns_config": { + "D": { + "date_format": { + "dateStyle": "long" + } + } + } + }, + "output": { + "version": "CURRENT", + "plugin": "Debug", + "columns_config": { + "D": { + "date_format": { + "dateStyle": "long" + } + } + } + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/workspace/2.0.0/empty.json b/rust/perspective-migrate/corpus/workspace/2.0.0/empty.json new file mode 100644 index 0000000000..fb0ce8b5ab --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/2.0.0/empty.json @@ -0,0 +1,10 @@ +{ + "kind": "workspace", + "source": "hand-written", + "input": {}, + "output": { + "version": "CURRENT" + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/workspace/2.0.0/superstore.json b/rust/perspective-migrate/corpus/workspace/2.0.0/superstore.json new file mode 100644 index 0000000000..704ed9e21e --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/2.0.0/superstore.json @@ -0,0 +1,253 @@ +{ + "kind": "workspace", + "from": "2.0.0", + "source": "examples/blocks/src/superstore/layout.json@v2.0.0", + "input": { + "sizes": [ + 1 + ], + "detail": { + "main": { + "type": "split-area", + "orientation": "horizontal", + "children": [ + { + "type": "tab-area", + "widgets": [ + "PERSPECTIVE_GENERATED_ID_0", + "PERSPECTIVE_GENERATED_ID_1" + ], + "currentIndex": 0 + }, + { + "type": "tab-area", + "widgets": [ + "PERSPECTIVE_GENERATED_ID_2" + ], + "currentIndex": 0 + } + ], + "sizes": [ + 0.5, + 0.5 + ] + } + }, + "mode": "globalFilters", + "viewers": { + "PERSPECTIVE_GENERATED_ID_0": { + "plugin": "datagrid", + "group_by": [ + "Region", + "State" + ], + "split_by": [ + "Category", + "Sub-Category" + ], + "columns": [ + "Sales", + "Profit" + ], + "master": false, + "name": "Sales Report", + "table": "superstore", + "linked": false + }, + "PERSPECTIVE_GENERATED_ID_1": { + "plugin": "datagrid", + "columns": [ + "Row ID", + "Order ID", + "Ship Mode", + "Customer ID", + "Customer Name", + "Segment", + "Country", + "City", + "State", + "Postal Code", + "Region", + "Product ID", + "Category", + "Sub-Category", + "Product Name", + "Sales", + "Quantity", + "Discount", + "Profit", + "Order Date", + "Ship Date" + ], + "master": false, + "name": "superstore.csv", + "table": "superstore", + "linked": false + }, + "PERSPECTIVE_GENERATED_ID_2": { + "plugin": "Treemap", + "group_by": [ + "State" + ], + "columns": [ + "Sales", + "Profit", + null + ], + "sort": [ + [ + "Profit", + "desc" + ] + ], + "plugin_config": { + "realValues": [ + "Sales", + "Profit", + null + ], + "treemaps": { + "": { + "treemapLevel": 0, + "treemapRoute": [ + "" + ] + } + } + }, + "master": false, + "name": "Sales Report (by State)", + "table": "superstore", + "linked": false + } + } + }, + "output": { + "version": "CURRENT", + "layout": { + "type": "split-layout", + "orientation": "horizontal", + "children": [ + { + "type": "tab-layout", + "tabs": [ + "PERSPECTIVE_GENERATED_ID_0", + "PERSPECTIVE_GENERATED_ID_1" + ], + "selected": 0 + }, + { + "type": "tab-layout", + "tabs": [ + "PERSPECTIVE_GENERATED_ID_2" + ], + "selected": 0 + } + ], + "sizes": [ + 0.5, + 0.5 + ] + }, + "panels": { + "PERSPECTIVE_GENERATED_ID_0": { + "plugin": "Datagrid", + "group_by": [ + "Region", + "State" + ], + "split_by": [ + "Category", + "Sub-Category" + ], + "columns": [ + "Sales", + "Profit" + ], + "table": "superstore", + "title": "Sales Report" + }, + "PERSPECTIVE_GENERATED_ID_1": { + "plugin": "Datagrid", + "columns": [ + "Row ID", + "Order ID", + "Ship Mode", + "Customer ID", + "Customer Name", + "Segment", + "Country", + "City", + "State", + "Postal Code", + "Region", + "Product ID", + "Category", + "Sub-Category", + "Product Name", + "Sales", + "Quantity", + "Discount", + "Profit", + "Order Date", + "Ship Date" + ], + "table": "superstore", + "title": "superstore.csv" + }, + "PERSPECTIVE_GENERATED_ID_2": { + "plugin": "Treemap", + "group_by": [ + "State" + ], + "columns": [ + "Sales", + "Profit", + null + ], + "sort": [ + [ + "Profit", + "desc" + ] + ], + "plugin_config": {}, + "table": "superstore", + "title": "Sales Report (by State)" + } + } + }, + "notices": [ + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/viewers/PERSPECTIVE_GENERATED_ID_2/plugin_config/realValues", + "value": [ + "Sales", + "Profit", + null + ] + }, + { + "kind": "Dropped", + "step": "charts.d3fc_removed", + "pointer": "/viewers/PERSPECTIVE_GENERATED_ID_2/plugin_config/treemaps", + "value": { + "": { + "treemapLevel": 0, + "treemapRoute": [ + "" + ] + } + } + } + ], + "steps": [ + "ws.name_title", + "ws.drop_linked", + "ws.drop_master_flag", + "charts.d3fc_removed", + "ws.regular_layout", + "viewer.plugin_names" + ] +} diff --git a/rust/perspective-migrate/corpus/workspace/3.1.2/master_detail.json b/rust/perspective-migrate/corpus/workspace/3.1.2/master_detail.json new file mode 100644 index 0000000000..5e9445dffc --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/3.1.2/master_detail.json @@ -0,0 +1,196 @@ +{ + "kind": "workspace", + "from": "3.1.2", + "source": "hand-written from v3.1.2:packages/perspective-workspace/src/js/workspace/workspace.js", + "input": { + "sizes": [ + 1, + 3 + ], + "master": { + "widgets": [ + { + "slot": "M1", + "plugin": "Datagrid", + "table": "t", + "group_by": [ + "Region" + ], + "selectable": "", + "master": true + }, + "M2" + ], + "sizes": [ + 0.5, + 0.5 + ] + }, + "detail": { + "main": { + "type": "tab-area", + "widgets": [ + "D1", + "GONE" + ], + "currentIndex": 0 + } + }, + "mode": "linked", + "viewers": { + "M2": { + "plugin": "Datagrid", + "table": "t", + "group_by": [ + "State" + ], + "master": true, + "linked": true + }, + "D1": { + "plugin": "Datagrid", + "table": "t", + "filter": [ + [ + "Region", + "==", + "East" + ] + ], + "settings": true, + "editable": "true" + }, + "O1": { + "plugin": "Datagrid", + "table": "t" + } + } + }, + "output": { + "version": "CURRENT", + "layout": { + "type": "split-layout", + "orientation": "horizontal", + "children": [ + { + "type": "split-layout", + "orientation": "vertical", + "children": [ + { + "type": "tab-layout", + "tabs": [ + "M1" + ] + }, + { + "type": "tab-layout", + "tabs": [ + "M2" + ] + } + ], + "sizes": [ + 0.5, + 0.5 + ] + }, + { + "type": "tab-layout", + "tabs": [ + "D1" + ], + "selected": 0 + } + ], + "sizes": [ + 0.25, + 0.75 + ] + }, + "panels": { + "M2": { + "plugin": "Datagrid", + "table": "t", + "group_by": [ + "State" + ] + }, + "D1": { + "plugin": "Datagrid", + "table": "t", + "filter": [ + [ + "Region", + "==", + "East" + ] + ] + }, + "M1": { + "plugin": "Datagrid", + "table": "t", + "group_by": [ + "Region" + ] + } + }, + "masters": [ + "M1", + "M2" + ], + "active": "D1" + }, + "notices": [ + { + "kind": "Dropped", + "step": "ws.drop_linked", + "pointer": "/mode", + "value": "linked" + }, + { + "kind": "Dropped", + "step": "ws.drop_linked", + "pointer": "/viewers/M2/linked", + "value": true + }, + { + "kind": "Dropped", + "step": "ws.drop_linked", + "pointer": "/viewers/D1/editable", + "value": "true" + }, + { + "kind": "Dropped", + "step": "ws.drop_linked", + "pointer": "/viewers/M1/selectable", + "value": "" + }, + { + "kind": "Approximated", + "step": "ws.regular_layout", + "pointer": "/layout", + "reason": "the master panels are placed on the left" + }, + { + "kind": "Approximated", + "step": "ws.regular_layout", + "pointer": "/masters", + "reason": "master selections remain in each detail panel's saved `filter`" + }, + { + "kind": "Dropped", + "step": "ws.regular_layout", + "pointer": "/viewers/O1", + "value": { + "plugin": "Datagrid", + "table": "t" + } + } + ], + "steps": [ + "ws.inline_widgets", + "ws.drop_linked", + "ws.drop_master_flag", + "ws.regular_layout" + ] +} diff --git a/rust/perspective-migrate/corpus/workspace/4.5.2/dataset.json b/rust/perspective-migrate/corpus/workspace/4.5.2/dataset.json new file mode 100644 index 0000000000..2e7674eb56 --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/4.5.2/dataset.json @@ -0,0 +1,265 @@ +{ + "kind": "workspace", + "from": "4.5.2", + "source": "examples/blocks/src/dataset/layout.json@v4.5.2", + "input": { + "sizes": [ + 1 + ], + "detail": { + "main": { + "type": "split-area", + "orientation": "vertical", + "children": [ + { + "type": "tab-area", + "widgets": [ + "PERSPECTIVE_GENERATED_ID_0" + ], + "currentIndex": 0 + }, + { + "type": "split-area", + "orientation": "horizontal", + "children": [ + { + "type": "tab-area", + "widgets": [ + "PERSPECTIVE_GENERATED_ID_1" + ], + "currentIndex": 0 + }, + { + "type": "tab-area", + "widgets": [ + "PERSPECTIVE_GENERATED_ID_2" + ], + "currentIndex": 0 + } + ], + "sizes": [ + 0.5, + 0.5 + ] + } + ], + "sizes": [ + 0.5, + 0.5 + ] + } + }, + "viewers": { + "PERSPECTIVE_GENERATED_ID_0": { + "version": "4.5.2", + "plugin": "Datagrid", + "plugin_config": { + "columns": {}, + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": {}, + "title": "Generated Data", + "group_by": [ + "String 0", + "String 1" + ], + "split_by": [ + "String 2" + ], + "sort": [], + "filter": [], + "expressions": {}, + "columns": [ + "Float 2", + "Float 3", + "Float 4", + "Integer 0", + "Integer 1", + "Datetime 4", + "Boolean 0" + ], + "aggregates": { + "Boolean 0": "dominant", + "Datetime 4": "dominant" + }, + "table": "superstore", + "settings": false + }, + "PERSPECTIVE_GENERATED_ID_1": { + "version": "4.5.2", + "plugin": "Y Area", + "plugin_config": {}, + "columns_config": {}, + "table": "stats", + "title": "Server Memory", + "group_by": [ + "timestamp" + ], + "split_by": [], + "sort": [], + "filter": [], + "expressions": { + "Free (mb)": "\"heap_size\" / 1024 / 1024", + "Heap (mb)": "\"used_size\" / 1024 / 1024" + }, + "columns": [ + "Free (mb)", + "Heap (mb)" + ], + "aggregates": {} + }, + "PERSPECTIVE_GENERATED_ID_2": { + "version": "4.5.2", + "plugin": "Y Line", + "plugin_config": {}, + "columns_config": {}, + "title": "Server CPU", + "group_by": [ + "timestamp" + ], + "split_by": [], + "sort": [], + "filter": [], + "expressions": { + "CPU (%)": "\"cpu_time\" / \"cpu_time_epoch\"" + }, + "columns": [ + "CPU (%)" + ], + "aggregates": {}, + "table": "stats", + "settings": false + } + } + }, + "output": { + "version": "CURRENT", + "layout": { + "type": "split-layout", + "orientation": "vertical", + "children": [ + { + "type": "tab-layout", + "tabs": [ + "PERSPECTIVE_GENERATED_ID_0" + ], + "selected": 0 + }, + { + "type": "split-layout", + "orientation": "horizontal", + "children": [ + { + "type": "tab-layout", + "tabs": [ + "PERSPECTIVE_GENERATED_ID_1" + ], + "selected": 0 + }, + { + "type": "tab-layout", + "tabs": [ + "PERSPECTIVE_GENERATED_ID_2" + ], + "selected": 0 + } + ], + "sizes": [ + 0.5, + 0.5 + ] + } + ], + "sizes": [ + 0.5, + 0.5 + ] + }, + "panels": { + "PERSPECTIVE_GENERATED_ID_0": { + "version": "CURRENT", + "plugin": "Datagrid", + "plugin_config": { + "edit_mode": "READ_ONLY", + "scroll_lock": false + }, + "columns_config": {}, + "title": "Generated Data", + "group_by": [ + "String 0", + "String 1" + ], + "split_by": [ + "String 2" + ], + "sort": [], + "filter": [], + "expressions": {}, + "columns": [ + "Float 2", + "Float 3", + "Float 4", + "Integer 0", + "Integer 1", + "Datetime 4", + "Boolean 0" + ], + "aggregates": { + "Boolean 0": "dominant", + "Datetime 4": "dominant" + }, + "table": "superstore" + }, + "PERSPECTIVE_GENERATED_ID_1": { + "version": "CURRENT", + "plugin": "Y Area", + "plugin_config": {}, + "columns_config": {}, + "table": "stats", + "title": "Server Memory", + "group_by": [ + "timestamp" + ], + "split_by": [], + "sort": [], + "filter": [], + "expressions": { + "Free (mb)": "\"heap_size\" / 1024 / 1024", + "Heap (mb)": "\"used_size\" / 1024 / 1024" + }, + "columns": [ + "Free (mb)", + "Heap (mb)" + ], + "aggregates": {} + }, + "PERSPECTIVE_GENERATED_ID_2": { + "version": "CURRENT", + "plugin": "Y Line", + "plugin_config": {}, + "columns_config": {}, + "title": "Server CPU", + "group_by": [ + "timestamp" + ], + "split_by": [], + "sort": [], + "filter": [], + "expressions": { + "CPU (%)": "\"cpu_time\" / \"cpu_time_epoch\"" + }, + "columns": [ + "CPU (%)" + ], + "aggregates": {}, + "table": "stats" + } + } + }, + "notices": [], + "steps": [ + "ws.regular_layout", + "datagrid.size_overrides" + ] +} diff --git a/rust/perspective-migrate/corpus/workspace/4.5.2/panel_versions.json b/rust/perspective-migrate/corpus/workspace/4.5.2/panel_versions.json new file mode 100644 index 0000000000..6e72466e41 --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/4.5.2/panel_versions.json @@ -0,0 +1,85 @@ +{ + "kind": "workspace", + "source": "hand-written", + "input": { + "detail": { + "main": { + "type": "tab-area", + "widgets": [ + "P", + "Q" + ], + "currentIndex": 1 + } + }, + "viewers": { + "P": { + "version": "4.5.2", + "plugin": "Map Scatter", + "table": "t", + "columns": [ + "Lon", + "Lat", + "Color", + "Size", + "Label", + "Tooltip" + ] + }, + "Q": { + "plugin": "Map Scatter", + "table": "t", + "columns": [ + "Lon", + "Lat", + "Color", + "Size", + "Tooltip" + ] + } + } + }, + "output": { + "layout": { + "type": "tab-layout", + "tabs": [ + "P", + "Q" + ], + "selected": 1 + }, + "panels": { + "P": { + "version": "CURRENT", + "plugin": "Map Scatter", + "table": "t", + "columns": [ + "Lon", + "Lat", + "Color", + "Size", + "Label", + "Tooltip" + ] + }, + "Q": { + "plugin": "Map Scatter", + "table": "t", + "columns": [ + "Lon", + "Lat", + "Color", + "Size", + null, + "Tooltip" + ] + } + }, + "version": "CURRENT" + }, + "notices": [], + "steps": [ + "maps.label_slot", + "ws.regular_layout" + ] +} diff --git a/rust/perspective-migrate/corpus/workspace/5.1.0/null_table.json b/rust/perspective-migrate/corpus/workspace/5.1.0/null_table.json new file mode 100644 index 0000000000..3f422151e1 --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/5.1.0/null_table.json @@ -0,0 +1,27 @@ +{ + "kind": "workspace", + "source": "hand-written from v5.1.0:rust/perspective-viewer/src/rust/config/workspace_config.rs", + "input": { + "version": "5.1.0", + "layout": { + "type": "tab-layout", + "tabs": [ + "A" + ] + }, + "panels": { + "A": { + "version": "5.1.0", + "plugin": "Datagrid", + "table": null + } + } + }, + "error": { + "kind": "MissingTable", + "step": "ws.panel_table", + "panel": "A" + }, + "notices": [], + "steps": [] +} diff --git a/rust/perspective-migrate/corpus/workspace/5.6.1/ignored_keys.json b/rust/perspective-migrate/corpus/workspace/5.6.1/ignored_keys.json new file mode 100644 index 0000000000..38c1345330 --- /dev/null +++ b/rust/perspective-migrate/corpus/workspace/5.6.1/ignored_keys.json @@ -0,0 +1,83 @@ +{ + "kind": "workspace", + "source": "hand-written", + "input": { + "version": "5.6.1", + "layout": { + "type": "tab-layout", + "tabs": [ + "P" + ] + }, + "panels": { + "P": { + "version": "5.6.1", + "plugin": "Datagrid", + "table": "t", + "settings": true + } + }, + "derived_tables": { + "d": { + "table": "t", + "group_by": [ + "x" + ], + "bogus": true + } + }, + "sizes": [ + 1 + ] + }, + "output": { + "version": "CURRENT", + "layout": { + "type": "tab-layout", + "tabs": [ + "P" + ] + }, + "panels": { + "P": { + "version": "CURRENT", + "plugin": "Datagrid", + "table": "t" + } + }, + "derived_tables": { + "d": { + "table": "t", + "group_by": [ + "x" + ] + } + } + }, + "notices": [ + { + "kind": "Dropped", + "step": "viewer.unknown_keys", + "pointer": "/panels/P/settings", + "value": true + }, + { + "kind": "Dropped", + "step": "ws.unknown_keys", + "pointer": "/sizes", + "value": [ + 1 + ] + }, + { + "kind": "Dropped", + "step": "ws.unknown_keys", + "pointer": "/derived_tables/d/bogus", + "value": true + } + ], + "steps": [ + "viewer.unknown_keys", + "ws.unknown_keys" + ] +} diff --git a/rust/perspective-migrate/src/column_style/mod.rs b/rust/perspective-migrate/src/column_style/mod.rs new file mode 100644 index 0000000000..c9e1cc8b14 --- /dev/null +++ b/rust/perspective-migrate/src/column_style/mod.rs @@ -0,0 +1,115 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::json::column_styles; +use crate::step::{Step, StepCx}; +use crate::version::Version; + +pub(crate) const STEPS: &[Step] = &[ + Step { + at: Version::new(5, 7, 0), + id: "style.date_format", + apply: date_format, + }, + Step { + at: Version::new(5, 7, 0), + id: "style.unit_gallon", + apply: unit_gallon, + }, +]; + +const PRESET_KEYS: &[&str] = &["timeZone", "dateStyle", "timeStyle"]; + +const PART_FALLBACKS: &[(&str, &str)] = &[ + ("year", "2-digit"), + ("month", "numeric"), + ("day", "numeric"), + ("hour", "numeric"), + ("minute", "numeric"), + ("second", "numeric"), +]; + +const SHAPE_KEYS: &[&str] = &[ + "dateStyle", + "timeStyle", + "weekday", + "year", + "month", + "day", + "hour", + "minute", + "second", + "fractionalSecondDigits", +]; + +fn date_format(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + for (column, style) in column_styles(token) { + let Some(Value::Object(format)) = style.get_mut("date_format") else { + continue; + }; + + if format.shift_remove("format").is_some() { + for (key, fallback) in PART_FALLBACKS { + format.entry(*key).or_insert_with(|| Value::from(*fallback)); + } + + format.entry("hour12").or_insert(Value::Bool(true)); + if format.get("fractionalSecondDigits").and_then(Value::as_u64) == Some(0) { + format.shift_remove("fractionalSecondDigits"); + } + } else { + let ignored = format + .keys() + .filter(|key| !PRESET_KEYS.contains(&key.as_str())) + .cloned() + .collect::>(); + + for key in ignored { + if let Some(value) = format.shift_remove(&key) { + cx.dropped(&["columns_config", column, "date_format", &key], value); + } + } + + format + .entry("dateStyle") + .or_insert_with(|| Value::from("short")); + format + .entry("timeStyle") + .or_insert_with(|| Value::from("medium")); + } + + format.retain(|_, value| value.as_str() != Some("disabled")); + if !SHAPE_KEYS.iter().any(|key| format.contains_key(*key)) { + for key in ["year", "month", "day"] { + format.insert(key.to_owned(), Value::from("numeric")); + } + } + } + + Ok(()) +} + +fn unit_gallon(token: &mut Map, _cx: &mut StepCx) -> Result<(), MigrateError> { + for (_, style) in column_styles(token) { + if let Some(Value::Object(format)) = style.get_mut("number_format") + && let Some(unit) = format.get_mut("unit") + && unit.as_str() == Some("gallob") + { + *unit = Value::from("gallon"); + } + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/css.rs b/rust/perspective-migrate/src/css.rs new file mode 100644 index 0000000000..3646cc6101 --- /dev/null +++ b/rust/perspective-migrate/src/css.rs @@ -0,0 +1,244 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +pub(crate) struct Color { + pub hex: String, + pub had_alpha: bool, +} + +pub(crate) fn normalize(src: &str) -> Option { + let src = src.trim().to_ascii_lowercase(); + if let Some(hex) = src.strip_prefix('#') { + return parse_hex(hex); + } + + if let Some((_, hex)) = NAMED.iter().find(|(name, _)| *name == src) { + return Some(Color { + hex: format!("#{hex}"), + had_alpha: false, + }); + } + + let body = src + .strip_prefix("rgba(") + .or_else(|| src.strip_prefix("rgb("))? + .strip_suffix(')')?; + + let (channels, alpha) = match body.split_once('/') { + Some((channels, alpha)) => (channels, Some(alpha)), + None => (body, None), + }; + + let tokens = channels + .split(|c: char| c == ',' || c.is_whitespace()) + .filter(|x| !x.is_empty()) + .collect::>(); + + let (rgb, alpha) = match (tokens.as_slice(), alpha) { + ([r, g, b], alpha) => ([*r, *g, *b], alpha.map(str::trim)), + ([r, g, b, a], None) => ([*r, *g, *b], Some(*a)), + _ => return None, + }; + + let [r, g, b] = [channel(rgb[0])?, channel(rgb[1])?, channel(rgb[2])?]; + Some(Color { + hex: format!("#{r:02x}{g:02x}{b:02x}"), + had_alpha: alpha.is_some_and(|a| !is_opaque(a)), + }) +} + +fn parse_hex(hex: &str) -> Option { + let digit = |i: usize, n: usize| u8::from_str_radix(hex.get(i..i + n)?, 16).ok(); + let (r, g, b) = match hex.len() { + 3 | 4 => (digit(0, 1)? * 17, digit(1, 1)? * 17, digit(2, 1)? * 17), + 6 | 8 => (digit(0, 2)?, digit(2, 2)?, digit(4, 2)?), + _ => return None, + }; + + let had_alpha = match hex.len() { + 4 => digit(3, 1)? != 15, + 8 => digit(6, 2)? != 255, + _ => false, + }; + + Some(Color { + hex: format!("#{r:02x}{g:02x}{b:02x}"), + had_alpha, + }) +} + +fn channel(token: &str) -> Option { + let value = match token.strip_suffix('%') { + Some(pct) => pct.parse::().ok()? / 100.0 * 255.0, + None => token.parse::().ok()?, + }; + + Some(value.round().clamp(0.0, 255.0) as u8) +} + +fn is_opaque(alpha: &str) -> bool { + match alpha.strip_suffix('%') { + Some(pct) => pct.parse::().is_ok_and(|x| x >= 100.0), + None => alpha.parse::().is_ok_and(|x| x >= 1.0), + } +} + +const NAMED: &[(&str, &str)] = &[ + ("aliceblue", "f0f8ff"), + ("antiquewhite", "faebd7"), + ("aqua", "00ffff"), + ("aquamarine", "7fffd4"), + ("azure", "f0ffff"), + ("beige", "f5f5dc"), + ("bisque", "ffe4c4"), + ("black", "000000"), + ("blanchedalmond", "ffebcd"), + ("blue", "0000ff"), + ("blueviolet", "8a2be2"), + ("brown", "a52a2a"), + ("burlywood", "deb887"), + ("cadetblue", "5f9ea0"), + ("chartreuse", "7fff00"), + ("chocolate", "d2691e"), + ("coral", "ff7f50"), + ("cornflowerblue", "6495ed"), + ("cornsilk", "fff8dc"), + ("crimson", "dc143c"), + ("cyan", "00ffff"), + ("darkblue", "00008b"), + ("darkcyan", "008b8b"), + ("darkgoldenrod", "b8860b"), + ("darkgray", "a9a9a9"), + ("darkgreen", "006400"), + ("darkgrey", "a9a9a9"), + ("darkkhaki", "bdb76b"), + ("darkmagenta", "8b008b"), + ("darkolivegreen", "556b2f"), + ("darkorange", "ff8c00"), + ("darkorchid", "9932cc"), + ("darkred", "8b0000"), + ("darksalmon", "e9967a"), + ("darkseagreen", "8fbc8f"), + ("darkslateblue", "483d8b"), + ("darkslategray", "2f4f4f"), + ("darkslategrey", "2f4f4f"), + ("darkturquoise", "00ced1"), + ("darkviolet", "9400d3"), + ("deeppink", "ff1493"), + ("deepskyblue", "00bfff"), + ("dimgray", "696969"), + ("dimgrey", "696969"), + ("dodgerblue", "1e90ff"), + ("firebrick", "b22222"), + ("floralwhite", "fffaf0"), + ("forestgreen", "228b22"), + ("fuchsia", "ff00ff"), + ("gainsboro", "dcdcdc"), + ("ghostwhite", "f8f8ff"), + ("gold", "ffd700"), + ("goldenrod", "daa520"), + ("gray", "808080"), + ("green", "008000"), + ("greenyellow", "adff2f"), + ("grey", "808080"), + ("honeydew", "f0fff0"), + ("hotpink", "ff69b4"), + ("indianred", "cd5c5c"), + ("indigo", "4b0082"), + ("ivory", "fffff0"), + ("khaki", "f0e68c"), + ("lavender", "e6e6fa"), + ("lavenderblush", "fff0f5"), + ("lawngreen", "7cfc00"), + ("lemonchiffon", "fffacd"), + ("lightblue", "add8e6"), + ("lightcoral", "f08080"), + ("lightcyan", "e0ffff"), + ("lightgoldenrodyellow", "fafad2"), + ("lightgray", "d3d3d3"), + ("lightgreen", "90ee90"), + ("lightgrey", "d3d3d3"), + ("lightpink", "ffb6c1"), + ("lightsalmon", "ffa07a"), + ("lightseagreen", "20b2aa"), + ("lightskyblue", "87cefa"), + ("lightslategray", "778899"), + ("lightslategrey", "778899"), + ("lightsteelblue", "b0c4de"), + ("lightyellow", "ffffe0"), + ("lime", "00ff00"), + ("limegreen", "32cd32"), + ("linen", "faf0e6"), + ("magenta", "ff00ff"), + ("maroon", "800000"), + ("mediumaquamarine", "66cdaa"), + ("mediumblue", "0000cd"), + ("mediumorchid", "ba55d3"), + ("mediumpurple", "9370db"), + ("mediumseagreen", "3cb371"), + ("mediumslateblue", "7b68ee"), + ("mediumspringgreen", "00fa9a"), + ("mediumturquoise", "48d1cc"), + ("mediumvioletred", "c71585"), + ("midnightblue", "191970"), + ("mintcream", "f5fffa"), + ("mistyrose", "ffe4e1"), + ("moccasin", "ffe4b5"), + ("navajowhite", "ffdead"), + ("navy", "000080"), + ("oldlace", "fdf5e6"), + ("olive", "808000"), + ("olivedrab", "6b8e23"), + ("orange", "ffa500"), + ("orangered", "ff4500"), + ("orchid", "da70d6"), + ("palegoldenrod", "eee8aa"), + ("palegreen", "98fb98"), + ("paleturquoise", "afeeee"), + ("palevioletred", "db7093"), + ("papayawhip", "ffefd5"), + ("peachpuff", "ffdab9"), + ("peru", "cd853f"), + ("pink", "ffc0cb"), + ("plum", "dda0dd"), + ("powderblue", "b0e0e6"), + ("purple", "800080"), + ("rebeccapurple", "663399"), + ("red", "ff0000"), + ("rosybrown", "bc8f8f"), + ("royalblue", "4169e1"), + ("saddlebrown", "8b4513"), + ("salmon", "fa8072"), + ("sandybrown", "f4a460"), + ("seagreen", "2e8b57"), + ("seashell", "fff5ee"), + ("sienna", "a0522d"), + ("silver", "c0c0c0"), + ("skyblue", "87ceeb"), + ("slateblue", "6a5acd"), + ("slategray", "708090"), + ("slategrey", "708090"), + ("snow", "fffafa"), + ("springgreen", "00ff7f"), + ("steelblue", "4682b4"), + ("tan", "d2b48c"), + ("teal", "008080"), + ("thistle", "d8bfd8"), + ("tomato", "ff6347"), + ("turquoise", "40e0d0"), + ("violet", "ee82ee"), + ("wheat", "f5deb3"), + ("white", "ffffff"), + ("whitesmoke", "f5f5f5"), + ("yellow", "ffff00"), + ("yellowgreen", "9acd32"), +]; diff --git a/rust/perspective-migrate/src/error.rs b/rust/perspective-migrate/src/error.rs new file mode 100644 index 0000000000..c462565431 --- /dev/null +++ b/rust/perspective-migrate/src/error.rs @@ -0,0 +1,82 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::fmt; + +use serde::Serialize; + +/// Why a config could not be migrated to a valid current form. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "kind")] +pub enum MigrateError { + /// `version` is not `x.y.z`. + InvalidVersion { version: String }, + + /// `version` predates `2.0.0`. + Unsupported { version: String }, + + /// `version` is newer than this crate's schema. + FromFuture { version: String }, + + /// The config is not a JSON object. + NotAnObject, + + /// The value at `pointer` is not the shape its declared version wrote. + Malformed { + step: &'static str, + pointer: String, + expected: &'static str, + }, + + /// An alias and its canonical key are both present at `pointer`. + Conflict { + step: &'static str, + pointer: String, + keys: Vec, + }, + + /// The workspace panel `panel` has no `table` to bind. + MissingTable { step: &'static str, panel: String }, +} + +impl fmt::Display for MigrateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidVersion { version } => write!(f, "invalid version `{version}`"), + Self::Unsupported { version } => { + write!( + f, + "configs written before 2.0.0 are unsupported (`{version}`)" + ) + }, + Self::FromFuture { version } => { + write!(f, "`{version}` is newer than this migrator") + }, + Self::NotAnObject => write!(f, "expected a JSON object"), + Self::Malformed { + step, + pointer, + expected, + } => write!(f, "{step}: expected {expected} at `{pointer}`"), + Self::Conflict { + step, + pointer, + keys, + } => write!(f, "{step}: conflicting keys {keys:?} at `{pointer}`"), + Self::MissingTable { step, panel } => { + write!(f, "{step}: panel `{panel}` has no `table`") + }, + } + } +} + +impl std::error::Error for MigrateError {} diff --git a/rust/perspective-migrate/src/json.rs b/rust/perspective-migrate/src/json.rs new file mode 100644 index 0000000000..0083bab7f3 --- /dev/null +++ b/rust/perspective-migrate/src/json.rs @@ -0,0 +1,43 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +pub(crate) fn escape(segment: &str) -> String { + segment.replace('~', "~0").replace('/', "~1") +} + +pub(crate) fn object_mut<'a>( + map: &'a mut Map, + key: &str, +) -> Option<&'a mut Map> { + map.get_mut(key).and_then(Value::as_object_mut) +} + +pub(crate) fn column_styles( + token: &mut Map, +) -> impl Iterator)> { + object_mut(token, "columns_config") + .into_iter() + .flat_map(|config| config.iter_mut()) + .filter_map(|(name, style)| style.as_object_mut().map(|style| (name, style))) +} + +pub(crate) fn rename(map: &mut Map, from: &str, to: &str) -> Option { + let value = map.shift_remove(from)?; + if map.contains_key(to) { + Some(value) + } else { + map.insert(to.to_owned(), value); + None + } +} diff --git a/rust/perspective-migrate/src/lib.rs b/rust/perspective-migrate/src/lib.rs new file mode 100644 index 0000000000..b178f580b0 --- /dev/null +++ b/rust/perspective-migrate/src/lib.rs @@ -0,0 +1,270 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +//! Upgrades persisted Perspective configs — engine `ViewConfig`s, +//! `` tokens and workspaces — written by any release since +//! 2.0.0 to the current schema. +//! +//! ``` +//! use perspective_migrate::{Kind, Migrator}; +//! use serde_json::json; +//! +//! let migrated = Migrator::new() +//! .migrate(Kind::View, Some("2.6.0"), json!({ +//! "row_pivot": ["State"], +//! "expressions": ["//profit\n\"Sales\" - \"Cost\""] +//! })) +//! .unwrap(); +//! +//! assert_eq!(migrated.value, json!({ +//! "group_by": ["State"], +//! "expressions": {"profit": "\"Sales\" - \"Cost\""} +//! })); +//! ``` + +mod column_style; +mod css; +mod error; +mod json; +mod notice; +mod plugin; +mod plugins; +mod step; +mod version; +mod view; +mod viewer; +mod workspace; + +pub use error::MigrateError; +pub use notice::Notice; +pub use plugin::PluginMigration; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +pub use step::{Apply, Step, StepCx}; +use step::{Entry, Layer, Run, panels}; +pub use version::Version; + +/// The kind of document a config is. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + /// An engine `ViewConfig` or `ViewConfigUpdate`. + View, + + /// A `` `save()` / `restore()` token. + Viewer, + + /// A workspace `save()` / `restoreWorkspace()` token. + Workspace, +} + +/// A migrated config and what the migration could not carry over exactly. +#[derive(Clone, Debug, PartialEq)] +pub struct Migrated { + /// The config in the current schema. + pub value: Value, + + /// Information dropped or approximated along the way. + pub notices: Vec, + + /// The ids of the steps that changed the config, in the order they ran. + pub steps: Vec<&'static str>, +} + +/// Upgrades configs through every schema change since their source version. +pub struct Migrator { + plugins: Vec>, +} + +impl Default for Migrator { + fn default() -> Self { + Self::new() + } +} + +impl Migrator { + /// A migrator for the engine, the viewer and the first-party plugins. + pub fn new() -> Self { + Self { + plugins: plugins::first_party(), + } + } + + /// Add the migrations of a third-party plugin. + /// + /// # Panics + /// + /// When `plugin` claims a plugin name another migration already owns. + pub fn with_plugin(mut self, plugin: impl PluginMigration + 'static) -> Self { + let owned = self + .plugins + .iter() + .flat_map(|x| x.plugins().iter().copied()) + .collect::>(); + + if let Some(name) = plugin.plugins().iter().find(|x| owned.contains(x)) { + panic!("plugin `{name}` already has a migration"); + } + + self.plugins.push(Box::new(plugin)); + self + } + + /// Upgrade `value`, a `kind` document, to the current schema. + /// + /// The release that wrote each object is read from its own `version` + /// field, falling back to `from` and then to 2.0.0, so re-migrating an + /// output is a no-op. + /// + /// # Arguments + /// + /// - `kind` - Which document `value` is. + /// - `from` - The release that wrote `value`, as `x.y.z`, used only when + /// `value` has no `version` field. + /// - `value` - The config, as JSON. + pub fn migrate( + &self, + kind: Kind, + from: Option<&str>, + value: Value, + ) -> Result { + let mut value = value; + if !value.is_object() { + return Err(MigrateError::NotAnObject); + } + + let schema = self.schema_version(); + let from = match value.get("version").and_then(Value::as_str).or(from) { + Some(version) => check_version(version, schema)?, + None => Version::new(2, 0, 0), + }; + + let entries = self.entries(); + let known_plugins = self.plugin_names(); + let mut run = Run { + kind, + from, + schema, + known_plugins: &known_plugins, + notices: vec![], + steps: vec![], + }; + + run.apply(&entries, &mut value)?; + stamp(kind, &schema.to_string(), &mut value); + Ok(Migrated { + value, + notices: run.notices, + steps: run.steps, + }) + } + + /// The id of every step this migrator can run. + pub fn step_ids(&self) -> Vec<&'static str> { + self.entries().iter().map(|x| x.step.id).collect() + } + + /// The newest release any step migrates to. + pub fn newest_step_version(&self) -> Version { + self.entries() + .iter() + .map(|x| x.step.at) + .max() + .unwrap_or_else(Version::crate_version) + } + + /// The newest release whose configs this migrator accepts. + pub fn schema_version(&self) -> Version { + self.newest_step_version().max(Version::crate_version()) + } + + fn plugin_names(&self) -> Vec<&'static str> { + let mut names = plugins::REGISTRATION_ORDER.to_vec(); + for name in self.plugins.iter().flat_map(|x| x.plugins().iter()) { + if !names.contains(name) { + names.push(name); + } + } + + names + } + + fn entries(&self) -> Vec> { + let fixed = [ + (Layer::View, view::STEPS), + (Layer::Viewer, viewer::STEPS), + (Layer::ColumnStyle, column_style::STEPS), + (Layer::Workspace, workspace::STEPS), + ]; + + let mut entries = fixed + .into_iter() + .flat_map(|(layer, steps)| { + steps.iter().map(move |step| Entry { + layer, + plugins: &[], + step: *step, + }) + }) + .collect::>(); + + for plugin in &self.plugins { + entries.extend(plugin.steps().into_iter().map(|step| Entry { + layer: Layer::Plugin, + plugins: plugin.plugins(), + step, + })); + } + + entries.sort_by_key(|x| (x.step.at, x.layer)); + entries + } +} + +pub(crate) fn check_version(version: &str, schema: Version) -> Result { + let parsed = Version::parse(version).ok_or_else(|| MigrateError::InvalidVersion { + version: version.to_owned(), + })?; + + if parsed < Version::new(2, 0, 0) { + Err(MigrateError::Unsupported { + version: version.to_owned(), + }) + } else if parsed > schema { + Err(MigrateError::FromFuture { + version: version.to_owned(), + }) + } else { + Ok(parsed) + } +} + +fn stamp(kind: Kind, version: &str, value: &mut Value) { + let version = Value::from(version); + match kind { + Kind::View => {}, + Kind::Viewer => { + value["version"] = version; + }, + Kind::Workspace => { + for pointer in panels(value) { + if let Some(Value::Object(panel)) = value.pointer_mut(&pointer) + && panel.contains_key("version") + { + panel.insert("version".to_owned(), version.clone()); + } + } + + value["version"] = version; + }, + } +} diff --git a/rust/perspective-migrate/src/notice.rs b/rust/perspective-migrate/src/notice.rs new file mode 100644 index 0000000000..4c22d38a7c --- /dev/null +++ b/rust/perspective-migrate/src/notice.rs @@ -0,0 +1,33 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde::Serialize; +use serde_json::Value; + +/// Information a migration could not carry into the current schema exactly. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "kind")] +pub enum Notice { + /// `value` at `pointer` has no current representation and was removed. + Dropped { + step: &'static str, + pointer: String, + value: Value, + }, + + /// The value at `pointer` was rewritten to the nearest current form. + Approximated { + step: &'static str, + pointer: String, + reason: &'static str, + }, +} diff --git a/rust/perspective-migrate/src/plugin.rs b/rust/perspective-migrate/src/plugin.rs new file mode 100644 index 0000000000..2da37268df --- /dev/null +++ b/rust/perspective-migrate/src/plugin.rs @@ -0,0 +1,71 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use crate::step::Step; + +/// The migrations of a plugin's `plugin_config` and `columns_config`, whose +/// [`Step::at`] versions are the Perspective release that wrote a token rather +/// than releases of the plugin itself. +/// +/// # Example +/// +/// ``` +/// use perspective_migrate::{Kind, Migrator, PluginMigration, Step, StepCx, Version}; +/// use serde_json::{Map, Value, json}; +/// +/// struct Sparkline; +/// +/// fn rename_width( +/// token: &mut Map, +/// _cx: &mut StepCx, +/// ) -> Result<(), perspective_migrate::MigrateError> { +/// if let Some(Value::Object(config)) = token.get_mut("plugin_config") +/// && let Some(width) = config.remove("width") +/// { +/// config.insert("line_width".to_owned(), width); +/// } +/// +/// Ok(()) +/// } +/// +/// impl PluginMigration for Sparkline { +/// fn plugins(&self) -> &[&'static str] { +/// &["Sparkline"] +/// } +/// +/// fn steps(&self) -> Vec { +/// vec![Step { +/// at: Version::new(3, 0, 0), +/// id: "sparkline.line_width", +/// apply: rename_width, +/// }] +/// } +/// } +/// +/// let migrated = Migrator::new() +/// .with_plugin(Sparkline) +/// .migrate(Kind::Viewer, None, json!({ +/// "version": "2.10.0", +/// "plugin": "Sparkline", +/// "plugin_config": {"width": 2} +/// })) +/// .unwrap(); +/// +/// assert_eq!(migrated.value["plugin_config"], json!({"line_width": 2})); +/// ``` +pub trait PluginMigration { + /// The `plugin` names whose payloads this migration owns. + fn plugins(&self) -> &[&'static str]; + + /// The plugin-layer steps, in any order. + fn steps(&self) -> Vec; +} diff --git a/rust/perspective-migrate/src/plugins/charts.rs b/rust/perspective-migrate/src/plugins/charts.rs new file mode 100644 index 0000000000..d718602a55 --- /dev/null +++ b/rust/perspective-migrate/src/plugins/charts.rs @@ -0,0 +1,235 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use super::{clear_plugin_config, retain_column_keys}; +use crate::error::MigrateError; +use crate::json::{column_styles, object_mut}; +use crate::plugin::PluginMigration; +use crate::step::{Step, StepCx}; +use crate::version::Version; + +pub(crate) struct Charts; + +impl PluginMigration for Charts { + fn plugins(&self) -> &[&'static str] { + &[ + "X Bar", + "Y Bar", + "Y Line", + "Y Scatter", + "Y Area", + "X/Y Scatter", + "X/Y Line", + "Density", + "Treemap", + "Sunburst", + "Heatmap", + "Candlestick", + "OHLC", + ] + } + + fn steps(&self) -> Vec { + vec![ + Step { + at: Version::new(2, 7, 0), + id: "charts.symbols_map", + apply: symbols_map, + }, + Step { + at: Version::new(2, 9, 0), + id: "charts.symbols_move", + apply: symbols_move, + }, + Step { + at: Version::new(4, 5, 0), + id: "charts.alt_axis", + apply: alt_axis, + }, + Step { + at: Version::new(4, 5, 0), + id: "charts.scatter_symbol_slot", + apply: scatter_symbol_slot, + }, + Step { + at: Version::new(4, 5, 0), + id: "charts.d3fc_removed", + apply: d3fc_removed, + }, + ] + } +} + +fn is_xy_scatter(token: &Map, cx: &StepCx) -> bool { + token + .get("plugin") + .and_then(Value::as_str) + .and_then(|name| cx.resolve_plugin(name)) + == Some("X/Y Scatter") +} + +fn symbol_columns(token: &mut Map) -> Option<&mut Map> { + match object_mut(token, "plugin_config")?.get_mut("columns")? { + Value::Object(columns) => Some(columns), + _ => None, + } +} + +fn symbols_map(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if !is_xy_scatter(token, cx) { + return Ok(()); + } + + for (column, config) in symbol_columns(token).into_iter().flatten() { + let Some(Value::Array(pairs)) = config.get("symbols") else { + continue; + }; + + let mut symbols = Map::new(); + for pair in pairs { + let (Some(Value::String(key)), Some(value)) = (pair.get("key"), pair.get("value")) + else { + return Err(cx.malformed( + &["plugin_config", "columns", column, "symbols"], + "an array of {key, value}", + )); + }; + + symbols.insert(key.clone(), value.clone()); + } + + config["symbols"] = Value::Object(symbols); + } + + Ok(()) +} + +fn symbols_move(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if !is_xy_scatter(token, cx) { + return Ok(()); + } + + let mut moved = vec![]; + if let Some(columns) = symbol_columns(token) { + for (column, config) in columns.iter_mut() { + if let Some(symbols) = config + .as_object_mut() + .and_then(|x| x.shift_remove("symbols")) + { + moved.push((column.clone(), symbols)); + } + } + + columns.retain(|_, config| config.as_object().is_none_or(|x| !x.is_empty())); + if columns.is_empty() + && let Some(config) = object_mut(token, "plugin_config") + { + config.shift_remove("columns"); + } + } + + if moved.is_empty() { + return Ok(()); + } + + let Value::Object(config) = token + .entry("columns_config") + .or_insert_with(|| Value::Object(Map::new())) + else { + return Err(cx.malformed(&["columns_config"], "an object")); + }; + + for (column, symbols) in moved { + if let Value::Object(style) = config + .entry(column) + .or_insert_with(|| Value::Object(Map::new())) + { + style.entry("symbols").or_insert(symbols); + } + } + + Ok(()) +} + +fn alt_axis(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let Some(values) = object_mut(token, "plugin_config") + .and_then(|config| config.shift_remove("splitMainValues")) + else { + return Ok(()); + }; + + let Value::Array(values) = values else { + return Err(cx.malformed(&["plugin_config", "splitMainValues"], "an array of names")); + }; + + if values.is_empty() { + return Ok(()); + } + + let Value::Object(config) = token + .entry("columns_config") + .or_insert_with(|| Value::Object(Map::new())) + else { + return Err(cx.malformed(&["columns_config"], "an object")); + }; + + for value in values { + let Some(name) = value.as_str() else { + return Err(cx.malformed(&["plugin_config", "splitMainValues"], "an array of names")); + }; + + let base = name.rsplit('|').next().unwrap_or(name); + if let Value::Object(style) = config + .entry(base) + .or_insert_with(|| Value::Object(Map::new())) + { + style.insert("alt_axis".to_owned(), Value::Bool(true)); + } + } + + Ok(()) +} + +fn scatter_symbol_slot( + token: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if !is_xy_scatter(token, cx) { + return Ok(()); + } + + if cx.from() >= Version::new(2, 6, 0) + && let Some(Value::Array(columns)) = token.get_mut("columns") + && columns.len() > 4 + { + let symbol = columns.remove(4); + if !symbol.is_null() { + cx.dropped(&["columns", "4"], symbol); + } + } + + for (column, style) in column_styles(token) { + if let Some(symbols) = style.shift_remove("symbols") { + cx.dropped(&["columns_config", column, "symbols"], symbols); + } + } + + Ok(()) +} + +fn d3fc_removed(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + clear_plugin_config(token, cx); + retain_column_keys(token, &["number_format", "date_format", "alt_axis"], cx); + Ok(()) +} diff --git a/rust/perspective-migrate/src/plugins/datagrid.rs b/rust/perspective-migrate/src/plugins/datagrid.rs new file mode 100644 index 0000000000..f1af069b8a --- /dev/null +++ b/rust/perspective-migrate/src/plugins/datagrid.rs @@ -0,0 +1,409 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value, json}; + +use crate::css; +use crate::error::MigrateError; +use crate::json::{column_styles, object_mut, rename}; +use crate::plugin::PluginMigration; +use crate::step::{Step, StepCx}; +use crate::version::Version; + +pub(crate) struct Datagrid; + +impl PluginMigration for Datagrid { + fn plugins(&self) -> &[&'static str] { + &["Datagrid"] + } + + fn steps(&self) -> Vec { + vec![ + Step { + at: Version::new(2, 9, 0), + id: "datagrid.columns_config", + apply: columns_config, + }, + Step { + at: Version::new(3, 0, 0), + id: "datagrid.edit_mode", + apply: edit_mode, + }, + Step { + at: Version::new(4, 5, 0), + id: "datagrid.chart_keys", + apply: chart_keys, + }, + Step { + at: Version::new(5, 3, 0), + id: "datagrid.size_overrides", + apply: size_overrides, + }, + Step { + at: Version::new(5, 3, 0), + id: "datagrid.colors", + apply: colors, + }, + Step { + at: Version::new(5, 5, 0), + id: "datagrid.mode_renames", + apply: mode_renames, + }, + Step { + at: Version::new(5, 5, 0), + id: "datagrid.format_flags", + apply: format_flags, + }, + Step { + at: Version::new(5, 6, 0), + id: "datagrid.role_keys", + apply: role_keys, + }, + ] + } +} + +const DATE_KEYS: &[&str] = &[ + "timeZone", + "fractionalSecondDigits", + "second", + "minute", + "hour", + "day", + "weekday", + "month", + "year", + "hour12", + "dateStyle", + "timeStyle", +]; + +fn columns_config(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let Some(Value::Object(columns)) = + object_mut(token, "plugin_config").and_then(|config| config.get_mut("columns")) + else { + return Ok(()); + }; + + let mut moved = Map::new(); + for (column, old) in columns.iter_mut() { + let Value::Object(old) = old else { + continue; + }; + + let mut style = Map::new(); + let mut date_format = Map::new(); + for (key, value) in std::mem::take(old) { + match key.as_str() { + "column_size_override" => { + old.insert(key, value); + }, + "format" if value.as_str() == Some("custom") => { + date_format.insert(key, value); + }, + "format" if value.as_str() == Some("image") => { + cx.dropped(&["plugin_config", "columns", column, "format"], value); + }, + "fixed" => { + style.insert( + "number_format".to_owned(), + json!({"minimumFractionDigits": value, "maximumFractionDigits": value}), + ); + }, + key if DATE_KEYS.contains(&key) => { + date_format.insert(key.to_owned(), value); + }, + _ => { + style.insert(key, value); + }, + } + } + + if !date_format.is_empty() { + style.insert("date_format".to_owned(), Value::Object(date_format)); + } + + if !style.is_empty() { + moved.insert(column.clone(), Value::Object(style)); + } + } + + columns.retain(|_, old| old.as_object().is_none_or(|old| !old.is_empty())); + merge_columns_config(token, moved, cx) +} + +fn merge_columns_config( + token: &mut Map, + moved: Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if moved.is_empty() { + return Ok(()); + } + + let config = token + .entry("columns_config") + .or_insert_with(|| Value::Object(Map::new())); + + let Value::Object(config) = config else { + return Err(cx.malformed(&["columns_config"], "an object")); + }; + + for (column, style) in moved { + let Value::Object(style) = style else { + continue; + }; + + let entry = config + .entry(column) + .or_insert_with(|| Value::Object(Map::new())); + + if let Value::Object(entry) = entry { + for (key, value) in style { + entry.entry(key).or_insert(value); + } + } + } + + Ok(()) +} + +fn edit_mode(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let Some(config) = object_mut(token, "plugin_config") else { + return Ok(()); + }; + + let Some(editable) = config.shift_remove("editable") else { + return Ok(()); + }; + + match (config.contains_key("edit_mode"), editable.as_bool()) { + (false, Some(editable)) => { + let mode = if editable { "EDIT" } else { "READ_ONLY" }; + config.insert("edit_mode".to_owned(), Value::from(mode)); + }, + _ => cx.dropped(&["plugin_config", "editable"], editable), + } + + Ok(()) +} + +fn chart_keys(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + for (column, style) in column_styles(token) { + if let Some(value) = style.shift_remove("symbols") { + cx.dropped(&["columns_config", column, "symbols"], value); + } + } + + Ok(()) +} + +fn size_overrides(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let is_split = token + .get("split_by") + .and_then(Value::as_array) + .is_some_and(|x| !x.is_empty()); + + let Some(Value::Object(columns)) = + object_mut(token, "plugin_config").and_then(|config| config.shift_remove("columns")) + else { + return Ok(()); + }; + + let mut moved = Map::new(); + for (path, old) in columns { + let size = old.get("column_size_override"); + let px = size + .and_then(Value::as_f64) + .filter(|x| x.is_finite() && *x > 0.0); + match px { + Some(_) if !is_split && path != "__ROW_PATH__" => { + moved.insert(path, json!({"column_size_override": size})); + }, + _ if old.as_object().is_some_and(Map::is_empty) => {}, + _ => cx.dropped(&["plugin_config", "columns", &path], old), + } + } + + merge_columns_config(token, moved, cx) +} + +fn colors(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + for (column, style) in column_styles(token) { + if let Some(Value::String(color)) = style.get("color") { + match css::normalize(color) { + Some(color) => { + if color.had_alpha { + cx.approximated( + &["columns_config", column, "color"], + "alpha is unsupported", + ); + } + + style.insert("color".to_owned(), Value::from(color.hex)); + }, + None => { + let value = style.shift_remove("color").unwrap_or_default(); + cx.dropped(&["columns_config", column, "color"], value); + }, + } + } + + for side in ["fg", "bg"] { + let pos = style.shift_remove(&format!("pos_{side}_color")); + let neg = style.shift_remove(&format!("neg_{side}_color")); + let key = format!("{side}_colors"); + match (pos, neg) { + (None, None) => {}, + (Some(pos), Some(neg)) => match (stop(&pos), stop(&neg)) { + (Some(pos), Some(neg)) => { + let mode = style.get("number_bg_mode").and_then(Value::as_str); + if side == "bg" && matches!(mode, Some("gradient" | "pulse")) { + cx.approximated( + &["columns_config", column, &key], + "the theme background middle stop is not reconstructed", + ); + } + + let gradient = format!("linear-gradient(to right, {neg} 0%, {pos} 100%)"); + style.insert(key, Value::from(gradient)); + }, + _ => cx.dropped(&["columns_config", column, &key], json!([neg, pos])), + }, + (pos, neg) => { + let value = pos.or(neg).unwrap_or_default(); + cx.dropped(&["columns_config", column, &key], value); + }, + } + } + } + + Ok(()) +} + +fn stop(color: &Value) -> Option { + color.as_str().and_then(css::normalize).map(|x| x.hex) +} + +fn mode_renames(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + for (column, style) in column_styles(token) { + for (from, to) in [ + ("number_fg_mode", "fg_mode"), + ("number_bg_mode", "bg_mode"), + ("fg_colors", "fg_color"), + ("bg_colors", "bg_color"), + ] { + if let Some(value) = rename(style, from, to) { + cx.dropped(&["columns_config", column, from], value); + } + } + + let mode_key = ["string_color_mode", "datetime_color_mode"] + .into_iter() + .find(|key| style.contains_key(*key)); + + let mode = mode_key.and_then(|key| style.shift_remove(key)); + let color = style.shift_remove("color"); + let palette = style.shift_remove("palette"); + let (assigned, unused) = match mode.as_ref().and_then(Value::as_str) { + Some("foreground") => (Some(("fg", "color", color)), vec![("palette", palette)]), + Some("background") => (Some(("bg", "color", color)), vec![("palette", palette)]), + Some("series") => (Some(("bg", "series", palette)), vec![("color", color)]), + Some("none") | None => (None, vec![("color", color), ("palette", palette)]), + Some(_) => { + let key = mode_key.unwrap_or_default(); + cx.dropped( + &["columns_config", column, key], + mode.clone().unwrap_or_default(), + ); + (None, vec![("color", color), ("palette", palette)]) + }, + }; + + if let Some((side, mode, value)) = assigned { + style.insert(format!("{side}_mode"), Value::from(mode)); + if let Some(value) = value { + style.insert(format!("{side}_color"), value); + } + } + + for (key, value) in unused { + if let Some(value) = value { + cx.dropped(&["columns_config", column, key], value); + } + } + } + + Ok(()) +} + +fn format_flags(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + for (column, style) in column_styles(token) { + let Some(format) = style.shift_remove("format") else { + continue; + }; + + let flag = match format.as_str() { + Some("link") => "link", + Some("bold") => "bold", + Some("italics") => "italic", + _ => { + cx.dropped(&["columns_config", column, "format"], format); + continue; + }, + }; + + style.entry(flag).or_insert(Value::Bool(true)); + } + + Ok(()) +} + +fn role_keys(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let names = |key: &str| -> Option> { + token.get(key)?.as_array().map(|x| { + x.iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + }; + + let (Some(group_by), Some(split_by), Some(columns)) = + (names("group_by"), names("split_by"), names("columns")) + else { + return Ok(()); + }; + + for (column, style) in column_styles(token) { + let pivot_only = + (group_by.contains(column) || split_by.contains(column)) && !columns.contains(column); + + if !pivot_only { + continue; + } + + let dropped = style + .keys() + .filter(|key| *key != "number_format" && *key != "date_format") + .cloned() + .collect::>(); + + for key in dropped { + if let Some(value) = style.shift_remove(&key) { + cx.dropped(&["columns_config", column, &key], value); + } + } + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/plugins/maps.rs b/rust/perspective-migrate/src/plugins/maps.rs new file mode 100644 index 0000000000..4c9cb59fcd --- /dev/null +++ b/rust/perspective-migrate/src/plugins/maps.rs @@ -0,0 +1,78 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use super::{clear_plugin_config, retain_column_keys}; +use crate::error::MigrateError; +use crate::json::object_mut; +use crate::plugin::PluginMigration; +use crate::step::{Step, StepCx}; +use crate::version::Version; + +pub(crate) struct Maps; + +impl PluginMigration for Maps { + fn plugins(&self) -> &[&'static str] { + &["Map Scatter", "Map Line", "Map Density"] + } + + fn steps(&self) -> Vec { + vec![ + Step { + at: Version::new(4, 5, 0), + id: "maps.label_slot", + apply: label_slot, + }, + Step { + at: Version::new(4, 5, 0), + id: "maps.openlayers_removed", + apply: openlayers_removed, + }, + Step { + at: Version::new(5, 3, 1), + id: "maps.carto_tiles", + apply: carto_tiles, + }, + ] + } +} + +fn label_slot(token: &mut Map, _cx: &mut StepCx) -> Result<(), MigrateError> { + if let Some(Value::Array(columns)) = token.get_mut("columns") + && columns.len() > 4 + { + columns.insert(4, Value::Null); + } + + Ok(()) +} + +fn openlayers_removed(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + clear_plugin_config(token, cx); + retain_column_keys(token, &["number_format", "date_format"], cx); + Ok(()) +} + +fn carto_tiles(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if let Some(config) = object_mut(token, "plugin_config") + && config + .get("map_tile_provider") + .and_then(Value::as_str) + .is_some_and(|x| x.starts_with("carto-")) + && let Some(provider) = config.shift_remove("map_tile_provider") + { + cx.dropped(&["plugin_config", "map_tile_provider"], provider); + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/plugins/mod.rs b/rust/perspective-migrate/src/plugins/mod.rs new file mode 100644 index 0000000000..9c01bb43b2 --- /dev/null +++ b/rust/perspective-migrate/src/plugins/mod.rs @@ -0,0 +1,92 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +mod charts; +mod datagrid; +mod maps; + +use serde_json::{Map, Value}; + +use crate::json::object_mut; +use crate::plugin::PluginMigration; +use crate::step::StepCx; + +pub(crate) const REGISTRATION_ORDER: &[&str] = &[ + "Datagrid", + "X Bar", + "Y Bar", + "Y Line", + "Y Scatter", + "Y Area", + "X/Y Scatter", + "X/Y Line", + "Density", + "Treemap", + "Sunburst", + "Heatmap", + "Candlestick", + "OHLC", + "Map Scatter", + "Map Line", + "Map Density", + "Debug", +]; + +pub(crate) fn first_party() -> Vec> { + vec![ + Box::new(datagrid::Datagrid), + Box::new(charts::Charts), + Box::new(maps::Maps), + ] +} + +pub(crate) fn resolve_plugin(name: &str, known: &[&'static str]) -> Option<&'static str> { + let short = short_name(name); + known.iter().find(|x| short_name(x) == short).copied() +} + +fn clear_plugin_config(token: &mut Map, cx: &mut StepCx) { + if let Some(config) = object_mut(token, "plugin_config") { + for (key, value) in std::mem::take(config) { + cx.dropped(&["plugin_config", &key], value); + } + } +} + +fn retain_column_keys(token: &mut Map, keep: &[&str], cx: &mut StepCx) { + let Some(config) = object_mut(token, "columns_config") else { + return; + }; + + for (column, style) in config.iter_mut() { + let Value::Object(style) = style else { + continue; + }; + + for (key, value) in std::mem::take(style) { + if keep.contains(&key.as_str()) { + style.insert(key, value); + } else { + cx.dropped(&["columns_config", column, &key], value); + } + } + } + + config.retain(|_, style| style.as_object().is_none_or(|x| !x.is_empty())); +} + +fn short_name(name: &str) -> String { + name.to_lowercase() + .chars() + .filter(|x| x.is_alphabetic()) + .collect() +} diff --git a/rust/perspective-migrate/src/step.rs b/rust/perspective-migrate/src/step.rs new file mode 100644 index 0000000000..4ecee920ff --- /dev/null +++ b/rust/perspective-migrate/src/step.rs @@ -0,0 +1,235 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::json::escape; +use crate::notice::Notice; +use crate::plugins::resolve_plugin; +use crate::version::Version; +use crate::{Kind, check_version}; + +/// Rewrites one config object from the shape before [`Step::at`] to the shape +/// after it. +pub type Apply = fn(&mut Map, &mut StepCx) -> Result<(), MigrateError>; + +/// One schema change: `apply` runs on every object it concerns when the +/// source version predates `at`. +#[derive(Clone, Copy)] +pub struct Step { + /// The first release that shipped the new shape. + pub at: Version, + + /// A stable identifier, reported in [`crate::Migrated::steps`]. + pub id: &'static str, + + /// The rewrite, which must only touch keys present in its input. + pub apply: Apply, +} + +/// The context a [`Step`] runs in. +pub struct StepCx<'a> { + kind: Kind, + from: Version, + step: &'static str, + pointer: &'a str, + known_plugins: &'a [&'static str], + notices: &'a mut Vec, +} + +impl StepCx<'_> { + /// The kind of document being migrated. + pub fn kind(&self) -> Kind { + self.kind + } + + /// The version the document was written by. + pub fn from(&self) -> Version { + self.from + } + + /// Record that `value`, found at `path` below this object, was removed. + pub fn dropped(&mut self, path: &[&str], value: Value) { + let pointer = self.pointer_to(path); + self.notices.push(Notice::Dropped { + step: self.step, + pointer, + value, + }); + } + + /// Record that the value at `path` below this object was approximated. + pub fn approximated(&mut self, path: &[&str], reason: &'static str) { + let pointer = self.pointer_to(path); + self.notices.push(Notice::Approximated { + step: self.step, + pointer, + reason, + }); + } + + /// A [`MigrateError::Malformed`] for the value at `path` below this object. + pub fn malformed(&self, path: &[&str], expected: &'static str) -> MigrateError { + MigrateError::Malformed { + step: self.step, + pointer: self.pointer_to(path), + expected, + } + } + + /// A [`MigrateError::Conflict`] between `keys` of this object. + pub fn conflict(&self, keys: &[&str]) -> MigrateError { + MigrateError::Conflict { + step: self.step, + pointer: self.pointer.to_owned(), + keys: keys.iter().map(|x| (*x).to_owned()).collect(), + } + } + + /// A [`MigrateError::MissingTable`] for the workspace panel `panel`. + pub fn missing_table(&self, panel: &str) -> MigrateError { + MigrateError::MissingTable { + step: self.step, + panel: panel.to_owned(), + } + } + + pub(crate) fn resolve_plugin(&self, name: &str) -> Option<&'static str> { + resolve_plugin(name, self.known_plugins) + } + + fn pointer_to(&self, path: &[&str]) -> String { + path.iter().fold(self.pointer.to_owned(), |acc, seg| { + format!("{acc}/{}", escape(seg)) + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum Layer { + View, + Viewer, + ColumnStyle, + Plugin, + Workspace, +} + +pub(crate) struct Entry<'a> { + pub layer: Layer, + pub plugins: &'a [&'static str], + pub step: Step, +} + +pub(crate) struct Run<'a> { + pub kind: Kind, + pub from: Version, + pub schema: Version, + pub known_plugins: &'a [&'static str], + pub notices: Vec, + pub steps: Vec<&'static str>, +} + +impl Run<'_> { + pub fn apply(&mut self, entries: &[Entry], root: &mut Value) -> Result<(), MigrateError> { + for entry in entries { + let notices = self.notices.len(); + let mut changed = false; + for pointer in targets(self.kind, entry.layer, root) { + let Some(Value::Object(target)) = root.pointer_mut(&pointer) else { + continue; + }; + + let from = match target.get("version").and_then(Value::as_str) { + Some(version) => check_version(version, self.schema)?, + None => self.from, + }; + + if from >= entry.step.at + || (entry.layer == Layer::Plugin && !self.owns(entry.plugins, target)) + { + continue; + } + + let before = target.clone(); + let mut cx = StepCx { + kind: self.kind, + from, + step: entry.step.id, + pointer: &pointer, + known_plugins: self.known_plugins, + notices: &mut self.notices, + }; + + (entry.step.apply)(target, &mut cx)?; + changed |= *target != before; + } + + if changed || self.notices.len() > notices { + self.steps.push(entry.step.id); + } + } + + Ok(()) + } + + fn owns(&self, plugins: &[&'static str], target: &Map) -> bool { + target + .get("plugin") + .and_then(Value::as_str) + .and_then(|name| resolve_plugin(name, self.known_plugins)) + .is_some_and(|name| plugins.contains(&name)) + } +} + +fn targets(kind: Kind, layer: Layer, root: &Value) -> Vec { + match (kind, layer) { + (Kind::View, Layer::View) => vec![String::new()], + (Kind::View, _) => vec![], + (Kind::Viewer, Layer::View) => { + let mut out = vec![String::new()]; + let mut pointer = String::from("/table"); + while root.pointer(&pointer).is_some_and(Value::is_object) { + out.push(pointer.clone()); + pointer.push_str("/table"); + } + + out + }, + (Kind::Viewer, Layer::Workspace) => vec![], + (Kind::Viewer, _) => vec![String::new()], + (Kind::Workspace, Layer::Workspace) => vec![String::new()], + (Kind::Workspace, Layer::View) => { + let mut out = panels(root); + out.extend(children(root, "derived_tables")); + out + }, + (Kind::Workspace, _) => panels(root), + } +} + +pub(crate) fn panels(root: &Value) -> Vec { + let mut out = children(root, "panels"); + out.extend(children(root, "viewers")); + out +} + +fn children(root: &Value, key: &str) -> Vec { + root.get(key) + .and_then(Value::as_object) + .map(|map| { + map.keys() + .map(|name| format!("/{key}/{}", escape(name))) + .collect() + }) + .unwrap_or_default() +} diff --git a/rust/perspective-migrate/src/version.rs b/rust/perspective-migrate/src/version.rs new file mode 100644 index 0000000000..c833a90bd2 --- /dev/null +++ b/rust/perspective-migrate/src/version.rs @@ -0,0 +1,50 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::fmt; + +/// A Perspective release, ordered by `major.minor.patch` with any +/// pre-release or build suffix ignored. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Version { + pub major: u64, + pub minor: u64, + pub patch: u64, +} + +impl Version { + pub const fn new(major: u64, minor: u64, patch: u64) -> Self { + Self { + major, + minor, + patch, + } + } + + /// Parse `x.y.z`, `x.y.z-pre` or `x.y.z+build`. + pub fn parse(version: &str) -> Option { + let core = version.trim().split(['-', '+']).next()?; + let mut parts = core.split('.').map(|x| x.parse::().ok()); + let version = Self::new(parts.next()??, parts.next()??, parts.next()??); + parts.next().is_none().then_some(version) + } + + pub(crate) fn crate_version() -> Self { + Self::parse(env!("CARGO_PKG_VERSION")).expect("CARGO_PKG_VERSION") + } +} + +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch) + } +} diff --git a/rust/perspective-migrate/src/view/mod.rs b/rust/perspective-migrate/src/view/mod.rs new file mode 100644 index 0000000000..a43cd15e48 --- /dev/null +++ b/rust/perspective-migrate/src/view/mod.rs @@ -0,0 +1,88 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +mod v3_0_0; +mod v3_8_0; +mod v5_2_0; +mod v5_6_0; +mod v5_7_0; + +use crate::step::Step; +use crate::version::Version; + +pub(crate) const STEPS: &[Step] = &[ + Step { + at: Version::new(3, 0, 0), + id: "view.v2_aliases", + apply: v3_0_0::v2_aliases, + }, + Step { + at: Version::new(3, 0, 0), + id: "view.drop_split_by_depth", + apply: v3_0_0::drop_split_by_depth, + }, + Step { + at: Version::new(3, 0, 0), + id: "view.drop_viewport", + apply: v3_0_0::drop_viewport, + }, + Step { + at: Version::new(3, 0, 0), + id: "view.filter_op_symbols", + apply: v3_0_0::filter_op_symbols, + }, + Step { + at: Version::new(3, 0, 0), + id: "view.expression_tuples", + apply: v3_0_0::expression_tuples, + }, + Step { + at: Version::new(3, 8, 0), + id: "view.multi_aggregate_args", + apply: v3_8_0::multi_aggregate_args, + }, + Step { + at: Version::new(5, 2, 0), + id: "view.prune_unused_aggregates", + apply: v5_2_0::prune_unused_aggregates, + }, + Step { + at: Version::new(5, 6, 0), + id: "view.dedupe_group_by", + apply: v5_6_0::dedupe_group_by, + }, + Step { + at: Version::new(5, 7, 0), + id: "view.expressions_map", + apply: v5_7_0::expressions_map, + }, + Step { + at: Version::new(5, 7, 0), + id: "view.canonical_names", + apply: v5_7_0::canonical_names, + }, +]; + +pub(crate) const VIEW_KEYS: &[&str] = &[ + "group_by", + "split_by", + "columns", + "filter", + "sort", + "expressions", + "windows", + "aggregates", + "group_by_depth", + "filter_op", + "group_rollup_mode", + "split_rollup_mode", +]; diff --git a/rust/perspective-migrate/src/view/v3_0_0.rs b/rust/perspective-migrate/src/view/v3_0_0.rs new file mode 100644 index 0000000000..b8f644766c --- /dev/null +++ b/rust/perspective-migrate/src/view/v3_0_0.rs @@ -0,0 +1,153 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::Kind; +use crate::error::MigrateError; +use crate::step::StepCx; + +const ALIASES: &[(&str, &str)] = &[ + ("row_pivot", "group_by"), + ("row-pivot", "group_by"), + ("row-pivots", "group_by"), + ("col_pivot", "split_by"), + ("col_pivots", "split_by"), + ("column_pivot", "split_by"), + ("column-pivot", "split_by"), + ("column-pivots", "split_by"), + ("filters", "filter"), + ("sorts", "sort"), +]; + +pub(super) fn v2_aliases( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if cx.kind() != Kind::View { + return Ok(()); + } + + for (alias, key) in ALIASES { + if view.contains_key(*alias) && view.contains_key(*key) { + return Err(cx.conflict(&[alias, key])); + } + + if let Some(value) = view.shift_remove(*alias) { + view.insert((*key).to_owned(), value); + } + } + + if let Some(aggregate) = view.shift_remove("aggregate") { + if view.contains_key("aggregates") || view.contains_key("columns") { + return Err(cx.conflict(&["aggregate", "aggregates", "columns"])); + } + + let Value::Array(specs) = aggregate else { + return Err(cx.malformed(&["aggregate"], "an array of {column, op}")); + }; + + let mut aggregates = Map::new(); + let mut columns = vec![]; + for spec in specs { + let (Some(Value::String(column)), Some(op)) = (spec.get("column"), spec.get("op")) + else { + return Err(cx.malformed(&["aggregate"], "an array of {column, op}")); + }; + + aggregates.insert(column.clone(), op.clone()); + columns.push(Value::String(column.clone())); + } + + view.insert("aggregates".to_owned(), Value::Object(aggregates)); + view.insert("columns".to_owned(), Value::Array(columns)); + } + + if let Some(value) = view.shift_remove("computed_columns") { + cx.dropped(&["computed_columns"], value); + } + + if let Some(value) = view.shift_remove("version") { + cx.dropped(&["version"], value); + } + + Ok(()) +} + +pub(super) fn drop_split_by_depth( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if let Some(value) = view.shift_remove("split_by_depth") { + cx.dropped(&["split_by_depth"], value); + } + + Ok(()) +} + +pub(super) fn drop_viewport( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if let Some(value) = view.shift_remove("viewport") { + cx.dropped(&["viewport"], value); + } + + Ok(()) +} + +pub(super) fn filter_op_symbols( + view: &mut Map, + _cx: &mut StepCx, +) -> Result<(), MigrateError> { + if let Some(op) = view.get_mut("filter_op") { + match op.as_str() { + Some("&") => *op = Value::from("and"), + Some("|") => *op = Value::from("or"), + _ => {}, + } + } + + Ok(()) +} + +pub(super) fn expression_tuples( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + let Some(Value::Array(items)) = view.get("expressions") else { + return Ok(()); + }; + + if !items.iter().any(Value::is_array) { + return Ok(()); + } + + let mut map = Map::new(); + for item in items { + let (Some(Value::String(alias)), Some(Value::String(expression))) = + (item.get(0), item.get(1)) + else { + return Err(cx.malformed(&["expressions"], "[alias, expression, …] tuples")); + }; + + let body = expression + .split_once('\n') + .filter(|(line, body)| line.starts_with("//") && !body.is_empty()) + .map_or(expression.as_str(), |(_, body)| body); + + map.insert(alias.clone(), Value::from(body)); + } + + view.insert("expressions".to_owned(), Value::Object(map)); + Ok(()) +} diff --git a/rust/perspective-migrate/src/view/v3_8_0.rs b/rust/perspective-migrate/src/view/v3_8_0.rs new file mode 100644 index 0000000000..ba0f57c4d3 --- /dev/null +++ b/rust/perspective-migrate/src/view/v3_8_0.rs @@ -0,0 +1,36 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::json::object_mut; +use crate::step::StepCx; + +pub(super) fn multi_aggregate_args( + view: &mut Map, + _cx: &mut StepCx, +) -> Result<(), MigrateError> { + let Some(aggregates) = object_mut(view, "aggregates") else { + return Ok(()); + }; + + for aggregate in aggregates.values_mut() { + if let Value::Array(parts) = aggregate + && let [_, arg @ Value::String(_)] = parts.as_mut_slice() + { + *arg = Value::Array(vec![arg.take()]); + } + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/view/v5_2_0.rs b/rust/perspective-migrate/src/view/v5_2_0.rs new file mode 100644 index 0000000000..b51b3cccef --- /dev/null +++ b/rust/perspective-migrate/src/view/v5_2_0.rs @@ -0,0 +1,145 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::step::StepCx; + +const UNIMPLEMENTED: &[&str] = &[ + "identity", + "mean by count", + "mean_by_count", + "py_agg", + "div", + "add", +]; + +const TAKES_ARGUMENT: &[&str] = &[ + "weighted mean", + "weighted_mean", + "max by", + "max_by", + "min by", + "min_by", +]; + +const IMPLEMENTED: &[&str] = &[ + "distinct count", + "distinctcount", + "distinct", + "distinct_count", + "sum", + "mul", + "avg", + "mean", + "count", + "unique", + "any", + "q1", + "q3", + "median", + "join", + "dominant", + "first by index", + "first", + "last by index", + "last minus first", + "and", + "or", + "last", + "last_value", + "max", + "min", + "high", + "high_water_mark", + "low", + "low_water_mark", + "high minus low", + "sum abs", + "sum_abs", + "abs sum", + "abs_sum", + "sum or zero", + "sum_or_zero", + "sum not null", + "sum_not_null", + "distinct leaf", + "distinct_leaf", + "pct sum parent", + "pct_sum_parent", + "pct sum total", + "pct_sum_total", + "pct sum grand total", + "pct_sum_grand_total", + "var", + "variance", + "stddev", + "standard deviation", + "gmv", +]; + +pub(super) fn prune_unused_aggregates( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + let used = view + .get("columns") + .and_then(Value::as_array) + .map(|columns| { + columns + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect::>() + }) + .unwrap_or_default(); + + let Some(Value::Object(aggregates)) = view.get_mut("aggregates") else { + return Ok(()); + }; + + let invalid = aggregates + .iter() + .filter(|(column, aggregate)| !used.contains(column) && !is_valid(aggregate)) + .map(|(column, _)| column.clone()) + .collect::>(); + + for column in invalid { + if let Some(value) = aggregates.shift_remove(&column) { + cx.dropped(&["aggregates", &column], value); + } + } + + Ok(()) +} + +fn is_valid(aggregate: &Value) -> bool { + match aggregate { + Value::String(name) => is_known(name) && !TAKES_ARGUMENT.contains(&name.as_str()), + Value::Array(parts) => match parts.as_slice() { + [Value::String(name)] => is_known(name) && !TAKES_ARGUMENT.contains(&name.as_str()), + [Value::String(name), Value::Array(args)] => { + is_known(name) && (!TAKES_ARGUMENT.contains(&name.as_str()) || !args.is_empty()) + }, + _ => false, + }, + _ => false, + } +} + +fn is_known(name: &str) -> bool { + !UNIMPLEMENTED.contains(&name) + && !name.contains("udf_combiner_") + && !name.contains("udf_reducer_") + && (IMPLEMENTED.contains(&name) || TAKES_ARGUMENT.contains(&name)) +} diff --git a/rust/perspective-migrate/src/view/v5_6_0.rs b/rust/perspective-migrate/src/view/v5_6_0.rs new file mode 100644 index 0000000000..68029cc649 --- /dev/null +++ b/rust/perspective-migrate/src/view/v5_6_0.rs @@ -0,0 +1,43 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::step::StepCx; + +pub(super) fn dedupe_group_by( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + let Some(Value::Array(group_by)) = view.get_mut("group_by") else { + return Ok(()); + }; + + let mut seen = vec![]; + let mut dropped = vec![]; + group_by.retain(|column| { + if seen.contains(column) { + dropped.push(column.clone()); + false + } else { + seen.push(column.clone()); + true + } + }); + + for column in dropped { + cx.dropped(&["group_by"], column); + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/view/v5_7_0.rs b/rust/perspective-migrate/src/view/v5_7_0.rs new file mode 100644 index 0000000000..68329d0f1d --- /dev/null +++ b/rust/perspective-migrate/src/view/v5_7_0.rs @@ -0,0 +1,138 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::json::object_mut; +use crate::step::StepCx; +use crate::version::Version; + +const AGGREGATE_RENAMES: &[(&str, &str)] = &[ + ("sum not null", "sum"), + ("sum_not_null", "sum"), + ("pct sum grand total", "pct sum total"), + ("pct_sum_grand_total", "pct sum total"), + ("last value", "last"), + ("high water mark", "high"), + ("low water mark", "low"), + ("high minuslow", "high minus low"), +]; + +const AGGREGATES_UNIMPLEMENTED: &[&str] = &["py agg", "scaled div", "scaled add", "scaled mul"]; + +const FILTER_OP_RENAMES: &[(&str, &str)] = + &[("is None", "is null"), ("is not None", "is not null")]; + +pub(super) fn expressions_map( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + let Some(Value::Array(items)) = view.get("expressions") else { + return Ok(()); + }; + + let mut map = Map::new(); + for item in items { + let Value::String(source) = item else { + return Err(cx.malformed(&["expressions"], "an array of expression strings")); + }; + + let (name, expression) = if cx.from() < Version::new(3, 0, 0) { + split_v2(source) + } else { + split_v3(source) + }; + + map.insert(name.to_owned(), Value::from(expression)); + } + + let unordered = map.len() > 1 && !view.contains_key("columns"); + view.insert("expressions".to_owned(), Value::Object(map)); + if unordered { + cx.approximated( + &["expressions"], + "expression columns without `columns` are no longer ordered by the array", + ); + } + + Ok(()) +} + +fn split_v2(source: &str) -> (&str, &str) { + let Some((line, body)) = source.split_once('\n') else { + return (source, source); + }; + + let alias = line.strip_prefix("//").map(str::trim).unwrap_or_default(); + if alias.is_empty() { + (source, source) + } else if body.is_empty() { + (alias, source) + } else { + (alias, body) + } +} + +fn split_v3(source: &str) -> (&str, &str) { + match source.split_once('\n') { + Some((line, body)) if !body.is_empty() && line.starts_with("//") => { + (line[2..].trim(), body) + }, + _ => (source, source), + } +} + +pub(super) fn canonical_names( + view: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if let Some(aggregates) = object_mut(view, "aggregates") { + let mut unimplemented = vec![]; + for (column, aggregate) in aggregates.iter_mut() { + let name = match aggregate { + Value::Array(parts) => parts.first_mut(), + name => Some(name), + }; + + let Some(Value::String(name)) = name else { + continue; + }; + + if AGGREGATES_UNIMPLEMENTED.contains(&name.as_str()) { + unimplemented.push(column.clone()); + } else if let Some((_, canonical)) = AGGREGATE_RENAMES.iter().find(|(x, _)| x == name) { + *name = (*canonical).to_owned(); + } + } + + for column in unimplemented { + if let Some(value) = aggregates.shift_remove(&column) { + cx.dropped(&["aggregates", &column], value); + } + } + } + + if let Some(Value::Array(filters)) = view.get_mut("filter") { + for filter in filters { + if let Some(op) = filter.get_mut(1) + && let Some((_, canonical)) = FILTER_OP_RENAMES + .iter() + .find(|(x, _)| op.as_str() == Some(*x)) + { + *op = Value::from(*canonical); + } + } + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/viewer/mod.rs b/rust/perspective-migrate/src/viewer/mod.rs new file mode 100644 index 0000000000..ad8a5f84dc --- /dev/null +++ b/rust/perspective-migrate/src/viewer/mod.rs @@ -0,0 +1,129 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value}; + +use crate::Kind; +use crate::error::MigrateError; +use crate::step::{Step, StepCx}; +use crate::version::Version; +use crate::view::VIEW_KEYS; + +pub(crate) const STEPS: &[Step] = &[ + Step { + at: Version::new(4, 5, 0), + id: "viewer.plugin_config_object", + apply: plugin_config_object, + }, + Step { + at: Version::new(5, 2, 0), + id: "viewer.null_table", + apply: null_table, + }, + Step { + at: Version::new(5, 7, 0), + id: "viewer.plugin_names", + apply: plugin_names, + }, + Step { + at: Version::new(5, 7, 0), + id: "viewer.unknown_keys", + apply: unknown_keys, + }, +]; + +const PANEL_KEYS: &[&str] = &[ + "version", + "plugin", + "title", + "table", + "theme", + "plugin_config", + "columns_config", +]; + +fn plugin_config_object( + token: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if let Some(config) = token.get_mut("plugin_config") + && !config.is_object() + && !config.is_null() + { + let value = std::mem::replace(config, Value::Object(Map::new())); + cx.dropped(&["plugin_config"], value); + } + + Ok(()) +} + +fn null_table(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if cx.kind() == Kind::Viewer && token.get("table").is_some_and(Value::is_null) { + token.shift_remove("table"); + } + + Ok(()) +} + +fn plugin_names(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if let Some(Value::String(plugin)) = token.get_mut("plugin") + && let Some(canonical) = cx.resolve_plugin(plugin) + { + canonical.clone_into(plugin); + } + + Ok(()) +} + +fn unknown_keys(token: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let settings: &[&str] = if cx.kind() == Kind::Viewer { + &["settings"] + } else { + &[] + }; + + retain_keys(token, &[PANEL_KEYS, VIEW_KEYS, settings], &[], cx); + let mut path = vec!["table".to_owned()]; + while let Some(Value::Object(derived)) = path_mut(token, &path) { + retain_keys(derived, &[&["table"], VIEW_KEYS], &path, cx); + path.push("table".to_owned()); + } + + Ok(()) +} + +fn path_mut<'a>(token: &'a mut Map, path: &[String]) -> Option<&'a mut Value> { + let (first, rest) = path.split_first()?; + rest.iter() + .try_fold(token.get_mut(first)?, |value, key| value.get_mut(key)) +} + +pub(crate) fn retain_keys( + map: &mut Map, + allowed: &[&[&str]], + path: &[String], + cx: &mut StepCx, +) { + let unknown = map + .keys() + .filter(|key| !allowed.iter().any(|keys| keys.contains(&key.as_str()))) + .cloned() + .collect::>(); + + for key in unknown { + if let Some(value) = map.shift_remove(&key) { + let mut pointer = path.iter().map(String::as_str).collect::>(); + pointer.push(&key); + cx.dropped(&pointer, value); + } + } +} diff --git a/rust/perspective-migrate/src/workspace/mod.rs b/rust/perspective-migrate/src/workspace/mod.rs new file mode 100644 index 0000000000..5602cbdb0c --- /dev/null +++ b/rust/perspective-migrate/src/workspace/mod.rs @@ -0,0 +1,187 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +mod regular_layout; + +use serde_json::{Map, Value}; + +use crate::error::MigrateError; +use crate::json::object_mut; +use crate::step::{Step, StepCx}; +use crate::version::Version; +use crate::view::VIEW_KEYS; +use crate::viewer::retain_keys; + +pub(crate) const STEPS: &[Step] = &[ + Step { + at: Version::new(2, 1, 0), + id: "ws.name_title", + apply: name_title, + }, + Step { + at: Version::new(3, 1, 3), + id: "ws.inline_widgets", + apply: inline_widgets, + }, + Step { + at: Version::new(3, 1, 3), + id: "ws.drop_linked", + apply: drop_linked, + }, + Step { + at: Version::new(3, 1, 4), + id: "ws.drop_master_flag", + apply: drop_master_flag, + }, + Step { + at: Version::new(5, 0, 0), + id: "ws.regular_layout", + apply: regular_layout::regular_layout, + }, + Step { + at: Version::new(5, 2, 0), + id: "ws.panel_table", + apply: panel_table, + }, + Step { + at: Version::new(5, 7, 0), + id: "ws.unknown_keys", + apply: unknown_keys, + }, +]; + +const WORKSPACE_KEYS: &[&str] = &[ + "version", + "active", + "layout", + "panels", + "derived_tables", + "global_filters", + "masters", + "palette", +]; + +fn lumino_panels( + root: &mut Map, +) -> impl Iterator)> { + object_mut(root, "viewers") + .into_iter() + .flat_map(|viewers| viewers.iter_mut()) + .filter_map(|(id, panel)| panel.as_object_mut().map(|panel| (id, panel))) +} + +fn name_title(root: &mut Map, _cx: &mut StepCx) -> Result<(), MigrateError> { + for (_, panel) in lumino_panels(root) { + if let Some(name) = panel.shift_remove("name") { + panel.insert("title".to_owned(), name); + } + } + + Ok(()) +} + +fn inline_widgets(root: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let Some(Value::Array(widgets)) = object_mut(root, "master").and_then(|x| x.get_mut("widgets")) + else { + return Ok(()); + }; + + let mut hoisted = vec![]; + for widget in widgets.iter_mut() { + if let Value::Object(config) = widget { + let Some(Value::String(slot)) = config.shift_remove("slot") else { + return Err(cx.malformed(&["master", "widgets"], "slot names or {slot, …}")); + }; + + hoisted.push((slot.clone(), std::mem::take(config))); + *widget = Value::String(slot); + } + } + + if hoisted.is_empty() { + return Ok(()); + } + + let Value::Object(viewers) = root + .entry("viewers") + .or_insert_with(|| Value::Object(Map::new())) + else { + return Err(cx.malformed(&["viewers"], "an object")); + }; + + for (slot, config) in hoisted { + viewers.entry(slot).or_insert(Value::Object(config)); + } + + Ok(()) +} + +fn drop_linked(root: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + if let Some(mode) = root.shift_remove("mode") + && mode.as_str() == Some("linked") + { + cx.dropped(&["mode"], mode); + } + + let mut dropped = vec![]; + for (id, panel) in lumino_panels(root) { + for key in ["linked", "selectable", "editable"] { + if let Some(value) = panel.shift_remove(key) + && !matches!(value, Value::Null | Value::Bool(false)) + { + dropped.push((id.clone(), key, value)); + } + } + } + + for (id, key, value) in dropped { + cx.dropped(&["viewers", &id, key], value); + } + + Ok(()) +} + +fn drop_master_flag(root: &mut Map, _cx: &mut StepCx) -> Result<(), MigrateError> { + for (_, panel) in lumino_panels(root) { + panel.shift_remove("master"); + } + + Ok(()) +} + +fn panel_table(root: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + let Some(panels) = object_mut(root, "panels") else { + return Ok(()); + }; + + for (id, panel) in panels.iter() { + if !panel.get("table").is_some_and(Value::is_string) { + return Err(cx.missing_table(id)); + } + } + + Ok(()) +} + +fn unknown_keys(root: &mut Map, cx: &mut StepCx) -> Result<(), MigrateError> { + retain_keys(root, &[WORKSPACE_KEYS], &[], cx); + if let Some(derived) = object_mut(root, "derived_tables") { + for (name, config) in derived.iter_mut() { + if let Value::Object(config) = config { + let path = ["derived_tables".to_owned(), name.clone()]; + retain_keys(config, &[&["table"], VIEW_KEYS], &path, cx); + } + } + } + + Ok(()) +} diff --git a/rust/perspective-migrate/src/workspace/regular_layout.rs b/rust/perspective-migrate/src/workspace/regular_layout.rs new file mode 100644 index 0000000000..cf8392a552 --- /dev/null +++ b/rust/perspective-migrate/src/workspace/regular_layout.rs @@ -0,0 +1,247 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use serde_json::{Map, Value, json}; + +use crate::error::MigrateError; +use crate::step::StepCx; + +const DEFAULT_MASTER_SIZES: [f64; 2] = [0.25, 0.75]; + +pub(super) fn regular_layout( + root: &mut Map, + cx: &mut StepCx, +) -> Result<(), MigrateError> { + if !root.contains_key("viewers") && !root.contains_key("detail") { + return Ok(()); + } + + let mut panels = match root.shift_remove("viewers") { + Some(Value::Object(viewers)) => viewers, + None => Map::new(), + Some(_) => return Err(cx.malformed(&["viewers"], "an object")), + }; + + let sizes = root.shift_remove("sizes"); + let master = root.shift_remove("master"); + let detail = root + .shift_remove("detail") + .and_then(|mut detail| detail.get_mut("main").map(Value::take)) + .filter(|main| !main.is_null()); + + let mut placed = vec![]; + let detail = detail + .map(|main| area(&main, &panels, &mut placed, cx)) + .transpose()? + .flatten(); + + let master_ids = master + .as_ref() + .and_then(|x| x.get("widgets")) + .and_then(Value::as_array) + .map(|widgets| { + widgets + .iter() + .filter_map(Value::as_str) + .filter(|id| panels.contains_key(*id) && !placed.iter().any(|x| x == id)) + .map(str::to_owned) + .collect::>() + }) + .unwrap_or_default(); + + let layout = if master_ids.is_empty() { + detail + } else { + placed.extend(master_ids.iter().cloned()); + let master_sizes = master.as_ref().and_then(|x| x.get("sizes")); + let master_node = if let [id] = master_ids.as_slice() { + tab(vec![id.clone()], None) + } else { + json!({ + "type": "split-layout", + "orientation": "vertical", + "children": master_ids.iter().map(|id| tab(vec![id.clone()], None)).collect::>(), + "sizes": normalize(master_sizes, master_ids.len()), + }) + }; + + cx.approximated(&["layout"], "the master panels are placed on the left"); + cx.approximated( + &["masters"], + "master selections remain in each detail panel's saved `filter`", + ); + + match detail { + Some(detail) => { + let outer = sizes + .as_ref() + .and_then(Value::as_array) + .filter(|x| x.len() == 2) + .map(|_| normalize(sizes.as_ref(), 2)) + .unwrap_or_else(|| DEFAULT_MASTER_SIZES.to_vec()); + + Some(json!({ + "type": "split-layout", + "orientation": "horizontal", + "children": [master_node, detail], + "sizes": outer, + })) + }, + None => Some(master_node), + } + }; + + let orphans = panels + .keys() + .filter(|id| !placed.contains(id)) + .cloned() + .collect::>(); + + for id in orphans { + if let Some(panel) = panels.shift_remove(&id) { + cx.dropped(&["viewers", &id], panel); + } + } + + let mut active = panels + .iter() + .filter(|(_, panel)| panel.get("settings").and_then(Value::as_bool) == Some(true)) + .map(|(id, _)| id.clone()); + + let active = match (active.next(), active.next()) { + (Some(id), None) => Some(id), + _ => None, + }; + + for panel in panels.values_mut() { + if let Value::Object(panel) = panel { + panel.shift_remove("settings"); + } + } + + root.insert("layout".to_owned(), layout.unwrap_or(Value::Null)); + root.insert("panels".to_owned(), Value::Object(panels)); + if !master_ids.is_empty() { + root.insert("masters".to_owned(), Value::from(master_ids)); + } + + if let Some(active) = active { + root.insert("active".to_owned(), Value::String(active)); + } + + Ok(()) +} + +fn area( + node: &Value, + panels: &Map, + placed: &mut Vec, + cx: &StepCx, +) -> Result, MigrateError> { + match node.get("type").and_then(Value::as_str) { + Some("tab-area") => { + let current = node.get("currentIndex").and_then(Value::as_u64); + let mut selected = None; + let mut tabs = vec![]; + for (index, widget) in node + .get("widgets") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + let Some(id) = widget.as_str() else { + continue; + }; + + if panels.contains_key(id) && !placed.iter().any(|x| x == id) { + if current == Some(index as u64) { + selected = Some(tabs.len()); + } + + placed.push(id.to_owned()); + tabs.push(id.to_owned()); + } + } + + Ok((!tabs.is_empty()).then(|| tab(tabs, selected))) + }, + Some("split-area") => { + let orientation = match node.get("orientation").and_then(Value::as_str) { + Some(orientation @ ("horizontal" | "vertical")) => orientation, + _ => return Err(cx.malformed(&["detail"], "a split-area orientation")), + }; + + let sizes = node.get("sizes").and_then(Value::as_array); + let mut children = vec![]; + let mut kept_sizes = vec![]; + for (index, child) in node + .get("children") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + if let Some(child) = area(child, panels, placed, cx)? { + children.push(child); + kept_sizes.push( + sizes + .and_then(|x| x.get(index)) + .cloned() + .unwrap_or(Value::Null), + ); + } + } + + Ok(match children.len() { + 0 => None, + 1 => children.pop(), + len => Some(json!({ + "type": "split-layout", + "orientation": orientation, + "children": children, + "sizes": normalize(Some(&Value::Array(kept_sizes)), len), + })), + }) + }, + _ => Err(cx.malformed(&["detail"], "a tab-area or split-area")), + } +} + +fn tab(tabs: Vec, selected: Option) -> Value { + let mut node = json!({"type": "tab-layout", "tabs": tabs}); + if let Some(selected) = selected { + node["selected"] = Value::from(selected); + } + + node +} + +fn normalize(sizes: Option<&Value>, len: usize) -> Vec { + let sizes = sizes + .and_then(Value::as_array) + .filter(|x| x.len() == len) + .map(|x| { + x.iter() + .map(|x| x.as_f64().unwrap_or(0.0)) + .collect::>() + }) + .filter(|x| x.iter().all(|x| x.is_finite() && *x > 0.0)) + .unwrap_or_else(|| vec![1.0; len]); + + let total = sizes.iter().sum::(); + if (total - 1.0).abs() <= 1e-6 { + sizes + } else { + sizes.iter().map(|x| x / total).collect() + } +} diff --git a/rust/perspective-migrate/tests/corpus.rs b/rust/perspective-migrate/tests/corpus.rs new file mode 100644 index 0000000000..5de14e3b4a --- /dev/null +++ b/rust/perspective-migrate/tests/corpus.rs @@ -0,0 +1,223 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use perspective_migrate::{Kind, Migrator}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const CURRENT: &str = "CURRENT"; + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Case { + kind: Kind, + #[serde(default, skip_serializing_if = "Option::is_none")] + from: Option, + source: String, + input: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, + #[serde(default)] + notices: Vec, + #[serde(default)] + steps: Vec, +} + +fn root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf() +} + +fn corpus() -> Vec<(PathBuf, Case)> { + fn walk(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + walk(&path, out); + } else if path.extension().is_some_and(|x| x == "json") { + out.push(path); + } + } + } + + let mut paths = vec![]; + walk(&root().join("corpus"), &mut paths); + paths.sort(); + paths + .into_iter() + .map(|path| { + let text = std::fs::read_to_string(&path).unwrap(); + let case = + serde_json::from_str(&text).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + (path, case) + }) + .collect() +} + +fn swap_stamps(kind: Kind, value: &mut Value, from: &str, to: &str) { + let replace = |slot: Option<&mut Value>| { + if let Some(slot) = slot + && slot.as_str() == Some(from) + { + *slot = Value::from(to); + } + }; + + replace(value.get_mut("version")); + if kind == Kind::Workspace + && let Some(Value::Object(panels)) = value.get_mut("panels") + { + for panel in panels.values_mut() { + replace(panel.get_mut("version")); + } + } +} + +fn actual(migrator: &Migrator, case: &Case) -> Case { + let mut out = Case { + output: None, + error: None, + notices: vec![], + steps: vec![], + ..case.clone() + }; + + let schema = migrator.schema_version().to_string(); + match migrator.migrate(case.kind, case.from.as_deref(), case.input.clone()) { + Ok(mut migrated) => { + swap_stamps(case.kind, &mut migrated.value, &schema, CURRENT); + + out.output = Some(migrated.value); + out.notices = migrated + .notices + .iter() + .map(|x| serde_json::to_value(x).unwrap()) + .collect(); + out.steps = migrated.steps.iter().map(|x| (*x).to_owned()).collect(); + }, + Err(error) => out.error = Some(serde_json::to_value(error).unwrap()), + } + + out +} + +#[test] +fn corpus_matches() { + let migrator = Migrator::new(); + let bless = std::env::var("PSP_MIGRATE_BLESS").is_ok(); + let mut failures = vec![]; + for (path, case) in corpus() { + let actual = actual(&migrator, &case); + if bless { + let mut text = vec![]; + let formatter = serde_json::ser::PrettyFormatter::with_indent(b" "); + let mut ser = serde_json::Serializer::with_formatter(&mut text, formatter); + actual.serialize(&mut ser).unwrap(); + text.push(b'\n'); + std::fs::write(&path, text).unwrap(); + continue; + } + + let expected = json!({ + "output": case.output, + "error": case.error, + "notices": case.notices, + "steps": case.steps, + }); + + let got = json!({ + "output": actual.output, + "error": actual.error, + "notices": actual.notices, + "steps": actual.steps, + }); + + if expected != got { + failures.push(format!( + "{}\nexpected: {}\n got: {}", + path.display(), + serde_json::to_string_pretty(&expected).unwrap(), + serde_json::to_string_pretty(&got).unwrap() + )); + } + } + + assert!(failures.is_empty(), "{}", failures.join("\n\n")); +} + +#[test] +fn outputs_are_not_migrated_again() { + let migrator = Migrator::new(); + let schema = migrator.schema_version().to_string(); + for (path, case) in corpus() { + let Some(output) = case.output else { + continue; + }; + + let mut input = output.clone(); + swap_stamps(case.kind, &mut input, CURRENT, &schema); + let mut migrated = migrator + .migrate(case.kind, None, input) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + + swap_stamps(case.kind, &mut migrated.value, &schema, CURRENT); + assert_eq!(migrated.value, output, "{}", path.display()); + assert!(migrated.notices.is_empty(), "{}", path.display()); + assert!(migrated.steps.is_empty(), "{}", path.display()); + } +} + +#[test] +fn every_step_is_covered() { + let migrator = Migrator::new(); + let covered = corpus() + .into_iter() + .flat_map(|(_, case)| { + let failed = case + .error + .as_ref() + .and_then(|x| x.get("step")) + .and_then(Value::as_str) + .map(str::to_owned); + + case.steps.into_iter().chain(failed) + }) + .collect::>(); + + let all = migrator + .step_ids() + .into_iter() + .map(str::to_owned) + .collect::>(); + + let missing = all.difference(&covered).collect::>(); + assert!( + missing.is_empty(), + "steps without a corpus case: {missing:?}" + ); +} + +#[test] +fn view_outputs_are_current() { + for (path, case) in corpus() { + if case.kind == Kind::View + && let Some(output) = case.output + { + serde_json::from_value::(output) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + } + } +} diff --git a/rust/perspective-python/Cargo.toml b/rust/perspective-python/Cargo.toml index 5df1f18c13..4557674ba3 100644 --- a/rust/perspective-python/Cargo.toml +++ b/rust/perspective-python/Cargo.toml @@ -58,6 +58,7 @@ pyo3-build-config = "0.29.0" [dependencies] perspective-client = { version = "5.6.1" } +perspective-migrate = { version = "5.6.1" } perspective-server = { version = "5.6.1" } bytes = "1.10.1" chrono = "0.4" @@ -68,6 +69,7 @@ extend = "1.1.2" indexmap = "2.12.1" futures = "0.3.28" serde = { version = "1.0" } +serde_json = { version = "1.0.107" } pyo3 = { version = "0.29.0", features = [ "experimental-async", "extension-module", diff --git a/rust/perspective-python/perspective/__init__.py b/rust/perspective-python/perspective/__init__.py index d8767006f2..81add388c5 100644 --- a/rust/perspective-python/perspective/__init__.py +++ b/rust/perspective-python/perspective/__init__.py @@ -26,6 +26,7 @@ "VirtualServer", "num_cpus", "set_num_cpus", + "migrate", "system_info", ] @@ -363,6 +364,7 @@ def delete_callback(): View, # noqa: F401 num_cpus, set_num_cpus, + migrate, ) diff --git a/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py b/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py index 8f2c21a8db..4c7ca91ffb 100644 --- a/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py +++ b/rust/perspective-python/perspective/tests/multi_threaded/test_multi_threaded.py @@ -147,7 +147,7 @@ def run_perspective(): s = Server() c = s.new_local_client() t = c.table({"a": "string", "b": int}) - t.view(expressions=["//tmp\nfalse"]) + t.view(expressions={"tmp": "false"}) while x > 0: t.update([{"a": "foo", "b": 42}]) x -= 1 diff --git a/rust/perspective-python/perspective/tests/table/test_table_datetime.py b/rust/perspective-python/perspective/tests/table/test_table_datetime.py index c9d432d291..b20ef24bac 100644 --- a/rust/perspective-python/perspective/tests/table/test_table_datetime.py +++ b/rust/perspective-python/perspective/tests/table/test_table_datetime.py @@ -1325,7 +1325,7 @@ def test_table_now_in_EST(self, util): table = Table(data) now = datetime.now() - view = table.view(expressions=["now()"]) + view = table.view(expressions={"now()": "now()"}) result = view.to_columns() for item in result["now()"]: @@ -1343,7 +1343,7 @@ def test_table_now_in_CST(self, util): table = Table(data) now = datetime.now() - view = table.view(expressions=["now()"]) + view = table.view(expressions={"now()": "now()"}) result = view.to_columns() for item in result["now()"]: @@ -1361,7 +1361,7 @@ def test_table_now_in_PST(self, util): table = Table(data) now = datetime.now() - view = table.view(expressions=["now()"]) + view = table.view(expressions={"now()": "now()"}) result = view.to_columns() for item in result["now()"]: @@ -1375,7 +1375,7 @@ def test_table_hour_of_day_in_EST(self): data = {"a": LOCAL_DATETIMES} table = Table(data) - view = table.view(expressions=['hour_of_day("a")']) + view = table.view(expressions={'hour_of_day("a")': 'hour_of_day("a")'}) result = view.to_columns() assert result['hour_of_day("a")'] == [0, 11, 19] @@ -1387,7 +1387,7 @@ def test_table_hour_of_day_in_CST(self): data = {"a": LOCAL_DATETIMES} table = Table(data) - view = table.view(expressions=['hour_of_day("a")']) + view = table.view(expressions={'hour_of_day("a")': 'hour_of_day("a")'}) result = view.to_columns() assert result['hour_of_day("a")'] == [0, 11, 19] @@ -1399,7 +1399,7 @@ def test_table_hour_of_day_in_PST(self): data = {"a": LOCAL_DATETIMES} table = Table(data) - view = table.view(expressions=['hour_of_day("a")']) + view = table.view(expressions={'hour_of_day("a")': 'hour_of_day("a")'}) result = view.to_columns() assert result['hour_of_day("a")'] == [0, 11, 19] @@ -1409,7 +1409,7 @@ def test_table_day_of_week_edge_in_EST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['day_of_week("a")']) + view = table.view(expressions={'day_of_week("a")': 'day_of_week("a")'}) result = view.to_columns() assert result['day_of_week("a")'] == ["6 Friday"] @@ -1420,7 +1420,7 @@ def test_table_day_of_week_edge_in_CST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['day_of_week("a")']) + view = table.view(expressions={'day_of_week("a")': 'day_of_week("a")'}) result = view.to_columns() assert result['day_of_week("a")'] == ["6 Friday"] @@ -1431,7 +1431,7 @@ def test_table_day_of_week_edge_in_PST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['day_of_week("a")']) + view = table.view(expressions={'day_of_week("a")': 'day_of_week("a")'}) result = view.to_columns() assert result['day_of_week("a")'] == ["6 Friday"] @@ -1441,7 +1441,7 @@ def test_table_month_of_year_edge_in_EST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['month_of_year("a")']) + view = table.view(expressions={'month_of_year("a")': 'month_of_year("a")'}) result = view.to_columns() assert result['month_of_year("a")'] == ["01 January"] @@ -1452,7 +1452,7 @@ def test_table_month_of_year_edge_in_CST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['month_of_year("a")']) + view = table.view(expressions={'month_of_year("a")': 'month_of_year("a")'}) result = view.to_columns() assert result['month_of_year("a")'] == ["01 January"] @@ -1463,7 +1463,7 @@ def test_table_month_of_year_edge_in_PST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['month_of_year("a")']) + view = table.view(expressions={'month_of_year("a")': 'month_of_year("a")'}) result = view.to_columns() assert result['month_of_year("a")'] == ["01 January"] @@ -1474,7 +1474,7 @@ def test_table_day_bucket_edge_in_EST(self, util): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'D')"]) + view = table.view(expressions={"bucket(\"a\", 'D')": "bucket(\"a\", 'D')"}) result = view.to_columns() assert result["bucket(\"a\", 'D')"] == [ util.to_timestamp(date(2020, 1, 31)) @@ -1487,7 +1487,7 @@ def test_table_day_bucket_edge_in_CST(self, util): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'D')"]) + view = table.view(expressions={"bucket(\"a\", 'D')": "bucket(\"a\", 'D')"}) result = view.to_columns() assert result["bucket(\"a\", 'D')"] == [ util.to_timestamp(date(2020, 1, 31)) @@ -1500,7 +1500,7 @@ def test_table_day_bucket_edge_in_PST(self, util): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'D')"]) + view = table.view(expressions={"bucket(\"a\", 'D')": "bucket(\"a\", 'D')"}) result = view.to_columns() assert result["bucket(\"a\", 'D')"] == [ util.to_timestamp(date(2020, 1, 31)) @@ -1513,7 +1513,7 @@ def test_table_week_bucket_edge_in_EST(self, util): data = {"a": [datetime(2020, 2, 2, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 1, 27)) @@ -1526,7 +1526,7 @@ def test_table_week_bucket_edge_in_CST(self, util): data = {"a": [datetime(2020, 2, 2, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 1, 27)) @@ -1539,7 +1539,7 @@ def test_table_week_bucket_edge_in_PST(self, util): data = {"a": [datetime(2020, 2, 2, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 1, 27)) @@ -1550,7 +1550,7 @@ def test_table_week_bucket_edge_flip_in_EST(self, util): data = {"a": [datetime(2020, 3, 1, 12, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 2, 24)) @@ -1562,7 +1562,7 @@ def test_table_week_bucket_edge_flip_in_CST(self, util): data = {"a": [datetime(2020, 3, 1, 12, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 2, 24)) @@ -1574,7 +1574,7 @@ def test_table_week_bucket_edge_flip_in_PST(self, util): data = {"a": [datetime(2020, 3, 1, 12, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'W')"]) + view = table.view(expressions={"bucket(\"a\", 'W')": "bucket(\"a\", 'W')"}) result = view.to_columns() assert result["bucket(\"a\", 'W')"] == [ util.to_timestamp(date(2020, 2, 24)) @@ -1587,7 +1587,7 @@ def test_table_month_bucket_edge_in_EST(self, util): data = {"a": [datetime(2020, 6, 30, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'M')"]) + view = table.view(expressions={"bucket(\"a\", 'M')": "bucket(\"a\", 'M')"}) result = view.to_columns() assert result["bucket(\"a\", 'M')"] == [ util.to_timestamp(date(2020, 6, 1)) @@ -1600,7 +1600,7 @@ def test_table_month_bucket_edge_in_CST(self, util): data = {"a": [datetime(2020, 6, 30, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'M')"]) + view = table.view(expressions={"bucket(\"a\", 'M')": "bucket(\"a\", 'M')"}) result = view.to_columns() assert result["bucket(\"a\", 'M')"] == [ util.to_timestamp(date(2020, 6, 1)) @@ -1613,7 +1613,7 @@ def test_table_month_bucket_edge_in_PST(self, util): data = {"a": [datetime(2020, 6, 30, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'M')"]) + view = table.view(expressions={"bucket(\"a\", 'M')": "bucket(\"a\", 'M')"}) result = view.to_columns() assert result["bucket(\"a\", 'M')"] == [ util.to_timestamp(date(2020, 6, 1)) @@ -1626,7 +1626,7 @@ def test_table_year_bucket_edge_in_EST(self, util): data = {"a": [datetime(2019, 12, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'Y')"]) + view = table.view(expressions={"bucket(\"a\", 'Y')": "bucket(\"a\", 'Y')"}) result = view.to_columns() assert result["bucket(\"a\", 'Y')"] == [ util.to_timestamp(date(2019, 1, 1)) @@ -1638,7 +1638,7 @@ def test_table_year_bucket_edge_in_CST(self, util): data = {"a": [datetime(2019, 12, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'Y')"]) + view = table.view(expressions={"bucket(\"a\", 'Y')": "bucket(\"a\", 'Y')"}) result = view.to_columns() assert result["bucket(\"a\", 'Y')"] == [ util.to_timestamp(date(2019, 1, 1)) @@ -1650,7 +1650,7 @@ def test_table_year_bucket_edge_in_PST(self, util): data = {"a": [datetime(2019, 12, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'Y')"]) + view = table.view(expressions={"bucket(\"a\", 'Y')": "bucket(\"a\", 'Y')"}) result = view.to_columns() assert result["bucket(\"a\", 'Y')"] == [ util.to_timestamp(date(2019, 1, 1)) @@ -2494,7 +2494,7 @@ def test_table_day_bucket_edge_in_JST(self, util): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=["bucket(\"a\", 'D')"]) + view = table.view(expressions={"bucket(\"a\", 'D')": "bucket(\"a\", 'D')"}) result = view.to_columns() assert result["bucket(\"a\", 'D')"] == [ util.to_timestamp(date(2020, 1, 31)) @@ -2506,7 +2506,7 @@ def test_table_day_of_week_edge_in_JST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['day_of_week("a")']) + view = table.view(expressions={'day_of_week("a")': 'day_of_week("a")'}) result = view.to_columns() assert result['day_of_week("a")'] == ["6 Friday"] @@ -2516,7 +2516,7 @@ def test_table_month_of_year_edge_in_JST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['month_of_year("a")']) + view = table.view(expressions={'month_of_year("a")': 'month_of_year("a")'}) result = view.to_columns() assert result['month_of_year("a")'] == ["01 January"] @@ -2526,7 +2526,7 @@ def test_table_hour_of_day_edge_in_JST(self): data = {"a": [datetime(2020, 1, 31, 23, 59)]} table = Table(data) - view = table.view(expressions=['hour_of_day("a")']) + view = table.view(expressions={'hour_of_day("a")': 'hour_of_day("a")'}) result = view.to_columns() assert result['hour_of_day("a")'] == [23] diff --git a/rust/perspective-python/perspective/tests/table/test_view_expression.py b/rust/perspective-python/perspective/tests/table/test_view_expression.py index 9f88f1ebea..359cdf12f8 100644 --- a/rust/perspective-python/perspective/tests/table/test_view_expression.py +++ b/rust/perspective-python/perspective/tests/table/test_view_expression.py @@ -33,7 +33,7 @@ def randstr(length, input=ascii_letters): class TestViewExpression(object): def test_table_validate_expressions_empty(self): table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) - validate = table.validate_expressions([]) + validate = table.validate_expressions({}) assert validate["expression_schema"] == {} assert validate["expression_alias"] == {} assert validate["errors"] == {} @@ -150,7 +150,7 @@ def test_view_expression_schema_all_types(self): "length('abcd')", ] - view = table.view(expressions=expressions) + view = table.view(expressions={expr: expr for expr in expressions}) assert view.expression_schema() == { '"a"': "integer", '"b" * 0.5': "float", @@ -178,14 +178,14 @@ def test_view_expression_schema_all_types(self): "length('abcd')": [4 for _ in range(4)], } - validated = table.validate_expressions(expressions) + validated = table.validate_expressions({expr: expr for expr in expressions}) for expr in expressions: assert validated["expression_alias"][expr] == expr def test_table_validate_expressions_with_errors(self): table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) expressions = ['"Sales" + "a"', "datetime()", "string()", "for () {}"] - validate = table.validate_expressions(expressions) + validate = table.validate_expressions({expr: expr for expr in expressions}) assert validate["expression_schema"] == {} assert validate["expression_alias"] == {expr: expr for expr in expressions} assert validate["errors"] == { @@ -291,13 +291,17 @@ def make_expression(idx): return { "expression_name": expr[0][2:], - "expression": "\n".join(expr), + "expression": "\n".join(expr[1:]), "output": "".join(concat_result), } expressions = [make_expression(i) for i in range(10)] - view = table.view(expressions=[expr["expression"] for expr in expressions]) + view = table.view( + expressions={ + expr["expression_name"]: expr["expression"] for expr in expressions + } + ) result = view.to_columns() schema = view.expression_schema() @@ -608,9 +612,9 @@ def test_view_expression_string_literal_var(self): for _ in range(10): view = table.view( - expressions=[ - "var x := 'Eabcdefghijklmn'; var y := '0123456789'; concat(x, y)" - ] + expressions={ + "var x := 'Eabcdefghijklmn'; var y := '0123456789'; concat(x, y)": "var x := 'Eabcdefghijklmn'; var y := '0123456789'; concat(x, y)" + } ) assert view.to_columns() == { "a": [1, 2, 3], @@ -626,7 +630,7 @@ def data(): return [{"a": random()} for _ in range(50)] table = Table(data()) - view = table.view(expressions=["123"]) + view = table.view(expressions={"123": "123"}) for _ in range(5): table.update(data()) @@ -640,7 +644,7 @@ def data(): return [{"a": random()} for _ in range(55)] table = Table(data(), limit=50) - view = table.view(expressions=["123"]) + view = table.view(expressions={"123": "123"}) for _ in range(5): table.update(data()) @@ -679,7 +683,7 @@ def data(): def test_view_expression_create_no_alias(self): table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) - view = table.view(expressions=['"a" + "b"']) + view = table.view(expressions={'"a" + "b"': '"a" + "b"'}) assert view.to_columns() == { "a": [1, 2, 3, 4], "b": [5, 6, 7, 8], @@ -698,14 +702,13 @@ def test_view_expression_should_not_overwrite_real(self): == 'Abort(): Value Error - expression "a" cannot overwrite an existing column.' ) - def test_legacy_view_duplicate_expression_should_resolve_to_last_alias(self): + def test_legacy_view_expression_list_is_rejected(self): table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) - view = table.view( - columns=["abc"], - expressions=['//abc\n"a" + "b"', '//abc\n"a" - "b"'], - ) - - assert view.to_columns() == {"abc": [-4, -4, -4, -4]} + with pytest.raises(Exception): + table.view( + columns=["abc"], + expressions=['//abc\n"a" + "b"', '//abc\n"a" - "b"'], + ) def test_view_expression_multiple_alias( self, @@ -1754,7 +1757,11 @@ def test_view_string_expression(self): def test_view_expession_multicomment(self): table = Table({"a": [1, 2, 3, 4]}) - view = table.view(expressions=["var x := 1 + 2;\n// def\nx + 100 // cdefghijk"]) + view = table.view( + expressions={ + "var x := 1 + 2;\n// def\nx + 100 // cdefghijk": "var x := 1 + 2;\n// def\nx + 100 // cdefghijk" + } + ) assert view.expression_schema() == { "var x := 1 + 2;\n// def\nx + 100 // cdefghijk": "float" } @@ -1976,16 +1983,12 @@ def digits(): # XXX: This test was broken because it thought `b` was an integer column. table = Table({"a": "string", "b": "string"}) table.update({"a": data, "b": [str(i) for i in range(1000)]}) - expressions = [ - """//w - replace('abc-def-hijk', '-', '')""", - """//x - replace("a", '[0-9]{4}$', "b")""", - """//y - replace("a", '[a-z]{4}$', "b")""", - """//z - var x := 'long string, very cool!'; replace("a", '^[0-9]{4}', x)""", - ] + expressions = { + "w": """replace('abc-def-hijk', '-', '')""", + "x": """replace("a", '[0-9]{4}$', "b")""", + "y": """replace("a", '[a-z]{4}$', "b")""", + "z": """var x := 'long string, very cool!'; replace("a", '^[0-9]{4}', x)""", + } validate = table.validate_expressions(expressions) assert validate["expression_schema"] == { @@ -2017,18 +2020,13 @@ def digits(): def test_view_replace_invalid(self): table = Table({"a": "string", "b": "string"}) - expressions = [ - """//v - replace('abc-def-hijk', '-', 123)""", - """//w - replace('', '-', today())""", - """//x - replace("a", '[0-9]{4}$', today())""", - """//y - replace("a", '[a-z]{4}$', null)""", - """//z - var x := 123; replace("a", '^[0-9]{4}', x)""", - ] + expressions = { + "v": """replace('abc-def-hijk', '-', 123)""", + "w": """replace('', '-', today())""", + "x": """replace("a", '[0-9]{4}$', today())""", + "y": """replace("a", '[a-z]{4}$', null)""", + "z": """var x := 123; replace("a", '^[0-9]{4}', x)""", + } validate = table.validate_expressions(expressions) assert validate["expression_schema"] == {} @@ -2053,16 +2051,12 @@ def digits(): ) table = Table({"a": "string", "b": "string"}) table.update({"a": data, "b": [str(i) for i in range(1000)]}) - expressions = [ - """//w - replace_all('abc-def-hijk', '-', '')""", - """//x - replace_all("a", '[0-9]{4}$', "b")""", - """//y - replace_all("a", '[a-z]{4}$', "b")""", - """//z - var x := 'long string, very cool!'; replace_all("a", '^[0-9]{4}', x)""", - ] + expressions = { + "w": """replace_all('abc-def-hijk', '-', '')""", + "x": """replace_all("a", '[0-9]{4}$', "b")""", + "y": """replace_all("a", '[a-z]{4}$', "b")""", + "z": """var x := 'long string, very cool!'; replace_all("a", '^[0-9]{4}', x)""", + } validate = table.validate_expressions(expressions) assert validate["expression_schema"] == { @@ -2095,18 +2089,13 @@ def digits(): def test_view_replace_invalid_variation(self): table = Table({"a": "string", "b": "string"}) - expressions = [ - """//v - replace_all('abc-def-hijk', '-', 123)""", - """//w - replace_all('', '-', today())""", - """//x - replace_all("a", '[0-9]{4}$', today())""", - """//y - replace_all("a", '[a-z]{4}$', null)""", - """//z - var x := 123; replace_all("a", '^[0-9]{4}', x)""", - ] + expressions = { + "v": """replace_all('abc-def-hijk', '-', 123)""", + "w": """replace_all('', '-', today())""", + "x": """replace_all("a", '[0-9]{4}$', today())""", + "y": """replace_all("a", '[a-z]{4}$', null)""", + "z": """var x := 123; replace_all("a", '^[0-9]{4}', x)""", + } validate = table.validate_expressions(expressions) assert validate["expression_schema"] == {} diff --git a/rust/perspective-python/perspective/tests/test_migrate.py b/rust/perspective-python/perspective/tests/test_migrate.py new file mode 100644 index 0000000000..693771a2ef --- /dev/null +++ b/rust/perspective-python/perspective/tests/test_migrate.py @@ -0,0 +1,61 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import pytest + +import perspective +from perspective import PerspectiveError, migrate + +V2_VIEW = { + "row_pivot": ["State"], + "expressions": ['//profit\n"Sales" - "Cost"'], +} + + +class TestMigrate: + def test_upgrades_a_2x_engine_config(self): + assert migrate("view", V2_VIEW, "2.6.0") == { + "value": { + "group_by": ["State"], + "expressions": {"profit": '"Sales" - "Cost"'}, + }, + "notices": [], + "steps": ["view.v2_aliases", "view.expressions_map"], + } + + def test_an_upgraded_config_creates_a_view(self): + client = perspective.Server().new_local_client() + table = client.table({"State": ["NY", "NJ"], "Sales": [5, 7], "Cost": [1, 2]}) + view = table.view(**migrate("view", V2_VIEW, "2.6.0")["value"]) + assert view.to_columns()["profit"] == [9, 5, 4] + + def test_a_token_version_overrides_from_version(self): + result = migrate( + "viewer", + { + "version": "5.4.0", + "plugin": "Datagrid", + "columns_config": {"Sales": {"number_fg_mode": "bar"}}, + }, + from_version="2.0.0", + ) + + assert result["value"]["columns_config"] == {"Sales": {"fg_mode": "bar"}} + assert result["steps"] == ["datagrid.mode_renames"] + + def test_rejects_an_unknown_kind(self): + with pytest.raises(ValueError, match="Unknown config kind"): + migrate("table", {}) + + def test_names_the_step_which_rejected_a_config(self): + with pytest.raises(PerspectiveError, match="view.v2_aliases"): + migrate("view", {"row_pivot": ["a"], "group_by": ["b"]}, "2.0.0") diff --git a/rust/perspective-python/perspective/widget/__init__.py b/rust/perspective-python/perspective/widget/__init__.py index 3a0545c1c4..12be8900fe 100644 --- a/rust/perspective-python/perspective/widget/__init__.py +++ b/rust/perspective-python/perspective/widget/__init__.py @@ -169,7 +169,7 @@ def __init__( ... group_by=["a"], ... sort=[["b", "desc"]], ... filter=[["a", ">", 1]], - ... expressions=["\"a\" + 100"]) + ... expressions={"plus_100": "\"a\" + 100"}) """ self.binding_mode = binding_mode diff --git a/rust/perspective-python/perspective/widget/viewer/viewer.py b/rust/perspective-python/perspective/widget/viewer/viewer.py index 50ef157a7a..5112d244b3 100644 --- a/rust/perspective-python/perspective/widget/viewer/viewer.py +++ b/rust/perspective-python/perspective/widget/viewer/viewer.py @@ -96,8 +96,9 @@ def __init__( filter (:obj:`list` of :obj:`list` of :obj:`str`): A list of lists, each list containing a column name, a filter comparator, and a value to filter by. - expressions (:obj:`list` of :obj:`str`): A list of string - expressions which are applied to the view. + expressions (:obj:`dict` of :obj:`str` to :obj:`str`): A map of + expression column names to the expressions which are applied + to the view. plugin (:obj:`str`/:obj:`perspective.Plugin`): Which plugin to select by default. plugin_config (:obj:`dict`): A configuration for the plugin, i.e. @@ -115,7 +116,7 @@ def __init__( ... group_by=["a"], ... sort=[["b", "desc"]], ... filter=[["a", ">", 1]], - ... expressions=["\"a\" + 100"] + ... expressions={"plus_100": "\"a\" + 100"} ... ) """ diff --git a/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py b/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py index d00cea786f..e11e9fa934 100644 --- a/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py +++ b/rust/perspective-python/perspective/widget/viewer/viewer_traitlets.py @@ -44,7 +44,7 @@ class PerspectiveTraitlets(HasTraits): aggregates = Dict(default_value={}).tag(sync=True) sort = List(default_value=[]).tag(sync=True) filter = List(default_value=[]).tag(sync=True) - expressions = Dict(default_value=[]).tag(sync=True) + expressions = Dict(default_value={}).tag(sync=True) plugin_config = Dict(default_value={}).tag(sync=True) settings = Bool(True).tag(sync=True) theme = Unicode("Pro Light", allow_none=True).tag(sync=True) diff --git a/rust/perspective-python/src/lib.rs b/rust/perspective-python/src/lib.rs index 6485a7fb53..e80a4c797b 100644 --- a/rust/perspective-python/src/lib.rs +++ b/rust/perspective-python/src/lib.rs @@ -14,6 +14,7 @@ #![warn(unstable_features)] mod client; +mod migrate; pub(crate) mod py_async; mod py_err; mod server; @@ -77,5 +78,6 @@ fn perspective(py: Python, m: &Bound) -> PyResult<()> { m.add("PerspectiveError", py.get_type::())?; m.add_function(wrap_pyfunction!(num_cpus, m)?)?; m.add_function(wrap_pyfunction!(set_num_cpus, m)?)?; + m.add_function(wrap_pyfunction!(migrate::migrate, m)?)?; Ok(()) } diff --git a/rust/perspective-python/src/migrate.rs b/rust/perspective-python/src/migrate.rs new file mode 100644 index 0000000000..fa53a72e04 --- /dev/null +++ b/rust/perspective-python/src/migrate.rs @@ -0,0 +1,59 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use perspective_migrate::{Kind, Migrator}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +use crate::py_err::PyPerspectiveError; + +/// Upgrade a persisted Perspective config written by any release since 2.0.0 +/// to the current schema, returning `{"value", "notices", "steps"}`. +/// +/// # Arguments +/// +/// - `kind` - `"view"`, `"viewer"` or `"workspace"`. +/// - `config` - The config, as a `dict`. +/// - `from_version` - The release that wrote `config`, used only when it has no +/// `version` field. +#[pyfunction] +#[pyo3(signature = (kind, config, from_version = None))] +pub fn migrate( + py: Python<'_>, + kind: &str, + config: &Bound<'_, PyAny>, + from_version: Option<&str>, +) -> PyResult> { + let kind = match kind { + "view" => Kind::View, + "viewer" => Kind::Viewer, + "workspace" => Kind::Workspace, + _ => { + return Err(PyValueError::new_err(format!( + "Unknown config kind `{kind}`; expected `view`, `viewer` or `workspace`" + ))); + }, + }; + + let value: serde_json::Value = pythonize::depythonize(config)?; + let migrated = Migrator::new() + .migrate(kind, from_version, value) + .map_err(|error| PyPerspectiveError::new_err(error.to_string()))?; + + let result = serde_json::json!({ + "value": migrated.value, + "notices": migrated.notices, + "steps": migrated.steps, + }); + + Ok(pythonize::pythonize(py, &result)?.unbind()) +} diff --git a/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp b/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp index cdc04fe0f4..0bdcf85987 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp @@ -186,7 +186,7 @@ t_aggspec::agg_str() const { return "or"; } break; case AGGTYPE_LAST_VALUE: { - return "last value"; + return "last"; } case AGGTYPE_MAX_BY: { return "max by"; @@ -201,13 +201,13 @@ t_aggspec::agg_str() const { return "min"; } case AGGTYPE_HIGH_WATER_MARK: { - return "high water mark"; + return "high"; } case AGGTYPE_LOW_WATER_MARK: { - return "low water mark"; + return "low"; } case AGGTYPE_HIGH_MINUS_LOW: { - return "high minuslow"; + return "high minus low"; } break; case AGGTYPE_UDF_COMBINER: { std::stringstream ss; @@ -242,7 +242,7 @@ t_aggspec::agg_str() const { return "pct sum parent"; } case AGGTYPE_PCT_SUM_GRAND_TOTAL: { - return "pct sum grand total"; + return "pct sum total"; } case AGGTYPE_VARIANCE: { return "variance"; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp index 3922fc9065..fe3e4dd8cf 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/base.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/base.cpp @@ -431,10 +431,10 @@ str_to_filter_op(const std::string& str) { if (str == "|" || str == "or") { return t_filter_op::FILTER_OP_OR; } - if (str == "is null" || str == "is None") { + if (str == "is null") { return t_filter_op::FILTER_OP_IS_NULL; } - if (str == "is not null" || str == "is not None") { + if (str == "is not null") { return t_filter_op::FILTER_OP_IS_NOT_NULL; } @@ -591,9 +591,6 @@ maybe_str_to_aggtype(const std::string& str) { if (str == "sum or zero" || str == "sum_or_zero") { return t_aggtype::AGGTYPE_SUM_OR_ZERO; } - if (str == "sum not null" || str == "sum_not_null") { - return t_aggtype::AGGTYPE_SUM; - } if (str == "mean by count" || str == "mean_by_count") { return t_aggtype::AGGTYPE_MEAN_BY_COUNT; } @@ -606,8 +603,7 @@ maybe_str_to_aggtype(const std::string& str) { if (str == "pct sum parent" || str == "pct_sum_parent") { return t_aggtype::AGGTYPE_PCT_SUM_PARENT; } - if (str == "pct sum total" || str == "pct_sum_total" - || str == "pct sum grand total" || str == "pct_sum_grand_total") { + if (str == "pct sum total" || str == "pct_sum_total") { return t_aggtype::AGGTYPE_PCT_SUM_GRAND_TOTAL; } if (str.find("udf_combiner_") != std::string::npos) { diff --git a/rust/perspective-viewer/src/rust/components/viewer/panels.rs b/rust/perspective-viewer/src/rust/components/viewer/panels.rs index 1349847c51..d878345495 100644 --- a/rust/perspective-viewer/src/rust/components/viewer/panels.rs +++ b/rust/perspective-viewer/src/rust/components/viewer/panels.rs @@ -19,6 +19,7 @@ //! [`Workspace`]: crate::workspace::Workspace use perspective_js::utils::*; +use wasm_bindgen::JsValue; use yew::prelude::*; use super::PerspectiveViewer; @@ -228,7 +229,7 @@ impl PerspectiveViewer { .await?; // TODO(texodus): what the ****? - let update = ViewerConfigInitial::decode(&config.encode()?)?; + let update = ViewerConfigInitial::decode(&JsValue::from_serde_ext(&config.panel)?)?; let client = panel.session.get_client(); let new_id = create_panel( &elem, diff --git a/rust/perspective-viewer/src/rust/config/datetime_column_style.rs b/rust/perspective-viewer/src/rust/config/datetime_column_style.rs index b477db0e65..e3146cd4ed 100644 --- a/rust/perspective-viewer/src/rust/config/datetime_column_style.rs +++ b/rust/perspective-viewer/src/rust/config/datetime_column_style.rs @@ -10,11 +10,9 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -mod migrate; mod part_format; mod simple_format; -pub use migrate::*; pub use part_format::*; use serde::{Deserialize, Serialize}; pub use simple_format::*; diff --git a/rust/perspective-viewer/src/rust/config/datetime_column_style/migrate.rs b/rust/perspective-viewer/src/rust/config/datetime_column_style/migrate.rs deleted file mode 100644 index abc570f292..0000000000 --- a/rust/perspective-viewer/src/rust/config/datetime_column_style/migrate.rs +++ /dev/null @@ -1,95 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -use serde_json::{Map, Value}; - -use crate::config::API_VERSION; -use crate::renderer::ColumnConfigMap; - -/// The first release whose stored `date_format` fields inherit when missing. -const INHERITING_RELEASE: (u64, u64, u64) = (5, 7, 0); - -const PART_FALLBACKS: [(&str, &str); 6] = [ - ("year", "2-digit"), - ("month", "numeric"), - ("day", "numeric"), - ("hour", "numeric"), - ("minute", "numeric"), - ("second", "numeric"), -]; - -const SHAPE_KEYS: [&str; 10] = [ - "dateStyle", - "timeStyle", - "weekday", - "year", - "month", - "day", - "hour", - "minute", - "second", - "fractionalSecondDigits", -]; - -/// Rewrite every `date_format` in `columns` written before 5.7 into the form -/// that renders identically under inheriting semantics. -pub fn migrate_date_formats(version: Option<&str>, columns: &mut ColumnConfigMap) { - let versioned_legacy = version.is_some_and(is_legacy_version); - for entry in columns.values_mut() { - if let Some(Value::Object(format)) = entry.get_mut("date_format") { - migrate_date_format(format, versioned_legacy); - } - } -} - -fn is_legacy_version(version: &str) -> bool { - version != *API_VERSION && parse_version(version).is_some_and(|v| v < INHERITING_RELEASE) -} - -fn parse_version(version: &str) -> Option<(u64, u64, u64)> { - let core = version.split(['-', '+']).next()?; - let mut parts = core.split('.').map(|x| x.parse::().ok()); - Some((parts.next()??, parts.next()??, parts.next()??)) -} - -fn is_disabled(value: &Value) -> bool { - value.as_str() == Some("disabled") -} - -fn migrate_date_format(format: &mut Map, versioned_legacy: bool) { - if format.remove("format").is_some() { - for (key, fallback) in PART_FALLBACKS { - format.entry(key).or_insert_with(|| Value::from(fallback)); - } - - format.entry("hour12").or_insert(Value::Bool(true)); - if format.get("fractionalSecondDigits").and_then(Value::as_u64) == Some(0) { - format.remove("fractionalSecondDigits"); - } - } else if versioned_legacy || format.values().any(is_disabled) { - format - .entry("dateStyle") - .or_insert_with(|| Value::from("short")); - format - .entry("timeStyle") - .or_insert_with(|| Value::from("medium")); - } else { - return; - } - - format.retain(|_, value| !is_disabled(value)); - if !SHAPE_KEYS.iter().any(|key| format.contains_key(*key)) { - for key in ["year", "month", "day"] { - format.insert(key.to_owned(), Value::from("numeric")); - } - } -} diff --git a/rust/perspective-viewer/src/rust/config/number_string_format/enums.rs b/rust/perspective-viewer/src/rust/config/number_string_format/enums.rs index 7b4fcdb876..db385ede81 100644 --- a/rust/perspective-viewer/src/rust/config/number_string_format/enums.rs +++ b/rust/perspective-viewer/src/rust/config/number_string_format/enums.rs @@ -224,7 +224,7 @@ pub enum Unit { FluidOunce, #[serde(rename = "foot")] Foot, - #[serde(rename = "gallob")] + #[serde(rename = "gallon")] Gallon, #[serde(rename = "gigabit")] Gigabit, diff --git a/rust/perspective-viewer/src/rust/config/viewer_config.rs b/rust/perspective-viewer/src/rust/config/viewer_config.rs index 70bad7e734..0b1176e4fe 100644 --- a/rust/perspective-viewer/src/rust/config/viewer_config.rs +++ b/rust/perspective-viewer/src/rust/config/viewer_config.rs @@ -20,7 +20,7 @@ use serde_json::Value; use ts_rs::TS; use wasm_bindgen::prelude::*; -use crate::config::{TableRef, migrate_date_formats}; +use crate::config::TableRef; use crate::renderer::ColumnConfigMap; /// The state of an entire `custom_elements::PerspectiveViewerElement` component @@ -44,8 +44,8 @@ pub struct ViewerConfig { #[derive(Debug, Default, Serialize, PartialEq, TS)] pub struct PanelViewerConfig { /// The `@perspective-dev/viewer` version that wrote this config, - /// stamped on save and used to migrate older tokens. Callers do not - /// set it. + /// stamped on save and named in the error when such a config is rejected. + /// Callers do not set it. pub version: V, /// Per-column styling, keyed by column name. The viewer treats the @@ -129,6 +129,71 @@ pub static API_VERSION: LazyLock<&'static str> = LazyLock::new(|| { pkg.version }); +/// The `version` a raw config, or else one of its workspace panels, was saved +/// by. +pub fn saved_version(config: &JsValue) -> Option { + let version = |value: &JsValue| { + js_sys::Reflect::get(value, &JsValue::from_str("version")) + .ok()? + .as_string() + }; + + version(config).or_else(|| { + ["panels", "viewers"].into_iter().find_map(|key| { + let panels = js_sys::Reflect::get(config, &JsValue::from_str(key)).ok()?; + let panels = panels.dyn_into::().ok()?; + js_sys::Object::values(&panels) + .iter() + .find_map(|panel| version(&panel)) + }) + }) +} + +/// Await `result`, naming the release a config was `saved` by in its error +/// when that is not this release. +pub async fn noting_saved_version( + saved: Option, + result: impl std::future::Future>, +) -> ApiResult { + result.await.map_err(|error| match saved { + Some(version) if version != *API_VERSION => ApiError::new(format!( + "{error} (this config was saved by {version}, and this is {}; configs from older \ + releases can be upgraded with `perspective-migrate`)", + *API_VERSION + )), + _ => error, + }) +} + +/// A trailing `#[serde(flatten)]` field which fails deserialization when its +/// struct is given a key it does not define. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct UnknownKeys; + +impl<'de> Deserialize<'de> for UnknownKeys { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let keys = + std::collections::BTreeMap::::deserialize(deserializer)?; + + if keys.is_empty() { + Ok(Self) + } else { + let keys = keys + .keys() + .map(|key| format!("`{key}`")) + .collect::>() + .join(", "); + + Err(serde::de::Error::custom(format!( + "Unknown config keys {keys}" + ))) + } + } +} + impl ViewerConfig { /// Encode a `ViewerConfig` to a `JsValue` in a supported type. pub fn encode(&self) -> ApiResult { @@ -151,8 +216,8 @@ impl Deref for PluginConfig { #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] pub struct ViewerConfigUpdate { /// The `@perspective-dev/viewer` version a saved config was written - /// by, used to migrate older tokens. Omit it — the viewer stamps - /// the current version on save. + /// by, named in the error when such a config is rejected. Omit it — the + /// viewer stamps the current version on save. #[serde(default)] #[ts(as = "Option<_>")] #[ts(optional)] @@ -225,27 +290,17 @@ pub struct ViewerConfigUpdate { #[serde(flatten)] pub view_config: ViewConfigUpdate, + + #[serde(flatten, skip_serializing)] + #[ts(skip)] + pub unknown: UnknownKeys, } impl ViewerConfigUpdate { - /// Decode a `JsValue` into a `ViewerConfigUpdate` by auto-detecting format - /// from JavaScript type. + /// Decode a `JsValue` into a `ViewerConfigUpdate`, rejecting unknown keys. pub fn decode(update: &JsValue) -> ApiResult { Ok(update.into_serde_ext()?) } - - /// Rewrite the parts of this config written by an older viewer into their - /// current form. - pub fn migrate(&mut self) { - if let OptionalUpdate::Update(columns) = &mut self.columns_config { - let version = match &self.version { - OptionalUpdate::Update(version) => Some(version.as_str()), - _ => None, - }; - - migrate_date_formats(version, columns); - } - } } /// The initial configuration of a NEW panel (`addPanel`, `restore`'s @@ -301,22 +356,18 @@ pub struct ViewerConfigInitial { #[serde(flatten)] pub view_config: ViewConfigUpdate, + + #[serde(flatten, skip_serializing)] + #[ts(skip)] + pub unknown: UnknownKeys, } impl ViewerConfigInitial { - /// Decode a `JsValue` by auto-detecting format from JavaScript type. + /// Decode a `JsValue`, rejecting unknown keys. pub fn decode(config: &JsValue) -> ApiResult { Ok(config.into_serde_ext()?) } - /// Rewrite the parts of this config written by an older viewer into their - /// current form. - pub fn migrate(&mut self) { - if let Some(columns) = &mut self.columns_config { - migrate_date_formats(self.version.as_deref(), columns); - } - } - /// A default-config panel bound to `table` (the "New panel" menus). /// Exhaustive on purpose — same drift alarm as the `From` impl below. pub fn new(table: impl Into) -> Self { @@ -329,6 +380,7 @@ impl ViewerConfigInitial { plugin_config: None, columns_config: None, view_config: ViewConfigUpdate::default(), + unknown: UnknownKeys, } } } @@ -355,6 +407,7 @@ impl From for ViewerConfigUpdate { plugin_config: up(value.plugin_config), columns_config: up(value.columns_config), view_config: value.view_config, + unknown: value.unknown, } } } @@ -388,6 +441,7 @@ impl TryFrom for ViewerConfigInitial { plugin_config, columns_config, view_config, + unknown, } = value; let OptionalUpdate::Update(table) = table else { @@ -405,6 +459,7 @@ impl TryFrom for ViewerConfigInitial { plugin_config: down(plugin_config), columns_config: down(columns_config), view_config, + unknown, }) } } diff --git a/rust/perspective-viewer/src/rust/config/workspace_config.rs b/rust/perspective-viewer/src/rust/config/workspace_config.rs index b69ce8e0bf..d805d0718d 100644 --- a/rust/perspective-viewer/src/rust/config/workspace_config.rs +++ b/rust/perspective-viewer/src/rust/config/workspace_config.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; use perspective_client::config::{Filter, ViewConfig}; -use crate::config::{OptionalUpdate, PanelViewerConfig, ViewerConfigInitial}; +use crate::config::{OptionalUpdate, PanelViewerConfig, UnknownKeys, ViewerConfigInitial}; use crate::utils::CssKind; /// A derived `Table`'s definition: the `Table` its source `View` reads, and @@ -29,6 +29,10 @@ pub struct DerivedTableConfig { #[serde(flatten)] pub view_config: ViewConfig, + + #[serde(flatten, skip_serializing)] + #[ts(skip)] + pub unknown: UnknownKeys, } /// What may occupy a `table` position in a single-panel config: a hosted @@ -183,7 +187,7 @@ pub struct WorkspaceConfig { /// `save()` outputs byte-unstable. pub panels: BTreeMap>, - /// Named derived `Table`s, each a `View` of its `parent` promoted to a + /// Named derived `Table`s, each a `View` of its `table` promoted to a /// `Table`. A `BTreeMap` for the same determinism reason as `panels`. #[serde(skip_serializing_if = "BTreeMap::is_empty")] #[ts(as = "Option<_>")] @@ -222,6 +226,12 @@ pub struct WorkspaceConfig { /// replaces. #[derive(serde::Deserialize, ts_rs::TS)] pub struct WorkspaceConfigUpdate { + /// The `@perspective-dev/viewer` version a saved workspace was written + /// by, named in the error when such a workspace is rejected. + #[serde(default)] + #[ts(optional)] + pub version: Option, + /// The panel to activate with the settings sidebar open, by `panels` key /// or existing panel id; `null` closes the sidebar. #[serde(default)] @@ -270,6 +280,10 @@ pub struct WorkspaceConfigUpdate { #[ts(as = "Option<_>")] #[ts(optional)] pub palette: OptionalUpdate>, + + #[serde(flatten)] + #[ts(skip)] + pub unknown: UnknownKeys, } /// Validate a restored palette map: each key's `--psp-user---` diff --git a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs index daf1a8c4f7..5c4a48580a 100644 --- a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs +++ b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs @@ -1048,7 +1048,9 @@ impl PerspectiveViewerElement { /// /// This restores a SINGLE panel; a workspace config (with a `panels` /// map) must be applied via [`Self::restoreWorkspace`] — its `panels` / - /// `layout` keys are ignored here. + /// `layout` keys are rejected here, as is any key the config format does + /// not define; a rejected config stamped by another release names that + /// release in the error (upgrade those with `perspective-migrate`). /// /// One of the best ways to use [`Self::restore`] is by first configuring /// a `` as you wish, then using either the `Debug` @@ -1057,8 +1059,7 @@ impl PerspectiveViewerElement { /// /// # Arguments /// - /// - `update` - The config to restore to, as returned by [`Self::save`] in - /// either "json", "string" or "arraybuffer" format. + /// - `update` - The config to restore to, as returned by [`Self::save`]. /// - `options.panel` - The panel to target, or the active panel when /// omitted. /// - `options.suppress_errors` - when `true`, a config that was applied and @@ -1090,7 +1091,8 @@ impl PerspectiveViewerElement { ) -> JsVoidPromise { let effect = self.workspace.effects().guard(); let this = self.clone(); - let fut = ApiFuture::new_throttled(async move { + let saved = saved_version(&update); + let fut = ApiFuture::new_throttled(noting_saved_version(saved, async move { let RestoreOptions { panel: name, suppress_errors, @@ -1108,7 +1110,6 @@ impl PerspectiveViewerElement { let _effect = effect; let id = name.map(PanelId::from); let mut update = ViewerConfigUpdate::decode(&update)?; - update.migrate(); let settings = std::mem::replace(&mut update.settings, OptionalUpdate::Missing); enum Target { Existing { panel: Panel, active: bool }, @@ -1206,7 +1207,7 @@ impl PerspectiveViewerElement { Ok(()) }, } - }); + })); js_sys::Promise::from(fut).unchecked_into() } @@ -1243,7 +1244,8 @@ impl PerspectiveViewerElement { ) -> JsVoidPromise { let effect = self.workspace.effects().guard(); let this = self.clone(); - let fut = ApiFuture::new(async move { + let saved = saved_version(&update); + let fut = ApiFuture::new(noting_saved_version(saved, async move { let _effect = effect; let RestoreWorkspaceOptions { wait_for_table } = parse_options(options)?; let missing = MissingTable::from_wait(wait_for_table); @@ -1308,7 +1310,7 @@ impl PerspectiveViewerElement { ejected?; Ok(()) - }); + })); js_sys::Promise::from(fut).unchecked_into() } @@ -2024,11 +2026,11 @@ impl PerspectiveViewerElement { clone!(self.elem, self.presentation, self.workspace); let effect = workspace.effects().guard(); let notify = self.layout_changed_notify(); - ApiFuture::new(async move { + let saved = saved_version(&config); + ApiFuture::new(noting_saved_version(saved, async move { let _effect = effect; let AddPanelOptions { wait_for_table } = parse_options(options)?; - let mut config = ViewerConfigInitial::decode(&config)?; - config.migrate(); + let config = ViewerConfigInitial::decode(&config)?; let id = create_panel( &elem, &presentation, @@ -2041,7 +2043,7 @@ impl PerspectiveViewerElement { ) .await?; Ok(JsValue::from_str(id.as_str())) - }) + })) } /// Get the ids of all panels in this viewer's layout, in insertion order. diff --git a/rust/perspective-viewer/src/rust/derived.rs b/rust/perspective-viewer/src/rust/derived.rs index 6fc7f1b902..e8cb5dec88 100644 --- a/rust/perspective-viewer/src/rust/derived.rs +++ b/rust/perspective-viewer/src/rust/derived.rs @@ -16,7 +16,7 @@ use perspective_client::config::ViewConfig; use perspective_client::{DeleteOptions, Table, View}; use perspective_js::utils::*; -use crate::config::DerivedTableConfig; +use crate::config::{DerivedTableConfig, UnknownKeys}; use crate::session::ViewSubscription; use crate::utils::PtrEqRc; @@ -76,6 +76,7 @@ impl DerivedTable { DerivedTableConfig { table: self.parent.clone(), view_config: self.view_config.clone(), + unknown: UnknownKeys, } } diff --git a/rust/perspective-viewer/src/rust/queries/validate_expression.rs b/rust/perspective-viewer/src/rust/queries/validate_expression.rs index c47a7a12d3..ef13d23854 100644 --- a/rust/perspective-viewer/src/rust/queries/validate_expression.rs +++ b/rust/perspective-viewer/src/rust/queries/validate_expression.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use perspective_client::ExprValidationError; -use perspective_client::config::ExpressionsDeserde; +use perspective_client::config::Expressions; use perspective_js::utils::ApiError; use crate::session::Session; @@ -28,10 +28,10 @@ pub async fn validate_expr( .get_table() .ok_or_else(|| ApiError::from("No table set"))?; let errors = table - .validate_expressions( - ExpressionsDeserde::Map(HashMap::from_iter([("_".to_string(), expr.to_string())])) - .into(), - ) + .validate_expressions(Expressions(HashMap::from_iter([( + "_".to_string(), + expr.to_string(), + )]))) .await? .errors; diff --git a/rust/perspective-viewer/src/rust/renderer.rs b/rust/perspective-viewer/src/rust/renderer.rs index 7ad1c5905a..4e4772b5c0 100644 --- a/rust/perspective-viewer/src/rust/renderer.rs +++ b/rust/perspective-viewer/src/rust/renderer.rs @@ -807,17 +807,9 @@ impl Renderer { } fn find_plugin_idx(&self, name: &str) -> Option { - let short_name = make_short_name(name); let mut borrowed = self.0.borrow_mut(); let configs = borrowed.plugin_store.plugin_configs(); - let short_names: Vec = configs.iter().map(|c| make_short_name(&c.name)).collect(); - if let Some(i) = short_names.iter().position(|n| n == &short_name) { - return Some(i); - } - - short_names - .iter() - .position(|n: &String| n.contains(&short_name)) + configs.iter().position(|config| config.name == name) } } diff --git a/rust/perspective-viewer/src/rust/session/op_queue.rs b/rust/perspective-viewer/src/rust/session/op_queue.rs index 9bf47d388b..d4808f4fc8 100644 --- a/rust/perspective-viewer/src/rust/session/op_queue.rs +++ b/rust/perspective-viewer/src/rust/session/op_queue.rs @@ -23,7 +23,7 @@ use futures::future::LocalBoxFuture; use perspective_client::config::ViewConfigUpdate; use perspective_js::utils::*; -use crate::config::{OptionalUpdate, ViewerConfigUpdate}; +use crate::config::{OptionalUpdate, UnknownKeys, ViewerConfigUpdate}; use crate::utils::InFlightGuard; /// The render an op's ticket waits on after its write is finished. @@ -373,6 +373,7 @@ fn overwrites((later_view, later): Writes, (earlier_view, earlier): Writes) -> b title, table, view_config, + unknown: UnknownKeys, } = earlier.unwrap_or(&empty); let _ = (version, view_config); diff --git a/rust/perspective-viewer/src/rust/tasks/derive.rs b/rust/perspective-viewer/src/rust/tasks/derive.rs index ec7f1935e5..11f657d484 100644 --- a/rust/perspective-viewer/src/rust/tasks/derive.rs +++ b/rust/perspective-viewer/src/rust/tasks/derive.rs @@ -17,7 +17,9 @@ use perspective_client::config::{ViewConfig, ViewConfigUpdate}; use perspective_client::{Client, DeleteOptions, TableData, TableInitOptions}; use perspective_js::utils::*; -use crate::config::{DerivedTableConfig, OptionalUpdate, TableRef, ViewerConfigUpdate}; +use crate::config::{ + DerivedTableConfig, OptionalUpdate, TableRef, UnknownKeys, ViewerConfigUpdate, +}; use crate::derived::{DerivedSource, DerivedTable, suggest_name}; use crate::presentation::Presentation; use crate::queries::fetch_hosted_tables; @@ -78,6 +80,7 @@ pub async fn resolve_table_ref( let config = DerivedTableConfig { table: parent.clone(), view_config, + unknown: UnknownKeys, }; if let Some(existing) = workspace.derived_find_structural(&config) { diff --git a/rust/perspective-viewer/src/rust/tasks/sync_update_panels.rs b/rust/perspective-viewer/src/rust/tasks/sync_update_panels.rs index 3509fc68a9..8e20a1fee7 100644 --- a/rust/perspective-viewer/src/rust/tasks/sync_update_panels.rs +++ b/rust/perspective-viewer/src/rust/tasks/sync_update_panels.rs @@ -45,18 +45,18 @@ pub fn sync_update_panels( update: JsWorkspaceConfigUpdate, ) -> ApiResult { let WorkspaceConfigUpdate { + version, active, layout, - mut panels, + panels, derived_tables, global_filters, masters, palette, + unknown: UnknownKeys, } = update.into_serde_ext()?; - for panel in panels.iter_mut().flat_map(|panels| panels.values_mut()) { - panel.migrate(); - } + let _ = version; let palette = match palette { OptionalUpdate::Missing => None, diff --git a/rust/perspective-viewer/src/rust/workspace.rs b/rust/perspective-viewer/src/rust/workspace.rs index 0eb09472c9..e4f7dbef1e 100644 --- a/rust/perspective-viewer/src/rust/workspace.rs +++ b/rust/perspective-viewer/src/rust/workspace.rs @@ -19,7 +19,7 @@ use std::rc::Rc; use perspective_client::Client; use perspective_client::config::Filter; -use crate::config::{DerivedTableConfig, TableRef}; +use crate::config::{DerivedTableConfig, TableRef, UnknownKeys}; use crate::derived::{DerivedRegistry, DerivedTable}; use crate::renderer::Renderer; use crate::session::{OverlayClause, Session}; @@ -1120,6 +1120,7 @@ impl Workspace { out = TableRef::Derived(Box::new(DerivedTableConfig { table: out, view_config: config.view_config, + unknown: UnknownKeys, })); }, None => return TableRef::Name(name.to_owned()), diff --git a/rust/perspective-viewer/src/ts/bootstrap.ts b/rust/perspective-viewer/src/ts/bootstrap.ts index 69f2948814..2037aad5c2 100644 --- a/rust/perspective-viewer/src/ts/bootstrap.ts +++ b/rust/perspective-viewer/src/ts/bootstrap.ts @@ -13,11 +13,16 @@ import type * as wasm_module_type from "../../dist/wasm/perspective-viewer.js"; import { init as registry_init, + MigrateLoader, provide_viewer_runtime, + resolve_migrate, + shared_migrate, } from "@perspective-dev/client/registry"; import type { ClientRuntime, InitOptions, + MigrateKind, + MigrateResult, WasmRegistration, } from "@perspective-dev/client/registry"; @@ -42,6 +47,26 @@ export async function init( await registry_init(wasm, options); } +/** + * A WebAssembly binary in any form `init_client` accepts; thunks are only + * invoked when the binary is needed. + */ +export type ViewerWasmSource = + | Promise + | Response + | ArrayBuffer + | Uint8Array + | (() => Promise); + +/** + * The `` binary, and the `perspective-migrate` binary + * behind `migrate()`. + */ +export interface ViewerWasmRegistration { + viewer?: ViewerWasmSource; + migrate?: ViewerWasmSource; +} + let GLUE_OVERRIDE: typeof wasm_module_type | undefined; provide_viewer_runtime(async (wasm) => { @@ -59,21 +84,70 @@ provide_viewer_runtime(async (wasm) => { return { glue: glue as ClientRuntime["glue"], module }; }); +const MIGRATE = new MigrateLoader(); +MIGRATE.register(resolve_migrate); + +function is_registration( + wasm: ViewerWasmSource | ViewerWasmRegistration, +): wasm is ViewerWasmRegistration { + return ( + typeof wasm === "object" && + !(wasm instanceof Promise) && + !(wasm instanceof Response) && + !(wasm instanceof ArrayBuffer) && + !ArrayBuffer.isView(wasm) + ); +} + /** - * Initialize `` from its WebAssembly binary; a wrapper - * over `init()`'s `viewer` role. + * Initialize `` from its WebAssembly binary, and register + * the `perspective-migrate` binary which `migrate()` downloads on first use; + * a wrapper over `init()`'s `viewer` and `migrate` roles. + * + * # Examples + * + * ```javascript + * await init_client({ + * viewer: () => fetch("perspective-viewer.wasm"), + * migrate: () => fetch("perspective-migrate-js.wasm"), + * }); + * ``` */ export async function init_client( - wasm_binary: - | Promise - | Response - | ArrayBuffer - | Uint8Array, + wasm: ViewerWasmSource | ViewerWasmRegistration, wasm_module?: typeof wasm_module_type, ) { + const registration = is_registration(wasm) ? wasm : { viewer: wasm }; + const { viewer, migrate } = registration; + if (viewer === undefined && migrate === undefined) { + throw new Error( + "init_client requires a wasm source or a {viewer, migrate} registration", + ); + } + if (wasm_module !== undefined) { GLUE_OVERRIDE = wasm_module; } - await init({ viewer: wasm_binary }); + await init({ viewer, migrate }); +} + +/** + * Upgrade a persisted config written by any Perspective release since 2.0.0 + * to the current schema, using the `migrate` binary registered with `init()`. + * + * # Arguments + * + * - `kind` - `"view"` for a `ViewConfig`, `"viewer"` for a `save()` token, or + * `"workspace"` for a `saveWorkspace()` token. + * - `config` - The config to upgrade. + * - `from_version` - The release which wrote `config`, used only when it has + * no `version` field. + */ +export async function migrate>( + kind: MigrateKind, + config: unknown, + from_version?: string, +): Promise> { + return await shared_migrate(MIGRATE).migrate(kind, config, from_version); } diff --git a/rust/perspective-viewer/src/ts/perspective-viewer.cdn.ts b/rust/perspective-viewer/src/ts/perspective-viewer.cdn.ts index 262f1d4700..3c96e1e282 100644 --- a/rust/perspective-viewer/src/ts/perspective-viewer.cdn.ts +++ b/rust/perspective-viewer/src/ts/perspective-viewer.cdn.ts @@ -13,6 +13,7 @@ import perspective_viewer from "./perspective-viewer.ts"; import { register_default, + resolve_package_wasm_url, resolve_viewer, } from "@perspective-dev/client/registry"; export * from "./perspective-viewer.ts"; @@ -20,6 +21,15 @@ export * from "./perspective-viewer.ts"; register_default({ viewer: () => fetch(new URL("../wasm/perspective-viewer.wasm", import.meta.url)), + migrate: () => + fetch( + resolve_package_wasm_url( + import.meta.url, + "viewer", + "client", + "perspective-migrate-js.wasm", + ), + ), }); await resolve_viewer(); diff --git a/rust/perspective-viewer/src/ts/perspective-viewer.ts b/rust/perspective-viewer/src/ts/perspective-viewer.ts index 9177051d7a..dfaa014d4c 100644 --- a/rust/perspective-viewer/src/ts/perspective-viewer.ts +++ b/rust/perspective-viewer/src/ts/perspective-viewer.ts @@ -68,13 +68,20 @@ export type * from "../../dist/ts-rs/GetClientOptions.d.ts"; export type * from "../../dist/ts-rs/GetViewOptions.d.ts"; export type * from "../../dist/ts-rs/GetViewMode.d.ts"; -export { init, init_client } from "./bootstrap.ts"; +export { init, init_client, migrate } from "./bootstrap.ts"; export type { InitOptions, + ViewerWasmRegistration, + ViewerWasmSource, WasmBinary, WasmRegistration, WasmSource, } from "./bootstrap.ts"; -import { init, init_client } from "./bootstrap.ts"; +export type { + MigrateKind, + MigrateNotice, + MigrateResult, +} from "@perspective-dev/client"; +import { init, init_client, migrate } from "./bootstrap.ts"; -export default { init, init_client }; +export default { init, init_client, migrate }; diff --git a/rust/perspective-viewer/test/js/viewer_config/config_strictness.spec.ts b/rust/perspective-viewer/test/js/viewer_config/config_strictness.spec.ts new file mode 100644 index 0000000000..fd77fe7ac0 --- /dev/null +++ b/rust/perspective-viewer/test/js/viewer_config/config_strictness.spec.ts @@ -0,0 +1,176 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect, PageView } from "../helpers.ts"; + +test.beforeEach(async ({ page }) => { + await page.goto("/rust/perspective-viewer/test/html/superstore-debug.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +}); + +async function restoreError(page, method: string, config: unknown) { + return await page.evaluate( + async ({ method, config }) => { + const viewer = document.querySelector("perspective-viewer")! as any; + try { + await viewer[method](config); + return null; + } catch (e) { + return e.message ?? `${e}`; + } + }, + { method, config }, + ); +} + +test.describe("config strictness", () => { + test("restore accepts a valid config stamped by an older release", async ({ + page, + }) => { + const error = await restoreError(page, "restore", { + version: "5.5.0", + plugin: "Debug Styled", + group_by: ["State"], + }); + + expect(error).toBeNull(); + }); + + test("restore rejects an unknown key", async ({ page }) => { + const error = await restoreError(page, "restore", { + plugin: "Debug Styled", + "row-pivots": ["State"], + }); + + expect(error).toContain("row-pivots"); + expect(error).not.toContain("perspective-migrate"); + }); + + test("a rejected config names the release that saved it", async ({ + page, + }) => { + const error = await restoreError(page, "restore", { + version: "4.4.1", + plugin: "Debug Styled", + "row-pivots": ["State"], + }); + + expect(error).toContain("row-pivots"); + expect(error).toContain("4.4.1"); + expect(error).toContain("perspective-migrate"); + }); + + test("restore rejects a workspace config", async ({ page }) => { + const error = await restoreError(page, "restore", { + layout: null, + panels: {}, + }); + + expect(error).toContain("panels"); + }); + + test("restore rejects the legacy expressions array", async ({ page }) => { + const error = await restoreError(page, "restore", { + plugin: "Debug Styled", + expressions: ['"Sales" + 1'], + }); + + expect(error).not.toBeNull(); + }); + + test("addPanel rejects an unknown key", async ({ page }) => { + const error = await restoreError(page, "addPanel", { + table: "superstore", + selectable: true, + }); + + expect(error).toContain("selectable"); + }); + + test("restoreWorkspace rejects a lumino workspace", async ({ page }) => { + const error = await restoreError(page, "restoreWorkspace", { + sizes: [1], + detail: { main: null }, + viewers: { + one: { + version: "4.5.2", + plugin: "Debug Styled", + table: "superstore", + }, + }, + }); + + expect(error).toContain("viewers"); + expect(error).toContain("4.5.2"); + expect(error).toContain("perspective-migrate"); + }); + + test("restoreWorkspace names the release that saved a rejected panel", async ({ + page, + }) => { + const error = await restoreError(page, "restoreWorkspace", { + layout: { type: "tab-layout", tabs: ["one"] }, + panels: { + one: { + version: "5.0.0", + table: "superstore", + plugin: "Debug Styled", + settings: true, + }, + }, + }); + + expect(error).toContain("settings"); + expect(error).toContain("5.0.0"); + }); + + test("an unversioned date_format takes the inheriting meaning", async ({ + page, + }) => { + const view = new PageView(page); + await view.restore({ + plugin: "Debug Format", + columns: ["Order Date"], + columns_config: { + "Order Date": { date_format: { dateStyle: "long" } }, + }, + } as any); + + const config = (await view.save()) as any; + expect(config.columns_config["Order Date"].date_format).toEqual({ + dateStyle: "long", + }); + }); + + test("a config saved by this build restores", async ({ page }) => { + const error = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore({ + plugin: "Debug Styled", + group_by: ["State"], + }); + const saved = await viewer.save(); + try { + await viewer.restore(saved); + return null; + } catch (e) { + return e.message ?? `${e}`; + } + }); + + expect(error).toBeNull(); + }); +}); diff --git a/rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts b/rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts deleted file mode 100644 index 299aec2d07..0000000000 --- a/rust/perspective-viewer/test/js/viewer_config/date_format_migration.spec.ts +++ /dev/null @@ -1,120 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -import { test, expect, PageView, API_VERSION } from "../helpers.ts"; - -test.beforeEach(async ({ page }) => { - await page.goto("/rust/perspective-viewer/test/html/superstore-debug.html"); - await page.evaluate(async () => { - while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { - await new Promise((x) => setTimeout(x, 10)); - } - }); -}); - -async function restoredDateFormat( - view: PageView, - plugin: string, - date_format: Record, - version?: string, -) { - await view.restore({ - plugin, - columns: ["Order Date"], - columns_config: { "Order Date": { date_format } }, - ...(version ? { version } : {}), - } as any); - - const config = (await view.save()) as any; - return config.columns_config["Order Date"]?.date_format; -} - -test.describe("date_format migration", () => { - test("a per-part value gains its old fallbacks", async ({ page }) => { - const view = new PageView(page); - expect( - await restoredDateFormat(view, "Debug Styled", { - format: "custom", - timeZone: "UTC", - second: "disabled", - minute: "disabled", - hour: "disabled", - }), - ).toEqual({ - timeZone: "UTC", - year: "2-digit", - month: "numeric", - day: "numeric", - hour12: true, - }); - }); - - test("a preset with a disabled field keeps its rendering", async ({ - page, - }) => { - const view = new PageView(page); - expect( - await restoredDateFormat(view, "Debug Format", { - timeStyle: "disabled", - }), - ).toEqual({ dateStyle: "short" }); - }); - - test("a preset with both fields disabled renders a numeric date", async ({ - page, - }) => { - const view = new PageView(page); - expect( - await restoredDateFormat(view, "Debug Styled", { - dateStyle: "disabled", - timeStyle: "disabled", - }), - ).toEqual({ year: "numeric", month: "numeric", day: "numeric" }); - }); - - test("an older versioned preset gains its missing style", async ({ - page, - }) => { - const view = new PageView(page); - expect( - await restoredDateFormat( - view, - "Debug Format", - { dateStyle: "long" }, - "5.5.0", - ), - ).toEqual({ dateStyle: "long", timeStyle: "medium" }); - }); - - test("an unversioned preset takes the inheriting meaning", async ({ - page, - }) => { - const view = new PageView(page); - expect( - await restoredDateFormat(view, "Debug Format", { - dateStyle: "long", - }), - ).toEqual({ dateStyle: "long" }); - }); - - test("a preset saved by this build is not migrated", async ({ page }) => { - const view = new PageView(page); - expect( - await restoredDateFormat( - view, - "Debug Format", - { dateStyle: "long" }, - API_VERSION, - ), - ).toEqual({ dateStyle: "long" }); - }); -}); diff --git a/rust/perspective-viewer/test/js/viewer_config/migrate_corpus.spec.ts b/rust/perspective-viewer/test/js/viewer_config/migrate_corpus.spec.ts new file mode 100644 index 0000000000..ddc10dba98 --- /dev/null +++ b/rust/perspective-viewer/test/js/viewer_config/migrate_corpus.spec.ts @@ -0,0 +1,127 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "../helpers.ts"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const CORPUS = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../../perspective-migrate/corpus", +); + +const CASES = [ + "viewer/2.6.0/xy_scatter_symbol_list.json", + "viewer/2.7.1/datagrid.json", + "viewer/2.7.1/xy_scatter.json", + "viewer/2.10.0/datagrid_editable.json", + "viewer/4.4.1/d3fc_y_bar.json", + "viewer/5.2.0/datagrid_sizes_and_colors.json", + "viewer/5.4.0/datagrid_formats.json", +]; + +const HOST_KEYS = ["version", "table", "theme", "settings"]; + +function corpus_case(name: string): Record { + return JSON.parse(fs.readFileSync(path.join(CORPUS, name), "utf8")); +} + +function migrated(name: string): Record { + const output = { ...corpus_case(name).output }; + for (const key of HOST_KEYS) { + delete output[key]; + } + + return output; +} + +test.beforeEach(async ({ page }) => { + await page.goto("/rust/perspective-viewer/test/html/superstore-all.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); +}); + +test.describe("perspective-migrate corpus", () => { + for (const name of CASES) { + test(`${name} migrates in the browser to its corpus output`, async ({ + page, + }) => { + const { input, from, output, steps } = corpus_case(name); + const result = await page.evaluate( + async ({ input, from }) => { + const url = + "/node_modules/@perspective-dev/viewer/dist/cdn/perspective-viewer.js"; + const viewer = await import(url); + return await viewer.default.migrate("viewer", input, from); + }, + { input, from }, + ); + + expect({ ...result.value, version: "CURRENT" }).toEqual(output); + expect(result.steps).toEqual(steps); + }); + + test(`${name} restores into the current viewer`, async ({ page }) => { + const config = migrated(name); + const saved = await page.evaluate(async (config) => { + const viewer = document.querySelector( + "perspective-viewer", + )! as any; + await viewer.restore(config); + return await viewer.save(); + }, config); + + for (const key of [ + "plugin", + "columns", + "group_by", + "split_by", + "expressions", + "aggregates", + ]) { + if (key in config) { + expect(saved[key]).toEqual(config[key]); + } + } + + for (const [column, style] of Object.entries( + saved.columns_config ?? {}, + )) { + for (const [key, value] of Object.entries(style)) { + const migrated = config.columns_config?.[column]?.[key]; + if ( + typeof value === "object" && + value !== null && + !Array.isArray(value) + ) { + expect(typeof migrated).toEqual("object"); + expect(value).toEqual( + expect.objectContaining(migrated), + ); + } else { + expect(migrated).toEqual(value); + } + } + } + + for (const [key, value] of Object.entries( + saved.plugin_config ?? {}, + )) { + expect(config.plugin_config?.[key]).toEqual(value); + } + }); + } +}); diff --git a/tools/scripts/publish.mjs b/tools/scripts/publish.mjs index 7eefddb221..4f52eedc94 100644 --- a/tools/scripts/publish.mjs +++ b/tools/scripts/publish.mjs @@ -95,6 +95,11 @@ async function publish_release_assets(releases) { SH_ENV, ); + execSync( + `cargo publish --config .cargo/release.toml -p perspective-migrate --allow-dirty --no-verify`, + SH_ENV, + ); + execSync( `cargo publish --config .cargo/release.toml -p perspective-python --allow-dirty --no-verify`, SH_ENV, diff --git a/tools/test/load-viewer-all-types-small-multi-arrow.js b/tools/test/load-viewer-all-types-small-multi-arrow.js index 38d4c6df91..c070af0711 100644 --- a/tools/test/load-viewer-all-types-small-multi-arrow.js +++ b/tools/test/load-viewer-all-types-small-multi-arrow.js @@ -25,7 +25,7 @@ async function load() { const table = worker.table(buffer); await viewer.load(table); const config = { - plugin: "datagrid", + plugin: "Datagrid", }; await viewer.restore(config); } diff --git a/tools/test/load-viewer-superstore.js b/tools/test/load-viewer-superstore.js index 5aa7ecfce2..c6848464c7 100644 --- a/tools/test/load-viewer-superstore.js +++ b/tools/test/load-viewer-superstore.js @@ -25,14 +25,11 @@ async function load() { const table = worker.table(csv, { name: "load-viewer-superstore" }); await viewer.load(table); const config = { - plugin: "datagrid", + plugin: "Datagrid", group_by: ["Region", "State"], split_by: ["Category", "Sub-Category"], columns: ["Sales", "Profit"], - master: false, - name: "Sales Report", table: "load-viewer-superstore", - linked: false, title: "Sales Report 2", }; await viewer.restore(config);