diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a64cab7..88b8653a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,4 +81,43 @@ jobs: - name: Run unit tests env: VIRTUAL_ENV: ${{ github.workspace }}/.venv - run: uv run --no-project pytest -v . -m "not integration" + # The ML demos run in their own job, below. + run: uv run --no-project pytest -v . -m "not integration" --ignore=tests/test_ml_demos.py + + ml-demos: + # Each ML demo in benchmarks/ml, at its smallest size, checked against + # JAX (tests/test_ml_demos.py). + name: "ml demos" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Configure sccache + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: ~/.cargo + key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }} + + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + enable-cache: true + + - name: Install xarray_sql + run: uv sync --dev --no-install-package xarray-sql + - name: build rust + run: uv run --no-project maturin develop --uv + - name: Run the demos + env: + VIRTUAL_ENV: ${{ github.workspace }}/.venv + run: uv run --no-project pytest -v tests/test_ml_demos.py diff --git a/AGENTS.md b/AGENTS.md index fc419a15..dfb638b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,16 @@ summarizes recurring maintainer review feedback so changes land clean. - Keep docstrings and comments self-contained. Do **not** put GitHub issue or PR numbers in docstrings or code comments; a reader should not need the issue tracker to understand the code. Issue references belong in the commit message - and PR description (e.g. `Closes #189`), not in the source. + and PR description (e.g. `Closes #189`), not in the source. The one exception + is a workaround's `TODO`, which names the issue that would remove it: + `TODO(#NN): `, or `TODO(owner/repo#NN): ` for an + issue in another repository. The comment still explains the workaround. +- Introduce every reference before relying on it. The first time a document + (a docstring, comment, README or docs page) mentions a project, paper, + algorithm, dataset or term of art, such as einfold, FlashAttention or a + section of another project's design, say in a few words what it is, and link + it where there is a link. Don't assume the reader has the context of the + conversation that produced the code. - Do not reference the review conversation, chat, or "the reporter" in comments. Describe the behavior, not how it came up. diff --git a/benchmarks/ml/01_attention.py b/benchmarks/ml/01_attention.py new file mode 100644 index 00000000..7709d2eb --- /dev/null +++ b/benchmarks/ml/01_attention.py @@ -0,0 +1,280 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "xarray-sql[ml,duckdb]", +# "ddxdb[duckdb]", +# "jax", +# ] +# +# [tool.uv.sources] +# xarray-sql = { path = "../../", editable = true } +# /// +"""Demo 1: causal attention in SQL, on several engines, with its gradient. + +Attention, the core of a transformer, is ``softmax(Q·Kᵀ/√d)·V``: each +token's output is an average of the value vectors ``V``, weighted by how +well the token's query in ``Q`` matches each earlier token's key in ``K``. +Here ``Q``, ``K`` and ``V`` are tables of ``(t, d, x)`` (token, feature, +value), registered from xarray Datasets, and attention is two sums over +joins: the scores, a sum of ``Q`` times ``K`` over features, and the +average of ``V`` weighted by ``exp(score)``. + +This SQL is the naive plan: it stores every one of the N×N scores. +FlashAttention (Dao et al., 2022, https://arxiv.org/abs/2205.14135) +computes the same result without storing them, by streaming blocks of +keys past blocks of queries and keeping a running softmax. einfold +(https://github.com/xqlsystems/einfold), a query optimizer that rewrites +SQL plans into array algorithms like this one, aims to find that plan +from this SQL, in both of the spellings below, with no rule written for +attention. It is the first of einfold's three target demos +(``docs/demos.md`` in its repository). This script is the baseline that +plan must match and then beat: + +* **forward**: each spelling of the model on each *host* (einfold's word for an engine + that runs a plan; here DataFusion and DuckDB), checked against the same + attention in JAX, Google's library for differentiable array programs; +* **gradient**: a loss that reads the model, and its gradient with + respect to every value of Q, K and V, as one SQL statement. ddx (https://github.com/xqlsystems/ddx), + which differentiates SQL queries, computes it, and ``jax.grad`` checks + it; +* **masks are absent rows**: an infinity in V reaches only the queries + allowed to see it, as SQL's semantics require of any faster plan; +* **a control that is not attention**: the Nadaraya–Watson estimator, a + Gaussian-kernel weighted average used in regridding. Any rule that + speeds up attention must speed up this query too. + +Run ``uv run benchmarks/ml/01_attention.py --help`` for the options. +""" + +from __future__ import annotations + +import jax +import jax.numpy as jnp +import numpy as np +import xarray as xr +from _bench import Report, assert_close, cli, hosts + +import xarray_sql as xql + +jax.config.update("jax_enable_x64", True) + +D = 64 # features per head; the scores are scaled by 1/√64 = 1/8. + +# The model and its loss are separate, as in a neural network written in +# Python (compare attention() and loss() below). The model is a view, a +# named query: each spelling of attention defines the view ``attention``, +# and the forward pass and the loss read it as if it were a table. + +# Attention in einfold's window spelling. The causal mask is a join +# predicate: a masked (i, j) pair is a row that does not exist. Subtracting +# each row's max only keeps exp() from overflowing: it cancels out of the +# ratio w / z, so its gradient sums to zero, and ddx differentiates it +# like any other window. +ATTENTION_WINDOW = """ +CREATE OR REPLACE VIEW attention AS +WITH scores AS ( + SELECT q.t AS i, k.t AS j, SUM(q.x * k.x) / 8.0 AS s + FROM q JOIN k ON q.d = k.d + WHERE q.t >= k.t + GROUP BY q.t, k.t), +weights AS ( + SELECT i, j, exp(s - MAX(s) OVER (PARTITION BY i)) AS w + FROM scores), +norm AS ( + SELECT i, SUM(w) AS z FROM weights GROUP BY i) +SELECT w.i AS t, v.d, SUM(w.w / n.z * v.x) AS x +FROM weights w +JOIN v ON w.j = v.t +JOIN norm n ON w.i = n.i +GROUP BY w.i, v.d +""" + +# The same attention as ddx spells it: each row's max as a GROUP BY, +# joined back to the scores. +ATTENTION_GROUP_BY = """ +CREATE OR REPLACE VIEW attention AS +WITH scores AS ( + SELECT q.t AS i, k.t AS j, SUM(q.x * k.x) / 8.0 AS s + FROM q JOIN k ON q.d = k.d + WHERE q.t >= k.t + GROUP BY q.t, k.t), +row_max AS ( + SELECT i, MAX(s) AS m FROM scores GROUP BY i), +weights AS ( + SELECT s.i, s.j, exp(s.s - r.m) AS w + FROM scores s JOIN row_max r ON s.i = r.i), +norm AS ( + SELECT i, SUM(w) AS z FROM weights GROUP BY i) +SELECT w.i AS t, v.d, SUM(w.w / n.z * v.x) AS x +FROM weights w +JOIN v ON w.j = v.t +JOIN norm n ON w.i = n.i +GROUP BY w.i, v.d +""" + +SPELLINGS = {"window": ATTENTION_WINDOW, "group by": ATTENTION_GROUP_BY} + +# The forward pass: run the model. +FORWARD = "SELECT t, d, x FROM attention ORDER BY t, d" + +# The loss, half the output's squared norm, calls the model by reading the +# view. The SELECT asks for its gradient with respect to every value of Q, +# K and V. The three grad() calls differentiate one loss, so ddx runs one +# backward pass for all three, as jax.grad(loss, argnums=(0, 1, 2)) does. +GRADIENT = """ +WITH loss AS ( + SELECT SUM(x * x) / 2 AS l FROM attention) +SELECT q.t, q.d, + grad(loss, q.x) AS dq, + grad(loss, k.x) AS dk, + grad(loss, v.x) AS dv +FROM q +JOIN k ON q.t = k.t AND q.d = k.d +JOIN v ON q.t = v.t AND q.d = v.d +ORDER BY q.t, q.d +""" + +# Nadaraya–Watson: at each target point, the average of the sources' y +# weighted by a Gaussian kernel of their distance (bandwidth h = 0.5). +# Structurally attention: a weighted average with weights exp(score). +NADARAYA_WATSON = """ +WITH kernel AS ( + SELECT t.p, s.q, exp(-(power(t.x - s.x, 2) + power(t.y - s.y, 2)) / 0.25) AS w + FROM target t CROSS JOIN source s) +SELECT k.p, SUM(k.w * s.v) / SUM(k.w) AS v +FROM kernel k JOIN source s ON k.q = s.q +GROUP BY k.p +ORDER BY k.p +""" + + +def attention(q, k, v): + """The reference: causal softmax attention, as JAX writes it.""" + n = q.shape[0] + scores = q @ k.T / np.sqrt(D) + causal = jnp.tril(jnp.ones((n, n), dtype=bool)) + weights = jax.nn.softmax(jnp.where(causal, scores, -jnp.inf), axis=-1) + return weights @ v + + +def loss(q, k, v): + return jnp.sum(attention(q, k, v) ** 2) / 2 + + +def table(values: np.ndarray) -> xr.Dataset: + """A (t, d) matrix as the XQL table ``(t, d, x)``.""" + t, d = values.shape + return xr.Dataset( + {"x": (("t", "d"), values)}, + coords={"t": np.arange(t), "d": np.arange(d)}, + ) + + +def matrix(rows, name: str = "x") -> np.ndarray: + """The rows of a ``(t, d, …)`` result as a (t, d) matrix.""" + return xql.to_dataset(rows, dims=["t", "d"])[name].values + + +def main(): + parser = cli(__doc__.splitlines()[0], sizes=[64, 256], size_name="tokens") + args = parser.parse_args() + report = Report("attention", "tokens", args.reps) + rng = np.random.default_rng(0) + + for n in args.sizes: + q, k, v = (rng.normal(size=(n, D)) for _ in range(3)) + want = attention(q, k, v) + want_grads = jax.grad(loss, argnums=(0, 1, 2))(q, k, v) + + for host in hosts(args.hosts): + for name, data in zip("qkv", (q, k, v)): + host.load(name, table(data)) + + for spelling, model in SPELLINGS.items(): + host.execute(model) + rows = report.time( + f"forward ({spelling})", host, n, host.query, FORWARD + ) + report.check(assert_close("forward", matrix(rows), want)) + report.note("operators", host.operators(FORWARD)) + + step = f"gradient ({spelling})" + if not host.differentiates: + report.skip(step, host, n, "no ddx backend") + continue + known = {} + if spelling == "window": + # TODO(xqlsystems/ddx#204): drop once ddx differentiates + # a view whose CTE computes a window on DuckDB. + known = {"duckdb": "Not implemented Error: CTE"} + rows = report.attempt( + step, host, n, host.query, GRADIENT, known=known + ) + if rows is None: + continue + errors = [ + assert_close(f"d/d{name}", matrix(rows, f"d{name}"), ref) + for name, ref in zip("qkv", want_grads) + ] + report.check(max(errors)) + + check_masks_are_absent_rows(args.hosts) + check_nadaraya_watson(args.hosts, report) + report.finish(args.csv) + + +def check_masks_are_absent_rows(host_names): + """An infinity in V at token j reaches the queries at or after j, and + no others: a masked pair is skipped, not multiplied by a zero weight, + which would turn 0·∞ into NaN. (xarray-sql reads NaN as missing data, + a NULL that SUM skips, so an infinity is the value that tells.)""" + n, j = 16, 9 + rng = np.random.default_rng(1) + q, k, v = (rng.normal(size=(n, D)) for _ in range(3)) + v[j, 0] = np.inf + print(f"\n masks: +inf in V at token {j}, feature 0") + for host in hosts(host_names): + for name, data in zip("qkv", (q, k, v)): + host.load(name, table(data)) + for spelling, model in SPELLINGS.items(): + host.execute(model) + out = matrix(host.query(FORWARD)) + assert np.isfinite(out[:j]).all(), (host.name, spelling) + assert np.isposinf(out[j:, 0]).all(), (host.name, spelling) + assert np.isfinite(out[:, 1:]).all(), (host.name, spelling) + print(f" {spelling:<9} {host.name:<11} ok") + + +def check_nadaraya_watson(host_names, report): + """The control: a kernel-weighted average that is not attention.""" + rng = np.random.default_rng(2) + targets, sources = rng.uniform(size=(64, 2)), rng.uniform(size=(256, 2)) + y = np.sin(4 * sources[:, 0]) + np.cos(3 * sources[:, 1]) + dist2 = ((targets[:, None, :] - sources[None, :, :]) ** 2).sum(-1) + w = np.exp(-dist2 / 0.25) + want = (w @ y) / w.sum(axis=1) + + target = xr.Dataset( + {"x": ("p", targets[:, 0]), "y": ("p", targets[:, 1])}, + coords={"p": np.arange(len(targets))}, + ) + source = xr.Dataset( + {"x": ("q", sources[:, 0]), "y": ("q", sources[:, 1]), "v": ("q", y)}, + coords={"q": np.arange(len(sources))}, + ) + for host in hosts(host_names): + host.load("target", target) + host.load("source", source) + rows = report.time( + f"nadaraya-watson ({len(sources)} sources)", + host, + len(targets), + host.query, + NADARAYA_WATSON, + ) + got = xql.to_dataset(rows, dims=["p"])["v"].values + report.check(assert_close("nadaraya-watson", got, want)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ml/02_nanogpt.py b/benchmarks/ml/02_nanogpt.py new file mode 100644 index 00000000..b5023ef0 --- /dev/null +++ b/benchmarks/ml/02_nanogpt.py @@ -0,0 +1,675 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "xarray-sql[ml,duckdb]", +# "ddxdb[duckdb]", +# "jax", +# ] +# +# [tool.uv.sources] +# xarray-sql = { path = "../../", editable = true } +# /// +"""Demo 2: a GPT in SQL, trained by gradient descent. + +nanoGPT (https://github.com/karpathy/nanoGPT) is Andrej Karpathy's small, +readable implementation of GPT-2, the language model. This is its model, +``model.py``, as one SQL query: the cross-entropy loss of a batch of +characters from Shakespeare, computed by a transformer whose parameters +are tables. ddx (https://github.com/xqlsystems/ddx), which differentiates +SQL queries, takes the loss's gradient with respect to every parameter, +so a training step is SQL too. + +================= ============================================ +Model piece In SQL +================= ============================================ +Embeddings a join of the batch with ``wte`` and ``wpe`` +Layer norm per-position ``AVG``s, joined back, times a gain +Attention per head and causal, as in ``01_attention.py`` +MLP two einsums around GELU +Output layer an einsum with ``wte`` again (tied, as in GPT-2) +Loss log-sum-exp per position, minus the target's logit +Gradients ddx: ``grad(loss, w)``, one statement per table +Optimizer step ``w - lr * grad(loss, w)``, the new table; or AdamW, + with Adam's moments as columns beside ``w`` and + the learning rate schedule as a table +================= ============================================ + +As in nanoGPT, the model and the loss are separate. The model is the +view ``logits``; the loss is SQL that reads it, as nanoGPT's +``F.cross_entropy`` reads the model's logits. + +Each training step updates every parameter table with its own +statement; the statements share one backward pass. By default the +optimizer is SGD at a fixed learning rate, the simplest step to read. +``--optimizer adamw`` trains as nanoGPT does instead: it stores the +gradients as tables, then updates each parameter table with AdamW. The same model in JAX +(Google's library for differentiable array programs), from the same +initial parameters and the same batches, is the reference: its loss at +every step, and its parameters after every step, must match the SQL's. + +This is Part A of the second target demo of einfold +(https://github.com/xqlsystems/einfold), a query optimizer that rewrites +SQL plans into array algorithms; the demo is described in einfold's +``docs/demos.md``. Part B builds on it: it writes the tricks behind a +record of the NanoGPT speedrun (a public race to train GPT-2 fastest) +as a small diff to this SQL. + +Differences from nanoGPT's ``model.py`` and ``train.py``: + +* **Size.** nanoGPT's Shakespeare model has 6 layers, 6 heads, 384 + channels and 256 characters of context, 10.65M parameters. This one has + 1 layer, 2 heads, 16 channels and 16 characters, 4,416 parameters (the + constants below). +* **GELU** is its tanh approximation, as GPT-2 computed it. nanoGPT's + ``nn.GELU()`` uses the exact form, through the error function ``erf``, + which DataFusion doesn't have. The two differ by less than 5e-4. +* **The optimizer**, by default, is SGD at a fixed learning rate. + ``--optimizer adamw`` matches nanoGPT's: AdamW with its Shakespeare + settings, weight decay on all but the layer norms, gradients clipped to + a norm of 1, and the learning rate warming up, then decaying along a + cosine. The schedule is stretched to ``--steps``, where nanoGPT's spans + 5,000. The JAX reference for this step matches ``torch.optim.AdamW`` + after ``clip_grad_norm_`` exactly. +* **No dropout**, nanoGPT's default, so that a run is deterministic and + can be checked. Its Shakespeare configuration trains with 0.2. +* **Q, K and V are three tables.** nanoGPT's ``c_attn`` holds the same + parameters as one matrix, split after the product. + +Like nanoGPT's default (``bias = False`` in ``train.py``), the model has +no biases, in linear layers or layer norms. + +Run ``uv run benchmarks/ml/02_nanogpt.py --help`` for the options. +""" + +from __future__ import annotations + +import pathlib +import urllib.request + +import jax +import jax.numpy as jnp +import numpy as np +import xarray as xr +from _bench import Report, assert_close, cli, hosts + +import xarray_sql as xql + +jax.config.update("jax_enable_x64", True) + +# A tiny character-level GPT. The comments name each constant in nanoGPT's +# GPTConfig (https://github.com/karpathy/nanoGPT/blob/master/model.py) and +# its value in nanoGPT's Shakespeare model +# (https://github.com/karpathy/nanoGPT/blob/master/config/train_shakespeare_char.py). +LAYERS = 1 # n_layer = 6: transformer blocks +EMBED = 16 # n_embd = 384; e: channels of the residual stream +HEADS = 2 # n_head = 6; h: attention heads +HEAD = EMBED // HEADS # c: channels per head +HIDDEN = 4 * EMBED # f: channels of the MLP, 4 × n_embd as in nanoGPT +CONTEXT = 16 # block_size = 256; pos: characters per sequence +LEARNING_RATE = 0.5 # for SGD; nanoGPT's AdamW uses 1e-3, below + +# AdamW (``--optimizer adamw``), with the settings of nanoGPT's Shakespeare +# model: train.py's defaults, and config/train_shakespeare_char.py's. +MAX_LR = 1e-3 # learning_rate +MIN_LR = 1e-4 # min_lr +WARMUP = 100 / 5000 # warmup_iters, as a fraction of max_iters +BETA1 = 0.9 # beta1 +BETA2 = 0.99 # beta2 +EPS = 1e-8 # torch.optim.AdamW's default +WEIGHT_DECAY = 0.1 # weight_decay, on every table but the layer norms' gains +GRAD_CLIP = 1.0 # grad_clip: the largest norm of all gradients together + +# Each parameter table's dimensions; its values are the column w. In +# nanoGPT, wq, wk and wv are attn.c_attn; wo is attn.c_proj; w_fc and +# w_proj are mlp.c_fc and mlp.c_proj; ln_1, ln_2 and ln_f are the layer +# norms' weights. +PARAMS = { + "wte": ("tok", "e"), + "wpe": ("pos", "e"), + "ln_1": ("layer", "e"), + "wq": ("layer", "e", "h", "c"), + "wk": ("layer", "e", "h", "c"), + "wv": ("layer", "e", "h", "c"), + "wo": ("layer", "h", "c", "e"), + "ln_2": ("layer", "e"), + "w_fc": ("layer", "e", "f"), + "w_proj": ("layer", "f", "e"), + "ln_f": ("e",), +} + +# -------------------------------------------------------------------------- +# The model, in SQL. Activations are tables of (seq, pos, e, val): one row +# per sequence in the batch, position, and channel. +# -------------------------------------------------------------------------- + + +def layer_norm(x: str, out: str, gain: str, layer: int | None = None) -> str: + """Normalize each position of ``x`` to zero mean and unit variance, + then scale each channel by the table ``gain`` (of ``layer``, if set).""" + of_layer = "" if layer is None else f" AND g.layer = {layer}" + return f""" +{out}_mean AS ( + SELECT seq, pos, AVG(val) AS mean FROM {x} GROUP BY seq, pos), +{out}_var AS ( + SELECT x.seq, x.pos, AVG(power(x.val - m.mean, 2)) AS var + FROM {x} x JOIN {out}_mean m ON x.seq = m.seq AND x.pos = m.pos + GROUP BY x.seq, x.pos), +{out} AS ( + SELECT x.seq, x.pos, x.e, (x.val - m.mean) / sqrt(v.var + 1e-5) * g.w AS val + FROM {x} x + JOIN {out}_mean m ON x.seq = m.seq AND x.pos = m.pos + JOIN {out}_var v ON x.seq = v.seq AND x.pos = v.pos + JOIN {gain} g ON x.e = g.e{of_layer}),""" + + +# GELU's tanh approximation: x/2 · (1 + tanh(√(2/π) · (x + 0.044715 x³))). +GELU_SCALE = np.sqrt(2 / np.pi) + + +def attention(layer: int, x: str, out: str) -> str: + """nanoGPT's ``CausalSelfAttention``: each position of ``x`` attends to + itself and the positions before it, in each head, and ``out`` is the + heads' outputs, projected back to the residual stream.""" + l = f"l{layer}" + projections = "".join( + f""" +{l}_{name} AS ( + SELECT x.seq, x.pos, p.h, p.c, SUM(x.val * p.w) AS val + FROM {x} x JOIN w{name} p ON x.e = p.e + WHERE p.layer = {layer} + GROUP BY x.seq, x.pos, p.h, p.c),""" + for name in ("q", "k", "v") + ) + return f"""{projections} +{l}_scores AS ( + SELECT q.seq, q.h, q.pos AS i, k.pos AS j, + SUM(q.val * k.val) / {np.sqrt(HEAD)} AS s + FROM {l}_q q JOIN {l}_k k ON q.seq = k.seq AND q.h = k.h AND q.c = k.c + WHERE q.pos >= k.pos + GROUP BY q.seq, q.h, q.pos, k.pos), +{l}_weights AS ( + -- The max only keeps exp() from overflowing: it cancels out of p / z. + SELECT seq, h, i, j, + exp(s - MAX(s) OVER (PARTITION BY seq, h, i)) AS p + FROM {l}_scores), +{l}_norm AS ( + SELECT seq, h, i, SUM(p) AS z FROM {l}_weights GROUP BY seq, h, i), +{l}_heads AS ( + SELECT a.seq, a.i AS pos, a.h, v.c, SUM(a.p / n.z * v.val) AS val + FROM {l}_weights a + JOIN {l}_v v ON a.seq = v.seq AND a.h = v.h AND a.j = v.pos + JOIN {l}_norm n ON a.seq = n.seq AND a.h = n.h AND a.i = n.i + GROUP BY a.seq, a.i, a.h, v.c), +{out} AS ( + SELECT a.seq, a.pos, p.e, SUM(a.val * p.w) AS val + FROM {l}_heads a JOIN wo p ON a.h = p.h AND a.c = p.c + WHERE p.layer = {layer} + GROUP BY a.seq, a.pos, p.e),""" + + +def mlp(layer: int, x: str, out: str) -> str: + """nanoGPT's ``MLP``: each position of ``x`` through a hidden layer + four times as wide, and GELU, back to the residual stream as ``out``.""" + l = f"l{layer}" + return f""" +{l}_fc AS ( + SELECT x.seq, x.pos, p.f, SUM(x.val * p.w) AS val + FROM {x} x JOIN w_fc p ON x.e = p.e + WHERE p.layer = {layer} + GROUP BY x.seq, x.pos, p.f), +{l}_hidden AS ( + SELECT seq, pos, f, + 0.5 * val * (1 + tanh({GELU_SCALE} * (val + 0.044715 * val * val * val))) + AS val + FROM {l}_fc), +{out} AS ( + SELECT x.seq, x.pos, p.e, SUM(x.val * p.w) AS val + FROM {l}_hidden x JOIN w_proj p ON x.f = p.f + WHERE p.layer = {layer} + GROUP BY x.seq, x.pos, p.e),""" + + +def residual(x: str, y: str, out: str) -> str: + """``out = x + y``, channel by channel: a residual connection.""" + return f""" +{out} AS ( + SELECT x.seq, x.pos, x.e, x.val + y.val AS val + FROM {x} x JOIN {y} y ON x.seq = y.seq AND x.pos = y.pos AND x.e = y.e),""" + + +def block(layer: int, x: str, out: str) -> str: + """nanoGPT's ``Block``: ``mid = x + attention(ln_1(x))``, then + ``out = mid + mlp(ln_2(mid))``.""" + l = f"l{layer}" + return "".join( + [ + layer_norm(x, f"{l}_ln1", "ln_1", layer), + attention(layer, f"{l}_ln1", f"{l}_attn"), + residual(x, f"{l}_attn", f"{l}_mid"), + layer_norm(f"{l}_mid", f"{l}_ln2", "ln_2", layer), + mlp(layer, f"{l}_ln2", f"{l}_mlp"), + residual(f"{l}_mid", f"{l}_mlp", out), + ] + ) + + +def model() -> str: + """The model, as the view ``logits``: for each position of each + sequence in ``batch``, a score for every character that could come + next. A DataFusion view reads the tables that existed when it was + created (DuckDB's looks them up by name), so it is recreated whenever + the batch or the parameters are.""" + blocks = "".join( + block(layer, f"x{layer}", f"x{layer + 1}") for layer in range(LAYERS) + ) + return f""" +CREATE OR REPLACE VIEW logits AS +WITH x0 AS ( + SELECT b.seq, b.pos, te.e, te.w + pe.w AS val + FROM batch b + JOIN wte te ON b.tok = te.tok + JOIN wpe pe ON b.pos = pe.pos AND te.e = pe.e), +{blocks} +{layer_norm(f"x{LAYERS}", "final", "ln_f").rstrip(",")} +SELECT x.seq, x.pos, te.tok, SUM(x.val * te.w) AS val +FROM final x JOIN wte te ON x.e = te.e +GROUP BY x.seq, x.pos, te.tok +""" + + +# The loss, separate from the model, as in nanoGPT: the mean cross-entropy +# of the model's logits against each next character of the batch. +# Subtracting each row's max only keeps exp() from overflowing: it cancels +# out of the log-sum-exp. +# +# The loss reads the model's view once, into the CTE ``predicted``, and +# uses that twice: a host expands a view at every read, but DuckDB +# computes a CTE that is read twice only once. +LOSS = """ +predicted AS ( + SELECT seq, pos, tok, val FROM logits), +shifted AS ( + SELECT seq, pos, tok, val, + MAX(val) OVER (PARTITION BY seq, pos) AS m + FROM predicted), +log_sum_exp AS ( + SELECT seq, pos, MAX(m) + ln(SUM(exp(val - m))) AS lse + FROM shifted GROUP BY seq, pos), +loss AS ( + SELECT AVG(s.lse - l.val) AS l + FROM log_sum_exp s + JOIN batch b ON s.seq = b.seq AND s.pos = b.pos + JOIN predicted l ON l.seq = b.seq AND l.pos = b.pos AND l.tok = b.next)""" + + +def step_statements(lr: float) -> list[str]: + """One SGD step: each parameter table's next version, as a statement.""" + return [ + f"WITH {LOSS} SELECT {', '.join(dims)}, " + f"w - {lr} * grad(loss, w) AS w FROM {table} " + f"ORDER BY {', '.join(dims)}" + for table, dims in PARAMS.items() + ] + + +def gradient_statements() -> list[str]: + """The gradient of the loss with respect to each parameter table, as + the table ``{name}_grad``'s rows. The statements share one backward + pass.""" + return [ + f"WITH {LOSS} SELECT {', '.join(dims)}, grad(loss, w) AS g " + f"FROM {table} ORDER BY {', '.join(dims)}" + for table, dims in PARAMS.items() + ] + + +def adamw_statement(table: str, step: int) -> str: + """One AdamW step on a parameter table with columns ``w``, ``m`` and + ``v`` (the weight and Adam's two moments), from its gradient in + ``{table}_grad`` and the learning rate in ``schedule``. This is + ``torch.optim.AdamW``, after nanoGPT clips the gradients.""" + dims = PARAMS[table] + keys = ", ".join(f"p.{d}" for d in dims) + same_row = " AND ".join(f"p.{d} = g.{d}" for d in dims) + decay = 0.0 if table.startswith("ln_") else WEIGHT_DECAY + every_grad = "\n UNION ALL ".join( + f"SELECT g FROM {t}_grad" for t in PARAMS + ) + return f""" +WITH grad_norm AS ( + SELECT sqrt(SUM(g * g)) AS norm FROM ( + {every_grad})), +clip AS ( + SELECT LEAST(1.0, {GRAD_CLIP} / (norm + 1e-6)) AS c FROM grad_norm), +moments AS ( + SELECT {keys}, p.w, + {BETA1} * p.m + (1 - {BETA1}) * g.g * clip.c AS m, + {BETA2} * p.v + (1 - {BETA2}) * power(g.g * clip.c, 2) AS v + FROM {table} p + JOIN {table}_grad g ON {same_row} + CROSS JOIN clip) +SELECT {keys}, + p.w * (1 - s.lr * {decay}) + - s.lr * (p.m / (1 - power({BETA1}, s.step + 1))) + / (sqrt(p.v / (1 - power({BETA2}, s.step + 1))) + {EPS}) AS w, + p.m, + p.v +FROM moments p JOIN schedule s ON s.step = {step} +ORDER BY {keys} +""" + + +def loss_statement() -> str: + return f"WITH {LOSS} SELECT l FROM loss" + + +# -------------------------------------------------------------------------- +# The same model in JAX: the reference. +# -------------------------------------------------------------------------- + + +def jax_layer_norm(x, gain): + mean = x.mean(-1, keepdims=True) + var = ((x - mean) ** 2).mean(-1, keepdims=True) + return (x - mean) / jnp.sqrt(var + 1e-5) * gain + + +def jax_loss(p, tok, nxt): + x = p["wte"][tok] + p["wpe"][None] + causal = jnp.tril(jnp.ones((CONTEXT, CONTEXT), dtype=bool)) + for layer in range(LAYERS): + h = jax_layer_norm(x, p["ln_1"][layer]) + q, k, v = ( + jnp.einsum("bte,ehc->bhtc", h, p[w][layer]) + for w in ("wq", "wk", "wv") + ) + s = jnp.einsum("bhic,bhjc->bhij", q, k) / np.sqrt(HEAD) + a = jax.nn.softmax(jnp.where(causal, s, -jnp.inf), axis=-1) + heads = jnp.einsum("bhij,bhjc->bhic", a, v) + x = x + jnp.einsum("bhtc,hce->bte", heads, p["wo"][layer]) + h = jax_layer_norm(x, p["ln_2"][layer]) + hidden = jax.nn.gelu(h @ p["w_fc"][layer], approximate=True) + x = x + hidden @ p["w_proj"][layer] + logits = jax_layer_norm(x, p["ln_f"]) @ p["wte"].T + target = jnp.take_along_axis(logits, nxt[..., None], -1)[..., 0] + return jnp.mean(jax.nn.logsumexp(logits, -1) - target) + + +def jax_adamw(params, moments, grads, step, lr): + """The reference AdamW step: new parameters and moments.""" + norm = np.sqrt(sum(float(jnp.sum(g**2)) for g in grads.values())) + clip = min(1.0, GRAD_CLIP / (norm + 1e-6)) + new_params, new_moments = {}, {} + for name, w in params.items(): + g = grads[name] * clip + m, v = moments[name] + m = BETA1 * m + (1 - BETA1) * g + v = BETA2 * v + (1 - BETA2) * g**2 + decay = 0.0 if name.startswith("ln_") else WEIGHT_DECAY + m_hat = m / (1 - BETA1 ** (step + 1)) + v_hat = v / (1 - BETA2 ** (step + 1)) + new_params[name] = w * (1 - lr * decay) - lr * m_hat / ( + jnp.sqrt(v_hat) + EPS + ) + new_moments[name] = (m, v) + return new_params, new_moments + + +# -------------------------------------------------------------------------- +# Data and parameters, as xarray Datasets. +# -------------------------------------------------------------------------- + +SHAKESPEARE = ( + "https://raw.githubusercontent.com/karpathy/char-rnn/master/" + "data/tinyshakespeare/input.txt" +) + + +def shakespeare() -> str: + """Tiny Shakespeare, cached in ~/.cache; a sonnet offline.""" + cache = ( + pathlib.Path.home() / ".cache" / "xarray-sql" / "tinyshakespeare.txt" + ) + if not cache.exists(): + cache.parent.mkdir(parents=True, exist_ok=True) + try: + with urllib.request.urlopen(SHAKESPEARE, timeout=10) as r: + cache.write_bytes(r.read()) + except OSError: + return SONNET_18 * 4 + return cache.read_text() + + +def batches(text: str, size: int, rng): + """Endless batches of (seq, pos) characters and the characters after + them, as one Dataset: the table ``batch(seq, pos, tok, next)``.""" + vocab = sorted(set(text)) + ids = np.array([vocab.index(c) for c in text]) + while True: + starts = rng.integers(0, len(ids) - CONTEXT - 1, size) + tok = np.stack([ids[s : s + CONTEXT] for s in starts]) + nxt = np.stack([ids[s + 1 : s + CONTEXT + 1] for s in starts]) + yield xr.Dataset( + {"tok": (("seq", "pos"), tok), "next": (("seq", "pos"), nxt)}, + coords={"seq": np.arange(size), "pos": np.arange(CONTEXT)}, + ) + + +def init_params(vocab: int, rng) -> dict[str, np.ndarray]: + """nanoGPT's initialization, from GPT-2: weights normal with standard + deviation 0.02, or 0.02/√(2 × layers) for the two projections that add + to the residual stream; layer norm gains 1.""" + sizes = { + "tok": vocab, + "pos": CONTEXT, + "layer": LAYERS, + "e": EMBED, + "h": HEADS, + "c": HEAD, + "f": HIDDEN, + } + params = {} + for name, dims in PARAMS.items(): + shape = [sizes[d] for d in dims] + if name.startswith("ln_"): + params[name] = np.ones(shape) + elif name in ("wo", "w_proj"): + params[name] = rng.normal(0, 0.02 / np.sqrt(2 * LAYERS), shape) + else: + params[name] = rng.normal(0, 0.02, shape) + return params + + +def as_table(name: str, w: np.ndarray, m=None, v=None) -> xr.Dataset: + """A parameter as a table: its weights ``w``, and Adam's moments ``m`` + and ``v`` when they are given.""" + dims = PARAMS[name] + columns = {"w": w, "m": m, "v": v} + return xr.Dataset( + {k: (dims, x) for k, x in columns.items() if x is not None}, + coords={d: np.arange(n) for d, n in zip(dims, w.shape)}, + ) + + +def as_array(name: str, rows, column: str = "w") -> np.ndarray: + return xql.to_dataset(rows, dims=list(PARAMS[name]))[column].values + + +def schedule(steps: int) -> xr.Dataset: + """nanoGPT's learning rate at each step, as the table + ``schedule(step, lr)``: a linear warmup, then a cosine decay to the + minimum, with train.py's ``max_iters`` and ``lr_decay_iters`` both + ``steps``.""" + warmup = round(WARMUP * steps) + lr = [] + for it in range(steps): + if it < warmup: + lr.append(MAX_LR * (it + 1) / (warmup + 1)) + else: + decay = (it - warmup) / (steps - warmup) + coeff = 0.5 * (1 + np.cos(np.pi * decay)) + lr.append(MIN_LR + coeff * (MAX_LR - MIN_LR)) + return xr.Dataset({"lr": ("step", lr)}, coords={"step": np.arange(steps)}) + + +def adamw_step(host, step: int) -> list: + """One AdamW step on every parameter table: the gradients, in one + backward pass, registered as tables; then each table's update.""" + gradients = host.query_all(gradient_statements()) + for name, rows in zip(PARAMS, gradients): + host.load(f"{name}_grad", xql.to_dataset(rows, dims=list(PARAMS[name]))) + return [host.query(adamw_statement(name, step)) for name in PARAMS] + + +def main(): + # The model's size, from the command line, as module constants. + global LAYERS, EMBED, HEADS, HEAD, HIDDEN, CONTEXT + parser = cli(__doc__.splitlines()[0], sizes=[4], size_name="batch") + parser.add_argument("--steps", type=int, default=5, help="training steps") + parser.add_argument( + "--optimizer", + choices=["sgd", "adamw"], + default="sgd", + help="SGD at a fixed learning rate (default), or nanoGPT's AdamW", + ) + parser.add_argument("--layers", type=int, default=LAYERS, help="n_layer") + parser.add_argument("--embed", type=int, default=EMBED, help="n_embd") + parser.add_argument("--heads", type=int, default=HEADS, help="n_head") + parser.add_argument( + "--context", type=int, default=CONTEXT, help="block_size" + ) + args = parser.parse_args() + if args.embed % args.heads: + parser.error("--embed must be a multiple of --heads") + LAYERS, EMBED, HEADS, CONTEXT = ( + args.layers, + args.embed, + args.heads, + args.context, + ) + HEAD, HIDDEN = EMBED // HEADS, 4 * EMBED + adamw = args.optimizer == "adamw" + lr = schedule(args.steps) + report = Report("nanogpt", "batch", args.reps) + text = shakespeare() + vocab = len(set(text)) + lines = len(model().splitlines()) + len(loss_statement().splitlines()) + count = sum( + v.size for v in init_params(vocab, np.random.default_rng(0)).values() + ) + print( + f" {LAYERS}-layer GPT, vocab {vocab}, {count:,} parameters: " + f"the loss is {lines} lines of SQL" + ) + + for size in args.sizes: + init = init_params(vocab, np.random.default_rng(0)) + data = list( + zip( + range(args.steps), batches(text, size, np.random.default_rng(1)) + ) + ) + + for host in hosts(args.hosts): + reference = dict(init) + moments = { + n: (np.zeros_like(w), np.zeros_like(w)) for n, w in init.items() + } + for name, values in init.items(): + state = moments[name] if adamw else () + host.load(name, as_table(name, values, *state)) + host.load("schedule", lr) + + for step, batch in data: + host.load("batch", batch) + host.execute(model()) + tok, nxt = batch["tok"].values, batch["next"].values + + rows = report.time( + f"loss, step {step}", + host, + size, + host.query, + loss_statement(), + ) + want = jax_loss(reference, tok, nxt) + report.check( + assert_close("loss", rows["l"][0].as_py(), want, rtol=1e-9) + ) + report.note("loss", round(float(want), 6)) + report.note("operators", host.operators(loss_statement())) + + train = f"train ({args.optimizer}), step {step}" + if not host.differentiates: + report.skip(train, host, size, "no ddx backend") + continue + # TODO(xqlsystems/ddx#204): drop once ddx differentiates a + # view whose CTE computes a window on DuckDB. + known = {"duckdb": "Not implemented Error: CTE"} + if adamw: + updated = report.attempt( + train, host, size, adamw_step, host, step, known=known + ) + else: + updated = report.attempt( + train, + host, + size, + host.query_all, + step_statements(LEARNING_RATE), + known=known, + ) + if updated is None: + # Every later step starts from this one's weights. + break + grads = jax.grad(jax_loss)(reference, tok, nxt) + if adamw: + reference, moments = jax_adamw( + reference, moments, grads, step, lr["lr"].values[step] + ) + else: + reference = { + name: w - LEARNING_RATE * grads[name] + for name, w in reference.items() + } + + errors = [] + for name, rows in zip(PARAMS, updated): + expected = {"w": reference[name]} + if adamw: + expected["m"], expected["v"] = moments[name] + for column, values in expected.items(): + errors.append( + assert_close( + f"{name}.{column}, step {step}", + as_array(name, rows, column), + values, + ) + ) + host.load( + name, xql.to_dataset(rows, dims=list(PARAMS[name])) + ) + report.check(max(errors)) + + report.finish(args.csv) + + +SONNET_18 = """Shall I compare thee to a summer's day? +Thou art more lovely and more temperate: +Rough winds do shake the darling buds of May, +And summer's lease hath all too short a date; +Sometime too hot the eye of heaven shines, +And often is his gold complexion dimm'd; +And every fair from fair sometime declines, +By chance or nature's changing course untrimm'd; +But thy eternal summer shall not fade, +Nor lose possession of that fair thou ow'st; +Nor shall death brag thou wander'st in his shade, +When in eternal lines to time thou grow'st: +So long as men can breathe or eyes can see, +So long lives this, and this gives life to thee. +""" + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ml/CONTRIBUTING.md b/benchmarks/ml/CONTRIBUTING.md new file mode 100644 index 00000000..4837b962 --- /dev/null +++ b/benchmarks/ml/CONTRIBUTING.md @@ -0,0 +1,146 @@ +# Contributing an ML demo + +A demo here is a machine-learning model written as SQL over tables that +xarray-sql registers from xarray Datasets. It is trained or differentiated +with [ddx](https://github.com/xqlsystems/ddx), which differentiates SQL +queries, and checked against the same model in +[JAX](https://github.com/jax-ml/jax), Google's library for differentiable +array programs. Each demo is also a benchmark: the same SQL runs on every +engine in [`_hosts.py`](_hosts.py), and its times are what ddx, xarray-sql +and [einfold](https://github.com/xqlsystems/einfold) (a query optimizer +that rewrites SQL plans into array algorithms) work to bring down. The +performance work is tracked in +[#285](https://github.com/xqlsystems/xarray-sql/issues/285). + +Good demos are models whose SQL teaches something: a model people know +from another library, written so a reader can follow it line by line. +Some that would fit well: + +- linear or logistic regression on a gridded dataset, such as ERA5 + temperatures; +- a small convolutional network over a grid, where a convolution is a + join on shifted coordinates; +- an MLP on MNIST; +- matrix factorization, or word2vec; +- fitting a physical model's parameters to observations, such as a + diffusion coefficient in the heat equation. + +## What a demo is made of + +Read [`01_attention.py`](01_attention.py) (short) and +[`02_nanogpt.py`](02_nanogpt.py) (a whole training loop) first. A demo +is one script, `NN_name.py`, numbered after the last one, with: + +1. **Inline script metadata**, copied from the other demos, so that + `uv run benchmarks/ml/NN_name.py` runs it against this checkout. Add + any dependency the demo needs beyond `xarray-sql[ml,duckdb]`, + `ddxdb[duckdb]` and `jax`, both there and in the `dev` dependency + group in [`pyproject.toml`](../../pyproject.toml). CI runs the demos + in the project's environment, which reads no script's metadata. +2. **A module docstring** that says: + - what the model is, and links its reference implementation or paper; + - how each piece of the model maps to SQL (`02_nanogpt.py` has a + table for this); + - every way the demo differs from the reference, and why. +3. **The model as SQL**, built by small Python functions that each return + one piece of SQL, named after the pieces of the reference. In + `02_nanogpt.py`, `block()` is `layer_norm()`, `attention()`, `mlp()` + and `residual()`, as nanoGPT's `Block` is. Write the model as a view + (`CREATE OR REPLACE VIEW … AS`), and the loss as a separate query + that reads it, as a Python loss calls its model. +4. **The same model in JAX**, from the same inputs and parameters, as the + reference. Turn on 64-bit floats + (`jax.config.update("jax_enable_x64", True)`) so the two agree to + about 1e-15. +5. **A `main()`** that: + - parses the shared command line from `_bench.cli` (`--hosts`, + `--sizes`, `--reps`, `--csv`), plus the demo's own options; + - loads the tables on each host; + - times each step with `Report.time`, or `Report.attempt` for a + gradient that a host may refuse, naming the refusals it expects + (`known=`); + - checks each result with `assert_close` and `Report.check`; + - ends with `Report.finish`. + + A mismatch with JAX must fail the run, which `assert_close` does. +6. **A line in `DEMOS`** in [`tests/test_ml_demos.py`](../../tests/test_ml_demos.py), + with arguments that make the demo run in a few seconds. CI's `ml demos` + job runs every demo there, and fails if one is missing. +7. **A row in the table in [`README.md`](README.md)**, and anything it + found under "Findings". + +## Writing the SQL + +- **Legibility over speed.** Write the model as its definition reads, + even where a faster spelling exists. The slow parts are what the + engines and optimizers need to make fast, and a demo that hides them + measures nothing. +- **Introduce every reference before relying on it**, in docstrings, + comments and the README: say in a few words what a project, paper, + algorithm or term is, and link it. This rule, and the others in + [`AGENTS.md`](../../AGENTS.md), apply to demos too. +- **Name issues only in TODOs.** Docstrings and comments name no issue + numbers, except a workaround's `TODO(#NN): `, or + `TODO(owner/repo#NN)` for another repository's issue. +- **Make results deterministic** with `ORDER BY`, so they can be checked + against JAX. +- **Use no NaN as a value.** xarray-sql reads a NaN as SQL `NULL`, which + aggregates skip, so it can't show where a value went. To test that a + value never reaches an output, plant an infinity instead. Demo 1 does + this to check that its causal mask leaves the masked pairs out, as rows + the query drops (`WHERE q.t >= k.t`): an infinity at a masked position + must not reach that output. A plan that multiplied it by a zero weight + instead would give NaN (0 · ∞). +- **Run the same SQL on every host.** If an engine needs different + SQL, that difference is a finding: write it up rather than fork the + query. + +## Running + +```shell +uv run benchmarks/ml/NN_name.py --sizes 16 --hosts datafusion +uv run pytest tests/test_ml_demos.py # every demo, small +``` + +`uv run pytest` syncs the project's environment, with the `dev` group, +which holds JAX and ddx's DuckDB backend. + +Against a ddx checkout rather than the release: + +1. Build ddxdb into this checkout's `.venv`, with `maturin develop + --release` in ddx's `python/ddxdb`. +2. Run the demos with the venv's `python`: + `.venv/bin/python benchmarks/ml/NN_name.py`. `uv run` runs a script + with inline metadata in an environment of its own, with ddxdb from + PyPI, and `--no-sync` doesn't change that. A `uv sync` reinstalls the + release in `.venv` too. + +## When ddx or an engine can't run your model + +That's a useful result. Each of these is a finding, not a failure: + +- ddx raises a typed error (a `ddxdb.DdxError`) for anything it can't + differentiate; +- an engine may refuse a plan; +- an engine may compute the wrong number. + +`Report.attempt` records a refusal in the results table as `refused:` +with the error. Unless the demo passes it in `known=`, the run then +fails, so that CI notices a refusal that's new. Then: + +1. Cut the query down to a minimal repro. +2. File it on [ddx](https://github.com/xqlsystems/ddx/issues), or on + xarray-sql if it's about how a table is read. +3. List the limit in the README's findings. +4. Pass it in `known=`, the host and the start of its error, with a + `TODO(xqlsystems/ddx#NN)` to drop it once the issue is fixed. + +A demo may land with a refused step. + +## Measuring + +The output ends with a table: one row per step, host and size, with the +median time, the error against JAX, and the number of operators in the +host's plan. To compare runs, pass `--reps 3 --csv results.csv`, and post +the table on [#285](https://github.com/xqlsystems/xarray-sql/issues/285), +with your CPU and the versions of DataFusion, DuckDB and ddxdb. diff --git a/benchmarks/ml/README.md b/benchmarks/ml/README.md new file mode 100644 index 00000000..26c5f7c6 --- /dev/null +++ b/benchmarks/ml/README.md @@ -0,0 +1,233 @@ +# Machine learning in SQL + +These scripts write machine-learning models as SQL over tables that +[xarray-sql](../../README.md) registers from xarray Datasets, and time the +same SQL on several engines. Two related projects shape them: + +- [ddx](https://github.com/xqlsystems/ddx) differentiates SQL queries: it + turns the gradient of a query into more SQL, which trains the models. +- [einfold](https://github.com/xqlsystems/einfold) is a query optimizer + that rewrites SQL plans into array algorithms. Two of these models + are its [target demos](https://github.com/xqlsystems/einfold/blob/main/docs/demos.md), + and this SQL is the baseline its plans must beat. + +Each script checks its results against the same model in +[JAX](https://github.com/jax-ml/jax), Google's library for differentiable +array programs. + +| # | Script | Model | What it measures | +|---|---|---|---| +| 01 | [`01_attention.py`](01_attention.py) | causal softmax attention, the core of a transformer, as a view (the model) in einfold's window and `GROUP BY` spellings, and a loss that reads it | the naive plan, which stores every score, for [FlashAttention](https://arxiv.org/abs/2205.14135)'s tiled plan to beat; gradients with respect to Q, K and V; masked pairs as absent rows; a control that isn't attention, the Nadaraya–Watson kernel average | +| 02 | [`02_nanogpt.py`](02_nanogpt.py) | [nanoGPT](https://github.com/karpathy/nanoGPT)'s GPT-2 model, at 1 layer and 4,416 parameters, as a view, and a cross-entropy loss that reads it, on Tiny Shakespeare | the loss on each host; training steps with SGD or, as nanoGPT trains, AdamW, every parameter table checked after every step | + +The SQL favors legibility over speed. Each model reads like its definition, +and the slow parts are left slow on purpose: they are what ddx, xarray-sql +and einfold need to make fast. + +To add a demo, see [CONTRIBUTING.md](CONTRIBUTING.md). The work to make +these models fast is tracked in +[#285](https://github.com/xqlsystems/xarray-sql/issues/285), which lists +places to start. + +Every result is checked before it is timed into the table. Each script +computes the same model in JAX from the same inputs and parameters, and a +mismatch fails the run. + +## Running + +```shell +uv run benchmarks/ml/01_attention.py +uv run benchmarks/ml/02_nanogpt.py --steps 5 +``` + +Each script carries [inline `uv` metadata](https://docs.astral.sh/uv/guides/scripts/) +and runs against the xarray-sql in this checkout, with the `ml` extra. The +options are the same in each: + +```shell +--hosts datafusion duckdb # the engines to run on (default: all) +--sizes 64 256 # problem sizes: tokens, or batch size +--reps 3 # timed repetitions; the median is reported +--csv results.csv # also write the results table +``` + +Demo 2 adds `--steps`, the number of training steps, and `--optimizer`: +`sgd` (the default), or `adamw` to train as nanoGPT does, and the +model's size: `--layers`, `--embed`, `--heads` and `--context`, nanoGPT's +`n_layer`, `n_embd`, `n_head` and `block_size` (1, 16, 2 and 16 by +default; nanoGPT's Shakespeare model is 6, 384, 6 and 256). + +The output ends with one table: a row per step, host and size, with its +median time, its error against JAX, and the number of operators in the +host's physical plan. + +To run against a ddx checkout rather than the release, build ddxdb into +this checkout's `.venv` (`maturin develop --release` in ddx's +`python/ddxdb`) and run the scripts with the venv's `python`, as in +`.venv/bin/python benchmarks/ml/01_attention.py`. `uv run` runs a script +with inline metadata in an environment of its own, with ddxdb from PyPI, +and `--no-sync` doesn't change that. + +## The `ml` extra + +```shell +pip install "xarray-sql[ml]" +``` + +The extra installs [ddxdb](https://pypi.org/project/ddxdb/). With it, an +`xarray_sql.Context` understands `grad`, `vjp` and `jvp` in a select list, +and `ddx_stop_gradient` as a function. A gradient-descent step on a +parameter table is one statement: + +```sql +WITH pred AS ( + SELECT x.sample, SUM(x.value * w.val) AS y + FROM x JOIN w ON x.feature = w.feature GROUP BY x.sample), +loss AS ( + SELECT AVG(power(pred.y - t.value, 2)) AS l + FROM pred JOIN t ON pred.sample = t.sample) +SELECT feature, val - 0.1 * grad(loss, val) AS val FROM w +``` + +`xarray_sql.ad.sql_all(ctx, statements)` runs several such statements for +one backward pass, which is how Demo 2 updates its eleven parameter tables. + +## Hosts + +einfold calls the engine that runs a plan its *host*. +[`_hosts.py`](_hosts.py) defines each one in a few lines: how it registers +an `xr.Dataset` (through xarray-sql), runs SQL, and counts the operators +in its plan. + +| Host | Forward pass | Gradients | +|---|---|---| +| `datafusion` | `xarray_sql.Context` | ddx | +| `duckdb` | DuckDB, through `xql.register` | ddx, with the `duckdb` extra of ddxdb, which pins the DuckDB it supports | + +A host that cannot differentiate still runs every forward query, and its +gradient rows say `no ddx backend`. A gradient that ddx, or the engine +it plans with, refuses is reported as `refused:` with the error, and a +refused training step ends that host's training run. Each demo lists the +refusals it expects, by host and error. Any other refusal fails the run, +so a refusal that's new is noticed. To add a host, such as DataFusion with +einfold's optimizer or DuckDB with a GPU extension, write one class in +`_hosts.py` and add it to `HOSTS`. + +## Findings + +Measured on 2026-10-17 on an Intel Core i7-8700 (6 cores, 12 threads) +with DataFusion 54.0.0, DuckDB 1.5.5 and ddxdb 0.3.0, one repetition each. +The GPT and attention demos were timed again on 2026-10-21 with +DataFusion 55.0.0 and ddxdb 0.4.0, after xarray-sql began registering +small in-memory Datasets as DataFusion in-memory tables. Both demos were +timed again on 2026-10-25 with DataFusion 55.0.0, DuckDB 1.5.5 and +ddxdb 0.4.2; where those runs give new numbers, they are given here. + +**ddx's gradients agree with JAX.** The gradients of attention with +respect to Q, K and V, in both spellings, match `jax.grad` to 3e-15 of the +largest value at every size. After each of five SGD steps, all eleven of +the GPT's parameter tables match JAX's to 1e-15, and the loss falls from +4.18 to 3.84 (each step on a different batch). With AdamW, the weights +and both of Adam's moments match a JAX transcription of +`torch.optim.AdamW` to 6e-15 after each of five steps, and that +transcription matches PyTorch exactly. + +**DataFusion computes a CTE once for every place that reads it.** A +model reads its activations more than once: a layer norm reads its input +three times, a residual connection twice. DataFusion inlines each read, +so the plan grows with the model's depth; DuckDB computes a CTE that is +read more than once a single time. + +| Query | DataFusion operators | DuckDB operators | DataFusion | DuckDB | +|---|---:|---:|---:|---:| +| GPT loss, 1 layer, batch 4 | 13,756 | 126 | 3.0 s | 0.063 s | + +Setting up a lazy read of a Dataset takes xarray-sql about 1.5 ms on its +own (1.7 ms for a `SELECT *` of 1,040 rows, against 0.26 ms for an +in-memory table). In the GPT's forward pass, which reads its Datasets +2,989 times, the lazy reads cost 6.7 s of its 9.2 s, about 2.2 ms each. A small +in-memory Dataset is now an in-memory table, which takes a quarter of a +millisecond per read. A GPT training step, forward and backward through +ddx, takes 0.65 s on DataFusion with SGD and 0.74 s with AdamW, a quarter +of its forward query alone (1.2 s and 1.55 s with ddxdb 0.3.3 on +DataFusion 54). ddx builds the gradient program on the first step, which +takes 1.2 s, and runs it again on later ones. It plans each CTE the loss +reads twice on its own, and stores the result of each step of its program +as a table, so it computes each shared subexpression once. Doing the same +for forward queries is the first thing xarray-sql's DataFusion path needs +for these models, and einfold should plan them as graphs rather than +trees. + +**DataFusion spends the GPT's time planning, not running.** Measured on +2026-10-24 with DataFusion 55.0.0. Of the 1-layer loss's 2.5 s, turning +its SQL into a physical plan takes nearly all; the same physical plan, +built once and run again, takes 1.8 ms (with one partition, and the same +tables). A second layer multiplies the plan by about 100, to 1.74 +million logical nodes and 313,789 table scans: the SQL planner takes 8.6 +s to build it, and planning it runs out of 8 GB. DuckDB plans the 2-layer +loss in 211 operators and runs it in 0.12 s. DataFusion 56 plans a CTE +written `WITH … AS MATERIALIZED` once, which the GPT needs to grow past +one layer on DataFusion. A ddx training step runs 76 small queries, of +5.4 ms each, and about 4 ms of each is planning; with one partition +instead of twelve, and one optimizer pass, a step takes 0.40 s instead +of 0.66 s. + +**JAX is the target.** JAX, compiled with `jax.jit` on the same CPU, +computes the GPT's loss in 0.15 ms and an SGD step in 0.37 ms, and +attention at 512 tokens in 0.92 ms forward and 2.4 ms with its +gradients: 300 to 400 times faster than DataFusion on attention, and up +to 1,000 times faster than DuckDB on its gradients, whose time is in +running their joins; and thousands of times faster on the GPT, whose time +is in planning. + +**Attention stores every score.** Causal attention's forward pass at 512 +tokens takes 0.27 s on DataFusion in the window spelling and 0.39 s in the +`GROUP BY` spelling; DuckDB takes 0.47 s and 0.26 s. Both grow with the +square of the tokens. The gradients take 2.5 to 3.5 times the forward +pass: 0.84 s and 0.95 s on DataFusion at 512 tokens, against 1.1 s and 1.3 +s with ddxdb 0.3.3 on DataFusion 54, and 2.8 s and 2.7 s with ddxdb 0.3.2, +before ddx differentiated the masked product as `jax.grad` does. + +**ddx differentiates on DuckDB.** Measured with ddxdb 0.4.2. The +gradients of attention in the `GROUP BY` spelling match `jax.grad` to +2e-15 on DuckDB, and take 0.25 s, 0.78 s and 2.4 s at 64, 256 and 512 +tokens, against 0.095 s, 0.31 s and 0.95 s on DataFusion. Nearly all of DuckDB's +time is in running ddx's program, not planning it: two contractions over +about 8 and 17 million joined rows take half of it. DuckDB orders those +joins worse when it reads a table through an Arrow scan, as +`xql.register` gives it, than when it reads a native table (0.64 s +against 0.20 s for one step), but copying Q, K and V into native tables +only brings the gradient to 1.7 s, and slows the forward pass (measured +on 2026-10-24, with ddx's development branch and DuckDB 1.5.6). + +**What ddx cannot differentiate yet.** Each of these raised a typed +error naming the construct, never a wrong number: + +- A `GROUP BY` key that depends on a differentiated column. einfold's + window spelling groups by the normalizer `z`; the demos divide inside + the sum instead, `SUM(w / z * v)`, which einfold also lists. +- On DuckDB, a view whose CTE computes a window and is read more than + once, as the softmax's weights are in attention's window spelling and + in the GPT, so neither trains on DuckDB yet. ddx reads DuckDB's plans + as [Substrait](https://substrait.io/), a portable format for query + plans, and DuckDB's Substrait producer has no plan for a CTE it + computes once for several reads. ddxdb 0.4.2 writes such a view's SQL + into the query instead, except when the CTE computes a window + ([ddx#204](https://github.com/xqlsystems/ddx/issues/204)). These rows + say `refused: Not implemented Error: CTE`. + +**xarray-sql reads NaN as missing.** A NaN in a Dataset becomes SQL +`NULL`, which `SUM` skips, so Demo 1 checks masks with an infinity: a +plan that multiplies a masked pair's zero weight by it gives NaN where +SQL gives a number. + +## Next + +The open work, and how to measure it, is in +[#285](https://github.com/xqlsystems/xarray-sql/issues/285). Part B of +the NanoGPT demo is also still to come: the four tricks behind a record +of the [NanoGPT speedrun](https://github.com/KellerJordan/modded-nanogpt), +a public race to train GPT-2 fastest, as a small diff to `02_nanogpt.py`. + +[`_bench.py`](_bench.py) holds the shared command line, timing, checks and +results table. diff --git a/benchmarks/ml/_bench.py b/benchmarks/ml/_bench.py new file mode 100644 index 00000000..dcb044e8 --- /dev/null +++ b/benchmarks/ml/_bench.py @@ -0,0 +1,186 @@ +"""Timing, checking and reporting for the ML demos. + +Each demo is a script with the same command line:: + + uv run benchmarks/ml/01_attention.py --hosts datafusion duckdb \\ + --sizes 64 256 --reps 3 --csv attention.csv + +It runs every step on every chosen host (an engine that runs SQL; see +``_hosts.py``) and size, checks each result against the same model in +JAX, Google's library for differentiable array programs, and ends with +one table: a row per step, host +and size, with the median time and whether the result matched. Checks +come first: a wrong answer fails the run, however fast it was. +""" + +from __future__ import annotations + +import argparse +import csv +import statistics +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field + +import numpy as np +from _hosts import HOSTS, Host + + +def cli(description: str, sizes: Sequence[int], size_name: str = "n"): + """The shared command line, for a demo to extend and parse.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + "--hosts", + nargs="+", + default=list(HOSTS), + choices=list(HOSTS), + help="engines to run on (default: all)", + ) + parser.add_argument( + "--sizes", + nargs="+", + type=int, + default=list(sizes), + help=f"problem sizes, as {size_name} (default: %(default)s)", + ) + parser.add_argument( + "--reps", + type=int, + default=1, + help="timed repetitions per step; the median is reported", + ) + parser.add_argument("--csv", help="also write the results table here") + return parser + + +def hosts(names: Sequence[str]) -> list[Host]: + """A fresh instance of each named host.""" + return [HOSTS[name]() for name in names] + + +def assert_close(name: str, got, want, *, rtol=1e-9, atol=1e-9) -> float: + """Assert two arrays match, NaN for NaN. Returns the largest error, + relative to the largest value.""" + got, want = np.asarray(got), np.asarray(want) + np.testing.assert_allclose( + got, want, rtol=rtol, atol=atol, equal_nan=True, err_msg=name + ) + finite = np.isfinite(want) + scale = np.max(np.abs(want[finite]), initial=0.0) or 1.0 + return float(np.max(np.abs(got - want)[finite], initial=0.0) / scale) + + +@dataclass +class Report: + """The demo's results: one row per step, host and size.""" + + demo: str + size_name: str + reps: int = 1 + rows: list[dict] = field(default_factory=list) + # The refusals :meth:`attempt` wasn't told to expect. + unexpected: list[str] = field(default_factory=list) + + def time(self, step: str, host: Host, size: int, run: Callable, *args): + """Run ``run(*args)`` ``reps`` times, record the median, and return + the last result.""" + times = [] + for _ in range(self.reps): + start = time.perf_counter() + result = run(*args) + times.append(time.perf_counter() - start) + self.rows.append( + { + "demo": self.demo, + "step": step, + "host": host.name, + self.size_name: size, + "seconds": statistics.median(times), + "check": "", + } + ) + print( + f" {step:<28} {host.name:<11} {self.size_name}={size:<6} " + f"{statistics.median(times):8.3f}s", + flush=True, + ) + return result + + def attempt( + self, + step: str, + host: Host, + size: int, + run: Callable, + *args, + known: Mapping[str, str] | None = None, + ): + """As :meth:`time`, but a step the host refuses (one of its + ``refusals``) is recorded with the reason, and returns None. + + ``known`` maps the name of each host expected to refuse the step to + the start of its error. Any other refusal is recorded too, and + :meth:`finish` then fails the run, so that a new refusal is noticed + rather than leaving the step unchecked.""" + try: + return self.time(step, host, size, run, *args) + except host.refusals as e: + reason = str(e).splitlines()[0] + expected = (known or {}).get(host.name) + if expected is None or not reason.startswith(expected): + self.unexpected.append(f"{step} on {host.name}: {reason}") + print(f" {step:<28} {host.name:<11} refused: {reason}", flush=True) + self.skip(step, host, size, f"refused: {reason}") + return None + + def check(self, error: float) -> None: + """Mark the last row's result as checked, with its error (from + :func:`assert_close`).""" + self.rows[-1]["check"] = f"ok ({error:.0e})" + + def skip(self, step: str, host: Host, size: int, why: str) -> None: + """Record a step the host cannot run, and why.""" + self.rows.append( + { + "demo": self.demo, + "step": step, + "host": host.name, + self.size_name: size, + "seconds": None, + "check": why, + } + ) + + def note(self, key: str, value) -> None: + """Attach a measurement besides time to the last row.""" + self.rows[-1][key] = value + + def finish(self, csv_path: str | None = None) -> None: + """Print the table, and write it as CSV if asked. Exits with an + error if a host refused a step it wasn't expected to.""" + columns = list(dict.fromkeys(k for row in self.rows for k in row)) + print(f"\n| {' | '.join(columns)} |") + print(f"|{'|'.join('---' for _ in columns)}|") + for row in self.rows: + cells = [_cell(row.get(c)) for c in columns] + print(f"| {' | '.join(cells)} |") + if csv_path: + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + writer.writeheader() + writer.writerows(self.rows) + print(f"\nwrote {csv_path}", file=sys.stderr) + if self.unexpected: + sys.exit( + "refused unexpectedly:\n" + + "\n".join(f" {r}" for r in self.unexpected) + ) + + +def _cell(value) -> str: + if value is None: + return "n/a" + if isinstance(value, float): + return f"{value:.3f}" + return str(value) diff --git a/benchmarks/ml/_hosts.py b/benchmarks/ml/_hosts.py new file mode 100644 index 00000000..d5bcadcf --- /dev/null +++ b/benchmarks/ml/_hosts.py @@ -0,0 +1,141 @@ +"""The SQL engines ("hosts") the ML demos run on. + +einfold (https://github.com/xqlsystems/einfold), a query optimizer that +rewrites SQL plans into array algorithms, calls the engine that runs a +plan its *host*. Every demo writes +its model once, as SQL, and runs that same text on each host, so a +difference in time is a difference between engines, never between +queries. A host does five things: + +* ``load(name, data, **options)`` registers an ``xr.Dataset`` as the + table ``name`` through xarray-sql, replacing any table of that name + (``options`` such as ``table_names`` go to xarray-sql); +* ``execute(sql)`` runs a statement for its effect, such as + ``CREATE VIEW``; +* ``query(sql)`` runs a statement and returns its rows as Arrow; +* ``query_all(sqls)`` runs several statements, and, on a host that + differentiates, shares the backward pass between those that take + ``grad`` of the same loss; +* ``operators(sql)`` counts the operators in the physical plan it would + run, which shows, for one, whether a CTE read twice is computed twice. + +Forward queries run everywhere. Gradients (``grad``, ``vjp``, ``jvp``) +need ddx (https://github.com/xqlsystems/ddx), which differentiates SQL +queries and runs its programs on DataFusion and DuckDB; a host that +cannot differentiate says so with ``differentiates = False``, and the +demos report its gradient steps as not run rather than skip it silently. +A gradient that ddx (or the engine under it) refuses to plan raises one +of the host's ``refusals``, and the demos report that step with the reason. + +Adding a host (einfold's optimizer on DataFusion, or DuckDB with a GPU +extension) is one class here and one entry in ``HOSTS``. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence + +import duckdb +import pyarrow as pa +import xarray as xr +from ddxdb import DdxError +from ddxdb import duckdb as ddx_duckdb + +import xarray_sql as xql + + +class Host: + """An engine that runs the demos' SQL. Subclasses fill in the rest.""" + + name = "" + differentiates = False + refusals: tuple[type[Exception], ...] = () + + def load(self, name: str, data: xr.Dataset, **options) -> None: + raise NotImplementedError + + def execute(self, sql: str) -> None: + raise NotImplementedError + + def query(self, sql: str) -> pa.Table: + raise NotImplementedError + + def query_all(self, sqls: Sequence[str]) -> list[pa.Table]: + return [self.query(sql) for sql in sqls] + + def operators(self, sql: str) -> int: + raise NotImplementedError + + +class DataFusion(Host): + """xarray-sql's ``Context``, with ddx's gradients (the ``ml`` extra).""" + + name = "datafusion" + differentiates = True + refusals = (DdxError,) + + def __init__(self): + self.ctx = xql.Context() + + def load(self, name, data, **options): + self.ctx.deregister_table(name) + self.ctx.from_dataset(name, data, **options) + + def execute(self, sql): + self.ctx.sql(sql).collect() + + def query(self, sql): + return self.ctx.sql(sql).to_arrow_table() + + def query_all(self, sqls): + return [df.to_arrow_table() for df in xql.ad.sql_all(self.ctx, sqls)] + + def operators(self, sql): + plan = self.ctx.sql(sql).execution_plan().display_indent() + return len(plan.splitlines()) + + +class DuckDB(Host): + """DuckDB, reading the same Datasets through xarray-sql's adapter, with + ddx's gradients (``ddxdb[duckdb]``).""" + + name = "duckdb" + differentiates = True + # ddx reads DuckDB's plans as Substrait, a portable format for query + # plans, and DuckDB's Substrait producer raises its own error for a + # plan it can't write. + refusals = (DdxError, duckdb.NotImplementedException) + + def __init__(self): + self.con = duckdb.connect() + + def load(self, name, data, **options): + xql.register(self.con, name, data, **options) + + def execute(self, sql): + self.con.execute(sql) + + def query(self, sql): + # As xarray-sql's Context does, send ddx only the statements that + # take a gradient. + if xql.ad.calls_ad(sql): + return self.query_all([sql])[0] + return self.con.sql(sql).fetch_arrow_table() + + def query_all(self, sqls): + return [ + rel.fetch_arrow_table() + for rel in ddx_duckdb.sql_all(self.con, sqls) + ] + + def operators(self, sql): + (_, plan), *_ = self.con.sql(f"EXPLAIN (FORMAT JSON) {sql}").fetchall() + return sum(_count(node) for node in json.loads(plan)) + + +def _count(node: dict) -> int: + return 1 + sum(_count(child) for child in node.get("children", [])) + + +HOSTS = {host.name: host for host in (DataFusion, DuckDB)} diff --git a/pyproject.toml b/pyproject.toml index 8d80eae7..59e68eb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,11 +58,17 @@ polars = [ geo = [ "pyproj", ] +# Gradients in SQL: grad(), vjp() and jvp() from ddx. ddxdb is pre-1.0 +# and breaks its API between minor versions. It reads and writes the +# Substrait of DataFusion 55, so bump it together with that pin. +ml = [ + "ddxdb>=0.4.2,<0.5; sys_platform != 'emscripten'", +] test = [ "adbc-driver-flightsql>=1.12", "adbc-driver-sqlite>=1.12", "cftime", - "xarray-sql[adbc,dask,duckdb,polars,geo]", + "xarray-sql[adbc,dask,duckdb,polars,geo,ml]", "pytest", "xarray[io]", "gcsfs", @@ -130,6 +136,10 @@ dev = [ "py-spy>=0.4.0", "ruff>=0.15.10", "maturin>=1.9.1", + # The ML demos: ddx's DuckDB backend, at the DuckDB it declares, and + # JAX, which the demos check their results against. + "ddxdb[duckdb]>=0.4.2,<0.5; sys_platform != 'emscripten'", + "jax", ] [tool.uv] diff --git a/tests/test_ad.py b/tests/test_ad.py new file mode 100644 index 00000000..13bb28e4 --- /dev/null +++ b/tests/test_ad.py @@ -0,0 +1,127 @@ +"""Tests for the ddx gradient extension (`xarray_sql.ad`).""" + +import numpy as np +import pytest +import xarray as xr + +pytest.importorskip("ddxdb") + +from ddxdb import UnsupportedExpression + +from xarray_sql import Context, ad + +# Least squares: pred = x @ w, loss = sum((pred - y)^2). +LOSS = """ +pred AS ( + SELECT x.sample, SUM(x.x * w.w) AS p + FROM data x JOIN weights w ON x.feature = w.feature + GROUP BY x.sample), +loss AS ( + SELECT SUM(power(pred.p - y.y, 2)) AS l + FROM pred JOIN targets y ON pred.sample = y.sample) +""" + + +@pytest.fixture +def problem(): + rng = np.random.default_rng(0) + x = rng.normal(size=(6, 3)) + w = rng.normal(size=3) + y = rng.normal(size=6) + sample, feature = np.arange(6), np.arange(3) + ctx = Context() + ctx.from_dataset( + "data", + xr.Dataset( + {"x": (("sample", "feature"), x)}, + coords={"sample": sample, "feature": feature}, + ), + ) + ctx.from_dataset( + "weights", + xr.Dataset({"w": ("feature", w)}, coords={"feature": feature}), + ) + ctx.from_dataset( + "targets", xr.Dataset({"y": ("sample", y)}, coords={"sample": sample}) + ) + return ctx, x, w, y + + +def test_grad_matches_closed_form(problem): + ctx, x, w, y = problem + got = ctx.sql( + f"WITH {LOSS} SELECT feature, grad(loss, w) AS w FROM weights ORDER BY feature" + ).to_dataset() + np.testing.assert_allclose(got["w"].values, 2 * x.T @ (x @ w - y)) + + +def test_gradient_step_reads_like_sql(problem): + ctx, x, w, y = problem + got = ctx.sql( + f"WITH {LOSS} " + "SELECT feature, w - 0.01 * grad(loss, w) AS w FROM weights " + "ORDER BY feature" + ).to_pandas() + np.testing.assert_allclose(got["w"], w - 0.01 * 2 * x.T @ (x @ w - y)) + + +def test_sql_all_shares_one_loss(problem): + ctx, x, w, y = problem + grad = 2 * x.T @ (x @ w - y) + select = "SELECT feature, {} AS w FROM weights ORDER BY feature" + gradient, step = ad.sql_all( + ctx, + [ + f"WITH {LOSS} " + select.format("grad(loss, w)"), + f"WITH {LOSS} " + select.format("w - 0.01 * grad(loss, w)"), + ], + ) + np.testing.assert_allclose(gradient.to_dataset()["w"].values, grad) + np.testing.assert_allclose(step.to_dataset()["w"].values, w - 0.01 * grad) + + +def test_vjp_matches_closed_form(problem): + # Pulled back along the targets, pred = x @ w gives x.T @ y. + ctx, x, _, y = problem + got = ctx.sql( + f"WITH {LOSS}, cotangent AS (SELECT sample, y AS p FROM targets) " + "SELECT feature, vjp(pred, w, cotangent) AS w FROM weights " + "ORDER BY feature" + ).to_dataset() + np.testing.assert_allclose(got["w"].values, x.T @ y) + + +def test_jvp_matches_closed_form(problem): + # Pushed forward along t, pred = x @ w gives x @ t. + ctx, x, _, _ = problem + got = ctx.sql( + "WITH tangent AS (" + " SELECT feature, CAST(feature AS DOUBLE) AS w FROM weights) " + "SELECT x.sample, jvp(SUM(x.x * weights.w), weights.w, tangent) AS p " + "FROM data x JOIN weights ON x.feature = weights.feature " + "GROUP BY x.sample ORDER BY sample" + ).to_dataset() + np.testing.assert_allclose(got["p"].values, x @ np.arange(3)) + + +def test_stop_gradient_blocks_the_derivative(problem): + ctx, _, w, _ = problem + got = ctx.sql( + "WITH loss AS (SELECT SUM(w * ddx_stop_gradient(w)) AS l " + "FROM weights) " + "SELECT feature, grad(loss, w) AS w FROM weights ORDER BY feature" + ).to_dataset() + np.testing.assert_allclose(got["w"].values, w) + + +def test_unsupported_expression_raises(problem): + # The loss groups by a value of w, and then reads it. + ctx, *_ = problem + with pytest.raises(UnsupportedExpression, match="GROUP BY key"): + ctx.sql( + "WITH buckets AS (" + " SELECT ROUND(w) AS k, SUM(w * w) AS s FROM weights" + " GROUP BY ROUND(w)), " + "loss AS (SELECT SUM(k * s) AS l FROM buckets) " + "SELECT feature, grad(loss, w) AS g FROM weights" + ).collect() diff --git a/tests/test_ml_demos.py b/tests/test_ml_demos.py new file mode 100644 index 00000000..d8dc23ea --- /dev/null +++ b/tests/test_ml_demos.py @@ -0,0 +1,50 @@ +"""The ML demos in ``benchmarks/ml`` run, at their smallest sizes. + +Each demo checks its results against JAX and exits with an error on a +mismatch, so this runs each one and checks that it succeeds. It needs the +``ml`` extra, ddx's DuckDB backend and JAX, which the ``dev`` dependency +group installs. + +A new demo adds a line to ``DEMOS``: its script, and for each run of +it, the arguments that make it small enough to run in a few seconds. A +demo fails if a host refuses a step the demo doesn't expect it to. +""" + +import pathlib +import subprocess +import sys + +import pytest + +DEMOS_DIR = pathlib.Path(__file__).parent.parent / "benchmarks" / "ml" + +DEMOS = { + "01_attention.py": [["--sizes", "16"]], + "02_nanogpt.py": [ + ["--steps", "1"], + ["--steps", "1", "--optimizer", "adamw"], + ], +} + + +def test_every_demo_is_listed(): + scripts = {p.name for p in DEMOS_DIR.glob("[0-9][0-9]_*.py")} + assert scripts == set(DEMOS) + + +@pytest.mark.parametrize( + "script, args", + [(script, args) for script in sorted(DEMOS) for args in DEMOS[script]], + ids=lambda v: " ".join(v) if isinstance(v, list) else v, +) +def test_the_demo_runs_and_matches_jax(script, args): + for module in ("ddxdb", "duckdb", "jax"): + pytest.importorskip(module) + result = subprocess.run( + [sys.executable, str(DEMOS_DIR / script), *args], + check=False, + capture_output=True, + text=True, + timeout=600, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/uv.lock b/uv.lock index f98b1b8c..8e717d2f 100644 --- a/uv.lock +++ b/uv.lock @@ -520,6 +520,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/d0/bdc7ea7072c3925f3977e13e1e754e145fa42eb40bb5c1abff8028e0f791/datafusion-55.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b7191ff60defee8d9d9e5ff6aeefdd961bc200897cd60440b02d2a0f316902ef", size = 45733022, upload-time = "2026-10-12T16:20:42.607Z" }, ] +[[package]] +name = "ddxdb" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/49/58d710ea5363481e7bdd27567eb3944d0215459f4970eeb9fa8c7cee0266/ddxdb-0.4.2.tar.gz", hash = "sha256:169867a6a9bf7f9c9fd7093b079e26258a89b1614b147c7895a6b36fc3d8ee5c", size = 338983, upload-time = "2026-10-25T20:06:47.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/3b/599e254908a3b36867309841e1cb5a53f4a0c9b0cafb98ab4dedc5a609a9/ddxdb-0.4.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7e3eaea2ed3b21c834626ef793b66ed927d2d2ff1d4fd077e15277ab2f4d4c61", size = 6828615, upload-time = "2026-10-25T20:06:37.132Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/6d205060124dec023b6ee8f343275ca9a0365c36502f1695db590ea6991f/ddxdb-0.4.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:9c2129ebc9e6e1113c23fdecc2d168c940fe9240fd57b01a82d1468c31e132e1", size = 6706126, upload-time = "2026-10-25T20:06:39.012Z" }, + { url = "https://files.pythonhosted.org/packages/d7/4d/c060c21b3e3b31bc29d3e773ffc3ec89f84762d1f644cc40a81e1441701b/ddxdb-0.4.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9301f571a3ffb8b546db9b6c0c1b820d68ab911f6477a7f5b03a1d75ef3f28e0", size = 6799863, upload-time = "2026-10-25T20:06:41.11Z" }, + { url = "https://files.pythonhosted.org/packages/3d/43/13004abb0b3ea778c5aab57150a4d09f7990995814764a002efeb81c1643/ddxdb-0.4.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c6dcff7743858582f63a6da2474889cc6b633bb4fd272a7a9f505e87c4f4a867", size = 7297851, upload-time = "2026-10-25T20:06:43.219Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a4/bf14d45ce82aa23725d384145106926f191390ca39f3fb9cb4631fefddfe/ddxdb-0.4.2-cp310-abi3-win_amd64.whl", hash = "sha256:937aee96a2fbc61a12361d5c9ceee52205e02d214de605bbb16efd5790565ba1", size = 7253013, upload-time = "2026-10-25T20:06:45.579Z" }, +] + +[package.optional-dependencies] +duckdb = [ + { name = "duckdb" }, + { name = "pyarrow" }, +] + [[package]] name = "decorator" version = "5.2.1" @@ -561,44 +580,44 @@ wheels = [ [[package]] name = "duckdb" -version = "1.5.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/d4/298acf9331a80b3ce6ac64dd940e7e13f4058fb69d18914445f02e3c7bfe/duckdb-1.5.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b805507f88171b428b21c966c30e9a3d54e30b24528918a44ed0032542bc26f", size = 32702934, upload-time = "2026-07-22T10:53:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/d5/90/c489fb63d64b2e7ee109ce8460bdede003a0f256e5b41a03a2a1c4764058/duckdb-1.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b08e19cc856220d8a26fa62abc2264b349aff67255e9373c6a3f607addd56dc6", size = 17343604, upload-time = "2026-07-22T10:53:22.767Z" }, - { url = "https://files.pythonhosted.org/packages/0b/27/effa80a15b1f0c61c235622f797868485359e8c9ad6a8e358e7a0c479151/duckdb-1.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a17e6a922e42a5c06ed2353fe78c5dff2610f6632d603836f9606ad0bf754079", size = 15488179, upload-time = "2026-07-22T10:53:25.945Z" }, - { url = "https://files.pythonhosted.org/packages/5d/07/21212345c8d24ba62dceaa20be3b21f5c46f1510b1b42ce93bb058afe0c4/duckdb-1.5.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc38922c365c37720149f90d90b1e9823eb82dad6830855b5f87537fa6fc0c", size = 19367323, upload-time = "2026-07-22T10:53:30.23Z" }, - { url = "https://files.pythonhosted.org/packages/3f/d0/10371ae875fb4b5ef61bb892743b4b2e90c512b371fdf29317deb744857d/duckdb-1.5.5-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e238060db5ca59879882a6e9b015e2c65d5c64ddf281ba1d7a9a2033764152cf", size = 21476568, upload-time = "2026-07-22T10:53:33.486Z" }, - { url = "https://files.pythonhosted.org/packages/b2/35/09568ce617dd7bc0757b3d7b6a981660b9e4f0b7594de8ed776755eae740/duckdb-1.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:4acc72798ba1885a9c17d1242903d2cd502f13b1271c7677f7cab25d8578eceb", size = 13156129, upload-time = "2026-07-22T10:53:37.55Z" }, - { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, - { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, - { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, - { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, - { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, - { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, - { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, - { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, - { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, - { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, - { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, - { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, - { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, - { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, - { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, - { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, - { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, - { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, - { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, - { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, - { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, - { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, - { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, +version = "1.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/59/0b/d65ea3be00ea79aa276a8388bec588a9cbf409ce637c6d306e5316210d15/duckdb-1.5.6.tar.gz", hash = "sha256:166a91dbfacfc0c9f08cc76c0243cb6d3d4296bfab5bad72a3cfb63140a5b7c8", size = 18032957, upload-time = "2026-09-28T13:38:37.978Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/e1/5d05ecb59e3fd401414dacc9c969a326fe3a0b1eb07920058b656fe728d6/duckdb-1.5.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64db8a6700e81fe419fba130d8f1780686ad40fbf2eb69f78d2a1533728a0549", size = 32758341, upload-time = "2026-09-28T13:37:14.588Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d0/a382d9677097a1493049ae38f8219d751db989bfc72bf3a3766dc5af038e/duckdb-1.5.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6d1eac4de11779bb249b89b0544916ad65751da031df5c5f6d779c85b753109", size = 17372329, upload-time = "2026-09-28T13:37:17.997Z" }, + { url = "https://files.pythonhosted.org/packages/5c/dc/76577ce6520db9e4e8b33f90ec2f503cbf79652a1fd34e391b8043f921f2/duckdb-1.5.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:56355a543a79c7f4d8576d27edcbd9aaed19a562a0901188b021c10f4c818800", size = 15511297, upload-time = "2026-09-28T13:37:20.236Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3e/eeeef69e0c3cf3bb463b544435695647a4802437cfcc2b94035026bf5f84/duckdb-1.5.6-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a6b91bb9149950baeb5d02466c006550d0ea98b9d10f15f7d614a8eb32e174", size = 19428638, upload-time = "2026-09-28T13:37:22.436Z" }, + { url = "https://files.pythonhosted.org/packages/58/05/4ed0a651d55c8cbf9f7e826cfa95e67c9955a5db22a0c7c0cc5378f4a90c/duckdb-1.5.6-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dbd348e9ebdc8b28f1f9930efb5a74a382063c35d9c43901075566fbae50ab5c", size = 21534632, upload-time = "2026-09-28T13:37:25.139Z" }, + { url = "https://files.pythonhosted.org/packages/33/34/66f49f13f4286871e54b8d5478fb0b10e1f334f6ffe81536213e7fb55f09/duckdb-1.5.6-cp310-cp310-win_amd64.whl", hash = "sha256:f14551eef9180fc72869e2d9a2896410a8826169e22495e98a825abaa0eac1a7", size = 13178288, upload-time = "2026-09-28T13:37:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/e5/01e03d30b7ba33a030a4269fdca16ce445ce10f9d29b84a10fdbe0636ad2/duckdb-1.5.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c88700d0ee68ad149a0cc624df21b0f21efc136ea2449aaadd7cd0c9a564962a", size = 32757482, upload-time = "2026-09-28T13:37:29.916Z" }, + { url = "https://files.pythonhosted.org/packages/ba/4f/7f7be626a4649a3948ca646c84d6afc1a00121f292f98e6f0d9ed68330df/duckdb-1.5.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03e4f1b10a8b8ff476eb2b73955590fadbcef978da1167c593114c5edf763960", size = 17372997, upload-time = "2026-09-28T13:37:32.363Z" }, + { url = "https://files.pythonhosted.org/packages/1a/66/9d57573729348d800a0eebdd508f1a833d3714f72e984fef79b47f0e6c45/duckdb-1.5.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34623eaabd2c66ba5c20f1a39486321c3b7d32e4e0e001ced95f81e3372dd361", size = 15514224, upload-time = "2026-09-28T13:37:34.467Z" }, + { url = "https://files.pythonhosted.org/packages/57/ec/97f595214b3a27b4ca42b8cab6d8121c06f3537dcc4d2da7bca0332de4c5/duckdb-1.5.6-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56c0f71c6bee982e9c30568bb12371bf66b26bf129c75d8d7f60bc69d6590a2c", size = 19428776, upload-time = "2026-09-28T13:37:36.689Z" }, + { url = "https://files.pythonhosted.org/packages/68/4a/ab59f4c1f76fb89e28d23f19b2729538e0723c8d328a07e1b8c37f9ee128/duckdb-1.5.6-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73b108c04c932b36c2fa4e41110cc1c3c8cd510eb49f065f92d050be8e6929fd", size = 21537771, upload-time = "2026-09-28T13:37:39.548Z" }, + { url = "https://files.pythonhosted.org/packages/31/4f/9306c442ecad76f2a4d19f249e7fc8861f139dcf748315102eb69de8ca56/duckdb-1.5.6-cp311-cp311-win_amd64.whl", hash = "sha256:dda311932cf5aae955a53fe28a4fc1700c2ab5fa02dc1f165abdd5ec6c39141e", size = 13179009, upload-time = "2026-09-28T13:37:41.981Z" }, + { url = "https://files.pythonhosted.org/packages/a0/40/8a370e998293d3ebbbac4d926db30bb4ac5f700851a06ac31e7093bee386/duckdb-1.5.6-cp311-cp311-win_arm64.whl", hash = "sha256:df5ae02af278e084f54a9730a9f4f211ed736d0bd8f3bc12af925c2effb5b33d", size = 14046340, upload-time = "2026-09-28T13:37:44.187Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d5/d0ab77a0a1702a43171c93874f44c1f6481e30038bd3987df0d77a16a5c6/duckdb-1.5.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:48d07d0651aaeac2c3974afd37599970154b7b79b54c18f27c319c14ccf98d9d", size = 32810486, upload-time = "2026-09-28T13:37:47.254Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cd/b22201de5377faa3be6c38d5f3eaa504cb480392a448bed6a4d2239469b4/duckdb-1.5.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:79de3dfa8705b1ba0d59e7e3252e40ff399e0afd12f485502a6c7bf7c2fd809a", size = 17405278, upload-time = "2026-09-28T13:37:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/f9cfb1493bbdc2f095693a402e42dce1192077f9e11573f00baed6a748de/duckdb-1.5.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dcccce20965e6986cd083fdf192c461685ad0b93cd1ccd0b2a8207f1185f078b", size = 15532943, upload-time = "2026-09-28T13:37:52.927Z" }, + { url = "https://files.pythonhosted.org/packages/53/04/f65ccfaa5a833f2e570c4a140f03c8f95da416da9fe8ed08401f81f8242a/duckdb-1.5.6-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce89a1025a5317ebe9c520876c48032b5247ac574865486648b1a004f6009875", size = 19454940, upload-time = "2026-09-28T13:37:55.732Z" }, + { url = "https://files.pythonhosted.org/packages/4c/99/be75c788a492f8d77b7a1cdc1b19939ae7be0007f2028691ad371a1a33ee/duckdb-1.5.6-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc9619ed7d4ffa117b5155d84b44794366bb6635178d78ed5e13a6024845c757", size = 21568087, upload-time = "2026-09-28T13:37:58.191Z" }, + { url = "https://files.pythonhosted.org/packages/b5/95/889f8508960e47c0a7c75cc5bf57cde8512fc24f8db7b3129cca5388da42/duckdb-1.5.6-cp312-cp312-win_amd64.whl", hash = "sha256:09ff51b230219f0d8b47fc8a1e17fb595ba9fab0c3d96a6de4d00b8ff86b3cf1", size = 13190189, upload-time = "2026-09-28T13:38:00.407Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c9/baab503364a68309f8368c88e77f5341e7d94927bdf3e6d703f0e5035f3e/duckdb-1.5.6-cp312-cp312-win_arm64.whl", hash = "sha256:b8d795c8b2d5634b3269f974aa97f1fdf878f62f032317a52252a151b693fb1e", size = 14021977, upload-time = "2026-09-28T13:38:02.682Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/a476197fcba557738a588ec844747a19bc0a24b0e6f1809e308f29d68c0e/duckdb-1.5.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ae352646374cacf48e9981cf031191c494865192fc436d13667a2531fc5d1da3", size = 32810376, upload-time = "2026-09-28T13:38:05.148Z" }, + { url = "https://files.pythonhosted.org/packages/0c/6d/5466a2b53ddd557644dfa47a763f68748efccdf282e6ae7c4f1bcfb3da69/duckdb-1.5.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a1261e90785e9d29953293e44f60fa073bd1137098924e8de21a037a861b051", size = 17405385, upload-time = "2026-09-28T13:38:07.363Z" }, + { url = "https://files.pythonhosted.org/packages/d4/a0/bf87071170835ee4a34fe764fc11c1c6e7040a0e021b36c1b6f834a4c22f/duckdb-1.5.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97dd7a555b8f5298b76bc7d48a11cb2c64336e8de9bfde783cffb86ea9f54807", size = 15533132, upload-time = "2026-09-28T13:38:09.681Z" }, + { url = "https://files.pythonhosted.org/packages/31/e0/38095c8e140ecfbe847519ac07bcba94301b8fbb76b2870015e33e07f179/duckdb-1.5.6-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:364992ba1089a2b327391cfcb68fd0bd0ce9090cf293baef861a0ba6847abfee", size = 19454994, upload-time = "2026-09-28T13:38:11.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/61dd2876bbaa69cf77d7b5c620e52e8b25faae7096f4d2e4a812b52095d7/duckdb-1.5.6-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:644f54ce99b3b61844bc9a3fe80e0aecb1ea4084b1fffc4396d1569db6111679", size = 21568700, upload-time = "2026-09-28T13:38:14.258Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/100730e7785e85268be4d4d5bd62cfc8314e261d2f42efa208243eef35cb/duckdb-1.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:ced693d33ddcee2e5345f077d342c87d2aaa80e41c514e64c9ff2d4e5963c251", size = 13190707, upload-time = "2026-09-28T13:38:16.875Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2e/bc7f44eab4e89ee5c1cb427bb1168ad021d985042e6841ec0694c3d3d501/duckdb-1.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:41ecc75bb9328d72d154a705c1a653d2c5c60f686a5c0c6578aa80020753c884", size = 14020962, upload-time = "2026-09-28T13:38:19.007Z" }, + { url = "https://files.pythonhosted.org/packages/fb/62/a8a30a4c6b94c0861d348ed5633b963f6745a5525527530f02f3c1a7c931/duckdb-1.5.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:aa21d2ad803b2524326e8622d7d96b2bb1ff1d5b60368e1978ee805df9c21fb3", size = 32828003, upload-time = "2026-09-28T13:38:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/71/b7/1dcca0005eb8c67adf9fc06bf0cbb1d2bf4ea1974cc89e7a7c2ad66aac28/duckdb-1.5.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8a1b2ad27d414068cbca06c55cfa802eece10f86ea4812ff082f8ab4cb25fc85", size = 17413912, upload-time = "2026-09-28T13:38:23.915Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/e3ac175443550f3464f2d95731a8b0aae9b4dc3875c3a186c352262b43c2/duckdb-1.5.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c79c6d222b1d015cde73b5139087186b00db65357fb4e2c94c2308fbbf465a72", size = 15543122, upload-time = "2026-09-28T13:38:26.317Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/cc510a7952aba69d5cdca17f3ef61c95713d86143f2ee9aa3e097d38f50b/duckdb-1.5.6-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1052b8050ef5696e2c0d8c836949c72f3dd11f0690466acbea739613e8e2750b", size = 19457946, upload-time = "2026-09-28T13:38:28.877Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a5/6f8099d9a5a02ddff89e5c85875df3465054845b0920fb0703fbdf8dd2ec/duckdb-1.5.6-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c5e485e59613b8878d1670bcaa7a010f53c5a4da5ae8e08863e5e529ca6182", size = 21575132, upload-time = "2026-09-28T13:38:31.231Z" }, + { url = "https://files.pythonhosted.org/packages/9f/58/762f7159662d7859e201fa05ca29f306795daeabf84f3e087215a966b001/duckdb-1.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:ebcbd09cd8578ab1093393e9b16289cda0e8f1791ac595bf00eb5bad75c3cf00", size = 13713963, upload-time = "2026-09-28T13:38:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/69/64d165db322de13f5c3e75d377b6b9694df1821155ad1fa4b14b04601abc/duckdb-1.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:820a8384faef11cd86068ea48c5da57ce2d8f1c7b3d2bdb9be3398317a7c3728", size = 14514368, upload-time = "2026-09-28T13:38:35.676Z" }, ] [[package]] @@ -998,6 +1017,174 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "jax" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "jaxlib", version = "0.6.2", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "opt-einsum" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/1e/267f59c8fb7f143c3f778c76cb7ef1389db3fd7e4540f04b9f42ca90764d/jax-0.6.2.tar.gz", hash = "sha256:a437d29038cbc8300334119692744704ca7941490867b9665406b7f90665cd96", size = 2334091, upload-time = "2025-06-17T23:10:27.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/a8/97ef0cbb7a17143ace2643d600a7b80d6705b2266fc31078229e406bdef2/jax-0.6.2-py3-none-any.whl", hash = "sha256:bb24a82dc60ccf704dcaf6dbd07d04957f68a6c686db19630dd75260d1fb788c", size = 2722396, upload-time = "2025-06-17T23:10:25.293Z" }, +] + +[[package]] +name = "jax" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "jaxlib", version = "0.10.2", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes" }, + { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" } }, + { name = "opt-einsum" }, + { name = "scipy", version = "1.16.0", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/73/eb91d98fcadfa2cbcfdd4e417ab116e47eb20882acc5ee678e47c35d6b57/jax-0.10.2.tar.gz", hash = "sha256:bf77428a8c2e6904c4f46d5ab12aa5cfc6cad2179f07f7e4c0fc75ac86ef0639", size = 2775110, upload-time = "2026-06-17T23:44:57.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/82/5ab5211079a151b6f661529369c0c8e98ec64cabf5c0cf22a0a05af124d8/jax-0.10.2-py3-none-any.whl", hash = "sha256:724d73c4678d8b06f6a6ab4db1b8a2fea8cd4f1e2c2564f99601634ec7b8d1c6", size = 3219515, upload-time = "2026-06-17T23:42:41.259Z" }, +] + +[[package]] +name = "jax" +version = "0.12.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", +] +dependencies = [ + { name = "jaxlib", version = "0.12.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes" }, + { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" } }, + { name = "opt-einsum" }, + { name = "scipy", version = "1.16.0", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/5b/5b7f007852809973310ed99eede7db6e9e1cd60914f8fe90b091ae264db4/jax-0.12.0.1.tar.gz", hash = "sha256:f543fb1151c351d328b7dd9e40865c01bf9ae10cdf7e21637196172891d51914", size = 3024361, upload-time = "2026-10-21T21:07:07.969Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/5d/be6a4b67161462d1a0a97c7405459c2f76a4c23c66b39ad496527f68a17d/jax-0.12.0.1-py3-none-any.whl", hash = "sha256:e9dfd256ba090259b10f97f0755d4fc92535b8cb38c0a40df6488a8c8a7dd8ae", size = 3466959, upload-time = "2026-10-21T21:04:05.013Z" }, +] + +[[package]] +name = "jaxlib" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/c5/41598634c99cbebba46e6777286fb76abc449d33d50aeae5d36128ca8803/jaxlib-0.6.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4601b2b5dc8c23d6afb293eacfb9aec4e1d1871cb2f29c5a151d103e73b0f8", size = 54298019, upload-time = "2025-06-17T23:10:36.916Z" }, + { url = "https://files.pythonhosted.org/packages/81/af/db07d746cd5867d5967528e7811da53374e94f64e80a890d6a5a4b95b130/jaxlib-0.6.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4205d098ce8efb5f7fe2fe5098bae6036094dc8d8829f5e0e0d7a9b155326336", size = 79440052, upload-time = "2025-06-17T23:10:41.282Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d8/b7ae9e819c62c1854dbc2c70540a5c041173fbc8bec5e78ab7fd615a4aee/jaxlib-0.6.2-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:c087a0eb6fb7f6f8f54d56f4730328dfde5040dd3b5ddfa810e7c28ea7102b42", size = 89917034, upload-time = "2025-06-17T23:10:45.897Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e5/87e91bc70569ac5c3e3449eefcaf47986e892f10cfe1d5e5720dceae3068/jaxlib-0.6.2-cp310-cp310-win_amd64.whl", hash = "sha256:153eaa51f778b60851720729d4f461a91edd9ba3932f6f3bc598d4413870038b", size = 57896337, upload-time = "2025-06-17T23:10:50.179Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/6899b0aed36a4acc51319465ddd83c7c300a062a9e236cceee00984ffe0b/jaxlib-0.6.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a208ff61c58128d306bb4e5ad0858bd2b0960f2c1c10ad42c548f74a60c0020e", size = 54300346, upload-time = "2025-06-17T23:10:54.591Z" }, + { url = "https://files.pythonhosted.org/packages/e6/03/34bb6b346609079a71942cfbf507892e3c877a06a430a0df8429c455cebc/jaxlib-0.6.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:11eae7e05bc5a79875da36324afb9eddd4baeaef2a0386caf6d4f3720b9aef28", size = 79438425, upload-time = "2025-06-17T23:10:58.356Z" }, + { url = "https://files.pythonhosted.org/packages/80/02/49b05cbab519ffd3cb79586336451fbbf8b6523f67128a794acc9f179000/jaxlib-0.6.2-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:335d7e3515ce78b52a410136f46aa4a7ea14d0e7d640f34e1e137409554ad0ac", size = 89920354, upload-time = "2025-06-17T23:11:03.086Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7a/93b28d9452b46c15fc28dd65405672fc8a158b35d46beabaa0fe9631afb0/jaxlib-0.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6815509997d6b05e5c9daa7994b9ad473ce3e8c8a17bdbbcacc3c744f76f7a0", size = 57895707, upload-time = "2025-06-17T23:11:07.074Z" }, + { url = "https://files.pythonhosted.org/packages/ac/db/05e702d2534e87abf606b1067b46a273b120e6adc7d459696e3ce7399317/jaxlib-0.6.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:34d8a684a8be949dd87dd4acc97101b4106a0dc9ad151ec891da072319a57b99", size = 54301644, upload-time = "2025-06-17T23:11:10.977Z" }, + { url = "https://files.pythonhosted.org/packages/0d/8a/b0a96887b97a25d45ae2c30e4acecd2f95acd074c18ec737dda8c5cc7016/jaxlib-0.6.2-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87ec2dc9c3ed9ab936eec8535160c5fbd2c849948559f1c5daa75f63fabe5942", size = 79439161, upload-time = "2025-06-17T23:11:14.822Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e8/71c2555431edb5dd115cf86a7b599aa7e1be26728d89ae59aa11251d299c/jaxlib-0.6.2-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f1dd09b481a93c1d4c750013f467f74194493ba7bd29fcd4d1cec16e3a214f65", size = 89942952, upload-time = "2025-06-17T23:11:19.181Z" }, + { url = "https://files.pythonhosted.org/packages/de/3a/06849113c844b86d20174df54735c84202ccf82cbd36d805f478c834418b/jaxlib-0.6.2-cp312-cp312-win_amd64.whl", hash = "sha256:921dbd4db214eba19a29ba9f2450d880e08b2b2c7b968f28cc89da3e62366af4", size = 57919603, upload-time = "2025-06-17T23:11:23.207Z" }, + { url = "https://files.pythonhosted.org/packages/af/38/bed4279c2a3407820ed8bcd72dbad43c330ada35f88fafe9952b35abf785/jaxlib-0.6.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bff67b188133ce1f0111c7b163ac321fd646b59ed221ea489063e2e0f85cb967", size = 54300638, upload-time = "2025-06-17T23:11:26.372Z" }, + { url = "https://files.pythonhosted.org/packages/52/dc/9e35a1dc089ddf3d6be53ef2e6ba4718c5b6c0f90bccc535a20edac0c895/jaxlib-0.6.2-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:70498837caf538bd458ff6858c8bfd404db82015aba8f663670197fa9900ff02", size = 79439983, upload-time = "2025-06-17T23:11:30.016Z" }, + { url = "https://files.pythonhosted.org/packages/34/16/e93f0184b80a4e1ad38c6998aa3a2f7569c0b0152cbae39f7572393eda04/jaxlib-0.6.2-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:f94163f14c8fd3ba93ae14b631abacf14cb031bba0b59138869984b4d10375f8", size = 89941720, upload-time = "2025-06-17T23:11:34.62Z" }, + { url = "https://files.pythonhosted.org/packages/06/b9/ea50792ee0333dba764e06c305fe098bce1cb938dcb66fbe2fc47ef5dd02/jaxlib-0.6.2-cp313-cp313-win_amd64.whl", hash = "sha256:b977604cd36c74b174d25ed685017379468138eb747d865f75e466cb273c801d", size = 57919073, upload-time = "2025-06-17T23:11:39.344Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/9596391c104a0547fcaf6a8c72078bbae79dbc8e7f0843dc8318f6606328/jaxlib-0.6.2-cp313-cp313t-manylinux2014_aarch64.whl", hash = "sha256:39cf9555f85ae1ce2e2c1a59fc71f2eca4f9867a7cb934fef881ba56b11371d1", size = 79579638, upload-time = "2025-06-17T23:11:43.054Z" }, + { url = "https://files.pythonhosted.org/packages/10/79/f6e80f7f4cacfc9f03e64ac57ecb856b140de7c2f939b25f8dcf1aff63f9/jaxlib-0.6.2-cp313-cp313t-manylinux2014_x86_64.whl", hash = "sha256:3abd536e44b05fb1657507e3ff1fc3691f99613bae3921ecab9e82f27255f784", size = 90066675, upload-time = "2025-06-17T23:11:47.454Z" }, +] + +[[package]] +name = "jaxlib" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.16.0", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/2c/7038fc73154307389631b5b2dbe5ac529e1918eecc19a27e6644ad114bbf/jaxlib-0.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5a98873fc867623b81f2bee15d554b8edd6588a183d01fa50d21b1e3db96ff2b", size = 61429039, upload-time = "2026-06-17T23:43:44.858Z" }, + { url = "https://files.pythonhosted.org/packages/66/c6/d69a0a33046f84930b89387861c061996d5207671b35080898679ca9960a/jaxlib-0.10.2-cp311-cp311-manylinux_2_27_aarch64.whl", hash = "sha256:d44565dcfd1b4f60f76d911c6512118a8a4fc764bdef92663fecb8bfccd54f23", size = 81079180, upload-time = "2026-06-17T23:43:48.245Z" }, + { url = "https://files.pythonhosted.org/packages/e2/27/fb54e3265c0ffcb687f93e9fb761c589acebbe958c3fed1b2c74c3f0e782/jaxlib-0.10.2-cp311-cp311-manylinux_2_27_x86_64.whl", hash = "sha256:1faca3c5d4662cb4a6130a68105d68bb520764817e165d6eebfd6786c0d1f30f", size = 85448560, upload-time = "2026-06-17T23:43:51.724Z" }, + { url = "https://files.pythonhosted.org/packages/21/bc/31fbb3d892c3cb97c73af9226eca63d60d8e224017145bdb6871d1d24da6/jaxlib-0.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:e7a9214e6b0b9e0825d905573d1bbf2253c20e9d7464a63e085b60519975553f", size = 65867603, upload-time = "2026-06-17T23:43:54.939Z" }, + { url = "https://files.pythonhosted.org/packages/ca/93/ee9cc8743191544f65d26ab7eeb82d65968fe60905662d1a5554d056654b/jaxlib-0.10.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47bb7c011515ea862be7e8313f40f9c56cbec09dc98a0fcb5016785fcd454c01", size = 61434612, upload-time = "2026-06-17T23:43:57.808Z" }, + { url = "https://files.pythonhosted.org/packages/11/06/8cc36021bf74d617c312eeed94c280282bb1bcbb32b63f2a42b10ae41575/jaxlib-0.10.2-cp312-cp312-manylinux_2_27_aarch64.whl", hash = "sha256:53b72977ae582c03a9e8e1cdee1efbf8ebc1418270965b0e69eade57acf40331", size = 81085366, upload-time = "2026-06-17T23:44:01.067Z" }, + { url = "https://files.pythonhosted.org/packages/48/17/38b718af2353dba7753300871e83fbb64a88a772e12727ae27373ab675ce/jaxlib-0.10.2-cp312-cp312-manylinux_2_27_x86_64.whl", hash = "sha256:fe88ec443714c4379968b6c109f9fa617c7ad19b802828e4d7bf861cd66da4b7", size = 85467828, upload-time = "2026-06-17T23:44:04.238Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c2/d41d13826ebdfe62e56cd87ba70fab3bb9fcbea4a6c9086739a91667e5bf/jaxlib-0.10.2-cp312-cp312-win_amd64.whl", hash = "sha256:4b08f5fbc596b83f76308181863996f93d901d1f09cfd4e130a65c1998e1b371", size = 65900139, upload-time = "2026-06-17T23:44:07.476Z" }, + { url = "https://files.pythonhosted.org/packages/c2/68/eaa4cebe253359196a8e80a33b242959e27d8d2a6ae3d09339f21da2acb8/jaxlib-0.10.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4df530afa354a22dc1747a5d560640450cbb895d49889338a3f58c76a4c76c8e", size = 61434805, upload-time = "2026-06-17T23:44:10.511Z" }, + { url = "https://files.pythonhosted.org/packages/25/c1/4b884ea5962b6beb3c0f93742db54246bbf8b3274e48b0aca47908e454be/jaxlib-0.10.2-cp313-cp313-manylinux_2_27_aarch64.whl", hash = "sha256:45b28b0238697ab74bbcf20411aafb6db42acc31836cc2fd711e5cf056bf9556", size = 81084260, upload-time = "2026-06-17T23:44:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/36/ac/4ee28c65861605945223145fbcd3c9362ec2255ddf7d917574e205548c82/jaxlib-0.10.2-cp313-cp313-manylinux_2_27_x86_64.whl", hash = "sha256:9e4818b4a8756fd3918766ca2aa5342125809f4f08a6fe46026d4386e7c23644", size = 85467706, upload-time = "2026-06-17T23:44:17.471Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/9918b0f77a25a1299818c0610305ca2bea38ed90584f4489b60357e2dd39/jaxlib-0.10.2-cp313-cp313-win_amd64.whl", hash = "sha256:c75d6f1df1c9cff08e110b4a21c79560fdc502f4288972d6b117d25dafd44352", size = 65897894, upload-time = "2026-06-17T23:44:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/70df11da52a8b1a826184cccc05a3fec8aed76058a980021873fba3069cb/jaxlib-0.10.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4c202d8ff7c1f3b5049dbd8f1e30e52759cd4e0a5835f0b3c7ae076a05818e28", size = 61564746, upload-time = "2026-06-17T23:44:24.421Z" }, + { url = "https://files.pythonhosted.org/packages/23/5c/184a648ea5db6c8b1a08fc5784c157b4c557255e009fb56091393df3c6de/jaxlib-0.10.2-cp313-cp313t-manylinux_2_27_aarch64.whl", hash = "sha256:b7b029bb95d981566750475b9719a9d6b66ed5dd2748851667899b6cfe075299", size = 81204888, upload-time = "2026-06-17T23:44:27.57Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5c/539596a55265711d74147913278bcdc38412980be7d74d9c9d860297c486/jaxlib-0.10.2-cp313-cp313t-manylinux_2_27_x86_64.whl", hash = "sha256:e8b126097d609b0c6e6786e89f6dd6978adc02ebd5f63a1c61293fbac7821305", size = 85583810, upload-time = "2026-06-17T23:44:30.962Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0d/27471ec9f1d04674f6e62de809412371e097aed3eca7d9483e677c54c214/jaxlib-0.10.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:72eba28b12fee02616fa42aa4b881b4ab62d7757c7843c462401d3fb34a27be4", size = 61446097, upload-time = "2026-06-17T23:44:34.196Z" }, + { url = "https://files.pythonhosted.org/packages/af/c8/941a7f7f37510f51290a5bd1a413aeef977fb8ba8adc0cfe8391233a764c/jaxlib-0.10.2-cp314-cp314-manylinux_2_27_aarch64.whl", hash = "sha256:f18f56fee90699cfba9b6627045a7a299702cb0e2af82ce180d9a6a7c8048093", size = 81096546, upload-time = "2026-06-17T23:44:37.486Z" }, + { url = "https://files.pythonhosted.org/packages/69/77/ac054882c220872512df28d16aeb648fe0e651efbb5be4fd7c4817fd88b0/jaxlib-0.10.2-cp314-cp314-manylinux_2_27_x86_64.whl", hash = "sha256:ca34f363197fb0ac4082582ca755007910369e33f8a8ba3d35ed94b71070107d", size = 85472993, upload-time = "2026-06-17T23:44:40.904Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7d/c592d1fa69c210be0d2743fffc598dfc2f54efa9671c5f6f5d1e151c6f4a/jaxlib-0.10.2-cp314-cp314-win_amd64.whl", hash = "sha256:99818b0a18adc0b899abf4873795e8d65169441d87ab2e5cbb228e73d0f25808", size = 68376553, upload-time = "2026-06-17T23:44:44.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/9b/91b00ec74985d29708b50420b4103c1f651c8f1c253d4fcb49d1bbb532cd/jaxlib-0.10.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fc62997fce8831819551a2a5469a818169b09582b5b648c102d11ac7205bb812", size = 61564620, upload-time = "2026-06-17T23:44:48.217Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c7/49d2b19c3b3105c30e1d3af2062e82e1977fb239d3dc3cbb583ef676dda7/jaxlib-0.10.2-cp314-cp314t-manylinux_2_27_aarch64.whl", hash = "sha256:a24d6e3cba263978293eae8b41330d5ccf24d6cdd1a6bcd4e82aff34e767620d", size = 81206365, upload-time = "2026-06-17T23:44:51.419Z" }, + { url = "https://files.pythonhosted.org/packages/bf/99/006cedf443f4a01f2088651facce79b2105bfb4905bfe9162eb0920a6dfb/jaxlib-0.10.2-cp314-cp314t-manylinux_2_27_x86_64.whl", hash = "sha256:5a2ac7aed7c4e661f67600bbcdec9e589151c1efec91f4cdb8d484af1a45c895", size = 85584458, upload-time = "2026-06-17T23:44:55.377Z" }, +] + +[[package]] +name = "jaxlib" +version = "0.12.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", +] +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.16.0", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/73/47d66eeab3bb5a48009772f4fa572e46bdf8049ecb180f83d544c933462f/jaxlib-0.12.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db2cc8584d7d0d166293ef2475b726988a10f6f9b5a8e3b8e3712415e5570380", size = 60710018, upload-time = "2026-10-21T21:05:22.443Z" }, + { url = "https://files.pythonhosted.org/packages/c1/cf/f9baa1ba672ced2a7bce86a6afebbc4b565daf04508b7fdeddefee2e9ac0/jaxlib-0.12.0.1-cp312-cp312-manylinux_2_27_aarch64.whl", hash = "sha256:40812201c3bcd4cb104b88f71f58fec4cbd0c32ba90e5c92b3fa8f40819f6774", size = 80858298, upload-time = "2026-10-21T21:05:27.085Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e0/d39abd53ba408219062c1156bf3b9f965718f66bda8d625cb28453d63c2c/jaxlib-0.12.0.1-cp312-cp312-manylinux_2_27_x86_64.whl", hash = "sha256:78318e9ac79c92972f81c77090f557bdb4a5a49821751c6ee3f6a9c66a948842", size = 86848931, upload-time = "2026-10-21T21:05:31.389Z" }, + { url = "https://files.pythonhosted.org/packages/de/45/e87c51650ccaba0c08dea0206e132db3093b4e5265ebbb5449f664a955b6/jaxlib-0.12.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:aa24baf4aafd78bbf7104a81f01eadc40332b8b90098634d14d585cd0477eeaf", size = 71385367, upload-time = "2026-10-21T21:05:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/df/ce/49eab4b18b65406dd80059a3cbfb27c8d75596d5710a0623ced2bbd7c3d0/jaxlib-0.12.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8be160b0b847bc57168e4305713756f6da0d0bba26715e64f9d2f46c0a7a6b42", size = 60709320, upload-time = "2026-10-21T21:05:39.353Z" }, + { url = "https://files.pythonhosted.org/packages/f6/8d/5c4c74d419be62a09bdf44a13293f726c399623b048805360aaf0788b34e/jaxlib-0.12.0.1-cp313-cp313-manylinux_2_27_aarch64.whl", hash = "sha256:c8de1cad382713114f657f406383edc53ee9fda4679db4e2b09d1dfefe1c4e45", size = 80854031, upload-time = "2026-10-21T21:05:43.268Z" }, + { url = "https://files.pythonhosted.org/packages/78/9d/225a525821b08b4ce85a7343fc8d21dcda6e75a825b90e9b6a29feedc2ee/jaxlib-0.12.0.1-cp313-cp313-manylinux_2_27_x86_64.whl", hash = "sha256:fdc56999ba690bd915f7029613269848eaaf85d231ec8abc46610d3b840c189c", size = 86849021, upload-time = "2026-10-21T21:05:48.262Z" }, + { url = "https://files.pythonhosted.org/packages/d8/34/e020e9279dc6bde837a20c31f97f9c7bbead144caa9b89bf36f7d94801a1/jaxlib-0.12.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:3011a4ef1fc3dbe7c03fee30dee8f0b85a9ce92e5673271f5065d4cb400587a5", size = 71385208, upload-time = "2026-10-21T21:05:52.509Z" }, + { url = "https://files.pythonhosted.org/packages/74/d4/f5870719e158fdf7d4edc3b2ac3f81f48341a813bdf442fcb24a3f4547d3/jaxlib-0.12.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:93b8da183f0235603c460b530da8cd6eb23ea7f1c6adc4f82a43a6035497d466", size = 60710508, upload-time = "2026-10-21T21:05:55.865Z" }, + { url = "https://files.pythonhosted.org/packages/67/a7/dd962170691604061286ef750ddd9317efd20c554d587ea2486713033441/jaxlib-0.12.0.1-cp314-cp314-manylinux_2_27_aarch64.whl", hash = "sha256:609deffaea388d1ed9d1c5104d8f6041791f00771adc305f7316b742f502fd6b", size = 80859721, upload-time = "2026-10-21T21:05:59.819Z" }, + { url = "https://files.pythonhosted.org/packages/29/bf/14bbe929413ed61e13d2cc5dd2368b710cc638fcaf4632997b75fceff568/jaxlib-0.12.0.1-cp314-cp314-manylinux_2_27_x86_64.whl", hash = "sha256:b6079a7669d624ebd42d1571339566e62cfe7225f4b5a20b8e16773ea9a8357a", size = 86849658, upload-time = "2026-10-21T21:06:04.735Z" }, + { url = "https://files.pythonhosted.org/packages/e5/7b/940134f193e705dea624e55f597ef475f824e4ec2da34973bf7c3794938c/jaxlib-0.12.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfba501b599e4083a4463cd58c36a1271249a415a69f2c8054083554f281efcc", size = 73913218, upload-time = "2026-10-21T21:06:11.627Z" }, + { url = "https://files.pythonhosted.org/packages/39/c3/5402e68283e2d07f8544b777ed1628b0514da62f2a50c1605686b15f3597/jaxlib-0.12.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d35f6faca955676b4b46418028bf7fa7af88c6c9bb6b691c6e3005fc37839fce", size = 60809745, upload-time = "2026-10-21T21:06:15.299Z" }, + { url = "https://files.pythonhosted.org/packages/12/c5/dd642aa73dbc29fd1c1b830a221b9c4483ae7e48086d7e0761c11de0ff71/jaxlib-0.12.0.1-cp314-cp314t-manylinux_2_27_aarch64.whl", hash = "sha256:f5d9e527c1736eca5ca82bdc6d6f45421d8962ec7c9a8db153e4547311cc6876", size = 80951733, upload-time = "2026-10-21T21:06:20.105Z" }, + { url = "https://files.pythonhosted.org/packages/23/4b/1df08833de32175aff13d04c0b02f09ac5e24b797e95a2f8d7d4ebdefa5b/jaxlib-0.12.0.1-cp314-cp314t-manylinux_2_27_x86_64.whl", hash = "sha256:d8f14f5253c032f9a6f688734e3b63d911dbdcac9fd698ceb71f660f1956ddef", size = 86943877, upload-time = "2026-10-21T21:06:24.846Z" }, + { url = "https://files.pythonhosted.org/packages/51/1f/cde063eb5e2e4f9b80b4b6317450027d86c0c4525e9e1ba9f3db477c0ea1/jaxlib-0.12.0.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00acba6d262f20110289e5d3384cd92298ae362226b66f77f92c518fab55c2c6", size = 60712539, upload-time = "2026-10-21T21:06:34.593Z" }, + { url = "https://files.pythonhosted.org/packages/b3/5b/932cb1f2ea25048fe7107eb01a0b2fd704e736a238ed676973e9fc9ef7cf/jaxlib-0.12.0.1-cp315-cp315-manylinux_2_27_aarch64.whl", hash = "sha256:7978a182ce9d1a8314dd693ab466ff24b961950ce6617f03498395e467f72e42", size = 80858593, upload-time = "2026-10-21T21:06:38.689Z" }, + { url = "https://files.pythonhosted.org/packages/af/c2/640a003f7c331e2d3b4dfd7af2283bd4af3f752c385abd30b7882f24e760/jaxlib-0.12.0.1-cp315-cp315-manylinux_2_27_x86_64.whl", hash = "sha256:44d3d534664fe762f32c5b831d556384bd96f9afad2e40aa207050f0b6e2c9f6", size = 86851688, upload-time = "2026-10-21T21:06:42.649Z" }, + { url = "https://files.pythonhosted.org/packages/94/a6/f3c13d4c3c0e6958331f38c6508a7f73d19de1cf44c49612a6286e7853cf/jaxlib-0.12.0.1-cp315-cp315-win_amd64.whl", hash = "sha256:9fa1147a3ccc916c77ab2063229d6ddaf82699d6b485bf6ad4dac265cfd8e0ee", size = 73915243, upload-time = "2026-10-21T21:06:46.511Z" }, + { url = "https://files.pythonhosted.org/packages/44/59/78b61a557fa58feee23e7e4ac4c1cec7c4ad5085536ddb22485d6e2e8a8f/jaxlib-0.12.0.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:78d16c6d6342cbe8de117f23b5c6d98339e582453221e977920dcbe5e05d72d6", size = 60812353, upload-time = "2026-10-21T21:06:49.984Z" }, + { url = "https://files.pythonhosted.org/packages/b3/b1/76a8e1368610b210684f8d74e24873365af89db0a6a29693f20ee8032ac3/jaxlib-0.12.0.1-cp315-cp315t-manylinux_2_27_aarch64.whl", hash = "sha256:b8435a50e7f09481156ab15a7522e3dd96ee6655f57f494933c5b9ffc6101332", size = 80950599, upload-time = "2026-10-21T21:06:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/71/f3d7e2408ddb179fd517e5c3b6f8722261d04ba6b5aa583f7e4d25535323/jaxlib-0.12.0.1-cp315-cp315t-manylinux_2_27_x86_64.whl", hash = "sha256:085c5196ac9e735a9313dee67a536e70ca09fe61c4ce19e694849509add2ecac", size = 86944972, upload-time = "2026-10-21T21:07:04.98Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1318,6 +1505,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/28/79f0f8de97cce916d5ae88a7bee1ad724855e83e6019c0b4d5b3fabc80f3/mkdocstrings_python-2.0.3-py3-none-any.whl", hash = "sha256:0b83513478bdfd803ff05aa43e9b1fca9dd22bcd9471f09ca6257f009bc5ee12", size = 104779, upload-time = "2026-02-20T10:38:34.517Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/72/307d7c4bd0600601c7133fba5cb78af7db968152951c1cd473abb1cda782/ml_dtypes-0.6.0.tar.gz", hash = "sha256:5e60251d32ced5598972e4d5e06a2f044341f9291402551a3f6f0ec44f9299b0", size = 3032327, upload-time = "2026-08-13T14:14:40.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/15/01285c64133ea38abf3b990a704d7d30e50daea2806d150bcc4163495d35/ml_dtypes-0.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bad8d1dd5bed060a29332b99d63d0e5c2969081e1c6ea54adfbccfdfa783be44", size = 566808, upload-time = "2026-08-13T14:13:50.012Z" }, + { url = "https://files.pythonhosted.org/packages/e7/54/850d9b8b35549182f7c7f2cf742ce75c853ee880101bbc51cca0d62732e3/ml_dtypes-0.6.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:008382aeab529df5d3f00501ad9a7dcd64494d4b5b1971fc4c79019e6c1f5010", size = 356865, upload-time = "2026-08-13T14:13:51.339Z" }, + { url = "https://files.pythonhosted.org/packages/e9/15/844f5402145ce73bec8eb3afeb9f41d2bf99e0c8617c93f9e9886f26b419/ml_dtypes-0.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6ec0d244a5bba12239025389ad88bbfb45f9f10e25ab4f678e9a4768ebd47532", size = 412036, upload-time = "2026-08-13T14:13:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/f8/63/efc9257a1ef0f53dfc76dedfe70d7d35118fbcdb810bb48cb7323ebd0b87/ml_dtypes-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:03ce583adfce34ad33aa9e1fc7a8344dcf90ea776cc4ef0e5a48d4eae84e5d20", size = 433668, upload-time = "2026-08-13T14:13:53.668Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2c/318cd1a9014c63939ffe687e19559ae12831fcc37d66c71ad1f616f1ffd6/ml_dtypes-0.6.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f4f59f83c82ab480e924b988e7b1b4eb4de836dfcf5390c6f59148d1a00e1d02", size = 566813, upload-time = "2026-08-13T14:13:55.053Z" }, + { url = "https://files.pythonhosted.org/packages/d9/83/706b8a39449f0d55a7d5f7d07a169da4decfafae8a1f4983a9236d4b49e8/ml_dtypes-0.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7728c0420ec1c338564fc8b01015ff2d58567e70f17fedce5a0a7c0308c0d5b9", size = 356864, upload-time = "2026-08-13T14:13:56.249Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b1/135a7bf47633f5b9184f0d0316af819884124d12b40965064bd216266514/ml_dtypes-0.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c8e39b53e90afda8ce52859c93de4dba3e02b76d85dcf091cc469f9184c6dae", size = 412043, upload-time = "2026-08-13T14:13:57.614Z" }, + { url = "https://files.pythonhosted.org/packages/07/23/8870bb62d6e499d6bcbc1242b9f11689bae00a3d39d3684a9aefad8b6ee6/ml_dtypes-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:3035518e3e19add1a4cac9236ab22888b208a4074912514313ccb2d6d242cde8", size = 433670, upload-time = "2026-08-13T14:13:59.097Z" }, + { url = "https://files.pythonhosted.org/packages/cf/7a/5d8fbe24d0bffd0d7cb5165a89f8ab7c3de000f26d6705242aeed99d583c/ml_dtypes-0.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:5a519c9e95a216fbcb8e759793ef7fb40793fc803ed839142d6dc5be9be5bc89", size = 551915, upload-time = "2026-08-13T14:14:00.368Z" }, + { url = "https://files.pythonhosted.org/packages/84/6a/441eb053b078954f7fea284dfb288701884d0a1404d39babb858e1649023/ml_dtypes-0.6.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5359c588cc62de6f78d7430f06b65853d884955494d86d6ad90b6dd64a3f3a08", size = 565447, upload-time = "2026-08-13T14:14:01.737Z" }, + { url = "https://files.pythonhosted.org/packages/ed/cf/87e8a6c57eed63a91782a0d229856ddf73e138ce004dd71e2799a9dcdb33/ml_dtypes-0.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37da32aa97749251025666d62372775019594577b9c9e9cfda83bed48d778fdb", size = 360227, upload-time = "2026-08-13T14:14:02.938Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f9/7d76c1eae866f5d4636401b31b6d6dd90e4b4ced1fa7cfdfcca9c60e4bd3/ml_dtypes-0.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b4a480aa8fd54a1805b8ac10f3f91763926a74f73c0c364c10f9231854f4170", size = 409890, upload-time = "2026-08-13T14:14:04.248Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/9c61ec2760b5cbfb1c6558d5c991a6d8fd3271053c32db20506a9a90272b/ml_dtypes-0.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:2a3e9d53925597fbffafd2a37048dadeddd0bdaba58058f6ae0869ed709a184d", size = 439333, upload-time = "2026-08-13T14:14:05.501Z" }, + { url = "https://files.pythonhosted.org/packages/6a/57/780ca3e5ab135b9fbdd8e5441abf5f801b30398371b691291e05ab9834c0/ml_dtypes-0.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:6eaed129a4afe90694b8685e2f9b6294849f5eda4af9a15be83a4326eeebd775", size = 552268, upload-time = "2026-08-13T14:14:06.866Z" }, + { url = "https://files.pythonhosted.org/packages/50/51/fd1582b8f5ed8a9e7be0e161a6ea0dff70cb280479a12178df0b3a72700e/ml_dtypes-0.6.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:084dfe51a7ad58b171f05115f8226ed4233a454a1611371947e806e76f0c638d", size = 565468, upload-time = "2026-08-13T14:14:08.5Z" }, + { url = "https://files.pythonhosted.org/packages/d2/22/20fd70ca6ed12446cb92d5b2a7745bd185f9d8b8cdeeadad976574398e6b/ml_dtypes-0.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28d676428b104bb9717b0928bc5c5129f2d6b51b6727587cc4289e7bf8713cb5", size = 360232, upload-time = "2026-08-13T14:14:09.873Z" }, + { url = "https://files.pythonhosted.org/packages/89/a5/da8ae6c6f1babe4b68e3e55d43d39b529e29774f10e0910671a6b8c86eb8/ml_dtypes-0.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26b1f1fa4f0435a2946859823f6e2bf06796f1e9f10f5a05b08a5e3c8f46ff69", size = 410169, upload-time = "2026-08-13T14:14:11.036Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/4561acefa00fa4bcbfb82ca6a48578b41f372cd7dd7cdd6eb4720abc2e5f/ml_dtypes-0.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:fb87f46b4f7ad7b5d3ad8f4b452b024bd4229d44c8ff934798c1fe656210387a", size = 439357, upload-time = "2026-08-13T14:14:12.172Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5d/6a01538e507ef0ed5e879985b13a92467bf8960696fb1131f8b8cadc60ff/ml_dtypes-0.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:57ed0d6b4ac5e7868361303a9c57fbcf63b768236ee14456f585dfcf260d0292", size = 552278, upload-time = "2026-08-13T14:14:13.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7a/97dc35667b7c9db33c5344c673cd27f87e34771875ea7100138726132ac9/ml_dtypes-0.6.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:84fa136b8602c8c39e3b6cb24918960cd6f36cade7a70376f56770729cd56510", size = 562551, upload-time = "2026-08-13T14:14:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/db/48/77f0ede10558d0d935da2e3276ed7e9c8cc2bad3463b9a0b66b03fc60be2/ml_dtypes-0.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:317be9967fb84b0ce4e80e6b1bf71213d21971621cf6f1e501a63602a95297bf", size = 360334, upload-time = "2026-08-13T14:14:16.079Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b1/1831dd8c9b06c013085d31a2ac4f03392d43bd36bfc6ff591a08bcedc1cf/ml_dtypes-0.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f490c003369ce60e514a0c3b12374f05274c101fee1bead6740ec8a564032b0", size = 409966, upload-time = "2026-08-13T14:14:17.477Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ad/9c32c53f823dda3742df19a79c10bc198365937873ea125ba65747440c23/ml_dtypes-0.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:d574c2b28921dc72e869df248f1a278f6eee176a1f237c8642e1a71eb15f3977", size = 457224, upload-time = "2026-08-13T14:14:18.608Z" }, + { url = "https://files.pythonhosted.org/packages/41/3d/dd98205418a13353d41c52bf5326d8cbec515aace46174e23c6ea01c2978/ml_dtypes-0.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:f4adb4af61516510d786cf8c01851a66f6d3ddfa79e1144deaa5b40d8507231e", size = 568378, upload-time = "2026-08-13T14:14:19.843Z" }, + { url = "https://files.pythonhosted.org/packages/65/36/32e7beef3281fed74883451477ad976364323206dbfaa95e948ba788dac7/ml_dtypes-0.6.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:3e169214e0d80ff1c038e1b3017e33c23e43bdf948d42d31de8283111c7e2fa3", size = 590177, upload-time = "2026-08-13T14:14:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/99b3d9b3c984b3bd1e81d8244f1fa2f812e44060d853205b2df6271aa17c/ml_dtypes-0.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:573b11f3c327e17ef3826d266e676cf1149a1f3016f822a05f2306c55d8246bf", size = 363142, upload-time = "2026-08-13T14:14:22.463Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fb/8091c0aee7f2712de99c7fd4b1642382644dec6a4962effe4f5b9d16a973/ml_dtypes-0.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b76fa1d3f92967d58289ac47ab7458ede66e6f3527fff3e59142aee57d9307cd", size = 430645, upload-time = "2026-08-13T14:14:23.737Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6f/962d2c589513b5930d05b6eae5fbd22ad8bbcf26bb763449f3d8f912360f/ml_dtypes-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:3be9911d953f97cddded4b9961d7b650473b7e55806d20f6176f8356dfe7b38e", size = 465667, upload-time = "2026-08-13T14:14:25.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ca/bcb25e246edd19af5fa1cf6267040bd9977a7afca846e6cfd4a52078b44f/ml_dtypes-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:e74266ca8e97874a937b7646378c178025650a236584f7474d10d8086a6edea3", size = 572706, upload-time = "2026-08-13T14:14:26.296Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/46cb442648e3c774d8cb25f2e1e41d496cdcc91fbe9c2a6f75c0b8df7af6/ml_dtypes-0.6.0-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:b1b503864fada3f74fabf8d9fee7b4c1cbe956301e6fdece975d5f77c2fce958", size = 562550, upload-time = "2026-08-13T14:14:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/07/56/844eff5af7a2d1a09d75df12c70225c3a6b6a771f95876b2bf5f7d10ad44/ml_dtypes-0.6.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c6ad60af4102789a5c09824004beade2f7f28cd1cd581ee5c170d9dc2fbb00e", size = 360332, upload-time = "2026-08-13T14:14:28.767Z" }, + { url = "https://files.pythonhosted.org/packages/b6/29/b7165a3a76364a5baa6aa4ee82a0adf73a3c014b8cd126120b62cc087992/ml_dtypes-0.6.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4f1b9329a251e4affe3bb58f4d3e2db22a714396fd7ffb40d0b5db423c24d17", size = 409964, upload-time = "2026-08-13T14:14:30.023Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2e/f61c54a0544b6a170ac1bb89bcf406af53fb2deffc5476b6d2d3df5ba13e/ml_dtypes-0.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:488c99ab181a2f59d9ec3b12c5fa11ec904e92be2c4ba18cded54dd7501208fe", size = 457249, upload-time = "2026-08-13T14:14:31.213Z" }, + { url = "https://files.pythonhosted.org/packages/63/00/bee1bc9faa02a46e7a851019fd23f47ca1f906609edbec8b6ba5decc3cc3/ml_dtypes-0.6.0-cp315-cp315-win_arm64.whl", hash = "sha256:de9d14748dbf3968951436ef514a29c9d1fe438aa680d110134ee2f7a9f9df18", size = 568381, upload-time = "2026-08-13T14:14:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/72/f7/9a5edede28f73185fd51d75030ef7f11d76997bab3a92427d986e54fe2eb/ml_dtypes-0.6.0-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e25bb3b0ad1217b60626e4ed45b10ca170c41d99fbe44a12bebc1e07ec4aad55", size = 589877, upload-time = "2026-08-13T14:14:33.695Z" }, + { url = "https://files.pythonhosted.org/packages/fd/81/d5924a141b850b606eb027493c9c3ca3c665cca5163af3f5b6e5e3345503/ml_dtypes-0.6.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31f1ce979d31a357e95aa81812f20412c8c954fa43c44ee3ead1e1c8a78575ef", size = 362788, upload-time = "2026-08-13T14:14:34.996Z" }, + { url = "https://files.pythonhosted.org/packages/59/8f/3298e3f334832bc28dd144af6b99cdc93502a8687e71922ea68b0a319929/ml_dtypes-0.6.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2d6149f3a57f405bcad5fb41e03218b8373936253f23e1ca84c0108abbc3392", size = 430823, upload-time = "2026-08-13T14:14:36.44Z" }, + { url = "https://files.pythonhosted.org/packages/93/d2/f2dbf118f42ce4c325a139c9236737f436b7f8e00cd18701c99ef2405e6f/ml_dtypes-0.6.0-cp315-cp315t-win_amd64.whl", hash = "sha256:ce7563e0b1a4482cbc1b4a6272145e54e4489e54fe7428f94908c3d87103abfa", size = 465119, upload-time = "2026-08-13T14:14:37.776Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ff/bda40387b5c5c64254595f4d81a12351770856acc5de4e6d43606a31f161/ml_dtypes-0.6.0-cp315-cp315t-win_arm64.whl", hash = "sha256:f6cb525101b6b903779188c1e9e9490c343b455ab822883e02cf01e5547338d2", size = 572666, upload-time = "2026-08-13T14:14:38.993Z" }, +] + [[package]] name = "multidict" version = "6.6.3" @@ -1679,6 +1917,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] +[[package]] +name = "opt-einsum" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -2865,6 +3112,9 @@ geo = [ { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] +ml = [ + { name = "ddxdb", marker = "sys_platform != 'emscripten'" }, +] polars = [ { name = "polars" }, ] @@ -2874,6 +3124,7 @@ test = [ { name = "adbc-driver-sqlite" }, { name = "cftime" }, { name = "dask" }, + { name = "ddxdb", marker = "sys_platform != 'emscripten'" }, { name = "duckdb" }, { name = "gcsfs" }, { name = "polars" }, @@ -2886,6 +3137,10 @@ test = [ [package.dev-dependencies] dev = [ + { name = "ddxdb", extra = ["duckdb"], marker = "sys_platform != 'emscripten'" }, + { name = "jax", version = "0.6.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "jax", version = "0.10.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "jax", version = "0.12.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "maturin" }, { name = "py-spy" }, { name = "ruff" }, @@ -2900,6 +3155,7 @@ requires-dist = [ { name = "cftime", marker = "extra == 'test'" }, { name = "dask", marker = "extra == 'dask'", specifier = ">=2024.8.0" }, { name = "datafusion", marker = "sys_platform != 'emscripten'", specifier = "==55.0.0" }, + { name = "ddxdb", marker = "sys_platform != 'emscripten' and extra == 'ml'", specifier = ">=0.4.2,<0.5" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.4.0" }, { name = "gcsfs", marker = "extra == 'test'" }, { name = "mkdocstrings", extras = ["python"], marker = "extra == 'docs'" }, @@ -2912,14 +3168,16 @@ requires-dist = [ { name = "watchfiles", marker = "extra == 'dev'" }, { name = "xarray", specifier = ">=2024.7.0" }, { name = "xarray", extras = ["io"], marker = "extra == 'test'" }, - { name = "xarray-sql", extras = ["adbc", "dask", "duckdb", "polars", "geo"], marker = "extra == 'test'" }, + { name = "xarray-sql", extras = ["adbc", "dask", "duckdb", "polars", "geo", "ml"], marker = "extra == 'test'" }, { name = "xarray-sql", extras = ["docs"], marker = "extra == 'dev'" }, { name = "zensical", marker = "extra == 'docs'" }, ] -provides-extras = ["adbc", "dask", "dev", "docs", "duckdb", "geo", "polars", "test"] +provides-extras = ["adbc", "dask", "dev", "docs", "duckdb", "geo", "ml", "polars", "test"] [package.metadata.requires-dev] dev = [ + { name = "ddxdb", extras = ["duckdb"], marker = "sys_platform != 'emscripten'", specifier = ">=0.4.2,<0.5" }, + { name = "jax" }, { name = "maturin", specifier = ">=1.9.1" }, { name = "py-spy", specifier = ">=0.4.0" }, { name = "ruff", specifier = ">=0.15.10" }, diff --git a/xarray_sql/ad.py b/xarray_sql/ad.py new file mode 100644 index 00000000..edf4b09e --- /dev/null +++ b/xarray_sql/ad.py @@ -0,0 +1,133 @@ +"""Gradients of SQL queries: the optional ml extension, built on ddx. + +[ddx](https://github.com/xqlsystems/ddx) differentiates a whole SQL query +the way [JAX](https://github.com/jax-ml/jax), Google's library for +differentiable array programs, differentiates a function. With it installed +(``pip install xarray-sql[ml]``), [xarray_sql.Context][]'s ``.sql()`` +understands three calls in a select list: + +* ``grad(loss, column)``: at each row, the derivative of the one number + the CTE ``loss`` computes with respect to that row's ``column`` + (``table.column`` when the ``SELECT`` reads several tables). +* ``vjp(f, column, cotangent)``: the same for any CTE ``f``, pulled back + along ``cotangent``, a table or CTE shaped like ``f``'s output. +* ``jvp(e, table.column, tangent)``: the tangent of the expression ``e`` + along ``tangent``, pushed forward. + +A gradient-descent step on a parameter table is then one statement:: + + WITH pred AS ( + SELECT x.sample, SUM(x.value * w.val) AS y + FROM x JOIN w ON x.feature = w.feature GROUP BY x.sample), + loss AS ( + SELECT AVG(power(pred.y - t.value, 2)) AS l + FROM pred JOIN t ON pred.sample = t.sample) + SELECT feature, val - 0.1 * grad(loss, val) AS val FROM w + +The query being differentiated is ordinary SQL, over tables registered +with ``from_dataset`` or any other way. ddx plans the backward pass in +[Substrait](https://substrait.io), a portable format for query plans; +DataFusion runs it, and the statement reads what it computed. + +The one function ddx gives a meaning to is ``ddx_stop_gradient(x)``, the +identity whose derivative is zero (JAX's ``lax.stop_gradient``). It is +registered on every ``Context``. Its typical use is the stable softmax, +whose shift by the row maximum cancels and so carries no gradient:: + + exp(s - ddx_stop_gradient(MAX(s) OVER (PARTITION BY i))) + +A statement that calls none of ``grad``, ``vjp`` or ``jvp`` is planned +exactly as it would be without ddx. An expression ddx cannot +differentiate raises a ``ddxdb.DdxError`` subclass naming it, never a +wrong number. + +[sql_all][xarray_sql.ad.sql_all] runs several statements that take +``grad`` of the same loss for the price of one backward pass, which is +how a training step updates each of a model's parameter tables. For the +programs underneath (``grad``, ``vjp``, ``jvp`` and ``run``, to build a +backward pass once and run it every step), use ``ddxdb.ad`` directly; +they take a ``Context`` like any other ``SessionContext``. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence + +from datafusion import SessionContext +from ddxdb import ad as _ddx + +__all__ = ["STOP_GRADIENT", "calls_ad", "register", "sql", "sql_all"] + +STOP_GRADIENT = _ddx.STOP_GRADIENT +"""The SQL name of the stop-gradient function, ``ddx_stop_gradient``.""" + +_AD_CALL = re.compile(r"\b(grad|vjp|jvp)\s*\(", re.IGNORECASE) + + +def register(ctx: SessionContext) -> None: + """Register ``ddx_stop_gradient`` on ``ctx``. + + Works on any DataFusion ``SessionContext`` (``Context`` registers it + automatically when ddxdb is installed). ``grad``, ``vjp`` and ``jvp`` + are not UDFs: [sql][xarray_sql.ad.sql] rewrites them before DataFusion + plans the statement. + + Args: + ctx: The DataFusion session context to register the UDF on. + """ + _ddx.register_stop_gradient(ctx) + + +def calls_ad(statement: str) -> bool: + """Whether ``statement`` may call ``grad``, ``vjp`` or ``jvp``. + + A cheap textual check, so statements without a call never reach + ddx's parser. A false positive (the word inside a string literal, + say) is harmless: ddx plans a statement with no call as it is. + """ + return _AD_CALL.search(statement) is not None + + +def sql(ctx: SessionContext, statement: str): + """Run ``statement``, computing its ``grad``, ``vjp`` and ``jvp`` calls. + + Returns a DataFusion DataFrame; ``Context.sql`` wraps it as an + [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame]. + + Args: + ctx: Any DataFusion ``SessionContext``. + statement: A SQL statement. + """ + return _ddx.sql(ctx, statement) + + +def sql_all(ctx: SessionContext, statements: Sequence[str]) -> list: + """Run several statements, sharing the programs their calls need. + + Statements that take ``grad`` of the same loss share one backward + pass, so a training step can update each parameter table with its + own statement and pay for the gradient once:: + + steps = xql.ad.sql_all(ctx, [ + f"WITH loss AS ({LOSS}) " + "SELECT i, val - 0.1 * grad(loss, val) AS val FROM w1", + f"WITH loss AS ({LOSS}) " + "SELECT j, val - 0.1 * grad(loss, val) AS val FROM w2", + ]) + + On a [Context][xarray_sql.Context], each result is an + [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame]. + + Args: + ctx: Any DataFusion ``SessionContext``. + statements: SQL statements. + + Returns: + One DataFrame per statement, in order. + """ + if isinstance(statements, str): + raise TypeError("statements is a list of SQL statements") + frames: list = _ddx.sql_all(ctx, list(statements)) + wrap = getattr(ctx, "_wrap", None) + return [wrap(f) for f in frames] if wrap else frames diff --git a/xarray_sql/sql.py b/xarray_sql/sql.py index 632f9bca..d326837a 100644 --- a/xarray_sql/sql.py +++ b/xarray_sql/sql.py @@ -25,6 +25,14 @@ except ImportError: # pragma: no cover - depends on the environment _proj = None +_ad: ModuleType | None +try: # ddxdb is an optional dependency (`pip install xarray-sql[ml]`). + from . import ad + + _ad = ad +except ImportError: # pragma: no cover - depends on the environment + _ad = None + class Context(SessionContext): """A datafusion `SessionContext` that also supports `xarray.Dataset`s. @@ -64,6 +72,10 @@ def __init__(self, config: SessionConfig | None = None, *args, **kwargs): # reproject(x, y, src_crs, dst_crs), à la PostGIS ST_Transform. if _proj is not None: _proj.register(self) + # With ddxdb installed, .sql() also takes gradients: grad(), vjp() + # and jvp() in a select list, and ddx_stop_gradient(). + if _ad is not None: + _ad.register(self) def from_dataset( self, @@ -230,6 +242,10 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame: ``.to_dataset(dimension_columns=[...])`` for round-tripping the result back to an ``xr.Dataset``. + With the ml extension installed (``pip install xarray-sql[ml]``), + the query may also call ``grad``, ``vjp`` and ``jvp`` (see + [xarray_sql.ad][]). + Args: query: A SQL query string. *args: Forwarded to ``SessionContext.sql``. @@ -238,7 +254,17 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame: Returns: An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame. """ - inner = super().sql(query, *args, **kwargs) + if _ad is not None and _ad.calls_ad(query): + if args or kwargs: + raise TypeError( + "a query that calls grad, vjp or jvp takes no other " + "arguments" + ) + return self._wrap(_ad.sql(self, query)) + return self._wrap(super().sql(query, *args, **kwargs)) + + def _wrap(self, inner) -> XarrayDataFrame: + """Wrap a DataFusion DataFrame so it can round-trip to xarray.""" return XarrayDataFrame(inner, templates=self._registered_datasets)