From 73bfc5812f045a16fc4297241923c47b06a01abd Mon Sep 17 00:00:00 2001 From: ppraneth Date: Mon, 9 Nov 2026 09:33:25 +0530 Subject: [PATCH 1/2] Add rewrite, using Theorem 1 for zero-preserving functions on sparse tensors --- src/function.rs | 6 +- src/index_functions.rs | 54 +++++++++++++++ src/lattice.rs | 6 ++ src/lib.rs | 5 +- src/rewrite.rs | 62 +++++++++++++++++ tests/index_functions.rs | 47 ++++++++++++- tests/rewrite.rs | 143 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 src/rewrite.rs create mode 100644 tests/rewrite.rs diff --git a/src/function.rs b/src/function.rs index baf5cb4..c546f88 100644 --- a/src/function.rs +++ b/src/function.rs @@ -43,8 +43,8 @@ impl Function for Rc { /// `f ∘ g`: applies `g`, then `f` to the result. pub struct Composition { - f: F, - g: G, + pub(crate) f: F, + pub(crate) g: G, } /// Composes `f` after `g`. Composing a tensor after an index map re-indexes it, @@ -89,7 +89,7 @@ where /// of its elements, repeats included, so it goes from the multisets of the /// domain of `f` to those of its codomain. pub struct MultiFunction { - f: F, + pub(crate) f: F, } pub fn multiset_lift(f: F) -> MultiFunction { diff --git a/src/index_functions.rs b/src/index_functions.rs index 273d5ce..2190c0e 100644 --- a/src/index_functions.rs +++ b/src/index_functions.rs @@ -331,10 +331,30 @@ impl IndexFunction for Where { } } +/// Whether a multi-index function sends each index to at most one index, which +/// makes a sum over it a sparse tensor (Definition 4 of the paper). The answer +/// may be a conservative `false`, never a wrong `true`. +pub trait Sparsity { + fn is_sparse(&self) -> bool; +} + +impl Sparsity for Rc { + fn is_sparse(&self) -> bool { + (**self).is_sparse() + } +} + pub struct EmptyMapping { signature: MultiIndexFunctionSignature, } +// Every index is sent to no index at all. +impl Sparsity for EmptyMapping { + fn is_sparse(&self) -> bool { + true + } +} + impl EmptyMapping { pub fn new(domain: HRect, codomain: HRect) -> Self { Self { @@ -374,6 +394,18 @@ impl COO { } } +// A partial coordinate matches every index sharing its prefix, so two stored +// coordinates land on a common index when one is a prefix of the other, equal +// ones included. Sorted, the coordinates starting with a given one come right +// after it, so only neighbours need comparing. +impl Sparsity for COO { + fn is_sparse(&self) -> bool { + let mut coords: Vec<&Vec> = self.inverse_coords.iter().collect(); + coords.sort(); + coords.windows(2).all(|pair| !pair[1].starts_with(pair[0])) + } +} + impl IndexFunction for COO { type Codomain = Multisets; @@ -434,6 +466,13 @@ impl IndexFunction for Diagonal { } } +// An index on the diagonal is sent to one index, and any other to none. +impl Sparsity for Diagonal { + fn is_sparse(&self) -> bool { + true + } +} + /// Projection dropping axis `dim`. /// e.g. with dim=1, f(m, n, p) = [m, p]^T pub struct Projection { @@ -516,6 +555,13 @@ impl IndexFunction for InvProjection { } } +// Each index is sent to the whole line along `dim`. +impl Sparsity for InvProjection { + fn is_sparse(&self) -> bool { + self.signature.codomain().set().shape()[self.dim] <= 1 + } +} + /// The affine map `x ↦ basis·x + offset`. `basis` has a row per dimension of the /// codomain and a column per dimension of the domain. Its entries and `offset` /// may be negative, as long as every index of the domain lands in the codomain. @@ -642,3 +688,11 @@ impl IndexFunction for InvAffine { &self.signature } } + +// Each index is sent to its preimage under the affine map, which holds at most +// one index when the map is one-to-one. +impl Sparsity for InvAffine { + fn is_sparse(&self) -> bool { + self.preimage.is_injective() + } +} diff --git a/src/lattice.rs b/src/lattice.rs index 1c7d4d3..d7e740c 100644 --- a/src/lattice.rs +++ b/src/lattice.rs @@ -38,6 +38,12 @@ impl Preimage { } } + /// Whether `x ↦ basis·x` is one-to-one, i.e. the basis has no integer + /// kernel. A kernel the box is too small to reach still counts. + pub(crate) fn is_injective(&self) -> bool { + self.h_pivots.len() == self.extents.len() + } + /// Every index `x` of the box with `basis·x = c`. pub(crate) fn solve(&self, c: &[i64]) -> Vec> { let mut z = vec![0; self.h_pivots.len()]; diff --git a/src/lib.rs b/src/lib.rs index 89ab9db..4c40008 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod index_functions; mod lattice; pub mod multiset; pub mod reductions; +pub mod rewrite; pub mod sets; pub mod signature; pub mod tensor; @@ -21,13 +22,15 @@ pub use functions::{ }; pub use index_functions::{ Affine, COO, Concat, Diagonal, EmptyMapping, IndexFunction, InvAffine, InvProjection, - MultiIndexFunctionSignature, Permutation, Projection, Squeeze, Stride, Unsqueeze, Where, + MultiIndexFunctionSignature, Permutation, Projection, Sparsity, Squeeze, Stride, Unsqueeze, + Where, }; pub use multiset::Multiset; pub use reductions::{ Inf, InfReduction, Product, ProductReduction, Reduction, ReductionFunction, Sum, SumReduction, Sup, SupReduction, }; +pub use rewrite::{Rewrite, rewrite}; pub use sets::{HRect, IndexSet, JaggedDomain, Multisets, Set, Values}; pub use signature::{HRectSignature, Signature}; pub use tensor::{JaggedTensor, Physical, ReductionTensor, Tensor}; diff --git a/src/rewrite.rs b/src/rewrite.rs new file mode 100644 index 0000000..5588493 --- /dev/null +++ b/src/rewrite.rs @@ -0,0 +1,62 @@ +//! Rewriting a tensor into an equivalent one that is cheaper to evaluate, with +//! the identities between compositions of Section 4 of the paper. Only +//! Theorem 1 is used so far. + +use crate::function::{ + Composition, Function, MultiFunction, ValueFunction, compose, multiset_lift, +}; +use crate::index_functions::Sparsity; +use crate::reductions::{Reduction, Sum, SumReduction}; +use crate::sets::{Multisets, Values}; +use crate::tensor::ReductionTensor; + +/// A tensor that an identity of the paper can rewrite into an equivalent one. +pub trait Rewrite: Sized { + type Rewritten; + + /// The equivalent tensor, or `Err` with the tensor unchanged when the + /// identity does not apply to it. + fn rewrite(self) -> Result; +} + +/// Rewrites `tensor` into an equivalent one that is cheaper to evaluate, or +/// gives it back unchanged as `Err` when no identity applies to it. +pub fn rewrite(tensor: T) -> Result { + tensor.rewrite() +} + +/// Theorem 1: a function on values after a sparse tensor only has to be applied +/// to its stored values, `f ∘ Σ ∘ Ū ∘ ψ = Σ ∘ (f ∘ U)‾ ∘ ψ`. This holds when +/// `f(0) = 0` and `ψ` sends each index to at most one index, which is checked. +impl Rewrite for Composition, ReductionTensor, U, M>> +where + A: Clone + PartialEq, + SumReduction: Reduction, + U: Function>, + M: Function> + Sparsity, +{ + type Rewritten = ReductionTensor, Composition, U>, M>; + + fn rewrite(self) -> Result { + let zero = >::NEUTRAL; + if self.f.apply(&zero) != zero || !self.g.g.g.is_sparse() { + return Err(self); + } + let Composition { + f, + g: + Composition { + g: + Composition { + f: MultiFunction { f: u }, + g: psi, + }, + .. + }, + } = self; + Ok(compose( + Sum::new(), + compose(multiset_lift(compose(f, u)), psi), + )) + } +} diff --git a/tests/index_functions.rs b/tests/index_functions.rs index fa746a4..dfd93ca 100644 --- a/tests/index_functions.rs +++ b/tests/index_functions.rs @@ -2,8 +2,8 @@ use std::{assert_eq, collections::HashMap, rc::Rc}; use cten::{ Affine, COO, Concat, Diagonal, EmptyMapping, HRect, IndexFunction, IndexSet, InvAffine, - InvProjection, JaggedDomain, Multiset, Permutation, Physical, Projection, Squeeze, Stride, - Unsqueeze, Where, + InvProjection, JaggedDomain, Multiset, Permutation, Physical, Projection, Sparsity, Squeeze, + Stride, Unsqueeze, Where, }; /// The indices of a multiset in sorted order, to compare multisets whose @@ -407,3 +407,46 @@ fn a_where_sends_each_index_to_the_part_its_condition_picks() { assert_eq!(picks.map(&[1]), vec![1, 1]); assert_eq!(picks.map(&[2]), vec![0, 2]); } + +// A partial coordinate matches every index sharing its prefix, so it meets every +// coordinate it is a prefix of. +#[test] +fn a_coo_is_sparse_when_no_two_coordinates_meet() { + let coo = |coords: Vec>| COO::new(HRect::new(vec![3, 3]), coords); + + assert!(coo(vec![vec![0, 2], vec![1, 0], vec![1, 2]]).is_sparse()); + assert!(!coo(vec![vec![1, 2], vec![0, 0], vec![1, 2]]).is_sparse()); + assert!(!coo(vec![vec![2], vec![2, 1]]).is_sparse()); + assert!(coo(vec![vec![2], vec![1, 1]]).is_sparse()); +} + +#[test] +fn diagonal_and_empty_index_functions_are_sparse() { + assert!(Diagonal::new(HRect::new(vec![3, 3])).is_sparse()); + assert!(EmptyMapping::new(HRect::new(vec![3]), HRect::new(vec![0])).is_sparse()); +} + +#[test] +fn an_inv_projection_is_sparse_only_along_an_axis_of_extent_one() { + assert!(InvProjection::new(HRect::new(vec![2, 1]), 1).is_sparse()); + assert!(!InvProjection::new(HRect::new(vec![2, 3]), 1).is_sparse()); +} + +#[test] +fn an_inv_affine_is_sparse_when_its_affine_map_is_one_to_one() { + let diagonal = InvAffine::new(Affine::new( + HRect::new(vec![3]), + vec![vec![1], vec![1]], + vec![0, 0], + HRect::new(vec![3, 3]), + )); + let projection = InvAffine::new(Affine::new( + HRect::new(vec![2, 3, 4]), + vec![vec![1, 0, 0], vec![0, 0, 1]], + vec![0, 0], + HRect::new(vec![2, 4]), + )); + + assert!(diagonal.is_sparse()); + assert!(!projection.is_sparse()); +} diff --git a/tests/rewrite.rs b/tests/rewrite.rs new file mode 100644 index 0000000..c3e3f1c --- /dev/null +++ b/tests/rewrite.rs @@ -0,0 +1,143 @@ +use std::cell::Cell; +use std::rc::Rc; + +use cten::{ + COO, HRect, Physical, ReductionTensor, Stride, Sum, Tensor, ValueFunction, compose, + diagonalize, rewrite, scale, sparse_coo_tensor, square, sum, +}; + +/// The sparse matrix of torch's `sparse_coo_tensor` documentation, which stores +/// three values: `[[0, 0, 3, 0], [4, 0, 5, 0]]`. +fn sparse() -> ReductionTensor { + sparse_coo_tensor( + vec![vec![0, 1, 1], vec![2, 0, 2]], + Physical::new(vec![3.0, 4.0, 5.0]), + vec![2, 4], + ) +} + +fn collect_2d(tensor: &dyn Tensor, rows: usize, cols: usize) -> Vec> { + (0..rows) + .map(|i| (0..cols).map(|j| tensor.apply(&[i, j])).collect()) + .collect() +} + +/// `x ↦ x²`, counting how many times it is applied. +fn counting_square(calls: &Rc>) -> ValueFunction { + let calls = calls.clone(); + ValueFunction::new(move |x: f32| { + calls.set(calls.get() + 1); + x * x + }) +} + +#[test] +fn theorem_1_rewrites_a_function_after_a_sparse_tensor_into_a_sparse_tensor() { + let Ok(rewritten) = rewrite(square(sparse())) else { + panic!("squaring keeps 0, so theorem 1 applies"); + }; + + assert_eq!( + collect_2d(&rewritten, 2, 4), + vec![vec![0.0, 0.0, 9.0, 0.0], vec![16.0, 0.0, 25.0, 0.0]] + ); + assert_eq!( + collect_2d(&rewritten, 2, 4), + collect_2d(&square(sparse()), 2, 4) + ); +} + +// The point of theorem 1: evaluated as written, the function is applied at every +// index of the matrix; rewritten, only at the three stored values. +#[test] +fn a_rewritten_tensor_applies_the_function_to_the_stored_values_only() { + let calls = Rc::new(Cell::new(0)); + collect_2d(&compose(counting_square(&calls), sparse()), 2, 4); + assert_eq!(calls.get(), 8); + + let calls = Rc::new(Cell::new(0)); + let Ok(rewritten) = rewrite(compose(counting_square(&calls), sparse())) else { + panic!("squaring keeps 0, so theorem 1 applies"); + }; + // Checking that the function keeps 0 applied it once. + calls.set(0); + collect_2d(&rewritten, 2, 4); + assert_eq!(calls.get(), 3); +} + +#[test] +fn a_rewritten_tensor_prints_like_the_original() { + let Ok(rewritten) = rewrite(square(sparse())) else { + panic!("squaring keeps 0, so theorem 1 applies"); + }; + + assert_eq!(rewritten.to_string(), square(sparse()).to_string()); +} + +#[test] +fn theorem_1_applies_to_other_functions_and_sparse_formats() { + let relu = || ValueFunction::new(|x: f32| x.max(0.0)); + let diagonal = || diagonalize(Physical::new(vec![-1.0, 2.0, -3.0]), 2); + let Ok(rewritten) = rewrite(compose(relu(), diagonal())) else { + panic!("relu keeps 0, so theorem 1 applies"); + }; + + assert_eq!( + collect_2d(&rewritten, 3, 3), + vec![ + vec![0.0, 0.0, 0.0], + vec![0.0, 2.0, 0.0], + vec![0.0, 0.0, 0.0] + ] + ); + assert_eq!( + collect_2d(&rewritten, 3, 3), + collect_2d(&compose(relu(), diagonal()), 3, 3) + ); +} + +// The result is a sparse tensor again, so a second function rewrites the same way. +#[test] +fn a_rewritten_tensor_can_be_rewritten_again() { + let Ok(squared) = rewrite(square(sparse())) else { + panic!("squaring keeps 0, so theorem 1 applies"); + }; + let Ok(rewritten) = rewrite(scale(squared, 2.0)) else { + panic!("scaling keeps 0, so theorem 1 applies"); + }; + + assert_eq!( + collect_2d(&rewritten, 2, 4), + vec![vec![0.0, 0.0, 18.0, 0.0], vec![32.0, 0.0, 50.0, 0.0]] + ); +} + +// Adding 1 does not keep 0: off the stored values, the tensor is 1 and not 0. The +// tensor comes back unchanged. +#[test] +fn theorem_1_needs_a_function_that_keeps_zero() { + let plus_one = ValueFunction::new(|x: f32| x + 1.0); + let Err(unchanged) = rewrite(compose(plus_one, sparse())) else { + panic!("adding 1 does not keep 0, so theorem 1 does not apply"); + }; + + assert_eq!( + collect_2d(&unchanged, 2, 4), + vec![vec![1.0, 1.0, 4.0, 1.0], vec![5.0, 1.0, 6.0, 1.0]] + ); +} + +// With several values at one index, squaring their sum is not summing their +// squares, so theorem 1 does not apply. +#[test] +fn theorem_1_needs_at_most_one_stored_value_per_index() { + let repeated = sparse_coo_tensor(vec![vec![1, 1]], Physical::new(vec![2.0, 3.0]), vec![3]); + let matrix = compose( + Physical::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + Stride::new(HRect::new(vec![2, 3]), vec![3, 1]), + ); + let row_sums = sum(matrix, 1); + + assert!(rewrite(square(repeated)).is_err()); + assert!(rewrite(square(row_sums)).is_err()); +} From 545abe8bac642e7f770782917bd2e0eed3c08f8e Mon Sep 17 00:00:00 2001 From: ppraneth Date: Sun, 15 Nov 2026 19:23:55 +0530 Subject: [PATCH 2/2] Make rewrite work on any tensor, leaving it unchanged where nothing applies --- src/display.rs | 13 ++- src/function.rs | 158 +++++++++++++++++++++++++++++++++-- src/functions.rs | 14 ++-- src/index_functions.rs | 72 +++++++--------- src/lib.rs | 5 +- src/reductions.rs | 22 ++++- src/rewrite.rs | 75 +++++------------ tests/index_functions.rs | 4 +- tests/rewrite.rs | 176 ++++++++++++++++++++++++++------------- 9 files changed, 359 insertions(+), 180 deletions(-) diff --git a/src/display.rs b/src/display.rs index 7f38a3e..ef423ac 100644 --- a/src/display.rs +++ b/src/display.rs @@ -15,6 +15,13 @@ pub fn format_boxed(tensor: &dyn Tensor) -> String { format_tensor(tensor.domain().shape(), &|index| tensor.apply(index)) } +// A tensor whose type was erased, as `rewrite` returns, prints the same way. +impl Display for dyn Tensor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&format_boxed(self)) + } +} + impl Display for Physical { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&format_boxed(self)) @@ -25,8 +32,10 @@ impl Display for Physical { // one applied last gives values that print, multisets included. impl Display for Composition where - F: Function, - G: Function, + F: Function + 'static, + G: Function + 'static, + F::Codomain: 'static, + G::Codomain: 'static, ::Owned: Display, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/src/function.rs b/src/function.rs index c546f88..09feed0 100644 --- a/src/function.rs +++ b/src/function.rs @@ -2,6 +2,7 @@ //! multisets. Tensors, index maps, index functions and reductions are all //! functions, and every operation on them is built from these. +use std::any::Any; use std::borrow::Borrow; use std::rc::Rc; @@ -11,6 +12,10 @@ use crate::sets::{Multisets, Set, Values}; /// of values, or of multisets of either. A tensor goes from indices to values, /// an index map from indices to indices, an index function from indices to /// multisets of indices, and a reduction from multisets of values to values. +/// +/// Besides `apply`, a function can describe itself so that `rewrite` can find +/// the identities of the paper that apply to it. Every such method has a +/// default that finds nothing, which is always correct. pub trait Function { type Domain: Set; type Codomain: Set; @@ -20,10 +25,51 @@ pub trait Function { fn domain(&self) -> Self::Domain; fn codomain(&self) -> Self::Codomain; + + /// An equivalent function that is cheaper to evaluate, or `None` when no + /// identity applies anywhere inside this one. + fn rewritten( + &self, + ) -> Option>> { + None + } + + /// When this is a reduction `Ω` after a function returning multisets of at + /// most one element, and `f` keeps the neutral element of `Ω`: `f` after + /// this, rewritten as `Ω ∘ f̄` after that function. `f` is given as an + /// `Rc` from the codomain of this to itself, behind `Any`. + fn absorbed( + &self, + _f: &dyn Any, + ) -> Option>> { + None + } + + /// When this is a reduction and `f` keeps its neutral element: `f̄`, the + /// extension of `f` to the multisets this reduces. `f` is given as for + /// `absorbed`. + fn lifted_keeping_neutral( + &self, + _f: &dyn Any, + ) -> Option>> { + None + } + + /// Whether every multiset this returns holds at most one element. + fn returns_at_most_one(&self) -> bool { + false + } + + /// Whether this maps each multiset to one of the same size, as the + /// extension of a function to multisets does. + fn keeps_sizes(&self) -> bool { + false + } } // A shared function is the same function, so that one can be used in several -// compositions, or have its type erased as an `Rc>`. +// compositions, or have its type erased as an `Rc>`. Rewriting +// sees through it. impl Function for Rc { type Domain = T::Domain; type Codomain = T::Codomain; @@ -39,12 +85,41 @@ impl Function for Rc { fn codomain(&self) -> Self::Codomain { (**self).codomain() } + + fn rewritten( + &self, + ) -> Option>> { + (**self).rewritten() + } + + fn absorbed( + &self, + f: &dyn Any, + ) -> Option>> { + (**self).absorbed(f) + } + + fn lifted_keeping_neutral( + &self, + f: &dyn Any, + ) -> Option>> { + (**self).lifted_keeping_neutral(f) + } + + fn returns_at_most_one(&self) -> bool { + (**self).returns_at_most_one() + } + + fn keeps_sizes(&self) -> bool { + (**self).keeps_sizes() + } } /// `f ∘ g`: applies `g`, then `f` to the result. pub struct Composition { - pub(crate) f: F, - pub(crate) g: G, + // Shared, so that rewriting can reuse them without taking this apart. + pub(crate) f: Rc, + pub(crate) g: Rc, } /// Composes `f` after `g`. Composing a tensor after an index map re-indexes it, @@ -60,13 +135,19 @@ where codomain.lies_in(&domain), "the inner function lands in {codomain:?}, which does not lie in the domain {domain:?} of the outer one" ); - Composition { f, g } + Composition { + f: Rc::new(f), + g: Rc::new(g), + } } impl Function for Composition where - F: Function, - G: Function, + F: Function + 'static, + G: Function + 'static, + F::Codomain: 'static, + G::Domain: 'static, + G::Codomain: 'static, { type Domain = G::Domain; type Codomain = F::Codomain; @@ -82,6 +163,54 @@ where fn codomain(&self) -> Self::Codomain { self.f.codomain() } + + // Rewrites both sides, then `f` after the rewritten `g` with Theorem 1, which + // `absorbed` checks the conditions of. + fn rewritten( + &self, + ) -> Option>> { + let new_f = self.f.rewritten(); + let new_g = self.g.rewritten(); + let changed = new_f.is_some() || new_g.is_some(); + let f: Rc> = + new_f.unwrap_or_else(|| self.f.clone()); + let g: Rc> = + new_g.unwrap_or_else(|| self.g.clone()); + if let Some(absorbed) = g.absorbed(&f) { + // `f` was taken for a function from the codomain of `g` to itself, so + // the codomain of `g` is that of `f`, as the downcast confirms. + let absorbed: Box = Box::new(absorbed); + if let Ok(absorbed) = absorbed + .downcast::>>() + { + return Some(*absorbed); + } + } + changed.then(|| { + Rc::new(compose(f, g)) + as Rc> + }) + } + + // Here this is `Ω ∘ H`, with `Ω` in `f` and `H` in `g`. + fn absorbed( + &self, + f: &dyn Any, + ) -> Option>> { + if !self.g.returns_at_most_one() { + return None; + } + let lifted = self.f.lifted_keeping_neutral(f)?; + // Erased, so that the result has the same type whatever this one is, and + // rewriting it again does not build ever larger types. + let omega: Rc> = self.f.clone(); + let inner: Rc> = self.g.clone(); + Some(Rc::new(compose(omega, compose(lifted, inner)))) + } + + fn returns_at_most_one(&self) -> bool { + self.g.returns_at_most_one() && self.f.keeps_sizes() + } } /// A function from multisets to multisets: the extension of `f` to multisets, @@ -96,7 +225,11 @@ pub fn multiset_lift(f: F) -> MultiFunction { MultiFunction { f } } -impl Function for MultiFunction { +impl Function for MultiFunction +where + F::Domain: 'static, + F::Codomain: 'static, +{ type Domain = Multisets; type Codomain = Multisets; @@ -113,6 +246,17 @@ impl Function for MultiFunction { fn codomain(&self) -> Self::Codomain { Multisets::new(self.f.codomain()) } + + fn rewritten( + &self, + ) -> Option>> { + let rewritten = self.f.rewritten()?; + Some(Rc::new(multiset_lift(rewritten))) + } + + fn keeps_sizes(&self) -> bool { + true + } } /// A function on values, given as a closure, from `A` to `B`. Composed after a diff --git a/src/functions.rs b/src/functions.rs index 14692c8..ff28d61 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -208,7 +208,7 @@ pub fn where_( /// Spreads a vector over the diagonal of a tensor of the given rank: rank two /// gives a diagonal matrix, rank three a tensor whose only non-zero indices /// are of the form `[i, i, i]`. -pub fn diagonalize>( +pub fn diagonalize + 'static>( u: U, rank: usize, ) -> ReductionTensor, U, Diagonal> @@ -224,7 +224,7 @@ where /// Reducing over the empty set gives the reduction's neutral element at every /// index, so a constant tensor needs no underlying data at all. -fn neutral>( +fn neutral + 'static>( shape: Vec, ) -> ReductionTensor, Physical, EmptyMapping> { let empty = EmptyMapping::new(HRect::new(shape), HRect::new(vec![0])); @@ -266,7 +266,7 @@ pub fn eye( /// `indices`, and zero everywhere else. As in torch, `indices` has one row per /// dimension, so column `k` is the coordinate of `values[k]`, and values at a /// repeated coordinate add up. -pub fn sparse_coo_tensor>( +pub fn sparse_coo_tensor + 'static>( indices: Vec>, values: U, size: Vec, @@ -312,7 +312,7 @@ where /// `physical` sits at `basis·x + offset`, and the tensor is zero off that /// lattice. `basis` has a row per dimension of `shape` and a column per /// dimension of `physical`. Values the lattice sends to one index add up. -pub fn sparse_latticed_tensor>( +pub fn sparse_latticed_tensor + 'static>( physical: U, basis: Vec>, offset: Vec, @@ -328,7 +328,7 @@ where ) } -pub fn sum>( +pub fn sum + 'static>( input: U, dim: usize, ) -> ReductionTensor, U, InvProjection> @@ -339,7 +339,7 @@ where compose(Sum::new(), compose(multiset_lift(input), rows)) } -pub fn prod>( +pub fn prod + 'static>( input: U, dim: usize, ) -> ReductionTensor, U, InvProjection> @@ -354,7 +354,7 @@ where /// matrices at each `[m, n, p]`. type Products = ReductionTensor, Composition, Concat>, InvProjection>; -pub fn mm + 'static, V: Tensor + 'static>( +pub fn mm + 'static, V: Tensor + 'static>( mat1: U, mat2: V, ) -> ReductionTensor, Products, InvProjection> diff --git a/src/index_functions.rs b/src/index_functions.rs index 2190c0e..1888a63 100644 --- a/src/index_functions.rs +++ b/src/index_functions.rs @@ -21,6 +21,13 @@ pub trait IndexFunction { fn map(&self, index: &[usize]) -> ::Owned; fn signature(&self) -> &Signature; + + /// Whether this sends each index to at most one index, which makes a sum + /// over it a sparse tensor (Definition 4 of the paper). The answer may be a + /// conservative `false`, never a wrong `true`. + fn is_sparse(&self) -> bool { + false + } } impl Function for M { @@ -38,6 +45,10 @@ impl Function for M { fn codomain(&self) -> M::Codomain { self.signature().codomain().clone() } + + fn returns_at_most_one(&self) -> bool { + self.is_sparse() + } } pub struct Stride { @@ -331,30 +342,10 @@ impl IndexFunction for Where { } } -/// Whether a multi-index function sends each index to at most one index, which -/// makes a sum over it a sparse tensor (Definition 4 of the paper). The answer -/// may be a conservative `false`, never a wrong `true`. -pub trait Sparsity { - fn is_sparse(&self) -> bool; -} - -impl Sparsity for Rc { - fn is_sparse(&self) -> bool { - (**self).is_sparse() - } -} - pub struct EmptyMapping { signature: MultiIndexFunctionSignature, } -// Every index is sent to no index at all. -impl Sparsity for EmptyMapping { - fn is_sparse(&self) -> bool { - true - } -} - impl EmptyMapping { pub fn new(domain: HRect, codomain: HRect) -> Self { Self { @@ -373,6 +364,11 @@ impl IndexFunction for EmptyMapping { fn signature(&self) -> &MultiIndexFunctionSignature { &self.signature } + + // Every index is sent to no index at all. + fn is_sparse(&self) -> bool { + true + } } pub struct COO { @@ -394,18 +390,6 @@ impl COO { } } -// A partial coordinate matches every index sharing its prefix, so two stored -// coordinates land on a common index when one is a prefix of the other, equal -// ones included. Sorted, the coordinates starting with a given one come right -// after it, so only neighbours need comparing. -impl Sparsity for COO { - fn is_sparse(&self) -> bool { - let mut coords: Vec<&Vec> = self.inverse_coords.iter().collect(); - coords.sort(); - coords.windows(2).all(|pair| !pair[1].starts_with(pair[0])) - } -} - impl IndexFunction for COO { type Codomain = Multisets; @@ -426,6 +410,16 @@ impl IndexFunction for COO { fn signature(&self) -> &MultiIndexFunctionSignature { &self.signature } + + // A partial coordinate matches every index sharing its prefix, so two stored + // coordinates land on a common index when one is a prefix of the other, equal + // ones included. Sorted, the coordinates starting with a given one come right + // after it, so only neighbours need comparing. + fn is_sparse(&self) -> bool { + let mut coords: Vec<&Vec> = self.inverse_coords.iter().collect(); + coords.sort(); + coords.windows(2).all(|pair| !pair[1].starts_with(pair[0])) + } } pub struct Diagonal { @@ -464,10 +458,8 @@ impl IndexFunction for Diagonal { fn signature(&self) -> &MultiIndexFunctionSignature { &self.signature } -} -// An index on the diagonal is sent to one index, and any other to none. -impl Sparsity for Diagonal { + // An index on the diagonal is sent to one index, and any other to none. fn is_sparse(&self) -> bool { true } @@ -553,10 +545,8 @@ impl IndexFunction for InvProjection { fn signature(&self) -> &MultiIndexFunctionSignature { &self.signature } -} -// Each index is sent to the whole line along `dim`. -impl Sparsity for InvProjection { + // Each index is sent to the whole line along `dim`. fn is_sparse(&self) -> bool { self.signature.codomain().set().shape()[self.dim] <= 1 } @@ -687,11 +677,9 @@ impl IndexFunction for InvAffine { fn signature(&self) -> &MultiIndexFunctionSignature { &self.signature } -} -// Each index is sent to its preimage under the affine map, which holds at most -// one index when the map is one-to-one. -impl Sparsity for InvAffine { + // Each index is sent to its preimage under the affine map, which holds at most + // one index when the map is one-to-one. fn is_sparse(&self) -> bool { self.preimage.is_injective() } diff --git a/src/lib.rs b/src/lib.rs index 4c40008..50548bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,15 +22,14 @@ pub use functions::{ }; pub use index_functions::{ Affine, COO, Concat, Diagonal, EmptyMapping, IndexFunction, InvAffine, InvProjection, - MultiIndexFunctionSignature, Permutation, Projection, Sparsity, Squeeze, Stride, Unsqueeze, - Where, + MultiIndexFunctionSignature, Permutation, Projection, Squeeze, Stride, Unsqueeze, Where, }; pub use multiset::Multiset; pub use reductions::{ Inf, InfReduction, Product, ProductReduction, Reduction, ReductionFunction, Sum, SumReduction, Sup, SupReduction, }; -pub use rewrite::{Rewrite, rewrite}; +pub use rewrite::rewrite; pub use sets::{HRect, IndexSet, JaggedDomain, Multisets, Set, Values}; pub use signature::{HRectSignature, Signature}; pub use tensor::{JaggedTensor, Physical, ReductionTensor, Tensor}; diff --git a/src/reductions.rs b/src/reductions.rs index a3d0e94..e50e53f 100644 --- a/src/reductions.rs +++ b/src/reductions.rs @@ -1,8 +1,10 @@ //! Reductions: functions from multisets of values to values, `Ω` in the paper. +use std::any::Any; use std::marker::PhantomData; +use std::rc::Rc; -use crate::function::Function; +use crate::function::{Function, multiset_lift}; use crate::multiset::Multiset; use crate::sets::{Multisets, Values}; @@ -80,7 +82,9 @@ impl Default for ReductionFunction { } } -impl, A: Clone> Function for ReductionFunction { +impl + 'static, A: Clone + PartialEq + 'static> Function + for ReductionFunction +{ type Domain = Multisets>; type Codomain = Values; @@ -95,6 +99,20 @@ impl, A: Clone> Function for ReductionFunction { fn codomain(&self) -> Values { Values::new() } + + // `f ∘ Ω` and `Ω ∘ f̄` agree on the multisets of at most one element when + // `f` keeps the neutral element of `Ω` (Lemma 6 of the paper; Theorem 1 is + // the case of a sum). + fn lifted_keeping_neutral( + &self, + f: &dyn Any, + ) -> Option>> { + let f = f.downcast_ref::, Codomain = Values>>>()?; + if f.apply(&R::NEUTRAL) != R::NEUTRAL { + return None; + } + Some(Rc::new(multiset_lift(f.clone()))) + } } pub type Sum = ReductionFunction; diff --git a/src/rewrite.rs b/src/rewrite.rs index 5588493..f656b00 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -1,62 +1,25 @@ //! Rewriting a tensor into an equivalent one that is cheaper to evaluate, with -//! the identities between compositions of Section 4 of the paper. Only -//! Theorem 1 is used so far. +//! the identities between compositions of Section 4 of the paper. So far: +//! Theorem 1, which Lemma 6 extends from sums to any reduction. -use crate::function::{ - Composition, Function, MultiFunction, ValueFunction, compose, multiset_lift, -}; -use crate::index_functions::Sparsity; -use crate::reductions::{Reduction, Sum, SumReduction}; -use crate::sets::{Multisets, Values}; -use crate::tensor::ReductionTensor; +use std::rc::Rc; -/// A tensor that an identity of the paper can rewrite into an equivalent one. -pub trait Rewrite: Sized { - type Rewritten; +use crate::sets::Set; +use crate::tensor::Tensor; - /// The equivalent tensor, or `Err` with the tensor unchanged when the - /// identity does not apply to it. - fn rewrite(self) -> Result; -} - -/// Rewrites `tensor` into an equivalent one that is cheaper to evaluate, or -/// gives it back unchanged as `Err` when no identity applies to it. -pub fn rewrite(tensor: T) -> Result { - tensor.rewrite() -} - -/// Theorem 1: a function on values after a sparse tensor only has to be applied -/// to its stored values, `f ∘ Σ ∘ Ū ∘ ψ = Σ ∘ (f ∘ U)‾ ∘ ψ`. This holds when -/// `f(0) = 0` and `ψ` sends each index to at most one index, which is checked. -impl Rewrite for Composition, ReductionTensor, U, M>> -where - A: Clone + PartialEq, - SumReduction: Reduction, - U: Function>, - M: Function> + Sparsity, -{ - type Rewritten = ReductionTensor, Composition, U>, M>; - - fn rewrite(self) -> Result { - let zero = >::NEUTRAL; - if self.f.apply(&zero) != zero || !self.g.g.g.is_sparse() { - return Err(self); - } - let Composition { - f, - g: - Composition { - g: - Composition { - f: MultiFunction { f: u }, - g: psi, - }, - .. - }, - } = self; - Ok(compose( - Sum::new(), - compose(multiset_lift(compose(f, u)), psi), - )) +/// Rewrites `tensor` into an equivalent one that is cheaper to evaluate, +/// wherever an identity of the paper applies inside it, and leaves it as it is +/// elsewhere. Any tensor can be given, and so can the result, again. +/// +/// A function `f` after a reduction tensor `Ω ∘ Ū ∘ ψ` is rewritten as +/// `Ω ∘ f̄ ∘ Ū ∘ ψ`, which is `Ω ∘ (f ∘ U)‾ ∘ ψ`, so that `f` is only applied to +/// the stored values. This needs `ψ` to send each index to at most one index, +/// and `f` to keep the neutral element of `Ω`, such as `f(0) = 0` for a sum. +pub fn rewrite + 'static>( + tensor: T, +) -> Rc> { + match tensor.rewritten() { + Some(rewritten) => Rc::new(rewritten), + None => Rc::new(tensor), } } diff --git a/tests/index_functions.rs b/tests/index_functions.rs index dfd93ca..852a803 100644 --- a/tests/index_functions.rs +++ b/tests/index_functions.rs @@ -2,8 +2,8 @@ use std::{assert_eq, collections::HashMap, rc::Rc}; use cten::{ Affine, COO, Concat, Diagonal, EmptyMapping, HRect, IndexFunction, IndexSet, InvAffine, - InvProjection, JaggedDomain, Multiset, Permutation, Physical, Projection, Sparsity, Squeeze, - Stride, Unsqueeze, Where, + InvProjection, JaggedDomain, Multiset, Permutation, Physical, Projection, Squeeze, Stride, + Unsqueeze, Where, }; /// The indices of a multiset in sorted order, to compare multisets whose diff --git a/tests/rewrite.rs b/tests/rewrite.rs index c3e3f1c..d39370c 100644 --- a/tests/rewrite.rs +++ b/tests/rewrite.rs @@ -2,8 +2,8 @@ use std::cell::Cell; use std::rc::Rc; use cten::{ - COO, HRect, Physical, ReductionTensor, Stride, Sum, Tensor, ValueFunction, compose, - diagonalize, rewrite, scale, sparse_coo_tensor, square, sum, + COO, Diagonal, HRect, Physical, Product, ReductionTensor, Stride, Sum, Sup, Tensor, + ValueFunction, compose, multiset_lift, rewrite, sparse_coo_tensor, square, sum, transpose, }; /// The sparse matrix of torch's `sparse_coo_tensor` documentation, which stores @@ -22,20 +22,18 @@ fn collect_2d(tensor: &dyn Tensor, rows: usize, cols: usize) -> Vec> { .collect() } -/// `x ↦ x²`, counting how many times it is applied. -fn counting_square(calls: &Rc>) -> ValueFunction { +/// `f`, counting how many times it is applied. +fn counting(calls: &Rc>, f: impl Fn(f32) -> f32 + 'static) -> ValueFunction { let calls = calls.clone(); ValueFunction::new(move |x: f32| { calls.set(calls.get() + 1); - x * x + f(x) }) } #[test] -fn theorem_1_rewrites_a_function_after_a_sparse_tensor_into_a_sparse_tensor() { - let Ok(rewritten) = rewrite(square(sparse())) else { - panic!("squaring keeps 0, so theorem 1 applies"); - }; +fn theorem_1_keeps_the_values() { + let rewritten = rewrite(square(sparse())); assert_eq!( collect_2d(&rewritten, 2, 4), @@ -52,13 +50,11 @@ fn theorem_1_rewrites_a_function_after_a_sparse_tensor_into_a_sparse_tensor() { #[test] fn a_rewritten_tensor_applies_the_function_to_the_stored_values_only() { let calls = Rc::new(Cell::new(0)); - collect_2d(&compose(counting_square(&calls), sparse()), 2, 4); + collect_2d(&compose(counting(&calls, |x| x * x), sparse()), 2, 4); assert_eq!(calls.get(), 8); let calls = Rc::new(Cell::new(0)); - let Ok(rewritten) = rewrite(compose(counting_square(&calls), sparse())) else { - panic!("squaring keeps 0, so theorem 1 applies"); - }; + let rewritten = rewrite(compose(counting(&calls, |x| x * x), sparse())); // Checking that the function keeps 0 applied it once. calls.set(0); collect_2d(&rewritten, 2, 4); @@ -67,77 +63,139 @@ fn a_rewritten_tensor_applies_the_function_to_the_stored_values_only() { #[test] fn a_rewritten_tensor_prints_like_the_original() { - let Ok(rewritten) = rewrite(square(sparse())) else { - panic!("squaring keeps 0, so theorem 1 applies"); - }; - - assert_eq!(rewritten.to_string(), square(sparse()).to_string()); + assert_eq!( + rewrite(square(sparse())).to_string(), + square(sparse()).to_string() + ); } +// Adding 1 does not keep 0, so theorem 1 does not apply, and the tensor is left +// as it is: the function is still applied at every index. #[test] -fn theorem_1_applies_to_other_functions_and_sparse_formats() { - let relu = || ValueFunction::new(|x: f32| x.max(0.0)); - let diagonal = || diagonalize(Physical::new(vec![-1.0, 2.0, -3.0]), 2); - let Ok(rewritten) = rewrite(compose(relu(), diagonal())) else { - panic!("relu keeps 0, so theorem 1 applies"); - }; +fn rewrite_leaves_a_function_that_does_not_keep_zero_alone() { + let calls = Rc::new(Cell::new(0)); + let unchanged = rewrite(compose(counting(&calls, |x| x + 1.0), sparse())); + calls.set(0); assert_eq!( - collect_2d(&rewritten, 3, 3), - vec![ - vec![0.0, 0.0, 0.0], - vec![0.0, 2.0, 0.0], - vec![0.0, 0.0, 0.0] - ] + collect_2d(&unchanged, 2, 4), + vec![vec![1.0, 1.0, 4.0, 1.0], vec![5.0, 1.0, 6.0, 1.0]] ); + assert_eq!(calls.get(), 8); +} + +// With several values at one index, squaring their sum is not summing their +// squares, so theorem 1 does not apply there and rewriting changes nothing. +#[test] +fn rewrite_leaves_several_values_at_one_index_alone() { + let repeated = || sparse_coo_tensor(vec![vec![1, 1]], Physical::new(vec![2.0, 3.0]), vec![3]); + let matrix = || { + compose( + Physical::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + Stride::new(HRect::new(vec![2, 3]), vec![3, 1]), + ) + }; + + let squared_sums = rewrite(square(repeated())); + let squared_row_sums = rewrite(square(sum(matrix(), 1))); + + let read = + |tensor: &dyn Tensor, len| (0..len).map(|i| tensor.apply(&[i])).collect::>(); + assert_eq!(read(&squared_sums, 3), vec![0.0, 25.0, 0.0]); + assert_eq!(read(&squared_row_sums, 2), vec![36.0, 225.0]); +} + +#[test] +fn rewrite_takes_any_tensor() { + let physical = rewrite(Physical::new(vec![1.0, 2.0, 3.0])); + assert_eq!( - collect_2d(&rewritten, 3, 3), - collect_2d(&compose(relu(), diagonal()), 3, 3) + (0..3).map(|i| physical.apply(&[i])).collect::>(), + vec![1.0, 2.0, 3.0] ); } -// The result is a sparse tensor again, so a second function rewrites the same way. +// The result of a rewrite is a sparse tensor again, so a second function after it +// is rewritten the same way, and neither touches more than the stored values. #[test] fn a_rewritten_tensor_can_be_rewritten_again() { - let Ok(squared) = rewrite(square(sparse())) else { - panic!("squaring keeps 0, so theorem 1 applies"); - }; - let Ok(rewritten) = rewrite(scale(squared, 2.0)) else { - panic!("scaling keeps 0, so theorem 1 applies"); - }; + let squares = Rc::new(Cell::new(0)); + let doubles = Rc::new(Cell::new(0)); + let once = rewrite(compose(counting(&squares, |x| x * x), sparse())); + let twice = rewrite(compose(counting(&doubles, |x| 2.0 * x), once)); + squares.set(0); + doubles.set(0); assert_eq!( - collect_2d(&rewritten, 2, 4), + collect_2d(&twice, 2, 4), vec![vec![0.0, 0.0, 18.0, 0.0], vec![32.0, 0.0, 50.0, 0.0]] ); + assert_eq!(squares.get(), 3); + assert_eq!(doubles.get(), 3); } -// Adding 1 does not keep 0: off the stored values, the tensor is 1 and not 0. The -// tensor comes back unchanged. +// Rewriting reaches inside a tensor: here the square under a transpose. #[test] -fn theorem_1_needs_a_function_that_keeps_zero() { - let plus_one = ValueFunction::new(|x: f32| x + 1.0); - let Err(unchanged) = rewrite(compose(plus_one, sparse())) else { - panic!("adding 1 does not keep 0, so theorem 1 does not apply"); - }; +fn rewrite_reaches_inside_a_tensor() { + let calls = Rc::new(Cell::new(0)); + let rewritten = rewrite(transpose(compose(counting(&calls, |x| x * x), sparse()))); + calls.set(0); assert_eq!( - collect_2d(&unchanged, 2, 4), - vec![vec![1.0, 1.0, 4.0, 1.0], vec![5.0, 1.0, 6.0, 1.0]] + collect_2d(&rewritten, 4, 2), + collect_2d(&transpose(square(sparse())), 4, 2) ); + assert_eq!(calls.get(), 3); } -// With several values at one index, squaring their sum is not summing their -// squares, so theorem 1 does not apply. +// Lemma 6 extends theorem 1 to any reduction: a function that keeps the neutral +// element of the reduction only has to be applied to the stored values. #[test] -fn theorem_1_needs_at_most_one_stored_value_per_index() { - let repeated = sparse_coo_tensor(vec![vec![1, 1]], Physical::new(vec![2.0, 3.0]), vec![3]); - let matrix = compose( - Physical::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), - Stride::new(HRect::new(vec![2, 3]), vec![3, 1]), +fn a_function_keeping_the_neutral_element_of_any_reduction_is_rewritten() { + let products = || { + compose( + Product::new(), + compose( + multiset_lift(Physical::new(vec![2.0, 3.0, 4.0])), + Diagonal::new(HRect::new(vec![3, 3])), + ), + ) + }; + let calls = Rc::new(Cell::new(0)); + // Squaring keeps 1, the neutral element of a product. + let rewritten = rewrite(compose(counting(&calls, |x| x * x), products())); + calls.set(0); + + assert_eq!( + collect_2d(&rewritten, 3, 3), + collect_2d(&square(products()), 3, 3) ); - let row_sums = sum(matrix, 1); + assert_eq!(calls.get(), 3); +} + +#[test] +fn a_function_changing_the_neutral_element_is_left_alone() { + let sups = || { + compose( + Sup::new(), + compose( + multiset_lift(Physical::new(vec![-2.0, 3.0, -4.0])), + Diagonal::new(HRect::new(vec![3, 3])), + ), + ) + }; + let calls = Rc::new(Cell::new(0)); + // Relu sends -∞, the neutral element of a sup, to 0. + let unchanged = rewrite(compose(counting(&calls, |x| x.max(0.0)), sups())); + calls.set(0); - assert!(rewrite(square(repeated)).is_err()); - assert!(rewrite(square(row_sums)).is_err()); + assert_eq!( + collect_2d(&unchanged, 3, 3), + vec![ + vec![0.0, 0.0, 0.0], + vec![0.0, 3.0, 0.0], + vec![0.0, 0.0, 0.0] + ] + ); + assert_eq!(calls.get(), 9); }