Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f3438f3
Record frozen editions as append-only files under vortex/editions
claude Aug 11, 2026
8781cc0
Record required_vortex_release in the frozen edition records
claude Aug 11, 2026
55b6e66
Record every edition, and lock a record only once its edition freezes
claude Aug 11, 2026
d1040c6
Export the edition records from an xtask instead of a test
claude Aug 11, 2026
d25151b
Group the edition records by family
claude Aug 11, 2026
479040e
Move the edition declarations into vortex-edition, and check records …
claude Aug 12, 2026
eca8b10
Document the unstable edition family
claude Aug 12, 2026
7a4e4e3
Declare edition families, and record what each one is for
claude Aug 12, 2026
5f1b7af
Port the frozen-record check to an xtask
claude Aug 12, 2026
925fbf1
Fix the CI failures from adding git2
claude Aug 12, 2026
f5d5ef0
Quote the step condition with double quotes
claude Aug 12, 2026
577d72e
Adopt develop's edition set after the rebase
robert3005 Aug 20, 2026
89e66b3
Require two approvals for edition changes, and document the freeze pr…
robert3005 Aug 20, 2026
fbce2ac
Declare the json and spatial edition families
joseph-isaacs Aug 21, 2026
7d54f5b
Add the SPDX header to .policy.yml
joseph-isaacs Aug 21, 2026
f64cb63
rework
robert3005 Aug 25, 2026
b6b43fb
more
robert3005 Aug 26, 2026
202a56e
more
robert3005 Aug 26, 2026
c1af399
reword
robert3005 Aug 28, 2026
dc2ca47
Move optional components into edition families
robert3005 Sep 1, 2026
a6c479b
Fix edition dependency lockfile
robert3005 Sep 1, 2026
e9761ed
Correct the OnPair minimum library version
robert3005 Sep 1, 2026
ae4d997
Split component editions and update core 2026.08
robert3005 Sep 1, 2026
047878e
more
robert3005 Sep 1, 2026
256ebf7
Correct the core 2026.08.1 minimum version
robert3005 Sep 1, 2026
1c49a41
Fix edition CI checks
robert3005 Sep 1, 2026
fe619d4
Remove the unstable encodings feature
robert3005 Sep 1, 2026
b77bcb3
Move golden snapshots to component editions
robert3005 Sep 1, 2026
9db048d
Fix edition-specific CI configurations
robert3005 Sep 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -517,21 +517,15 @@ jobs:
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
# The golden suite pins the compressor's decisions per feature variant, and the
# variants are mutually exclusive at compile time: the `default` variant only exists
# without `unstable_encodings` (which changes ALL_SCHEMES), so the `--all-features`
# workspace test jobs cannot run it. Run each feature combination explicitly.
- name: Golden corpus (default features)
# The golden suite pins compressor decisions for the core-only and explicitly enabled
# edition configurations. Compact compression additionally needs its optional schemes.
- name: Golden corpus (edition configurations)
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden
- name: Golden corpus (unstable_encodings)
- name: Golden corpus (compact, zstd + pco)
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \
--features unstable_encodings
- name: Golden corpus (compact, unstable_encodings + zstd + pco)
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \
--features unstable_encodings,zstd,pco
--features zstd,pco -E 'test(golden_compact)'

build-java:
name: "Java"
Expand Down Expand Up @@ -729,6 +723,9 @@ jobs:
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# check-editions compares against the merge base, so it needs real history.
fetch-depth: 0
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
Expand All @@ -738,6 +735,16 @@ jobs:
run: |
cargo run --profile ci -p xtask -- generate-fbs
cargo run --profile ci -p xtask -- generate-proto
- name: "regenerate the edition records"
run: |
cargo run --profile ci -p xtask -- generate-editions
- name: "check frozen edition records never change"
# Independent of the regeneration above: a stale record must not mask a frozen one
# being edited, nor the other way round.
if: "!cancelled()"
run: |
BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}"
cargo run --profile ci -p xtask -- check-editions --base "$BASE"
- name: "regenerate FFI header file"
run: |
cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi
Expand Down
49 changes: 49 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ geo-types = "0.7.19"
geoarrow = "0.8.0"
geoarrow-cast = "0.8.0"
get_dir = "0.5.0"
git2 = { version = "0.21", default-features = false }
glob = "0.3.2"
goldenfile = "1"
half = { version = "2.7.1", features = ["std", "num-traits"] }
Expand Down Expand Up @@ -276,6 +277,7 @@ thiserror = "2.0.3"
tokio = { version = "1.52" }
tokio-stream = "0.1.17"
tokio-util = "0.7.17"
toml = "0.9"
tpchgen = "3.0.0"
tpchgen-arrow = "3.0.0"
tracing = { version = "0.1.41", default-features = false }
Expand Down
2 changes: 1 addition & 1 deletion bench-orchestrator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,6 @@ Benchmarks are built with:

- Profile: `release_debug`
- RUSTFLAGS: `-C target-cpu=native -C force-frame-pointers=yes`
- Features: `unstable_encodings`
- Features: none

This enables native CPU optimizations while preserving debug symbols for profiling.
2 changes: 1 addition & 1 deletion bench-orchestrator/bench_orchestrator/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ class BuildConfig:

profile: str = "release_debug"
rustflags: str = "-C target-cpu=native -C force-frame-pointers=yes"
features: tuple[str, ...] = ("unstable_encodings",)
features: tuple[str, ...] = ()


def get_workspace_root() -> Path:
Expand Down
1 change: 0 additions & 1 deletion benchmarks/compress-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ vortex-cuda = { workspace = true, optional = true }
[features]
cuda = ["dep:tempfile", "dep:vortex-cuda"]
lance = ["dep:lance-bench"]
unstable_encodings = ["vortex/unstable_encodings", "vortex-cuda?/unstable_encodings"]

[[bin]]
name = "compress-bench"
Expand Down
6 changes: 3 additions & 3 deletions benchmarks/compress-bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ Both sides therefore decode all the way to device-resident arrays, which is what

```bash
cargo run -p compress-bench --profile release_debug \
--features cuda,unstable_encodings -- --gpu-decompress
--features cuda -- --gpu-decompress

# pick the Parquet page codec the GPU file is written with (default: snappy)
cargo run -p compress-bench --profile release_debug \
--features cuda,unstable_encodings -- --gpu-decompress --gpu-parquet-codec zstd
--features cuda -- --gpu-decompress --gpu-parquet-codec zstd
```

### cuDF
Expand Down Expand Up @@ -116,7 +116,7 @@ still publishes comparable numbers — it just takes considerably longer:

```bash
cargo run -p compress-bench --profile release_debug \
--features cuda,unstable_encodings -- --gpu-decompress --gpu-verify --iterations 1
--features cuda -- --gpu-decompress --gpu-verify --iterations 1
```

Any `--gpu-decompress` run reports on every dataset rather than stopping at the first failure, so
Expand Down
1 change: 0 additions & 1 deletion benchmarks/datafusion-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ custom-labels = { workspace = true }

[features]
cuda = ["dep:vortex-cuda"]
unstable_encodings = ["vortex/unstable_encodings"]

[lints]
workspace = true
1 change: 0 additions & 1 deletion benchmarks/duckdb-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ vortex-duckdb = { workspace = true }

[features]
cuda = ["dep:vortex-cuda"]
unstable_encodings = ["vortex/unstable_encodings"]

[lints]
workspace = true
3 changes: 0 additions & 3 deletions benchmarks/lance-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@ tokio = { workspace = true, features = ["full"] }
tracing = { workspace = true }
vortex-bench = { workspace = true }

[features]
unstable_encodings = ["vortex-bench/unstable_encodings"]

[lib]
test = false

Expand Down
1 change: 0 additions & 1 deletion benchmarks/random-access-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ vortex-bench = { workspace = true }

[features]
lance = ["dep:lance-bench"]
unstable_encodings = ["vortex-bench/unstable_encodings"]

[[bin]]
name = "random-access-bench"
Expand Down
7 changes: 0 additions & 7 deletions benchmarks/string-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,8 @@ vortex-btrblocks = { workspace = true }
vortex-fsst = { workspace = true }
vortex-onpair = { workspace = true }

[features]
unstable_encodings = [
"vortex/unstable_encodings",
"vortex-btrblocks/unstable_encodings",
]

[[bin]]
name = "string-bench"
required-features = ["unstable_encodings"]

[lints]
workspace = true
10 changes: 5 additions & 5 deletions benchmarks/string-bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,22 +80,22 @@ Input preparation is outside benchmark timing.

```bash
# Tracked metrics: size, write, read for every configured column and encoder.
cargo run -p string-bench --profile release_debug --features unstable_encodings
cargo run -p string-bench --profile release_debug

# Focus on selected columns or encoders.
cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
cargo run -p string-bench --profile release_debug -- \
--columns URL --encoders onpair

# Add the direct codec microbenchmark.
cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
cargo run -p string-bench --profile release_debug -- \
--suite both

# Emit benchmark-comparator JSONL.
cargo run -p string-bench --profile release_debug --features unstable_encodings -- \
cargo run -p string-bench --profile release_debug -- \
--display-format gh-json --output-path results.json
```

Run `cargo run -p string-bench --features unstable_encodings -- --help` for all
Run `cargo run -p string-bench -- --help` for all
filters and tuning options.

Before timing, the benchmark checks that each requested encoding was produced
Expand Down
2 changes: 0 additions & 2 deletions benchmarks/string-bench/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

#![cfg(feature = "unstable_encodings")]

//! String-compression benchmarks for Vortex.
//!
//! Two intentionally separate suites:
Expand Down
Loading
Loading