Add logical page indexing and selective access to PCO

.github/workflows/pco-attribution.yml

@@ -0,0 +1,81 @@
+name: PCO attribution
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/pco-attribution.yml"
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  profile:
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=pco-profile', github.run_id)
+          || 'ubuntu-latest' }}
+    timeout-minutes: 25
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
+      - name: Build frozen adverse candidate and baseline
+        run: |
+          git worktree add --detach "$RUNNER_TEMP/pco-profile-candidate" 9f468abe16dd070d2d8eed5df4f45709b50ae375
+          git worktree add --detach "$RUNNER_TEMP/pco-profile-base" 910f9933e8e2318c592c1eddcc2dfee8d24c6cd7
+          git worktree add --detach "$RUNNER_TEMP/pco-profile-latest" HEAD
+          mkdir -p "$RUNNER_TEMP/pco-profile-base/vortex-file/examples"
+          cp vortex-file/examples/pco_access_profile.rs "$RUNNER_TEMP/pco-profile-candidate/vortex-file/examples/"
+          cp vortex-file/examples/pco_access_profile.rs "$RUNNER_TEMP/pco-profile-base/vortex-file/examples/"
+          for engine in candidate base latest; do
+            cp vortex-file/tests/pco_logical_pages_compat.rs "$RUNNER_TEMP/pco-profile-$engine/vortex-file/tests/"
+            cargo build --manifest-path "$RUNNER_TEMP/pco-profile-$engine/Cargo.toml" --profile bench --locked -p vortex-file --example pco_access_profile --message-format=json-render-diagnostics > "$RUNNER_TEMP/$engine-build.jsonl"
+          done
+      - name: Bounded software attribution, separate from timings
+        shell: bash
+        run: |
+          mkdir -p "$RUNNER_TEMP/attribution"
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq linux-tools-generic
+          perf=$(find /usr/lib -path "*linux-tools*/perf" -type f | sort | tail -1)
+          test -n "$perf"
+          for engine in base candidate latest; do
+            executable=$(jq -r 'select(.reason == "compiler-artifact" and .target.name == "pco_access_profile") | .executable // empty' "$RUNNER_TEMP/$engine-build.jsonl")
+            sha256sum "$executable" > "$RUNNER_TEMP/attribution/$engine-executable.sha256"
+            git -C "$RUNNER_TEMP/pco-profile-$engine" rev-parse HEAD > "$RUNNER_TEMP/attribution/$engine-source.txt"
+            nm "$executable" | awk '/vortex_pco/ && /cached_scalar|5probe9scalar_at|scalar_at_once|scalar_at_uncached/ {print $NF}' > "$RUNNER_TEMP/attribution/$engine-symbols.txt"
+            while IFS= read -r symbol; do
+              timeout --kill-after=5s 20s objdump --disassemble="$symbol" --no-show-raw-insn "$executable" >> "$RUNNER_TEMP/attribution/$engine-scalar-assembly.txt"
+            done < "$RUNNER_TEMP/attribution/$engine-symbols.txt"
+            for request in probe:dense probe:nullable probe:all-null; do
+              IFS=: read -r mode pattern <<< "$request"
+              repeats=100000
+              stem="$RUNNER_TEMP/attribution/$engine-$mode-$pattern"
+              echo "Record $engine $mode $pattern"
+              timeout --signal=INT --kill-after=5s 25s sudo env DEBUGINFOD_URLS= "$perf" record -e cpu-clock -F 499 -o "$stem.data" -- "$executable" "$mode" "$pattern" 8388608 "$repeats" > "$RUNNER_TEMP/$engine-$mode-$pattern.jsonl" || echo "Record exit $?" > "$stem-status.txt"
+              echo "Flat report $engine $mode $pattern"
+              timeout --kill-after=5s 20s sudo env DEBUGINFOD_URLS= "$perf" report -i "$stem.data" --stdio --no-children --call-graph none --percent-limit 0.5 > "$stem-flat.txt" || echo "Flat report exit $?" >> "$stem-status.txt"
+              sudo rm -f "$stem.data"
+            done
+          done
+          printf '%s\n' 'Frozen 9f adverse candidate, develop910f, and current CI merge source. Software flat samples and exact linked symbol assembly only; profiled times are not uninstrumented latency controls. Startup/fixture compression is included. Public stock/metadata byte attribution is retained in run40119098145; it is not repeated here.' > "$RUNNER_TEMP/attribution/README.txt"
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7
+        if: always()
+        with:
+          name: pco-attribution
+          path: |
+            ${{ runner.temp }}/attribution
+            !${{ runner.temp }}/attribution/*.data
+          if-no-files-found: ignore

.github/workflows/pco-compat.yml

@@ -0,0 +1,100 @@
+name: PCO selective access compatibility
+
+on:
+  pull_request:
+    paths:
+      - "encodings/pco/**"
+      - "vortex-file/src/lib.rs"
+      - "vortex-file/tests/pco_logical_pages_compat.rs"
+      - ".github/workflows/pco-compat.yml"
+  workflow_dispatch: { }
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  pco-compat:
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=pco-compat', github.run_id)
+          || 'ubuntu-latest' }}
+    timeout-minutes: 45
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
+      - name: PCO public semantics
+        run: cargo test --profile ci --locked -p vortex-pco
+      - name: Generate new files with current public writer
+        env:
+          PCO_COMPAT_MODE: write
+          PCO_COMPAT_DIR: ${{ runner.temp }}/pco-new-files
+        run: cargo test --profile ci --locked -p vortex-file --test pco_logical_pages_compat
+      - name: Read new files using unchanged pre-change reader; generate legacy files
+        env:
+          PCO_COMPAT_DIR: ${{ runner.temp }}/pco-new-files
+        run: |
+          git worktree add --detach "$RUNNER_TEMP/pco-old-reader" 910f9933e8e2318c592c1eddcc2dfee8d24c6cd7
+          # Only the public test harness is added. The old reader/writer/library sources remain
+          # exactly the pinned pre-change revision and do not know the new metadata field.
+          cp vortex-file/tests/pco_logical_pages_compat.rs "$RUNNER_TEMP/pco-old-reader/vortex-file/tests/"
+          PCO_COMPAT_MODE=read cargo test --manifest-path "$RUNNER_TEMP/pco-old-reader/Cargo.toml" --profile ci --locked -p vortex-file --test pco_logical_pages_compat
+          PCO_COMPAT_MODE=write PCO_COMPAT_DIR="$RUNNER_TEMP/pco-old-files" cargo test --manifest-path "$RUNNER_TEMP/pco-old-reader/Cargo.toml" --profile ci --locked -p vortex-file --test pco_logical_pages_compat
+      - name: Read legacy files using current reader
+        env:
+          PCO_COMPAT_MODE: read
+          PCO_COMPAT_DIR: ${{ runner.temp }}/pco-old-files
+        run: cargo test --profile ci --locked -p vortex-file --test pco_logical_pages_compat
+      - name: Record actual public file sizes
+        run: |
+          mkdir -p "$RUNNER_TEMP/pco-resource-results"
+          wc -c "$RUNNER_TEMP"/pco-{new,old}-files/*.vortex > "$RUNNER_TEMP/pco-resource-results/public-file-sizes.txt"
+      - name: Build matching optimized public resource harnesses
+        run: |
+          mkdir -p "$RUNNER_TEMP/pco-old-reader/vortex-file/examples"
+          cp vortex-file/examples/pco_access_profile.rs "$RUNNER_TEMP/pco-old-reader/vortex-file/examples/"
+          cargo build --profile bench --locked -p vortex-file --example pco_access_profile --message-format=json-render-diagnostics > "$RUNNER_TEMP/pco-candidate-build.jsonl"
+          cargo build --manifest-path "$RUNNER_TEMP/pco-old-reader/Cargo.toml" --profile bench --locked -p vortex-file --example pco_access_profile --message-format=json-render-diagnostics > "$RUNNER_TEMP/pco-baseline-build.jsonl"
+      - name: Matched first and warmed resource controls
+        shell: bash
+        run: |
+          mkdir -p "$RUNNER_TEMP/pco-resource-results"
+          printf '%s\n' 'Wall timings exclude accounting and post-timing scalar inspection.' 'open-only/open-first include public array-plugin metadata decode/validation/index preparation with compressed buffers and validity children already available; no file IO.' 'probe-first starts with prepared metadata/index; probe retains touched pages across iterations.' 'CPU ticks are OS process user+system ticks; sub-tick readings are resolution-limited, not zero CPU.' 'RSS snapshots may miss transient peaks. GNU time maximum RSS is whole-process peak including fixtures/compression and post-timing inspection, not operation-only peak.' > "$RUNNER_TEMP/pco-resource-results/README.txt"
+          getconf CLK_TCK > "$RUNNER_TEMP/pco-resource-results/clock-ticks-per-second.txt"
+          git rev-parse HEAD > "$RUNNER_TEMP/pco-resource-results/candidate-source.txt"
+          git -C "$RUNNER_TEMP/pco-old-reader" rev-parse HEAD > "$RUNNER_TEMP/pco-resource-results/baseline-source.txt"
+          candidate=$(jq -r 'select(.reason == "compiler-artifact" and .target.name == "pco_access_profile") | .executable // empty' "$RUNNER_TEMP/pco-candidate-build.jsonl")
+          baseline=$(jq -r 'select(.reason == "compiler-artifact" and .target.name == "pco_access_profile") | .executable // empty' "$RUNNER_TEMP/pco-baseline-build.jsonl")
+          sha256sum "$candidate" "$baseline" > "$RUNNER_TEMP/pco-resource-results/executables.sha256"
+          for mode in write full slice slice-near-full take take-large take-repeated take-mixed take-narrow take-spans take-tiny take-contiguous take-range ranges open-only open-first probe-first probe; do
+            for pattern in dense nullable all-null long-null compressed nullable-legacy; do
+              # ABBA process order; every process retains all eight iterations including cold.
+              order=0
+              for engine in baseline candidate candidate baseline; do
+                order=$((order + 1))
+                executable="$candidate"
+                if [ "$engine" = baseline ]; then
+                  executable="$baseline"
+                fi
+                stem="$RUNNER_TEMP/pco-resource-results/$mode-$pattern-$order-$engine"
+                /usr/bin/time -v "$executable" "$mode" "$pattern" 8388608 8 > "$stem.jsonl" 2> "$stem.time"
+              done
+            done
+          done
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7
+        if: always()
+        with:
+          name: pco-resource-results
+          path: ${{ runner.temp }}/pco-resource-results
+          if-no-files-found: ignore