diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py
index fdd15e08e6..cbdff9dbce 100644
--- a/src/spikeinterface/core/basesorting.py
+++ b/src/spikeinterface/core/basesorting.py
@@ -11,11 +11,6 @@
 
 #: Makes each unit's spiketrain compact in memory (unit, then segment, then sample).
 LEXSORT_UNIT_COMPACT = ("sample_index", "segment_index", "unit_index")
-#: Makes each segment compact, units compact within a segment (segment, then unit, then sample).
-LEXSORT_SEGMENT_COMPACT = ("sample_index", "unit_index", "segment_index")
-
-# The reorderings that `BaseSorting.to_reordered_spike_vector()` can produce.
-_ALLOWED_LEXSORTS = (LEXSORT_UNIT_COMPACT, LEXSORT_SEGMENT_COMPACT)
 
 
 class BaseSorting(BaseExtractor):
@@ -35,14 +30,13 @@ def __init__(self, sampling_frequency: float, unit_ids: list):
         self._sorting_info = None
 
         # caching:
-        # 1. the spike vector : one vector complex dtype
-        self._cached_spike_vector = None
-        # 2. the segment slices in the spike vector
-        self._cached_spike_vector_segment_slices = None
-        # 3. the indices of spike train inside the spike_vector per segment (list) then unit (dict)
-        self._cached_spike_vector_to_indices = None
-        # 4. reordering of the spike vector
-        self._cached_lexsorted_spike_vector = {}
+        # 1. the spike vector : one vector complex dtype, with its segment slices (computed lazily if None)
+        #    {"spikes": spike vector, "segment_slices": None or (num_segments, 2) array}
+        self._cached_time_ordered = None
+        # 2. the indices of spike train inside the spike_vector per segment (list) then unit (dict)
+        self._cached_time_ordered_indices_by_unit = None
+        # 3. reordering of the spike vector
+        self._cached_unit_grouped = None
 
     def __repr__(self):
         return self._repr_header()
@@ -172,9 +166,9 @@ def search_cached_spikes_sorted(
         positions : np.ndarray
             The insertion positions, relative to the start of the segment in the spike vector.
         """
-        if self._cached_spike_vector is None:
+        if self._cached_time_ordered is None:
             self._compute_and_cache_spike_vector()
-        spikes = self._cached_spike_vector
+        spikes = self._cached_time_ordered["spikes"]
         if not isinstance(spikes, np.ndarray):  # np.memmap is an ndarray
             # np.searchsorted would materialise a lazy vector on every call
             raise TypeError(
@@ -243,13 +237,12 @@ def get_unit_spike_train(
 
         segment_index = self._check_segment_index(segment_index)
 
-        lexsort_key = LEXSORT_UNIT_COMPACT
-        if lexsort_key in self._cached_lexsorted_spike_vector.keys():
+        if self._cached_unit_grouped is not None:
             use_cache = True
 
         if use_cache:
             ordered_spike_vector, slices = self.to_reordered_spike_vector(
-                lexsort=lexsort_key,
+                lexsort=LEXSORT_UNIT_COMPACT,
                 return_order=False,
                 return_slices=True,
             )
@@ -676,12 +669,10 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None):
         """
 
         # speed strategy by order
-        # 1. if _cached_lexsorted_spike_vector has LEXSORT_UNIT_COMPACT then use it and sum
-        # 2. if _cached_spike_vector not None then use it with np.unique()
+        # 1. if _cached_unit_grouped is not None then use it and sum
+        # 2. if _cached_time_ordered not None then use it with np.unique()
         # 3. compute spikevector and do np.unique()
 
-        cache_key = LEXSORT_UNIT_COMPACT
-
         if unit_ids is not None:
             assert outputs == "dict", "count_num_spikes_per_unit() with unit_ids not None works only for output='dict'"
 
@@ -693,9 +684,9 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None):
             keep_mask = slice(None)
             unit_ids = self.unit_ids
 
-        if cache_key in self._cached_lexsorted_spike_vector:
+        if self._cached_unit_grouped is not None:
             # case 1
-            slices = self._cached_lexsorted_spike_vector[cache_key]["slices"]
+            slices = self._cached_unit_grouped["slices"]
             # end of last segment minus start of first segment
             num_spikes = slices[keep_mask, -1, 1] - slices[keep_mask, 0, 0]
 
@@ -965,9 +956,8 @@ def precompute_spike_trains(self):
         Pre-computes and caches all spike trains for this sorting.
         This is equivalent to cache lexsort LEXSORT_UNIT_COMPACT.
         """
-        cache_key = LEXSORT_UNIT_COMPACT
-        if cache_key not in self._cached_lexsorted_spike_vector:
-            self.to_reordered_spike_vector(lexsort=cache_key)
+        if self._cached_unit_grouped is None:
+            self.to_reordered_spike_vector(lexsort=LEXSORT_UNIT_COMPACT)
 
     def _compute_and_cache_spike_vector(self) -> None:
         #
@@ -977,7 +967,7 @@ def _compute_and_cache_spike_vector(self) -> None:
         Function that can be implemented by some children sorting to quickly
         cache the spike vector without computing it from spike trains
         (e.g. computing it from a sorting parent).
-        This function should set the `self._cached_spike_vector`, see for
+        This function should set the `self._cached_time_ordered`, see for
         instance the `UnitsSelectionSorting` implementation.
         """
 
@@ -1023,8 +1013,7 @@ def _compute_and_cache_spike_vector(self) -> None:
         # the spikes are not lexsorted here because the previous loop ensure that the spike vector is constructucted alway the same way.
         # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))]
 
-        self._cached_spike_vector = spikes
-        self._cached_spike_vector_segment_slices = segment_slices
+        self._cached_time_ordered = {"spikes": spikes, "segment_slices": segment_slices}
 
     def to_spike_vector(
         self,
@@ -1051,7 +1040,7 @@ def to_spike_vector(
             This can be convenient for computing spikes position after sorter.
             This dict can be given by analyzer.get_main_channels(outputs="index", with_dict=False)
         use_cache : bool, default: True
-            When True the spikes vector is cached as an attribute of the object (`_cached_spike_vector`).
+            When True the spikes vector is cached as an attribute of the object (`_cached_time_ordered`).
             This caching only occurs when main_channel_indices=None.
 
         Returns
@@ -1063,8 +1052,9 @@ def to_spike_vector(
 
         """
 
-        if self._cached_spike_vector is None:
+        if self._cached_time_ordered is None:
             self._compute_and_cache_spike_vector()
+        cached_spikes = self._cached_time_ordered["spikes"]
 
         if extremum_channel_inds is not None:
             warnings.warn(
@@ -1075,13 +1065,13 @@ def to_spike_vector(
             main_channel_indices = np.array([extremum_channel_inds[unit_id] for unit_id in self.unit_ids])
 
         if main_channel_indices is None:
-            spikes = self._cached_spike_vector
-        elif "channel_index" in self._cached_spike_vector.dtype.names:
-            spikes = self._cached_spike_vector
+            spikes = cached_spikes
+        elif "channel_index" in cached_spikes.dtype.names:
+            spikes = cached_spikes
         else:
             spike_dtype = minimum_spike_dtype + [("channel_index", "int64")]
-            spikes = np.zeros(self._cached_spike_vector.size, dtype=spike_dtype)
-            spikes[["sample_index", "unit_index", "segment_index"]] = self._cached_spike_vector
+            spikes = np.zeros(cached_spikes.size, dtype=spike_dtype)
+            spikes[["sample_index", "unit_index", "segment_index"]] = cached_spikes
             spikes["channel_index"] = main_channel_indices[spikes["unit_index"]]
 
         if not concatenated:
@@ -1107,31 +1097,30 @@ def get_spike_vector_to_indices(self):
         Note that, indices are return using absolute_index=True.
         """
 
-        if self._cached_spike_vector_to_indices is None:
+        if self._cached_time_ordered_indices_by_unit is None:
             from .sorting_tools import spike_vector_to_indices
 
             spike_vector = self.to_spike_vector(concatenated=False)
-            self._cached_spike_vector_to_indices = spike_vector_to_indices(
+            self._cached_time_ordered_indices_by_unit = spike_vector_to_indices(
                 spike_vector, self.unit_ids, absolute_index=True
             )
 
-        return self._cached_spike_vector_to_indices
+        return self._cached_time_ordered_indices_by_unit
 
     def _get_spike_vector_segment_slices(self):
-        if self._cached_spike_vector_segment_slices is None:
+        cache = self._cached_time_ordered
+        if cache["segment_slices"] is None:
             # compute the, this is needed when spikevector is loaded from format and not computed
             num_seg = self.get_num_segments()
             if num_seg == 1:
-                self._cached_spike_vector_segment_slices = np.array(
-                    [[0, self._cached_spike_vector.size]], dtype="int64"
-                )
+                cache["segment_slices"] = np.array([[0, cache["spikes"].size]], dtype="int64")
             else:
-                slices = np.searchsorted(self._cached_spike_vector["segment_index"], np.arange(num_seg + 1))
-                self._cached_spike_vector_segment_slices = np.zeros((num_seg, 2), dtype="int64")
+                slices = np.searchsorted(cache["spikes"]["segment_index"], np.arange(num_seg + 1))
+                cache["segment_slices"] = np.zeros((num_seg, 2), dtype="int64")
                 for seg_index in range(num_seg):
-                    self._cached_spike_vector_segment_slices[seg_index, 0] = slices[seg_index]
-                    self._cached_spike_vector_segment_slices[seg_index, 1] = slices[seg_index + 1]
-        return self._cached_spike_vector_segment_slices
+                    cache["segment_slices"][seg_index, 0] = slices[seg_index]
+                    cache["segment_slices"][seg_index, 1] = slices[seg_index + 1]
+        return cache["segment_slices"]
 
     def to_reordered_spike_vector(
         self,
@@ -1147,20 +1136,17 @@ def to_reordered_spike_vector(
 
         Please note that the lexsort syntax is the **reverse** of natural reading.
 
-        Two reorderings are supported:
+        One reordering is supported:
           - LEXSORT_UNIT_COMPACT: ("sample_index", "segment_index", "unit_index").
-            Makes each unit's spiketrain compact in memory. This is the default, and is what
+            Makes each unit's spiketrain compact in memory. This is what
             unit-by-unit computations (e.g. isi violations) want.
-          - LEXSORT_SEGMENT_COMPACT: ("sample_index", "unit_index", "segment_index").
-            Makes each segment compact, with each unit's spiketrain compact within a segment.
-            Rarely (if ever) used, but might be useful when iterating segment by segment.
 
         This operation is internally cached.
 
         Parameters
         ----------
         lexsort : tuple, default: LEXSORT_UNIT_COMPACT
-            The requested sort order. Must be one of the two orderings listed above.
+            The requested sort order. Must be the ordering listed above.
         return_order: bool, default: True
             Return the numpy array needed to sort the spike vector (given the requested sort).
         return_slices: bool, default: True
@@ -1175,14 +1161,14 @@ def to_reordered_spike_vector(
             Numpy array needed to sort the spike vector given the lexsort. Can be used
             to sort other external vectors like spike_amplitudes, spike_locations, ...
         slices : np.array
-            A 3D array of internal slices for fast access to a compact portion of the reordered spikes.
-            Depending on the lexsort, a numpy array of size (num_units, num_segments, 2) or (num_segments, num_units, 2).
+            A 3D array of internal slices for fast access to a compact portion of the reordered spikes,
+            of size (num_units, num_segments, 2).
             The last dimension contains the start and end indices of each segment-unit pair.
 
         Raises
         ------
         ValueError
-            If `lexsort` is not one of the two supported orderings.
+            If `lexsort` is not the supported ordering.
         """
         if lexsort == ("unit_index", "sample_index", "segment_index"):
             raise ValueError(
@@ -1190,21 +1176,20 @@ def to_reordered_spike_vector(
                 "Use `to_spike_vector()` to get the default order."
             )
 
-        if lexsort not in _ALLOWED_LEXSORTS:
-            raise ValueError(f"`lexsort` must be one of {_ALLOWED_LEXSORTS}; got {lexsort}.")
+        if lexsort != LEXSORT_UNIT_COMPACT:
+            raise ValueError(f"`lexsort` must be {LEXSORT_UNIT_COMPACT}; got {lexsort}.")
 
-        if lexsort not in self._cached_lexsorted_spike_vector.keys():
+        if self._cached_unit_grouped is None:
             from .sorting_tools import reorder_spike_vector_by_unit_and_segment
 
             spikes = self.to_spike_vector()
             num_units = len(self.unit_ids)
             num_segments = self.get_num_segments()
 
-            unit_major = lexsort == LEXSORT_UNIT_COMPACT
-            slices_shape = (num_units, num_segments) if unit_major else (num_segments, num_units)
+            slices_shape = (num_units, num_segments)
 
             ordered_spikes, order, counts = reorder_spike_vector_by_unit_and_segment(
-                spikes, num_units, num_segments, unit_major=unit_major
+                spikes, num_units, num_segments, unit_major=True
             )
 
             counts = counts.reshape(slices_shape)
@@ -1212,13 +1197,13 @@ def to_reordered_spike_vector(
             starts = stops - counts
             slices = np.stack([starts, stops], axis=-1).astype(np.int64, copy=False)
 
-            self._cached_lexsorted_spike_vector[lexsort] = {
+            self._cached_unit_grouped = {
                 "ordered_spikes": ordered_spikes,
                 "order": order,
                 "slices": slices,
             }
 
-        cached = self._cached_lexsorted_spike_vector[lexsort]
+        cached = self._cached_unit_grouped
         out = [cached["ordered_spikes"]]
         if return_order:
             out.append(cached["order"])
@@ -1241,12 +1226,14 @@ def to_numpy_sorting(self, propagate_cache=True):
 
         sorting = NumpySorting.from_sorting(self)
         if propagate_cache:
-            if len(self._cached_lexsorted_spike_vector) > 0:
-                sorting._cached_lexsorted_spike_vector = deepcopy(self._cached_lexsorted_spike_vector)
-            if self._cached_spike_vector_segment_slices is not None:
-                sorting._cached_spike_vector_segment_slices = self._cached_spike_vector_segment_slices.copy()
-            if self._cached_spike_vector_to_indices is not None:
-                sorting._cached_spike_vector_to_indices = deepcopy(self._cached_spike_vector_to_indices)
+            if self._cached_unit_grouped is not None:
+                sorting._cached_unit_grouped = deepcopy(self._cached_unit_grouped)
+            if self._cached_time_ordered is not None:
+                segment_slices = self._cached_time_ordered["segment_slices"]
+                if segment_slices is not None:
+                    sorting._cached_time_ordered["segment_slices"] = segment_slices.copy()
+            if self._cached_time_ordered_indices_by_unit is not None:
+                sorting._cached_time_ordered_indices_by_unit = deepcopy(self._cached_time_ordered_indices_by_unit)
 
         return sorting
 
diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py
index 28ae361fc0..c9591e2e88 100644
--- a/src/spikeinterface/core/generate.py
+++ b/src/spikeinterface/core/generate.py
@@ -366,18 +366,18 @@ def __init__(
         BaseSorting.__init__(self, sampling_frequency, unit_ids)
 
         self.parent_unit_ids = sorting.unit_ids
-        self._cached_spike_vector = sorting.to_spike_vector().copy()
+        spikes = sorting.to_spike_vector().copy()
         self.refractory_period_ms = refractory_period_ms
 
-        self.added_spikes_from_existing_mask = np.zeros(len(self._cached_spike_vector), dtype=bool)
-        self.added_spikes_from_new_mask = np.zeros(len(self._cached_spike_vector), dtype=bool)
+        self.added_spikes_from_existing_mask = np.zeros(len(spikes), dtype=bool)
+        self.added_spikes_from_new_mask = np.zeros(len(spikes), dtype=bool)
 
         if added_spikes_existing_units is not None and len(added_spikes_existing_units) > 0:
             assert (
                 added_spikes_existing_units.dtype == minimum_spike_dtype
             ), "added_spikes_existing_units should be a spike vector"
             added_unit_indices = np.arange(len(self.parent_unit_ids))
-            self._cached_spike_vector = np.concatenate((self._cached_spike_vector, added_spikes_existing_units))
+            spikes = np.concatenate((spikes, added_spikes_existing_units))
             self.added_spikes_from_existing_mask = np.concatenate(
                 (self.added_spikes_from_existing_mask, np.ones(len(added_spikes_existing_units), dtype=bool))
             )
@@ -389,7 +389,7 @@ def __init__(
             assert (
                 added_spikes_new_units.dtype == minimum_spike_dtype
             ), "added_spikes_new_units should be a spike vector"
-            self._cached_spike_vector = np.concatenate((self._cached_spike_vector, added_spikes_new_units))
+            spikes = np.concatenate((spikes, added_spikes_new_units))
             self.added_spikes_from_existing_mask = np.concatenate(
                 (self.added_spikes_from_existing_mask, np.zeros(len(added_spikes_new_units), dtype=bool))
             )
@@ -397,14 +397,20 @@ def __init__(
                 (self.added_spikes_from_new_mask, np.ones(len(added_spikes_new_units), dtype=bool))
             )
 
-        sort_idxs = np.lexsort([self._cached_spike_vector["sample_index"], self._cached_spike_vector["segment_index"]])
-        self._cached_spike_vector = self._cached_spike_vector[sort_idxs]
+        sort_idxs = np.lexsort(
+            [
+                spikes["sample_index"],
+                spikes["segment_index"],
+            ]
+        )
+        spikes = spikes[sort_idxs]
+        self._cached_time_ordered = {"spikes": spikes, "segment_slices": None}
         self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[sort_idxs]
         self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[sort_idxs]
 
         # We need to add the sorting segments
         for segment_index in range(sorting.get_num_segments()):
-            segment = SpikeVectorSortingSegment(self._cached_spike_vector, segment_index, unit_ids=self.unit_ids)
+            segment = SpikeVectorSortingSegment(spikes, segment_index, unit_ids=self.unit_ids)
             self.add_sorting_segment(segment)
 
         if self.refractory_period_ms is not None:
@@ -557,20 +563,18 @@ def clean_refractory_period(self):
         ## This function will remove the added spikes that will violate RPV, but does not affect the
         ## spikes in the original sorting. So if some RPV violation are present in this sorting,
         ## they will be left untouched
-        unit_indices = np.unique(self._cached_spike_vector["unit_index"])
+        spikes = self._cached_time_ordered["spikes"]
+        unit_indices = np.unique(spikes["unit_index"])
         rpv = int(self.get_sampling_frequency() * self.refractory_period_ms / 1000)
         to_keep = ~self.added_spikes_from_existing_mask.copy()
         for segment_index in range(self.get_num_segments()):
             for unit_ind in unit_indices:
-                (indices,) = np.nonzero(
-                    (self._cached_spike_vector["unit_index"] == unit_ind)
-                    * (self._cached_spike_vector["segment_index"] == segment_index)
-                )
+                (indices,) = np.nonzero((spikes["unit_index"] == unit_ind) * (spikes["segment_index"] == segment_index))
                 to_keep[indices[1:]] = np.logical_or(
-                    to_keep[indices[1:]], np.diff(self._cached_spike_vector[indices]["sample_index"]) > rpv
+                    to_keep[indices[1:]], np.diff(spikes[indices]["sample_index"]) > rpv
                 )
 
-        self._cached_spike_vector = self._cached_spike_vector[to_keep]
+        self._cached_time_ordered = {"spikes": spikes[to_keep], "segment_slices": None}
         self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[to_keep]
         self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[to_keep]
 
diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py
index 7f5f5321b9..717c925efa 100644
--- a/src/spikeinterface/core/numpyextractors.py
+++ b/src/spikeinterface/core/numpyextractors.py
@@ -287,7 +287,7 @@ def __init__(self, spikes, sampling_frequency, unit_ids):
             self.add_sorting_segment(SpikeVectorSortingSegment(spikes, segment_index, unit_ids))
 
         # important trick : the cache is already spikes vector
-        self._cached_spike_vector = spikes
+        self._cached_time_ordered = {"spikes": spikes, "segment_slices": None}
 
         self._kwargs = dict(spikes=spikes, sampling_frequency=sampling_frequency, unit_ids=unit_ids)
 
@@ -523,7 +523,7 @@ def __init__(self, shm_name, shape, sampling_frequency, unit_ids, dtype=minimum_
             self.add_sorting_segment(SpikeVectorSortingSegment(self.shm_spikes, segment_index, unit_ids))
 
         # important trick : the cache is already spikes vector
-        self._cached_spike_vector = self.shm_spikes
+        self._cached_time_ordered = {"spikes": self.shm_spikes, "segment_slices": None}
 
         # this is very important for the shm.unlink()
         # only the main instance need to call it
diff --git a/src/spikeinterface/core/sortingfolder.py b/src/spikeinterface/core/sortingfolder.py
index fefe5c1229..8879e8e2fc 100644
--- a/src/spikeinterface/core/sortingfolder.py
+++ b/src/spikeinterface/core/sortingfolder.py
@@ -53,7 +53,7 @@ def __init__(self, folder_path, mmap_mode: str | None = None):
         for segment_index in range(num_segments):
             self.add_sorting_segment(SpikeVectorSortingSegment(self.spikes, segment_index, unit_ids))
         # important trick : the cache is already spikes vector
-        self._cached_spike_vector = self.spikes
+        self._cached_time_ordered = {"spikes": self.spikes, "segment_slices": None}
 
         load_properties_from_folder(folder_path / "properties", self)
         load_annotations_from_folder(folder_path, self)
diff --git a/src/spikeinterface/core/tests/test_basesorting.py b/src/spikeinterface/core/tests/test_basesorting.py
index ec7c2cefb6..61ac0ad761 100644
--- a/src/spikeinterface/core/tests/test_basesorting.py
+++ b/src/spikeinterface/core/tests/test_basesorting.py
@@ -98,7 +98,7 @@ def test_BaseSorting(create_cache_folder):
 
     spikes = sorting.to_spike_vector()
     # print(spikes)
-    assert sorting._cached_spike_vector is not None
+    assert sorting._cached_time_ordered is not None
 
     spikes = sorting.to_spike_vector()
     ordered_spikes, order, slices = sorting.to_reordered_spike_vector(
@@ -108,17 +108,6 @@ def test_BaseSorting(create_cache_folder):
     )
     assert np.array_equal(spikes[order], ordered_spikes)
 
-    ordered_spikes, order, slices = sorting.to_reordered_spike_vector(
-        lexsort=(
-            "sample_index",
-            "unit_index",
-            "segment_index",
-        ),
-        return_order=True,
-        return_slices=True,
-    )
-    assert np.array_equal(spikes[order], ordered_spikes)
-
     num_spikes_per_unit = sorting.count_num_spikes_per_unit(outputs="dict")
     num_spikes_per_unit = sorting.count_num_spikes_per_unit(outputs="array")
     total_spikes = sorting.count_total_num_spikes()
@@ -185,11 +174,7 @@ def _make_sorting_with_shuffled_ties(num_units, num_segments, seed=42):
 
 
 @pytest.mark.parametrize("use_numba", [True, False], ids=["numba", "numpy"])
-@pytest.mark.parametrize(
-    "lexsort",
-    [("sample_index", "segment_index", "unit_index"), ("sample_index", "unit_index", "segment_index")],
-)
-def test_to_reordered_spike_vector(lexsort, use_numba, monkeypatch):
+def test_to_reordered_spike_vector(use_numba, monkeypatch):
     """`to_reordered_spike_vector` should group spikes by (unit, segment) without disturbing them."""
     if use_numba and importlib.util.find_spec("numba") is None:
         pytest.skip("numba not installed")
@@ -205,17 +190,12 @@ def test_to_reordered_spike_vector(lexsort, use_numba, monkeypatch):
     )
 
     ordered_spikes, order, slices = sorting.to_reordered_spike_vector(
-        lexsort=lexsort, return_order=True, return_slices=True
+        lexsort=LEXSORT_UNIT_COMPACT, return_order=True, return_slices=True
     )
 
-    # The buckets, in the order the requested lexsort puts them in.
-    unit_major = lexsort == ("sample_index", "segment_index", "unit_index")
-    if unit_major:
-        assert slices.shape == (num_units, num_segments, 2)
-        groups = [(u, s) for u in range(num_units) for s in range(num_segments)]
-    else:
-        assert slices.shape == (num_segments, num_units, 2)
-        groups = [(u, s) for s in range(num_segments) for u in range(num_units)]
+    # The buckets, unit by unit and segment by segment within each unit.
+    assert slices.shape == (num_units, num_segments, 2)
+    groups = [(u, s) for u in range(num_units) for s in range(num_segments)]
     masks = [(spikes["unit_index"] == u) & (spikes["segment_index"] == s) for u, s in groups]
 
     assert np.array_equal(ordered_spikes, np.concatenate([spikes[mask] for mask in masks]))
diff --git a/src/spikeinterface/core/tests/test_numpy_extractors.py b/src/spikeinterface/core/tests/test_numpy_extractors.py
index d0a8c5be70..a748ea2264 100644
--- a/src/spikeinterface/core/tests/test_numpy_extractors.py
+++ b/src/spikeinterface/core/tests/test_numpy_extractors.py
@@ -108,7 +108,7 @@ def test_NumpySorting(setup_NumpyRecording):
 
     # construct back from kwargs keep the same array
     sorting2 = load(sorting.to_dict())
-    assert np.shares_memory(sorting2._cached_spike_vector, sorting._cached_spike_vector)
+    assert np.shares_memory(sorting2._cached_time_ordered["spikes"], sorting._cached_time_ordered["spikes"])
 
 
 def test_SharedMemorySorting():
@@ -124,7 +124,7 @@ def test_SharedMemorySorting():
 
     sorting = SharedMemorySorting.from_sorting(np_sorting)
     # print(sorting)
-    assert sorting._cached_spike_vector is not None
+    assert sorting._cached_time_ordered is not None
 
     # print(sorting.to_spike_vector())
     d = sorting.to_dict()
diff --git a/src/spikeinterface/core/tests/test_unitsselectionsorting.py b/src/spikeinterface/core/tests/test_unitsselectionsorting.py
index 309563df62..53c8f47636 100644
--- a/src/spikeinterface/core/tests/test_unitsselectionsorting.py
+++ b/src/spikeinterface/core/tests/test_unitsselectionsorting.py
@@ -147,7 +147,7 @@ def test_selection_keeps_extra_fields():
     stay with their spike through a selection."""
     wide_dtype = minimum_spike_dtype + [("channel_index", "int64")]
     parent = _make_parent_with_shuffled_ties(dtype=wide_dtype)
-    parent._cached_spike_vector["channel_index"] = np.arange(parent._cached_spike_vector.size)
+    parent._cached_time_ordered["spikes"]["channel_index"] = np.arange(parent._cached_time_ordered["spikes"].size)
 
     unit_ids = ["3", "22", "29"]
     child = UnitsSelectionSorting(parent, unit_ids=unit_ids)
@@ -188,7 +188,7 @@ def _assert_shares_parent_caches(child, parent):
     assert child._is_identity_selection is True
     assert child.to_spike_vector() is parent.to_spike_vector()
     assert child._get_spike_vector_segment_slices() is parent._get_spike_vector_segment_slices()
-    assert child._cached_lexsorted_spike_vector is parent._cached_lexsorted_spike_vector
+    assert child._cached_unit_grouped is parent._cached_unit_grouped
 
 
 @pytest.mark.parametrize("make_child", IDENTITY_SELECTIONS.values(), ids=IDENTITY_SELECTIONS.keys())
@@ -203,13 +203,13 @@ def test_identity_selection_shares_parent_cache(make_child):
     parent_reordered = parent.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False)
     child_reordered = child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False)
     assert child_reordered is parent_reordered
-    assert len(parent._cached_lexsorted_spike_vector) == 1
+    assert parent._cached_unit_grouped is not None
 
     # The child computes the reordered spike vector first (via get_unit_spike_train); the parent must find it.
     parent = _make_parent_with_shuffled_ties()
     child = make_child(parent)
     child.get_unit_spike_train(child.unit_ids[0], segment_index=0)
-    assert len(parent._cached_lexsorted_spike_vector) == 1
+    assert parent._cached_unit_grouped is not None
     parent_reordered = parent.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False)
     child_reordered = child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False)
     assert child_reordered is parent_reordered
@@ -241,11 +241,10 @@ def test_non_identity_selection_does_not_share(unit_ids):
     child = parent.select_units(unit_ids)
     assert child._is_identity_selection is False
     assert child.to_spike_vector() is not parent.to_spike_vector()
-    assert child._cached_lexsorted_spike_vector is not parent._cached_lexsorted_spike_vector
 
     child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT)
-    assert len(child._cached_lexsorted_spike_vector) == 1
-    assert len(parent._cached_lexsorted_spike_vector) == 0
+    assert child._cached_unit_grouped is not None
+    assert parent._cached_unit_grouped is None
 
 
 @pytest.mark.requires_zarr_write
diff --git a/src/spikeinterface/core/unitsselectionsorting.py b/src/spikeinterface/core/unitsselectionsorting.py
index 33fec6fa94..0e8e1a4085 100644
--- a/src/spikeinterface/core/unitsselectionsorting.py
+++ b/src/spikeinterface/core/unitsselectionsorting.py
@@ -34,9 +34,6 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None):
         BaseSorting.__init__(self, sampling_frequency, self._renamed_unit_ids)
 
         self._is_identity_selection = bool(np.array_equal(self._unit_ids, parents_unit_ids))
-        if self._is_identity_selection:
-            # Same units (possibly renamed), same order => we can use the parent's cached spike vector
-            self._cached_lexsorted_spike_vector = parent_sorting._cached_lexsorted_spike_vector
 
         for parent_segment in self._parent_sorting.segments:
             sub_segment = UnitsSelectionSortingSegment(parent_segment, ids_conversion)
@@ -50,26 +47,40 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None):
 
         self._kwargs = dict(parent_sorting=parent_sorting, unit_ids=unit_ids, renamed_unit_ids=renamed_unit_ids)
 
+    @property
+    def _cached_unit_grouped(self):
+        # Same units (possibly renamed), same order => we read and fill the parent's unit-grouped cache
+        if getattr(self, "_is_identity_selection", False):
+            return self._parent_sorting._cached_unit_grouped
+        return self._own_cached_unit_grouped
+
+    @_cached_unit_grouped.setter
+    def _cached_unit_grouped(self, value):
+        if getattr(self, "_is_identity_selection", False):
+            self._parent_sorting._cached_unit_grouped = value
+        else:
+            self._own_cached_unit_grouped = value
+
     def _compute_and_cache_spike_vector(self) -> None:
         from spikeinterface.core.sorting_tools import remap_unit_indices_in_vector
 
-        if self._parent_sorting._cached_spike_vector is None:
+        if self._parent_sorting._cached_time_ordered is None:
             self._parent_sorting._compute_and_cache_spike_vector()
 
-            if self._parent_sorting._cached_spike_vector is None:
+            if self._parent_sorting._cached_time_ordered is None:
                 return
 
         if self._is_identity_selection:
-            self._cached_spike_vector = self._parent_sorting._cached_spike_vector
-            self._cached_spike_vector_segment_slices = self._parent_sorting._get_spike_vector_segment_slices()
+            self._parent_sorting._get_spike_vector_segment_slices()
+            self._cached_time_ordered = self._parent_sorting._cached_time_ordered
             return
 
         spike_vector, _ = remap_unit_indices_in_vector(
-            vector=self._parent_sorting._cached_spike_vector,
+            vector=self._parent_sorting._cached_time_ordered["spikes"],
             all_old_unit_ids=self._parent_sorting.unit_ids,
             all_new_unit_ids=self._unit_ids,
         )
-        self._cached_spike_vector = spike_vector
+        self._cached_time_ordered = {"spikes": spike_vector, "segment_slices": None}
 
     def search_cached_spikes_sorted(
         self,
diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py
index d2a604e38b..daa3f94302 100644
--- a/src/spikeinterface/core/zarrextractors.py
+++ b/src/spikeinterface/core/zarrextractors.py
@@ -595,10 +595,12 @@ def __init__(
         self._lazy_spike_vector = lazy_spike_vector
         self._spikes_group = spikes_group
         self._sample_index_search = None
-        self._cached_spike_vector = spikes
         # pre-populate segment slices so _get_spike_vector_segment_slices() never
         # needs to materialise the full segment_index array
-        self._cached_spike_vector_segment_slices = np.asarray(segment_slices_list, dtype="int64")
+        self._cached_time_ordered = {
+            "spikes": spikes,
+            "segment_slices": np.asarray(segment_slices_list, dtype="int64"),
+        }
 
         for segment_index in range(num_segments):
             soring_segment = SpikeVectorSortingSegment(spikes, segment_index, unit_ids)
@@ -642,7 +644,7 @@ def search_cached_spikes_sorted(
             if "sample_index_chunk_firsts" in self._spikes_group:
                 chunk_firsts = self._spikes_group["sample_index_chunk_firsts"][:]
             self._sample_index_search = ZarrSampleIndexSearch(self._spikes_group["sample_index"], chunk_firsts)
-        start, stop = self._cached_spike_vector_segment_slices[segment_index]
+        start, stop = self._cached_time_ordered["segment_slices"][segment_index]
         return self._sample_index_search.searchsorted(indices, int(start), int(stop))
 
     @staticmethod
diff --git a/src/spikeinterface/curation/remove_excess_spikes.py b/src/spikeinterface/curation/remove_excess_spikes.py
index 04169808f5..a2b8ebb20a 100644
--- a/src/spikeinterface/curation/remove_excess_spikes.py
+++ b/src/spikeinterface/curation/remove_excess_spikes.py
@@ -46,13 +46,13 @@ def __init__(self, sorting: BaseSorting, recording: BaseRecording) -> None:
         self._kwargs = {"sorting": sorting, "recording": recording}
 
     def _compute_and_cache_spike_vector(self) -> None:
-        if self._parent_sorting._cached_spike_vector is None:
+        if self._parent_sorting._cached_time_ordered is None:
             self._parent_sorting._compute_and_cache_spike_vector()
 
-            if self._parent_sorting._cached_spike_vector is None:
+            if self._parent_sorting._cached_time_ordered is None:
                 return
 
-        parent_spike_vector = self._parent_sorting._cached_spike_vector
+        parent_spike_vector = self._parent_sorting._cached_time_ordered["spikes"]
         num_segments = self._parent_sorting.get_num_segments()
 
         list_spike_vectors = []
@@ -64,7 +64,7 @@ def _compute_and_cache_spike_vector(self) -> None:
             list_spike_vectors.append(spike_vector[start:end])
 
         spike_vector = np.concatenate(list_spike_vectors)
-        self._cached_spike_vector = spike_vector
+        self._cached_time_ordered = {"spikes": spike_vector, "segment_slices": None}
 
 
 class RemoveExcessSpikesSortingSegment(BaseSortingSegment):
diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py
index 0d1946d39a..2ac4950a5a 100644
--- a/src/spikeinterface/extractors/nwbextractors.py
+++ b/src/spikeinterface/extractors/nwbextractors.py
@@ -1468,8 +1468,10 @@ def _compute_and_cache_spike_vector(self) -> None:
         spikes["sample_index"] = sample_indices[order]
         spikes["unit_index"] = unit_indices[order]
         # segment_index stays 0 for the single segment.
-        self._cached_spike_vector = spikes
-        self._cached_spike_vector_segment_slices = np.array([[0, num_spikes]], dtype="int64")
+        self._cached_time_ordered = {
+            "spikes": spikes,
+            "segment_slices": np.array([[0, num_spikes]], dtype="int64"),
+        }
 
     def get_last_spike_frame(self, segment_index: int | None = None) -> int:
         # Performance: the generic BaseSorting version builds the full spike vector to take one maximum.
diff --git a/src/spikeinterface/extractors/phykilosortextractors.py b/src/spikeinterface/extractors/phykilosortextractors.py
index 95cc7c9c64..300a9b4662 100644
--- a/src/spikeinterface/extractors/phykilosortextractors.py
+++ b/src/spikeinterface/extractors/phykilosortextractors.py
@@ -268,9 +268,10 @@ def _compute_and_cache_spike_vector(self) -> None:
         # This is useless because phy is always one segment
         # spikes["segment_index"] = 0
 
-        self._cached_spike_vector = spikes
-        self._cached_spike_vector_segment_slices = np.zeros((1, 2), dtype="int64")
-        self._cached_spike_vector_segment_slices[0, 1] = n
+        self._cached_time_ordered = {
+            "spikes": spikes,
+            "segment_slices": np.array([[0, n]], dtype="int64"),
+        }
 
 
 class PhySortingSegment(BaseSortingSegment):