From 58a8ae5ec5e98a54e96c348bc7c7ed00eba0cc7a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 2 Oct 2026 10:14:08 -0600 Subject: [PATCH 1/6] Rename the spike vector caches to say which order they hold --- src/spikeinterface/core/basesorting.py | 76 +++++++++---------- src/spikeinterface/core/generate.py | 37 +++++---- src/spikeinterface/core/numpyextractors.py | 4 +- src/spikeinterface/core/sortingfolder.py | 2 +- .../core/tests/test_basesorting.py | 2 +- .../core/tests/test_numpy_extractors.py | 4 +- .../core/unitsselectionsorting.py | 8 +- src/spikeinterface/core/zarrextractors.py | 4 +- .../curation/remove_excess_spikes.py | 8 +- .../extractors/nwbextractors.py | 4 +- .../extractors/phykilosortextractors.py | 6 +- 11 files changed, 83 insertions(+), 72 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 1e136f2dff..73722dd33c 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -36,13 +36,13 @@ def __init__(self, sampling_frequency: float, unit_ids: list): # caching: # 1. the spike vector : one vector complex dtype - self._cached_spike_vector = None + self._cached_time_ordered_spike_vector = None # 2. the segment slices in the spike vector - self._cached_spike_vector_segment_slices = None + self._cached_time_ordered_segment_slices = None # 3. the indices of spike train inside the spike_vector per segment (list) then unit (dict) - self._cached_spike_vector_to_indices = None + self._cached_time_ordered_indices_by_unit = None # 4. reordering of the spike vector - self._cached_lexsorted_spike_vector = {} + self._cached_unit_grouped_spike_vectors = {} def __repr__(self): return self._repr_header() @@ -205,7 +205,7 @@ def get_unit_spike_train( segment_index = self._check_segment_index(segment_index) lexsort_key = LEXSORT_UNIT_COMPACT - if lexsort_key in self._cached_lexsorted_spike_vector.keys(): + if lexsort_key in self._cached_unit_grouped_spike_vectors.keys(): use_cache = True if use_cache: @@ -631,8 +631,8 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): """ # speed strategy by order - # 1. if _cached_lexsorted_spike_vector has LEXSORT_UNIT_COMPACT then use it and sum - # 2. if _cached_spike_vector not None then use it with np.unique() + # 1. if _cached_unit_grouped_spike_vectors has LEXSORT_UNIT_COMPACT then use it and sum + # 2. if _cached_time_ordered_spike_vector not None then use it with np.unique() # 3. compute spikevector and do np.unique() cache_key = LEXSORT_UNIT_COMPACT @@ -648,9 +648,9 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): keep_mask = slice(None) unit_ids = self.unit_ids - if cache_key in self._cached_lexsorted_spike_vector: + if cache_key in self._cached_unit_grouped_spike_vectors: # case 1 - slices = self._cached_lexsorted_spike_vector[cache_key]["slices"] + slices = self._cached_unit_grouped_spike_vectors[cache_key]["slices"] # end of last segment minus start of first segment num_spikes = slices[keep_mask, -1, 1] - slices[keep_mask, 0, 0] @@ -921,7 +921,7 @@ def precompute_spike_trains(self): This is equivalent to cache lexsort LEXSORT_UNIT_COMPACT. """ cache_key = LEXSORT_UNIT_COMPACT - if cache_key not in self._cached_lexsorted_spike_vector: + if cache_key not in self._cached_unit_grouped_spike_vectors: self.to_reordered_spike_vector(lexsort=cache_key) def _compute_and_cache_spike_vector(self) -> None: @@ -932,7 +932,7 @@ def _compute_and_cache_spike_vector(self) -> None: Function that can be implemented by some children sorting to quickly cache the spike vector without computing it from spike trains (e.g. computing it from a sorting parent). - This function should set the `self._cached_spike_vector`, see for + This function should set the `self._cached_time_ordered_spike_vector`, see for instance the `UnitsSelectionSorting` implementation. """ @@ -978,8 +978,8 @@ def _compute_and_cache_spike_vector(self) -> None: # the spikes are not lexsorted here because the previous loop ensure that the spike vector is constructucted alway the same way. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] - self._cached_spike_vector = spikes - self._cached_spike_vector_segment_slices = segment_slices + self._cached_time_ordered_spike_vector = spikes + self._cached_time_ordered_segment_slices = segment_slices def to_spike_vector( self, @@ -1006,7 +1006,7 @@ def to_spike_vector( This can be convenient for computing spikes position after sorter. This dict can be given by analyzer.get_main_channels(outputs="index", with_dict=False) use_cache : bool, default: True - When True the spikes vector is cached as an attribute of the object (`_cached_spike_vector`). + When True the spikes vector is cached as an attribute of the object (`_cached_time_ordered_spike_vector`). This caching only occurs when main_channel_indices=None. Returns @@ -1018,7 +1018,7 @@ def to_spike_vector( """ - if self._cached_spike_vector is None: + if self._cached_time_ordered_spike_vector is None: self._compute_and_cache_spike_vector() if extremum_channel_inds is not None: @@ -1030,13 +1030,13 @@ def to_spike_vector( main_channel_indices = np.array([extremum_channel_inds[unit_id] for unit_id in self.unit_ids]) if main_channel_indices is None: - spikes = self._cached_spike_vector - elif "channel_index" in self._cached_spike_vector.dtype.names: - spikes = self._cached_spike_vector + spikes = self._cached_time_ordered_spike_vector + elif "channel_index" in self._cached_time_ordered_spike_vector.dtype.names: + spikes = self._cached_time_ordered_spike_vector else: spike_dtype = minimum_spike_dtype + [("channel_index", "int64")] - spikes = np.zeros(self._cached_spike_vector.size, dtype=spike_dtype) - spikes[["sample_index", "unit_index", "segment_index"]] = self._cached_spike_vector + spikes = np.zeros(self._cached_time_ordered_spike_vector.size, dtype=spike_dtype) + spikes[["sample_index", "unit_index", "segment_index"]] = self._cached_time_ordered_spike_vector spikes["channel_index"] = main_channel_indices[spikes["unit_index"]] if not concatenated: @@ -1062,26 +1062,26 @@ def get_spike_vector_to_indices(self): Note that, indices are return using absolute_index=True. """ - if self._cached_spike_vector_to_indices is None: + if self._cached_time_ordered_indices_by_unit is None: from .sorting_tools import spike_vector_to_indices spike_vector = self.to_spike_vector(concatenated=False) - self._cached_spike_vector_to_indices = spike_vector_to_indices( + self._cached_time_ordered_indices_by_unit = spike_vector_to_indices( spike_vector, self.unit_ids, absolute_index=True ) - return self._cached_spike_vector_to_indices + return self._cached_time_ordered_indices_by_unit def _get_spike_vector_segment_slices(self): - if self._cached_spike_vector_segment_slices is None: + if self._cached_time_ordered_segment_slices is None: # compute the, this is needed when spikevector is loaded from format and not computed num_seg = self.get_num_segments() - slices = np.searchsorted(self._cached_spike_vector["segment_index"], np.arange(num_seg + 1)) - self._cached_spike_vector_segment_slices = np.zeros((num_seg, 2), dtype="int64") + slices = np.searchsorted(self._cached_time_ordered_spike_vector["segment_index"], np.arange(num_seg + 1)) + self._cached_time_ordered_segment_slices = np.zeros((num_seg, 2), dtype="int64") for seg_index in range(num_seg): - self._cached_spike_vector_segment_slices[seg_index, 0] = slices[seg_index] - self._cached_spike_vector_segment_slices[seg_index, 1] = slices[seg_index + 1] - return self._cached_spike_vector_segment_slices + self._cached_time_ordered_segment_slices[seg_index, 0] = slices[seg_index] + self._cached_time_ordered_segment_slices[seg_index, 1] = slices[seg_index + 1] + return self._cached_time_ordered_segment_slices def to_reordered_spike_vector( self, @@ -1143,7 +1143,7 @@ def to_reordered_spike_vector( if lexsort not in _ALLOWED_LEXSORTS: raise ValueError(f"`lexsort` must be one of {_ALLOWED_LEXSORTS}; got {lexsort}.") - if lexsort not in self._cached_lexsorted_spike_vector.keys(): + if lexsort not in self._cached_unit_grouped_spike_vectors.keys(): from .sorting_tools import reorder_spike_vector_by_unit_and_segment spikes = self.to_spike_vector() @@ -1162,13 +1162,13 @@ def to_reordered_spike_vector( starts = stops - counts slices = np.stack([starts, stops], axis=-1).astype(np.int64, copy=False) - self._cached_lexsorted_spike_vector[lexsort] = { + self._cached_unit_grouped_spike_vectors[lexsort] = { "ordered_spikes": ordered_spikes, "order": order, "slices": slices, } - cached = self._cached_lexsorted_spike_vector[lexsort] + cached = self._cached_unit_grouped_spike_vectors[lexsort] out = [cached["ordered_spikes"]] if return_order: out.append(cached["order"]) @@ -1191,12 +1191,12 @@ def to_numpy_sorting(self, propagate_cache=True): sorting = NumpySorting.from_sorting(self) if propagate_cache: - if len(self._cached_lexsorted_spike_vector) > 0: - sorting._cached_lexsorted_spike_vector = deepcopy(self._cached_lexsorted_spike_vector) - if self._cached_spike_vector_segment_slices is not None: - sorting._cached_spike_vector_segment_slices = self._cached_spike_vector_segment_slices.copy() - if self._cached_spike_vector_to_indices is not None: - sorting._cached_spike_vector_to_indices = deepcopy(self._cached_spike_vector_to_indices) + if len(self._cached_unit_grouped_spike_vectors) > 0: + sorting._cached_unit_grouped_spike_vectors = deepcopy(self._cached_unit_grouped_spike_vectors) + if self._cached_time_ordered_segment_slices is not None: + sorting._cached_time_ordered_segment_slices = self._cached_time_ordered_segment_slices.copy() + if self._cached_time_ordered_indices_by_unit is not None: + sorting._cached_time_ordered_indices_by_unit = deepcopy(self._cached_time_ordered_indices_by_unit) return sorting diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py index 28ae361fc0..e0cbc9c951 100644 --- a/src/spikeinterface/core/generate.py +++ b/src/spikeinterface/core/generate.py @@ -366,18 +366,20 @@ def __init__( BaseSorting.__init__(self, sampling_frequency, unit_ids) self.parent_unit_ids = sorting.unit_ids - self._cached_spike_vector = sorting.to_spike_vector().copy() + self._cached_time_ordered_spike_vector = sorting.to_spike_vector().copy() self.refractory_period_ms = refractory_period_ms - self.added_spikes_from_existing_mask = np.zeros(len(self._cached_spike_vector), dtype=bool) - self.added_spikes_from_new_mask = np.zeros(len(self._cached_spike_vector), dtype=bool) + self.added_spikes_from_existing_mask = np.zeros(len(self._cached_time_ordered_spike_vector), dtype=bool) + self.added_spikes_from_new_mask = np.zeros(len(self._cached_time_ordered_spike_vector), dtype=bool) if added_spikes_existing_units is not None and len(added_spikes_existing_units) > 0: assert ( added_spikes_existing_units.dtype == minimum_spike_dtype ), "added_spikes_existing_units should be a spike vector" added_unit_indices = np.arange(len(self.parent_unit_ids)) - self._cached_spike_vector = np.concatenate((self._cached_spike_vector, added_spikes_existing_units)) + self._cached_time_ordered_spike_vector = np.concatenate( + (self._cached_time_ordered_spike_vector, added_spikes_existing_units) + ) self.added_spikes_from_existing_mask = np.concatenate( (self.added_spikes_from_existing_mask, np.ones(len(added_spikes_existing_units), dtype=bool)) ) @@ -389,7 +391,9 @@ def __init__( assert ( added_spikes_new_units.dtype == minimum_spike_dtype ), "added_spikes_new_units should be a spike vector" - self._cached_spike_vector = np.concatenate((self._cached_spike_vector, added_spikes_new_units)) + self._cached_time_ordered_spike_vector = np.concatenate( + (self._cached_time_ordered_spike_vector, added_spikes_new_units) + ) self.added_spikes_from_existing_mask = np.concatenate( (self.added_spikes_from_existing_mask, np.zeros(len(added_spikes_new_units), dtype=bool)) ) @@ -397,14 +401,21 @@ def __init__( (self.added_spikes_from_new_mask, np.ones(len(added_spikes_new_units), dtype=bool)) ) - sort_idxs = np.lexsort([self._cached_spike_vector["sample_index"], self._cached_spike_vector["segment_index"]]) - self._cached_spike_vector = self._cached_spike_vector[sort_idxs] + sort_idxs = np.lexsort( + [ + self._cached_time_ordered_spike_vector["sample_index"], + self._cached_time_ordered_spike_vector["segment_index"], + ] + ) + self._cached_time_ordered_spike_vector = self._cached_time_ordered_spike_vector[sort_idxs] self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[sort_idxs] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[sort_idxs] # We need to add the sorting segments for segment_index in range(sorting.get_num_segments()): - segment = SpikeVectorSortingSegment(self._cached_spike_vector, segment_index, unit_ids=self.unit_ids) + segment = SpikeVectorSortingSegment( + self._cached_time_ordered_spike_vector, segment_index, unit_ids=self.unit_ids + ) self.add_sorting_segment(segment) if self.refractory_period_ms is not None: @@ -557,20 +568,20 @@ def clean_refractory_period(self): ## This function will remove the added spikes that will violate RPV, but does not affect the ## spikes in the original sorting. So if some RPV violation are present in this sorting, ## they will be left untouched - unit_indices = np.unique(self._cached_spike_vector["unit_index"]) + unit_indices = np.unique(self._cached_time_ordered_spike_vector["unit_index"]) rpv = int(self.get_sampling_frequency() * self.refractory_period_ms / 1000) to_keep = ~self.added_spikes_from_existing_mask.copy() for segment_index in range(self.get_num_segments()): for unit_ind in unit_indices: (indices,) = np.nonzero( - (self._cached_spike_vector["unit_index"] == unit_ind) - * (self._cached_spike_vector["segment_index"] == segment_index) + (self._cached_time_ordered_spike_vector["unit_index"] == unit_ind) + * (self._cached_time_ordered_spike_vector["segment_index"] == segment_index) ) to_keep[indices[1:]] = np.logical_or( - to_keep[indices[1:]], np.diff(self._cached_spike_vector[indices]["sample_index"]) > rpv + to_keep[indices[1:]], np.diff(self._cached_time_ordered_spike_vector[indices]["sample_index"]) > rpv ) - self._cached_spike_vector = self._cached_spike_vector[to_keep] + self._cached_time_ordered_spike_vector = self._cached_time_ordered_spike_vector[to_keep] self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[to_keep] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[to_keep] diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index b58a98eddb..89f60c91d1 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -288,7 +288,7 @@ def __init__(self, spikes, sampling_frequency, unit_ids): self.add_sorting_segment(SpikeVectorSortingSegment(spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_spike_vector = spikes + self._cached_time_ordered_spike_vector = spikes self._kwargs = dict(spikes=spikes, sampling_frequency=sampling_frequency, unit_ids=unit_ids) @@ -521,7 +521,7 @@ def __init__(self, shm_name, shape, sampling_frequency, unit_ids, dtype=minimum_ self.add_sorting_segment(SpikeVectorSortingSegment(self.shm_spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_spike_vector = self.shm_spikes + self._cached_time_ordered_spike_vector = self.shm_spikes # this is very important for the shm.unlink() # only the main instance need to call it diff --git a/src/spikeinterface/core/sortingfolder.py b/src/spikeinterface/core/sortingfolder.py index 3a068e0384..0ce21aaf9d 100644 --- a/src/spikeinterface/core/sortingfolder.py +++ b/src/spikeinterface/core/sortingfolder.py @@ -53,7 +53,7 @@ def __init__(self, folder_path, mmap_mode: str | None = None): for segment_index in range(num_segments): self.add_sorting_segment(SpikeVectorSortingSegment(self.spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_spike_vector = self.spikes + self._cached_time_ordered_spike_vector = self.spikes load_properties_from_folder(folder_path / "properties", self) load_annotations_from_folder(folder_path, self) diff --git a/src/spikeinterface/core/tests/test_basesorting.py b/src/spikeinterface/core/tests/test_basesorting.py index 25ba1a3c2d..891eafdaf3 100644 --- a/src/spikeinterface/core/tests/test_basesorting.py +++ b/src/spikeinterface/core/tests/test_basesorting.py @@ -97,7 +97,7 @@ def test_BaseSorting(create_cache_folder): spikes = sorting.to_spike_vector() # print(spikes) - assert sorting._cached_spike_vector is not None + assert sorting._cached_time_ordered_spike_vector is not None spikes = sorting.to_spike_vector() ordered_spikes, order, slices = sorting.to_reordered_spike_vector( diff --git a/src/spikeinterface/core/tests/test_numpy_extractors.py b/src/spikeinterface/core/tests/test_numpy_extractors.py index daaf99d72a..14ba2d006d 100644 --- a/src/spikeinterface/core/tests/test_numpy_extractors.py +++ b/src/spikeinterface/core/tests/test_numpy_extractors.py @@ -92,7 +92,7 @@ def test_NumpySorting(setup_NumpyRecording): # construct back from kwargs keep the same array sorting2 = load(sorting.to_dict()) - assert np.shares_memory(sorting2._cached_spike_vector, sorting._cached_spike_vector) + assert np.shares_memory(sorting2._cached_time_ordered_spike_vector, sorting._cached_time_ordered_spike_vector) def test_SharedMemorySorting(): @@ -108,7 +108,7 @@ def test_SharedMemorySorting(): sorting = SharedMemorySorting.from_sorting(np_sorting) # print(sorting) - assert sorting._cached_spike_vector is not None + assert sorting._cached_time_ordered_spike_vector is not None # print(sorting.to_spike_vector()) d = sorting.to_dict() diff --git a/src/spikeinterface/core/unitsselectionsorting.py b/src/spikeinterface/core/unitsselectionsorting.py index fdba9b361a..e93d35564c 100644 --- a/src/spikeinterface/core/unitsselectionsorting.py +++ b/src/spikeinterface/core/unitsselectionsorting.py @@ -48,14 +48,14 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None): def _compute_and_cache_spike_vector(self) -> None: from spikeinterface.core.sorting_tools import remap_unit_indices_in_vector - if self._parent_sorting._cached_spike_vector is None: + if self._parent_sorting._cached_time_ordered_spike_vector is None: self._parent_sorting._compute_and_cache_spike_vector() - if self._parent_sorting._cached_spike_vector is None: + if self._parent_sorting._cached_time_ordered_spike_vector is None: return spike_vector, _ = remap_unit_indices_in_vector( - vector=self._parent_sorting._cached_spike_vector, + vector=self._parent_sorting._cached_time_ordered_spike_vector, all_old_unit_ids=self._parent_sorting.unit_ids, all_new_unit_ids=self._unit_ids, ) @@ -75,7 +75,7 @@ def _compute_and_cache_spike_vector(self) -> None: (spike_vector["unit_index"], spike_vector["sample_index"], spike_vector["segment_index"]) ) spike_vector = spike_vector[sort_indices] - self._cached_spike_vector = spike_vector + self._cached_time_ordered_spike_vector = spike_vector class UnitsSelectionSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py index 78c10df8d4..eeaf9c97eb 100644 --- a/src/spikeinterface/core/zarrextractors.py +++ b/src/spikeinterface/core/zarrextractors.py @@ -525,10 +525,10 @@ def __init__( # In version 0.104.X this was fully lexsorted, but we don't need it anymore because it's only important in the context of SpikeVectorBased extensions in the SortingAnalyzer, which stores its own copy of the Sorting object. This makes the extension data and the spike vector always matching their order. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] - self._cached_spike_vector = spikes + self._cached_time_ordered_spike_vector = spikes # pre-populate segment slices so _get_spike_vector_segment_slices() never # needs to materialise the full segment_index array - self._cached_spike_vector_segment_slices = np.asarray(segment_slices_list, dtype="int64") + self._cached_time_ordered_segment_slices = np.asarray(segment_slices_list, dtype="int64") for segment_index in range(num_segments): soring_segment = SpikeVectorSortingSegment(spikes, segment_index, unit_ids) diff --git a/src/spikeinterface/curation/remove_excess_spikes.py b/src/spikeinterface/curation/remove_excess_spikes.py index 04169808f5..628a5997a3 100644 --- a/src/spikeinterface/curation/remove_excess_spikes.py +++ b/src/spikeinterface/curation/remove_excess_spikes.py @@ -46,13 +46,13 @@ def __init__(self, sorting: BaseSorting, recording: BaseRecording) -> None: self._kwargs = {"sorting": sorting, "recording": recording} def _compute_and_cache_spike_vector(self) -> None: - if self._parent_sorting._cached_spike_vector is None: + if self._parent_sorting._cached_time_ordered_spike_vector is None: self._parent_sorting._compute_and_cache_spike_vector() - if self._parent_sorting._cached_spike_vector is None: + if self._parent_sorting._cached_time_ordered_spike_vector is None: return - parent_spike_vector = self._parent_sorting._cached_spike_vector + parent_spike_vector = self._parent_sorting._cached_time_ordered_spike_vector num_segments = self._parent_sorting.get_num_segments() list_spike_vectors = [] @@ -64,7 +64,7 @@ def _compute_and_cache_spike_vector(self) -> None: list_spike_vectors.append(spike_vector[start:end]) spike_vector = np.concatenate(list_spike_vectors) - self._cached_spike_vector = spike_vector + self._cached_time_ordered_spike_vector = spike_vector class RemoveExcessSpikesSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 5f7f88e996..f666c94c8d 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -1453,8 +1453,8 @@ def _compute_and_cache_spike_vector(self) -> None: spikes["sample_index"] = sample_indices[order] spikes["unit_index"] = unit_indices[order] # segment_index stays 0 for the single segment. - self._cached_spike_vector = spikes - self._cached_spike_vector_segment_slices = np.array([[0, num_spikes]], dtype="int64") + self._cached_time_ordered_spike_vector = spikes + self._cached_time_ordered_segment_slices = np.array([[0, num_spikes]], dtype="int64") @staticmethod def fetch_available_units_tables( diff --git a/src/spikeinterface/extractors/phykilosortextractors.py b/src/spikeinterface/extractors/phykilosortextractors.py index 95cc7c9c64..5ed470f2ef 100644 --- a/src/spikeinterface/extractors/phykilosortextractors.py +++ b/src/spikeinterface/extractors/phykilosortextractors.py @@ -268,9 +268,9 @@ def _compute_and_cache_spike_vector(self) -> None: # This is useless because phy is always one segment # spikes["segment_index"] = 0 - self._cached_spike_vector = spikes - self._cached_spike_vector_segment_slices = np.zeros((1, 2), dtype="int64") - self._cached_spike_vector_segment_slices[0, 1] = n + self._cached_time_ordered_spike_vector = spikes + self._cached_time_ordered_segment_slices = np.zeros((1, 2), dtype="int64") + self._cached_time_ordered_segment_slices[0, 1] = n class PhySortingSegment(BaseSortingSegment): From e8186ada4d15204519430d4c9d7148bba55a9e6e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 6 Oct 2026 11:11:03 -0600 Subject: [PATCH 2/6] Sam first suggestion --- src/spikeinterface/core/basesorting.py | 59 ++++++++----------- .../core/tests/test_basesorting.py | 30 ++-------- 2 files changed, 28 insertions(+), 61 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 73722dd33c..5ee212a36d 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -11,11 +11,6 @@ #: Makes each unit's spiketrain compact in memory (unit, then segment, then sample). LEXSORT_UNIT_COMPACT = ("sample_index", "segment_index", "unit_index") -#: Makes each segment compact, units compact within a segment (segment, then unit, then sample). -LEXSORT_SEGMENT_COMPACT = ("sample_index", "unit_index", "segment_index") - -# The reorderings that `BaseSorting.to_reordered_spike_vector()` can produce. -_ALLOWED_LEXSORTS = (LEXSORT_UNIT_COMPACT, LEXSORT_SEGMENT_COMPACT) class BaseSorting(BaseExtractor): @@ -42,7 +37,7 @@ def __init__(self, sampling_frequency: float, unit_ids: list): # 3. the indices of spike train inside the spike_vector per segment (list) then unit (dict) self._cached_time_ordered_indices_by_unit = None # 4. reordering of the spike vector - self._cached_unit_grouped_spike_vectors = {} + self._cached_unit_grouped_spike_vector = None def __repr__(self): return self._repr_header() @@ -204,13 +199,12 @@ def get_unit_spike_train( segment_index = self._check_segment_index(segment_index) - lexsort_key = LEXSORT_UNIT_COMPACT - if lexsort_key in self._cached_unit_grouped_spike_vectors.keys(): + if self._cached_unit_grouped_spike_vector is not None: use_cache = True if use_cache: ordered_spike_vector, slices = self.to_reordered_spike_vector( - lexsort=lexsort_key, + lexsort=LEXSORT_UNIT_COMPACT, return_order=False, return_slices=True, ) @@ -631,12 +625,10 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): """ # speed strategy by order - # 1. if _cached_unit_grouped_spike_vectors has LEXSORT_UNIT_COMPACT then use it and sum + # 1. if _cached_unit_grouped_spike_vector is not None then use it and sum # 2. if _cached_time_ordered_spike_vector not None then use it with np.unique() # 3. compute spikevector and do np.unique() - cache_key = LEXSORT_UNIT_COMPACT - if unit_ids is not None: assert outputs == "dict", "count_num_spikes_per_unit() with unit_ids not None works only for output='dict'" @@ -648,9 +640,9 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): keep_mask = slice(None) unit_ids = self.unit_ids - if cache_key in self._cached_unit_grouped_spike_vectors: + if self._cached_unit_grouped_spike_vector is not None: # case 1 - slices = self._cached_unit_grouped_spike_vectors[cache_key]["slices"] + slices = self._cached_unit_grouped_spike_vector["slices"] # end of last segment minus start of first segment num_spikes = slices[keep_mask, -1, 1] - slices[keep_mask, 0, 0] @@ -920,9 +912,8 @@ def precompute_spike_trains(self): Pre-computes and caches all spike trains for this sorting. This is equivalent to cache lexsort LEXSORT_UNIT_COMPACT. """ - cache_key = LEXSORT_UNIT_COMPACT - if cache_key not in self._cached_unit_grouped_spike_vectors: - self.to_reordered_spike_vector(lexsort=cache_key) + if self._cached_unit_grouped_spike_vector is None: + self.to_reordered_spike_vector(lexsort=LEXSORT_UNIT_COMPACT) def _compute_and_cache_spike_vector(self) -> None: # @@ -1097,20 +1088,17 @@ def to_reordered_spike_vector( Please note that the lexsort syntax is the **reverse** of natural reading. - Two reorderings are supported: + One reordering is supported: - LEXSORT_UNIT_COMPACT: ("sample_index", "segment_index", "unit_index"). - Makes each unit's spiketrain compact in memory. This is the default, and is what + Makes each unit's spiketrain compact in memory. This is what unit-by-unit computations (e.g. isi violations) want. - - LEXSORT_SEGMENT_COMPACT: ("sample_index", "unit_index", "segment_index"). - Makes each segment compact, with each unit's spiketrain compact within a segment. - Rarely (if ever) used, but might be useful when iterating segment by segment. This operation is internally cached. Parameters ---------- lexsort : tuple, default: LEXSORT_UNIT_COMPACT - The requested sort order. Must be one of the two orderings listed above. + The requested sort order. Must be the ordering listed above. return_order: bool, default: True Return the numpy array needed to sort the spike vector (given the requested sort). return_slices: bool, default: True @@ -1125,14 +1113,14 @@ def to_reordered_spike_vector( Numpy array needed to sort the spike vector given the lexsort. Can be used to sort other external vectors like spike_amplitudes, spike_locations, ... slices : np.array - A 3D array of internal slices for fast access to a compact portion of the reordered spikes. - Depending on the lexsort, a numpy array of size (num_units, num_segments, 2) or (num_segments, num_units, 2). + A 3D array of internal slices for fast access to a compact portion of the reordered spikes, + of size (num_units, num_segments, 2). The last dimension contains the start and end indices of each segment-unit pair. Raises ------ ValueError - If `lexsort` is not one of the two supported orderings. + If `lexsort` is not the supported ordering. """ if lexsort == ("unit_index", "sample_index", "segment_index"): raise ValueError( @@ -1140,21 +1128,20 @@ def to_reordered_spike_vector( "Use `to_spike_vector()` to get the default order." ) - if lexsort not in _ALLOWED_LEXSORTS: - raise ValueError(f"`lexsort` must be one of {_ALLOWED_LEXSORTS}; got {lexsort}.") + if lexsort != LEXSORT_UNIT_COMPACT: + raise ValueError(f"`lexsort` must be {LEXSORT_UNIT_COMPACT}; got {lexsort}.") - if lexsort not in self._cached_unit_grouped_spike_vectors.keys(): + if self._cached_unit_grouped_spike_vector is None: from .sorting_tools import reorder_spike_vector_by_unit_and_segment spikes = self.to_spike_vector() num_units = len(self.unit_ids) num_segments = self.get_num_segments() - unit_major = lexsort == LEXSORT_UNIT_COMPACT - slices_shape = (num_units, num_segments) if unit_major else (num_segments, num_units) + slices_shape = (num_units, num_segments) ordered_spikes, order, counts = reorder_spike_vector_by_unit_and_segment( - spikes, num_units, num_segments, unit_major=unit_major + spikes, num_units, num_segments, unit_major=True ) counts = counts.reshape(slices_shape) @@ -1162,13 +1149,13 @@ def to_reordered_spike_vector( starts = stops - counts slices = np.stack([starts, stops], axis=-1).astype(np.int64, copy=False) - self._cached_unit_grouped_spike_vectors[lexsort] = { + self._cached_unit_grouped_spike_vector = { "ordered_spikes": ordered_spikes, "order": order, "slices": slices, } - cached = self._cached_unit_grouped_spike_vectors[lexsort] + cached = self._cached_unit_grouped_spike_vector out = [cached["ordered_spikes"]] if return_order: out.append(cached["order"]) @@ -1191,8 +1178,8 @@ def to_numpy_sorting(self, propagate_cache=True): sorting = NumpySorting.from_sorting(self) if propagate_cache: - if len(self._cached_unit_grouped_spike_vectors) > 0: - sorting._cached_unit_grouped_spike_vectors = deepcopy(self._cached_unit_grouped_spike_vectors) + if self._cached_unit_grouped_spike_vector is not None: + sorting._cached_unit_grouped_spike_vector = deepcopy(self._cached_unit_grouped_spike_vector) if self._cached_time_ordered_segment_slices is not None: sorting._cached_time_ordered_segment_slices = self._cached_time_ordered_segment_slices.copy() if self._cached_time_ordered_indices_by_unit is not None: diff --git a/src/spikeinterface/core/tests/test_basesorting.py b/src/spikeinterface/core/tests/test_basesorting.py index 891eafdaf3..78e97880a0 100644 --- a/src/spikeinterface/core/tests/test_basesorting.py +++ b/src/spikeinterface/core/tests/test_basesorting.py @@ -107,17 +107,6 @@ def test_BaseSorting(create_cache_folder): ) assert np.array_equal(spikes[order], ordered_spikes) - ordered_spikes, order, slices = sorting.to_reordered_spike_vector( - lexsort=( - "sample_index", - "unit_index", - "segment_index", - ), - return_order=True, - return_slices=True, - ) - assert np.array_equal(spikes[order], ordered_spikes) - num_spikes_per_unit = sorting.count_num_spikes_per_unit(outputs="dict") num_spikes_per_unit = sorting.count_num_spikes_per_unit(outputs="array") total_spikes = sorting.count_total_num_spikes() @@ -182,11 +171,7 @@ def _make_sorting_with_shuffled_ties(num_units, num_segments, seed=42): @pytest.mark.parametrize("use_numba", [True, False], ids=["numba", "numpy"]) -@pytest.mark.parametrize( - "lexsort", - [("sample_index", "segment_index", "unit_index"), ("sample_index", "unit_index", "segment_index")], -) -def test_to_reordered_spike_vector(lexsort, use_numba, monkeypatch): +def test_to_reordered_spike_vector(use_numba, monkeypatch): """`to_reordered_spike_vector` should group spikes by (unit, segment) without disturbing them.""" if use_numba and importlib.util.find_spec("numba") is None: pytest.skip("numba not installed") @@ -202,17 +187,12 @@ def test_to_reordered_spike_vector(lexsort, use_numba, monkeypatch): ) ordered_spikes, order, slices = sorting.to_reordered_spike_vector( - lexsort=lexsort, return_order=True, return_slices=True + lexsort=LEXSORT_UNIT_COMPACT, return_order=True, return_slices=True ) - # The buckets, in the order the requested lexsort puts them in. - unit_major = lexsort == ("sample_index", "segment_index", "unit_index") - if unit_major: - assert slices.shape == (num_units, num_segments, 2) - groups = [(u, s) for u in range(num_units) for s in range(num_segments)] - else: - assert slices.shape == (num_segments, num_units, 2) - groups = [(u, s) for s in range(num_segments) for u in range(num_units)] + # The buckets, unit by unit and segment by segment within each unit. + assert slices.shape == (num_units, num_segments, 2) + groups = [(u, s) for u in range(num_units) for s in range(num_segments)] masks = [(spikes["unit_index"] == u) & (spikes["segment_index"] == s) for u, s in groups] assert np.array_equal(ordered_spikes, np.concatenate([spikes[mask] for mask in masks])) From 58ac2f604f02c17661cf315b63a6d237a0e3b864 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 6 Oct 2026 12:35:30 -0600 Subject: [PATCH 3/6] samuel second request --- src/spikeinterface/core/basesorting.py | 42 ++++++++++--------- src/spikeinterface/core/generate.py | 35 +++++++--------- src/spikeinterface/core/numpyextractors.py | 4 +- src/spikeinterface/core/sortingfolder.py | 2 +- .../core/tests/test_numpy_extractors.py | 4 +- .../core/unitsselectionsorting.py | 4 +- src/spikeinterface/core/zarrextractors.py | 6 ++- .../curation/remove_excess_spikes.py | 4 +- .../extractors/nwbextractors.py | 6 ++- .../extractors/phykilosortextractors.py | 7 ++-- 10 files changed, 59 insertions(+), 55 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 5ee212a36d..c25932b690 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -30,13 +30,12 @@ def __init__(self, sampling_frequency: float, unit_ids: list): self._sorting_info = None # caching: - # 1. the spike vector : one vector complex dtype + # 1. the spike vector : one vector complex dtype, with its segment slices (computed lazily if None) + # {"spikes": spike vector, "segment_slices": None or (num_segments, 2) array} self._cached_time_ordered_spike_vector = None - # 2. the segment slices in the spike vector - self._cached_time_ordered_segment_slices = None - # 3. the indices of spike train inside the spike_vector per segment (list) then unit (dict) + # 2. the indices of spike train inside the spike_vector per segment (list) then unit (dict) self._cached_time_ordered_indices_by_unit = None - # 4. reordering of the spike vector + # 3. reordering of the spike vector self._cached_unit_grouped_spike_vector = None def __repr__(self): @@ -969,8 +968,7 @@ def _compute_and_cache_spike_vector(self) -> None: # the spikes are not lexsorted here because the previous loop ensure that the spike vector is constructucted alway the same way. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] - self._cached_time_ordered_spike_vector = spikes - self._cached_time_ordered_segment_slices = segment_slices + self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": segment_slices} def to_spike_vector( self, @@ -1011,6 +1009,7 @@ def to_spike_vector( if self._cached_time_ordered_spike_vector is None: self._compute_and_cache_spike_vector() + cached_spikes = self._cached_time_ordered_spike_vector["spikes"] if extremum_channel_inds is not None: warnings.warn( @@ -1021,13 +1020,13 @@ def to_spike_vector( main_channel_indices = np.array([extremum_channel_inds[unit_id] for unit_id in self.unit_ids]) if main_channel_indices is None: - spikes = self._cached_time_ordered_spike_vector - elif "channel_index" in self._cached_time_ordered_spike_vector.dtype.names: - spikes = self._cached_time_ordered_spike_vector + spikes = cached_spikes + elif "channel_index" in cached_spikes.dtype.names: + spikes = cached_spikes else: spike_dtype = minimum_spike_dtype + [("channel_index", "int64")] - spikes = np.zeros(self._cached_time_ordered_spike_vector.size, dtype=spike_dtype) - spikes[["sample_index", "unit_index", "segment_index"]] = self._cached_time_ordered_spike_vector + spikes = np.zeros(cached_spikes.size, dtype=spike_dtype) + spikes[["sample_index", "unit_index", "segment_index"]] = cached_spikes spikes["channel_index"] = main_channel_indices[spikes["unit_index"]] if not concatenated: @@ -1064,15 +1063,16 @@ def get_spike_vector_to_indices(self): return self._cached_time_ordered_indices_by_unit def _get_spike_vector_segment_slices(self): - if self._cached_time_ordered_segment_slices is None: + cache = self._cached_time_ordered_spike_vector + if cache["segment_slices"] is None: # compute the, this is needed when spikevector is loaded from format and not computed num_seg = self.get_num_segments() - slices = np.searchsorted(self._cached_time_ordered_spike_vector["segment_index"], np.arange(num_seg + 1)) - self._cached_time_ordered_segment_slices = np.zeros((num_seg, 2), dtype="int64") + slices = np.searchsorted(cache["spikes"]["segment_index"], np.arange(num_seg + 1)) + cache["segment_slices"] = np.zeros((num_seg, 2), dtype="int64") for seg_index in range(num_seg): - self._cached_time_ordered_segment_slices[seg_index, 0] = slices[seg_index] - self._cached_time_ordered_segment_slices[seg_index, 1] = slices[seg_index + 1] - return self._cached_time_ordered_segment_slices + cache["segment_slices"][seg_index, 0] = slices[seg_index] + cache["segment_slices"][seg_index, 1] = slices[seg_index + 1] + return cache["segment_slices"] def to_reordered_spike_vector( self, @@ -1180,8 +1180,10 @@ def to_numpy_sorting(self, propagate_cache=True): if propagate_cache: if self._cached_unit_grouped_spike_vector is not None: sorting._cached_unit_grouped_spike_vector = deepcopy(self._cached_unit_grouped_spike_vector) - if self._cached_time_ordered_segment_slices is not None: - sorting._cached_time_ordered_segment_slices = self._cached_time_ordered_segment_slices.copy() + if self._cached_time_ordered_spike_vector is not None: + segment_slices = self._cached_time_ordered_spike_vector["segment_slices"] + if segment_slices is not None: + sorting._cached_time_ordered_spike_vector["segment_slices"] = segment_slices.copy() if self._cached_time_ordered_indices_by_unit is not None: sorting._cached_time_ordered_indices_by_unit = deepcopy(self._cached_time_ordered_indices_by_unit) diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py index e0cbc9c951..07dfade9a2 100644 --- a/src/spikeinterface/core/generate.py +++ b/src/spikeinterface/core/generate.py @@ -366,20 +366,18 @@ def __init__( BaseSorting.__init__(self, sampling_frequency, unit_ids) self.parent_unit_ids = sorting.unit_ids - self._cached_time_ordered_spike_vector = sorting.to_spike_vector().copy() + spikes = sorting.to_spike_vector().copy() self.refractory_period_ms = refractory_period_ms - self.added_spikes_from_existing_mask = np.zeros(len(self._cached_time_ordered_spike_vector), dtype=bool) - self.added_spikes_from_new_mask = np.zeros(len(self._cached_time_ordered_spike_vector), dtype=bool) + self.added_spikes_from_existing_mask = np.zeros(len(spikes), dtype=bool) + self.added_spikes_from_new_mask = np.zeros(len(spikes), dtype=bool) if added_spikes_existing_units is not None and len(added_spikes_existing_units) > 0: assert ( added_spikes_existing_units.dtype == minimum_spike_dtype ), "added_spikes_existing_units should be a spike vector" added_unit_indices = np.arange(len(self.parent_unit_ids)) - self._cached_time_ordered_spike_vector = np.concatenate( - (self._cached_time_ordered_spike_vector, added_spikes_existing_units) - ) + spikes = np.concatenate((spikes, added_spikes_existing_units)) self.added_spikes_from_existing_mask = np.concatenate( (self.added_spikes_from_existing_mask, np.ones(len(added_spikes_existing_units), dtype=bool)) ) @@ -391,9 +389,7 @@ def __init__( assert ( added_spikes_new_units.dtype == minimum_spike_dtype ), "added_spikes_new_units should be a spike vector" - self._cached_time_ordered_spike_vector = np.concatenate( - (self._cached_time_ordered_spike_vector, added_spikes_new_units) - ) + spikes = np.concatenate((spikes, added_spikes_new_units)) self.added_spikes_from_existing_mask = np.concatenate( (self.added_spikes_from_existing_mask, np.zeros(len(added_spikes_new_units), dtype=bool)) ) @@ -403,19 +399,18 @@ def __init__( sort_idxs = np.lexsort( [ - self._cached_time_ordered_spike_vector["sample_index"], - self._cached_time_ordered_spike_vector["segment_index"], + spikes["sample_index"], + spikes["segment_index"], ] ) - self._cached_time_ordered_spike_vector = self._cached_time_ordered_spike_vector[sort_idxs] + spikes = spikes[sort_idxs] + self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": None} self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[sort_idxs] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[sort_idxs] # We need to add the sorting segments for segment_index in range(sorting.get_num_segments()): - segment = SpikeVectorSortingSegment( - self._cached_time_ordered_spike_vector, segment_index, unit_ids=self.unit_ids - ) + segment = SpikeVectorSortingSegment(spikes, segment_index, unit_ids=self.unit_ids) self.add_sorting_segment(segment) if self.refractory_period_ms is not None: @@ -568,20 +563,20 @@ def clean_refractory_period(self): ## This function will remove the added spikes that will violate RPV, but does not affect the ## spikes in the original sorting. So if some RPV violation are present in this sorting, ## they will be left untouched - unit_indices = np.unique(self._cached_time_ordered_spike_vector["unit_index"]) + spikes = self._cached_time_ordered_spike_vector["spikes"] + unit_indices = np.unique(spikes["unit_index"]) rpv = int(self.get_sampling_frequency() * self.refractory_period_ms / 1000) to_keep = ~self.added_spikes_from_existing_mask.copy() for segment_index in range(self.get_num_segments()): for unit_ind in unit_indices: (indices,) = np.nonzero( - (self._cached_time_ordered_spike_vector["unit_index"] == unit_ind) - * (self._cached_time_ordered_spike_vector["segment_index"] == segment_index) + (spikes["unit_index"] == unit_ind) * (spikes["segment_index"] == segment_index) ) to_keep[indices[1:]] = np.logical_or( - to_keep[indices[1:]], np.diff(self._cached_time_ordered_spike_vector[indices]["sample_index"]) > rpv + to_keep[indices[1:]], np.diff(spikes[indices]["sample_index"]) > rpv ) - self._cached_time_ordered_spike_vector = self._cached_time_ordered_spike_vector[to_keep] + self._cached_time_ordered_spike_vector = {"spikes": spikes[to_keep], "segment_slices": None} self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[to_keep] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[to_keep] diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index 89f60c91d1..eb4f674c78 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -288,7 +288,7 @@ def __init__(self, spikes, sampling_frequency, unit_ids): self.add_sorting_segment(SpikeVectorSortingSegment(spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = spikes + self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": None} self._kwargs = dict(spikes=spikes, sampling_frequency=sampling_frequency, unit_ids=unit_ids) @@ -521,7 +521,7 @@ def __init__(self, shm_name, shape, sampling_frequency, unit_ids, dtype=minimum_ self.add_sorting_segment(SpikeVectorSortingSegment(self.shm_spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = self.shm_spikes + self._cached_time_ordered_spike_vector = {"spikes": self.shm_spikes, "segment_slices": None} # this is very important for the shm.unlink() # only the main instance need to call it diff --git a/src/spikeinterface/core/sortingfolder.py b/src/spikeinterface/core/sortingfolder.py index 0ce21aaf9d..901924fd0c 100644 --- a/src/spikeinterface/core/sortingfolder.py +++ b/src/spikeinterface/core/sortingfolder.py @@ -53,7 +53,7 @@ def __init__(self, folder_path, mmap_mode: str | None = None): for segment_index in range(num_segments): self.add_sorting_segment(SpikeVectorSortingSegment(self.spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = self.spikes + self._cached_time_ordered_spike_vector = {"spikes": self.spikes, "segment_slices": None} load_properties_from_folder(folder_path / "properties", self) load_annotations_from_folder(folder_path, self) diff --git a/src/spikeinterface/core/tests/test_numpy_extractors.py b/src/spikeinterface/core/tests/test_numpy_extractors.py index 14ba2d006d..4cd0ce7786 100644 --- a/src/spikeinterface/core/tests/test_numpy_extractors.py +++ b/src/spikeinterface/core/tests/test_numpy_extractors.py @@ -92,7 +92,9 @@ def test_NumpySorting(setup_NumpyRecording): # construct back from kwargs keep the same array sorting2 = load(sorting.to_dict()) - assert np.shares_memory(sorting2._cached_time_ordered_spike_vector, sorting._cached_time_ordered_spike_vector) + assert np.shares_memory( + sorting2._cached_time_ordered_spike_vector["spikes"], sorting._cached_time_ordered_spike_vector["spikes"] + ) def test_SharedMemorySorting(): diff --git a/src/spikeinterface/core/unitsselectionsorting.py b/src/spikeinterface/core/unitsselectionsorting.py index e93d35564c..d82e4f1adf 100644 --- a/src/spikeinterface/core/unitsselectionsorting.py +++ b/src/spikeinterface/core/unitsselectionsorting.py @@ -55,7 +55,7 @@ def _compute_and_cache_spike_vector(self) -> None: return spike_vector, _ = remap_unit_indices_in_vector( - vector=self._parent_sorting._cached_time_ordered_spike_vector, + vector=self._parent_sorting._cached_time_ordered_spike_vector["spikes"], all_old_unit_ids=self._parent_sorting.unit_ids, all_new_unit_ids=self._unit_ids, ) @@ -75,7 +75,7 @@ def _compute_and_cache_spike_vector(self) -> None: (spike_vector["unit_index"], spike_vector["sample_index"], spike_vector["segment_index"]) ) spike_vector = spike_vector[sort_indices] - self._cached_time_ordered_spike_vector = spike_vector + self._cached_time_ordered_spike_vector = {"spikes": spike_vector, "segment_slices": None} class UnitsSelectionSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py index eeaf9c97eb..ba450cf4a1 100644 --- a/src/spikeinterface/core/zarrextractors.py +++ b/src/spikeinterface/core/zarrextractors.py @@ -525,10 +525,12 @@ def __init__( # In version 0.104.X this was fully lexsorted, but we don't need it anymore because it's only important in the context of SpikeVectorBased extensions in the SortingAnalyzer, which stores its own copy of the Sorting object. This makes the extension data and the spike vector always matching their order. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] - self._cached_time_ordered_spike_vector = spikes # pre-populate segment slices so _get_spike_vector_segment_slices() never # needs to materialise the full segment_index array - self._cached_time_ordered_segment_slices = np.asarray(segment_slices_list, dtype="int64") + self._cached_time_ordered_spike_vector = { + "spikes": spikes, + "segment_slices": np.asarray(segment_slices_list, dtype="int64"), + } for segment_index in range(num_segments): soring_segment = SpikeVectorSortingSegment(spikes, segment_index, unit_ids) diff --git a/src/spikeinterface/curation/remove_excess_spikes.py b/src/spikeinterface/curation/remove_excess_spikes.py index 628a5997a3..4f18a9d0c6 100644 --- a/src/spikeinterface/curation/remove_excess_spikes.py +++ b/src/spikeinterface/curation/remove_excess_spikes.py @@ -52,7 +52,7 @@ def _compute_and_cache_spike_vector(self) -> None: if self._parent_sorting._cached_time_ordered_spike_vector is None: return - parent_spike_vector = self._parent_sorting._cached_time_ordered_spike_vector + parent_spike_vector = self._parent_sorting._cached_time_ordered_spike_vector["spikes"] num_segments = self._parent_sorting.get_num_segments() list_spike_vectors = [] @@ -64,7 +64,7 @@ def _compute_and_cache_spike_vector(self) -> None: list_spike_vectors.append(spike_vector[start:end]) spike_vector = np.concatenate(list_spike_vectors) - self._cached_time_ordered_spike_vector = spike_vector + self._cached_time_ordered_spike_vector = {"spikes": spike_vector, "segment_slices": None} class RemoveExcessSpikesSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index f666c94c8d..863616fef7 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -1453,8 +1453,10 @@ def _compute_and_cache_spike_vector(self) -> None: spikes["sample_index"] = sample_indices[order] spikes["unit_index"] = unit_indices[order] # segment_index stays 0 for the single segment. - self._cached_time_ordered_spike_vector = spikes - self._cached_time_ordered_segment_slices = np.array([[0, num_spikes]], dtype="int64") + self._cached_time_ordered_spike_vector = { + "spikes": spikes, + "segment_slices": np.array([[0, num_spikes]], dtype="int64"), + } @staticmethod def fetch_available_units_tables( diff --git a/src/spikeinterface/extractors/phykilosortextractors.py b/src/spikeinterface/extractors/phykilosortextractors.py index 5ed470f2ef..b9ec8c8b44 100644 --- a/src/spikeinterface/extractors/phykilosortextractors.py +++ b/src/spikeinterface/extractors/phykilosortextractors.py @@ -268,9 +268,10 @@ def _compute_and_cache_spike_vector(self) -> None: # This is useless because phy is always one segment # spikes["segment_index"] = 0 - self._cached_time_ordered_spike_vector = spikes - self._cached_time_ordered_segment_slices = np.zeros((1, 2), dtype="int64") - self._cached_time_ordered_segment_slices[0, 1] = n + self._cached_time_ordered_spike_vector = { + "spikes": spikes, + "segment_slices": np.array([[0, n]], dtype="int64"), + } class PhySortingSegment(BaseSortingSegment): From 55e2e52f5d3bd43265e76bc6355a461933ca3f76 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:38:19 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/spikeinterface/core/generate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py index 07dfade9a2..bee2a2e3b7 100644 --- a/src/spikeinterface/core/generate.py +++ b/src/spikeinterface/core/generate.py @@ -569,9 +569,7 @@ def clean_refractory_period(self): to_keep = ~self.added_spikes_from_existing_mask.copy() for segment_index in range(self.get_num_segments()): for unit_ind in unit_indices: - (indices,) = np.nonzero( - (spikes["unit_index"] == unit_ind) * (spikes["segment_index"] == segment_index) - ) + (indices,) = np.nonzero((spikes["unit_index"] == unit_ind) * (spikes["segment_index"] == segment_index)) to_keep[indices[1:]] = np.logical_or( to_keep[indices[1:]], np.diff(spikes[indices]["sample_index"]) > rpv ) From 8ec567c1a0aa54ff0626c1685fa882c1a56cbca0 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 12 Oct 2026 12:31:59 -0600 Subject: [PATCH 5/6] Name the spikes caches by order only (Graham review) --- src/spikeinterface/core/basesorting.py | 44 +++++++++---------- src/spikeinterface/core/generate.py | 6 +-- src/spikeinterface/core/numpyextractors.py | 4 +- src/spikeinterface/core/sortingfolder.py | 2 +- .../core/tests/test_basesorting.py | 2 +- .../core/tests/test_numpy_extractors.py | 6 +-- .../core/unitsselectionsorting.py | 8 ++-- src/spikeinterface/core/zarrextractors.py | 2 +- .../curation/remove_excess_spikes.py | 8 ++-- .../extractors/nwbextractors.py | 2 +- .../extractors/phykilosortextractors.py | 2 +- 11 files changed, 42 insertions(+), 44 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index c25932b690..928869d8d8 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -32,11 +32,11 @@ def __init__(self, sampling_frequency: float, unit_ids: list): # caching: # 1. the spike vector : one vector complex dtype, with its segment slices (computed lazily if None) # {"spikes": spike vector, "segment_slices": None or (num_segments, 2) array} - self._cached_time_ordered_spike_vector = None + self._cached_time_ordered = None # 2. the indices of spike train inside the spike_vector per segment (list) then unit (dict) self._cached_time_ordered_indices_by_unit = None # 3. reordering of the spike vector - self._cached_unit_grouped_spike_vector = None + self._cached_unit_grouped = None def __repr__(self): return self._repr_header() @@ -198,7 +198,7 @@ def get_unit_spike_train( segment_index = self._check_segment_index(segment_index) - if self._cached_unit_grouped_spike_vector is not None: + if self._cached_unit_grouped is not None: use_cache = True if use_cache: @@ -624,8 +624,8 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): """ # speed strategy by order - # 1. if _cached_unit_grouped_spike_vector is not None then use it and sum - # 2. if _cached_time_ordered_spike_vector not None then use it with np.unique() + # 1. if _cached_unit_grouped is not None then use it and sum + # 2. if _cached_time_ordered not None then use it with np.unique() # 3. compute spikevector and do np.unique() if unit_ids is not None: @@ -639,9 +639,9 @@ def count_num_spikes_per_unit(self, outputs="dict", unit_ids=None): keep_mask = slice(None) unit_ids = self.unit_ids - if self._cached_unit_grouped_spike_vector is not None: + if self._cached_unit_grouped is not None: # case 1 - slices = self._cached_unit_grouped_spike_vector["slices"] + slices = self._cached_unit_grouped["slices"] # end of last segment minus start of first segment num_spikes = slices[keep_mask, -1, 1] - slices[keep_mask, 0, 0] @@ -911,7 +911,7 @@ def precompute_spike_trains(self): Pre-computes and caches all spike trains for this sorting. This is equivalent to cache lexsort LEXSORT_UNIT_COMPACT. """ - if self._cached_unit_grouped_spike_vector is None: + if self._cached_unit_grouped is None: self.to_reordered_spike_vector(lexsort=LEXSORT_UNIT_COMPACT) def _compute_and_cache_spike_vector(self) -> None: @@ -922,7 +922,7 @@ def _compute_and_cache_spike_vector(self) -> None: Function that can be implemented by some children sorting to quickly cache the spike vector without computing it from spike trains (e.g. computing it from a sorting parent). - This function should set the `self._cached_time_ordered_spike_vector`, see for + This function should set the `self._cached_time_ordered`, see for instance the `UnitsSelectionSorting` implementation. """ @@ -968,7 +968,7 @@ def _compute_and_cache_spike_vector(self) -> None: # the spikes are not lexsorted here because the previous loop ensure that the spike vector is constructucted alway the same way. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] - self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": segment_slices} + self._cached_time_ordered = {"spikes": spikes, "segment_slices": segment_slices} def to_spike_vector( self, @@ -995,7 +995,7 @@ def to_spike_vector( This can be convenient for computing spikes position after sorter. This dict can be given by analyzer.get_main_channels(outputs="index", with_dict=False) use_cache : bool, default: True - When True the spikes vector is cached as an attribute of the object (`_cached_time_ordered_spike_vector`). + When True the spikes vector is cached as an attribute of the object (`_cached_time_ordered`). This caching only occurs when main_channel_indices=None. Returns @@ -1007,9 +1007,9 @@ def to_spike_vector( """ - if self._cached_time_ordered_spike_vector is None: + if self._cached_time_ordered is None: self._compute_and_cache_spike_vector() - cached_spikes = self._cached_time_ordered_spike_vector["spikes"] + cached_spikes = self._cached_time_ordered["spikes"] if extremum_channel_inds is not None: warnings.warn( @@ -1063,7 +1063,7 @@ def get_spike_vector_to_indices(self): return self._cached_time_ordered_indices_by_unit def _get_spike_vector_segment_slices(self): - cache = self._cached_time_ordered_spike_vector + cache = self._cached_time_ordered if cache["segment_slices"] is None: # compute the, this is needed when spikevector is loaded from format and not computed num_seg = self.get_num_segments() @@ -1131,7 +1131,7 @@ def to_reordered_spike_vector( if lexsort != LEXSORT_UNIT_COMPACT: raise ValueError(f"`lexsort` must be {LEXSORT_UNIT_COMPACT}; got {lexsort}.") - if self._cached_unit_grouped_spike_vector is None: + if self._cached_unit_grouped is None: from .sorting_tools import reorder_spike_vector_by_unit_and_segment spikes = self.to_spike_vector() @@ -1149,13 +1149,13 @@ def to_reordered_spike_vector( starts = stops - counts slices = np.stack([starts, stops], axis=-1).astype(np.int64, copy=False) - self._cached_unit_grouped_spike_vector = { + self._cached_unit_grouped = { "ordered_spikes": ordered_spikes, "order": order, "slices": slices, } - cached = self._cached_unit_grouped_spike_vector + cached = self._cached_unit_grouped out = [cached["ordered_spikes"]] if return_order: out.append(cached["order"]) @@ -1178,12 +1178,12 @@ def to_numpy_sorting(self, propagate_cache=True): sorting = NumpySorting.from_sorting(self) if propagate_cache: - if self._cached_unit_grouped_spike_vector is not None: - sorting._cached_unit_grouped_spike_vector = deepcopy(self._cached_unit_grouped_spike_vector) - if self._cached_time_ordered_spike_vector is not None: - segment_slices = self._cached_time_ordered_spike_vector["segment_slices"] + if self._cached_unit_grouped is not None: + sorting._cached_unit_grouped = deepcopy(self._cached_unit_grouped) + if self._cached_time_ordered is not None: + segment_slices = self._cached_time_ordered["segment_slices"] if segment_slices is not None: - sorting._cached_time_ordered_spike_vector["segment_slices"] = segment_slices.copy() + sorting._cached_time_ordered["segment_slices"] = segment_slices.copy() if self._cached_time_ordered_indices_by_unit is not None: sorting._cached_time_ordered_indices_by_unit = deepcopy(self._cached_time_ordered_indices_by_unit) diff --git a/src/spikeinterface/core/generate.py b/src/spikeinterface/core/generate.py index bee2a2e3b7..c9591e2e88 100644 --- a/src/spikeinterface/core/generate.py +++ b/src/spikeinterface/core/generate.py @@ -404,7 +404,7 @@ def __init__( ] ) spikes = spikes[sort_idxs] - self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": None} + self._cached_time_ordered = {"spikes": spikes, "segment_slices": None} self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[sort_idxs] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[sort_idxs] @@ -563,7 +563,7 @@ def clean_refractory_period(self): ## This function will remove the added spikes that will violate RPV, but does not affect the ## spikes in the original sorting. So if some RPV violation are present in this sorting, ## they will be left untouched - spikes = self._cached_time_ordered_spike_vector["spikes"] + spikes = self._cached_time_ordered["spikes"] unit_indices = np.unique(spikes["unit_index"]) rpv = int(self.get_sampling_frequency() * self.refractory_period_ms / 1000) to_keep = ~self.added_spikes_from_existing_mask.copy() @@ -574,7 +574,7 @@ def clean_refractory_period(self): to_keep[indices[1:]], np.diff(spikes[indices]["sample_index"]) > rpv ) - self._cached_time_ordered_spike_vector = {"spikes": spikes[to_keep], "segment_slices": None} + self._cached_time_ordered = {"spikes": spikes[to_keep], "segment_slices": None} self.added_spikes_from_existing_mask = self.added_spikes_from_existing_mask[to_keep] self.added_spikes_from_new_mask = self.added_spikes_from_new_mask[to_keep] diff --git a/src/spikeinterface/core/numpyextractors.py b/src/spikeinterface/core/numpyextractors.py index eb4f674c78..5f5c5aaa29 100644 --- a/src/spikeinterface/core/numpyextractors.py +++ b/src/spikeinterface/core/numpyextractors.py @@ -288,7 +288,7 @@ def __init__(self, spikes, sampling_frequency, unit_ids): self.add_sorting_segment(SpikeVectorSortingSegment(spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = {"spikes": spikes, "segment_slices": None} + self._cached_time_ordered = {"spikes": spikes, "segment_slices": None} self._kwargs = dict(spikes=spikes, sampling_frequency=sampling_frequency, unit_ids=unit_ids) @@ -521,7 +521,7 @@ def __init__(self, shm_name, shape, sampling_frequency, unit_ids, dtype=minimum_ self.add_sorting_segment(SpikeVectorSortingSegment(self.shm_spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = {"spikes": self.shm_spikes, "segment_slices": None} + self._cached_time_ordered = {"spikes": self.shm_spikes, "segment_slices": None} # this is very important for the shm.unlink() # only the main instance need to call it diff --git a/src/spikeinterface/core/sortingfolder.py b/src/spikeinterface/core/sortingfolder.py index 901924fd0c..d92880d001 100644 --- a/src/spikeinterface/core/sortingfolder.py +++ b/src/spikeinterface/core/sortingfolder.py @@ -53,7 +53,7 @@ def __init__(self, folder_path, mmap_mode: str | None = None): for segment_index in range(num_segments): self.add_sorting_segment(SpikeVectorSortingSegment(self.spikes, segment_index, unit_ids)) # important trick : the cache is already spikes vector - self._cached_time_ordered_spike_vector = {"spikes": self.spikes, "segment_slices": None} + self._cached_time_ordered = {"spikes": self.spikes, "segment_slices": None} load_properties_from_folder(folder_path / "properties", self) load_annotations_from_folder(folder_path, self) diff --git a/src/spikeinterface/core/tests/test_basesorting.py b/src/spikeinterface/core/tests/test_basesorting.py index 78e97880a0..f23bd19d48 100644 --- a/src/spikeinterface/core/tests/test_basesorting.py +++ b/src/spikeinterface/core/tests/test_basesorting.py @@ -97,7 +97,7 @@ def test_BaseSorting(create_cache_folder): spikes = sorting.to_spike_vector() # print(spikes) - assert sorting._cached_time_ordered_spike_vector is not None + assert sorting._cached_time_ordered is not None spikes = sorting.to_spike_vector() ordered_spikes, order, slices = sorting.to_reordered_spike_vector( diff --git a/src/spikeinterface/core/tests/test_numpy_extractors.py b/src/spikeinterface/core/tests/test_numpy_extractors.py index 4cd0ce7786..d3af6d67d2 100644 --- a/src/spikeinterface/core/tests/test_numpy_extractors.py +++ b/src/spikeinterface/core/tests/test_numpy_extractors.py @@ -92,9 +92,7 @@ def test_NumpySorting(setup_NumpyRecording): # construct back from kwargs keep the same array sorting2 = load(sorting.to_dict()) - assert np.shares_memory( - sorting2._cached_time_ordered_spike_vector["spikes"], sorting._cached_time_ordered_spike_vector["spikes"] - ) + assert np.shares_memory(sorting2._cached_time_ordered["spikes"], sorting._cached_time_ordered["spikes"]) def test_SharedMemorySorting(): @@ -110,7 +108,7 @@ def test_SharedMemorySorting(): sorting = SharedMemorySorting.from_sorting(np_sorting) # print(sorting) - assert sorting._cached_time_ordered_spike_vector is not None + assert sorting._cached_time_ordered is not None # print(sorting.to_spike_vector()) d = sorting.to_dict() diff --git a/src/spikeinterface/core/unitsselectionsorting.py b/src/spikeinterface/core/unitsselectionsorting.py index d82e4f1adf..d1c12e1c0f 100644 --- a/src/spikeinterface/core/unitsselectionsorting.py +++ b/src/spikeinterface/core/unitsselectionsorting.py @@ -48,14 +48,14 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None): def _compute_and_cache_spike_vector(self) -> None: from spikeinterface.core.sorting_tools import remap_unit_indices_in_vector - if self._parent_sorting._cached_time_ordered_spike_vector is None: + if self._parent_sorting._cached_time_ordered is None: self._parent_sorting._compute_and_cache_spike_vector() - if self._parent_sorting._cached_time_ordered_spike_vector is None: + if self._parent_sorting._cached_time_ordered is None: return spike_vector, _ = remap_unit_indices_in_vector( - vector=self._parent_sorting._cached_time_ordered_spike_vector["spikes"], + vector=self._parent_sorting._cached_time_ordered["spikes"], all_old_unit_ids=self._parent_sorting.unit_ids, all_new_unit_ids=self._unit_ids, ) @@ -75,7 +75,7 @@ def _compute_and_cache_spike_vector(self) -> None: (spike_vector["unit_index"], spike_vector["sample_index"], spike_vector["segment_index"]) ) spike_vector = spike_vector[sort_indices] - self._cached_time_ordered_spike_vector = {"spikes": spike_vector, "segment_slices": None} + self._cached_time_ordered = {"spikes": spike_vector, "segment_slices": None} class UnitsSelectionSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py index ba450cf4a1..57327d4287 100644 --- a/src/spikeinterface/core/zarrextractors.py +++ b/src/spikeinterface/core/zarrextractors.py @@ -527,7 +527,7 @@ def __init__( # pre-populate segment slices so _get_spike_vector_segment_slices() never # needs to materialise the full segment_index array - self._cached_time_ordered_spike_vector = { + self._cached_time_ordered = { "spikes": spikes, "segment_slices": np.asarray(segment_slices_list, dtype="int64"), } diff --git a/src/spikeinterface/curation/remove_excess_spikes.py b/src/spikeinterface/curation/remove_excess_spikes.py index 4f18a9d0c6..a2b8ebb20a 100644 --- a/src/spikeinterface/curation/remove_excess_spikes.py +++ b/src/spikeinterface/curation/remove_excess_spikes.py @@ -46,13 +46,13 @@ def __init__(self, sorting: BaseSorting, recording: BaseRecording) -> None: self._kwargs = {"sorting": sorting, "recording": recording} def _compute_and_cache_spike_vector(self) -> None: - if self._parent_sorting._cached_time_ordered_spike_vector is None: + if self._parent_sorting._cached_time_ordered is None: self._parent_sorting._compute_and_cache_spike_vector() - if self._parent_sorting._cached_time_ordered_spike_vector is None: + if self._parent_sorting._cached_time_ordered is None: return - parent_spike_vector = self._parent_sorting._cached_time_ordered_spike_vector["spikes"] + parent_spike_vector = self._parent_sorting._cached_time_ordered["spikes"] num_segments = self._parent_sorting.get_num_segments() list_spike_vectors = [] @@ -64,7 +64,7 @@ def _compute_and_cache_spike_vector(self) -> None: list_spike_vectors.append(spike_vector[start:end]) spike_vector = np.concatenate(list_spike_vectors) - self._cached_time_ordered_spike_vector = {"spikes": spike_vector, "segment_slices": None} + self._cached_time_ordered = {"spikes": spike_vector, "segment_slices": None} class RemoveExcessSpikesSortingSegment(BaseSortingSegment): diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 863616fef7..6a0e0cb2f2 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -1453,7 +1453,7 @@ def _compute_and_cache_spike_vector(self) -> None: spikes["sample_index"] = sample_indices[order] spikes["unit_index"] = unit_indices[order] # segment_index stays 0 for the single segment. - self._cached_time_ordered_spike_vector = { + self._cached_time_ordered = { "spikes": spikes, "segment_slices": np.array([[0, num_spikes]], dtype="int64"), } diff --git a/src/spikeinterface/extractors/phykilosortextractors.py b/src/spikeinterface/extractors/phykilosortextractors.py index b9ec8c8b44..300a9b4662 100644 --- a/src/spikeinterface/extractors/phykilosortextractors.py +++ b/src/spikeinterface/extractors/phykilosortextractors.py @@ -268,7 +268,7 @@ def _compute_and_cache_spike_vector(self) -> None: # This is useless because phy is always one segment # spikes["segment_index"] = 0 - self._cached_time_ordered_spike_vector = { + self._cached_time_ordered = { "spikes": spikes, "segment_slices": np.array([[0, n]], dtype="int64"), } From d4e895158fdb402fe5f3f0c918ee4b2bd8536bd2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 12 Oct 2026 12:38:25 -0600 Subject: [PATCH 6/6] Resolve the merge with main on the order-named caches --- src/spikeinterface/core/basesorting.py | 34 +++-------- .../core/tests/test_unitsselectionsorting.py | 13 ++-- .../core/unitsselectionsorting.py | 59 +++++-------------- src/spikeinterface/core/zarrextractors.py | 10 +--- 4 files changed, 31 insertions(+), 85 deletions(-) diff --git a/src/spikeinterface/core/basesorting.py b/src/spikeinterface/core/basesorting.py index 0ffcb92c92..cbdff9dbce 100644 --- a/src/spikeinterface/core/basesorting.py +++ b/src/spikeinterface/core/basesorting.py @@ -166,9 +166,9 @@ def search_cached_spikes_sorted( positions : np.ndarray The insertion positions, relative to the start of the segment in the spike vector. """ - if self._cached_spike_vector is None: + if self._cached_time_ordered is None: self._compute_and_cache_spike_vector() - spikes = self._cached_spike_vector + spikes = self._cached_time_ordered["spikes"] if not isinstance(spikes, np.ndarray): # np.memmap is an ndarray # np.searchsorted would materialise a lazy vector on every call raise TypeError( @@ -1112,33 +1112,15 @@ def _get_spike_vector_segment_slices(self): if cache["segment_slices"] is None: # compute the, this is needed when spikevector is loaded from format and not computed num_seg = self.get_num_segments() -<<<<<<< HEAD - slices = np.searchsorted(cache["spikes"]["segment_index"], np.arange(num_seg + 1)) - cache["segment_slices"] = np.zeros((num_seg, 2), dtype="int64") - for seg_index in range(num_seg): - cache["segment_slices"][seg_index, 0] = slices[seg_index] - cache["segment_slices"][seg_index, 1] = slices[seg_index + 1] - return cache["segment_slices"] -||||||| 1666a0eab - slices = np.searchsorted(self._cached_spike_vector["segment_index"], np.arange(num_seg + 1)) - self._cached_spike_vector_segment_slices = np.zeros((num_seg, 2), dtype="int64") - for seg_index in range(num_seg): - self._cached_spike_vector_segment_slices[seg_index, 0] = slices[seg_index] - self._cached_spike_vector_segment_slices[seg_index, 1] = slices[seg_index + 1] - return self._cached_spike_vector_segment_slices -======= if num_seg == 1: - self._cached_spike_vector_segment_slices = np.array( - [[0, self._cached_spike_vector.size]], dtype="int64" - ) + cache["segment_slices"] = np.array([[0, cache["spikes"].size]], dtype="int64") else: - slices = np.searchsorted(self._cached_spike_vector["segment_index"], np.arange(num_seg + 1)) - self._cached_spike_vector_segment_slices = np.zeros((num_seg, 2), dtype="int64") + slices = np.searchsorted(cache["spikes"]["segment_index"], np.arange(num_seg + 1)) + cache["segment_slices"] = np.zeros((num_seg, 2), dtype="int64") for seg_index in range(num_seg): - self._cached_spike_vector_segment_slices[seg_index, 0] = slices[seg_index] - self._cached_spike_vector_segment_slices[seg_index, 1] = slices[seg_index + 1] - return self._cached_spike_vector_segment_slices ->>>>>>> main + cache["segment_slices"][seg_index, 0] = slices[seg_index] + cache["segment_slices"][seg_index, 1] = slices[seg_index + 1] + return cache["segment_slices"] def to_reordered_spike_vector( self, diff --git a/src/spikeinterface/core/tests/test_unitsselectionsorting.py b/src/spikeinterface/core/tests/test_unitsselectionsorting.py index 309563df62..53c8f47636 100644 --- a/src/spikeinterface/core/tests/test_unitsselectionsorting.py +++ b/src/spikeinterface/core/tests/test_unitsselectionsorting.py @@ -147,7 +147,7 @@ def test_selection_keeps_extra_fields(): stay with their spike through a selection.""" wide_dtype = minimum_spike_dtype + [("channel_index", "int64")] parent = _make_parent_with_shuffled_ties(dtype=wide_dtype) - parent._cached_spike_vector["channel_index"] = np.arange(parent._cached_spike_vector.size) + parent._cached_time_ordered["spikes"]["channel_index"] = np.arange(parent._cached_time_ordered["spikes"].size) unit_ids = ["3", "22", "29"] child = UnitsSelectionSorting(parent, unit_ids=unit_ids) @@ -188,7 +188,7 @@ def _assert_shares_parent_caches(child, parent): assert child._is_identity_selection is True assert child.to_spike_vector() is parent.to_spike_vector() assert child._get_spike_vector_segment_slices() is parent._get_spike_vector_segment_slices() - assert child._cached_lexsorted_spike_vector is parent._cached_lexsorted_spike_vector + assert child._cached_unit_grouped is parent._cached_unit_grouped @pytest.mark.parametrize("make_child", IDENTITY_SELECTIONS.values(), ids=IDENTITY_SELECTIONS.keys()) @@ -203,13 +203,13 @@ def test_identity_selection_shares_parent_cache(make_child): parent_reordered = parent.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False) child_reordered = child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False) assert child_reordered is parent_reordered - assert len(parent._cached_lexsorted_spike_vector) == 1 + assert parent._cached_unit_grouped is not None # The child computes the reordered spike vector first (via get_unit_spike_train); the parent must find it. parent = _make_parent_with_shuffled_ties() child = make_child(parent) child.get_unit_spike_train(child.unit_ids[0], segment_index=0) - assert len(parent._cached_lexsorted_spike_vector) == 1 + assert parent._cached_unit_grouped is not None parent_reordered = parent.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False) child_reordered = child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT, return_order=False, return_slices=False) assert child_reordered is parent_reordered @@ -241,11 +241,10 @@ def test_non_identity_selection_does_not_share(unit_ids): child = parent.select_units(unit_ids) assert child._is_identity_selection is False assert child.to_spike_vector() is not parent.to_spike_vector() - assert child._cached_lexsorted_spike_vector is not parent._cached_lexsorted_spike_vector child.to_reordered_spike_vector(LEXSORT_UNIT_COMPACT) - assert len(child._cached_lexsorted_spike_vector) == 1 - assert len(parent._cached_lexsorted_spike_vector) == 0 + assert child._cached_unit_grouped is not None + assert parent._cached_unit_grouped is None @pytest.mark.requires_zarr_write diff --git a/src/spikeinterface/core/unitsselectionsorting.py b/src/spikeinterface/core/unitsselectionsorting.py index 6cf04691e6..0e8e1a4085 100644 --- a/src/spikeinterface/core/unitsselectionsorting.py +++ b/src/spikeinterface/core/unitsselectionsorting.py @@ -34,9 +34,6 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None): BaseSorting.__init__(self, sampling_frequency, self._renamed_unit_ids) self._is_identity_selection = bool(np.array_equal(self._unit_ids, parents_unit_ids)) - if self._is_identity_selection: - # Same units (possibly renamed), same order => we can use the parent's cached spike vector - self._cached_lexsorted_spike_vector = parent_sorting._cached_lexsorted_spike_vector for parent_segment in self._parent_sorting.segments: sub_segment = UnitsSelectionSortingSegment(parent_segment, ids_conversion) @@ -50,6 +47,20 @@ def __init__(self, parent_sorting, unit_ids=None, renamed_unit_ids=None): self._kwargs = dict(parent_sorting=parent_sorting, unit_ids=unit_ids, renamed_unit_ids=renamed_unit_ids) + @property + def _cached_unit_grouped(self): + # Same units (possibly renamed), same order => we read and fill the parent's unit-grouped cache + if getattr(self, "_is_identity_selection", False): + return self._parent_sorting._cached_unit_grouped + return self._own_cached_unit_grouped + + @_cached_unit_grouped.setter + def _cached_unit_grouped(self, value): + if getattr(self, "_is_identity_selection", False): + self._parent_sorting._cached_unit_grouped = value + else: + self._own_cached_unit_grouped = value + def _compute_and_cache_spike_vector(self) -> None: from spikeinterface.core.sorting_tools import remap_unit_indices_in_vector @@ -60,8 +71,8 @@ def _compute_and_cache_spike_vector(self) -> None: return if self._is_identity_selection: - self._cached_spike_vector = self._parent_sorting._cached_spike_vector - self._cached_spike_vector_segment_slices = self._parent_sorting._get_spike_vector_segment_slices() + self._parent_sorting._get_spike_vector_segment_slices() + self._cached_time_ordered = self._parent_sorting._cached_time_ordered return spike_vector, _ = remap_unit_indices_in_vector( @@ -69,45 +80,7 @@ def _compute_and_cache_spike_vector(self) -> None: all_old_unit_ids=self._parent_sorting.unit_ids, all_new_unit_ids=self._unit_ids, ) -<<<<<<< HEAD - - # check if order is preserved - pos = np.searchsorted(self._parent_sorting.unit_ids, self.unit_ids) - order_is_preserved = np.all(np.diff(pos) > 0) - - if not order_is_preserved: - # Note: this can be a very high cost and make big dataset very slow - # the only goal of this is to ensure the unit_index order when the sample is the same - # TODO: maybe we can remove it, if we don't guarantee the order of unit_index - # when sample_index is the same, but it can be a problem for some downstream analysis - - # lexsort by segment_index, sample_index, unit_index - sort_indices = np.lexsort( - (spike_vector["unit_index"], spike_vector["sample_index"], spike_vector["segment_index"]) - ) - spike_vector = spike_vector[sort_indices] self._cached_time_ordered = {"spikes": spike_vector, "segment_slices": None} -||||||| 1666a0eab - - # check if order is preserved - pos = np.searchsorted(self._parent_sorting.unit_ids, self.unit_ids) - order_is_preserved = np.all(np.diff(pos) > 0) - - if not order_is_preserved: - # Note: this can be a very high cost and make big dataset very slow - # the only goal of this is to ensure the unit_index order when the sample is the same - # TODO: maybe we can remove it, if we don't guarantee the order of unit_index - # when sample_index is the same, but it can be a problem for some downstream analysis - - # lexsort by segment_index, sample_index, unit_index - sort_indices = np.lexsort( - (spike_vector["unit_index"], spike_vector["sample_index"], spike_vector["segment_index"]) - ) - spike_vector = spike_vector[sort_indices] - self._cached_spike_vector = spike_vector -======= - self._cached_spike_vector = spike_vector ->>>>>>> main def search_cached_spikes_sorted( self, diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py index 95cf1be498..daa3f94302 100644 --- a/src/spikeinterface/core/zarrextractors.py +++ b/src/spikeinterface/core/zarrextractors.py @@ -592,17 +592,9 @@ def __init__( # we do not need to lexsort at init (very high cost) because there already sorted by frame before to be saved. # In version 0.104.X this was fully lexsorted, but we don't need it anymore because it's only important in the context of SpikeVectorBased extensions in the SortingAnalyzer, which stores its own copy of the Sorting object. This makes the extension data and the spike vector always matching their order. # spikes = spikes[np.lexsort((spikes["unit_index"], spikes["sample_index"], spikes["segment_index"]))] -<<<<<<< HEAD - -||||||| 1666a0eab - - self._cached_spike_vector = spikes -======= self._lazy_spike_vector = lazy_spike_vector self._spikes_group = spikes_group self._sample_index_search = None - self._cached_spike_vector = spikes ->>>>>>> main # pre-populate segment slices so _get_spike_vector_segment_slices() never # needs to materialise the full segment_index array self._cached_time_ordered = { @@ -652,7 +644,7 @@ def search_cached_spikes_sorted( if "sample_index_chunk_firsts" in self._spikes_group: chunk_firsts = self._spikes_group["sample_index_chunk_firsts"][:] self._sample_index_search = ZarrSampleIndexSearch(self._spikes_group["sample_index"], chunk_firsts) - start, stop = self._cached_spike_vector_segment_slices[segment_index] + start, stop = self._cached_time_ordered["segment_slices"][segment_index] return self._sample_index_search.searchsorted(indices, int(start), int(stop)) @staticmethod