From af340b2a4b3eea94902c3d01225e907586b2a16d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 27 Oct 2026 17:07:04 -0600 Subject: [PATCH 1/2] Fall back to the cached test data when gin cannot be reached in CI --- .github/workflows/io-test.yml | 5 +++-- .github/workflows/plexon2-testing.yml | 5 +++-- neo/utils/datasets.py | 6 +++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 68a741680..92e23cfc1 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -42,10 +42,11 @@ jobs: # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git id: ephy_testing_data run: | + # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one set -o pipefail if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then - echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key" - exit 1 + echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache" + hash="" fi echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/plexon2-testing.yml b/.github/workflows/plexon2-testing.yml index c3b2f314a..8fdab0b5c 100644 --- a/.github/workflows/plexon2-testing.yml +++ b/.github/workflows/plexon2-testing.yml @@ -33,10 +33,11 @@ jobs: # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git id: ephy_testing_data run: | + # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one set -o pipefail if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then - echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key" - exit 1 + echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache" + hash="" fi echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT" diff --git a/neo/utils/datasets.py b/neo/utils/datasets.py index 5d9527551..d29427e6d 100644 --- a/neo/utils/datasets.py +++ b/neo/utils/datasets.py @@ -3,6 +3,7 @@ """ import os +import warnings from pathlib import Path default_testing_repo = "https://gin.g-node.org/NeuralEnsemble/ephy_testing_data" @@ -64,7 +65,10 @@ def download_dataset(repo=default_testing_repo, remote_path=None, local_folder=N # make sure git repo is in clean state repo = dataset.repo repo.call_git(["checkout", "--force", "master"]) - dataset.update(merge=True) + # the remote can be unreachable (gin answers 403 to CI runners at times), so keep the local copy then + results = dataset.update(merge=True, on_failure="ignore") + if any(result["status"] == "error" for result in results): + warnings.warn(f"Could not update {local_folder} from its remote, using the local copy as it is") else: dataset = datalad.api.install(path=local_folder, source=repo) From 56dd1d5f04ec1d25821d7ff37b5dda9fb5ad5bb4 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 28 Oct 2026 10:20:18 -0600 Subject: [PATCH 2/2] Update the test data at most once per process and warn on datalad's impossible status --- neo/utils/datasets.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/neo/utils/datasets.py b/neo/utils/datasets.py index d29427e6d..7e70ddad8 100644 --- a/neo/utils/datasets.py +++ b/neo/utils/datasets.py @@ -15,6 +15,9 @@ # set in home local_testing_data_folder = Path.home() / "ephy_testing_data" +# local copies already updated in this process, so each one asks its remote at most once +_updated_folders = set() + def get_local_testing_data_folder(): global local_testing_data_folder @@ -65,10 +68,12 @@ def download_dataset(repo=default_testing_repo, remote_path=None, local_folder=N # make sure git repo is in clean state repo = dataset.repo repo.call_git(["checkout", "--force", "master"]) - # the remote can be unreachable (gin answers 403 to CI runners at times), so keep the local copy then - results = dataset.update(merge=True, on_failure="ignore") - if any(result["status"] == "error" for result in results): - warnings.warn(f"Could not update {local_folder} from its remote, using the local copy as it is") + if local_folder not in _updated_folders: + _updated_folders.add(local_folder) + # the remote can be unreachable (gin answers 403 to CI runners at times), so keep the local copy then + results = dataset.update(merge=True, on_failure="ignore") + if any(result["status"] in ("impossible", "error") for result in results): + warnings.warn(f"Could not update {local_folder} from its remote, using the local copy as it is") else: dataset = datalad.api.install(path=local_folder, source=repo)