diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..32f8a92 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,42 @@ +# Scripts + +## Structure + +- `data_preparation.*`: Scripts for preparing the data for training, tuning and evaluation. Mainly converting large netCDF files to Zarr storage with specific chunking strategies. This allows executing training for larger-than-memory datasets. +- `example_training.*`: example training script +- `tuning.*`: Scripts for hyperparameter tuning. +- `run_best_tuned_model.*`: Scripts for running the best tuned model on the test set. +- `logs`: + - `eso4clima_24438134_subset.out`: example SLURM job output file of an execution on a subset of the global dataset. The dataset has two years of data (2020-2021) and the spatial coverage is from 30S to 30N and from 30W to 30E. + - `eso4clima_24449471_full.out`: example SLURM job output file of an execution on the full dataset, two years of data (2020-2021) and almost global coverage (from 80S to 80N and from 179.99W to 179.99E). The training only executed for 1 hour and cuted off by SLURM time limit. + +## Experiments + +### Tuning experiments + +- datasplit: train set = 2020, validation set = 2021, test set = 2022 +- path of tuning results: `/work//eso4clima/tune/`. +- test loss: 0.036662004509047774 +- hyperparameters of the best model: + + ``` + {'patch_size': 8, + 'overlap': 1, + 'embed_dim': 64, + 'dropout': 0.2, + 'hidden': 32, + 'spatial_depth': 3, + 'spatial_heads': 2, + 'optimizer_lr': 0.001787422899066508, + 'batch_config': {'batch_size': 100, 'accumulation_steps': 2}} + ``` + +### Training experiments + + - Use the best hyperparameters found in the tuning experiments to train the model on the training set and evaluate it on the test set. + - Use three years 2018-2020 for training, 2021 for validation and 2022 for testing. + - Prepared data is stored in `/work//eso4clima/preprocessed/sst/`. + - Load one year data following example in `run_best_tuned_model.py` script. Then concatenate the three years as `xr.concat([da_2018, da_2019, da_2020], dim="M")`. + - In dataloader, use `load_lazy=True`. + + diff --git a/scripts/logs/eso4clima_24438134_subset.out b/scripts/logs/eso4clima_24438134_subset.out new file mode 100644 index 0000000..377cb42 --- /dev/null +++ b/scripts/logs/eso4clima_24438134_subset.out @@ -0,0 +1,263 @@ +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +2026-04-23 12:05:45,831 - INFO - Creating the model... +2026-04-23 12:05:46,026 - INFO - Creating the dataset... +2026-04-23 12:06:16,938 - INFO - Starting training... +Epoch 0: best_loss = 3.379575 +Epoch 20: best_loss = 2.331017 +Epoch 40: best_loss = 2.142740 +Epoch 60: best_loss = 1.710370 +Epoch 80: best_loss = 1.189252 +Epoch 100: best_loss = 0.894883 +Epoch 120: best_loss = 0.688254 +Epoch 140: best_loss = 0.571773 +Epoch 160: best_loss = 0.505098 +Epoch 180: best_loss = 0.445357 +Epoch 200: best_loss = 0.412255 +Epoch 220: best_loss = 0.381430 +Epoch 240: best_loss = 0.361015 +Epoch 260: best_loss = 0.346520 +Epoch 280: best_loss = 0.325091 +Epoch 300: best_loss = 0.317928 +Epoch 320: best_loss = 0.310767 +Epoch 340: best_loss = 0.303774 +Epoch 360: best_loss = 0.296891 +Epoch 380: best_loss = 0.290118 +Epoch 400: best_loss = 0.285634 +Epoch 420: best_loss = 0.281635 +Epoch 440: best_loss = 0.278483 +Epoch 460: best_loss = 0.275278 +Epoch 480: best_loss = 0.272091 +Epoch 500: best_loss = 0.268902 +Training complete. Best loss: 0.268902 +Model saved to runs/best_model.pth +==== Slurm accounting summary 24438134 ==== +JobID|NTasks|AveCPU|AveRSS|MaxRSS|MaxVMSize|TRESUsageInAve|TRESUsageInMax +24438134.extern|1|00:00:00|856K|3752K|575840K|cpu=00:00:00,energy=0,fs/disk=2332,mem=856K,pages=2,vmem=217160K|cpu=00:00:00,energy=0,fs/disk=2332,mem=3752K,pages=2,vmem=575840K +24438134.batch|1|6-04:57:46|11960K|21279432K|53545480K|cpu=6-04:57:46,energy=0,fs/disk=6648661618,mem=11960K,pages=19,vmem=356728K|cpu=6-04:57:46,energy=0,fs/disk=6648661618,mem=21279432K,pages=8555,vmem=53545480K + +******************************************************************************** +* * +* This is the automated job summary provided by DKRZ. * +* If you encounter problems, need assistance or have any suggestion, please * +* write an email to * +* * +* -- support@dkrz.de -- * +* * +* We hope you enjoyed the DKRZ supercomputer LEVANTE ... * +* +* JobID : 24438134 +* JobName : eso4clima +* Account : bd0854 +* User : b383704 (202985), bd0854 (1473) +* Partition : compute +* QOS : normal +* Nodelist : l40346 (1) +* Submit date : 2026-04-23T12:03:09 +* Start time : 2026-04-23T12:04:45 +* End time : 2026-04-23T14:04:54 +* Elapsed time : 02:00:09 (Timelimit=04:00:00) +* Command : /home/b/b383704/eso4clima/train_twoyears/ +* example_subset.slurm +* WorkDir : /home/b/b383704/eso4clima/train_twoyears +* +* StepID | JobName NodeHours MaxRSS [Byte] (@task) +* ------------------------------------------------------------------------------ +* batch | batch 2.0 +* extern | extern 2.0 3752K (0) +* ------------------------------------------------------------------------------ + +JobID|JobName|Partition|AllocCPUS|Elapsed|TotalCPU|MaxRSS|State|ExitCode +24438134|eso4clima|compute|256|02:00:09|6-04:57:47||COMPLETED|0:0 +24438134.batch|batch||256|02:00:09|6-04:57:47|21279432K|COMPLETED|0:0 +24438134.extern|extern||256|02:00:09|00:00.001|3752K|COMPLETED|0:0 diff --git a/scripts/logs/eso4clima_24449471_full.out b/scripts/logs/eso4clima_24449471_full.out new file mode 100644 index 0000000..c5b28bb --- /dev/null +++ b/scripts/logs/eso4clima_24449471_full.out @@ -0,0 +1,232 @@ +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:39: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + daily_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lat" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +/home/b/b383704/eso4clima/train_twoyears/example_training_subset.py:52: UserWarning: The specified chunks separate the stored chunks along dimension "lon" starting at index 240. This could degrade performance. Instead, consider rechunking after loading. + monthly_data = xr.open_mfdataset( +2026-04-23 17:37:53,532 - INFO - Creating the model... +2026-04-23 17:37:53,750 - INFO - Creating the dataset... +/home/b/b383704/eso4clima/ClimaNet/climanet/dataset.py:112: UserWarning: Patch size (120, 120) does not evenly divide image dimensions (H=720, W=640). Uncovered pixels: 0 in height, 40 in width. Consider adjusting patch_size or image dimensions for full coverage. + warnings.warn( +2026-04-23 17:38:52,233 - INFO - Starting training... +Epoch 0: best_loss = 13.395518 +Epoch 20: best_loss = 5.020292 +slurmstepd: error: *** JOB 24449471 ON l10543 CANCELLED AT 2026-04-23T18:37:47 DUE TO TIME LIMIT *** + +******************************************************************************** +* * +* This is the automated job summary provided by DKRZ. * +* If you encounter problems, need assistance or have any suggestion, please * +* write an email to * +* * +* -- support@dkrz.de -- * +* * +* We hope you enjoyed the DKRZ supercomputer LEVANTE ... * +* +* JobID : 24449471 +* JobName : eso4clima +* Account : bd0854 +* User : b383704 (202985), bd0854 (1473) +* Partition : compute +* QOS : normal +* Nodelist : l10543 (1) +* Submit date : 2026-04-23T17:37:01 +* Start time : 2026-04-23T17:37:37 +* End time : 2026-04-23T18:37:47 +* Elapsed time : 01:00:10 (Timelimit=01:00:00) +* Command : /home/b/b383704/eso4clima/train_twoyears/ +* example_subset.slurm +* WorkDir : /home/b/b383704/eso4clima/train_twoyears +* +* StepID | JobName NodeHours MaxRSS [Byte] (@task) +* ------------------------------------------------------------------------------ +* batch | batch 1.0 +* extern | extern 1.0 3812K (0) +* ------------------------------------------------------------------------------ + diff --git a/scripts/run_best_tuned_model.py b/scripts/run_best_tuned_model.py new file mode 100644 index 0000000..7c2964d --- /dev/null +++ b/scripts/run_best_tuned_model.py @@ -0,0 +1,184 @@ +import argparse +from pathlib import Path + +import xarray as xr +from ray import tune + +from climanet.dataset import DataLoaderConfig, STDataset +from climanet.predict import PredictionConfig, predict_monthly_var +from climanet.utils import data_preparation, read_st_data + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Load the best Ray Tune checkpoint, prepare the test data, and evaluate the " + "trained model on the 2023 test period." + ) + ) + parser.add_argument( + "--experiment-path", + type=Path, + required=True, + help="Path to the Ray Tune experiment directory containing the checkpoint.", + ) + parser.add_argument( + "--test-data-dir", + type=Path, + required=True, + help="Directory containing the test NetCDF files.", + ) + parser.add_argument( + "--lsm-file-path", + type=Path, + required=True, + help="Path to the land-sea mask NetCDF file.", + ) + parser.add_argument( + "--run-dir", + type=Path, + default=Path("./run_dir_tune_test").resolve(), + help="Directory used for the evaluation run and saved logs.", + ) + parser.add_argument( + "--var-name", + type=str, + default="tos", + help="Variable name to evaluate in the NetCDF files.", + ) + parser.add_argument( + "--year", + type=str, + default="2022", + help="Year pattern to include in the test files (e.g. 2022).", + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + experiment_path = args.experiment_path.resolve() + test_data_dir = args.test_data_dir.resolve() + lsm_file_path = args.lsm_file_path.resolve() + run_dir = args.run_dir.resolve() + run_dir.mkdir(parents=True, exist_ok=True) + + if not experiment_path.exists(): + raise FileNotFoundError( + f"Experiment directory does not exist: {experiment_path}" + ) + if not test_data_dir.exists(): + raise FileNotFoundError(f"Test data directory does not exist: {test_data_dir}") + if not lsm_file_path.exists(): + raise FileNotFoundError(f"LSM file does not exist: {lsm_file_path}") + + daily_files = list( + test_data_dir.glob(f"{args.year}*_hr_ERA5dc_masked_{args.var_name}*.nc") + ) + monthly_files = list( + test_data_dir.glob(f"{args.year}*_mon_ERA5dc_masked_{args.var_name}*.nc") + ) + + if not daily_files: + raise FileNotFoundError( + f"No daily test files found for year '{args.year}' in '{test_data_dir}'" + ) + if not monthly_files: + raise FileNotFoundError( + f"No monthly test files found for year '{args.year}' in '{test_data_dir}'" + ) + + print(f"Using daily files ({len(daily_files)}): {daily_files[:3]} ...") + print(f"Using monthly files ({len(monthly_files)}): {monthly_files[:3]} ...") + + daily_data_test = xr.open_mfdataset( + daily_files, combine="by_coords", parallel=False + ) + monthly_data_test = xr.open_mfdataset( + monthly_files, combine="by_coords", parallel=False + ) + + test_data_zarr_dir = run_dir / "test_data_zarr" + test_data_zarr_dir.mkdir(parents=True, exist_ok=True) + + _ = data_preparation( + daily_data_test[args.var_name], + monthly_data_test[args.var_name], + calculate_residuals=True, + is_hourly=True, + save_to_zarr=True, + run_dir=test_data_zarr_dir, + ) + + input_da, input_da_nan_mask, monthly_da, padded_days_mask, time_features = ( + read_st_data( + data_path=test_data_zarr_dir, + var_name=args.var_name, + ) + ) + + lsm_mask = xr.open_dataset(lsm_file_path) + + num_patches = (10, 10) + patch_size = (1, 4, 4) + spatial_patch_size = ( + patch_size[1] * num_patches[0], + patch_size[2] * num_patches[1], + ) + stride = (spatial_patch_size[0] // 5, spatial_patch_size[1] // 5) + + dataset_test = STDataset( + input_da=input_da, + input_da_nan_mask=input_da_nan_mask, + monthly_da=monthly_da, + padded_days_mask=padded_days_mask, + time_features=time_features, + land_mask=lsm_mask["lsm"], + patch_size=(1, *spatial_patch_size), + stride=stride, + sh_embed_dim=96, + sh_order_L=10, + verbose=True, + load_lazy=False, + ) + print(f"Created test dataset with {len(dataset_test)} patches.") + + analysis = tune.ExperimentAnalysis(str(experiment_path)) + best_result = analysis.get_best_trial("loss", "min") + best_checkpoint = best_result.checkpoint + model_path = Path(best_checkpoint.path) / "checkpoint.pt" + print(f"Best checkpoint path: {model_path}") + + prediction_config = PredictionConfig( + calculate_residuals=True, + return_numpy=True, + save_predictions=False, + return_loss=True, + device="cpu", + verbose=False, + ) + + dataloader_config = DataLoaderConfig( + batch_size=10, + shuffle=True, + num_workers=0, + pin_memory=False, + persistent_workers=False, + device="cpu", + multiprocessing_context=None, + ) + + test_loss = predict_monthly_var( + model=model_path, + dataset=dataset_test, + dataloader_config=dataloader_config, + prediction_config=prediction_config, + run_dir=run_dir, + ) + + print("Test loss:") + print(test_loss) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_best_tuned_model.slurm b/scripts/run_best_tuned_model.slurm new file mode 100644 index 0000000..aa3f3b4 --- /dev/null +++ b/scripts/run_best_tuned_model.slurm @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH --job-name=climanet_eval +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=128 +#SBATCH --time=02:00:00 +#SBATCH --account=bd0854 +#SBATCH --partition=compute +#SBATCH --output=climanet_eval_%j.out +#SBATCH --error=climanet_eval_%j.err + +set -euo pipefail + +source /home/b/b383704/eso4clima/ClimaNet/.venv/bin/activate + +python -u /home/b/b383704/eso4clima/run_best_tuned_model/run_best_tuned_model.py \ + --experiment-path /work/bd0854/eso4clima/tune/sst_01 \ + --test-data-dir /work/bd0854/b380103/eso4clima/output/sst/concatenated/ \ + --lsm-file-path /home/b/b383704/eso4clima/data/era5_lsm_bool.nc \ + --run-dir /home/b/b383704/eso4clima/run_best_tuned_model/run_dir \ + --var-name tos \ + --year 2022 + +printf "\nFinished evaluation run.\n" diff --git a/scripts/training.py b/scripts/training.py new file mode 100644 index 0000000..ff8034a --- /dev/null +++ b/scripts/training.py @@ -0,0 +1,157 @@ +from pathlib import Path +import xarray as xr +import torch +import torch.nn.functional +from climanet.st_encoder_decoder import SpatioTemporalModel +from climanet.utils import ( + set_seed, + configure_compute_resources, +) +from climanet.train import train_monthly_model +from climanet import STDataset + +from torch.utils.data import random_split + + +def main(): + # Data settings + # Data folder + data_folder = Path("/work/bd0854/b380103/eso4clima/output/concatenated/") + # Path to land-sea mask file (need to setup in the experiment directory) + lsm_file = "/home/b/b383704/eso4clima/data/era5_lsm_bool.nc" + # Must be divisible by the model patch size + # Default input data has 720x1440 spatial dimensions + + # Training settings + patch_size_model = (1, 4, 4) # Size of model encoder (time, lat, lon). + num_patches = (30, 30) # Number of patches in spatial dimensions + spatial_patch_size = ( + patch_size_model[1] * num_patches[0], + patch_size_model[2] * num_patches[1], + ) # Spatial dimensions of the input data + stride = (spatial_patch_size[0] // 5, spatial_patch_size[1] // 5) + overlap = 2 # Overlap between patches (in pixels). + num_months = 2 # Number of months to predict (model output channels) + embed_dim = 128 + dropout = 0.2 + hidden = 128 + batch_size = 50 # Number of samples per batch in training + num_epoch = 100 # Maximum number of epochs to train + accumulation_steps = 2 # Number of batches to accumulate gradients over + sh_embed_dim = 96 + sh_order_L = 10 + compute_threads = 96 + dataloader_num_workers = 32 + run_dir = "./runs" # Directory to save logs and model checkpoints + + # Get list of daily and monthly files, sort by time + daily_files = sorted(data_folder.rglob("20*day_ERA5dc_masked_tos.nc")) + monthly_files = sorted(data_folder.rglob("20*mon_ERA5dc_full_tos.nc")) + + # Set seed for reproducibility + set_seed() + + # Open datasets with chunks + # The chunk sizes are chosen as twice the sample patch size + daily_data = xr.open_mfdataset( + daily_files, + combine="by_coords", + chunks={ + "time": 1, + "lat": spatial_patch_size[0] * 2, + "lon": spatial_patch_size[1] * 2, + }, + data_vars="minimal", + coords="minimal", + compat="override", + parallel=False, + ) + daily_data = daily_data.chunk( + {"time": 1, "lat": spatial_patch_size[0] * 2, "lon": spatial_patch_size[1] * 2} + ) # Mannually chunk the dataset after opening + + monthly_data = xr.open_mfdataset( + monthly_files, + combine="by_coords", + chunks={ + "time": 1, + "lat": spatial_patch_size[0] * 2, + "lon": spatial_patch_size[1] * 2, + }, + data_vars="minimal", + coords="minimal", + compat="override", + parallel=False, + ) + monthly_data = monthly_data.chunk( + {"time": 1, "lat": spatial_patch_size[0] * 2, "lon": spatial_patch_size[1] * 2} + ) # Mannually chunk the dataset after opening + + lsm_mask = xr.open_dataset(lsm_file) + + # Use Monthly residuals as target + daily_data_averaged = daily_data.resample(time="MS").mean(skipna=True) + daily_data_averaged["time"] = monthly_data["time"] + monthly_data_res = monthly_data - daily_data_averaged + + # create the model + print("Creating the model...") + model = SpatioTemporalModel( + patch_size=patch_size_model, + overlap=overlap, + num_months=num_months, + embed_dim=embed_dim, + dropout=dropout, + hidden=hidden, + ) + + # Make a dataset + print("Creating the dataset...") + dataset = STDataset( + daily_da=daily_data["tos"], + monthly_da=monthly_data_res["tos"], + land_mask=lsm_mask["lsm"], + patch_size=spatial_patch_size, # based on the patch_size in model + stride=stride, + sh_embed_dim=sh_embed_dim, + sh_order_L=sh_order_L, + ) + print(f"Total length training dataset: {len(dataset)}") + + # create train test data + generator = torch.Generator().manual_seed(42) + train_size = int(0.6 * len(dataset)) + validation_size = int(0.3 * len(dataset)) + test_size = len(dataset) - train_size - validation_size + train_dataset, validation_dataset, test_dataset = random_split( + dataset, [train_size, validation_size, test_size], generator=generator + ) + print( + f"Train dataset length: {len(train_dataset)}, Validation dataset length: {len(validation_dataset)}, Test dataset length: {len(test_dataset)}" + ) + + # Device and resources + model = configure_compute_resources( + model, + device="cpu", + compute_threads=compute_threads, + dataloader_num_workers=dataloader_num_workers, + ) + + # Train the model + # Results will be saved to runs/best_model.pth + print("Starting training...") + _ = train_monthly_model( + model, + train_dataset, + validation_dataset=validation_dataset, + batch_size=batch_size, + num_epoch=num_epoch, + accumulation_steps=accumulation_steps, + run_dir=run_dir, + dataloader_num_workers=dataloader_num_workers, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/training.slurm b/scripts/training.slurm new file mode 100644 index 0000000..01b3d96 --- /dev/null +++ b/scripts/training.slurm @@ -0,0 +1,18 @@ +#!/bin/bash +#SBATCH --job-name=eso4clima +#SBATCH --partition=compute +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=128 +#SBATCH --time=04:00:00 +#SBATCH --account=bd0854 +#SBATCH --output=eso4clima_%j.out + +source /home/b/b383704/eso4clima/ClimaNet/.venv/bin/activate + +# Run the training script +python -u /home/b/b383704/eso4clima/train_twoyears/example_training.py + +echo "==== Slurm accounting summary ${SLURM_JOB_ID} ====" +sstat --allsteps -j "$SLURM_JOB_ID" \ + --format=JobID,NTasks,AveCPU,AveRSS,MaxRSS,MaxVMSize,TresUsageInAve,TresUsageInMax \ + --parsable2 \ No newline at end of file