diff --git a/.github/workflows/test_urls.yml b/.github/workflows/test_urls.yml index 2c5616a..f065462 100644 --- a/.github/workflows/test_urls.yml +++ b/.github/workflows/test_urls.yml @@ -22,12 +22,12 @@ jobs: ping_urls: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.12 - name: Install testing dependencies run: | diff --git a/src/openmc_data/__init__.py b/src/openmc_data/__init__.py index 157f1c7..07081b6 100644 --- a/src/openmc_data/__init__.py +++ b/src/openmc_data/__init__.py @@ -17,9 +17,11 @@ __all__ = ["__version__"] from .utils import ( + ProgressTracker, calculate_download_size, download, extract, + format_duration, get_file_types, process_neutron, process_thermal, diff --git a/src/openmc_data/convert/convert_endf.py b/src/openmc_data/convert/convert_endf.py index 17817a7..685c109 100755 --- a/src/openmc_data/convert/convert_endf.py +++ b/src/openmc_data/convert/convert_endf.py @@ -11,7 +11,7 @@ from pathlib import Path import openmc.data -from openmc_data import download, extract, all_release_details, get_file_types, calculate_download_size +from openmc_data import download, extract, all_release_details, get_file_types, calculate_download_size, ProgressTracker # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -155,8 +155,10 @@ def main(): (openmc.data.IncidentNeutron, ace_files_dir.rglob(details["ace_files"])), (openmc.data.ThermalScattering, ace_files_dir.rglob(details["sab_files"])), ]: - for path in sorted(files): - print(f"Converting: {path.name}") + files = sorted(files) + tracker = ProgressTracker(len(files)) + for path in files: + tracker.starting(path.name) data = cls.from_ace(path) # Export HDF5 file h5_file = args.destination.joinpath(particle, data.name + ".h5") @@ -166,11 +168,13 @@ def main(): library.register_file(h5_file) elif particle == "photon": - for photo_path, atom_path in zip( + photon_pairs = list(zip( sorted(endf_files_dir.glob(details["photo_files"])), sorted(endf_files_dir.glob(details["atom_files"])) - ): + )) + tracker = ProgressTracker(len(photon_pairs)) + for photo_path, atom_path in photon_pairs: # Generate instance of IncidentPhoton - print("Converting:", photo_path.name, atom_path.name) + tracker.starting(f"{photo_path.name} {atom_path.name}") data = openmc.data.IncidentPhoton.from_endf(photo_path, atom_path) # Export HDF5 file diff --git a/src/openmc_data/convert/convert_fendl.py b/src/openmc_data/convert/convert_fendl.py index 6043877..682cd83 100755 --- a/src/openmc_data/convert/convert_fendl.py +++ b/src/openmc_data/convert/convert_fendl.py @@ -14,7 +14,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, all_release_details, calculate_download_size, get_file_types +from openmc_data import download, all_release_details, calculate_download_size, get_file_types, ProgressTracker class CustomFormatter( @@ -228,7 +228,9 @@ def main(): if not f.name.endswith("_") and not f.name.endswith(".xsd") ] - for filename in sorted(neutron_files): + neutron_files = sorted(neutron_files) + tracker = ProgressTracker(len(neutron_files)) + for filename in neutron_files: # Handling for special cases if filename.name in special_cases: ret = special_cases[filename.name](filename) @@ -237,7 +239,7 @@ def main(): if ret["skip_file"]: continue - print(f"Converting: {filename}") + tracker.starting(filename) data = openmc.data.IncidentNeutron.from_ace(filename) # Export HDF5 file @@ -258,7 +260,9 @@ def main(): release_details[args.release]["photon"][file_types[particle]]["endf_files"] ) - for photo_path in sorted(photon_files): + photon_files = sorted(photon_files) + photon_tracker = ProgressTracker(len(photon_files)) + for photo_path in photon_files: # Check if file requires special handling if photo_path.name in special_cases: @@ -268,7 +272,7 @@ def main(): if ret["skip_file"]: continue - print(f"Converting: {photo_path}") + photon_tracker.starting(photo_path) evaluations = openmc.data.endf.get_evaluations(photo_path) for ev in evaluations: # Export HDF5 file diff --git a/src/openmc_data/convert/convert_jeff32.py b/src/openmc_data/convert/convert_jeff32.py index dee634e..10c6386 100755 --- a/src/openmc_data/convert/convert_jeff32.py +++ b/src/openmc_data/convert/convert_jeff32.py @@ -14,7 +14,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, calculate_download_size, all_release_details, get_file_types +from openmc_data import download, calculate_download_size, all_release_details, get_file_types, ProgressTracker class CustomFormatter( @@ -196,9 +196,11 @@ def main(): library = openmc.data.DataLibrary() - for name, filenames in sorted(tables.items()): + neutron_tables = sorted(tables.items()) + tracker = ProgressTracker(len(neutron_tables)) + for name, filenames in neutron_tables: # Convert first temperature for the table - print("Converting: " + str(filenames[0])) + tracker.starting(filenames[0]) data = openmc.data.IncidentNeutron.from_ace(filenames[0]) # For each higher temperature, add cross sections to the existing table @@ -228,9 +230,11 @@ def main(): for name, filenames in sorted(tables.items()): filenames.sort(key=lambda x: int(x.name.split("-")[1].split(".")[0])) - for name, filenames in sorted(tables.items()): + sab_tables = sorted(tables.items()) + sab_tracker = ProgressTracker(len(sab_tables)) + for name, filenames in sab_tables: # Convert first temperature for the table - print(f"Converting: {filenames[0]}") + sab_tracker.starting(filenames[0]) # Take numbers out of table name, e.g. lw10.32t -> lw.32t table = openmc.data.ace.get_table(filenames[0]) diff --git a/src/openmc_data/convert/convert_jeff33.py b/src/openmc_data/convert/convert_jeff33.py index 74b05d8..05041a3 100755 --- a/src/openmc_data/convert/convert_jeff33.py +++ b/src/openmc_data/convert/convert_jeff33.py @@ -13,7 +13,7 @@ import openmc.data -from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types +from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types, ProgressTracker # Make sure Python version is sufficient @@ -153,8 +153,10 @@ def main(): lib = openmc.data.DataLibrary() - for p in sorted(ace_files_dir.glob(details['neutron_files']), key=key): - print(f"Converting: {p}") + neutron_paths = sorted(ace_files_dir.glob(details['neutron_files']), key=key) + tracker = ProgressTracker(len(neutron_paths)) + for p in neutron_paths: + tracker.starting(p) temp, z, a, m = key(p) data = openmc.data.IncidentNeutron.from_ace(p) @@ -203,12 +205,13 @@ def thermal_temp(p): thermal_dir = ace_files_dir / details["thermal_files"] + thermal_tracker = ProgressTracker(len(thermal_mats)) for mat in thermal_mats: for i, p in enumerate( sorted(thermal_dir.glob(f"{mat}*.ace"), key=thermal_temp) ): if i == 0: - print(f"Converting: {p}") + thermal_tracker.starting(p) data = openmc.data.ThermalScattering.from_ace(p) else: print(f"Adding temperature: {p}") diff --git a/src/openmc_data/convert/convert_lib80x.py b/src/openmc_data/convert/convert_lib80x.py index 36be10b..37a23e3 100755 --- a/src/openmc_data/convert/convert_lib80x.py +++ b/src/openmc_data/convert/convert_lib80x.py @@ -13,6 +13,8 @@ import openmc.data +from openmc_data import ProgressTracker + # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -76,10 +78,12 @@ def main(): library = openmc.data.DataLibrary() - for name, paths in sorted(tables.items()): + sorted_tables = sorted(tables.items()) + tracker = ProgressTracker(len(sorted_tables)) + for name, paths in sorted_tables: # Convert first temperature for the table p = paths[0] - print(f'Converting: {p}') + tracker.starting(p) if p.name.endswith('t'): data = openmc.data.ThermalScattering.from_ace(p) else: diff --git a/src/openmc_data/convert/convert_mcnp70.py b/src/openmc_data/convert/convert_mcnp70.py index 1006693..bd84ce3 100755 --- a/src/openmc_data/convert/convert_mcnp70.py +++ b/src/openmc_data/convert/convert_mcnp70.py @@ -15,6 +15,8 @@ import openmc.data +from openmc_data import ProgressTracker + # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -68,9 +70,11 @@ def main(): zaid, xs = table.name.split('.') tables[zaid].append(table) - for zaid, tables in sorted(tables.items()): + neutron_groups = sorted(tables.items()) + tracker = ProgressTracker(len(neutron_groups)) + for zaid, tables in neutron_groups: # Convert first temperature for the table - print(f'Converting: {tables[0].name}') + tracker.starting(tables[0].name) data = openmc.data.IncidentNeutron.from_ace(tables[0], 'mcnp') # For each higher temperature, add cross sections to the existing table @@ -97,9 +101,11 @@ def main(): name, xs = table.name.split('.') tables[name].append(table) - for zaid, tables in sorted(tables.items()): + sab_groups = sorted(tables.items()) + tracker = ProgressTracker(len(sab_groups)) + for zaid, tables in sab_groups: # Convert first temperature for the table - print(f'Converting: {tables[0].name}') + tracker.starting(tables[0].name) data = openmc.data.ThermalScattering.from_ace(tables[0]) # For each higher temperature, add cross sections to the existing table @@ -119,9 +125,10 @@ def main(): if args.photon is not None: lib = openmc.data.ace.Library(args.photon) + tracker = ProgressTracker(len(lib.tables)) for table in lib.tables: # Convert first temperature for the table - print(f'Converting: {table.name}') + tracker.starting(table.name) data = openmc.data.IncidentPhoton.from_ace(table) # Export HDF5 file diff --git a/src/openmc_data/convert/convert_mcnp71.py b/src/openmc_data/convert/convert_mcnp71.py index bcef339..2f393c1 100755 --- a/src/openmc_data/convert/convert_mcnp71.py +++ b/src/openmc_data/convert/convert_mcnp71.py @@ -15,6 +15,8 @@ import openmc.data +from openmc_data import ProgressTracker + # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -87,10 +89,12 @@ def main(): library = openmc.data.DataLibrary() - for name, paths in sorted(tables.items()): + sorted_tables = sorted(tables.items()) + tracker = ProgressTracker(len(sorted_tables)) + for name, paths in sorted_tables: # Convert first temperature for the table p = paths[0] - print(f'Converting: {p}') + tracker.starting(p) if p.name.endswith('t'): data = openmc.data.ThermalScattering.from_ace(p) else: @@ -116,9 +120,10 @@ def main(): if args.photon is not None: lib = openmc.data.ace.Library(args.photon) + tracker = ProgressTracker(len(lib.tables)) for table in lib.tables: # Convert first temperature for the table - print(f'Converting: {table.name}') + tracker.starting(table.name) data = openmc.data.IncidentPhoton.from_ace(table) # Export HDF5 file diff --git a/src/openmc_data/convert/convert_tendl.py b/src/openmc_data/convert/convert_tendl.py index 3b3ac34..c0e3f56 100755 --- a/src/openmc_data/convert/convert_tendl.py +++ b/src/openmc_data/convert/convert_tendl.py @@ -11,7 +11,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types +from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types, ProgressTracker # Make sure Python version is sufficient @@ -139,7 +139,9 @@ def main(): library = openmc.data.DataLibrary() - for filename in sorted(neutron_files): + neutron_files = sorted(neutron_files) + tracker = ProgressTracker(len(neutron_files)) + for filename in neutron_files: # this is a fix for the TENDL-2017 release where the B10 ACE file which has an error on one of the values if args.release == "2017" and filename.name == "B010": @@ -150,7 +152,7 @@ def main(): text = "".join(text[:423]) + "86896" + "".join(text[428:]) open(filename, "w").write(text) - print(f"Converting: {filename}") + tracker.starting(filename) data = openmc.data.IncidentNeutron.from_ace(filename) # Export HDF5 file diff --git a/src/openmc_data/generate/generate_cendl.py b/src/openmc_data/generate/generate_cendl.py index 7581e4f..a0ff41f 100755 --- a/src/openmc_data/generate/generate_cendl.py +++ b/src/openmc_data/generate/generate_cendl.py @@ -11,7 +11,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, extract, process_neutron, state_download_size +from openmc_data import download, extract, process_neutron, state_download_size, ProgressTracker class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, @@ -123,7 +123,9 @@ def main(): with Pool() as pool: results = [] - for filename in sorted(neutron_files): + neutron_files = sorted(neutron_files) + tracker = ProgressTracker(len(neutron_files), verb='Processed') + for filename in neutron_files: # this is a fix for the CENDL 3.1 release where the # 22-Ti-047.C31 and 5-B-010.C31 files contain non-ASCII characters @@ -137,7 +139,11 @@ def main(): open(filename, 'w').write('\r\n'.join(text)) func_args = (filename, args.destination, args.libver) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(filename.name), + error_callback=tracker.callback(filename.name), + ) results.append(r) for r in results: diff --git a/src/openmc_data/generate/generate_endf.py b/src/openmc_data/generate/generate_endf.py index a053acf..2b2406d 100755 --- a/src/openmc_data/generate/generate_endf.py +++ b/src/openmc_data/generate/generate_endf.py @@ -16,7 +16,7 @@ from shutil import rmtree, copy, copyfileobj import openmc.data -from openmc_data import download, process_neutron, process_thermal, state_download_size +from openmc_data import download, process_neutron, process_thermal, state_download_size, ProgressTracker # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -442,21 +442,29 @@ def main(): with Pool() as pool: details = release_details[args.release][particle] results = [] - for filename in details['endf_files']: - - # Skip neutron evaluation that fails the processing stage - if filename.name == 'n-000_n_001.endf': - continue - + # Skip neutron evaluation that fails the processing stage + endf_files = [f for f in details['endf_files'] + if f.name != 'n-000_n_001.endf'] + tracker = ProgressTracker( + len(endf_files) + len(details['sab_files']), verb='Processed') + for filename in endf_files: func_args = (filename, args.destination / particle, args.libver, args.temperatures) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(filename.name), + error_callback=tracker.callback(filename.name), + ) results.append(r) for path_neutron, path_thermal in details['sab_files']: func_args = (path_neutron, path_thermal, args.destination / particle, args.libver) - r = pool.apply_async(process_thermal, func_args) + r = pool.apply_async( + process_thermal, func_args, + callback=tracker.callback(path_thermal.name), + error_callback=tracker.callback(path_thermal.name), + ) results.append(r) for r in results: @@ -472,10 +480,12 @@ def main(): if 'photon' in args.particles: particle = 'photon' details = release_details[args.release][particle] - for photo_path, atom_path in zip(sorted(details['photo_files']), - sorted(details['atom_files'])): + photon_pairs = list(zip(sorted(details['photo_files']), + sorted(details['atom_files']))) + tracker = ProgressTracker(len(photon_pairs)) + for photo_path, atom_path in photon_pairs: # Generate instance of IncidentPhoton - print('Converting:', photo_path.name, atom_path.name) + tracker.starting(f'{photo_path.name} {atom_path.name}') data = openmc.data.IncidentPhoton.from_endf(photo_path, atom_path) # Export HDF5 file diff --git a/src/openmc_data/generate/generate_fendl.py b/src/openmc_data/generate/generate_fendl.py index a9cc317..4945d33 100644 --- a/src/openmc_data/generate/generate_fendl.py +++ b/src/openmc_data/generate/generate_fendl.py @@ -12,7 +12,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, extract, process_neutron, state_download_size, all_release_details +from openmc_data import download, extract, process_neutron, state_download_size, all_release_details, ProgressTracker class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, @@ -104,9 +104,15 @@ def main(): with Pool() as pool: results = [] - for filename in sorted(neutron_files): + neutron_files = sorted(neutron_files) + tracker = ProgressTracker(len(neutron_files), verb='Processed') + for filename in neutron_files: func_args = (filename, args.destination, args.libver) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(filename.name), + error_callback=tracker.callback(filename.name), + ) results.append(r) for r in results: diff --git a/src/openmc_data/generate/generate_jeff33.py b/src/openmc_data/generate/generate_jeff33.py index 43c4e8a..5828c5c 100755 --- a/src/openmc_data/generate/generate_jeff33.py +++ b/src/openmc_data/generate/generate_jeff33.py @@ -20,7 +20,7 @@ import openmc.data -from openmc_data import download, process_neutron, process_thermal +from openmc_data import download, process_neutron, process_thermal, ProgressTracker # Make sure Python version is sufficient assert sys.version_info >= (3, 6), "Python 3.6+ is required" @@ -162,15 +162,25 @@ def main(): # PROCESS INCIDENT NEUTRON AND THERMAL SCATTERING DATA IN PARALLEL with Pool() as pool: - neutron_paths = neutron_dir.glob('*.jeff33') + neutron_paths = sorted(neutron_dir.glob('*.jeff33')) results = [] + tracker = ProgressTracker( + len(neutron_paths) + len(thermal_paths), verb='Processed') for p in neutron_paths: func_args = (p, destination, args.libver, args.temperatures) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(p.name), + error_callback=tracker.callback(p.name), + ) results.append(r) for p_neut, p_therm in thermal_paths: func_args = (p_neut, p_therm, destination, args.libver) - r = pool.apply_async(process_thermal, func_args) + r = pool.apply_async( + process_thermal, func_args, + callback=tracker.callback(p_therm.name), + error_callback=tracker.callback(p_therm.name), + ) results.append(r) for r in results: r.wait() diff --git a/src/openmc_data/generate/generate_jendl.py b/src/openmc_data/generate/generate_jendl.py index 2736830..e1a062c 100755 --- a/src/openmc_data/generate/generate_jendl.py +++ b/src/openmc_data/generate/generate_jendl.py @@ -12,7 +12,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, extract, process_neutron, state_download_size, all_release_details +from openmc_data import download, extract, process_neutron, state_download_size, all_release_details, ProgressTracker class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, @@ -111,9 +111,15 @@ def main(): with Pool() as pool: results = [] - for filename in sorted(neutron_files): + neutron_files = sorted(neutron_files) + tracker = ProgressTracker(len(neutron_files), verb='Processed') + for filename in neutron_files: func_args = (filename, args.destination, args.libver, args.temperatures) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(filename.name), + error_callback=tracker.callback(filename.name), + ) results.append(r) for r in results: diff --git a/src/openmc_data/generate/generate_tendl.py b/src/openmc_data/generate/generate_tendl.py index e122673..c7edf74 100644 --- a/src/openmc_data/generate/generate_tendl.py +++ b/src/openmc_data/generate/generate_tendl.py @@ -12,7 +12,7 @@ from urllib.parse import urljoin import openmc.data -from openmc_data import download, extract, process_neutron, state_download_size, all_release_details +from openmc_data import download, extract, process_neutron, state_download_size, all_release_details, ProgressTracker class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, @@ -141,9 +141,14 @@ def endf_to_h5_name(endf_filename): with Pool() as pool: results = [] + tracker = ProgressTracker(len(neutron_files_to_process), verb='Processed') for filename in neutron_files_to_process: func_args = (filename, args.destination, args.libver) - r = pool.apply_async(process_neutron, func_args) + r = pool.apply_async( + process_neutron, func_args, + callback=tracker.callback(filename.name), + error_callback=tracker.callback(filename.name), + ) results.append(r) for r in results: diff --git a/src/openmc_data/utils.py b/src/openmc_data/utils.py index d5684b3..44eebd9 100644 --- a/src/openmc_data/utils.py +++ b/src/openmc_data/utils.py @@ -1,6 +1,8 @@ import hashlib import shutil import tarfile +import threading +import time from typing import Iterable import warnings import zipfile @@ -18,6 +20,75 @@ _BLOCK_SIZE = 16384 +def format_duration(seconds): + """Format a number of seconds as a ``H:MM:SS`` string.""" + if seconds is None or seconds != seconds: # None or NaN + return '--:--:--' + seconds = int(round(seconds)) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f'{hours:d}:{minutes:02d}:{secs:02d}' + + +class ProgressTracker: + """Track progress through a fixed number of items and estimate the time + remaining. + + The estimate is a simple extrapolation from the average time taken per + completed item so far. Use :meth:`starting` for serial loops (call it just + before processing each item) and :meth:`callback` with a + ``multiprocessing.Pool`` (pass the returned function as the ``apply_async`` + callback so progress is reported as each item completes). + + Parameters + ---------- + total : int + Total number of items that will be processed. + verb : str + Word printed at the start of each progress line, e.g. ``'Converting'``. + """ + + def __init__(self, total, verb='Converting'): + self.total = max(int(total), 0) + self.verb = verb + self.count = 0 + self.start = time.monotonic() + self._lock = threading.Lock() + + def _format(self, position, completed, name): + elapsed = time.monotonic() - self.start + eta = elapsed / completed * (self.total - completed) if completed > 0 else None + pct = 100 * position / self.total if self.total else 100 + line = (f'{self.verb} [{position}/{self.total} {pct:3.0f}%] ' + f'elapsed {format_duration(elapsed)} ETA {format_duration(eta)}') + return f'{line}: {name}' if name else line + + def starting(self, name=''): + """Print a progress line for an item that is about to be processed. + + The ETA is based on the items completed before this one, which is the + appropriate estimate for a serial loop. + """ + with self._lock: + self.count += 1 + line = self._format(self.count, self.count - 1, name) + print(line, flush=True) + + def complete(self, name=''): + """Advance the counter and print a progress line for a finished item.""" + with self._lock: + self.count += 1 + line = self._format(self.count, self.count, name) + print(line, flush=True) + + def callback(self, name=''): + """Return a one-argument function suitable as an ``apply_async`` + ``callback`` / ``error_callback`` that reports completion of ``name``.""" + def _callback(_result=None): + self.complete(name) + return _callback + + def _require_openmc(): if openmc is None: raise ModuleNotFoundError(