Add progress counter and ETA to convert/generate script output

.github/workflows/test_urls.yml

@@ -22,12 +22,12 @@ jobs:
     ping_urls:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Install testing dependencies
         run: |

src/openmc_data/__init__.py

@@ -17,9 +17,11 @@
 __all__ = ["__version__"]
 
 from .utils import (
+    ProgressTracker,
     calculate_download_size,
     download,
     extract,
+    format_duration,
     get_file_types,
     process_neutron,
     process_thermal,

src/openmc_data/convert/convert_endf.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 
 import openmc.data
-from openmc_data import download, extract, all_release_details, get_file_types, calculate_download_size
+from openmc_data import download, extract, all_release_details, get_file_types, calculate_download_size, ProgressTracker
 
 # Make sure Python version is sufficient
 assert sys.version_info >= (3, 6), "Python 3.6+ is required"
@@ -155,8 +155,10 @@ def main():
                 (openmc.data.IncidentNeutron, ace_files_dir.rglob(details["ace_files"])),
                 (openmc.data.ThermalScattering, ace_files_dir.rglob(details["sab_files"])),
             ]:
-                for path in sorted(files):
-                    print(f"Converting: {path.name}")
+                files = sorted(files)
+                tracker = ProgressTracker(len(files))
+                for path in files:
+                    tracker.starting(path.name)
                     data = cls.from_ace(path)
                     # Export HDF5 file
                     h5_file = args.destination.joinpath(particle, data.name + ".h5")
@@ -166,11 +168,13 @@ def main():
                     library.register_file(h5_file)
 
         elif particle == "photon":
-            for photo_path, atom_path in zip(
+            photon_pairs = list(zip(
                 sorted(endf_files_dir.glob(details["photo_files"])), sorted(endf_files_dir.glob(details["atom_files"]))
-            ):
+            ))
+            tracker = ProgressTracker(len(photon_pairs))
+            for photo_path, atom_path in photon_pairs:
                 # Generate instance of IncidentPhoton
-                print("Converting:", photo_path.name, atom_path.name)
+                tracker.starting(f"{photo_path.name} {atom_path.name}")
                 data = openmc.data.IncidentPhoton.from_endf(photo_path, atom_path)
 
                 # Export HDF5 file

src/openmc_data/convert/convert_fendl.py

@@ -14,7 +14,7 @@
 from urllib.parse import urljoin
 
 import openmc.data
-from openmc_data import download, all_release_details, calculate_download_size, get_file_types
+from openmc_data import download, all_release_details, calculate_download_size, get_file_types, ProgressTracker
 
 
 class CustomFormatter(
@@ -228,7 +228,9 @@ def main():
                 if not f.name.endswith("_") and not f.name.endswith(".xsd")
             ]
 
-            for filename in sorted(neutron_files):
+            neutron_files = sorted(neutron_files)
+            tracker = ProgressTracker(len(neutron_files))
+            for filename in neutron_files:
                 # Handling for special cases
                 if filename.name in special_cases:
                     ret = special_cases[filename.name](filename)
@@ -237,7 +239,7 @@ def main():
                     if ret["skip_file"]:
                         continue
 
-                print(f"Converting: {filename}")
+                tracker.starting(filename)
                 data = openmc.data.IncidentNeutron.from_ace(filename)
 
                 # Export HDF5 file
@@ -258,7 +260,9 @@ def main():
                 release_details[args.release]["photon"][file_types[particle]]["endf_files"]
             )
 
-            for photo_path in sorted(photon_files):
+            photon_files = sorted(photon_files)
+            photon_tracker = ProgressTracker(len(photon_files))
+            for photo_path in photon_files:
 
                 # Check if file requires special handling
                 if photo_path.name in special_cases:
@@ -268,7 +272,7 @@ def main():
                     if ret["skip_file"]:
                         continue
 
-                print(f"Converting: {photo_path}")
+                photon_tracker.starting(photo_path)
                 evaluations = openmc.data.endf.get_evaluations(photo_path)
                 for ev in evaluations:
                     # Export HDF5 file

src/openmc_data/convert/convert_jeff32.py

@@ -14,7 +14,7 @@
 from urllib.parse import urljoin
 
 import openmc.data
-from openmc_data import download, calculate_download_size, all_release_details, get_file_types
+from openmc_data import download, calculate_download_size, all_release_details, get_file_types, ProgressTracker
 
 
 class CustomFormatter(
@@ -196,9 +196,11 @@ def main():
 
     library = openmc.data.DataLibrary()
 
-    for name, filenames in sorted(tables.items()):
+    neutron_tables = sorted(tables.items())
+    tracker = ProgressTracker(len(neutron_tables))
+    for name, filenames in neutron_tables:
         # Convert first temperature for the table
-        print("Converting: " + str(filenames[0]))
+        tracker.starting(filenames[0])
         data = openmc.data.IncidentNeutron.from_ace(filenames[0])
 
         # For each higher temperature, add cross sections to the existing table
@@ -228,9 +230,11 @@ def main():
     for name, filenames in sorted(tables.items()):
         filenames.sort(key=lambda x: int(x.name.split("-")[1].split(".")[0]))
 
-    for name, filenames in sorted(tables.items()):
+    sab_tables = sorted(tables.items())
+    sab_tracker = ProgressTracker(len(sab_tables))
+    for name, filenames in sab_tables:
         # Convert first temperature for the table
-        print(f"Converting: {filenames[0]}")
+        sab_tracker.starting(filenames[0])
 
         # Take numbers out of table name, e.g. lw10.32t -> lw.32t
         table = openmc.data.ace.get_table(filenames[0])

src/openmc_data/convert/convert_jeff33.py

@@ -13,7 +13,7 @@
 
 import openmc.data
 
-from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types
+from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types, ProgressTracker
 
 
 # Make sure Python version is sufficient
@@ -153,8 +153,10 @@ def main():
 
     lib = openmc.data.DataLibrary()
 
-    for p in sorted(ace_files_dir.glob(details['neutron_files']), key=key):
-        print(f"Converting: {p}")
+    neutron_paths = sorted(ace_files_dir.glob(details['neutron_files']), key=key)
+    tracker = ProgressTracker(len(neutron_paths))
+    for p in neutron_paths:
+        tracker.starting(p)
         temp, z, a, m = key(p)
 
         data = openmc.data.IncidentNeutron.from_ace(p)
@@ -203,12 +205,13 @@ def thermal_temp(p):
 
     thermal_dir = ace_files_dir / details["thermal_files"]
 
+    thermal_tracker = ProgressTracker(len(thermal_mats))
     for mat in thermal_mats:
         for i, p in enumerate(
             sorted(thermal_dir.glob(f"{mat}*.ace"), key=thermal_temp)
         ):
             if i == 0:
-                print(f"Converting: {p}")
+                thermal_tracker.starting(p)
                 data = openmc.data.ThermalScattering.from_ace(p)
             else:
                 print(f"Adding temperature: {p}")

src/openmc_data/convert/convert_lib80x.py

@@ -13,6 +13,8 @@
 
 import openmc.data
 
+from openmc_data import ProgressTracker
+
 
 # Make sure Python version is sufficient
 assert sys.version_info >= (3, 6), "Python 3.6+ is required"
@@ -76,10 +78,12 @@ def main():
 
     library = openmc.data.DataLibrary()
 
-    for name, paths in sorted(tables.items()):
+    sorted_tables = sorted(tables.items())
+    tracker = ProgressTracker(len(sorted_tables))
+    for name, paths in sorted_tables:
         # Convert first temperature for the table
         p = paths[0]
-        print(f'Converting: {p}')
+        tracker.starting(p)
         if p.name.endswith('t'):
             data = openmc.data.ThermalScattering.from_ace(p)
         else:

src/openmc_data/convert/convert_mcnp70.py

@@ -15,6 +15,8 @@
 
 import openmc.data
 
+from openmc_data import ProgressTracker
+
 
 # Make sure Python version is sufficient
 assert sys.version_info >= (3, 6), "Python 3.6+ is required"
@@ -68,9 +70,11 @@ def main():
             zaid, xs = table.name.split('.')
             tables[zaid].append(table)
 
-        for zaid, tables in sorted(tables.items()):
+        neutron_groups = sorted(tables.items())
+        tracker = ProgressTracker(len(neutron_groups))
+        for zaid, tables in neutron_groups:
             # Convert first temperature for the table
-            print(f'Converting: {tables[0].name}')
+            tracker.starting(tables[0].name)
             data = openmc.data.IncidentNeutron.from_ace(tables[0], 'mcnp')
 
             # For each higher temperature, add cross sections to the existing table
@@ -97,9 +101,11 @@ def main():
             name, xs = table.name.split('.')
             tables[name].append(table)
 
-        for zaid, tables in sorted(tables.items()):
+        sab_groups = sorted(tables.items())
+        tracker = ProgressTracker(len(sab_groups))
+        for zaid, tables in sab_groups:
             # Convert first temperature for the table
-            print(f'Converting: {tables[0].name}')
+            tracker.starting(tables[0].name)
             data = openmc.data.ThermalScattering.from_ace(tables[0])
 
             # For each higher temperature, add cross sections to the existing table
@@ -119,9 +125,10 @@ def main():
     if args.photon is not None:
         lib = openmc.data.ace.Library(args.photon)
 
+        tracker = ProgressTracker(len(lib.tables))
         for table in lib.tables:
             # Convert first temperature for the table
-            print(f'Converting: {table.name}')
+            tracker.starting(table.name)
             data = openmc.data.IncidentPhoton.from_ace(table)
 
             # Export HDF5 file

src/openmc_data/convert/convert_mcnp71.py

@@ -15,6 +15,8 @@
 
 import openmc.data
 
+from openmc_data import ProgressTracker
+
 
 # Make sure Python version is sufficient
 assert sys.version_info >= (3, 6), "Python 3.6+ is required"
@@ -87,10 +89,12 @@ def main():
 
     library = openmc.data.DataLibrary()
 
-    for name, paths in sorted(tables.items()):
+    sorted_tables = sorted(tables.items())
+    tracker = ProgressTracker(len(sorted_tables))
+    for name, paths in sorted_tables:
         # Convert first temperature for the table
         p = paths[0]
-        print(f'Converting: {p}')
+        tracker.starting(p)
         if p.name.endswith('t'):
             data = openmc.data.ThermalScattering.from_ace(p)
         else:
@@ -116,9 +120,10 @@ def main():
     if args.photon is not None:
         lib = openmc.data.ace.Library(args.photon)
 
+        tracker = ProgressTracker(len(lib.tables))
         for table in lib.tables:
             # Convert first temperature for the table
-            print(f'Converting: {table.name}')
+            tracker.starting(table.name)
             data = openmc.data.IncidentPhoton.from_ace(table)
 
             # Export HDF5 file

src/openmc_data/convert/convert_tendl.py

@@ -11,7 +11,7 @@
 from urllib.parse import urljoin
 
 import openmc.data
-from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types
+from openmc_data import download, extract, calculate_download_size, all_release_details, get_file_types, ProgressTracker
 
 
 # Make sure Python version is sufficient
@@ -139,7 +139,9 @@ def main():
 
     library = openmc.data.DataLibrary()
 
-    for filename in sorted(neutron_files):
+    neutron_files = sorted(neutron_files)
+    tracker = ProgressTracker(len(neutron_files))
+    for filename in neutron_files:
 
         # this is a fix for the TENDL-2017 release where the B10 ACE file which has an error on one of the values
         if args.release == "2017" and filename.name == "B010":
@@ -150,7 +152,7 @@ def main():
                 text = "".join(text[:423]) + "86896" + "".join(text[428:])
                 open(filename, "w").write(text)
 
-        print(f"Converting: {filename}")
+        tracker.starting(filename)
         data = openmc.data.IncidentNeutron.from_ace(filename)
 
         # Export HDF5 file

src/openmc_data/generate/generate_cendl.py

@@ -11,7 +11,7 @@
 from urllib.parse import urljoin
 
 import openmc.data
-from openmc_data import download, extract, process_neutron, state_download_size
+from openmc_data import download, extract, process_neutron, state_download_size, ProgressTracker
 
 
 class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
@@ -123,7 +123,9 @@ def main():
 
     with Pool() as pool:
         results = []
-        for filename in sorted(neutron_files):
+        neutron_files = sorted(neutron_files)
+        tracker = ProgressTracker(len(neutron_files), verb='Processed')
+        for filename in neutron_files:
 
             # this is a fix for the CENDL 3.1 release where the
             # 22-Ti-047.C31 and 5-B-010.C31 files contain non-ASCII characters
@@ -137,7 +139,11 @@ def main():
                 open(filename, 'w').write('\r\n'.join(text))
 
             func_args = (filename, args.destination, args.libver)
-            r = pool.apply_async(process_neutron, func_args)
+            r = pool.apply_async(
+                process_neutron, func_args,
+                callback=tracker.callback(filename.name),
+                error_callback=tracker.callback(filename.name),
+            )
             results.append(r)
 
         for r in results: