Modified data loading for HXN scans using PandABox

.github/workflows/black.yml

@@ -13,7 +13,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.11
+          python-version: 3.13
       - name: Install Dependencies
         run: |
           # These packages are installed in the base environment but may be older

.github/workflows/docs.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.13]
       fail-fast: false
 
     steps:

.github/workflows/docs_publish.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.13]
       fail-fast: false
 
     steps:

.github/workflows/testing.yml

@@ -17,8 +17,8 @@ jobs:
     strategy:
       matrix:
         host-os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.10", "3.11"]
-        numpy-version: ["1.26"]
+        python-version: ["3.12", "3.13"]
+        numpy-version: ["2.4.0"]
         pyqt-version: ["5.15"]
         include:
           - host-os: "ubuntu-latest"

pyxrf/core/tests/test_map_processing.py

@@ -51,6 +51,7 @@ def test_dask_client_create(tmpdir):
     client = dask_client_create(n_workers=n_workers_requested)
     n_workers = len(client.scheduler_info()["workers"])
     assert n_workers == n_workers_requested, "The number of workers was set incorrectly"
+
     client.close()
 
     assert not os.path.exists(dask_worker_space_path), "Temporary directory was created in the current directory"

pyxrf/core/tests/test_quant_analysis.py

@@ -347,9 +347,11 @@ def test_get_quant_fluor_data_dict():
         ), "Generated object contains emission lines that are different from expected"
 
         mass_sum = sum([_["density"] for _ in quant_fluor_data_dict["element_lines"].values()])
-        assert (
-            mass_sum == mass_sum_expected
-        ), "The total mass (density) of the components is different from expected"
+        npt.assert_almost_equal(
+            mass_sum,
+            mass_sum_expected,
+            err_msg="The total mass (density) of the components is different from expected",
+        )
 
 
 def gen_xrf_map_dict(nx=10, ny=5, elines=["S_K", "Au_M", "Fe_K"]):

pyxrf/core/tests/test_yaml_param_files.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import jsonschema
 import numpy as np
@@ -135,13 +136,21 @@ def _generate_sample_docstring(param_dict, include_section_titles=True):
         d_str.append("    -------")
 
     d_str.extend([""] * n_empty_lines_after)
-
     d_str = "\n".join(d_str)  # Convert the list to a single string
 
+    # Remove initial 4 spaces from all lines to mimick behavior of func.__doc__ in Python 3.13
+    # and later (in Python 3.13, the initial spaces are removed automatically,
+    # but in earlier versions they are not)
+    is_py313 = sys.version_info >= (3, 13)
+    if is_py313:
+        d_str = d_str.split("\n")
+        d_str = [_[4:] if len(_) > 4 else "" for _ in d_str]
+        d_str = "\n".join(d_str)
+
     return d_str, parameters
 
 
-def test_parse_docstring_parameters():
+def test_parse_docstring_parameters_01():
     # Simple test for the successfully parsed docstring. It seems sufficient, since all error cases are trivial.
 
     param_dict = _generate_parameter_set()
@@ -164,7 +173,7 @@ def test_parse_docstring_parameters():
     # Check for exception if the section titles are required, but don't exist
     param_dict = _generate_parameter_set()
     d_str, parameters = _generate_sample_docstring(param_dict, include_section_titles=False)
-    with pytest.raises(AssertionError, match="'Parameters' or 'Return' statement was not found in the docstring"):
+    with pytest.raises(AssertionError, match="'Parameters' or 'Returns' statement was not found in the docstring"):
         _parse_docstring_parameters(d_str, search_param_section=True)
 
 

pyxrf/core/yaml_param_files.py

@@ -1,5 +1,6 @@
 import os
 import re
+import sys
 
 import yaml
 
@@ -51,6 +52,11 @@ def _parse_docstring_parameters(doc_string, search_param_section=True):
 
     str_list = doc_string.split("\n")
 
+    is_py313 = sys.version_info >= (3, 13)
+    if is_py313:
+        # Add initial 4 spaces to all lines
+        str_list = [f"    {_}" for _ in str_list]
+
     # Remove all spaces at the end of the strings (the should be no spaces there, but still)
     str_list = [s.rstrip() for s in str_list]
 
@@ -70,14 +76,15 @@ def _parse_docstring_parameters(doc_string, search_param_section=True):
 
     assert (n_first is not None) or (
         n_last is not None
-    ), "Incorrect docstring format: 'Parameters' or 'Return' statement was not found in the docstring"
+    ), "Incorrect docstring format: 'Parameters' or 'Returns' statement was not found in the docstring"
 
     # The list of strings contains parameter descriptions
     str_list = str_list[n_first : n_last + 1]
     # Each line must start with 4 spaces or be empty. Verify this
     assert all(
         [(not s) or re.search(r"^    ", s) for s in str_list]
     ), "Incorrect docstring format: parameter descriptions should be indented by at least FOUR spaces"
+
     # Now remove the spaces from nonempty lines
     str_list = [s[4:] if s else s for s in str_list]
 
@@ -99,7 +106,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True):
     #   The fist line of the description is actually the
     assert all(
         [len(s) > 1 for s in param_descriptions]
-    ), "Incomplete docstring: some parameters have not descriptions"
+    ), "Incomplete docstring: some parameters have no descriptions"
 
     params = list(zip(param_names, param_descriptions))
 

pyxrf/db_config/hxn_db_config.py

@@ -3,11 +3,17 @@
 except ModuleNotFoundError:
     from databroker import Broker
 
-from hxntools.handlers.timepix import TimepixHDF5Handler
-from hxntools.handlers.xspress3 import Xspress3HDF5Handler
-
 db = Broker.named("hxn")
+
+from hxntools.handlers import register  # noqa: E402
+
+register(db)
+
+# from hxntools.handlers.xspress3 import Xspress3HDF5Handler
+# from hxntools.handlers.timepix import TimepixHDF5Handler
+#
+# db = Broker.named("hxn")
 # db_analysis = Broker.named('hxn_analysis')
 
-db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True)
-db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True)
+# db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True)
+# db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True)

pyxrf/model/fileio.py

@@ -165,6 +165,8 @@ def _get_pyxrf_version_str(self):
         """
 
         # Determine the current version of PyXRF
+        global pyxrf_version  # noqa: F824
+
         pyxrf_version_str = pyxrf_version
         if pyxrf_version_str[0].lower() != "v":
             pyxrf_version_str = f"v{pyxrf_version_str}"

pyxrf/model/load_data_from_db.py

@@ -820,13 +820,19 @@ def map_data2D_hxn(
     data_output = []
 
     start_doc = hdr["start"]
-    logger.info("Plan type: '%s'", start_doc["plan_type"])
+
+    if "scan" in start_doc:
+        # print(" panda scan ")
+        plan_type = start_doc["scan"]["type"]
+
+    else:
+        plan_type = start_doc["plan_type"]
+
+    logger.info("Plan type: '%s'", plan_type)
 
     # Exclude certain types of plans based on data from the start document
-    if isinstance(skip_scan_types, (list, tuple)) and (start_doc["plan_type"] in skip_scan_types):
-        raise RuntimeError(
-            f"Failed to load the scan: plan type {start_doc['plan_type']!r} is in the list of skipped types"
-        )
+    if isinstance(skip_scan_types, (list, tuple)) and (plan_type in skip_scan_types):
+        raise RuntimeError(f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types")
 
     # The dictionary holding scan metadata
     mdata = _extract_metadata_from_header(hdr)
@@ -920,11 +926,17 @@ def map_data2D_hxn(
     else:
         raise ValueError(f"Invalid data shape: {datashape}. Must be a list with 1 or 2 elements.")
 
+    logger.info(f"Data shape: {datashape}.")
+
     # -----------------------------------------------------------------------------------------------
     # Determine fast axis and slow axis
     fast_axis, slow_axis, fast_axis_index = start_doc.get("fast_axis", None), None, None
     motors = start_doc.get("motors", None)
-    if motors and isinstance(motors, (list, tuple)) and len(motors) == 2:
+    if motors and isinstance(motors, (list, tuple)) and len(motors) == 1:
+        fast_axis = fast_axis if fast_axis else motors[0]
+        fast_axis_index = motors.index(fast_axis, 0)
+
+    elif motors and isinstance(motors, (list, tuple)) and len(motors) == 2:
         fast_axis = fast_axis if fast_axis else motors[0]
         fast_axis_index = motors.index(fast_axis, 0)
         slow_axis_index = 0 if (fast_axis_index == 1) else 1
@@ -940,14 +952,24 @@ def map_data2D_hxn(
     # -----------------------------------------------------------------------------------------------
     # Reconstruct scan input
     try:
-        plan_args = start_doc["plan_args"]
-        # px_motor = plan_args["motor1"]
-        px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"]
-        # py_motor = plan_args["motor2"]
-        py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"]
-        dwell_time = plan_args["exposure_time"]
-        param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
-        mdata["param_input"] = param_input
+        if "plan_args" in start_doc:  # dscan and fly1d/fly2d scan
+            plan_args = start_doc["plan_args"]
+            # px_motor = plan_args["motor1"]
+            px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"]
+            # py_motor = plan_args["motor2"]
+            py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"]
+            dwell_time = plan_args["exposure_time"]
+            param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
+            mdata["param_input"] = param_input
+        elif "scan" in start_doc:  # fly1dpd and fly2dpd scan
+            scan_input = start_doc["scan"]["scan_input"]
+            px_start, px_end, px_step = scan_input[0:3]
+            py_start, py_end, py_step = scan_input[3:6]
+            dwell_time = start_doc["scan"]["dwell"]
+            param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
+            mdata["param_input"] = param_input
+        else:
+            raise Exception("Unknown scan plan type")
     except Exception as ex:
         logger.warning(
             "Failed to reconstruct scan input: %s. Scan input is not saved as part of metadata to HDF5 file",
@@ -966,6 +988,7 @@ def map_data2D_hxn(
 
     keylist = hdr.descriptors[0].data_keys.keys()
     det_list = [v for v in keylist if "xspress3" in v]  # find xspress3 det with key word matching
+    det_list = [v for v in det_list if len(v) == 12]  # added to filter out other rois added by user
 
     scaler_list_all = config_data["scaler_list"]
 
@@ -978,7 +1001,7 @@ def map_data2D_hxn(
     if isinstance(db, databroker._core.Broker):
         fields = None
 
-    data = hdr.table(fields=fields, fill=True)
+    data = hdr.table(fields=fields, fill=False)  # HXN data is stored in h5 files, load them later in map_data2D.
 
     # This is for the case of 'dcan' (1D), where the slow axis positions are not saved
     if (slow_axis not in data) and (fast_axis in data):
@@ -994,6 +1017,7 @@ def map_data2D_hxn(
         fly_type=fly_type,
         subscan_dims=subscan_dims,
         spectrum_len=4096,
+        hdr=hdr,
     )
 
     # Transform coordinates for the fast axis if necessary:
@@ -3567,6 +3591,7 @@ def write_db_to_hdf(
         # position data
         dataGrp = f.create_group(interpath + "/positions")
 
+        # scanning position data
         pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)
 
         for i in range(len(pos_names)):
@@ -3593,6 +3618,7 @@ def write_db_to_hdf(
         # scaler data
         dataGrp = f.create_group(interpath + "/scalers")
 
+        # scaler data
         scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)
 
         if fly_type in ("pyramid",):
@@ -3767,6 +3793,7 @@ def map_data2D(
     fly_type=None,
     subscan_dims=None,
     spectrum_len=4096,
+    hdr=None,
 ):
     """
     Data is obained from databroker. Transfer items from data to a dictionary of
@@ -3806,7 +3833,10 @@ def map_data2D(
         if c_name in data:
             detname = "det" + str(n + 1)
             logger.info("read data from %s" % c_name)
-            channel_data = data[c_name]
+            if db.name == "hxn":
+                channel_data = np.squeeze(np.array(list(hdr.data(c_name))))
+            else:
+                channel_data = data[c_name]
 
             # new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans
             new_v_shape = len(channel_data) // datashape[1]
@@ -3839,8 +3869,21 @@ def map_data2D(
                 sum_data += new_data
     data_output["det_sum"] = sum_data
 
-    # scanning position data
-    pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)
+    if db.name == "hxn":
+        pos_names = pos_list
+        pos_data = np.zeros([datashape[0], datashape[1], len(pos_list)])
+        from hxntools.scan_info import get_scan_positions
+
+        pos = get_scan_positions(hdr)
+        if isinstance(pos, tuple):
+            for i in range(len(pos)):
+                pos_data[:, :, i] = pos[i].reshape((datashape[0], datashape[1]))
+        else:
+            pos_data[:, :, 0] = pos.reshape((datashape[0], datashape[1]))
+    else:
+        # scanning position data
+        pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)
+
     for i in range(len(pos_names)):
         if "x" in pos_names[i]:
             pos_names[i] = "x_pos"
@@ -3871,8 +3914,15 @@ def map_data2D(
     data_output["pos_names"] = pos_names
     data_output["pos_data"] = new_p
 
-    # scaler data
-    scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)
+    if db.name == "hxn":
+        scaler_names = scaler_list
+        scaler_data = np.zeros([datashape[0], datashape[1], len(scaler_list)])
+        for i in range(len(scaler_list)):
+            scaler_data[:, :, i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1]))
+    else:
+        # scaler data
+        scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)
+
     if fly_type in ("pyramid",):
         scaler_data = flip_data(scaler_data, subscan_dims=subscan_dims)
 

pyxrf/model/tests/test_hdf5_file_operations.py

@@ -23,7 +23,7 @@ def _prepare_raw_dataset(N=5, M=10, K=4096):
     pos_data = np.zeros(shape=[2, N, M])
     pos_data[0, :, :] = np.broadcast_to(np.linspace(1, 1 + (M - 1) * 0.1, M), shape=[N, M])
     pos_data[1, :, :] = np.broadcast_to(
-        np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), newshape=[N, 1]), shape=[N, M]
+        np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), shape=[N, 1]), shape=[N, M]
     )
 
     data = {