From 34598a95459a4da65a0a43e3e75ff993719bd0c7 Mon Sep 17 00:00:00 2001 From: zgao-bnl Date: Wed, 23 Jul 2025 13:16:57 -0400 Subject: [PATCH 01/10] Modified data loading for HXN scans using PandABox --- pyxrf/db_config/hxn_db_config.py | 14 +++-- pyxrf/model/load_data_from_db.py | 95 +++++++++++++++++++++++++------- 2 files changed, 84 insertions(+), 25 deletions(-) diff --git a/pyxrf/db_config/hxn_db_config.py b/pyxrf/db_config/hxn_db_config.py index fc919122..f891efce 100644 --- a/pyxrf/db_config/hxn_db_config.py +++ b/pyxrf/db_config/hxn_db_config.py @@ -3,11 +3,15 @@ except ModuleNotFoundError: from databroker import Broker -from hxntools.handlers.timepix import TimepixHDF5Handler -from hxntools.handlers.xspress3 import Xspress3HDF5Handler - db = Broker.named("hxn") +from hxntools.handlers import register +register(db) + +#from hxntools.handlers.xspress3 import Xspress3HDF5Handler +#from hxntools.handlers.timepix import TimepixHDF5Handler +# +#db = Broker.named("hxn") # db_analysis = Broker.named('hxn_analysis') -db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) -db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) +#db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) +#db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) diff --git a/pyxrf/model/load_data_from_db.py b/pyxrf/model/load_data_from_db.py index fc995b53..b4b02b11 100644 --- a/pyxrf/model/load_data_from_db.py +++ b/pyxrf/model/load_data_from_db.py @@ -820,12 +820,20 @@ def map_data2D_hxn( data_output = [] start_doc = hdr["start"] - logger.info("Plan type: '%s'", start_doc["plan_type"]) + + if "scan" in start_doc: + #print(" panda scan ") + plan_type = start_doc["scan"]["type"] + + else: + plan_type = start_doc["plan_type"] + + logger.info("Plan type: '%s'", plan_type) # Exclude certain types of plans based on data from the start document - if isinstance(skip_scan_types, (list, tuple)) and (start_doc["plan_type"] in skip_scan_types): + if isinstance(skip_scan_types, (list, tuple)) and (plan_type in skip_scan_types): raise RuntimeError( - f"Failed to load the scan: plan type {start_doc['plan_type']!r} is in the list of skipped types" + f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types" ) # The dictionary holding scan metadata @@ -920,11 +928,17 @@ def map_data2D_hxn( else: raise ValueError(f"Invalid data shape: {datashape}. Must be a list with 1 or 2 elements.") + logger.info(f'Data shape: {datashape}.') + # ----------------------------------------------------------------------------------------------- # Determine fast axis and slow axis fast_axis, slow_axis, fast_axis_index = start_doc.get("fast_axis", None), None, None motors = start_doc.get("motors", None) - if motors and isinstance(motors, (list, tuple)) and len(motors) == 2: + if motors and isinstance(motors, (list, tuple)) and len(motors) == 1: + fast_axis = fast_axis if fast_axis else motors[0] + fast_axis_index = motors.index(fast_axis, 0) + + elif motors and isinstance(motors, (list, tuple)) and len(motors) == 2: fast_axis = fast_axis if fast_axis else motors[0] fast_axis_index = motors.index(fast_axis, 0) slow_axis_index = 0 if (fast_axis_index == 1) else 1 @@ -940,14 +954,24 @@ def map_data2D_hxn( # ----------------------------------------------------------------------------------------------- # Reconstruct scan input try: - plan_args = start_doc["plan_args"] - # px_motor = plan_args["motor1"] - px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"] - # py_motor = plan_args["motor2"] - py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"] - dwell_time = plan_args["exposure_time"] - param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] - mdata["param_input"] = param_input + if "plan_args" in start_doc: # dscan and fly1d/fly2d scan + plan_args = start_doc["plan_args"] + # px_motor = plan_args["motor1"] + px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"] + # py_motor = plan_args["motor2"] + py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"] + dwell_time = plan_args["exposure_time"] + param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] + mdata["param_input"] = param_input + elif "scan" in start_doc: # fly1dpd and fly2dpd scan + scan_input = start_doc["scan"]["scan_input"] + px_start, px_end, px_step = scan_input[0:3] + py_start, py_end, py_step = scan_input[3:6] + dwell_time = start_doc["scan"]["dwell"] + param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] + mdata["param_input"] = param_input + else: + raise Exception("Unknown scan plan type") except Exception as ex: logger.warning( "Failed to reconstruct scan input: %s. Scan input is not saved as part of metadata to HDF5 file", @@ -966,7 +990,8 @@ def map_data2D_hxn( keylist = hdr.descriptors[0].data_keys.keys() det_list = [v for v in keylist if "xspress3" in v] # find xspress3 det with key word matching - + det_list = [v for v in det_list if len(v)==12] #added to filter out other rois added by user + scaler_list_all = config_data["scaler_list"] all_keys = hdr.descriptors[0].data_keys.keys() @@ -978,7 +1003,8 @@ def map_data2D_hxn( if isinstance(db, databroker._core.Broker): fields = None - data = hdr.table(fields=fields, fill=True) + + data = hdr.table(fields=fields, fill=False) # HXN data is stored in h5 files, load them later in map_data2D. # This is for the case of 'dcan' (1D), where the slow axis positions are not saved if (slow_axis not in data) and (fast_axis in data): @@ -994,6 +1020,7 @@ def map_data2D_hxn( fly_type=fly_type, subscan_dims=subscan_dims, spectrum_len=4096, + hdr=hdr ) # Transform coordinates for the fast axis if necessary: @@ -3452,6 +3479,7 @@ def map_data2D_xfm( create_each_det=create_each_det, scaler_list=config_data["scaler_list"], fly_type=fly_type, + hdr = hdr ) fpath_out = fpath @@ -3485,6 +3513,7 @@ def write_db_to_hdf( fly_type=None, subscan_dims=None, base_val=None, + hdr = None ): """ Assume data is obained from databroker, and save the data to hdf file. @@ -3531,6 +3560,7 @@ def write_db_to_hdf( dataGrp = f.create_group(interpath + "/" + detname) logger.info("read data from %s" % c_name) + channel_data = data[c_name] # new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans @@ -3567,8 +3597,9 @@ def write_db_to_hdf( # position data dataGrp = f.create_group(interpath + "/positions") + # scanning position data pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) - + for i in range(len(pos_names)): if "x" in pos_names[i]: pos_names[i] = "x_pos" @@ -3593,6 +3624,7 @@ def write_db_to_hdf( # scaler data dataGrp = f.create_group(interpath + "/scalers") + # scaler data scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) if fly_type in ("pyramid",): @@ -3767,6 +3799,7 @@ def map_data2D( fly_type=None, subscan_dims=None, spectrum_len=4096, + hdr = None ): """ Data is obained from databroker. Transfer items from data to a dictionary of @@ -3806,7 +3839,10 @@ def map_data2D( if c_name in data: detname = "det" + str(n + 1) logger.info("read data from %s" % c_name) - channel_data = data[c_name] + if db.name == 'hxn': + channel_data = np.squeeze(np.array(list(hdr.data(c_name)))) + else: + channel_data = data[c_name] # new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans new_v_shape = len(channel_data) // datashape[1] @@ -3839,8 +3875,20 @@ def map_data2D( sum_data += new_data data_output["det_sum"] = sum_data - # scanning position data - pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) + if db.name == 'hxn': + pos_names = pos_list + pos_data = np.zeros([datashape[0], datashape[1], len(pos_list)]) + from hxntools.scan_info import get_scan_positions + pos = get_scan_positions(hdr) + if isinstance(pos,tuple): + for i in range(len(pos)): + pos_data[:,:,i] = pos[i].reshape((datashape[0], datashape[1])) + else: + pos_data[:,:,0] = pos.reshape((datashape[0], datashape[1])) + else: + # scanning position data + pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) + for i in range(len(pos_names)): if "x" in pos_names[i]: pos_names[i] = "x_pos" @@ -3871,8 +3919,15 @@ def map_data2D( data_output["pos_names"] = pos_names data_output["pos_data"] = new_p - # scaler data - scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) + if db.name == 'hxn': + scaler_names = scaler_list + scaler_data = np.zeros([datashape[0], datashape[1], len(scaler_list)]) + for i in range(len(scaler_list)): + scaler_data[:,:,i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1])) + else: + # scaler data + scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) + if fly_type in ("pyramid",): scaler_data = flip_data(scaler_data, subscan_dims=subscan_dims) From 545085d525257b87d23a1b89ebe392a3ba5cac2f Mon Sep 17 00:00:00 2001 From: zgao-bnl Date: Wed, 23 Jul 2025 13:22:02 -0400 Subject: [PATCH 02/10] removed unnecessary changes --- pyxrf/model/load_data_from_db.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyxrf/model/load_data_from_db.py b/pyxrf/model/load_data_from_db.py index b4b02b11..fa688d4b 100644 --- a/pyxrf/model/load_data_from_db.py +++ b/pyxrf/model/load_data_from_db.py @@ -3479,7 +3479,6 @@ def map_data2D_xfm( create_each_det=create_each_det, scaler_list=config_data["scaler_list"], fly_type=fly_type, - hdr = hdr ) fpath_out = fpath @@ -3512,8 +3511,7 @@ def write_db_to_hdf( fname_add_version=False, fly_type=None, subscan_dims=None, - base_val=None, - hdr = None + base_val=None ): """ Assume data is obained from databroker, and save the data to hdf file. @@ -3560,7 +3558,6 @@ def write_db_to_hdf( dataGrp = f.create_group(interpath + "/" + detname) logger.info("read data from %s" % c_name) - channel_data = data[c_name] # new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans From b7268b6ef6f6e285c1ef5f87f0113dedb33d1562 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 24 May 2026 19:39:22 -0400 Subject: [PATCH 03/10] STY: formatting with black --- pyxrf/db_config/hxn_db_config.py | 11 ++++---- pyxrf/model/load_data_from_db.py | 44 +++++++++++++++----------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/pyxrf/db_config/hxn_db_config.py b/pyxrf/db_config/hxn_db_config.py index f891efce..5dd51d5b 100644 --- a/pyxrf/db_config/hxn_db_config.py +++ b/pyxrf/db_config/hxn_db_config.py @@ -5,13 +5,14 @@ db = Broker.named("hxn") from hxntools.handlers import register + register(db) -#from hxntools.handlers.xspress3 import Xspress3HDF5Handler -#from hxntools.handlers.timepix import TimepixHDF5Handler +# from hxntools.handlers.xspress3 import Xspress3HDF5Handler +# from hxntools.handlers.timepix import TimepixHDF5Handler # -#db = Broker.named("hxn") +# db = Broker.named("hxn") # db_analysis = Broker.named('hxn_analysis') -#db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) -#db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) +# db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) +# db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) diff --git a/pyxrf/model/load_data_from_db.py b/pyxrf/model/load_data_from_db.py index fa688d4b..2a5dfce5 100644 --- a/pyxrf/model/load_data_from_db.py +++ b/pyxrf/model/load_data_from_db.py @@ -822,7 +822,7 @@ def map_data2D_hxn( start_doc = hdr["start"] if "scan" in start_doc: - #print(" panda scan ") + # print(" panda scan ") plan_type = start_doc["scan"]["type"] else: @@ -832,9 +832,7 @@ def map_data2D_hxn( # Exclude certain types of plans based on data from the start document if isinstance(skip_scan_types, (list, tuple)) and (plan_type in skip_scan_types): - raise RuntimeError( - f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types" - ) + raise RuntimeError(f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types") # The dictionary holding scan metadata mdata = _extract_metadata_from_header(hdr) @@ -928,7 +926,7 @@ def map_data2D_hxn( else: raise ValueError(f"Invalid data shape: {datashape}. Must be a list with 1 or 2 elements.") - logger.info(f'Data shape: {datashape}.') + logger.info(f"Data shape: {datashape}.") # ----------------------------------------------------------------------------------------------- # Determine fast axis and slow axis @@ -954,7 +952,7 @@ def map_data2D_hxn( # ----------------------------------------------------------------------------------------------- # Reconstruct scan input try: - if "plan_args" in start_doc: # dscan and fly1d/fly2d scan + if "plan_args" in start_doc: # dscan and fly1d/fly2d scan plan_args = start_doc["plan_args"] # px_motor = plan_args["motor1"] px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"] @@ -963,7 +961,7 @@ def map_data2D_hxn( dwell_time = plan_args["exposure_time"] param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] mdata["param_input"] = param_input - elif "scan" in start_doc: # fly1dpd and fly2dpd scan + elif "scan" in start_doc: # fly1dpd and fly2dpd scan scan_input = start_doc["scan"]["scan_input"] px_start, px_end, px_step = scan_input[0:3] py_start, py_end, py_step = scan_input[3:6] @@ -990,8 +988,8 @@ def map_data2D_hxn( keylist = hdr.descriptors[0].data_keys.keys() det_list = [v for v in keylist if "xspress3" in v] # find xspress3 det with key word matching - det_list = [v for v in det_list if len(v)==12] #added to filter out other rois added by user - + det_list = [v for v in det_list if len(v) == 12] # added to filter out other rois added by user + scaler_list_all = config_data["scaler_list"] all_keys = hdr.descriptors[0].data_keys.keys() @@ -1003,8 +1001,7 @@ def map_data2D_hxn( if isinstance(db, databroker._core.Broker): fields = None - - data = hdr.table(fields=fields, fill=False) # HXN data is stored in h5 files, load them later in map_data2D. + data = hdr.table(fields=fields, fill=False) # HXN data is stored in h5 files, load them later in map_data2D. # This is for the case of 'dcan' (1D), where the slow axis positions are not saved if (slow_axis not in data) and (fast_axis in data): @@ -1020,7 +1017,7 @@ def map_data2D_hxn( fly_type=fly_type, subscan_dims=subscan_dims, spectrum_len=4096, - hdr=hdr + hdr=hdr, ) # Transform coordinates for the fast axis if necessary: @@ -3511,7 +3508,7 @@ def write_db_to_hdf( fname_add_version=False, fly_type=None, subscan_dims=None, - base_val=None + base_val=None, ): """ Assume data is obained from databroker, and save the data to hdf file. @@ -3596,7 +3593,7 @@ def write_db_to_hdf( # scanning position data pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) - + for i in range(len(pos_names)): if "x" in pos_names[i]: pos_names[i] = "x_pos" @@ -3796,7 +3793,7 @@ def map_data2D( fly_type=None, subscan_dims=None, spectrum_len=4096, - hdr = None + hdr=None, ): """ Data is obained from databroker. Transfer items from data to a dictionary of @@ -3836,7 +3833,7 @@ def map_data2D( if c_name in data: detname = "det" + str(n + 1) logger.info("read data from %s" % c_name) - if db.name == 'hxn': + if db.name == "hxn": channel_data = np.squeeze(np.array(list(hdr.data(c_name)))) else: channel_data = data[c_name] @@ -3872,20 +3869,21 @@ def map_data2D( sum_data += new_data data_output["det_sum"] = sum_data - if db.name == 'hxn': + if db.name == "hxn": pos_names = pos_list pos_data = np.zeros([datashape[0], datashape[1], len(pos_list)]) from hxntools.scan_info import get_scan_positions + pos = get_scan_positions(hdr) - if isinstance(pos,tuple): + if isinstance(pos, tuple): for i in range(len(pos)): - pos_data[:,:,i] = pos[i].reshape((datashape[0], datashape[1])) + pos_data[:, :, i] = pos[i].reshape((datashape[0], datashape[1])) else: - pos_data[:,:,0] = pos.reshape((datashape[0], datashape[1])) + pos_data[:, :, 0] = pos.reshape((datashape[0], datashape[1])) else: # scanning position data pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) - + for i in range(len(pos_names)): if "x" in pos_names[i]: pos_names[i] = "x_pos" @@ -3916,11 +3914,11 @@ def map_data2D( data_output["pos_names"] = pos_names data_output["pos_data"] = new_p - if db.name == 'hxn': + if db.name == "hxn": scaler_names = scaler_list scaler_data = np.zeros([datashape[0], datashape[1], len(scaler_list)]) for i in range(len(scaler_list)): - scaler_data[:,:,i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1])) + scaler_data[:, :, i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1])) else: # scaler data scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) From ffcdaf7fa6a6f7c1f473da63d346a9629bdbf765 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 24 May 2026 19:46:51 -0400 Subject: [PATCH 04/10] STY: flake8 --- pyxrf/db_config/hxn_db_config.py | 3 ++- pyxrf/model/fileio.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyxrf/db_config/hxn_db_config.py b/pyxrf/db_config/hxn_db_config.py index 5dd51d5b..83bbb834 100644 --- a/pyxrf/db_config/hxn_db_config.py +++ b/pyxrf/db_config/hxn_db_config.py @@ -4,7 +4,8 @@ from databroker import Broker db = Broker.named("hxn") -from hxntools.handlers import register + +from hxntools.handlers import register # noqa: E402 register(db) diff --git a/pyxrf/model/fileio.py b/pyxrf/model/fileio.py index 7f50be5f..1d5d910f 100644 --- a/pyxrf/model/fileio.py +++ b/pyxrf/model/fileio.py @@ -165,6 +165,8 @@ def _get_pyxrf_version_str(self): """ # Determine the current version of PyXRF + global pyxrf_version # noqa: F824 + pyxrf_version_str = pyxrf_version if pyxrf_version_str[0].lower() != "v": pyxrf_version_str = f"v{pyxrf_version_str}" From 5ada5c62c7727703955b3e6eac4a80ad7f7b0d5e Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 24 May 2026 22:15:46 -0400 Subject: [PATCH 05/10] CI: update testing matrix --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9bae88a7..ca3e02d4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: host-os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11"] - numpy-version: ["1.26"] + python-version: ["3.11", "3.12", "3.13"] + numpy-version: ["2.4.0"] pyqt-version: ["5.15"] include: - host-os: "ubuntu-latest" From b76d2734cccaebf1e2070defdd27551718c0e341 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 24 May 2026 22:15:58 -0400 Subject: [PATCH 06/10] CI: update testing matrix --- .github/workflows/black.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/docs_publish.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index b6a2ce20..40dae412 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -13,7 +13,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@v2 with: - python-version: 3.11 + python-version: 3.13 - name: Install Dependencies run: | # These packages are installed in the base environment but may be older diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 32a896ef..a586a014 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.13] fail-fast: false steps: diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 46a02ef0..11709771 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.13] fail-fast: false steps: From 374c258c651c337c823efb643271339a5bf84db7 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Mon, 25 May 2026 10:18:51 -0400 Subject: [PATCH 07/10] TST: fixed some unit tests --- pyxrf/core/tests/test_map_processing.py | 1 + pyxrf/core/tests/test_quant_analysis.py | 8 +++++--- pyxrf/model/tests/test_hdf5_file_operations.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyxrf/core/tests/test_map_processing.py b/pyxrf/core/tests/test_map_processing.py index 3dbe3971..e78228eb 100644 --- a/pyxrf/core/tests/test_map_processing.py +++ b/pyxrf/core/tests/test_map_processing.py @@ -51,6 +51,7 @@ def test_dask_client_create(tmpdir): client = dask_client_create(n_workers=n_workers_requested) n_workers = len(client.scheduler_info()["workers"]) assert n_workers == n_workers_requested, "The number of workers was set incorrectly" + client.close() assert not os.path.exists(dask_worker_space_path), "Temporary directory was created in the current directory" diff --git a/pyxrf/core/tests/test_quant_analysis.py b/pyxrf/core/tests/test_quant_analysis.py index 02f7b54c..ad901fca 100644 --- a/pyxrf/core/tests/test_quant_analysis.py +++ b/pyxrf/core/tests/test_quant_analysis.py @@ -347,9 +347,11 @@ def test_get_quant_fluor_data_dict(): ), "Generated object contains emission lines that are different from expected" mass_sum = sum([_["density"] for _ in quant_fluor_data_dict["element_lines"].values()]) - assert ( - mass_sum == mass_sum_expected - ), "The total mass (density) of the components is different from expected" + npt.assert_almost_equal( + mass_sum, + mass_sum_expected, + err_msg="The total mass (density) of the components is different from expected" + ) def gen_xrf_map_dict(nx=10, ny=5, elines=["S_K", "Au_M", "Fe_K"]): diff --git a/pyxrf/model/tests/test_hdf5_file_operations.py b/pyxrf/model/tests/test_hdf5_file_operations.py index c69bc22b..c4b213fb 100644 --- a/pyxrf/model/tests/test_hdf5_file_operations.py +++ b/pyxrf/model/tests/test_hdf5_file_operations.py @@ -23,7 +23,7 @@ def _prepare_raw_dataset(N=5, M=10, K=4096): pos_data = np.zeros(shape=[2, N, M]) pos_data[0, :, :] = np.broadcast_to(np.linspace(1, 1 + (M - 1) * 0.1, M), shape=[N, M]) pos_data[1, :, :] = np.broadcast_to( - np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), newshape=[N, 1]), shape=[N, M] + np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), shape=[N, 1]), shape=[N, M] ) data = { From 36c5eb913472c1956e97cf27de1851b533ae2f60 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Mon, 25 May 2026 10:19:41 -0400 Subject: [PATCH 08/10] STY: black --- pyxrf/core/tests/test_quant_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyxrf/core/tests/test_quant_analysis.py b/pyxrf/core/tests/test_quant_analysis.py index ad901fca..79fee31c 100644 --- a/pyxrf/core/tests/test_quant_analysis.py +++ b/pyxrf/core/tests/test_quant_analysis.py @@ -348,9 +348,9 @@ def test_get_quant_fluor_data_dict(): mass_sum = sum([_["density"] for _ in quant_fluor_data_dict["element_lines"].values()]) npt.assert_almost_equal( - mass_sum, + mass_sum, mass_sum_expected, - err_msg="The total mass (density) of the components is different from expected" + err_msg="The total mass (density) of the components is different from expected", ) From d51117843d99e8453958a6b97dd8f0a8037ee6db Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Mon, 25 May 2026 16:05:06 -0400 Subject: [PATCH 09/10] TST: fixed unit tests --- pyxrf/core/tests/test_yaml_param_files.py | 15 ++++++++++++--- pyxrf/core/yaml_param_files.py | 11 +++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pyxrf/core/tests/test_yaml_param_files.py b/pyxrf/core/tests/test_yaml_param_files.py index 5560e8d6..c557c58e 100644 --- a/pyxrf/core/tests/test_yaml_param_files.py +++ b/pyxrf/core/tests/test_yaml_param_files.py @@ -1,4 +1,5 @@ import os +import sys import jsonschema import numpy as np @@ -135,13 +136,21 @@ def _generate_sample_docstring(param_dict, include_section_titles=True): d_str.append(" -------") d_str.extend([""] * n_empty_lines_after) - d_str = "\n".join(d_str) # Convert the list to a single string + # Remove initial 4 spaces from all lines to mimick behavior of func.__doc__ in Python 3.13 + # and later (in Python 3.13, the initial spaces are removed automatically, + # but in earlier versions they are not) + is_py313 = sys.version_info >= (3, 13) + if is_py313: + d_str = d_str.split("\n") + d_str = [_[4:] if len(_) > 4 else "" for _ in d_str] + d_str = "\n".join(d_str) + return d_str, parameters -def test_parse_docstring_parameters(): +def test_parse_docstring_parameters_01(): # Simple test for the successfully parsed docstring. It seems sufficient, since all error cases are trivial. param_dict = _generate_parameter_set() @@ -164,7 +173,7 @@ def test_parse_docstring_parameters(): # Check for exception if the section titles are required, but don't exist param_dict = _generate_parameter_set() d_str, parameters = _generate_sample_docstring(param_dict, include_section_titles=False) - with pytest.raises(AssertionError, match="'Parameters' or 'Return' statement was not found in the docstring"): + with pytest.raises(AssertionError, match="'Parameters' or 'Returns' statement was not found in the docstring"): _parse_docstring_parameters(d_str, search_param_section=True) diff --git a/pyxrf/core/yaml_param_files.py b/pyxrf/core/yaml_param_files.py index 1fc4353b..712241a1 100644 --- a/pyxrf/core/yaml_param_files.py +++ b/pyxrf/core/yaml_param_files.py @@ -1,5 +1,6 @@ import os import re +import sys import yaml @@ -51,6 +52,11 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): str_list = doc_string.split("\n") + is_py313 = sys.version_info >= (3, 13) + if is_py313: + # Add initial 4 spaces to all lines + str_list = [f" {_}" for _ in str_list] + # Remove all spaces at the end of the strings (the should be no spaces there, but still) str_list = [s.rstrip() for s in str_list] @@ -70,7 +76,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): assert (n_first is not None) or ( n_last is not None - ), "Incorrect docstring format: 'Parameters' or 'Return' statement was not found in the docstring" + ), "Incorrect docstring format: 'Parameters' or 'Returns' statement was not found in the docstring" # The list of strings contains parameter descriptions str_list = str_list[n_first : n_last + 1] @@ -78,6 +84,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): assert all( [(not s) or re.search(r"^ ", s) for s in str_list] ), "Incorrect docstring format: parameter descriptions should be indented by at least FOUR spaces" + # Now remove the spaces from nonempty lines str_list = [s[4:] if s else s for s in str_list] @@ -99,7 +106,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): # The fist line of the description is actually the assert all( [len(s) > 1 for s in param_descriptions] - ), "Incomplete docstring: some parameters have not descriptions" + ), "Incomplete docstring: some parameters have no descriptions" params = list(zip(param_names, param_descriptions)) From cfb19fdbb251f19bcb287638ad209d746fd872ab Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Mon, 25 May 2026 16:21:11 -0400 Subject: [PATCH 10/10] CI: remove Python 3.11 from test matrix --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ca3e02d4..78530304 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: host-os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13"] numpy-version: ["2.4.0"] pyqt-version: ["5.15"] include: