diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index b6a2ce20..40dae412 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -13,7 +13,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@v2 with: - python-version: 3.11 + python-version: 3.13 - name: Install Dependencies run: | # These packages are installed in the base environment but may be older diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 32a896ef..a586a014 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.13] fail-fast: false steps: diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 46a02ef0..11709771 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.13] fail-fast: false steps: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9bae88a7..78530304 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: host-os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11"] - numpy-version: ["1.26"] + python-version: ["3.12", "3.13"] + numpy-version: ["2.4.0"] pyqt-version: ["5.15"] include: - host-os: "ubuntu-latest" diff --git a/pyxrf/core/tests/test_map_processing.py b/pyxrf/core/tests/test_map_processing.py index 3dbe3971..e78228eb 100644 --- a/pyxrf/core/tests/test_map_processing.py +++ b/pyxrf/core/tests/test_map_processing.py @@ -51,6 +51,7 @@ def test_dask_client_create(tmpdir): client = dask_client_create(n_workers=n_workers_requested) n_workers = len(client.scheduler_info()["workers"]) assert n_workers == n_workers_requested, "The number of workers was set incorrectly" + client.close() assert not os.path.exists(dask_worker_space_path), "Temporary directory was created in the current directory" diff --git a/pyxrf/core/tests/test_quant_analysis.py b/pyxrf/core/tests/test_quant_analysis.py index 02f7b54c..79fee31c 100644 --- a/pyxrf/core/tests/test_quant_analysis.py +++ b/pyxrf/core/tests/test_quant_analysis.py @@ -347,9 +347,11 @@ def test_get_quant_fluor_data_dict(): ), "Generated object contains emission lines that are different from expected" mass_sum = sum([_["density"] for _ in quant_fluor_data_dict["element_lines"].values()]) - assert ( - mass_sum == mass_sum_expected - ), "The total mass (density) of the components is different from expected" + npt.assert_almost_equal( + mass_sum, + mass_sum_expected, + err_msg="The total mass (density) of the components is different from expected", + ) def gen_xrf_map_dict(nx=10, ny=5, elines=["S_K", "Au_M", "Fe_K"]): diff --git a/pyxrf/core/tests/test_yaml_param_files.py b/pyxrf/core/tests/test_yaml_param_files.py index 5560e8d6..c557c58e 100644 --- a/pyxrf/core/tests/test_yaml_param_files.py +++ b/pyxrf/core/tests/test_yaml_param_files.py @@ -1,4 +1,5 @@ import os +import sys import jsonschema import numpy as np @@ -135,13 +136,21 @@ def _generate_sample_docstring(param_dict, include_section_titles=True): d_str.append(" -------") d_str.extend([""] * n_empty_lines_after) - d_str = "\n".join(d_str) # Convert the list to a single string + # Remove initial 4 spaces from all lines to mimick behavior of func.__doc__ in Python 3.13 + # and later (in Python 3.13, the initial spaces are removed automatically, + # but in earlier versions they are not) + is_py313 = sys.version_info >= (3, 13) + if is_py313: + d_str = d_str.split("\n") + d_str = [_[4:] if len(_) > 4 else "" for _ in d_str] + d_str = "\n".join(d_str) + return d_str, parameters -def test_parse_docstring_parameters(): +def test_parse_docstring_parameters_01(): # Simple test for the successfully parsed docstring. It seems sufficient, since all error cases are trivial. param_dict = _generate_parameter_set() @@ -164,7 +173,7 @@ def test_parse_docstring_parameters(): # Check for exception if the section titles are required, but don't exist param_dict = _generate_parameter_set() d_str, parameters = _generate_sample_docstring(param_dict, include_section_titles=False) - with pytest.raises(AssertionError, match="'Parameters' or 'Return' statement was not found in the docstring"): + with pytest.raises(AssertionError, match="'Parameters' or 'Returns' statement was not found in the docstring"): _parse_docstring_parameters(d_str, search_param_section=True) diff --git a/pyxrf/core/yaml_param_files.py b/pyxrf/core/yaml_param_files.py index 1fc4353b..712241a1 100644 --- a/pyxrf/core/yaml_param_files.py +++ b/pyxrf/core/yaml_param_files.py @@ -1,5 +1,6 @@ import os import re +import sys import yaml @@ -51,6 +52,11 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): str_list = doc_string.split("\n") + is_py313 = sys.version_info >= (3, 13) + if is_py313: + # Add initial 4 spaces to all lines + str_list = [f" {_}" for _ in str_list] + # Remove all spaces at the end of the strings (the should be no spaces there, but still) str_list = [s.rstrip() for s in str_list] @@ -70,7 +76,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): assert (n_first is not None) or ( n_last is not None - ), "Incorrect docstring format: 'Parameters' or 'Return' statement was not found in the docstring" + ), "Incorrect docstring format: 'Parameters' or 'Returns' statement was not found in the docstring" # The list of strings contains parameter descriptions str_list = str_list[n_first : n_last + 1] @@ -78,6 +84,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): assert all( [(not s) or re.search(r"^ ", s) for s in str_list] ), "Incorrect docstring format: parameter descriptions should be indented by at least FOUR spaces" + # Now remove the spaces from nonempty lines str_list = [s[4:] if s else s for s in str_list] @@ -99,7 +106,7 @@ def _parse_docstring_parameters(doc_string, search_param_section=True): # The fist line of the description is actually the assert all( [len(s) > 1 for s in param_descriptions] - ), "Incomplete docstring: some parameters have not descriptions" + ), "Incomplete docstring: some parameters have no descriptions" params = list(zip(param_names, param_descriptions)) diff --git a/pyxrf/db_config/hxn_db_config.py b/pyxrf/db_config/hxn_db_config.py index fc919122..83bbb834 100644 --- a/pyxrf/db_config/hxn_db_config.py +++ b/pyxrf/db_config/hxn_db_config.py @@ -3,11 +3,17 @@ except ModuleNotFoundError: from databroker import Broker -from hxntools.handlers.timepix import TimepixHDF5Handler -from hxntools.handlers.xspress3 import Xspress3HDF5Handler - db = Broker.named("hxn") + +from hxntools.handlers import register # noqa: E402 + +register(db) + +# from hxntools.handlers.xspress3 import Xspress3HDF5Handler +# from hxntools.handlers.timepix import TimepixHDF5Handler +# +# db = Broker.named("hxn") # db_analysis = Broker.named('hxn_analysis') -db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) -db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) +# db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True) +# db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True) diff --git a/pyxrf/model/fileio.py b/pyxrf/model/fileio.py index 7f50be5f..1d5d910f 100644 --- a/pyxrf/model/fileio.py +++ b/pyxrf/model/fileio.py @@ -165,6 +165,8 @@ def _get_pyxrf_version_str(self): """ # Determine the current version of PyXRF + global pyxrf_version # noqa: F824 + pyxrf_version_str = pyxrf_version if pyxrf_version_str[0].lower() != "v": pyxrf_version_str = f"v{pyxrf_version_str}" diff --git a/pyxrf/model/load_data_from_db.py b/pyxrf/model/load_data_from_db.py index fc995b53..2a5dfce5 100644 --- a/pyxrf/model/load_data_from_db.py +++ b/pyxrf/model/load_data_from_db.py @@ -820,13 +820,19 @@ def map_data2D_hxn( data_output = [] start_doc = hdr["start"] - logger.info("Plan type: '%s'", start_doc["plan_type"]) + + if "scan" in start_doc: + # print(" panda scan ") + plan_type = start_doc["scan"]["type"] + + else: + plan_type = start_doc["plan_type"] + + logger.info("Plan type: '%s'", plan_type) # Exclude certain types of plans based on data from the start document - if isinstance(skip_scan_types, (list, tuple)) and (start_doc["plan_type"] in skip_scan_types): - raise RuntimeError( - f"Failed to load the scan: plan type {start_doc['plan_type']!r} is in the list of skipped types" - ) + if isinstance(skip_scan_types, (list, tuple)) and (plan_type in skip_scan_types): + raise RuntimeError(f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types") # The dictionary holding scan metadata mdata = _extract_metadata_from_header(hdr) @@ -920,11 +926,17 @@ def map_data2D_hxn( else: raise ValueError(f"Invalid data shape: {datashape}. Must be a list with 1 or 2 elements.") + logger.info(f"Data shape: {datashape}.") + # ----------------------------------------------------------------------------------------------- # Determine fast axis and slow axis fast_axis, slow_axis, fast_axis_index = start_doc.get("fast_axis", None), None, None motors = start_doc.get("motors", None) - if motors and isinstance(motors, (list, tuple)) and len(motors) == 2: + if motors and isinstance(motors, (list, tuple)) and len(motors) == 1: + fast_axis = fast_axis if fast_axis else motors[0] + fast_axis_index = motors.index(fast_axis, 0) + + elif motors and isinstance(motors, (list, tuple)) and len(motors) == 2: fast_axis = fast_axis if fast_axis else motors[0] fast_axis_index = motors.index(fast_axis, 0) slow_axis_index = 0 if (fast_axis_index == 1) else 1 @@ -940,14 +952,24 @@ def map_data2D_hxn( # ----------------------------------------------------------------------------------------------- # Reconstruct scan input try: - plan_args = start_doc["plan_args"] - # px_motor = plan_args["motor1"] - px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"] - # py_motor = plan_args["motor2"] - py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"] - dwell_time = plan_args["exposure_time"] - param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] - mdata["param_input"] = param_input + if "plan_args" in start_doc: # dscan and fly1d/fly2d scan + plan_args = start_doc["plan_args"] + # px_motor = plan_args["motor1"] + px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"] + # py_motor = plan_args["motor2"] + py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"] + dwell_time = plan_args["exposure_time"] + param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] + mdata["param_input"] = param_input + elif "scan" in start_doc: # fly1dpd and fly2dpd scan + scan_input = start_doc["scan"]["scan_input"] + px_start, px_end, px_step = scan_input[0:3] + py_start, py_end, py_step = scan_input[3:6] + dwell_time = start_doc["scan"]["dwell"] + param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time] + mdata["param_input"] = param_input + else: + raise Exception("Unknown scan plan type") except Exception as ex: logger.warning( "Failed to reconstruct scan input: %s. Scan input is not saved as part of metadata to HDF5 file", @@ -966,6 +988,7 @@ def map_data2D_hxn( keylist = hdr.descriptors[0].data_keys.keys() det_list = [v for v in keylist if "xspress3" in v] # find xspress3 det with key word matching + det_list = [v for v in det_list if len(v) == 12] # added to filter out other rois added by user scaler_list_all = config_data["scaler_list"] @@ -978,7 +1001,7 @@ def map_data2D_hxn( if isinstance(db, databroker._core.Broker): fields = None - data = hdr.table(fields=fields, fill=True) + data = hdr.table(fields=fields, fill=False) # HXN data is stored in h5 files, load them later in map_data2D. # This is for the case of 'dcan' (1D), where the slow axis positions are not saved if (slow_axis not in data) and (fast_axis in data): @@ -994,6 +1017,7 @@ def map_data2D_hxn( fly_type=fly_type, subscan_dims=subscan_dims, spectrum_len=4096, + hdr=hdr, ) # Transform coordinates for the fast axis if necessary: @@ -3567,6 +3591,7 @@ def write_db_to_hdf( # position data dataGrp = f.create_group(interpath + "/positions") + # scanning position data pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) for i in range(len(pos_names)): @@ -3593,6 +3618,7 @@ def write_db_to_hdf( # scaler data dataGrp = f.create_group(interpath + "/scalers") + # scaler data scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) if fly_type in ("pyramid",): @@ -3767,6 +3793,7 @@ def map_data2D( fly_type=None, subscan_dims=None, spectrum_len=4096, + hdr=None, ): """ Data is obained from databroker. Transfer items from data to a dictionary of @@ -3806,7 +3833,10 @@ def map_data2D( if c_name in data: detname = "det" + str(n + 1) logger.info("read data from %s" % c_name) - channel_data = data[c_name] + if db.name == "hxn": + channel_data = np.squeeze(np.array(list(hdr.data(c_name)))) + else: + channel_data = data[c_name] # new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans new_v_shape = len(channel_data) // datashape[1] @@ -3839,8 +3869,21 @@ def map_data2D( sum_data += new_data data_output["det_sum"] = sum_data - # scanning position data - pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) + if db.name == "hxn": + pos_names = pos_list + pos_data = np.zeros([datashape[0], datashape[1], len(pos_list)]) + from hxntools.scan_info import get_scan_positions + + pos = get_scan_positions(hdr) + if isinstance(pos, tuple): + for i in range(len(pos)): + pos_data[:, :, i] = pos[i].reshape((datashape[0], datashape[1])) + else: + pos_data[:, :, 0] = pos.reshape((datashape[0], datashape[1])) + else: + # scanning position data + pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape) + for i in range(len(pos_names)): if "x" in pos_names[i]: pos_names[i] = "x_pos" @@ -3871,8 +3914,15 @@ def map_data2D( data_output["pos_names"] = pos_names data_output["pos_data"] = new_p - # scaler data - scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) + if db.name == "hxn": + scaler_names = scaler_list + scaler_data = np.zeros([datashape[0], datashape[1], len(scaler_list)]) + for i in range(len(scaler_list)): + scaler_data[:, :, i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1])) + else: + # scaler data + scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape) + if fly_type in ("pyramid",): scaler_data = flip_data(scaler_data, subscan_dims=subscan_dims) diff --git a/pyxrf/model/tests/test_hdf5_file_operations.py b/pyxrf/model/tests/test_hdf5_file_operations.py index c69bc22b..c4b213fb 100644 --- a/pyxrf/model/tests/test_hdf5_file_operations.py +++ b/pyxrf/model/tests/test_hdf5_file_operations.py @@ -23,7 +23,7 @@ def _prepare_raw_dataset(N=5, M=10, K=4096): pos_data = np.zeros(shape=[2, N, M]) pos_data[0, :, :] = np.broadcast_to(np.linspace(1, 1 + (M - 1) * 0.1, M), shape=[N, M]) pos_data[1, :, :] = np.broadcast_to( - np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), newshape=[N, 1]), shape=[N, M] + np.reshape(np.linspace(5, 5 + (N - 1) * 0.2, N), shape=[N, 1]), shape=[N, M] ) data = {