From c1ceb79c9d845811b1e254173ab2196071f49cbe Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Tue, 28 Jul 2026 20:57:24 +0000 Subject: [PATCH 1/3] Update link to gio submodule that contains fix to #234 See bug fix in https://github.com/wkliao/gio/pull/2 --- gio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gio b/gio index 0fb229897..8c53087ea 160000 --- a/gio +++ b/gio @@ -1 +1 @@ -Subproject commit 0fb229897f9ab6ef74233f9d7e1d1bcec959dda6 +Subproject commit 8c53087eaab39507b732aada28ed89b6f392c7dd From 0cfe9a812a531b4e66ae2c3906d3860372be0a1d Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Tue, 28 Jul 2026 21:39:55 +0000 Subject: [PATCH 2/3] Add test program for testing files using Lustre's Progressive File Layout. --- configure.ac | 1 + test/Lustre/Makefile.am | 57 ++++++++ test/Lustre/parallel_run.sh | 50 +++++++ test/Lustre/seq_runs.sh | 29 ++++ test/Lustre/tst_pfl.c | 280 ++++++++++++++++++++++++++++++++++++ test/Lustre/tst_seq.sh | 32 +++++ test/Makefile.am | 3 +- 7 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 test/Lustre/Makefile.am create mode 100755 test/Lustre/parallel_run.sh create mode 100755 test/Lustre/seq_runs.sh create mode 100644 test/Lustre/tst_pfl.c create mode 100755 test/Lustre/tst_seq.sh diff --git a/configure.ac b/configure.ac index 00bf13002..25c59b2f1 100644 --- a/configure.ac +++ b/configure.ac @@ -2937,6 +2937,7 @@ AC_CONFIG_FILES(Makefile \ test/F90/Makefile \ test/burst_buffer/Makefile \ test/cdl/Makefile \ + test/Lustre/Makefile \ ) dnl The following dependency is for configure.in and configure diff --git a/test/Lustre/Makefile.am b/test/Lustre/Makefile.am new file mode 100644 index 000000000..65caaee6f --- /dev/null +++ b/test/Lustre/Makefile.am @@ -0,0 +1,57 @@ +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# +# @configure_input@ + +SUFFIXES = .o .c + +AM_DEFAULT_SOURCE_EXT = .c + +AM_CPPFLAGS = + +LDADD = + +if DECL_MPI_OFFSET + AM_CPPFLAGS += -DHAVE_DECL_MPI_OFFSET +endif + +check_PROGRAMS = + +# autimake 1.11.3 has not yet implemented AM_TESTS_ENVIRONMENT +# For newer versions, we can use AM_TESTS_ENVIRONMENT instead +TESTS_ENVIRONMENT = export SED="$(SED)"; +TESTS_ENVIRONMENT += export srcdir="$(srcdir)"; +TESTS_ENVIRONMENT += export top_builddir="$(top_builddir)"; +TESTS_ENVIRONMENT += export TESTOUTDIR="$(FSTYPE_PREFIX)$(TESTOUTDIR)"; +TESTS_ENVIRONMENT += export TESTSEQRUN="$(TESTSEQRUN)"; +TESTS_ENVIRONMENT += export TESTMPIRUN="$(TESTMPIRUN)"; +TESTS_ENVIRONMENT += export check_PROGRAMS="$(check_PROGRAMS)"; + +TESTS_ENVIRONMENT += export ENABLE_BURST_BUFFER=@ENABLE_BURST_BUFFER@; +TESTS_ENVIRONMENT += export ENABLE_NETCDF4=@ENABLE_NETCDF4@; +TESTS_ENVIRONMENT += export ENABLE_THREAD_SAFE=@ENABLE_THREAD_SAFE@; +TESTS_ENVIRONMENT += export MIMIC_LUSTRE=@MIMIC_LUSTRE@; + +# build check targets but not invoke by not setting TESTS +# TESTS = $(check_PROGRAMS) +TEST_EXTENSIONS = .sh +LOG_COMPILER = $(srcdir)/seq_runs.sh +SH_LOG_COMPILER = + +check_SCRIPTS = seq_runs.sh parallel_run.sh + +EXTRA_DIST = seq_runs.sh parallel_run.sh + +CLEANFILES = core core.* *.gcda *.gcno *.gcov gmon.out + +../common/libtestutils.la: + set -e; cd ../common && $(MAKE) $(MFLAGS) tests + +ptest2 ptest4 ptest6 ptest8 ptest10: + +# build check targets but not invoke +tests-local: all $(check_PROGRAMS) + +.PHONY: ptest ptests ptest2 ptest4 ptest6 ptest8 ptest10 + diff --git a/test/Lustre/parallel_run.sh b/test/Lustre/parallel_run.sh new file mode 100755 index 000000000..20630dabc --- /dev/null +++ b/test/Lustre/parallel_run.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $MPIRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $MPIRUN $@ + fi +} + +MPIRUN=`echo ${TESTMPIRUN} | ${SED} -e "s/NP/$1/g"` +# echo "MPIRUN = ${MPIRUN}" +# echo "check_PROGRAMS=${check_PROGRAMS}" + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +if test "x$MIMIC_LUSTRE" != x1 ; then + PNETCDF_HINTS="cb_nodes=2" +fi + +for i in ${check_PROGRAMS} ; do + + # SECONDS=0 + # start_ns=$(date +%s.%4N) + + exe_name=`basename $i` + + run_cmd ./$i -q -o ${TESTOUTDIR}/${exe_name}.nc + + # # echo "Elapsed: $SECONDS seconds" + # end_ns=$(date +%s.%4N) + # # Calculate difference (requires bc for floating point math) + # elapsed_ns=$(echo "$end_ns - $start_ns" | bc) + # echo "Elapsed time: ${elapsed_ns} seconds" + +done # check_PROGRAMS + diff --git a/test/Lustre/seq_runs.sh b/test/Lustre/seq_runs.sh new file mode 100755 index 000000000..eb3c73faf --- /dev/null +++ b/test/Lustre/seq_runs.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright (C) 2026, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +DRY_RUN=no +VERBOSE=no + +run_cmd() { + local lineno=${BASH_LINENO[$((${#BASH_LINENO[@]} - 2))]} + if test "x$VERBOSE" = xyes || test "x$DRY_RUN" = xyes ; then + echo "Line $lineno CMD: $TESTSEQRUN $@" + fi + if test "x$DRY_RUN" = xno ; then + $TESTSEQRUN $@ + fi +} + +exe_name=`basename $1` + +# prevent user environment setting of PNETCDF_HINTS to interfere +unset PNETCDF_HINTS + +run_cmd ./$1 -q -o ${TESTOUTDIR}/${exe_name}.nc + diff --git a/test/Lustre/tst_pfl.c b/test/Lustre/tst_pfl.c new file mode 100644 index 000000000..21b9bc2d5 --- /dev/null +++ b/test/Lustre/tst_pfl.c @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2026, Northwestern University and Argonne National Laboratory + * See COPYRIGHT notice in top-level directory. + */ + +/* This program test whether PnetCDF can properly open/create a file in a + * folder configured with Lustre Progressive File Layout. + +First create a new directory and file with PFL (with 5 components): + + mkdir -p /eagle/radix-io/wkliao/pfl + lfs setstripe -E 4m -c 4 \ + -E 1g -c 8 \ + -E 10g -c 16 \ + -E 100g -c 32 \ + -E eof -c -1 \ + /eagle/radix-io/wkliao/pfl + +Copy a netCDF file over. The file should inherit the folder's PFL settings. + + cp /eagle/radix-io/wkliao/map_i_case_fill_1344p.nc \ + /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc + +To compile: + mpicc -Wall -O2 -o tst_pfl ./tst_pfl.c -I/home/wkliao/PnetCDF/Github/include -L/home/wkliao/PnetCDF/Github/lib -lpnetcdf -llustreapi + +To run: + ./tst_pfl /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32 + +==== test OPEN nc_driver = mpiio ========================== +---- print_info --------------------------- +MPI File Info: nkeys = 39 +MPI File Info: [15] key = striping_factor, value = 32 +MPI File Info: [22] key = cb_nodes, value = 1 +MPI File Info: [32] key = nc_driver, value = mpiio +MPI File Info: [34] key = file_striping, value = auto +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc +lmm_stripe_size = 1048576 +lmm_stripe_count = 8 +lmm_stripe_offset = 0 + +==== test OPEN nc_driver = gio ============================ +---- print_info --------------------------- +MPI File Info: nkeys = 30 +MPI File Info: [ 0] key = nc_driver, value = gio +MPI File Info: [ 9] key = cb_nodes, value = 1 +MPI File Info: [12] key = striping_factor, value = 32 +MPI File Info: [15] key = overstriping_ratio, value = 1 +MPI File Info: [26] key = file_striping, value = auto +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc +lmm_stripe_size = 1048576 +lmm_stripe_count = 8 +lmm_stripe_offset = 0 + + + +==== test CREATE nc_driver = mpiio ========================== +---- print_info --------------------------- +MPI File Info: nkeys = 39 +MPI File Info: [15] key = striping_factor, value = 1 +MPI File Info: [22] key = cb_nodes, value = 1 +MPI File Info: [32] key = nc_driver, value = mpiio +MPI File Info: [34] key = file_striping, value = auto +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc.mpiio.noinherit +lmm_stripe_size = 1048576 +lmm_stripe_count = 1 +lmm_stripe_offset = 0 + +==== test CREATE nc_driver = mpiio ========================== +---- print_info --------------------------- +MPI File Info: nkeys = 39 +MPI File Info: [15] key = striping_factor, value = 32 +MPI File Info: [22] key = cb_nodes, value = 1 +MPI File Info: [32] key = nc_driver, value = mpiio +MPI File Info: [34] key = file_striping, value = inherit +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc.mpiio.inherit +lmm_stripe_size = 1048576 +lmm_stripe_count = 4 +lmm_stripe_offset = 0 + +==== test CREATE nc_driver = gio ============================ +---- print_info --------------------------- +MPI File Info: nkeys = 30 +MPI File Info: [ 0] key = nc_driver, value = gio +MPI File Info: [ 1] key = file_striping, value = auto +MPI File Info: [10] key = cb_nodes, value = 1 +MPI File Info: [13] key = striping_factor, value = 1 +MPI File Info: [16] key = overstriping_ratio, value = 1 +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc.gio.noinherit +lmm_stripe_size = 1048576 +lmm_stripe_count = 1 +lmm_stripe_offset = 0 + +==== test CREATE nc_driver = gio ============================ +---- print_info --------------------------- +MPI File Info: nkeys = 30 +MPI File Info: [ 0] key = nc_driver, value = gio +MPI File Info: [ 1] key = file_striping, value = inherit +MPI File Info: [10] key = cb_nodes, value = 1 +MPI File Info: [13] key = striping_factor, value = 32 +MPI File Info: [16] key = overstriping_ratio, value = 1 +---- get_stripe --------------------------- +filename = /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc.gio.inherit +lmm_stripe_size = 1048576 +lmm_stripe_count = 4 +lmm_stripe_offset = 0 + + */ + +#include +#include +#include /* unlink() */ + +#include +#include + +/*----< print_info() >------------------------------------------------------*/ +static +void print_info(MPI_Info *info_used) +{ + int i, nkeys; + + printf("---- %s ---------------------------\n",__func__); + MPI_Info_get_nkeys(*info_used, &nkeys); + printf("MPI File Info: nkeys = %d\n",nkeys); + for (i=0; i +#include +#include +#include +#define MAX_LOV_UUID_COUNT 1000 + +void get_stripe(char *filename) +{ + int lumlen, err, fd; + struct lov_user_md *lum = NULL; + + lumlen = sizeof(struct lov_user_md) + MAX_LOV_UUID_COUNT * sizeof(struct lov_user_ost_data); + lum = (struct lov_user_md *) calloc(1, lumlen); + + lum->lmm_magic = LOV_USER_MAGIC; + + fd = open(filename, O_RDONLY, 0666); + + err = ioctl(fd, LL_IOC_LOV_GETSTRIPE, (void *) lum); + if (!err) { + printf("---- %s ---------------------------\n",__func__); + printf("filename = %s\n", filename); + printf("lmm_stripe_size = %u\n", lum->lmm_stripe_size); + printf("lmm_stripe_count = %u\n", lum->lmm_stripe_count); + printf("lmm_stripe_offset = %u\n", lum->lmm_stripe_offset); + } + + close(fd); +} + +int test_open(char *filename, MPI_Info info) +{ + int ncid, err, nerrs=0; + MPI_Info info_used; + + err = ncmpi_open(MPI_COMM_WORLD, filename, NC_NOWRITE, info, &ncid); + ERR + + err = ncmpi_inq_file_info(ncid, &info_used); + ERR + print_info(&info_used); + MPI_Info_free(&info_used); + + err = ncmpi_close(ncid); + + get_stripe(filename); + + return nerrs; +} + +int test_create(char *filename, MPI_Info info) +{ + int ncid, err, nerrs=0; + MPI_Info info_used; + + unlink(filename); + + err = ncmpi_create(MPI_COMM_WORLD, filename, NC_CLOBBER, info, &ncid); + ERR + + err = ncmpi_inq_file_info(ncid, &info_used); + ERR + print_info(&info_used); + MPI_Info_free(&info_used); + + err = ncmpi_close(ncid); + + get_stripe(filename); + + return nerrs; +} + +int main(int argc, char **argv) +{ + char filename[1024]; + int nerrs=0, rank; + MPI_Info info; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + if (argc != 2) { + if (rank == 0) printf("Usage: %s filename\n", argv[0]); + MPI_Finalize(); + return 1; + } + + MPI_Info_create(&info); + + printf("\n==== test OPEN nc_driver = mpiio ==========================\n"); + sprintf(filename, "%s.nc", argv[1]); + MPI_Info_set(info, "nc_driver", "mpiio"); + nerrs += test_open(filename, info); + + printf("\n==== test OPEN nc_driver = gio ============================\n"); + sprintf(filename, "%s.nc", argv[1]); + MPI_Info_set(info, "nc_driver", "gio"); + nerrs += test_open(filename, info); + + printf("\n\n"); + printf("\n==== test CREATE nc_driver = mpiio ==========================\n"); + sprintf(filename, "%s.nc.mpiio.noinherit", argv[1]); + MPI_Info_set(info, "nc_driver", "mpiio"); + nerrs += test_create(filename, info); + + printf("\n==== test CREATE nc_driver = mpiio ==========================\n"); + sprintf(filename, "%s.nc.mpiio.inherit", argv[1]); + MPI_Info_set(info, "nc_driver", "mpiio"); + MPI_Info_set(info, "file_striping", "inherit"); + nerrs += test_create(filename, info); + + printf("\n==== test CREATE nc_driver = gio ============================\n"); + sprintf(filename, "%s.nc.gio.noinherit", argv[1]); + MPI_Info_set(info, "nc_driver", "gio"); + MPI_Info_set(info, "file_striping", "auto"); + nerrs += test_create(filename, info); + + printf("\n==== test CREATE nc_driver = gio ============================\n"); + sprintf(filename, "%s.nc.gio.inherit", argv[1]); + MPI_Info_set(info, "nc_driver", "gio"); + MPI_Info_set(info, "file_striping", "inherit"); + nerrs += test_create(filename, info); + + MPI_Info_free(&info); + + MPI_Finalize(); + + return 0; +} + diff --git a/test/Lustre/tst_seq.sh b/test/Lustre/tst_seq.sh new file mode 100755 index 000000000..a6193158d --- /dev/null +++ b/test/Lustre/tst_seq.sh @@ -0,0 +1,32 @@ +#!/bin/bash -l + +# set a file with progressive striping (5 components) +# 1st component is from offset 0 to 4MB using striping count 4 +# 2nd component is from offset 4MB to 1GB using striping count 8 +# 2nd component is from offset 1GB to 10GB using striping count 16 +# 3rd component is from offset 10GB to 100GB using striping count 32 +# 3rd component is from offset 100GB to ~ using striping count -1, i.e. all +# lfs setstripe -E 4m -c 4 -E 1g -c 8 -E 10g -c 16 -E 100g -c 32 -E eof -c -1 /eagle/radix-io/wkliao/pfl +# cp /eagle/radix-io/wkliao/map_i_case_fill_1344p.nc /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32.nc + +# set -x + +INSTALL_DIR=$HOME/PnetCDF/Github + +OPTS=-O2 + +LUSTRE_DIR=/eagle/radix-io/wkliao + +# export LD_LIBRARY_PATH="$HOME/PnetCDF/Github/lib:${LD_LIBRARY_PATH}" + +EXE_FILE=tst_pfl + +SRC_DIR=. + +mpicc -Wall $OPTS -o ${EXE_FILE} $SRC_DIR/$EXE_FILE.c \ + -I${INSTALL_DIR}/include \ + -L${INSTALL_DIR}/lib -lpnetcdf -llustreapi + +exit + +./${EXE_FILE} /eagle/radix-io/wkliao/pfl/pfl_4_8_16_32 diff --git a/test/Makefile.am b/test/Makefile.am index 7d9407403..5c118a01d 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -14,7 +14,8 @@ C_SUBDIRS = common \ header \ testcases \ nonblocking \ - cdl + cdl \ + Lustre DIST_SUBDIRS = $(C_SUBDIRS) CXX nf_test nf90_test F90 subfile \ largefile burst_buffer nc4 adios From e76e823e4a5c66d8a55cefbf7368629f8fc2c58d Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Tue, 4 Aug 2026 15:08:07 -0500 Subject: [PATCH 3/3] Add note about PR #236 --- sneak_peek.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sneak_peek.md b/sneak_peek.md index ce36ff99e..8ededf43b 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -62,6 +62,10 @@ This is essentially a placeholder for the next release note ... See [PR #230](https://github.com/Parallel-NetCDF/PnetCDF/pull/230). * Bug fixes + + Fix a divide-by-zero error when opening a file that is striped using the + Lustre Progressive File Layout (PFL). The bug was reported in + [Issue #234](https://github.com/Parallel-NetCDF/PnetCDF/issues/234). + See fix in [PR #236](https://github.com/Parallel-NetCDF/PnetCDF/pull/236). + Fix detection of the MPI base compiler names when PrgEnv-cray is loaded, where cc, CC, and ftn are wrappers of Cray clang and Cray Fortran compilers.