scripts.py

################################################################
####
#### scripts.py : partial unix scripts
####
################################################################

import os
import re

from MrPackMod.basics  import module_version_from_env,\
    trace_string,echo_string,echo_warning,trace_var,\
    error_abort,abort_on_zero_keyword,nonzero_keyword,zero_keyword
from MrPackMod.error   import isnull,nonnull
from MrPackMod.names   import compilers_names,family_names,srcdir_name,scriptsdir_name,\
    dir_variable,\
    mode_has_mpi,mode_has_seq,mode_is_core,\
    ensure_download_path,DirNamesDict

from typing import Any,Optional

def compilers_flags( **kwargs: Any ) -> dict[str, str]:
    flags = { 'CFLAGS':"", 'CXXFLAGS':"", 'FFLAGS':"", }
    if cflags := nonzero_keyword( "cflags",**kwargs ):
        flags["CFLAGS"] = cflags
    if cxxflags := nonzero_keyword( "cxxflags",**kwargs ):
        flags["CXXFLAGS"] = cxxflags
    if fflags := nonzero_keyword( "fflags",**kwargs ):
        flags["FFLAGS"] = fflags
    return flags

def export_flags( **kwargs: Any ) -> str:
    flags = compilers_flags( **kwargs )
    cmdline = ""; cont = ""
    for lang in [ "CFLAGS", "CXXFLAGS", "FFLAGS", ]:
        if nonnull( flag := flags[lang] ):
            cmdline += f"{cont}export {lang}=\"{flag}\""
            cont = " && "
    return cmdline

def export_compilers_script( dummy : list[str],**kwargs: Any ) -> tuple[str,str]:
    trace_string( "Exporting compilers",**kwargs )
    compilers = compilers_names( **kwargs )
    script : str = "\necho \"Now exporting compilers for cmake and such\""
    for key,val in compilers.items():
        trace_string( f" .. Setting compiler: {key}={val}",**kwargs )
        script += f"""
echo Export {key}={val}
export {key}={val}
whichcomp=$( which {val} )
if [ $? -gt 0 ] ; then
  echo FAILURE: could not determine {val} && exit 1
fi
echo where {val}=${{whichcomp}}
            """
    return script,"Export compiler settings"

def load_compiler_and_mpi_and_modules_script( modulestoload : list[str],**kwargs: Any ) -> tuple[str,str]:
    title : str = f"Load compiler and mpi and modules: {modulestoload}"
    errmsg : str = f"Failed to load compiler and mpi and modules: {modulestoload}"
    _,compiler,compilerversion,_,mpi,mpiversion = family_names( **kwargs )
    trace_string( f"Loading {compiler=}/{compilerversion} and {mpi=}/{mpiversion}",
                  **kwargs )
    if ( modulepath := nonzero_keyword( "modulepath",**kwargs ) ) is None:
        error_abort( "Need a module path",**kwargs )
    if not ( redirect := nonzero_keyword( "redirect",**kwargs ) ):
        redirect = ""
    loadscript : str = ""
    #
    # Start by defining some functions
    #
    loadscript += successfailurefunction()
    loadscript += modulereportfunction()
    loadscript += modulelistfunction()
    loadscript += modulelonglistfunction()
    loadscript += modulecommandfunction()
    loadscript += moduleproperfunction()

    #
    # Now the actual script
    #
    loadscript += modulepurgefunction()
    if not mode_is_core( **kwargs ):
        if compiler is not None:
            if ( blasroot := kwargs.get("BLASLAPACK_ROOT") ) is None:
                trace_string( f"Not finding BLASLAPACK_ROOT in {kwargs}",**kwargs )
                blasroot = ""
            else: trace_string( f"Found BLASLAPACK_ROOT as {blasroot}",**kwargs )
            loadscript += compilerloadfunction( modulepath,compiler,compilerversion,blasroot )
        else: error_abort( "No compiler defined",**kwargs )
    if nonzero_keyword( "BLASLAPACK",**kwargs ):
        if comp := abort_on_zero_keyword( "COMPILER",**kwargs ):
            blas : str = ""
            # if comp=="gcc"    : blas = "mkl"
            # if comp=="nvidia" : blas = "nvpl"
            if nonnull(blas):
                loadscript += f"""
echo "Load blas/lapack library: {blas}"
modulecommand "load blas" "load {blas}"
                """
    if mode_has_mpi( **kwargs ):
        if mpi is not None:
            loadscript += mpiloadfunction( mpi,mpiversion )
        else: error_abort( "No mpi defined",**kwargs )
    if nonnull( modulestoload ) and zero_keyword( "skipmodules",**kwargs ) \
       and zero_keyword("NOMODULE",**kwargs) :
        scr,tit = modules_load_script( modulestoload,**kwargs )
        loadscript += scr
    else:
        trace_string( "Not loading any modules",**kwargs )
    loadscript += f"""
echo Module listing:
modulelonglist
    """
    return loadscript,title

def module_and_version_to_load( modver : str,**kwargs ) -> tuple[str,str,str]:
    if modslshver := re.match( r'^([^/]+)(/)([^/]+)$',modver ):
        # if there was a slash, keep original mod+ver
        m,s,v = modslshver.groups()
        return m,s,v
    else:
        # no slash, let's see if we can find a version
        module : str = modver
        # no version required, let's see if the environment has one
        if ( version := module_version_from_env(modver,**kwargs) ) is not None:
            return module,"/",version
            # modver = f"{module}/{version}"
        else:
            # no slash and no environment version, so load default
            return module,"",""

#
# This gets called only from the `test' action
#
def modules_proper_script( moduleslist : list[str],**kwargs : Any ) -> tuple[str,str]:
    modulestring : str = ','.join(moduleslist)
    title : str = f"Module proper testing for {modulestring}"
    script : str = f""
    if  nonzero_keyword("installing",**kwargs ):
        srcdir = srcdir_name( **kwargs )
        script += f"""
if [ ! -d "{srcdir}" ] ; then
    echo "FAILURE: Source directory {srcdir} does not exist"
    exit 1
fi
echo "Now testing properness of {moduleslist}"
        """
    for modver in moduleslist:
        onemodulescript,_ = one_module_proper_script( [modver],**kwargs )
        script += f"""
modulecommand "load {modver}" "load {modver}"
{onemodulescript}
        """
    script += f"\necho End of module proper testing\n"
    return script,title

#
# Assuming a module has been loaded,
# these lines test that the module is proper
#
def one_module_proper_script( modverlist : list[str],**kwargs : Any ) -> tuple[str,str]:
    modver : str = modverlist[0]
    title : str = f"test proper of module {modver}"
    module,_    = f"{modver}/".split("/",maxsplit=1)
    script : str = ""
    #
    # First make sure module is loaded
    #
    modvar : str = dir_variable(module)
    script += f"""
if [ -z "${{{modvar}}}" ] ; then
    echo "FAILURE: variable {modvar} is not set"
    exit 1
fi
"""
    #
    # Test all directories exist
    #
    f"""
echo \">>>> Test proper of module {modver}\"
testmoduleproper {modver}
    """
    if nonzero_keyword( "pkgconfig",**kwargs ) or nonzero_keyword( "pkgconfiglib",**kwargs ):
        script += f"""
echo " .. Finding pc files:"
find ${{pkgdir}} -name \\*.pc
echo "where PKG_CONFIG_PATH="
echo ${{PKG_CONFIG_PATH}} | tr ':' '\\n'
        """
    if nonzero_keyword( "cmakeconfig",**kwargs ):
        script += f"echo \" .. Finding cmake files:\"\nfind ${{pkgdir}} -name \\*.cmake\n"
    script += f"""
echo \"<<<< end of test proper of module {modver}\"
    """
    return script,title

##
## Some long literals,
## to make function above more readable
##
def successfailurefunction() -> str:
    return f"""
function report_success_failure () {{
    if [ $? -eq 0 ] ; then
        echo "SUCCESS: $1"
    else
        echo "FAILURE: $2"
        exit 1
    fi
}}
"""

def modulereportfunction( redirect="" ) -> str:
    return f"""
# Non-redirected return code reporting
# $1 : error code
# $2 : title
# $3 : actual command
function modulereport () {{
if [ $1 -gt 0 ] ; then
    echo FAILURE module command failed: $2
    echo Output: && module -t $3
    echo "Aborting this script" && exit
else
    echo SUCCESS module command succeeded: $2
    local cmd="$3"
    # echo "cmd was <<$cmd>>"
    echo -e "\\nNow loaded:"
    if [[ $cmd =~ load* ]] ; then
        module -t show ${{cmd##load}}
    fi 
    modulelist
fi {redirect}
}}
    """

def modulelistfunction() -> str:
    return """
# Single line module listing
function modulelist ()
{ module -t list 2>&1 | sort | tr '\n' ' ' && echo
}
        """

def modulelonglistfunction() -> str:
    return """
function modulelonglist ()
{
    local compiler=$( module -t list ${TACC_FAMILY_COMPILER} 2>&1 );
    local mpi=$( module -t list ${TACC_FAMILY_MPI} 2>&1 );
    local modules=$( module -t list 2>&1 | grep -v $compiler 2>/dev/null | grep -v $mpi 2>/dev/null | sort 2>/dev/null );
    for m in $compiler $mpi cont $modules;
    do
        if [ $m = "cont" ]; then
            echo "----------------";
        else
            if [ ! -z "${WORK}" ] ; then
                loc=$(module -t show $m 2>&1 | sed -e 's?'${WORK}'?WORK?' );
            else
                loc=$(module -t show $m 2>&1 );
            fi
            echo "$m : $loc";
        fi;
    done
    echo "----------------";
    echo -e "LD_LIBRARY_PATH=\\n$( echo ${LD_LIBRARY_PATH} | tr ':' '\\n' )"
    echo -e "PYTHONPATH=\\n$( echo ${PYTHONPATH} | tr ':' '\\n' )"
}
"""

def modulecommandfunction( redirect="" ) -> str:
    return f"""
# Execute a module command and report result:
# $1 : title, $2 actual command, $3 nonzero to display, otherwise redirected
function modulecommand () {{
    echo
    echo .... $1 : module $2 {redirect}
    if [ -z "$3" ] ; then 
      module -t $2 2>/dev/null {redirect}
    else
      module -t $2 {redirect}
    fi
    modulereport $? "$1" "$2"
}}
"""

def moduleproperfunction() -> str:
    return """
# Test whether modver is properly loaded
function testmoduleproper () {
    local modver=$1
    local module=${modver%%/*}
    local MODULE=$( echo $module | tr a-z A-Z )
    local nam=TACC_${MODULE}_DIR
    eval pkgdir=\\${$nam}
    if [ ! -d "${pkgdir}" ] ; then
        echo "FAILURE: package dir $nam=$pkgdir does not exist"
    else
        echo "SUCCESS: package ${modver} is at $pkgdir"
    fi
    for e in DIR BIN LIB INC ; do
        nam=TACC_${MODULE}_${e}
        eval cmpdir=\\${$nam}
        if [ ! -z "${cmpdir}" -a ! -d "${cmpdir}" ] ; then 
            echo "FAILURE: variable $nam set but dir $cmpdir does not exist"
        else
            echo "${nam}=${cmpdir}"
        fi
    done
}
    """

def modulepurgefunction( redirect="" ) -> str:
    return f"""
echo
echo Module setup starts here {redirect}

modulecommand "module purge" "purge"

# export LMOD_SYSTEM_DEFAULT_MODULES=TACC

# modulecommand "reset" "reset"
    """

def compilerloadfunction( modulepath : str,compiler : str,compilerversion : Optional[str],
                          blasroot : Optional[str],
                          redirect="" ) -> str:
    if compilerversion is None:
        compver : str = compiler
    else:
        compver = f"{compiler}/{compilerversion}"
    return f"""
if [ ! -z "${{TACC_FAMILY_MPI}}" ] ; then
  modulecommand "unload mpi" "-f unload ${{TACC_FAMILY_MPI}}"
fi

modulecommand "unload compiler" "unload ${{TACC_FAMILY_COMPILER}}"

echo .... After reset: {redirect}
modulelist {redirect}

echo ".... Set modulepath" {redirect}
export MODULEPATH={modulepath}
echo "MODULEPATH=${{MODULEPATH}}" {redirect}
modulecommand "Can we load compiler?" "avail {compver}" display

modulecommand "Load compiler" "load {compver}"

echo "\n.... Can we find BLASLAPACK_ROOT?" {redirect}
if [ -z "{blasroot}" ] ; then
    echo "WARNING: no BLASLAPACK_ROOT variable defined"
else
    echo "BLASLAPACK_ROOT={blasroot}"
fi {redirect}
    """

# VLE Should be insist on an mpi version?
def mpiloadfunction( mpi : str,mpiversion : Optional[str] ) -> str:
    if mpiversion is not None:
        return f"""
modulecommand "Load mpi" "load {mpi}/{mpiversion}"
        """
    else:
        return f"""
modulecommand "Load mpi" "load {mpi}"
    """

def modules_load_script( modulestoload : list[str],**kwargs ) -> tuple[str,str]:
    redirect : str = kwargs.get( "redirect","" )
    loadscript : str = f"""
echo ".... Load packages <<{modulestoload}>>" {redirect}
    """
    for modver in modulestoload:
        if isnull(modver): continue
        module,slash,version = module_and_version_to_load(modver,**kwargs )
        modulepropertest,_ = one_module_proper_script( [modver],**kwargs )
        loadscript += f"""
modulecommand "load module: {module}{slash}{version}" "load {module}{slash}{version}"
{modulepropertest}
        """
    modulesstring : str = re.sub( '/','',"-".join(modulestoload) )
    return loadscript,f"module loading {modulesstring}"

##
## Now the big scripts!
##

################################################################
####
#### Download
####
################################################################

def download_url_script( pdirs : tuple[str,DirNamesDict],**kwargs : dict[str,Any],
                        ) -> tuple[str,str]:
    package,dirnames = pdirs
    downloadpath : str = ensure_download_path( **kwargs )
    downloadurl  : str = dirnames["srcdir"]
    script : str = f"""
downloadpath={downloadpath}
echo "Using download location: ${{downloadpath}}"
cd ${{downloadpath}}

downloadurl={downloadurl}
tgz=${{downloadurl##*/}}
echo "First remove existing compressed file ${{tgz}}"
rm -f ${{tgz}}
echo "Now download from url: ${{downloadurl}}"
wget ${{downloadurl}}
if [ $? -gt 0 ] ; then
  echo "FAILURE Executing wget ${{downloadurl}}"
  exit
fi
echo "SUCCESS: package {package} downloaded as ${{tgz}}"
    """
    return script,"Download from url"

################################################################
####
#### CMake
####
################################################################

def cmake_configure_script( pcmakedirs : tuple[str,dict[str,Any]],
                            **kwargs : Any ) -> tuple[str,str]:
    program,dirnames = pcmakedirs
    program = re.sub( r'\..*','',program )
    cmakeopts = dirnames.get("cmakeopts")

    script : str = ""
    # setup
    if export_cmdline := nonzero_exports( **kwargs ):
        script += f"\n{export_cmdline}\n"
    if unset_cmdline := nonzero_unsets( **kwargs ):
        script += f"\n{unset_cmdline}\n"

    # remove old crud
    script += configure_preclean( dirnames,**kwargs )

    # cmake
    cmake         = cmake_basic_command( **kwargs )
    cmakeflags    = cmake_options( cmakeopts,**kwargs )
    buildsettings = cmake_build_settings( **kwargs )

    # set src, build, prefix
    pathsettings  = cmake_paths_settings( dirnames,**kwargs )
    builddir      = dirnames["builddir"]

    # for the regression case only: define project macro
    if nonnull(program) : pathsettings += f" -D PROJECTNAME={program}"
    script += f"""
echo "CMAKE_PREFIX_PATH=$( echo ${{CMAKE_PREFIX_PATH}} | tr ':' '\\n' )"
echo "PKG_CONFIG_PATH=$(   echo ${{PKG_CONFIG_PATH}}   | tr ':' '\\n' )"
cmdline="{cmake} {buildsettings} {cmakeflags} {pathsettings}"
echo "Doing cmake in pwd=${{PWD}}"
echo " .. with cmake=$( which cmake )"
echo " .. cmake cmdline=$cmdline" \
    | sed -e 's/-D/\\n    -D/g' -e 's/-S /\\n    -S /' -e 's/-B /\\n    -B /'
eval $cmdline
report_success_failure "configure succeeded" "cmake failed"
"""
    if ( grep := dirnames.get("grep") ) is not None:
        script += f"""
cd {builddir}

"""

    ## maybe only in the log file? script += configure_postreport( dirnames,**kwargs )
    script = script.replace( r'^ +-D(.*)$',r'  -D \1\\\n' )
    return script,"CMake configuring"

def configure_preclean( dirnames : DirNamesDict,**kwargs : Any ) -> str:
    builddir = dirnames ["builddir"]
    prefixdir = dirnames["prefixdir"]
    script : str =  f"""
echo "Remove any builddir: {builddir}"
rm -rf {builddir}
    """
    # netcdf installs c & f into the same prefix
    if kwargs.get("NOSCRATCHINSTALL") is not None:
        script += f"""
echo "Remove any prefixdir: {prefixdir}"
rm -rf {prefixdir}
        """
    return script

def configure_postreport( dirnames : DirNamesDict,**kwargs : Any ) -> str:
    builddir = dirnames ["builddir"]
    return f"""
echo "Builddir {builddir} contents:"
ls {builddir}
    """

def cmake_build_script( pcmakedirs : tuple[str,DirNamesDict],
                        **kwargs : dict[str,Any] ) -> tuple[str,str]:
    _,dirnames = pcmakedirs
    srcdir = dirnames["srcdir"]; builddir = dirnames["builddir"]; prefixdir = dirnames["prefixdir"]

    script : str = f"""
echo -e "\\n>>> Start of cmake build"
    """
    # flags and options
    jcount          : str = kwargs.get("jcount","6")
    make            : str = f"make --no-print-directory V=1 VERBOSE=1 -j {jcount}"
    makebuildtarget : str = kwargs.get("makebuildtarget","")
    # execute make & make install
    trace_string( f"Making in builddir: {builddir}",**kwargs )
    if ninja := kwargs.get( "CMAKEUSENINJA" ):
        makeline = f"ninja install"
    else:
        makeline = f"{make} --no-print-directory V=1 VERBOSE=1 -j {jcount} {makebuildtarget}"
    script += cmake_build_pre( dirnames,**kwargs )
    script += f"""
{makeline}
if [ $? -eq 0 ] ; then
    echo SUCCESS: compilation succeeded
else
    echo FAILURE: compilation failed
fi
    """
    if extra_targets := nonzero_keyword( "extrabuildtargets" ):
        script += f"""
{make} {extra_targets}
        """
    if nonnull(prefixdir) and not ninja:
        script += f"""
{make} install
if [ $? -eq 0 ] ; then
    echo SUCCESS: installation succeeded
else
    echo FAILURE: installation failed
fi
        """
    script += f"""
echo " .. end of cmake build"
    """
    return script,"CMake make and install"

def cmake_basic_command( **kwargs : Any ) -> str:
    cmake : str = kwargs.get( "CMAKENAME","cmake" )
    if nonzero_keyword( "CMAKEUSENINJA",**kwargs ):
        cmake = f"{cmake} -G Ninja"
    return f"TERM=dumb {cmake} --fresh -Wno-dev \
-D CMAKE_COMPILE_WARNING_AS_ERROR=OFF \
-D CMAKE_POLICY_VERSION_MINIMUM=3.13 \
-D CMAKE_VERBOSE_MAKEFILE=ON \
-D CMAKE_COLOR_MAKEFILE=OFF \
-D CMAKE_TERM_SUPPORTS_ANSI=OFF"

def cmake_options( opts : Optional[str],**kwargs: Any ) -> str:
    cmakeflags : str = "  -D CMAKE_VERBOSE_MAKEFILE=ON  -D CMAKE_EXPORT_COMPILE_COMMANDS=ON"
    if ( standard := kwargs.get("CPPSTANDARD") ) is not None:
        cmakeflags += f"  -D CMAKE_CXX_FLAGS=-std=c++{standard}"
    if ( conflags := nonzero_keyword("CMAKEFLAGS",**kwargs) ) is not None \
       or ( envflags := os.getenv("CMAKE_C_FLAGS") ) is not None \
       or nonnull( opts ):
        flags : str = f" -D MPM_CUSTOM_FLAGS=START "
        # VLE we need to cover the case that both are nonzero:
        # we need to edit the envflags into the conflags
        if conflags is not None:
            flags += conflags
        elif envflags is not None:
            flags += f" -D CMAKE_C_FLAGS={envflags}"
        if nonnull( opts ):
            flags += f" {opts}"
        cmakeflags += f"{flags} -D MPM_CUSTOM_FLAGS=END "
    return cmakeflags.lstrip(" ")

def cmake_build_settings( **kwargs ) -> str:
    if kwargs.get("CMAKEBUILDDEBUG"):
        defaultbuild = "Debug"
    else: defaultbuild = "RelWithDebInfo"
    cmakebuildtype = kwargs.get("CMAKEBUILDTYPE",defaultbuild)
    if static := kwargs.get("buildstaticlibs"):
        buildsharedlibs = "OFF"
    else: buildsharedlibs = "ON"
    return f""" -D BUILD_SHARED_LIBS={buildsharedlibs}  -D CMAKE_BUILD_TYPE={cmakebuildtype} """

def cmake_paths_settings( dirnames : DirNamesDict,**kwargs ) -> str:
    srcdir    = dirnames["srcdir"]
    builddir  = dirnames["builddir"]
    prefixdir = dirnames["prefixdir"]
    if nonnull( source := kwargs.get("CMAKESUBDIR") ):
        effectivesrcdir : str = f"{srcdir}/{source}"
    else: effectivesrcdir = srcdir
    if not os.path.isdir(effectivesrcdir):
        error_abort\
            ( f"Can not find source dir {effectivesrcdir}; did you forget to download?",
              **kwargs )
    settingsfile : str = f"{effectivesrcdir}/CMakeLists.txt"
    if not os.path.exists( f"{settingsfile}" ):
        error_abort( f"Can not find cmake settings file: {settingsfile}",**kwargs )
    cmakepathsetting : str = f"-S {effectivesrcdir} -B {builddir}"
    if nonnull(prefixdir):
        cmakepathsetting += f" -D CMAKE_INSTALL_PREFIX={prefixdir}"
    return cmakepathsetting

def cmake_build_pre( dirnames : DirNamesDict,**kwargs : Any ) -> str:
    builddir = dirnames["builddir"]
    prescript : str = f"""
if [ ! -d "{builddir}" ] ; then
    echo "FAILURE: no such build dir: {builddir}"
    exit  1
else
    echo "entering builddir: {builddir}"
fi
cd {builddir}
"""
    if kwargs.get( "CMAKEUSENINJA" ) is None:
        prescript += f"""
if [ ! -f makefile -a ! -f Makefile ] ; then
    echo "FAILURE: build dir {builddir} has no makefile or Makefile"
    exit 1
fi
"""
    return prescript

################################################################
####
#### Autotools
####
################################################################

def autotools_configure_script( pmakedirs : tuple[str,dict[str,Any]],**kwargs : dict[str,Any] ) \
    -> tuple[str,str]:
    program,dirnames = pmakedirs # pcmakedirs[0]; cmakedirs = pcmakedirs[1:]
    srcdir    : str = dirnames["srcdir"]
    prefixdir : str = dirnames["prefixdir"]

    if before := nonzero_keyword( "BEFORECONFIGURECMDS",**kwargs ):
        setup_script : str = f"\n{before}\n"
    else: setup_script = "\n"

    ##
    ## go to the right location for configure
    ## do autogen stuff before configure
    ##
    configsetupscript : str = config_setup_script( srcdir,**kwargs )

    ##
    ## do configure
    ##
    if ( option := nonzero_keyword( "PREFIXOPTION",**kwargs ) ) is not None:
        prefixoption = option # pdtoolkit
    else: prefixoption = "--prefix"
    if ( flags := nonzero_keyword( "CONFIGUREFLAGS",**kwargs ) ) is not None:
        flags = f" {flags}"
    else: flags = ""
    configurescript : str = f"""
./configure {prefixoption}={prefixdir} --libdir={prefixdir}/lib {flags}
echo "SUCCESS: autotools configure succeeded"
    """
    return setup_script+configsetupscript+configurescript,"Autotools configuring"

def config_setup_script( srcdir : str,**kwargs : dict[str,Any] ) -> str:    
    if ( builddir := nonzero_keyword( "CONFIGINBUILDDIR",**kwargs ) ) is not None:
        trace_string( f" .. going to configure in build dir {builddir}",**kwargs )
        configloc : str = builddir
        config_cmdline : str = f"{srcdir}/configure"
    elif ( subdir := nonzero_keyword( "CONFIGURESUBDIR",**kwargs ) ) is not None:
        trace_string( f" .. going to configure in subdir: {subdir}.",**kwargs )
        configloc = f"{srcdir}/{subdir}"
        config_cmdline = f"./configure"
    else:
        configloc = f"{srcdir}"
        config_cmdline = f"./configure"
    if nonzero_keyword( "AUTOUPDATE",**kwargs ):
        autoupdate : str = "./autoupdate"
    else: autoupdate = ""
    return f"""
cd {configloc}
echo Starting configure process in $(pwd)
if [ -f \"configure\" ] ; then
  has_configure=1
  echo has configure script
else has_configure= ; echo no configure script ; fi
if [ -f \"autogen.sh\" ] ; then
  has_autogen=1
  echo has autogen
else has_autogen= ; echo no autogen ; fi
if [ -f \"configure.ac\" ] ; then
  has_ac=1
  echo has configure.ac 
else has_ac= ; echo no configure.ac ; fi

if [ -z "${{has_configure}}" ] ; then 
  if [ ! -z "${{has_ac}}" ] ; then
    aclocal && autoconf
  elif [ ! -z "${{has_autogen}}" ] ; then 
    ./autogen.sh
  else
    echo FAILURE Need configure.ac or autogen.sh to generate configure script && exit 1
  fi
fi
{autoupdate}
    """

def autotools_build_script( pmakedirs : tuple[str,dict[str,Any]],**kwargs: dict[str,Any] ) \
    -> tuple[str,str]:
    program,dirnames = pmakedirs # pcmakedirs[0]; cmakedirs = pcmakedirs[1:]
    srcdir    : str = dirnames["srcdir"]
    prefixdir : str = dirnames["prefixdir"]

    if ( subdir := nonzero_keyword("MAKESUBDIR",**kwargs) ) is None:
        subdir = srcdir

    #
    # Make
    #
    jval : str = kwargs.get("jcount",6)
    makecommand : str = f"make --no-print-directory -j {jval}"
    script : str = f"""
cd {subdir}
{makecommand}
    """
    if extra := nonzero_keyword( "EXTRABUILDTARGETS",**kwargs ):
        trace_string( f" .. making extra targets: {extra}",**kwargs )
        script += f"\n{makecommand} {extra}\n"

    #
    # install
    #
    extra = kwargs.get( "EXTRAINSTALLTARGET","" )
    script += f"""
{makecommand} install {extra}
echo "SUCCESS: autotools build succeeded"
    """

    return script,"Autotools make and install"

################################################################
####
#### Scons
####
################################################################

def scons_configure_script( pmakedirs : tuple[str,dict[str,Any]],**kwargs : dict[str,Any] ) \
    -> tuple[str,str]:
    package,dirnames = pmakedirs # pcmakedirs[0]; cmakedirs = pcmakedirs[1:]
    srcdir    : str = dirnames["srcdir"]
    prefixdir : str = dirnames["prefixdir"]

    sconsflags : str = kwargs.get("SCONSFLAGS","")
    setupscript : str = ""

    ##
    ## do configure
    ##
    if ( option := nonzero_keyword( "PREFIXOPTION",**kwargs ) ) is not None:
        prefixoption = option # pdtoolkit
    else: prefixoption = "--prefix"
    if ( flags := nonzero_keyword( "CONFIGUREFLAGS",**kwargs ) ) is not None:
        flags = f" {flags}"
    else: flags = ""
    configurescript : str = f"""
echo "Test for Scons module"
PYTHONPATH=/scratch/00434/eijkhout/installation/scons/installation-scons-4.9.1-stampede3-gcc13.2.0 python3 -c "import SCons"
report_success_failure "python can load SCons" "python could not load SCons module"
python3 -c "from SCons.Script.Main import main"
report_success_failure "python can load scons script main" "python could not load scons module script main"

echo -e "\\nCheck validity of scons:\\n$(which scons)\\nuses:\\n$( head -1 $(which scons) )\\nand python is:\\n$(which python3)"

echo "Go to srcdir={srcdir}"
cd {srcdir}
echo -e "\\nTest for SConstruct file"
if [ ! -f SConstruct ] ; then
    echo "No SConstruct" && exit 1
else
    if [ -f {package}.conf ] ; then
        echo "removing {package}.conf"
        rm -f {package}.conf
    fi
    echo "scons=$(which scons)"
    PYTHONPATH=/scratch/00434/eijkhout/installation/scons/installation-scons-4.11.1-stampede3-gcc13.2.0/:${{PYTHONPATH}} \
    scons \
        build env_vars=all \
        CC=${{TACC_CC}} CXX=${{TACC_CXX}} FORTRAN=${{TACC_FC}} \
        prefix={prefixdir} \
        {sconsflags} \
        googletest='none'
    report_success_failure "scons build succeeded" "scons build failed"
fi
"""
    return setupscript+configurescript,"Scons configuring"

def scons_build_script( pmakedirs : tuple[str,dict[str,Any]],**kwargs: dict[str,Any] ) \
    -> tuple[str,str]:
    program,dirnames = pmakedirs # pcmakedirs[0]; cmakedirs = pcmakedirs[1:]
    srcdir    : str = dirnames["srcdir"]
    prefixdir : str = dirnames["prefixdir"]

    if ( subdir := nonzero_keyword("MAKESUBDIR",**kwargs) ) is None:
        subdir = srcdir

    #
    # Make
    #
    jval : str = kwargs.get("jcount",6)
    makecommand : str = f"scons install"
    script : str = f"""
cd {subdir}
{makecommand}
    """
    if extra := nonzero_keyword( "EXTRABUILDTARGETS",**kwargs ):
        trace_string( f" .. making extra targets: {extra}",**kwargs )
        script += f"\n{makecommand} {extra}\n"

    return script,"Scons make and install"

################################################################
####
#### PETSc
####
################################################################

def petsc_configure_script( pdirs : tuple[str,DirNamesDict],**kwargs : Any ) -> tuple[str,str]:
    program,dirnames = pdirs # pcmakedirs[0]; cmakedirs = pcmakedirs[1:]
    # srcdir,prefixdir = plist
    srcdir    : str = dirnames["srcdir"]
    prefixdir : str = dirnames["prefixdir"]

    petscflags : str = kwargs.get("PETSCFLAGS","")
    script : str = f"""
if [ ! -d "{srcdir}" ] ; then
    echo "FAILURE: src dir {srcdir} does not exist" && exit 1
fi
cd {srcdir}
    """
    if export_cmdline := nonzero_exports( **kwargs ):
        script += f"\n{export_cmdline}\n"
    script += f"""
if [ ! -f "configure" ] ; then
    echo "FAILURE: no petsc configure script found in ${{PWD}}" && exit 1
fi
which python3
cmdline="python3 ./configure \
    CC=${{CC}} CXX=${{CXX}} FC=${{FC}} \
    {petscflags} \
    --prefix={prefixdir}"
echo "Configure cmdline=$cmdline"
eval $cmdline
    """
    return script,"Make setup"

def petsc_build_script( dummy : list[str],**kwargs : Any ) -> tuple[str,str]:
    srcdir  : str = srcdir_name( **kwargs )
    jcount  : str = kwargs.get("jcount",6)
    targets : str = kwargs.get( "MAKETARGETS","" )
    trace_string( f"making targets: {targets}",**kwargs )

    script : str = f"\ncd {srcdir}\n"
    if export_cmdline := nonzero_exports( **kwargs ):
        script += f"\n{export_cmdline}\n"
    script += f"""
make -j {jcount} all
make -j {jcount} install
    """
    return script,"Make build"

################################################################
####
#### Make
####
################################################################

def make_build_script( dirnamesl : tuple[str,DirNamesDict],**kwargs : Any ) -> tuple[str,str]:
    program,dirnames = dirnamesl
    # srcdir,builddir,prefixdir = cmakedirs
    srcdir = dirnames["srcdir"]
    # flags and options
    jcount          : str = kwargs.get("jcount","6")
    make            : str = f"make --no-print-directory V=1 VERBOSE=1 -j {jcount}"
    makebuildtarget : str = kwargs.get("makebuildtarget","")

    script : str = f"""
make -f {srcdir}/Makefile SRCDIR={srcdir} PROJECTNAME={program} {program}
    """
    return script,"Make compilation"

#
# Auxs for the build scripts
#
def nonzero_exports( **kwargs : Any ) -> Optional[str]:
    if exports := nonzero_keyword( "exports",**kwargs ):
        export_cmdline : str = " && ".join(exports)
        trace_string( f"Using exports: {export_cmdline}",**kwargs )
        return export_cmdline
    else: return None

def nonzero_unsets( **kwargs : Any ) -> Optional[str]:
    if unsets := nonzero_keyword( "unsets",**kwargs ):
        unset_cmdline : str = " && ".join(unsets)
        trace_string( f"Using unsets: {unset_cmdline}",**kwargs )
        return unset_cmdline
    else: return None

################################################################
####
#### More scripts
####
################################################################

def ldd_script( dirnamesl : tuple[str,DirNamesDict],**kwargs ) -> tuple[str,str]:
    # old: program,programdir,cmakebuilddir,cmakeprefixdir = args
    # new:
    program,dirnames = dirnamesl
    program = re.sub( r'\..*','',program )

    # srcdir,builddir,prefixdir = cmakedirs
    if ( scriptsdir := kwargs.get("scriptsdir") ) is None:
        error_abort( f"need scriptsdir for ldd tmps",**kwargs )
    cmakebuilddir  = dirnames["builddir"]
    cmakeprefixdir = dirnames["prefixdir"]

    script : str = f"""
echo -e "\n>>> Start of ldd testing"
    """

    where : str = cmakeprefixdir if nonnull(cmakeprefixdir) else cmakebuilddir
    script += f"""
echo "ldd testing on file={program} in dir={where}"
if [ ! -d ] ; then
    echo "FAILURE: directory <<{where}>> does not exist" && exit 1
fi
cd {where}
if [ ! -d "{scriptsdir}" ] ; then
    echo "FAILURE: scripts dir {scriptsdir} does not exist" && exit 1
fi

program_full_path={where}/{program}"
if [ ! -f \"${{program_full_path}}\" ] ; then
    echo "FAILURE: could not find file=${{program_full_path}} to run ldd on"
else
    echo "Going to ldd on ${{program_full_path}}"
    lddout="{scriptsdir}/ldd_{program}.out"
    rm -f "${{lddout}}"
    ldd ${{program_full_path}} 2>&1 | tee "${{lddout}}"
    notfound=$( grep \"not found\" "${{lddout}}" | wc -l )
    if [ $notfound -eq 0 ] ; then
        echo "SUCCESS: all libraries resolved"
    else
        echo "FAILURE: $notfound references not found"
    fi
fi
    """
    return script,f"ldd test on {program}"

##
## Test file existence
##
def file_to_exist_script( args : list[str],**kwargs : Any, ) -> tuple[str,str]:
    package,dirtype,program,grep,executable = args
    dirvar : str = dir_variable(package,dirtype)
    title : str = f"Test existence of {dirtype}:{program}"
    script : str = f"""
echo "{title}"

echo "Using directory variable: {dirvar}"
eval filedir=\\${{{dirvar}}}
echo " .. expands to path: ${{filedir}}"
if [ ! -z "${{filedir}}" -a -d "${{filedir}}" ] ; then 
    echo " .. directory {dirvar}=${{filedir}} exists"
else 
    echo "FAILURE: {dirvar} does not exist"
    exit 1
fi

file_to_test=${{filedir}}/{program}
echo "File to test: ${{file_to_test}}"
if [ -f "${{file_to_test}}" ] ; then
    echo "SUCCESS: file exists: <<${{file_to_test}}>>"
else
    echo "FAILURE: file does not exist <<${{file_to_report}}>>"
    exit 1
fi
        """
    return script,title
    #
    if executable:
        script += f"""
if [ -x \"{file_to_test}\" ] ; then
    echo "SUCCESS: file is executable"
else
    echo "FAILURE: file is not executable"
fi
        """
    if nonnull( grep ):
        program_clean = re.sub( '/','',program )
        grep_output_file : str = f"{os.getcwd()}/{program_clean}_grep.out"
        script += f"""
if [ -f \"{file_to_test}\" ] ; then
    grep \"{grep}\" {file_to_test} >{grep_output_file} 2>&1
    echo INFORMATION: grep result is $( head -n 1 {grep_output_file} )
fi
        """
    return script,title

##
## Run a program
##
def run_script( dirnamesl : tuple[str,DirNamesDict,str,str],**kwargs : Any ) -> tuple[str,str]:
    program,dirnames,args,val = dirnamesl
    # strip any extension
    program = re.sub( r'\..*','',program )

    title   : str = f"run program {program}"

    script : str = ""
    # where do we run?
    rundir : str = dirnames.get("builddir","build")
    script += f"""
if [ ! -d "{rundir}" ] ; then
    echo "FAILURE: rundir does not exist: <<{rundir}>> in pwd=<<$(pwd)>>"
    exit 1
fi
cd {rundir}
echo "Running in rundir={rundir} full path=$( pwd )"
    """

    # what do we run?
    #  - prefix is empty for runing along path
    #  - prefix can be ./
    prefix  : str = dirnames.get("prefix","")
    #print( f"prefix=<<{prefix}>> out of dirnames={dirnames}" )
    cmdline : str = f"{prefix}{program}"
    if nonnull( args ):
        cmdline += f" {args}"
    if kwargs.get("MODE") in ["mpi","hybrid",]:
        cmdline = f"ibrun -n 1 {cmdline}"
    script += f"""
echo "cmdline={cmdline}"
echo ">>>> start execution"
runout=run_{program}.out
{cmdline} >${{runout}} 2>&1
runcode=$?
echo "<<<< end execution"
if [ ${{runcode}} -eq 0 ] ; then 
    echo "SUCCESS: running {program}"
else
    echo "FAILURE: running {program}"
fi 
echo ">>>> output"
cat ${{runout}}
echo "<<<< output"
    """
    return script,title