[com4]: Add com4 benchmark tests

avaframe/ana1Tests/testUtilities.py

@@ -92,6 +92,8 @@ def readAllBenchmarkDesDicts(info=False, inDir=""):
         inDir = pathlib.Path("..", "benchmarks")
 
     testDirs = list(inDir.glob("ava*"))
+    testDirsCom4 = list(inDir.glob("com4*"))
+    testDirs.extend(testDirsCom4)
     testDictList = []
 
     for testDir in testDirs:

avaframe/runStandardTestsCom4FlowPy.py

@@ -0,0 +1,240 @@
+"""
+    Run script for running the standard tests with com4FlowPy
+    in this test all the available tests tagged standardTest are performed
+"""
+
+# Load modules
+import time
+import pathlib
+import numpy as np
+from datetime import datetime
+import tempfile
+import os
+
+# Local imports
+import avaframe as avaf
+from avaframe.com4FlowPy import com4FlowPy
+from avaframe.runCom4FlowPy import readFlowPyinputs
+from avaframe.ana1Tests import testUtilities as tU
+from avaframe.in3Utils import fileHandlerUtils as fU
+from avaframe.in3Utils import initializeProject as initProj
+from avaframe.in3Utils import cfgUtils
+from avaframe.in3Utils import logUtils
+import avaframe.in2Trans.rasterUtils as rasterUtils
+
+
+def compareRasters(path, pathRef):
+    """
+    compare two rasters and compute the difference between them
+
+    Parameters
+    ----------
+    path: string or pathlib.Path
+        path to raster file
+    pathRef: string or pathlib.Path
+        path to reference raster file
+
+    Returns
+    -------
+    diff: np.array
+        difference of the rasters in every rastercell
+    equal: boolean
+        True if the rasters are equal
+    closePercentage: float
+        the proportion of cells that match closely between both rasters, out of all cells that were actually processed
+    """
+    rasterDict = rasterUtils.readRaster(path, noDataToNan=False)
+    raster = rasterDict["rasterData"]
+    rasterRefDict = rasterUtils.readRaster(pathRef, noDataToNan=False)
+    rasterRef = rasterRefDict["rasterData"]
+    # difference of both rasters
+    diff = rasterRef - raster
+
+    equal = np.array_equal(rasterRef, raster)
+
+    closeArray = np.isclose(raster, rasterRef, rtol=1e-04, equal_nan=True)
+    mask = np.logical_or(raster > 0, rasterRef > 0)
+    num_close = np.count_nonzero(closeArray[mask])
+    total = rasterRef[mask].size
+    closePercent = num_close / total
+
+    return diff, equal, closePercent
+
+def main():
+
+    # avaframe directory
+    _avaframeDir  = pathlib.Path(avaf.__file__).parents[0]
+    _benchmarkDir = pathlib.Path(_avaframeDir, '..', 'benchmarks')
+
+    # Which result types for comparison plots
+    # outputVariable = ['fpTravelAngleMax', 'zDelta', 'flux', 'cellCounts']
+
+    # log file name; leave empty to use default runLog.log
+    logName = 'runStandardTestsCom4FlowPy'
+
+    # Load settings from general configuration file
+    cfgMain = cfgUtils.getGeneralConfig()
+
+    # load all benchmark info as dictionaries from description files
+    testDictList = tU.readAllBenchmarkDesDicts(info=False, inDir = _benchmarkDir)
+
+    # filter benchmarks for tag standardTest
+    filterType = 'TAGS'
+    valuesList = ['standardTest', 'com4FlowPy']
+    # looking for 'com4FlowPy' and 'standardTest' in TAGS list
+    testList = tU.filterBenchmarks(testDictList, filterType, valuesList, condition='and')
+
+    # Set directory for full standard test report
+    outDir = _avaframeDir / 'tests' / 'reportsCom4FlowPy'
+    fU.makeADir(outDir)
+
+    # Start writing markdown style report for standard tests
+    reportFile = outDir / 'standardTestsReportCom4FlowPy.md'
+
+    _startDate = datetime.now()
+    with open(reportFile, 'w') as pfile:
+
+        # Write header
+        pfile.write('# Standard Tests Report \n\n')
+        pfile.write('Comparing __com4FlowPy__ simulations to selected benchmark results \n\n')
+        pfile.write(f'__tests started__ : {_startDate}\n\n')
+
+
+    log = logUtils.initiateLogger(outDir, logName)
+    log.info('The following benchmark tests will be fetched ')
+
+    with open(reportFile, 'a') as pfile:
+        pfile.write("__tests fetched__:\n")
+        for test in testList:
+            pfile.write(f"- {test['NAME']}\n")
+            log.info('%s' % test['NAME'])
+        pfile.write('\n* * * \n')
+
+
+    # create a temporary directory, where the outputs of all standard Tests are stored
+    # clean-up is automatic - this way we don't pollute the avaframe/data/ directory
+    with tempfile.TemporaryDirectory(prefix="avaframe_stdTests_") as tempDir:
+
+        tmpTestsDir = pathlib.Path(tempDir)
+
+        # run Standard Tests sequentially
+        for i, test in enumerate(testList):
+
+            with open(reportFile, 'a') as pfile:
+                pfile.write("\n")
+                pfile.write(f"### Test {i+1}: *{test['NAME']}*\n")
+                for descLine in str(test['DESCRIPTION']).split('\n'):
+                    pfile.write(f"{descLine}{' ' * 2}\n")
+                if 'REFERENCE' in test and isinstance(test['REFERENCE'], str):
+                    pfile.write(f"__Reference__: {test['REFERENCE']}{' ' * 2}\n")
+
+                testAvaFVersion = avaf.version.getVersion()
+                pfile.write(f"__tested AvaFrame Version__: {testAvaFVersion}{' ' * 2}\n")
+
+                if 'BENCHMARKED_AVAFRAME_VERSION' in test and isinstance(test['BENCHMARKED_AVAFRAME_VERSION'], str):
+                    benchAvaFVersion = test['BENCHMARKED_AVAFRAME_VERSION']
+                    if benchAvaFVersion == testAvaFVersion:
+                        pfile.write(f"__benchmarked AvaFrame::com4FlowPy version__: {benchAvaFVersion}{' ' * 2}\n")
+                    else:
+                        _str = "__benchmarked AvaFrame::com4FlowPy version__: "
+                        _str += f"<span style=\"color:red\"> {benchAvaFVersion} </span>{' ' * 2}\n"
+                        pfile.write(_str)
+
+
+                pfile.write("\n")
+                pfile.write("|Model Output|Result of comparison|status\n")
+                pfile.write("|----:|:-----:|:---:|\n")
+
+            avaDir = test['AVADIR']
+            cfgMain['MAIN']['avalancheDir'] = avaDir
+
+            # Fetch benchmark test info
+            refDir = pathlib.Path(_avaframeDir, '..', 'benchmarks', test['NAME'])
+
+            # Clean input directory(ies) of old work and output files
+            initProj.cleanSingleAvaDir(avaDir, deleteOutput=False)
+
+            # Load input parameters from configuration file for standard tests
+            benchmarkCfg = refDir / ('%s' % test['INI'])
+            modName = 'com4FlowPy'
+            cfg = cfgUtils.getModuleConfig(com4FlowPy, fileOverride=benchmarkCfg)
+            cfgGen = cfg["GENERAL"]
+            cfgGen["cpuCount"] = str(cfgUtils.getNumberOfProcesses(cfgMain, 9999))
+            cfgGen["overwriteResults"] = "reRunAndOverwrite"
+
+            avalancheDir = cfgMain["MAIN"]["avalancheDir"]
+            cfgPath = readFlowPyinputs(avalancheDir, cfg, log)
+
+            compDir = tmpTestsDir / pathlib.Path(avalancheDir)
+
+            cfgPath["customDirs"] = False
+            cfgPath["resDir"] = compDir
+            fU.makeADir(cfgPath["resDir"])
+            cfgPath["thalwegDir"] = cfgPath["resDir"] / "thalwegData"
+            cfgPath["tempDir"] = cfgPath["workDir"] / "temp"
+            fU.makeADir(cfgPath["tempDir"])
+            cfgPath["deleteTemp"] = "False"
+            cfgPath["outputFiles"] = cfg["PATHS"]["outputFiles"]
+            cfgPath["outputNoDataValue"] = cfg["PATHS"].getfloat("outputNoDataValue")
+            cfgPath["useCompression"] = cfg["PATHS"].getboolean("useCompression")
+            cfgPath["uid"] = cfgUtils.cfgHash(cfg)
+            cfgPath["timeString"] = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+            # Set timing
+            startTime = time.time()
+            # call com4FlowPy run
+            com4FlowPy.com4FlowPyMain(cfgPath, cfgGen)
+            endTime = time.time()
+            timeNeeded = endTime - startTime
+            log.info(('Took %s seconds to calculate.' % (timeNeeded)))
+
+            # files that are compared - must be defined in the .json inside the testDir
+            # every test can have its own set of output files / variables that are compared
+            outputVariable = test['FILES'] 
+
+            for variable in outputVariable:
+
+                for file in os.listdir(refDir):
+                    if file.endswith('%s.tif' % variable):
+                        pathRasterRef = refDir / file
+                        break
+                    else:
+                        continue
+                if os.path.isfile(pathRasterRef) is False:
+                    raise FileExistsError("in %s does not exist a file for variable %s" %(refDir, variable))
+
+                for file in os.listdir(compDir):
+                    if file.endswith('%s.tif' % variable) and ( str(file).split('_')[1] == cfgPath["uid"] ):
+                        pathRaster = compDir / file
+                        break
+                    else:
+                        continue
+
+                if os.path.isfile(pathRaster) is False:
+                    raise FileExistsError("in %s does not exist a file for variable %s" %(compDir, variable))
+                diff, eq, close = compareRasters(pathRaster, pathRasterRef)
+
+                if eq and np.sum(abs(diff[diff != 0])) == 0:
+                    message = f"|__{variable}__| rasters are equal |&check;\n"
+                    _logMsg = f"{variable} - rasters are equal"
+                else:
+                    message = f"|{variable}| rasters are __NOT(!)__ equal - {np.round(close, 4) * 100}%"
+                    message += f" of the affected area is close (relative tolerance: 10^-4)|&cross;\n"
+
+                    _logMsg = f"{variable} - rasters are NOT equal - {np.round(close, 4) * 100}% is close"
+
+                log.info(f"{test['NAME']}: {_logMsg}")
+
+                with open(reportFile, 'a') as pfile:
+                    pfile.write(message)
+
+            _endDate = datetime.now()
+            with open(reportFile, 'a') as pfile:
+                pfile.write('\n * * * \n')
+
+    with open(reportFile, 'a') as pfile:
+        pfile.write(f"__test(s) finished @__: {_endDate}\n")
+        pfile.write(f"__timeDelta__: {_endDate-_startDate}")
+
+if __name__ == "__main__":
+    main()

avaframe/tests/test_com4FlowPy.py

@@ -20,6 +20,7 @@
 import avaframe.in2Trans.rasterUtils as IOf
 import avaframe.runCom4FlowPy as runCom4FlowPy
 
+import avaframe.runStandardTestsCom4FlowPy as runStandardTestsCom4
 
 def test_add_os():
     cell = flowClass.Cell(
@@ -751,6 +752,113 @@ def test_runCom4FlowPy(tmp_path):
     for key in resDictTest6:
         assert resDictTest6[key] == resDict[key]
 
+def testCompareRasters(monkeypatch):
+    """Test the comparison of two raster arrays.
+
+    The test replaces ``rasterUtils.readRaster`` with a local fake
+    implementation. This keeps the test self-contained and avoids reading
+    actual raster files from disk.
+
+    The test verifies that:
+
+    * the cell-by-cell difference is calculated correctly;
+    * non-identical rasters are reported as unequal;
+    * the percentage of closely matching processed cells is correct;
+    * both rasters are read with ``noDataToNan=False``;
+    * the raster files are read in the expected order.
+
+    Parameters
+    ----------
+    monkeypatch : pytest.MonkeyPatch
+        Pytest fixture used to temporarily replace
+        ``rasterUtils.readRaster`` in the module under test.
+    """
+    rasterPath = pathlib.Path("raster.tif")
+    referenceRasterPath = pathlib.Path("referenceRaster.tif")
+
+    rasterData = np.array(
+        [
+            [1.0, 2.0, 0.0],
+            [4.0, np.nan, 6.0],
+        ]
+    )
+    referenceRasterData = np.array(
+        [
+            [1.0, 2.0001, 0.0],
+            [5.0, np.nan, 6.0],
+        ]
+    )
+
+    rasterDataByPath = {
+        rasterPath: rasterData,
+        referenceRasterPath: referenceRasterData,
+    }
+
+    readRasterCalls = []
+
+    def _fakeReadRaster(requestedPath, noDataToNan):
+        """Return predefined raster data for the requested path.
+
+        Parameters
+        ----------
+        requestedPath : pathlib.Path
+            Path of the raster requested by ``compareRasters``.
+        noDataToNan : bool
+            Value passed to the ``noDataToNan`` argument.
+
+        Returns
+        -------
+        dict
+            Dictionary containing the predefined NumPy array under the
+            ``"rasterData"`` key.
+        """
+        readRasterCalls.append((requestedPath, noDataToNan))
+
+        return {
+            "rasterData": rasterDataByPath[requestedPath],
+        }
+
+    # Patch readRaster where compareRasters looks it up. The patch is
+    # automatically removed by pytest after the test has completed.
+    monkeypatch.setattr(
+        runStandardTestsCom4.rasterUtils,
+        "readRaster",
+        _fakeReadRaster,
+    )
+
+    difference, areEqual, closePercentage = (
+        runStandardTestsCom4.compareRasters(
+            rasterPath,
+            referenceRasterPath,
+        )
+    )
+
+    expectedDifference = referenceRasterData - rasterData
+
+    np.testing.assert_allclose(
+        difference,
+        expectedDifference,
+        equal_nan=True,
+    )
+
+    assert areEqual is False
+
+    # Four cells are selected by the positive-value mask:
+    #
+    #   1.0 versus 1.0       -> close
+    #   2.0 versus 2.0001    -> close
+    #   4.0 versus 5.0       -> not close
+    #   6.0 versus 6.0       -> close
+    #
+    # The 0.0 pair and NaN pair are not selected by the mask.
+    # Therefore, three out of four processed cells are close.
+    assert closePercentage == pytest.approx(3 / 4)
+
+    assert readRasterCalls == [
+        (rasterPath, False),
+        (referenceRasterPath, False),
+    ]
+
 
 if __name__ == "__main__":
     test_add_os()