From ab2b68ccd8a60e9dc0f8b91b3d9966c9571a764b Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Tue, 29 Sep 2026 20:06:06 +0000 Subject: [PATCH 1/2] make covariance and pvlaue vlaidations fully detemrisnitc with seed --- CIValidations/CovarianceValidations.cpp | 4 ++++ CIValidations/PredictiveValidations.cpp | 23 ++++++++++++++++++++- CIValidations/README.md | 1 + CIValidations/TestOutputs/CovarianceOut.txt | 10 +++++++++ CIValidations/TestOutputs/Predictive.txt | 8 +++++++ CMakeLists.txt | 4 ++-- 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/CIValidations/CovarianceValidations.cpp b/CIValidations/CovarianceValidations.cpp index 697d48a8..8edbbb1d 100755 --- a/CIValidations/CovarianceValidations.cpp +++ b/CIValidations/CovarianceValidations.cpp @@ -236,6 +236,7 @@ void TestAdaptive(std::ostream& outFile, int main(int argc, char *argv[]) { SetMaCh3LoggerFormat(); + M3::rand::SetSeed(2137); if (argc != 1) { MACH3LOG_CRITICAL("You specified arguments, but none are needed. (Program name: {})", argv[0]); @@ -266,6 +267,9 @@ int main(int argc, char *argv[]) } xsec->ThrowParameters(); } + for (int i = 0; i < xsec->GetNumParams(); ++i) { + outFile << "Thrown param " << i << " is equal to "<< xsec->GetParCurr(i) << std::endl; + } xsec->SetParameters(ParProp); xsec->AcceptStep(); ///// Test Params from DetId ///// diff --git a/CIValidations/PredictiveValidations.cpp b/CIValidations/PredictiveValidations.cpp index 6cd56c16..4c03a31e 100755 --- a/CIValidations/PredictiveValidations.cpp +++ b/CIValidations/PredictiveValidations.cpp @@ -37,9 +37,27 @@ void PrepareConfig(const std::string& OriginalConfig) { configOut.close(); } + +void pValueCheck(std::ostream& outFile, TDirectory* predictiveDir) { + std::vector SampleNames = {"Tutorial_Beam", "Tutorial_ATM", "ND_CC0pi", "ND_CC1pip", "ND_CCPi0", "ND_NC0pi", "ND_NC1pi0", "Total"}; + + for (const auto& sampleName : SampleNames) { + TDirectory* sampleDir = static_cast(predictiveDir->Get(sampleName.c_str())); + if (!sampleDir) { + MACH3LOG_CRITICAL("Sample directory '{}' not found in 'Predictive'", sampleName); + throw MaCh3Exception(__FILE__, __LINE__); + } + auto histName = sampleName + "_predfluc_draw"; + TH2D* hist_pvalue = dynamic_cast(sampleDir->Get(histName.c_str())); + outFile << "Sample "<< sampleName<< " pvale " << hist_pvalue->GetTitle() << std::endl; + } +} + + int main(int argc, char *argv[]) { SetMaCh3LoggerFormat(); + M3::rand::SetSeed(2137); if (argc != 1) { MACH3LOG_CRITICAL("You specified arguments, but none are needed. (Program name: {})", argv[0]); @@ -52,7 +70,8 @@ int main(int argc, char *argv[]) std::string tutorialPath = std::string(rootEnv); std::string command = tutorialPath + "/bin/PredictiveTutorial " + - tutorialPath + "/mach3_temp_config.yaml"; + tutorialPath + "/mach3_temp_config.yaml General:Seed:2137"; + int ret = system(command.c_str()); if (ret != 0) { MACH3LOG_WARN("Error: system call failed with code {}", ret); @@ -144,6 +163,8 @@ int main(int argc, char *argv[]) } } + pValueCheck(outFile, predictiveDir); + const char* MaCh3Env = std::getenv("MaCh3_ROOT"); std::string MaCh3Path = std::string(MaCh3Env); diff --git a/CIValidations/README.md b/CIValidations/README.md index b5fbc2cb..01b4d0fc 100755 --- a/CIValidations/README.md +++ b/CIValidations/README.md @@ -39,6 +39,7 @@ ## PredictiveValidations * Checking if Predictive Thrower produces same format and posterior predictive value and errors are identical using stored toys +* Checking if pValue is always the same ## SplineValidations * Initialisation of SplineMonolith diff --git a/CIValidations/TestOutputs/CovarianceOut.txt b/CIValidations/TestOutputs/CovarianceOut.txt index 07d5a3f8..5ec091dc 100644 --- a/CIValidations/TestOutputs/CovarianceOut.txt +++ b/CIValidations/TestOutputs/CovarianceOut.txt @@ -1,4 +1,14 @@ Likelihood Xsec=266.956 +Thrown param 0 is equal to 0.977418 +Thrown param 1 is equal to 0.963047 +Thrown param 2 is equal to 1.26679 +Thrown param 3 is equal to 1.21713 +Thrown param 4 is equal to 1.21713 +Thrown param 5 is equal to 1.21713 +Thrown param 6 is equal to 1.21713 +Thrown param 7 is equal to 1.21713 +Thrown param 8 is equal to -0.00205295 +Thrown param 9 is equal to -0.00280439 Found 3 for Tutorial_Beam Params of type Norm Found 5 for Tutorial_Beam Params of type Spline Found 2 for Tutorial_Beam Params of type Functional diff --git a/CIValidations/TestOutputs/Predictive.txt b/CIValidations/TestOutputs/Predictive.txt index dd791516..2b92a27e 100644 --- a/CIValidations/TestOutputs/Predictive.txt +++ b/CIValidations/TestOutputs/Predictive.txt @@ -989,3 +989,11 @@ Sample ND_NC0pi bin (14, 5) content: 0 Sample ND_NC0pi bin (14, 5) error: 0 Sample ND_NC0pi bin (14, 6) content: 0 Sample ND_NC0pi bin (14, 6) error: 0 +Sample Tutorial_Beam pvale Tutorial_Beam_predfluc_draw_990/999=0.990991 +Sample Tutorial_ATM pvale Tutorial_ATM_predfluc_draw_937/999=0.937938 +Sample ND_CC0pi pvale ND_CC0pi_predfluc_draw_999/999=1 +Sample ND_CC1pip pvale ND_CC1pip_predfluc_draw_999/999=1 +Sample ND_CCPi0 pvale ND_CCPi0_predfluc_draw_999/999=1 +Sample ND_NC0pi pvale ND_NC0pi_predfluc_draw_999/999=1 +Sample ND_NC1pi0 pvale ND_NC1pi0_predfluc_draw_999/999=1 +Sample Total pvale Total_predfluc_draw_999/999=1 diff --git a/CMakeLists.txt b/CMakeLists.txt index efb488a1..d62b83b2 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ set(CMAKE_VERBOSE_MAKEFILE ON) cmake_minimum_required(VERSION 3.14 FATAL_ERROR) # Create project -project(MaCh3Tutorial VERSION 2.7.1 LANGUAGES CXX) +project(MaCh3Tutorial VERSION 2.8.0 LANGUAGES CXX) set(MaCh3Tutorial_VERSION ${PROJECT_VERSION}) set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -32,7 +32,7 @@ LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake/Modules) # download CPM.cmake file( DOWNLOAD - https://github.com/cpm-cmake/CPM.cmake/releases/download/v0.43.1/CPM.cmake + https://github.com/cpm-cmake/CPM.cmake/releases/download/v0.43.2/CPM.cmake ${CMAKE_CURRENT_BINARY_DIR}/cmake/CPM.cmake ) include(${CMAKE_CURRENT_BINARY_DIR}/cmake/CPM.cmake) From 345debd58fc5309fe059064da0150ad368ed5f80 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Wed, 30 Sep 2026 07:33:07 +0000 Subject: [PATCH 2/2] make fully reproducible CI --- CIValidations/CovarianceValidations.cpp | 6 ++++++ CIValidations/TestOutputs/CovarianceOut.txt | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CIValidations/CovarianceValidations.cpp b/CIValidations/CovarianceValidations.cpp index 8edbbb1d..3f79ab9a 100755 --- a/CIValidations/CovarianceValidations.cpp +++ b/CIValidations/CovarianceValidations.cpp @@ -237,6 +237,10 @@ int main(int argc, char *argv[]) { SetMaCh3LoggerFormat(); M3::rand::SetSeed(2137); + #ifdef MULTITHREAD + int oldThreads = omp_get_num_threads(); + omp_set_num_threads(4); + #endif if (argc != 1) { MACH3LOG_CRITICAL("You specified arguments, but none are needed. (Program name: {})", argv[0]); @@ -267,11 +271,13 @@ int main(int argc, char *argv[]) } xsec->ThrowParameters(); } + for (int i = 0; i < xsec->GetNumParams(); ++i) { outFile << "Thrown param " << i << " is equal to "<< xsec->GetParCurr(i) << std::endl; } xsec->SetParameters(ParProp); xsec->AcceptStep(); + ///// Test Params from DetId ///// const std::vector AffectedSamples = {"Tutorial_Beam", "Tutorial_ATM", "tutorial_beam", "blarb" "ATM"}; for (size_t id = 0; id < AffectedSamples.size(); ++id) diff --git a/CIValidations/TestOutputs/CovarianceOut.txt b/CIValidations/TestOutputs/CovarianceOut.txt index 5ec091dc..5e698566 100644 --- a/CIValidations/TestOutputs/CovarianceOut.txt +++ b/CIValidations/TestOutputs/CovarianceOut.txt @@ -1,14 +1,14 @@ Likelihood Xsec=266.956 -Thrown param 0 is equal to 0.977418 -Thrown param 1 is equal to 0.963047 -Thrown param 2 is equal to 1.26679 -Thrown param 3 is equal to 1.21713 -Thrown param 4 is equal to 1.21713 -Thrown param 5 is equal to 1.21713 -Thrown param 6 is equal to 1.21713 -Thrown param 7 is equal to 1.21713 -Thrown param 8 is equal to -0.00205295 -Thrown param 9 is equal to -0.00280439 +Thrown param 0 is equal to 0.955297 +Thrown param 1 is equal to 0.934278 +Thrown param 2 is equal to 0.847934 +Thrown param 3 is equal to 1.3331 +Thrown param 4 is equal to 1.61124 +Thrown param 5 is equal to 1.33056 +Thrown param 6 is equal to 1.09364 +Thrown param 7 is equal to 1.54087 +Thrown param 8 is equal to -0.000381615 +Thrown param 9 is equal to 0.00454828 Found 3 for Tutorial_Beam Params of type Norm Found 5 for Tutorial_Beam Params of type Spline Found 2 for Tutorial_Beam Params of type Functional