Merge branch 'master' into 'QPR-12101'

Author: farahkhashman

Docker/.env

@@ -3,7 +3,7 @@ DEBIAN_TAG=11.7
 # it's recommended to include CMAKE_BUILD_TYPE and BOOST_VARIANT in QL_TAG, ORE_TAG, BOOST_TAG
 # to distinguish a release build from a debug build
 
-QL_TAG=1.31.1_f5b9affc3
+QL_TAG=1.31.1_f5b9affc
 ORE_TAG=latest
 
 # debug or release

OREAnalytics/orea/CMakeLists.txt

@@ -26,6 +26,7 @@ app/analytics/simmanalytic.cpp
 app/analytics/varanalytic.cpp
 app/analytics/xvaanalytic.cpp
 app/analyticsmanager.cpp
+app/cleanupsingletons.cpp
 app/inputparameters.cpp
 app/marketcalibrationreport.cpp
 app/marketdatacsvloader.cpp
@@ -169,6 +170,7 @@ app/analytics/simmanalytic.hpp
 app/analytics/varanalytic.hpp
 app/analytics/xvaanalytic.hpp
 app/analyticsmanager.hpp
+app/cleanupsingletons.hpp
 app/inputparameters.hpp
 app/marketcalibrationreport.hpp
 app/marketdatacsvloader.hpp

OREAnalytics/orea/app/cleanupsingletons.cpp

@@ -0,0 +1,62 @@
+/*
+ Copyright (C) 2017 Quaternion Risk Management Ltd
+ All rights reserved.
+
+ This file is part of ORE, a free-software/open-source library
+ for transparent pricing and risk analysis - http://opensourcerisk.org
+
+ ORE is free software: you can redistribute it and/or modify it
+ under the terms of the Modified BSD License.  You should have received a
+ copy of the license along with this program.
+ The license is also available online at <http://opensourcerisk.org>
+
+ This program is distributed on the basis that it will form a useful
+ contribution to risk analytics and model standardisation, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
+*/
+
+#include <orea/app/cleanupsingletons.hpp>
+#include <orea/engine/observationmode.hpp>
+
+#include <ored/portfolio/scriptedtrade.hpp>
+#include <ored/utilities/calendarparser.hpp>
+#include <ored/utilities/currencyparser.hpp>
+#include <ored/utilities/indexnametranslator.hpp>
+#include <ored/utilities/log.hpp>
+
+#include <qle/math/computeenvironment.hpp>
+#include <qle/math/randomvariable.hpp>
+#include <qle/pricingengines/mcmultilegbaseengine.hpp>
+#include <qle/utilities/savedobservablesettings.hpp>
+
+namespace ore::analytics {
+
+CleanUpThreadLocalSingletons::~CleanUpThreadLocalSingletons() {
+    QuantLib::IndexManager::instance().clearHistories();
+    QuantExt::DividendManager::instance().clearHistories();
+    ore::analytics::ObservationMode::instance().setMode(ore::analytics::ObservationMode::Mode::None);
+    QuantExt::ComputeEnvironment::instance().reset();
+    QuantExt::RandomVariableStats::instance().reset();
+    QuantExt::McEngineStats::instance().reset();
+}
+
+CleanUpThreadGlobalSingletons::~CleanUpThreadGlobalSingletons() {
+    ore::data::InstrumentConventions::instance().clear();
+    ore::data::IndexNameTranslator::instance().clear();
+    ore::data::CalendarParser::instance().reset();
+    ore::data::CurrencyParser::instance().reset();
+    ore::data::ScriptLibraryStorage::instance().clear();
+}
+
+CleanUpLogSingleton::CleanUpLogSingleton(const bool removeLoggers, const bool clearIndependentLoggers)
+    : removeLoggers_(removeLoggers), clearIndependentLoggers_(clearIndependentLoggers) {}
+
+CleanUpLogSingleton::~CleanUpLogSingleton() {
+    if (clearIndependentLoggers_)
+        ore::data::Log::instance().clearAllIndependentLoggers();
+    if (removeLoggers_)
+        ore::data::Log::instance().removeAllLoggers();
+}
+
+} // namespace ore::data

OREAnalytics/orea/app/cleanupsingletons.hpp

@@ -0,0 +1,47 @@
+/*
+ Copyright (C) 2017 Quaternion Risk Management Ltd
+ All rights reserved.
+
+ This file is part of ORE, a free-software/open-source library
+ for transparent pricing and risk analysis - http://opensourcerisk.org
+
+ ORE is free software: you can redistribute it and/or modify it
+ under the terms of the Modified BSD License.  You should have received a
+ copy of the license along with this program.
+ The license is also available online at <http://opensourcerisk.org>
+
+ This program is distributed on the basis that it will form a useful
+ contribution to risk analytics and model standardisation, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
+*/
+
+#pragma once
+
+#include <ql/settings.hpp>
+#include <qle/utilities/savedobservablesettings.hpp>
+
+namespace ore::analytics {
+
+class CleanUpThreadLocalSingletons {
+public:
+    QuantLib::SavedSettings savedSettings;
+    QuantExt::SavedObservableSettings savedObservableSettings;
+    ~CleanUpThreadLocalSingletons();
+};
+
+class CleanUpThreadGlobalSingletons {
+public:
+    ~CleanUpThreadGlobalSingletons();
+};
+
+class CleanUpLogSingleton {
+public:
+    CleanUpLogSingleton(const bool removeLoggers = false, const bool clearIndependentLoggers = true);
+    ~CleanUpLogSingleton();
+
+private:
+    bool removeLoggers_, clearIndependentLoggers_;
+};
+
+} // namespace ore::analytics

OREAnalytics/orea/app/oreapp.cpp

@@ -25,6 +25,7 @@
 #pragma warning(disable : 4503)
 #endif
 
+#include <orea/app/cleanupsingletons.hpp>
 #include <orea/app/marketdatacsvloader.hpp>
 #include <orea/app/marketdatainmemoryloader.hpp>
 #include <orea/app/oreapp.hpp>
@@ -40,6 +41,7 @@
 
 #include <qle/version.hpp>
 
+
 #include <ql/cashflows/floatingratecoupon.hpp>
 #include <ql/time/calendars/all.hpp>
 #include <ql/time/daycounters/all.hpp>
@@ -157,7 +159,7 @@ boost::shared_ptr<AggregationScenarioData> OREApp::getMarketCube(std::string cub
 }
 
 std::vector<std::string> OREApp::getErrors() {
-    return structuredLogger_->messages();
+    return errorMessages_;
 }
 
 Real OREApp::getRunTime() {
@@ -287,60 +289,57 @@ void OREApp::analytics() {
     LOG("ORE analytics done");
 }
 
-OREApp::OREApp(boost::shared_ptr<Parameters> params, bool console, 
-               const boost::filesystem::path& logRootPath)
-    : params_(params), inputs_(nullptr) {
 
-    if (console) {
+void OREApp::initFromParams() {
+    if (console_) {
         ConsoleLog::instance().switchOn();
     }
 
-    string outputPath = params_->get("setup", "outputPath");
-    string logFile = outputPath + "/" + params_->get("setup", "logFile");
-    Size logMask = 15;
+    outputPath_ = params_->get("setup", "outputPath");
+    logFile_ = outputPath_ + "/" + params_->get("setup", "logFile");
+    logMask_ = 15;
     // Get log mask if available
     if (params_->has("setup", "logMask")) {
-        logMask = static_cast<Size>(parseInteger(params_->get("setup", "logMask")));
+        logMask_ = static_cast<Size>(parseInteger(params_->get("setup", "logMask")));
     }
 
-    string progressLogFile, structuredLogFile;
-    Size progressLogRotationSize = 0;
-    bool progressLogToConsole = false;
-    Size structuredLogRotationSize = 0;
+    progressLogRotationSize_ = 0;
+    progressLogToConsole_ = false;
+    structuredLogRotationSize_ = 0;
 
     if (params_->hasGroup("logging")) {
-        string logFileOverride = params_->get("logging", "logFile", false);
-        if (!logFileOverride.empty()) {
-            logFile = outputPath + '/' + logFileOverride;
+        string tmp = params_->get("logging", "logFile", false);
+        if (!tmp.empty()) {
+            logFile_ = outputPath_ + '/' + tmp;
         }
-        string logMaskOverride = params_->get("logging", "logMask", false);
-        if (!logMaskOverride.empty()) {
-            logMask = static_cast<Size>(parseInteger(logMaskOverride));
+        tmp = params_->get("logging", "logMask", false);
+        if (!tmp.empty()) {
+            logMask_ = static_cast<Size>(parseInteger(tmp));
         }
-        progressLogFile = params_->get("logging", "progressLogFile", false);
-        if (!progressLogFile.empty()) {
-            progressLogFile = outputPath + '/' + progressLogFile;
+        tmp = params_->get("logging", "progressLogFile", false);
+        if (!tmp.empty()) {
+            progressLogFile_ = outputPath_ + '/' + tmp;
         }
-        string tmp = params_->get("logging", "progressLogRotationSize", false);
+        tmp = params_->get("logging", "progressLogRotationSize", false);
         if (!tmp.empty()) {
-            progressLogRotationSize = static_cast<Size>(parseInteger(tmp));
+            progressLogRotationSize_ = static_cast<Size>(parseInteger(tmp));
         }
         tmp = params_->get("logging", "progressLogToConsole", false);
         if (!tmp.empty()) {
-            progressLogToConsole = ore::data::parseBool(tmp);
+            progressLogToConsole_ = ore::data::parseBool(tmp);
         }
-        structuredLogFile = params_->get("logging", "structuredLogFile", false);
-        if (!structuredLogFile.empty()) {
-            structuredLogFile = outputPath + '/' + structuredLogFile;
+        tmp = params_->get("logging", "structuredLogFile", false);
+        if (!tmp.empty()) {
+            structuredLogFile_ = outputPath_ + '/' + tmp;
         }
         tmp = params_->get("logging", "structuredLogRotationSize", false);
         if (!tmp.empty()) {
-            structuredLogRotationSize = static_cast<Size>(parseInteger(tmp));
+            structuredLogRotationSize_ = static_cast<Size>(parseInteger(tmp));
         }
     }
 
-    setupLog(outputPath, logFile, logMask, logRootPath, progressLogFile, progressLogRotationSize, progressLogToConsole,
-             structuredLogFile, structuredLogRotationSize);
+    setupLog(outputPath_, logFile_, logMask_, logRootPath_, progressLogFile_, progressLogRotationSize_, progressLogToConsole_,
+             structuredLogFile_, structuredLogRotationSize_);
 
     // Log the input parameters
     params_->log();
@@ -353,20 +352,18 @@ OREApp::OREApp(boost::shared_ptr<Parameters> params, bool console,
     CONSOLE("OK");
 
     Settings::instance().evaluationDate() = inputs_->asof();
-}
-
-OREApp::OREApp(const boost::shared_ptr<InputParameters>& inputs, const std::string& logFile, Size logLevel,
-               bool console, const boost::filesystem::path& logRootPath)
-    : params_(nullptr), inputs_(inputs) {
-
+} 
+  
+void OREApp::initFromInputs() {
     // Initialise Singletons
     Settings::instance().evaluationDate() = inputs_->asof();
     InstrumentConventions::instance().setConventions(inputs_->conventions());
-    if (console) {
+    if (console_) {
         ConsoleLog::instance().switchOn();
     }
 
-    setupLog(inputs_->resultsPath().string(), logFile, logLevel, logRootPath);
+    outputPath_ = inputs_->resultsPath().string();
+    setupLog(outputPath_, logFile_, logMask_, logRootPath_);
 }
 
 OREApp::~OREApp() {
@@ -376,6 +373,24 @@ OREApp::~OREApp() {
 
 void OREApp::run() {
 
+    // only one thread at a time should call run
+    static std::mutex _s_mutex;
+    std::lock_guard<std::mutex> lock(_s_mutex);
+
+    // clean up after finishing the run
+    CleanUpThreadLocalSingletons cleanupThreadLocalSingletons;
+    CleanUpThreadGlobalSingletons cleanupThreadGloablSingletons;
+    CleanUpLogSingleton cleanupLogSingleton(true, true);
+
+    if (inputs_ == nullptr)
+        initFromParams();
+    else if (params_ == nullptr)
+        initFromInputs();
+    else {
+        ALOG("both inputs are empty");
+        return;      
+    }
+    
     runTimer_.start();
 
     try {
@@ -389,13 +404,35 @@ void OREApp::run() {
 
     runTimer_.stop();
 
+    // cache the error messages because we reset the loggers 
+    errorMessages_ = structuredLogger_->messages();
+
     CONSOLE("run time: " << runTimer_.format(default_places, "%w") << " sec");
     CONSOLE("ORE done.");
     LOG("ORE done.");
 }
 
 void OREApp::run(const std::vector<std::string>& marketData,
                  const std::vector<std::string>& fixingData) {
+
+    // only one thread at a time should call run
+    static std::mutex _s_mutex;
+    std::lock_guard<std::mutex> lock(_s_mutex);
+
+    // clean up after finishing the run
+    CleanUpThreadLocalSingletons cleanupThreadLocalSingletons;
+    CleanUpThreadGlobalSingletons cleanupThreadGloablSingletons;
+    CleanUpLogSingleton cleanupLogSingleton(true, true);
+
+    if (inputs_ == nullptr)
+        initFromParams();
+    else if (params_ == nullptr)
+        initFromInputs();
+    else {
+        ALOG("both inputs are empty");
+        return;      
+    }
+
     runTimer_.start();
 
     try {

OREAnalytics/orea/app/oreapp.hpp

@@ -41,10 +41,14 @@ class OREApp {
 public:
     //! Constructor that uses ORE parameters and input data from files
     OREApp(boost::shared_ptr<Parameters> params, bool console = false, 
-           const boost::filesystem::path& = boost::filesystem::path());
+           const boost::filesystem::path& logRootPath = boost::filesystem::path())
+      : params_(params), inputs_(nullptr), console_(console), logRootPath_(logRootPath) {}
+
     //! Constructor that assumes we have already assembled input parameters via API
     OREApp(const boost::shared_ptr<InputParameters>& inputs, const std::string& logFile, Size logLevel = 31,
-           bool console = false, const boost::filesystem::path& = boost::filesystem::path());
+           bool console = false, const boost::filesystem::path& logRootPath = boost::filesystem::path())
+      : params_(nullptr), inputs_(inputs), logFile_(logFile), logMask_(logLevel), console_(console),
+	logRootPath_(logRootPath) {}
 
     //! Destructor
     virtual ~OREApp();
@@ -93,6 +97,9 @@ class OREApp {
     //! remove logs
     void closeLog();
 
+    void initFromParams();
+    void initFromInputs();
+  
     //! ORE Input parameters
     boost::shared_ptr<Parameters> params_;
     boost::shared_ptr<InputParameters> inputs_;
@@ -101,6 +108,21 @@ class OREApp {
     boost::shared_ptr<AnalyticsManager> analyticsManager_;
     boost::shared_ptr<StructuredLogger> structuredLogger_;
     boost::timer::cpu_timer runTimer_;
+
+    //! Logging
+    string logFile_;
+    Size logMask_;
+    bool console_;
+    string outputPath_;
+    boost::filesystem::path logRootPath_;
+    string progressLogFile_ = "";
+    QuantLib::Size progressLogRotationSize_ = 100 * 1024 * 1024;
+    bool progressLogToConsole_ = false;
+    string structuredLogFile_ = "";
+    QuantLib::Size structuredLogRotationSize_ = 100 * 1024 * 1024;
+
+    // Cached error messages of a run
+    std::vector<std::string> errorMessages_;
 };
 
 class OREAppInputParameters : virtual public InputParameters {