git subrepo pull (merge) ore

subrepo:
  subdir:   "ore"
  merged:   "ab14f4f822"
upstream:
  origin:   "git@gitlab.acadiasoft.net:qs/ore.git"
  branch:   "master"
  commit:   "9c5daf7b1d"
git-subrepo:
  version:  "0.4.6"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "110b9eb"

Author: jenkins

Docker/Dockerfile-Test

@@ -5,7 +5,7 @@ RUN apt-get update \
   && DEBIAN_FRONTEND=noninteractive apt-get install -y dos2unix python3 python3-pip libxml2-utils xsltproc \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
-  && pip3 install matplotlib pandas nose nose_xunitmp datacompy jsondiff lxml
+  && pip3 install matplotlib pandas nose nose_xunitmp datacompy jsondiff lxml xmldiff
 
 CMD bash
 

OREAnalytics/orea/CMakeLists.txt

@@ -93,6 +93,7 @@ scenario/shiftscenariogenerator.cpp
 scenario/simplescenario.cpp
 scenario/stressscenariodata.cpp
 scenario/stressscenariogenerator.cpp
+simm/crif.cpp
 simm/crifloader.cpp
 simm/crifrecord.cpp
 simm/simmbasicnamemapper.cpp
@@ -242,6 +243,7 @@ scenario/simplescenario.hpp
 scenario/simplescenariofactory.hpp
 scenario/stressscenariodata.hpp
 scenario/stressscenariogenerator.hpp
+simm/crif.hpp
 simm/crifloader.hpp
 simm/crifrecord.hpp
 simm/simmbasicnamemapper.hpp

OREAnalytics/orea/app/analytics/parconversionanalytic.cpp

@@ -82,10 +82,18 @@ void ParConversionAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::I
 
         auto& configs = analytic()->configurations();
 
-        auto simMarket = buildScenarioSimMarketForSensitivityAnalysis(
-            analytic()->market(), configs.simMarketParams, configs.sensiScenarioData, configs.curveConfig,
-            configs.todaysMarketParams, nullptr, inputs_->marketConfig("pricing"), true, false,
-            *inputs_->iborFallbackConfig());
+        auto simMarket = boost::make_shared<ScenarioSimMarket>(
+            analytic()->market(), configs.simMarketParams, inputs_->marketConfig("pricing"),
+            configs.curveConfig ? *configs.curveConfig : ore::data::CurveConfigurations(),
+            configs.todaysMarketParams ? *configs.todaysMarketParams : ore::data::TodaysMarketParameters(), true,
+            configs.sensiScenarioData->useSpreadedTermStructures(), false, false, *inputs_->iborFallbackConfig());
+
+        auto scenarioGenerator = boost::make_shared<SensitivityScenarioGenerator>(
+            configs.sensiScenarioData, simMarket->baseScenario(), configs.simMarketParams, simMarket,
+            boost::make_shared<DeltaScenarioFactory>(simMarket->baseScenario()), true, std::string(), true,
+            simMarket->baseScenarioAbsolute());
+
+        simMarket->scenarioGenerator() = scenarioGenerator;
 
         parAnalysis->computeParInstrumentSensitivities(simMarket);
 

OREAnalytics/orea/app/analytics/simmanalytic.cpp

@@ -46,20 +46,16 @@ void SimmAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLo
     LOG("Get CRIF records from CRIF loader and fill amountUSD");        
     simmAnalytic->loadCrifRecords(loader);
 
-    auto simmConfiguration = inputs_->crifLoader()->simmConfiguration();
-
     if (analytic()->getWriteIntermediateReports()) {
         boost::shared_ptr<InMemoryReport> crifReport = boost::make_shared<InMemoryReport>();
-        ReportWriter(inputs_->reportNaString()).writeCrifReport(crifReport, *simmAnalytic->crifRecords());
+        ReportWriter(inputs_->reportNaString()).writeCrifReport(crifReport, simmAnalytic->crif());
         analytic()->reports()[LABEL]["crif"] = crifReport;
         LOG("CRIF report generated");
 
-        auto crifLoader = boost::make_shared<CrifLoader>(simmConfiguration, CrifRecord::additionalHeaders, true, true);
-        for (const auto& cr : *simmAnalytic->crifRecords())
-            crifLoader->add(cr);
+        Crif simmDataCrif = simmAnalytic->crif().aggregate();
         boost::shared_ptr<InMemoryReport> simmDataReport = boost::make_shared<InMemoryReport>();
         ReportWriter(inputs_->reportNaString())
-            .writeSIMMData(crifLoader->netRecords(true), simmDataReport);
+            .writeSIMMData(simmAnalytic->crif(), simmDataReport);
         analytic()->reports()[LABEL]["simm_data"] = simmDataReport;
         LOG("SIMM data report generated");
     }
@@ -72,8 +68,8 @@ void SimmAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLo
         inputs_->simmCalibrationData()->toFile((inputs_->resultsPath() / "simmcalibration.xml").string());
 
     // Calculate SIMM
-    auto simm = boost::make_shared<SimmCalculator>(*simmAnalytic->crifRecords(),
-                                                   simmConfiguration,
+    auto simm = boost::make_shared<SimmCalculator>(simmAnalytic->crif(),
+                                                   inputs_->getSimmConfiguration(),
                                                    inputs_->simmCalculationCurrency(),
                                                    inputs_->simmResultCurrency(),
                                                    analytic()->market(),
@@ -110,12 +106,11 @@ void SimmAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLo
 
 void SimmAnalytic::loadCrifRecords(const boost::shared_ptr<ore::data::InMemoryLoader>& loader) {
     QL_REQUIRE(inputs_, "Inputs not set");
-    QL_REQUIRE(inputs_->crifLoader(), "CRIF loader not set");
-    QL_REQUIRE(inputs_->crifLoader()->hasCrifRecords(), "CRIF loader does not contain any records");
+    QL_REQUIRE(!inputs_->crif().empty(), "CRIF loader does not contain any records");
 
-    inputs_->crifLoader()->fillAmountUsd(market());
-    crifRecords_ = boost::make_shared<SimmNetSensitivities>(inputs_->crifLoader()->netRecords(true));
-    hasNettingSetDetails_ = inputs_->crifLoader()->hasNettingSetDetails();
+    crif_ = inputs_->crif();
+    crif_.fillAmountUsd(market());
+    hasNettingSetDetails_ = crif_.hasNettingSetDetails();
 }
 
 } // namespace analytics

OREAnalytics/orea/app/analytics/simmanalytic.hpp

@@ -22,6 +22,7 @@
 #pragma once
 
 #include <orea/app/analytic.hpp>
+#include <orea/simm/crif.hpp>
 
 namespace ore {
 namespace analytics {
@@ -40,23 +41,23 @@ class SimmAnalyticImpl : public Analytic::Impl {
 
 class SimmAnalytic : public Analytic {
 public:
-    SimmAnalytic(const boost::shared_ptr<InputParameters>& inputs,
-                 const boost::shared_ptr<SimmNetSensitivities>& crifRecords = nullptr,
+    SimmAnalytic(const boost::shared_ptr<InputParameters>& inputs, const Crif& crif = Crif(),
                  const bool hasNettingSetDetails = false,
                  const bool determineWinningRegulations = true)
         : Analytic(std::make_unique<SimmAnalyticImpl>(inputs), {"SIMM"}, inputs, false, false, false, false),
+          crif_(crif),
           hasNettingSetDetails_(hasNettingSetDetails),
           determineWinningRegulations_(determineWinningRegulations) {}
 
-    const boost::shared_ptr<SimmNetSensitivities>& crifRecords() const { return crifRecords_; }
+    const Crif& crif() const { return crif_; }
     bool hasNettingSetDetails() { return hasNettingSetDetails_; }
     bool determineWinningRegulations() { return determineWinningRegulations_; }
 
     //! Load CRIF from external source, override to generate CRIF
     virtual void loadCrifRecords(const boost::shared_ptr<ore::data::InMemoryLoader>& loader);
 
 private:
-    boost::shared_ptr<SimmNetSensitivities> crifRecords_;
+    Crif crif_;
     bool hasNettingSetDetails_;
     bool determineWinningRegulations_;
 };

OREAnalytics/orea/app/inputparameters.cpp

@@ -27,6 +27,7 @@
 #include <ored/utilities/currencyconfig.hpp>
 #include <ored/utilities/parsers.hpp>
 #include <ored/portfolio/scriptedtrade.hpp>
+#include <orea/simm/crifloader.hpp>
 
 namespace ore {
 namespace analytics {
@@ -43,6 +44,7 @@ vector<string> getFileNames(const string& fileString, const string& path) {
 
 InputParameters::InputParameters() {
     iborFallbackConfig_ = boost::make_shared<IborFallbackConfig>(IborFallbackConfig::defaultConfig());
+    simmBucketMapper_ = boost::make_shared<SimmBucketMapperBase>();
     loadParameters();
 }
 
@@ -427,26 +429,23 @@ void InputParameters::setCreditSimulationParametersFromBuffer(const std::string&
     creditSimulationParameters_ = boost::make_shared<CreditSimulationParameters>();
     creditSimulationParameters_->fromXMLString(xml);
 } 
-
-void InputParameters::setCrifLoader() {
-    boost::shared_ptr<SimmConfiguration> configuration = buildSimmConfiguration(
-        simmVersion_, boost::make_shared<SimmBucketMapperBase>(), simmCalibrationData(), mporDays());
-    bool updateMappings = true;
-    bool aggregateTrades = false;
-    crifLoader_ =
-        boost::make_shared<CrifLoader>(configuration, CrifRecord::additionalHeaders, updateMappings, aggregateTrades);
-}
 
 void InputParameters::setCrifFromFile(const std::string& fileName, char eol, char delim, char quoteChar, char escapeChar) {
-    if (!crifLoader_)
-        setCrifLoader();
-    crifLoader_->loadFromFile(fileName, eol, delim, quoteChar, escapeChar);
+    boost::shared_ptr<SimmConfiguration> configuration =
+        buildSimmConfiguration(simmVersion_, boost::make_shared<SimmBucketMapperBase>(),simmCalibrationData(), mporDays());
+    bool updateMappings = true;
+    bool aggregateTrades = false;
+    auto crifLoader = CsvFileCrifLoader(fileName, configuration, CrifRecord::additionalHeaders, updateMappings, aggregateTrades, eol, delim, quoteChar, escapeChar, reportNaString());
+    crif_ = crifLoader.loadCrif();
 }
 
 void InputParameters::setCrifFromBuffer(const std::string& csvBuffer, char eol, char delim, char quoteChar, char escapeChar) {
-    if (!crifLoader_)
-        setCrifLoader();
-    crifLoader_->loadFromString(csvBuffer, eol, delim, quoteChar, escapeChar);
+    boost::shared_ptr<SimmConfiguration> configuration =
+        buildSimmConfiguration(simmVersion_, boost::make_shared<SimmBucketMapperBase>(), simmCalibrationData(), mporDays());
+    bool updateMappings = true;
+    bool aggregateTrades = false;
+    auto crifLoader = CsvBufferCrifLoader(csvBuffer, configuration, CrifRecord::additionalHeaders, updateMappings, aggregateTrades, eol, delim, quoteChar, escapeChar, reportNaString());
+    crif_ = crifLoader.loadCrif();
 }
 
 void InputParameters::setSimmNameMapper(const std::string& xml) {
@@ -608,5 +607,13 @@ Date InputParameters::mporDate() {
     }
     return mporDate_;
 }
+
+boost::shared_ptr<SimmConfiguration> InputParameters::getSimmConfiguration() {
+    QL_REQUIRE(simmBucketMapper() != nullptr,
+               "Internal error, load simm bucket mapper before retrieving simmconfiguration");
+    return buildSimmConfiguration(simmVersion(), simmBucketMapper(), simmCalibrationData(), mporDays());
+}
+
+
 } // namespace analytics
 } // namespace ore

OREAnalytics/orea/app/inputparameters.hpp

@@ -22,9 +22,13 @@
 
 #pragma once
 
+#include <boost/filesystem/path.hpp>
+#include <orea/aggregation/creditsimulationparameters.hpp>
 #include <orea/app/parameters.hpp>
 #include <orea/cube/npvcube.hpp>
-#include <orea/aggregation/creditsimulationparameters.hpp>
+#include <orea/engine/sensitivitystream.hpp>
+#include <orea/scenario/scenariogenerator.hpp>
+#include <orea/scenario/scenariogeneratorbuilder.hpp>
 #include <orea/scenario/scenariosimmarketparameters.hpp>
 #include <orea/scenario/sensitivityscenariodata.hpp>
 #include <orea/scenario/stressscenariodata.hpp>
@@ -33,18 +37,19 @@
 #include <orea/engine/sensitivitystream.hpp>
 #include <orea/simm/crifloader.hpp>
 #include <orea/simm/simmcalibration.hpp>
+#include <orea/simm/crif.hpp>
 #include <orea/simm/simmbasicnamemapper.hpp>
 #include <orea/simm/simmbucketmapper.hpp>
+#include <orea/simm/simmconfiguration.hpp>
 #include <ored/configuration/curveconfigurations.hpp>
 #include <ored/configuration/iborfallbackconfig.hpp>
-#include <ored/model/crossassetmodeldata.hpp>
+#include <ored/marketdata/csvloader.hpp>
 #include <ored/marketdata/todaysmarketparameters.hpp>
+#include <ored/model/crossassetmodeldata.hpp>
 #include <ored/portfolio/nettingsetmanager.hpp>
 #include <ored/portfolio/portfolio.hpp>
 #include <ored/portfolio/referencedata.hpp>
-#include <ored/marketdata/csvloader.hpp>
 #include <ored/utilities/csvfilereader.hpp>
-#include <boost/filesystem/path.hpp>
 
 namespace ore {
 namespace analytics {
@@ -283,7 +288,7 @@ class InputParameters {
 
     // Setters for SIMM
     void setSimmVersion(const std::string& s) { simmVersion_ = s; }
-    void setCrifLoader();
+    
     void setCrifFromFile(const std::string& fileName,
                          char eol = '\n', char delim = ',', char quoteChar = '\0', char escapeChar = '\\');
     void setCrifFromBuffer(const std::string& csvBuffer,
@@ -553,14 +558,15 @@ class InputParameters {
      * Getters for SIMM
      ******************/
     const std::string& simmVersion() { return simmVersion_; }
-    const boost::shared_ptr<ore::analytics::CrifLoader>& crifLoader() { return crifLoader_; }
+    const ore::analytics::Crif& crif() { return crif_; }
     const boost::shared_ptr<ore::analytics::SimmBasicNameMapper>& simmNameMapper() { return simmNameMapper_; }
     const boost::shared_ptr<ore::analytics::SimmBucketMapper>& simmBucketMapper() { return simmBucketMapper_; }
     const boost::shared_ptr<ore::analytics::SimmCalibrationData>& simmCalibrationData() { return simmCalibrationData_; }
     const std::string& simmCalculationCurrency() { return simmCalculationCurrency_; }
     const std::string& simmResultCurrency() { return simmResultCurrency_; }
     const std::string& simmReportingCurrency() { return simmReportingCurrency_; }
     bool enforceIMRegulations() { return enforceIMRegulations_; }
+    boost::shared_ptr<SimmConfiguration> getSimmConfiguration();
 
     /**************************************************
      * Getters for Zero to Par Sensi conversion
@@ -805,7 +811,7 @@ class InputParameters {
      * SIMM analytic
      ***************/
     std::string simmVersion_;
-    boost::shared_ptr<ore::analytics::CrifLoader> crifLoader_;
+    ore::analytics::Crif crif_;
     boost::shared_ptr<ore::analytics::SimmBasicNameMapper> simmNameMapper_;
     boost::shared_ptr<ore::analytics::SimmBucketMapper> simmBucketMapper_;
     boost::shared_ptr<ore::analytics::SimmCalibrationData> simmCalibrationData_;

OREAnalytics/orea/app/reportwriter.cpp

@@ -1491,7 +1491,7 @@ void ReportWriter::writeCube(ore::data::Report& report, const boost::shared_ptr<
 }
 
 // Ease notation again
-typedef SimmConfiguration::ProductClass ProductClass;
+typedef CrifRecord::ProductClass ProductClass;
 typedef SimmConfiguration::RiskClass RiskClass;
 typedef SimmConfiguration::MarginType MarginType;
 typedef SimmConfiguration::SimmSide SimmSide;
@@ -1660,7 +1660,7 @@ void ReportWriter::writeSIMMReport(
     }
 }
 
-void ReportWriter::writeSIMMData(const SimmNetSensitivities& simmData, const boost::shared_ptr<Report>& dataReport,
+void ReportWriter::writeSIMMData(const ore::analytics::Crif& simmData, const boost::shared_ptr<Report>& dataReport,
                                  const bool hasNettingSetDetails) {
 
     LOG("Writing SIMM data report.");
@@ -1708,7 +1708,7 @@ void ReportWriter::writeSIMMData(const SimmNetSensitivities& simmData, const boo
         // Same check as above, but for backwards compatibility, if im_model is not used
         // but Risk::Type is PV or Notional
         if (cr.imModel.empty() &&
-            (cr.riskType == SimmConfiguration::RiskType::Notional || cr.riskType == SimmConfiguration::RiskType::PV))
+            (cr.riskType == CrifRecord::RiskType::Notional || cr.riskType == CrifRecord::RiskType::PV))
             continue;
 
         // Write current netted CRIF record
@@ -1743,13 +1743,13 @@ void ReportWriter::writeSIMMData(const SimmNetSensitivities& simmData, const boo
     LOG("SIMM data report written.");
 }
 
-void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, const SimmNetSensitivities& crifRecords) {
+void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, const Crif& crif) {
 
     // If we have SIMM parameters, check if at least one of them uses netting set details optional field/s
     // It is easier to check here than to pass the flag from other places, since otherwise we'd have to handle certain edge cases
     // e.g. SIMM parameters use optional NSDs, but trades don't. So SIMM report should not display NSDs, but CRIF report still should.
     bool hasNettingSetDetails = false;
-    for (const CrifRecord& cr : crifRecords) {
+    for (const auto& cr : crif) {
         if (!cr.nettingSetDetails.emptyOptionalFields())
             hasNettingSetDetails = true;
     }
@@ -1758,7 +1758,7 @@ void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, cons
     bool hasCollectRegulations = false;
     bool hasPostRegulations = false;
     bool hasScheduleTrades = false;
-    for (const auto& cr : crifRecords) {
+    for (const auto& cr : crif) {
         // Check which additional fields are being used/populated
         for (const auto& af : cr.additionalFields) {
             if (std::find(addFields.begin(), addFields.end(), af.first) == addFields.end()) {
@@ -1803,9 +1803,18 @@ void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, cons
         .addColumn("IMModel", string())
         .addColumn("TradeType", string());
 
-    if (hasScheduleTrades)
+    if (hasScheduleTrades || crif.type() == Crif::CrifType::Frtb)
         report->addColumn("end_date", string());
 
+    if (crif.type() == Crif::CrifType::Frtb) {
+        report->addColumn("Label3", string())
+            .addColumn("CreditQuality", string())
+            .addColumn("LongShortInd", string())
+            .addColumn("CoveredBondInd", string())
+            .addColumn("TrancheThickness", string())
+            .addColumn("BB_RW", string());
+    }
+
     if (hasCollectRegulations)
         report->addColumn("collect_regulations", string());
 
@@ -1818,7 +1827,7 @@ void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, cons
     }
 
     // Write individual CRIF records
-    for (const auto& cr : crifRecords) {
+    for (const auto& cr : crif) {
 
         report->next().add(cr.tradeId).add(cr.portfolioId);
 
@@ -1840,9 +1849,18 @@ void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, cons
             .add(cr.imModel)
             .add(cr.tradeType);
 
-        if (hasScheduleTrades)
+        if (hasScheduleTrades || crif.type() == Crif::CrifType::Frtb)
             report->add(cr.endDate);
 
+        if (crif.type() == Crif::CrifType::Frtb) {
+            report->add(cr.label3)
+                .add(cr.creditQuality)
+                .add(cr.longShortInd)
+                .add(cr.coveredBondInd)
+                .add(cr.trancheThickness)
+                .add(cr.bb_rw);
+        }
+
         if (hasCollectRegulations) {
             string regString = escapeCommaSeparatedList(cr.collectRegulations, '\0');
             report->add(regString);
@@ -1857,7 +1875,7 @@ void ReportWriter::writeCrifReport(const boost::shared_ptr<Report>& report, cons
             if (cr.additionalFields.find(af) == cr.additionalFields.end())
                 report->add("");
             else
-                report->add(cr.additionalFields.at(af));
+                report->add(cr.getAdditionalFieldAsStr(af));
         }
     }