Merge branch 'feature/QPR-13626' into 'master'

QPR-13626 HistVaR: report historical pnl vectors on risk factor level

Closes QPR-13626

See merge request qs/oreplus!3086

Author: damienbarker

Docs/UserGuide/examples/examples.tex

@@ -615,8 +615,9 @@ \subsubsection{Historical Simulation VaR}
 The analytic is specified as usual in {\tt ore.xml} with the following parameters:
 \begin{itemize}
 \item outputFile: csv file name of the resulting VaR report
-%\item breakdown: boolean, if true the VaR report will contain a breakdown by risk class and risk type, otherwise the report shows the portfolio-lvel VaR only.
-\item tradePnl: boolean, if true the VaR report will contain a breakdown by tradeID, risk class and risk type, otherwise the report shows the portfolio-lvel VaR only.
+%\item breakdown: boolean, if true the VaR report will contain a breakdown by risk class and risk type, otherwise the report shows the portfolio-level VaR only.
+\item tradePnl: boolean, if true the VaR report will contain a breakdown by tradeID, risk class and risk type, otherwise the report shows the portfolio-level VaR only.
+\item riskFactorBreakdown: boolean, if true the VaR report will contain a breakdown by risk factor.
 \item quantiles: comma separated list of quantiles to be reported
 \item portfolioFilter (optional): Only trades with {\tt portfolioId} equal to the provided filter name are processed, see {\tt portfolio.xml}; the entire portfolio is processed, if omitted
 \item historicalPeriod: comma-separated date list, an even number of ordered dates is required (d1, d2, d3, d4, ...), where each pair (d1-d2, d3-d4, ...) defines the start and end of historical observation periods used

Examples/MarketRisk/Input/ore_histsimvar.xml

@@ -38,6 +38,7 @@
       <Parameter name="includeExpectedShortfall">Y</Parameter>
       <!--<Parameter name="breakdown">Y</Parameter>-->
       <Parameter name="tradePnl">N</Parameter>
+      <!--<Parameter name="riskFactorBreakdown">Y</Parameter>-->
       <!--<Parameter name="portfolioFilter">PF1</Parameter>-->
       <Parameter name="outputFile">var.csv</Parameter>
     </Analytic>

OREAnalytics/orea/app/analytics/varanalytic.cpp

@@ -120,7 +120,7 @@ void ParametricVarAnalyticImpl::setVarReport(const QuantLib::ext::shared_ptr<ore
         if (inputs_->outputHistoricalScenarios())
             ReportWriter().writeHistoricalScenarios(
                 scenarios->scenarioLoader(),
-                QuantLib::ext::make_shared<CSVFileReport>(path(inputs_->resultsPath() / "backtest_histscenrios.csv").string(),
+                QuantLib::ext::make_shared<CSVFileReport>(path(inputs_->resultsPath() / "backtest_histscenarios.csv").string(),
                                                   ',', false, inputs_->csvQuoteChar(), inputs_->reportNaString()));
 
         auto simMarket = QuantLib::ext::make_shared<ScenarioSimMarket>(
@@ -146,6 +146,13 @@ void ParametricVarAnalyticImpl::setVarReport(const QuantLib::ext::shared_ptr<ore
 void HistoricalSimulationVarAnalyticImpl::setUpConfigurations() {
     VarAnalyticImpl::setUpConfigurations();
     analytic()->configurations().simMarketParams = inputs_->histVarSimMarketParams();
+    // ORE Swig does not handle that yet
+    // inputs_->loadParameter<bool>(riskFactorBreakdown_, "historicalSimulationVar", "riskFactorBreakdown", false,
+    //                              std::function<bool(const string&)>(parseBool));
+    riskFactorBreakdown_ = inputs_->riskFactorBreakdown();
+    if(riskFactorBreakdown_){
+        allowPartialScenarios_ = true;
+    }
 }
 
 void HistoricalSimulationVarAnalyticImpl::setVarReport(
@@ -163,18 +170,17 @@ void HistoricalSimulationVarAnalyticImpl::setVarReport(
     auto scenarios = buildHistoricalScenarioGenerator(
         inputs_->scenarioReader(), adjFactors, benchmarkVarPeriod, inputs_->mporCalendar(), inputs_->mporDays(),
         analytic()->configurations().simMarketParams, analytic()->configurations().todaysMarketParams,
-        defaultReturnConfig, inputs_->mporOverlappingPeriods());
+        defaultReturnConfig, inputs_->mporOverlappingPeriods(), riskFactorBreakdown_);
 
     if (inputs_->outputHistoricalScenarios())
         ore::analytics::ReportWriter().writeHistoricalScenarios(
             scenarios->scenarioLoader(),
             QuantLib::ext::make_shared<CSVFileReport>(path(inputs_->resultsPath() / "var_histscenarios.csv").string(), ',',
                                               false, inputs_->csvQuoteChar(), inputs_->reportNaString()));
-
     auto simMarket = QuantLib::ext::make_shared<ScenarioSimMarket>(
         analytic()->market(), analytic()->configurations().simMarketParams, Market::defaultConfiguration,
         *analytic()->configurations().curveConfig, *analytic()->configurations().todaysMarketParams, true, false, false,
-        false, inputs_->iborFallbackConfig());
+        allowPartialScenarios_, inputs_->iborFallbackConfig());
     simMarket->scenarioGenerator() = scenarios;
     scenarios->baseScenario() = simMarket->baseScenario();
 
@@ -184,7 +190,7 @@ void HistoricalSimulationVarAnalyticImpl::setVarReport(
     varReport_ = ext::make_shared<HistoricalSimulationVarReport>(
         inputs_->baseCurrency(), analytic()->portfolio(), inputs_->portfolioFilter(), inputs_->varQuantiles(),
         benchmarkVarPeriod, scenarios, std::move(fullRevalArgs), inputs_->varBreakDown(),
-        inputs_->includeExpectedShortfall(), inputs_->tradePnl(), inputs_->useAtParCouponsCurves(),
+        inputs_->includeExpectedShortfall(), inputs_->tradePnl(), riskFactorBreakdown_, inputs_->useAtParCouponsCurves(),
         inputs_->useAtParCouponsTrades());
 }
 
@@ -195,8 +201,14 @@ void HistoricalSimulationVarAnalyticImpl::addAdditionalReports(
         QuantLib::ext::make_shared<InMemoryReport>(inputs_->reportBufferSize());
 
         reports->add(histPnLReport);
-
         analytic()->addReport(label_, "historical_PnL", histPnLReport);
+
+        if(riskFactorBreakdown_){
+            QuantLib::ext::shared_ptr<InMemoryReport> histPnLRFReport =
+                QuantLib::ext::make_shared<InMemoryReport>(inputs_->reportBufferSize());
+            reports->add(histPnLRFReport);
+            analytic()->addReport(label_, "riskFactor_PnL", histPnLRFReport);
+        }
 }
 
 } // namespace analytics

OREAnalytics/orea/app/analytics/varanalytic.hpp

@@ -93,6 +93,8 @@ class HistoricalSimulationVarAnalyticImpl : public VarAnalyticImpl {
 protected:
     void setVarReport(const QuantLib::ext::shared_ptr<ore::data::InMemoryLoader>& loader) override;
     void addAdditionalReports(const QuantLib::ext::shared_ptr<MarketRiskReport::Reports>& reports) override;
+    bool riskFactorBreakdown_ = false;
+    bool allowPartialScenarios_ = false;
 };
 
 class HistoricalSimulationVarAnalytic : public VarAnalytic {

OREAnalytics/orea/app/inputparameters.hpp

@@ -266,6 +266,7 @@ class InputParameters {
     void setVarQuantiles(const std::string& s); // parse to vector<Real>
     void setVarBreakDown(bool b) { varBreakDown_ = b; }
     void setTradePnl(bool b) { tradePnL_ = b; }
+    void setRiskFactorBreakdown(bool b) { riskFactorBreakdown_ = b; }
     void setIncludeExpectedShortfall(bool b) { includeExpectedShortfall_ = b; }
     void setPortfolioFilter(const std::string& s) { portfolioFilter_ = s; }
     void setVarMethod(const std::string& s) { varMethod_ = s; }
@@ -753,6 +754,7 @@ class InputParameters {
     const std::vector<Real>& varQuantiles() const { return varQuantiles_; }
     bool varBreakDown() const { return varBreakDown_; }
     bool tradePnl() const { return tradePnL_; }
+    bool riskFactorBreakdown() const { return riskFactorBreakdown_; }
     bool includeExpectedShortfall() const { return includeExpectedShortfall_; }
     const std::string& portfolioFilter() const { return portfolioFilter_; }
     const std::string& varMethod() const { return varMethod_; }
@@ -1227,6 +1229,7 @@ class InputParameters {
     std::vector<Real> varQuantiles_;
     bool varBreakDown_ = false;
     bool tradePnL_ = false;
+    bool riskFactorBreakdown_ = false;
     bool includeExpectedShortfall_ = false;
     std::string portfolioFilter_;
     // Delta, DeltaGammaNormal, MonteCarlo, Cornish-Fisher, Saddlepoint 

OREAnalytics/orea/app/oreapp.cpp

@@ -1416,8 +1416,8 @@ void OREAppInputParameters::loadParameters() {
             setVarBreakDown(parseBool(tmp));
 
         tmp = params_->get("historicalSimulationVar", "tradePnl", false);
-            if (tmp != "")
-                setTradePnl(parseBool(tmp));
+        if (tmp != "")
+            setTradePnl(parseBool(tmp));
 
         tmp = params_->get("historicalSimulationVar", "portfolioFilter", false);
         if (tmp != "")
@@ -1426,6 +1426,11 @@ void OREAppInputParameters::loadParameters() {
         tmp = params_->get("historicalSimulationVar", "outputHistoricalScenarios", false);
         if (tmp != "")
             setOutputHistoricalScenarios(parseBool(tmp));
+        
+        tmp = params_->get("historicalSimulationVar", "riskFactorBreakdown", false);
+        if (tmp != "")
+            setRiskFactorBreakdown(parseBool(tmp));
+        
     }
 
     /*************

OREAnalytics/orea/engine/historicalpnlgenerator.cpp

@@ -20,6 +20,7 @@
 
 #include <orea/engine/multithreadedvaluationengine.hpp>
 #include <orea/engine/valuationcalculator.hpp>
+#include <orea/engine/historicalsensipnlcalculator.hpp>
 
 #include <orea/cube/jointnpvcube.hpp>
 #include <orea/cube/inmemorycube.hpp>
@@ -94,7 +95,7 @@ HistoricalPnlGenerator::HistoricalPnlGenerator(
           return {QuantLib::ext::make_shared<NPVCalculator>(baseCurrency)};
       }) {}
 
-void HistoricalPnlGenerator::generateCube(const QuantLib::ext::shared_ptr<ScenarioFilter>& filter) {
+void HistoricalPnlGenerator::generateCube(const QuantLib::ext::shared_ptr<ScenarioFilter>& filter, const bool runRiskFactorBreakdown) {
 
     DLOG("Filling historical P&L cube for " << portfolio_->size() << " trades and " << hisScenGen_->numScenarios()
                                             << " scenarios.");
@@ -112,9 +113,32 @@ void HistoricalPnlGenerator::generateCube(const QuantLib::ext::shared_ptr<Scenar
         simMarket_->reset();
         simMarket_->scenarioGenerator() = hisScenGen_;
         hisScenGen_->baseScenario() = simMarket_->baseScenario();
+        if(hisScenGen_->isRiskFactorBreakdown() && runRiskFactorBreakdown){
+            //Run for RiskFactor PnL Breakdown
+            ext::shared_ptr<Scenario> sc = hisScenGen_->baseScenario();
+            std::vector<RiskFactorKey> deltaKeys = sc->keys();
+            for(auto const& key: deltaKeys){
+                // If we already have a non-empty shared_ptr for this key, skip rebuilding
+                auto it = mapCube_.find(key);
+                if (it != mapCube_.end() && it->second) {
+                    continue;
+                }
+                hisScenGen_->setCurrentKey(key);
+                hisScenGen_->setIterator(0);
+                ext::shared_ptr<NPVCube> newCube = ext::make_shared<InMemoryCubeOpt<double>>(simMarket_->asofDate(), portfolio_->ids(),
+                    vector<Date>(1, simMarket_->asofDate()), hisScenGen_->numScenarios());
+
+                valuationEngine_->buildCube(portfolio_, newCube, npvCalculator_(), ValuationEngine::ErrorPolicy::RemoveAll, true,
+                                        nullptr, nullptr, {}, dryRun_);
+                mapCube_[key] = newCube;
+                hisScenGen_->reset();
+            }
+            //Run for Full report - Remove risk factor key - reset counter
+            hisScenGen_->setCurrentKey(RiskFactorKey());
+            hisScenGen_->setIterator(0);
+        }
         valuationEngine_->buildCube(portfolio_, cube_, npvCalculator_(), ValuationEngine::ErrorPolicy::RemoveAll, true,
                                     nullptr, nullptr, {}, dryRun_);
-
     } else {
         MultiThreadedValuationEngine engine(
             nThreads_, today_, QuantLib::ext::make_shared<ore::analytics::DateGrid>(), hisScenGen_->numScenarios(), loader_,
@@ -225,6 +249,54 @@ TradePnlStore HistoricalPnlGenerator::tradeLevelPnl(const set<pair<string, Size>
 
 TradePnlStore HistoricalPnlGenerator::tradeLevelPnl() const { return tradeLevelPnl(timePeriod()); }
 
+using RiskFactorPnLSeries = HistoricalPnlGenerator::RiskFactorPnLSeries;
+RiskFactorPnLSeries HistoricalPnlGenerator::riskFactorLevelPnlSeries(const ore::data::TimePeriod& period) const {
+    RiskFactorPnLSeries series;
+    if (mapCube_.empty()) {
+        DLOG("riskFactorLevelPnlSeries: mapCube_ is empty; returning empty result");
+        return series;
+    }
+
+    // Build per-scenario maps for scenarios within the period
+    // Determine how many scenarios we have from any cube in the map
+    Size samples = mapCube_.begin()->second->samples();
+    series.resize(samples);
+
+    Size dateIdx = 0; // risk factor cubes have a single date at asof
+    for (Size s = 0; s < samples; ++s) {
+        Date start = hisScenGen_->startDates()[s];
+        Date end = hisScenGen_->endDates()[s];
+        if (!(period.contains(start) && period.contains(end))) {
+            continue; // leave this scenario's map empty
+        }
+        // Aggregate PnL across trades and collapse duplicates by risk factor name
+        std::unordered_map<std::string, std::pair<RiskFactorKey, Real>> byName;
+        for (const auto& kv : mapCube_) {
+            const RiskFactorKey& rfKey = kv.first;
+            const ext::shared_ptr<NPVCube>& rfCube = kv.second;
+            Real pnl = 0.0;
+            Size tradeCount = rfCube->numIds();
+            for (Size i = 0; i < tradeCount; ++i) {
+                pnl += rfCube->get(i, dateIdx, s) - rfCube->getT0(i);
+            }
+            if (pnl == 0.0)
+                continue;
+            auto it = byName.find(rfKey.name);
+            if (it == byName.end()) {
+                byName.emplace(rfKey.name, std::make_pair(rfKey, pnl));
+            } else {
+                it->second.second += pnl;
+            }
+        }
+        // Write one entry per risk factor name into the scenario map
+        for (const auto& e : byName) {
+            series[s].emplace(e.second.first, e.second.second);
+        }  
+    }
+
+    return series;
+}
+
 const QuantLib::ext::shared_ptr<NPVCube>& HistoricalPnlGenerator::cube() const { return cube_; }
 
 set<pair<string, Size>> HistoricalPnlGenerator::tradeIdIndexPairs() const {

OREAnalytics/orea/engine/historicalpnlgenerator.hpp

@@ -85,7 +85,7 @@ class HistoricalPnlGenerator : public ore::data::ProgressReporter {
         the scenarios provided by the historical scenario generator. The historical
         scenarios will have the given \p filter applied.
     */
-    void generateCube(const QuantLib::ext::shared_ptr<ScenarioFilter>& filter);
+    void generateCube(const QuantLib::ext::shared_ptr<ScenarioFilter>& filter, const bool runRiskFactorBreakdown = false);
 
     /*! Return a vector of historical portfolio P&L values restricted to scenarios
         falling in \p period and restricted to the given \p tradeIds. The P&L values
@@ -136,6 +136,10 @@ class HistoricalPnlGenerator : public ore::data::ProgressReporter {
         time and the second dimension is tradeId.
     */
     TradePnlStore tradeLevelPnl() const;
+    
+    // Series of per-scenario portfolio PnL by risk factor (aligned with scenarios)
+    using RiskFactorPnLSeries = std::vector<std::map<RiskFactorKey,QuantLib::Real>>;
+    RiskFactorPnLSeries riskFactorLevelPnlSeries(const ore::data::TimePeriod& period) const;
 
     /*! Return the last cube generated by generateCube.
      */
@@ -155,6 +159,7 @@ class HistoricalPnlGenerator : public ore::data::ProgressReporter {
     QuantLib::ext::shared_ptr<HistoricalScenarioGenerator> hisScenGen_;
     QuantLib::ext::shared_ptr<NPVCube> cube_;
     QuantLib::ext::shared_ptr<ValuationEngine> valuationEngine_;
+    std::map<RiskFactorKey, ext::shared_ptr<NPVCube>> mapCube_;
 
     // additional parameters needed for multi-threaded ctor
     QuantLib::ext::shared_ptr<ore::data::EngineData> engineData_;