git subrepo pull (merge) ore

subrepo:
  subdir:   "ore"
  merged:   "628188fc7a"
upstream:
  origin:   "git@gitlab.acadiasoft.net:qs/ore.git"
  branch:   "master"
  commit:   "3545afeea3"
git-subrepo:
  version:  "0.4.6"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "73a0129"

Author: jenkins

OREAnalytics/orea/app/inputparameters.cpp

@@ -192,6 +192,11 @@ void InputParameters::setScenarioSimMarketParamsFromFile(const std::string& file
     scenarioSimMarketParams_->fromFile(fileName);
 }
 
+void InputParameters::setHistVarSimMarketParams(const std::string& xml) {
+    histVarSimMarketParams_ = QuantLib::ext::make_shared<ScenarioSimMarketParameters>();
+    histVarSimMarketParams_->fromXMLString(xml);
+}
+
 void InputParameters::setHistVarSimMarketParamsFromFile(const std::string& fileName) {
     histVarSimMarketParams_ = QuantLib::ext::make_shared<ScenarioSimMarketParameters>();
     histVarSimMarketParams_->fromFile(fileName);

OREAnalytics/orea/app/inputparameters.hpp

@@ -185,6 +185,7 @@ class InputParameters {
     void setBenchmarkVarPeriod(const std::string& period);
     void setHistoricalScenarioReader(const std::string& fileName);
     void setSensitivityStreamFromBuffer(const std::string& buffer);
+    void setHistVarSimMarketParams(const std::string& xml);
     void setHistVarSimMarketParamsFromFile(const std::string& fileName);
     void setOutputHistoricalScenarios(const bool b) { outputHistoricalScenarios_ = b; }
 

OREAnalytics/orea/cube/sensitivitycube.cpp

@@ -156,6 +156,16 @@ RiskFactorKey SensitivityCube::upDownFactor(const Size index) const {
     }
 }
 
+SensitivityCube::FactorData SensitivityCube::upThenDownFactorData(const RiskFactorKey& rfkey) {
+    if (auto f = upFactors_.find(rfkey); f != upFactors_.end())
+        return f->second;
+    else if (auto f = downFactors_.find(rfkey); f != downFactors_.end())
+        return f->second;
+    else {
+        QL_FAIL("SensitivityCube::upThenDownFactorData(): no up or down factor data found for " << rfkey);
+    }
+}
+
 SensitivityCube::crossPair SensitivityCube::crossFactor(const Size crossIndex) const {
     if (auto k = crossIndexToKey_.find(crossIndex); k != crossIndexToKey_.end()) {
         return k->second;

OREAnalytics/orea/cube/sensitivitycube.hpp

@@ -101,6 +101,9 @@ class SensitivityCube {
     //! Return the map of down risk factors to its factor data
     const std::map<RiskFactorKey, SensitivityCube::FactorData>& downFactors() const { return downFactors_; };
 
+    //! Return the factor data for an up shift of a rf key, if that does not exist for a down shift of the same rf key
+    SensitivityCube::FactorData upThenDownFactorData(const RiskFactorKey& rfkey);
+
     //! Returns the set of pairs of risk factor keys for which a cross gamma is available
     const std::map<crossPair, std::tuple<SensitivityCube::FactorData, SensitivityCube::FactorData, QuantLib::Size>>&
     crossFactors() const;

OREAnalytics/orea/engine/sensitivitycubestream.cpp

@@ -84,7 +84,7 @@ SensitivityRecord SensitivityCubeStream::next() {
     sr.baseNpv = cubes_[currentCubeIdx_]->npv(tradeIdx);
 
     if (currentDeltaKey_ != currentDeltaKeys_.end()) {
-        auto fd = cubes_[currentCubeIdx_]->upFactors().at(*currentDeltaKey_);
+        auto const& fd = cubes_[currentCubeIdx_]->upThenDownFactorData(*currentDeltaKey_);
         sr.key_1 = *currentDeltaKey_;
         sr.desc_1 = fd.factorDesc;
         sr.shift_1 = fd.targetShiftSize;
@@ -95,7 +95,7 @@ SensitivityRecord SensitivityCubeStream::next() {
             sr.gamma = Null<Real>();
         ++currentDeltaKey_;
     } else if (currentCrossGammaKey_ != currentCrossGammaKeys_.end()) {
-        auto fd = cubes_[currentCubeIdx_]->crossFactors().at(*currentCrossGammaKey_);
+        auto const& fd = cubes_[currentCubeIdx_]->crossFactors().at(*currentCrossGammaKey_);
         sr.key_1 = currentCrossGammaKey_->first;
         sr.desc_1 = std::get<0>(fd).factorDesc;
         sr.shift_1 = std::get<0>(fd).targetShiftSize;

OREAnalytics/test/cube.cpp

@@ -53,6 +53,8 @@
 #include <ql/time/daycounters/actualactual.hpp>
 #include <oret/toplevelfixture.hpp>
 #include <test/oreatoplevelfixture.hpp>
+#include <ored/report/inmemoryreport.hpp>
+#include <orea/app/reportwriter.hpp>
 
 #include "testmarket.hpp"
 
@@ -537,6 +539,49 @@ BOOST_AUTO_TEST_CASE(testDoublePrecisionJaggedCube) {
     IndexManager::instance().clearHistories();
 }
 
+string writeCube(const QuantLib::ext::shared_ptr<NPVCube>& cube, Size bufferSize) {
+    auto report = QuantLib::ext::make_shared<InMemoryReport>(bufferSize);
+    ReportWriter().writeCube(*report, cube);
+    string fileName = boost::filesystem::unique_path().string();
+    report->toFile(fileName);
+    return fileName;
+}
+
+void diffFiles(string filename1, string filename2) {
+    std::ifstream ifs1(filename1);
+    std::ifstream ifs2(filename2);
+
+    std::istream_iterator<char> b1(ifs1), e1;
+    std::istream_iterator<char> b2(ifs2), e2;
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(b1, e1, b2, e2);
+}
+
+// Test the functionality of class InMemoryReport to cache data on disk
+BOOST_AUTO_TEST_CASE(testInMemoryReportBuffer) {
+
+    // Generate a cube
+    std::set<string> ids{string("id")}; // the overlap doesn't matter
+    vector<Date> dates(50, Date());
+    Size samples = 200;
+    Size depth = 6;
+    auto c = QuantLib::ext::make_shared<SinglePrecisionInMemoryCubeN>(Date(), ids, dates, samples, depth);
+
+    // From the cube, generate multiple copies of the report, each of which which will have ~60K rows.
+    // Specify different values for the buffer size in InMemoryReport:
+    string filename_0 = writeCube(c, 0);            // no buffering
+    string filename_100 = writeCube(c, 100);
+    string filename_1000 = writeCube(c, 1000);
+    string filename_10000 = writeCube(c, 10000);
+    string filename_100000 = writeCube(c, 100000);  // buffer size > report size, resulting in no buffering
+
+    // Verify that buffering generates the same output as no buffering
+    diffFiles(filename_0, filename_100);
+    diffFiles(filename_0, filename_1000);
+    diffFiles(filename_0, filename_10000);
+    diffFiles(filename_0, filename_100000);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 BOOST_AUTO_TEST_SUITE_END()

OREData/ored/portfolio/legdata.cpp

@@ -942,7 +942,9 @@ Leg makeSimpleLeg(const LegData& data) {
     Leg leg;
     for (Size i = 0; i < dates.size(); i++) {
         Date d = parseDate(dates[i]);
-        leg.push_back(QuantLib::ext::shared_ptr<CashFlow>(new SimpleCashFlow(amounts[i], d)));
+        if (!data.paymentCalendar().empty() && !data.paymentConvention().empty())
+            d = parseCalendar(data.paymentCalendar()).adjust(d, parseBusinessDayConvention(data.paymentConvention()));
+        leg.push_back(QuantLib::ext::make_shared<SimpleCashFlow>(amounts[i], d));
     }
     return leg;
 }

OREData/ored/report/inmemoryreport.cpp

@@ -17,8 +17,17 @@
 */
 
 #include <ored/report/inmemoryreport.hpp>
+#include <ored/utilities/serializationdate.hpp>
+#include <ored/utilities/serializationperiod.hpp>
 
 #include <boost/algorithm/string/join.hpp>
+#include <boost/serialization/serialization.hpp>
+#include <boost/serialization/vector.hpp>
+#include <boost/serialization/variant.hpp>
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/binary_iarchive.hpp>
+
+#include <fstream>
 
 namespace ore {
 namespace data {
@@ -27,15 +36,26 @@ Report& InMemoryReport::addColumn(const string& name, const ReportType& rt, Size
     headers_.push_back(name);
     columnTypes_.push_back(rt);
     columnPrecision_.push_back(precision);
-    data_.push_back(vector<ReportType>()); // Initialise vector for
+    data_.push_back(vector<ReportType>()); // Initialise vector for column
     i_++;
     return *this;
 }
 
 Report& InMemoryReport::next() {
-    QL_REQUIRE(i_ == headers_.size(), "Cannot go to next line, only " << i_ << " entires filled, report headers are: "
+    QL_REQUIRE(i_ == headers_.size(), "Cannot go to next line, only " << i_ << " entries filled, report headers are: "
                                                                       << boost::join(headers_, ","));
     i_ = 0;
+    if (bufferSize_ && data_[0].size() == bufferSize_ && !headers_.empty()) {
+        std::string s = std::tmpnam(nullptr);
+        std::ofstream os(s.c_str(), std::ios::binary);
+        boost::archive::binary_oarchive oa(os, boost::archive::no_header);
+        for (Size i = 0; i < headers_.size(); i++) {
+            oa << data_[i];
+            data_[i].clear();
+        }
+        os.close();
+        files_.push_back(s);
+    }
     return *this;
 }
 
@@ -85,6 +105,8 @@ void InMemoryReport::end() {
 }
 
 const vector<Report::ReportType>& InMemoryReport::data(Size i) const {
+    QL_REQUIRE(files_.empty(), "Member function InMemoryReport::data() is not supported "
+        "when buffering is active");
     QL_REQUIRE(data_[i].size() == rows(), "internal error: report column "
                                               << i << " (" << header(i) << ") contains " << data_[i].size()
                                               << " rows, expected are " << rows()
@@ -103,6 +125,24 @@ void InMemoryReport::toFile(const string& filename, const char sep, const bool c
 
     auto numColumns = columns();
     if (numColumns > 0) {
+
+        for (auto &f : files_) {
+            vector<vector<ReportType>> data(numColumns);
+            std::ifstream is(f.c_str(), std::ios::binary);
+            boost::archive::binary_iarchive ia(is, boost::archive::no_header);
+            for (Size i = 0; i < numColumns; i++) {
+                ia >> data[i];
+            }
+            is.close();
+
+            for (Size i = 0; i < data[0].size(); i++) {
+                cReport.next();
+                for (Size j = 0; j < numColumns; j++) {
+                    cReport.add(data[j][i]);
+                }
+            }
+        }
+
         auto numRows = data_[0].size();
 
         for (Size i = 0; i < numRows; i++) {

OREData/ored/report/inmemoryreport.hpp

@@ -40,7 +40,7 @@ using std::vector;
  */
 class InMemoryReport : public Report {
 public:
-    InMemoryReport() : i_(0) {}
+    explicit InMemoryReport(Size bufferSize=100000) : i_(0), bufferSize_(bufferSize) {}
 
     Report& addColumn(const string& name, const ReportType& rt, Size precision = 0) override;
     Report& next() override;
@@ -50,7 +50,7 @@ class InMemoryReport : public Report {
 
     // InMemoryInterface
     Size columns() const { return headers_.size(); }
-    Size rows() const { return columns() == 0 ? 0 : data_[0].size(); }
+    Size rows() const { return columns() == 0 ? 0 : files_.size() * bufferSize_ + data_[0].size(); }
     const string& header(Size i) const { return headers_[i]; }
     bool hasHeader(string h) const { return std::find(headers_.begin(), headers_.end(), h) != headers_.end(); }
     ReportType columnType(Size i) const { return columnTypes_[i]; }
@@ -63,10 +63,12 @@ class InMemoryReport : public Report {
 
 private:
     Size i_;
+    Size bufferSize_;
     vector<string> headers_;
     vector<ReportType> columnTypes_;
     vector<Size> columnPrecision_;
     vector<vector<ReportType>> data_;
+    vector<string> files_;
 };
 
 //! InMemoryReport with access to plain types instead of boost::variant<>, to facilitate language bindings