QPR-12517 - Enhance class InMemoryReport to support caching of data to disk

Author: pcaspers

OREAnalytics/test/cube.cpp

@@ -53,6 +53,8 @@
 #include <ql/time/daycounters/actualactual.hpp>
 #include <oret/toplevelfixture.hpp>
 #include <test/oreatoplevelfixture.hpp>
+#include <ored/report/inmemoryreport.hpp>
+#include <orea/app/reportwriter.hpp>
 
 #include "testmarket.hpp"
 
@@ -537,6 +539,49 @@ BOOST_AUTO_TEST_CASE(testDoublePrecisionJaggedCube) {
     IndexManager::instance().clearHistories();
 }
 
+string writeCube(const QuantLib::ext::shared_ptr<NPVCube>& cube, Size bufferSize) {
+    auto report = QuantLib::ext::make_shared<InMemoryReport>(bufferSize);
+    ReportWriter().writeCube(*report, cube);
+    string fileName = boost::filesystem::unique_path().string();
+    report->toFile(fileName);
+    return fileName;
+}
+
+void diffFiles(string filename1, string filename2) {
+    std::ifstream ifs1(filename1);
+    std::ifstream ifs2(filename2);
+
+    std::istream_iterator<char> b1(ifs1), e1;
+    std::istream_iterator<char> b2(ifs2), e2;
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(b1, e1, b2, e2);
+}
+
+// Test the functionality of class InMemoryReport to cache data on disk
+BOOST_AUTO_TEST_CASE(testInMemoryReportBuffer) {
+
+    // Generate a cube
+    std::set<string> ids{string("id")}; // the overlap doesn't matter
+    vector<Date> dates(50, Date());
+    Size samples = 200;
+    Size depth = 6;
+    auto c = QuantLib::ext::make_shared<SinglePrecisionInMemoryCubeN>(Date(), ids, dates, samples, depth);
+
+    // From the cube, generate multiple copies of the report, each of which which will have ~60K rows.
+    // Specify different values for the buffer size in InMemoryReport:
+    string filename_0 = writeCube(c, 0);            // no buffering
+    string filename_100 = writeCube(c, 100);
+    string filename_1000 = writeCube(c, 1000);
+    string filename_10000 = writeCube(c, 10000);
+    string filename_100000 = writeCube(c, 100000);  // buffer size > report size, resulting in no buffering
+
+    // Verify that buffering generates the same output as no buffering
+    diffFiles(filename_0, filename_100);
+    diffFiles(filename_0, filename_1000);
+    diffFiles(filename_0, filename_10000);
+    diffFiles(filename_0, filename_100000);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 BOOST_AUTO_TEST_SUITE_END()

OREData/ored/report/inmemoryreport.cpp

@@ -17,8 +17,17 @@
 */
 
 #include <ored/report/inmemoryreport.hpp>
+#include <ored/utilities/serializationdate.hpp>
+#include <ored/utilities/serializationperiod.hpp>
 
 #include <boost/algorithm/string/join.hpp>
+#include <boost/serialization/serialization.hpp>
+#include <boost/serialization/vector.hpp>
+#include <boost/serialization/variant.hpp>
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/binary_iarchive.hpp>
+
+#include <fstream>
 
 namespace ore {
 namespace data {
@@ -27,15 +36,26 @@ Report& InMemoryReport::addColumn(const string& name, const ReportType& rt, Size
     headers_.push_back(name);
     columnTypes_.push_back(rt);
     columnPrecision_.push_back(precision);
-    data_.push_back(vector<ReportType>()); // Initialise vector for
+    data_.push_back(vector<ReportType>()); // Initialise vector for column
     i_++;
     return *this;
 }
 
 Report& InMemoryReport::next() {
-    QL_REQUIRE(i_ == headers_.size(), "Cannot go to next line, only " << i_ << " entires filled, report headers are: "
+    QL_REQUIRE(i_ == headers_.size(), "Cannot go to next line, only " << i_ << " entries filled, report headers are: "
                                                                       << boost::join(headers_, ","));
     i_ = 0;
+    if (bufferSize_ && data_[0].size() == bufferSize_ && !headers_.empty()) {
+        std::string s = std::tmpnam(nullptr);
+        std::ofstream os(s.c_str(), std::ios::binary);
+        boost::archive::binary_oarchive oa(os, boost::archive::no_header);
+        for (Size i = 0; i < headers_.size(); i++) {
+            oa << data_[i];
+            data_[i].clear();
+        }
+        os.close();
+        files_.push_back(s);
+    }
     return *this;
 }
 
@@ -85,6 +105,8 @@ void InMemoryReport::end() {
 }
 
 const vector<Report::ReportType>& InMemoryReport::data(Size i) const {
+    QL_REQUIRE(files_.empty(), "Member function InMemoryReport::data() is not supported "
+        "when buffering is active");
     QL_REQUIRE(data_[i].size() == rows(), "internal error: report column "
                                               << i << " (" << header(i) << ") contains " << data_[i].size()
                                               << " rows, expected are " << rows()
@@ -103,6 +125,24 @@ void InMemoryReport::toFile(const string& filename, const char sep, const bool c
 
     auto numColumns = columns();
     if (numColumns > 0) {
+
+        for (auto &f : files_) {
+            vector<vector<ReportType>> data(numColumns);
+            std::ifstream is(f.c_str(), std::ios::binary);
+            boost::archive::binary_iarchive ia(is, boost::archive::no_header);
+            for (Size i = 0; i < numColumns; i++) {
+                ia >> data[i];
+            }
+            is.close();
+
+            for (Size i = 0; i < data[0].size(); i++) {
+                cReport.next();
+                for (Size j = 0; j < numColumns; j++) {
+                    cReport.add(data[j][i]);
+                }
+            }
+        }
+
         auto numRows = data_[0].size();
 
         for (Size i = 0; i < numRows; i++) {

OREData/ored/report/inmemoryreport.hpp

@@ -40,7 +40,7 @@ using std::vector;
  */
 class InMemoryReport : public Report {
 public:
-    InMemoryReport() : i_(0) {}
+    explicit InMemoryReport(Size bufferSize=100000) : i_(0), bufferSize_(bufferSize) {}
 
     Report& addColumn(const string& name, const ReportType& rt, Size precision = 0) override;
     Report& next() override;
@@ -50,7 +50,7 @@ class InMemoryReport : public Report {
 
     // InMemoryInterface
     Size columns() const { return headers_.size(); }
-    Size rows() const { return columns() == 0 ? 0 : data_[0].size(); }
+    Size rows() const { return columns() == 0 ? 0 : files_.size() * bufferSize_ + data_[0].size(); }
     const string& header(Size i) const { return headers_[i]; }
     bool hasHeader(string h) const { return std::find(headers_.begin(), headers_.end(), h) != headers_.end(); }
     ReportType columnType(Size i) const { return columnTypes_[i]; }
@@ -63,10 +63,12 @@ class InMemoryReport : public Report {
 
 private:
     Size i_;
+    Size bufferSize_;
     vector<string> headers_;
     vector<ReportType> columnTypes_;
     vector<Size> columnPrecision_;
     vector<vector<ReportType>> data_;
+    vector<string> files_;
 };
 
 //! InMemoryReport with access to plain types instead of boost::variant<>, to facilitate language bindings