Merge branch 'feature/QPR-13658' into 'master'

QPR-13658 Zip reports that have suffix .gz

Closes QPR-13658

See merge request qs/oreplus!3091

Author: damienbarker

OREAnalytics/orea/app/analyticsmanager.cpp

@@ -280,12 +280,22 @@ void AnalyticsManager::toFile(const ore::analytics::Analytic::analytic_reports&
 
             // attach a suffix only if it does not have one already
             string suffix = "";
-            if (!endsWith(fileName,".csv") && !endsWith(fileName, ".txt"))
-                suffix = ".csv";
             std::string fullFileName = outputPath + "/" + fileName + suffix;
-
-            report->toFile(fullFileName, sep, commentCharacter, quoteChar, nullString,
-                           lowerHeaderReportNames.find(reportName) != lowerHeaderReportNames.end());
+            if (!endsWith(fileName,".csv") && !endsWith(fileName, ".txt") && !endsWith(fileName, ".gz")){
+                suffix = ".csv";
+                fullFileName = outputPath + "/" + fileName + suffix;
+                report->toFile(fullFileName, sep, commentCharacter, quoteChar, nullString,
+                            lowerHeaderReportNames.find(reportName) != lowerHeaderReportNames.end());
+            }else if(endsWith(fileName,".gz")){
+                fullFileName = outputPath + "/" + fileName;
+                report->toZip(fullFileName, sep, commentCharacter, quoteChar, nullString,
+                            lowerHeaderReportNames.find(reportName) != lowerHeaderReportNames.end());  
+            }else{
+                fullFileName = outputPath + "/" + fileName + suffix;
+                report->toFile(fullFileName, sep, commentCharacter, quoteChar, nullString,
+                            lowerHeaderReportNames.find(reportName) != lowerHeaderReportNames.end());
+            }
+          
             LOG("report " << reportName << " written to " << fullFileName); 
         }
     }

OREData/CMakeLists.txt

@@ -14,7 +14,19 @@ else()
     SET(COMPONENTS_CONDITIONAL "")
 endif()
 
-find_package (Boost REQUIRED COMPONENTS ${COMPONENTS_CONDITIONAL} date_time thread serialization timer log filesystem OPTIONAL_COMPONENTS system chrono)
+if(MSVC)
+	add_compile_definitions(BOOST_IOSTREAMS_NO_LIB)
+endif()
+
+if(ORE_USE_ZLIB)
+    find_package(ZLIB REQUIRED)
+endif()
+
+if(MSVC AND ORE_USE_ZLIB)
+    LIST(APPEND COMPONENT_LIST zlib)
+endif()
+
+find_package (Boost REQUIRED COMPONENTS ${COMPONENTS_CONDITIONAL} date_time thread serialization timer log filesystem OPTIONAL_COMPONENTS system chrono iostreams)
 
 
 include_directories(${Boost_INCLUDE_DIRS})

OREData/ored/report/inmemoryreport.cpp

@@ -28,6 +28,12 @@
 #include <boost/archive/binary_iarchive.hpp>
 #include <boost/filesystem.hpp>
 
+#include <boost/iostreams/device/file_descriptor.hpp>
+#ifdef ORE_USE_ZLIB
+#include <boost/iostreams/filter/gzip.hpp>
+#endif
+#include <boost/iostreams/filtering_stream.hpp>
+
 #include <fstream>
 
 namespace ore {
@@ -197,5 +203,78 @@ void InMemoryReport::toFile(const string& filename, const char sep, const bool c
     cReport.end();
 }
 
+bool use_compression(const std::string& filename) {
+#ifdef ORE_USE_ZLIB
+    // assume compression for all filenames that do not end with csv or txt
+
+    std::string extension = boost::filesystem::path(filename).extension().string();
+    return extension != ".csv" && extension != ".txt";
+#else
+    return false;
+#endif
+}
+
+void InMemoryReport::toZip(const string& filename, const char sep, const bool commentCharacter, char quoteChar,
+                            const string& nullString, bool lowerHeader) {
+
+    bool gzip = use_compression(filename);
+    std::ofstream out1(filename, gzip ? (std::ios::binary | std::ios::out) : std::ios::out);
+    boost::iostreams::filtering_stream<boost::iostreams::output> out;
+#ifdef ORE_USE_ZLIB
+    if (gzip)
+        out.push(boost::iostreams::gzip_compressor(/*boost::iostreams::gzip_params(9)*/));
+#endif
+    out.push(out1);
+
+    for (Size i = 0; i < headers_.size(); ++i) {
+        if (i > 0)
+            out << sep;
+
+        std::string h = headers_[i];
+        if (lowerHeader)
+            std::transform(h.begin(), h.end(), h.begin(), ::tolower);
+        out << quoteChar << h << quoteChar;
+    }
+    out << "\n";
+
+    Size numColumns = columns();
+    if (numColumns > 0) {
+        for (Size cacheIndex = 0; cacheIndex < files_.size(); ++cacheIndex) {
+            const auto& data = cache(cacheIndex);
+            Size numRows = data[0].size();
+
+            for (Size row = 0; row < numRows; ++row) {
+                for (Size col = 0; col < numColumns; ++col) {
+                    if (col > 0)
+                        out << sep;
+
+                    const ReportType& val = data[col][row];
+                    if (val.empty())
+                        out << nullString;
+                    else
+                        out << quoteChar << val << quoteChar;
+                }
+                out << "\n";
+            }
+        }
+
+        Size numRows = data_[0].size();
+        for (Size row = 0; row < numRows; ++row) {
+            for (Size col = 0; col < numColumns; ++col) {
+                if (col > 0)
+                    out << sep;
+
+                const ReportType& val = data_[col][row];
+                if (val.empty())
+                    out << nullString;
+                else
+                    out << quoteChar << val << quoteChar;
+            }
+            out << "\n";
+        }
+    }
+    out.flush();
+}
+
 } // namespace data
 } // namespace ore

OREData/ored/report/inmemoryreport.hpp

@@ -61,6 +61,8 @@ class InMemoryReport : public Report {
     const ReportType& data(Size i, Size j) const;
     void toFile(const string& filename, const char sep = ',', const bool commentCharacter = true, char quoteChar = '\0',
                 const string& nullString = "#N/A", bool lowerHeader = false);
+    void toZip(const string& filename, const char sep = ',', const bool commentCharacter = true, char quoteChar = '\0',
+                const string& nullString = "#N/A", bool lowerHeader = false);
     void jumpToColumn(Size i) { i_ = i; }
 
     //! Return the position of a column, throws an exception if columnName not in report