Skip to content

Commit ec438ea

Browse files
NathanielVolfangojenkins
NathanielVolfango
authored and
jenkins
committed
QPR-11814 -- Extend ORE logging with some Boost logging loggers, supporting file rotation. Expose some logging configurations to XML.
Consolidate writing of JSON-like strings into a single parent class JSONMessage.
1 parent 1de3d6c commit ec438ea

18 files changed

Lines changed: 569 additions & 201 deletions

‎Docs/UserGuide/userguide.tex‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3580,6 +3580,7 @@ \subsection{Master Input File: {\tt ore.xml}}\label{sec:master_input}
35803580
</ORE>} with three sections
35813581
\begin{itemize}
35823582
\item Setup
3583+
\item Logging
35833584
\item Markets
35843585
\item Analytics
35853586
\end{itemize}
@@ -3679,6 +3680,41 @@ \subsubsection{Setup}
36793680
\medskip If the parameter {\tt nThreads} is given, multiple threads will be used for valuation engine runs where
36803681
applicable (Sensitivity, Exposure Classic, Exposure AMC). If not given, the parameter defaults to $1$.
36813682

3683+
\subsubsection{Logging}\label{sec:master_input_logging}
3684+
3685+
The {\tt Logging} section (see listing \ref{lst:ore_logging}) is used to configure some ORE logging options.
3686+
3687+
\begin{listing}[H]
3688+
%\hrule\medskip
3689+
\begin{minted}[fontsize=\footnotesize]{xml}
3690+
<Logging>
3691+
<Parameter name="logFile">log.txt</Parameter>
3692+
<Parameter name="logMask">31</Parameter>
3693+
<Parameter name="progressLogFile">my_log_progress_%N.json</Parameter>
3694+
<Parameter name="progressLogRotationSize">102400</Parameter>
3695+
<Parameter name="progressLogToConsole">false</Parameter>
3696+
<Parameter name="structuredLogFile">my_structured_logs_%N.txt</Parameter>
3697+
<Parameter name="structuredLogRotationSize">102400</Parameter>
3698+
</Logging>
3699+
\end{minted}
3700+
%\hrule
3701+
\caption{ORE logging}
3702+
\label{lst:ore_logging}
3703+
\end{listing}
3704+
3705+
Parameter {\tt logFile} and {\tt logMask} will override the same parameters in the {\tt Setup} section.
3706+
3707+
Parameters {\tt progressLogFile} and {\tt structuredLogFile} are the filename where progress log messages
3708+
and structured log messages are written out to, respectively, which supports Boost string patterns.This defaults to ``log\_progress\_\%N.json'' and ``log\_structured\_\%N.json'', respectively, where {\tt N} will be an integer (beginning at 0) used for log file rotation.
3709+
3710+
Parameters {\tt progressLogRotationSize} and {\tt structuredLogRotationSize} are the size limit (in bytes)
3711+
of each log file before applying log file rotation to the progress log file and structured log message file,
3712+
respectively.. For example, $10 * 1024 * 1024 = 10 \text{MiB}$. Defaults to 100 MiB.
3713+
3714+
If the parameter {\tt progressLogToConsole} is set to true, then progress logs will be written to std::cout.
3715+
This can be used simultaneously with {\tt progressLogFile}, i.e.\ progress logs can be written out
3716+
to both file and std::cout.
3717+
36823718
\subsubsection{Markets}\label{sec:master_input_markets}
36833719

36843720
The {\tt Markets} section (see listing \ref{lst:ore_markets}) is used to choose market configurations for calibrating

‎Examples/Example_1/Input/ore.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@
1919
<Parameter name="calendarAdjustment">../../Input/calendaradjustment.xml</Parameter>
2020
<Parameter name="currencyConfiguration">../../Input/currencies.xml</Parameter>
2121
</Setup>
22+
<Logging>
23+
<Parameter name="logFile">log.txt</Parameter>
24+
<Parameter name="logMask">31</Parameter>
25+
<Parameter name="progressLogFile">my_log_progress_%N.txt</Parameter>
26+
<Parameter name="progressLogRotationSize">102400</Parameter>
27+
<Parameter name="progressLogToConsole">N</Parameter>
28+
<Parameter name="structuredLogFile">my_structured_logs_%N.txt</Parameter>
29+
<Parameter name="structuredLogRotationSize">102400</Parameter>
30+
</Logging>
2231
<Markets>
2332
<Parameter name="lgmcalibration">libor</Parameter>
2433
<Parameter name="fxcalibration">libor</Parameter>

‎OREAnalytics/orea/app/analytics/pricinganalytic.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ void PricingAnalyticImpl::runAnalytic(
209209
}
210210

211211
LOG("Sensi analysis - generate");
212-
sensiAnalysis->registerProgressIndicator(boost::make_shared<ProgressLog>("sensitivities", 100, ORE_NOTICE));
212+
sensiAnalysis->registerProgressIndicator(boost::make_shared<ProgressLog>("sensitivities", 100, oreSeverity::notice));
213213
sensiAnalysis->generateSensitivities();
214214

215215
LOG("Sensi analysis - write sensitivity report in memory");

‎OREAnalytics/orea/app/analytics/xvaanalytic.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ void XvaAnalyticImpl::buildClassicCube(const boost::shared_ptr<Portfolio>& portf
304304
// set up progress indicators
305305

306306
auto progressBar = boost::make_shared<SimpleProgressBar>(o.str(), ConsoleLog::instance().width(), ConsoleLog::instance().progressBarWidth());
307-
auto progressLog = boost::make_shared<ProgressLog>("Building cube", 100, ORE_NOTICE);
307+
auto progressLog = boost::make_shared<ProgressLog>("Building cube", 100, oreSeverity::notice);
308308

309309
if(inputs_->nThreads() == 1) {
310310

@@ -446,7 +446,7 @@ void XvaAnalyticImpl::amcRun(bool doClassicRun) {
446446
std::string message = "XVA: Build AMC Cube " + std::to_string(amcPortfolio_->size()) + " x " +
447447
std::to_string(grid_->valuationDates().size()) + " x " + std::to_string(samples_) + "... ";
448448
auto progressBar = boost::make_shared<SimpleProgressBar>(message, ConsoleLog::instance().width(), ConsoleLog::instance().progressBarWidth());
449-
auto progressLog = boost::make_shared<ProgressLog>("Building AMC Cube...", 100, ORE_NOTICE);
449+
auto progressLog = boost::make_shared<ProgressLog>("Building AMC Cube...", 100, oreSeverity::notice);
450450

451451
if (inputs_->nThreads() == 1) {
452452
initCube(amcCube_, amcPortfolio_->ids(), cubeDepth_);
@@ -579,6 +579,7 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
579579
const std::set<std::string>& runTypes) {
580580

581581
LOG("XVA analytic called with asof " << io::iso_date(inputs_->asof()));
582+
ProgressMessage("Running XVA Analytic", 0, 1).log();
582583

583584
if (runTypes.find("EXPOSURE") != runTypes.end() || runTypes.empty())
584585
runSimulation_ = true;
@@ -852,6 +853,7 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
852853

853854
// reset that mode
854855
ObservationMode::instance().setMode(inputs_->observationModel());
856+
ProgressMessage("Running XVA Analytic", 1, 1).log();
855857
}
856858

857859
Matrix XvaAnalyticImpl::creditStateCorrelationMatrix() const {

‎OREAnalytics/orea/app/oreapp.cpp‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,8 +302,45 @@ OREApp::OREApp(boost::shared_ptr<Parameters> params, bool console,
302302
if (params_->has("setup", "logMask")) {
303303
logMask = static_cast<Size>(parseInteger(params_->get("setup", "logMask")));
304304
}
305+
306+
string progressLogFile, structuredLogFile;
307+
Size progressLogRotationSize = 100 * 1024 * 1024;
308+
bool progressLogToConsole = false;
309+
Size structuredLogRotationSize = 100 * 1024 * 1024;
305310

306-
setupLog(outputPath, logFile, logMask, logRootPath);
311+
if (params_->hasGroup("logging")) {
312+
string logFileOverride = params_->get("logging", "logFile", false);
313+
if (!logFileOverride.empty()) {
314+
logFile = outputPath + '/' + logFileOverride;
315+
}
316+
string logMaskOverride = params_->get("logging", "logMask", false);
317+
if (!logMaskOverride.empty()) {
318+
logMask = static_cast<Size>(parseInteger(logMaskOverride));
319+
}
320+
progressLogFile = params_->get("logging", "progressLogFile", false);
321+
if (!progressLogFile.empty()) {
322+
progressLogFile = outputPath + '/' + progressLogFile;
323+
}
324+
string tmp = params_->get("logging", "progressLogRotationSize", false);
325+
if (!tmp.empty()) {
326+
progressLogRotationSize = static_cast<Size>(parseInteger(tmp));
327+
}
328+
tmp = params_->get("logging", "progressLogToConsole", false);
329+
if (!tmp.empty()) {
330+
progressLogToConsole = ore::data::parseBool(tmp);
331+
}
332+
structuredLogFile = params_->get("logging", "structuredLogFile", false);
333+
if (!structuredLogFile.empty()) {
334+
structuredLogFile = outputPath + '/' + structuredLogFile;
335+
}
336+
tmp = params_->get("logging", "structuredLogRotationSize", false);
337+
if (!tmp.empty()) {
338+
structuredLogRotationSize = static_cast<Size>(parseInteger(tmp));
339+
}
340+
}
341+
342+
setupLog(outputPath, logFile, logMask, logRootPath, progressLogFile, progressLogRotationSize, progressLogToConsole,
343+
structuredLogFile, structuredLogRotationSize);
307344

308345
// Log the input parameters
309346
params_->log();
@@ -1348,7 +1385,7 @@ void OREApp::buildInputParameters(boost::shared_ptr<InputParameters> inputs,
13481385
}
13491386

13501387
void OREApp::setupLog(const std::string& path, const std::string& file, Size mask,
1351-
const boost::filesystem::path& logRootPath) {
1388+
const boost::filesystem::path& logRootPath, const std::string& progressLogFile, Size progressLogRotationSize, bool progressLogToConsole, const std::string& structuredLogFile, Size structuredLogRotationSize) {
13521389
closeLog();
13531390

13541391
boost::filesystem::path p{path};
@@ -1366,6 +1403,19 @@ void OREApp::setupLog(const std::string& path, const std::string& file, Size mas
13661403
Log::instance().setRootPath(oreRootPath);
13671404
Log::instance().setMask(mask);
13681405
Log::instance().switchOn();
1406+
1407+
// Progress logger
1408+
auto progressLogger = boost::make_shared<ProgressLogger>();
1409+
string progressLogFilePath = progressLogFile.empty() ? path + '/' + "log_progress_%N.json" : progressLogFile;
1410+
progressLogger->setFileLog(progressLogFilePath, path, progressLogRotationSize);
1411+
progressLogger->setCoutLog(progressLogToConsole);
1412+
Log::instance().registerIndependentLogger(progressLogger);
1413+
1414+
// Structured message logger
1415+
auto structuredLogger = boost::make_shared<StructuredLogger>();
1416+
string structuredLogFilePath = structuredLogFile.empty() ? path + '/' + "log_structured_%N.json" : structuredLogFile;
1417+
structuredLogger->setFileLog(structuredLogFilePath, path, structuredLogRotationSize);
1418+
Log::instance().registerIndependentLogger(structuredLogger);
13691419
}
13701420

13711421
void OREApp::closeLog() { Log::instance().removeAllLoggers(); }

‎OREAnalytics/orea/app/oreapp.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,10 @@ class OREApp {
8585
vector<string> getFileNames(const string& fileString, const string& path);
8686
boost::shared_ptr<CSVLoader> buildCsvLoader(const boost::shared_ptr<Parameters>& params);
8787
//! set up logging
88-
void setupLog(const std::string& path, const std::string& file, Size mask,
89-
const boost::filesystem::path& logRootPath);
88+
void setupLog(const std::string& path, const std::string& file, QuantLib::Size mask,
89+
const boost::filesystem::path& logRootPath, const std::string& progressLogFile = "",
90+
QuantLib::Size progressLogRotationSize = 100 * 1024 * 1024, bool progressLogToConsole = false,
91+
const std::string& structuredLogFile = "", QuantLib::Size structuredLogRotationSize = 100 * 1024 * 1024);
9092
//! remove logs
9193
void closeLog();
9294

‎OREAnalytics/orea/app/parameters.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,17 @@ void Parameters::fromXML(XMLNode* node) {
8686
}
8787
data_["setup"] = setupMap;
8888

89+
XMLNode* loggingNode = XMLUtils::getChildNode(node, "Logging");
90+
if (loggingNode) {
91+
map<string, string> loggingMap;
92+
for (XMLNode* child = XMLUtils::getChildNode(loggingNode); child; child = XMLUtils::getNextSibling(child)) {
93+
string key = XMLUtils::getAttribute(child, "name");
94+
string value = XMLUtils::getNodeValue(child);
95+
loggingMap[key] = value;
96+
}
97+
data_["logging"] = loggingMap;
98+
}
99+
89100
XMLNode* marketsNode = XMLUtils::getChildNode(node, "Markets");
90101
if (marketsNode) {
91102
map<string, string> marketsMap;

‎OREAnalytics/orea/app/structuredanalyticserror.hpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ class StructuredAnalyticsErrorMessage : public ore::data::StructuredMessage {
3737
Category::Error, Group::Analytics, exceptionWhat,
3838
std::map<std::string, std::string>({{"exceptionType", exceptionType}, {"analyticType", analyticType}})) {
3939

40-
if (!subFields.empty())
41-
subFields_.insert(subFields.begin(), subFields.end());
40+
addSubFields(subFields);
4241
}
4342
};
4443

‎OREAnalytics/orea/app/structuredanalyticswarning.hpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ class StructuredAnalyticsWarningMessage : public ore::data::StructuredMessage {
3737
Category::Warning, Group::Analytics, warningWhat,
3838
std::map<std::string, std::string>({{"warningType", warningType}, {"analyticType", analyticType}})) {
3939

40-
if (!subFields.empty())
41-
subFields_.insert(subFields.begin(), subFields.end());
40+
addSubFields(subFields);
4241
}
4342
};
4443

‎OREData/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ if (ORE_BUILD_TESTS)
1616
else()
1717
SET(COMPONENTS_CONDITIONAL "")
1818
endif()
19-
find_package (Boost REQUIRED COMPONENTS ${COMPONENTS_CONDITIONAL} regex system date_time serialization filesystem timer OPTIONAL_COMPONENTS chrono)
19+
find_package (Boost REQUIRED COMPONENTS ${COMPONENTS_CONDITIONAL} regex system date_time serialization filesystem timer log OPTIONAL_COMPONENTS chrono)
2020

2121
include_directories(${Boost_INCLUDE_DIRS})
2222
include_directories(${QUANTLIB_SOURCE_DIR})

0 commit comments

Comments
 (0)