Skip to content

Commit 514ff19

Browse files
NathanielVolfangojenkins
NathanielVolfango
authored and
jenkins
committed
QPR-11679 -- Change how JSON messages (i.e. structured, progress, event messages) are logged, using Boost logging. Also fix incorrerct parameter assignnment in pricer_app
1 parent 0f33164 commit 514ff19

37 files changed

Lines changed: 445 additions & 211 deletions

‎Docker/Dockerfile-Test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ RUN apt-get update \
55
&& DEBIAN_FRONTEND=noninteractive apt-get install -y dos2unix python3 python3-pip libxml2-utils xsltproc \
66
&& apt-get clean \
77
&& rm -rf /var/lib/apt/lists/* \
8-
&& pip3 install matplotlib pandas nose nose_xunitmp datacompy
8+
&& pip3 install matplotlib pandas nose nose_xunitmp datacompy jsondiff
99

1010
CMD bash
1111

‎OREAnalytics/orea/aggregation/xvacalculator.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -359,8 +359,9 @@ void ValueAdjustmentCalculator::build() {
359359
nettingSetSumCva_[nid] += tradeCva_[tid];
360360
nettingSetSumDva_[nid] += tradeDva_[tid];
361361
} catch (const std::exception& e) {
362-
ALOG(StructuredAnalyticsErrorMessage("ValueAdjustmentCalculator", "Error processing trade.", e.what(),
363-
{{"tradeId", tid}}));
362+
StructuredAnalyticsErrorMessage("ValueAdjustmentCalculator", "Error processing trade.", e.what(),
363+
{{"tradeId", tid}})
364+
.log();
364365
}
365366
}
366367

@@ -442,8 +443,9 @@ void ValueAdjustmentCalculator::build() {
442443
}
443444
}
444445
} catch (const std::exception& e) {
445-
ALOG(StructuredAnalyticsErrorMessage("ValueAdjustmentCalculator", "Error processing netting set.", e.what(),
446-
{{"nettingSetId", nid}}));
446+
StructuredAnalyticsErrorMessage("ValueAdjustmentCalculator", "Error processing netting set.", e.what(),
447+
{{"nettingSetId", nid}})
448+
.log();
447449
}
448450
}
449451
}

‎OREAnalytics/orea/app/analytics/parconversionanalytic.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,9 +120,10 @@ void ParConversionAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::I
120120
if (it == factorToIndex.end()) {
121121
if (ParSensitivityAnalysis::isParType(rf.keytype) && typesDisabled.count(rf.keytype) != 1) {
122122

123-
ALOG(StructuredAnalyticsErrorMessage("Par conversion", "",
124-
"Par factor " + ore::data::to_string(rf) +
125-
" not found in factorToIndex map"));
123+
StructuredAnalyticsErrorMessage("Par conversion", "",
124+
"Par factor " + ore::data::to_string(rf) +
125+
" not found in factorToIndex map")
126+
.log();
126127
} else {
127128
SensitivityRecord sr;
128129
sr.tradeId = id;

‎OREAnalytics/orea/app/analytics/xvaanalytic.cpp‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,10 @@ void XvaAnalyticImpl::checkConfigurations(const boost::shared_ptr<Portfolio>& p
7878
Period mpor_simulation = analytic()->configurations().scenarioGeneratorData->closeOutLag();
7979
Period mpor_netting = inputs_->nettingSetManager()->get(key)->csaDetails()->marginPeriodOfRisk();
8080
if (mpor_simulation != mpor_netting)
81-
WLOG(StructuredAnalyticsWarningMessage(
81+
StructuredAnalyticsWarningMessage(
8282
"XvaAnalytic", "Inconsistent MPoR period",
83-
"For netting set " + key +", close-out lag is not consistent with the netting-set's mpor "));
83+
"For netting set " + key + ", close-out lag is not consistent with the netting-set's mpor ")
84+
.log();
8485
}
8586
}
8687
}
@@ -425,7 +426,7 @@ void XvaAnalyticImpl::buildAmcPortfolio() {
425426
amcPortfolio_->add(trade);
426427
DLOG("trade " << tradeId << " is added to amc portfolio");
427428
} catch (const std::exception& e) {
428-
ALOG(StructuredTradeErrorMessage(trade, "Error building trade for AMC simulation", e.what()));
429+
StructuredTradeErrorMessage(trade, "Error building trade for AMC simulation", e.what()).log();
429430
}
430431
}
431432
}
@@ -773,8 +774,9 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
773774
ReportWriter(inputs_->reportNaString()).writeTradeExposures(*report, postProcess_, tradeId);
774775
analytic()->reports()["XVA"]["exposure_trade_" + tradeId] = report;
775776
} catch (const std::exception& e) {
776-
ALOG(StructuredAnalyticsErrorMessage("Trade Exposure Report", "Error processing trade.", e.what(),
777-
{{"tradeId", tradeId}}));
777+
StructuredAnalyticsErrorMessage("Trade Exposure Report", "Error processing trade.", e.what(),
778+
{{"tradeId", tradeId}})
779+
.log();
778780
}
779781
}
780782
}
@@ -787,8 +789,9 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
787789
.writeNettingSetExposures(*exposureReport, postProcess_, nettingSet);
788790
analytic()->reports()["XVA"]["exposure_nettingset_" + nettingSet] = exposureReport;
789791
} catch (const std::exception& e) {
790-
ALOG(StructuredAnalyticsErrorMessage("Netting Set Exposure Report", "Error processing netting set.",
791-
e.what(), {{"nettingSetId", nettingSet}}));
792+
StructuredAnalyticsErrorMessage("Netting Set Exposure Report", "Error processing netting set.",
793+
e.what(), {{"nettingSetId", nettingSet}})
794+
.log();
792795
}
793796

794797
auto colvaReport = boost::make_shared<InMemoryReport>();
@@ -797,8 +800,9 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
797800
.writeNettingSetColva(*colvaReport, postProcess_, nettingSet);
798801
analytic()->reports()["XVA"]["colva_nettingset_" + nettingSet] = colvaReport;
799802
} catch (const std::exception& e) {
800-
ALOG(StructuredAnalyticsErrorMessage("Netting Set Colva Report", "Error processing netting set.",
801-
e.what(), {{"nettingSetId", nettingSet}}));
803+
StructuredAnalyticsErrorMessage("Netting Set Colva Report", "Error processing netting set.",
804+
e.what(), {{"nettingSetId", nettingSet}})
805+
.log();
802806
}
803807

804808
auto cvaSensiReport = boost::make_shared<InMemoryReport>();
@@ -807,8 +811,9 @@ void XvaAnalyticImpl::runAnalytic(const boost::shared_ptr<ore::data::InMemoryLoa
807811
.writeNettingSetCvaSensitivities(*cvaSensiReport, postProcess_, nettingSet);
808812
analytic()->reports()["XVA"]["cva_sensitivity_nettingset_" + nettingSet] = cvaSensiReport;
809813
} catch (const std::exception& e) {
810-
ALOG(StructuredAnalyticsErrorMessage("Cva Sensi Report", "Error processing netting set.", e.what(),
811-
{{"nettingSetId", nettingSet}}));
814+
StructuredAnalyticsErrorMessage("Cva Sensi Report", "Error processing netting set.", e.what(),
815+
{{"nettingSetId", nettingSet}})
816+
.log();
812817
}
813818
}
814819
}

‎OREAnalytics/orea/app/marketdataloader.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,8 @@ void MarketDataLoader::populateFixings(
215215
fixingErr = ", error: " + ore::data::to_string(e.what());
216216
}
217217
}
218-
WLOG(StructuredFixingWarningMessage(f.first, d, "Missing fixing",
219-
"Could not find required fixing ID."));
218+
StructuredFixingWarningMessage(f.first, d, "Missing fixing", "Could not find required fixing ID.")
219+
.log();
220220
}
221221
}
222222
}

‎OREAnalytics/orea/app/oreapp.cpp‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -308,9 +308,9 @@ OREApp::OREApp(boost::shared_ptr<Parameters> params, bool console,
308308
}
309309

310310
string progressLogFile, structuredLogFile;
311-
Size progressLogRotationSize = 100 * 1024 * 1024;
311+
Size progressLogRotationSize = 0;
312312
bool progressLogToConsole = false;
313-
Size structuredLogRotationSize = 100 * 1024 * 1024;
313+
Size structuredLogRotationSize = 0;
314314

315315
if (params_->hasGroup("logging")) {
316316
string logFileOverride = params_->get("logging", "logFile", false);
@@ -385,7 +385,7 @@ void OREApp::run() {
385385
try {
386386
analytics();
387387
} catch (std::exception& e) {
388-
ALOG(StructuredAnalyticsWarningMessage("OREApp::run()", "Error", e.what()));
388+
StructuredAnalyticsWarningMessage("OREApp::run()", "Error", e.what()).log();
389389
CONSOLE("Error: " << e.what());
390390
return;
391391
}
@@ -443,7 +443,7 @@ void OREApp::run(const std::vector<std::string>& marketData,
443443
catch (std::exception& e) {
444444
ostringstream oss;
445445
oss << "Error in ORE analytics: " << e.what();
446-
ALOG(StructuredAnalyticsWarningMessage("OREApp::run()", oss.str(), e.what()));
446+
StructuredAnalyticsWarningMessage("OREApp::run()", oss.str(), e.what()).log();
447447
MEM_LOG_USING_LEVEL(ORE_WARNING)
448448
CONSOLE(oss.str());
449449
QL_FAIL(oss.str());
@@ -1422,16 +1422,21 @@ void OREApp::setupLog(const std::string& path, const std::string& file, Size mas
14221422

14231423
// Progress logger
14241424
auto progressLogger = boost::make_shared<ProgressLogger>();
1425-
string progressLogFilePath = progressLogFile.empty() ? path + '/' + "log_progress_%N.json" : progressLogFile;
1425+
string progressLogFilePath = progressLogFile.empty() ? path + "/log_progress.json" : progressLogFile;
14261426
progressLogger->setFileLog(progressLogFilePath, path, progressLogRotationSize);
14271427
progressLogger->setCoutLog(progressLogToConsole);
14281428
Log::instance().registerIndependentLogger(progressLogger);
14291429

14301430
// Structured message logger
14311431
auto structuredLogger = boost::make_shared<StructuredLogger>();
1432-
string structuredLogFilePath = structuredLogFile.empty() ? path + '/' + "log_structured_%N.json" : structuredLogFile;
1432+
string structuredLogFilePath = structuredLogFile.empty() ? path + "/log_structured.json" : structuredLogFile;
14331433
structuredLogger->setFileLog(structuredLogFilePath, path, structuredLogRotationSize);
14341434
Log::instance().registerIndependentLogger(structuredLogger);
1435+
1436+
// Event message logger
1437+
auto eventLogger = boost::make_shared<EventLogger>();
1438+
eventLogger->setFileLog(path + "/log_event_");
1439+
ore::data::Log::instance().registerIndependentLogger(eventLogger);
14351440
}
14361441

14371442
void OREApp::closeLog() { Log::instance().removeAllLoggers(); }

‎OREAnalytics/orea/app/reportwriter.cpp‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ void ReportWriter::writeNpv(ore::data::Report& report, const std::string& baseCu
122122
.add(trade->envelope().nettingSetId())
123123
.add(trade->envelope().counterparty());
124124
} catch (std::exception& e) {
125-
ALOG(StructuredTradeErrorMessage(trade->id(), trade->tradeType(), "Error during trade pricing", e.what()));
125+
StructuredTradeErrorMessage(trade->id(), trade->tradeType(), "Error during trade pricing", e.what()).log();
126126
Date maturity = trade->maturity();
127127
report.next()
128128
.add(trade->id())
@@ -588,8 +588,9 @@ void ReportWriter::writeCashflow(ore::data::Report& report, const std::string& b
588588
}
589589

590590
} catch (std::exception& e) {
591-
ALOG(StructuredTradeErrorMessage(trade->id(), trade->tradeType(), "Error during cashflow report generation",
592-
e.what()));
591+
StructuredTradeErrorMessage(trade->id(), trade->tradeType(), "Error during cashflow report generation",
592+
e.what())
593+
.log();
593594
}
594595
}
595596
report.end();
@@ -629,9 +630,10 @@ void ReportWriter::writeCashflowNpv(ore::data::Report& report, const ore::data::
629630
Real fx = 1.0;
630631
// There shouldn't be entries in the cf report without ccy. We assume ccy = baseCcy in this case and log an error.
631632
if (ccy.empty()) {
632-
ALOG(StructuredTradeErrorMessage(tradeId, tradeType, "Error during CashflowNpv calculation.",
633-
"Cashflow in row " + std::to_string(i) +
634-
" has no ccy. Assuming ccy = baseCcy = " + baseCcy + "."));
633+
StructuredTradeErrorMessage(tradeId, tradeType, "Error during CashflowNpv calculation.",
634+
"Cashflow in row " + std::to_string(i) +
635+
" has no ccy. Assuming ccy = baseCcy = " + baseCcy + ".")
636+
.log();
635637
}
636638
if (!ccy.empty() && ccy != baseCcy)
637639
fx = market->fxRate(ccy + baseCcy, configuration)->value();
@@ -952,8 +954,9 @@ void ReportWriter::writeXVA(ore::data::Report& report, const string& allocationM
952954
.add(postProcess->netEPE_B(n))
953955
.add(postProcess->netEEPE_B(n));
954956
} catch (const std::exception& e) {
955-
ALOG(StructuredAnalyticsErrorMessage("XVA Report", "Error during writing xva for netting set.", e.what(),
956-
{{"nettingSetId", n}}));
957+
StructuredAnalyticsErrorMessage("XVA Report", "Error during writing xva for netting set.", e.what(),
958+
{{"nettingSetId", n}})
959+
.log();
957960
}
958961

959962
for (auto& [tid, trade] : portfolio->trades()) {
@@ -987,8 +990,9 @@ void ReportWriter::writeXVA(ore::data::Report& report, const string& allocationM
987990
.add(postProcess->tradeEPE_B(tid))
988991
.add(postProcess->tradeEEPE_B(tid));
989992
} catch (const std::exception& e) {
990-
ALOG(StructuredAnalyticsErrorMessage("XVA Report", "Error during writing xva for trade.", e.what(),
991-
{{"tradeId", n}}));
993+
StructuredAnalyticsErrorMessage("XVA Report", "Error during writing xva for trade.", e.what(),
994+
{{"tradeId", n}})
995+
.log();
992996
}
993997
}
994998
}
@@ -1303,8 +1307,9 @@ void ReportWriter::writeAdditionalResultsReport(Report& report, boost::shared_pt
13031307
}
13041308
}
13051309
} catch (const std::exception& e) {
1306-
ALOG(StructuredTradeErrorMessage(trade->id(), trade->tradeType(),
1307-
"Error during trade pricing (additional results)", e.what()));
1310+
StructuredTradeErrorMessage(trade->id(), trade->tradeType(),
1311+
"Error during trade pricing (additional results)", e.what())
1312+
.log();
13081313
}
13091314
}
13101315

‎OREAnalytics/orea/engine/amcvaluationengine.cpp‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,8 @@ simulatePathInterface2(const boost::shared_ptr<AmcCalculator>& amcCalc, const st
9595
try {
9696
return amcCalc->simulatePath(pathTimes, paths, isRelevantTime, moveStateToPreviousTime);
9797
} catch (const std::exception& e) {
98-
ALOG(StructuredTradeErrorMessage(tradeLabel, tradeType, "error during amc path simulation for trade.",
99-
e.what()));
98+
StructuredTradeErrorMessage(tradeLabel, tradeType, "error during amc path simulation for trade.", e.what())
99+
.log();
100100
return std::vector<QuantExt::RandomVariable>(std::count(isRelevantTime.begin(), isRelevantTime.end(), true) + 1,
101101
RandomVariable(paths.front().front().size()));
102102
}
@@ -247,8 +247,9 @@ void runCoreEngine(const boost::shared_ptr<ore::data::Portfolio>& portfolio,
247247
tradeFees.back().push_back(std::make_tuple(model->ccyIndex(p->currency()), p->cashFlow()->amount(),
248248
p->cashFlow()->date()));
249249
} else {
250-
ALOG(StructuredTradeErrorMessage(trade.second, "Additional instrument is ignored in AMC simulation",
251-
"only QuantExt::Payment is handled as additional instrument."));
250+
StructuredTradeErrorMessage(trade.second, "Additional instrument is ignored in AMC simulation",
251+
"only QuantExt::Payment is handled as additional instrument.")
252+
.log();
252253
}
253254
}
254255
}
@@ -295,7 +296,7 @@ void runCoreEngine(const boost::shared_ptr<ore::data::Portfolio>& portfolio,
295296
extractAmcCalculator(trade, amcCalc, multiplier, true);
296297

297298
} catch (const std::exception& e) {
298-
ALOG(StructuredTradeErrorMessage(trade.second, "Error building trade for AMC simulation", e.what()));
299+
StructuredTradeErrorMessage(trade.second, "Error building trade for AMC simulation", e.what()).log();
299300
}
300301
progressIndicator->updateProgress(++progressCounter, portfolio->size() + 1);
301302
}
@@ -805,9 +806,9 @@ void AMCValuationEngine::buildCube(const boost::shared_ptr<ore::data::Portfolio>
805806

806807
// log error and return code 1 = not ok
807808

808-
ALOG(ore::analytics::StructuredAnalyticsErrorMessage("AMC Valuation Engine (multithreaded mode)",
809-
"",
810-
e.what()));
809+
ore::analytics::StructuredAnalyticsErrorMessage("AMC Valuation Engine (multithreaded mode)", "",
810+
e.what())
811+
.log();
811812
rc = 1;
812813
}
813814

‎OREAnalytics/orea/engine/multithreadedvaluationengine.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ void MultiThreadedValuationEngine::buildCube(
341341

342342
// log error and return code 1 = not ok
343343

344-
ALOG(ore::analytics::StructuredAnalyticsErrorMessage("Multithreaded Valuation Engine", "", e.what()));
344+
ore::analytics::StructuredAnalyticsErrorMessage("Multithreaded Valuation Engine", "", e.what()).log();
345345
rc = 1;
346346
}
347347

0 commit comments

Comments
 (0)