Skip to content

Commit 6f0541c

Browse files
NathanielVolfangojenkins
NathanielVolfango
authored and
jenkins
committed
QPR-12190 -- Add progress log 'detail' key, and set jsondiff abs/rel tolerance defaults
1 parent 28f77bc commit 6f0541c

7 files changed

Lines changed: 45 additions & 31 deletions

File tree

‎OREAnalytics/orea/engine/amcvaluationengine.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,9 @@ void runCoreEngine(const boost::shared_ptr<ore::data::Portfolio>& portfolio,
145145
const Size aggDataNumberCreditStates, boost::shared_ptr<ore::analytics::AggregationScenarioData> asd,
146146
boost::shared_ptr<NPVCube> outputCube, boost::shared_ptr<ProgressIndicator> progressIndicator) {
147147

148-
progressIndicator->updateProgress(0, portfolio->size());
148+
std::ostringstream detail;
149+
detail << portfolio->size() << " trade" << (portfolio->size() == 1 ? "" : "s");
150+
progressIndicator->updateProgress(0, portfolio->size(), detail.str());
149151

150152
// base currency is the base currency of the cam
151153

@@ -547,7 +549,9 @@ void runCoreEngine(const boost::shared_ptr<ore::data::Portfolio>& portfolio,
547549
}
548550
}
549551
}
550-
progressIndicator->updateProgress(++progressCounter, portfolio->size());
552+
std::ostringstream detail;
553+
detail << portfolio->size() << " trade" << (portfolio->size() == 1 ? "" : "s");
554+
progressIndicator->updateProgress(++progressCounter, portfolio->size(), detail.str());
551555
}
552556
timer.stop();
553557
valuationTime += timer.elapsed().wall * 1e-9;

‎OREAnalytics/orea/engine/valuationengine.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,11 @@ void ValuationEngine::buildCube(const boost::shared_ptr<data::Portfolio>& portfo
239239
}
240240
}
241241
}
242+
std::ostringstream detail;
243+
detail << nTrades << " trade" << (nTrades == 1 ? "" : "s") << ", " << outputCube->samples() << " sample"
244+
<< (outputCube->samples() == 1 ? "" : "s");
245+
updateProgress(sample * nTrades, outputCube->samples() * nTrades, detail.str());
246+
242247
timer.start();
243248
simMarket_->fixingManager()->reset();
244249
fixingTime += timer.elapsed().wall * 1e-9;
@@ -260,7 +265,10 @@ void ValuationEngine::buildCube(const boost::shared_ptr<data::Portfolio>& portfo
260265
}
261266
}
262267

263-
updateProgress(outputCube->samples() * nTrades, outputCube->samples() * nTrades);
268+
std::ostringstream detail;
269+
detail << nTrades << " trade" << (nTrades == 1 ? "" : "s") << ", " << outputCube->samples() << " sample"
270+
<< (outputCube->samples() == 1 ? "" : "s");
271+
updateProgress(outputCube->samples() * nTrades, outputCube->samples() * nTrades, detail.str());
264272
loopTimer.stop();
265273
LOG("ValuationEngine completed: loop " << setprecision(2) << loopTimer.format(2, "%w") << " sec, "
266274
<< "pricing " << pricingTime << " sec, "

‎OREData/ored/utilities/log.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,8 +620,10 @@ void EventMessage::emitLog() const {
620620
MLOG(oreSeverity::alert, msg());
621621
}
622622

623-
ProgressMessage::ProgressMessage(const string& key, const Size progressCurrent, const Size progressTotal) {
623+
ProgressMessage::ProgressMessage(const string& key, const Size progressCurrent, const Size progressTotal, const string& detail) {
624624
data_["key"] = key;
625+
if (!detail.empty())
626+
data_["detail"] = detail;
625627
data_["progress"] = progressCurrent;
626628
data_["total"] = progressTotal;
627629
data_["@timestamp"] =

‎OREData/ored/utilities/log.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -730,7 +730,7 @@ class EventMessage : public JSONMessage {
730730

731731
class ProgressMessage : public JSONMessage {
732732
public:
733-
ProgressMessage(const std::string&, const QuantLib::Size, const QuantLib::Size);
733+
ProgressMessage(const std::string&, const QuantLib::Size, const QuantLib::Size, const std::string& detail = "");
734734

735735
static constexpr const char* name = "ProgressMessage";
736736

‎OREData/ored/utilities/progressbar.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ void ProgressReporter::unregisterProgressIndicator(const boost::shared_ptr<Progr
3434

3535
void ProgressReporter::unregisterAllProgressIndicators() { indicators_.clear(); }
3636

37-
void ProgressReporter::updateProgress(const unsigned long progress, const unsigned long total) {
37+
void ProgressReporter::updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) {
3838
for (const auto& i : indicators_)
39-
i->updateProgress(progress, total);
39+
i->updateProgress(progress, total, detail);
4040
}
4141

4242
void ProgressReporter::resetProgress() {
@@ -46,32 +46,32 @@ void ProgressReporter::resetProgress() {
4646

4747
SimpleProgressBar::SimpleProgressBar(const std::string& message, const QuantLib::Size messageWidth,
4848
const QuantLib::Size barWidth, const QuantLib::Size numberOfScreenUpdates)
49-
: message_(message), messageWidth_(messageWidth), barWidth_(barWidth),
49+
: key_(message), messageWidth_(messageWidth), barWidth_(barWidth),
5050
numberOfScreenUpdates_(numberOfScreenUpdates), updateCounter_(0), finalized_(false) {
51-
updateProgress(0, 1);
51+
updateProgress(0, 1, "");
5252
updateCounter_--;
5353
}
5454

55-
void SimpleProgressBar::updateProgress(const unsigned long progress, const unsigned long total) {
55+
void SimpleProgressBar::updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) {
5656
if (!ConsoleLog::instance().enabled())
5757
return;
5858
if (finalized_)
5959
return;
6060
double ratio = static_cast<double>(progress) / static_cast<double>(total);
6161
if (progress >= total) {
62-
std::cout << "\r" << std::setw(messageWidth_) << std::left << message_;
62+
std::cout << "\r" << std::setw(messageWidth_) << std::left << key_;
6363
for (unsigned int i = 0; i < barWidth_; ++i)
6464
std::cout << " ";
6565
std::cout << " \r";
66-
std::cout << std::setw(messageWidth_) << std::left << message_;
66+
std::cout << std::setw(messageWidth_) << std::left << key_;
6767
std::cout.flush();
6868
finalized_ = true;
6969
return;
7070
}
7171
if (updateCounter_ > 0 && progress * numberOfScreenUpdates_ < updateCounter_ * total) {
7272
return;
7373
}
74-
std::cout << "\r" << std::setw(messageWidth_) << std::left << message_;
74+
std::cout << "\r" << std::setw(messageWidth_) << std::left << key_;
7575
if (barWidth_ > 0)
7676
std::cout << "[";
7777
unsigned int pos = static_cast<unsigned int>(static_cast<double>(barWidth_) * ratio);
@@ -96,16 +96,16 @@ void SimpleProgressBar::reset() {
9696
}
9797

9898
ProgressLog::ProgressLog(const std::string& message, const unsigned int numberOfMessages, const oreSeverity logLevel)
99-
: message_(message), numberOfMessages_(numberOfMessages), logLevel_(logLevel), messageCounter_(0) {}
99+
: key_(message), numberOfMessages_(numberOfMessages), logLevel_(logLevel), messageCounter_(0) {}
100100

101-
void ProgressLog::updateProgress(const unsigned long progress, const unsigned long total) {
101+
void ProgressLog::updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) {
102102
if (messageCounter_ > 0 && progress * numberOfMessages_ < (messageCounter_ * total)) {
103103
return;
104104
}
105-
MLOG(logLevel_, message_ << " " << progress << " out of " << total << " steps ("
105+
MLOG(logLevel_, key_ << " (" << detail << "): " << progress << " out of " << total << " steps ("
106106
<< static_cast<int>(static_cast<double>(progress) / static_cast<double>(total) * 100.0)
107107
<< "%) completed");
108-
ProgressMessage(message_, progress, total).log();
108+
ProgressMessage(key_, progress, total, detail).log();
109109
messageCounter_++;
110110
}
111111

@@ -115,7 +115,7 @@ MultiThreadedProgressIndicator::MultiThreadedProgressIndicator(
115115
const std::set<boost::shared_ptr<ProgressIndicator>>& indicators)
116116
: indicators_(indicators) {}
117117

118-
void MultiThreadedProgressIndicator::updateProgress(const unsigned long progress, const unsigned long total) {
118+
void MultiThreadedProgressIndicator::updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) {
119119
boost::unique_lock<boost::shared_mutex> lock(mutex_);
120120
threadData_[std::this_thread::get_id()] = std::make_pair(progress, total);
121121
unsigned long progressTmp = 0;
@@ -125,7 +125,7 @@ void MultiThreadedProgressIndicator::updateProgress(const unsigned long progress
125125
totalTmp += d.second.second;
126126
}
127127
for (auto& i : indicators_)
128-
i->updateProgress(progressTmp, totalTmp);
128+
i->updateProgress(progressTmp, totalTmp, detail);
129129
}
130130

131131
void MultiThreadedProgressIndicator::reset() {

‎OREData/ored/utilities/progressbar.hpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class ProgressIndicator {
4242
public:
4343
ProgressIndicator() {}
4444
virtual ~ProgressIndicator() {}
45-
virtual void updateProgress(const unsigned long progress, const unsigned long total) = 0;
45+
virtual void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) = 0;
4646
virtual void reset() = 0;
4747
};
4848

@@ -63,7 +63,7 @@ class ProgressReporter {
6363
void unregisterAllProgressIndicators();
6464

6565
//! update progress
66-
void updateProgress(const unsigned long progress, const unsigned long total);
66+
void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail = "");
6767

6868
//! reset
6969
void resetProgress();
@@ -88,11 +88,11 @@ class SimpleProgressBar : public ProgressIndicator {
8888
const QuantLib::Size barWidth = 40, const QuantLib::Size numberOfScreenUpdates = 100);
8989

9090
//! ProgressIndicator interface
91-
void updateProgress(const unsigned long progress, const unsigned long total) override;
91+
void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) override;
9292
void reset() override;
9393

9494
private:
95-
std::string message_;
95+
std::string key_;
9696
unsigned int messageWidth_, barWidth_, numberOfScreenUpdates_, updateCounter_;
9797
bool finalized_;
9898
};
@@ -102,14 +102,15 @@ class SimpleProgressBar : public ProgressIndicator {
102102
*/
103103
class ProgressLog : public ProgressIndicator {
104104
public:
105-
ProgressLog(const std::string& message, const unsigned int numberOfMessages = 100, const oreSeverity logLevel = oreSeverity::debug);
105+
ProgressLog(const std::string& message, const unsigned int numberOfMessages = 100,
106+
const oreSeverity logLevel = oreSeverity::debug);
106107

107108
//! ProgressIndicator interface
108-
void updateProgress(const unsigned long progress, const unsigned long total) override;
109+
void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) override;
109110
void reset() override;
110111

111112
private:
112-
std::string message_;
113+
std::string key_;
113114
unsigned int numberOfMessages_;
114115
oreSeverity logLevel_;
115116
unsigned int messageCounter_;
@@ -121,15 +122,15 @@ class NoProgressBar : public ProgressIndicator {
121122
NoProgressBar(const std::string& message, const unsigned int messageWidth = 40);
122123

123124
/*! ProgressIndicator interface */
124-
void updateProgress(const unsigned long progress, const unsigned long total) override {}
125+
void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) override {}
125126
void reset() override {}
126127
};
127128

128129
/*! Progress Manager that consolidates updates from multiple threads */
129130
class MultiThreadedProgressIndicator : public ProgressIndicator {
130131
public:
131132
explicit MultiThreadedProgressIndicator(const std::set<boost::shared_ptr<ProgressIndicator>>& indicators);
132-
void updateProgress(const unsigned long progress, const unsigned long total) override;
133+
void updateProgress(const unsigned long progress, const unsigned long total, const std::string& detail) override;
133134
void reset() override;
134135

135136
private:

‎Tools/PythonTools/compare_files.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -655,8 +655,8 @@ def validate_json_diff(json_diff: dict, config: dict, path: str) -> None:
655655
if settings:
656656
for s in settings:
657657
names = s.get('names')
658-
abs_tol = s.get('abs_tol')
659-
rel_tol = s.get('rel_tol')
658+
abs_tol = s.get('abs_tol') or 0
659+
rel_tol = s.get('rel_tol') or 0
660660

661661
# Number diffs
662662
for n in names:
@@ -676,7 +676,6 @@ def validate_json_diff(json_diff: dict, config: dict, path: str) -> None:
676676
abs_diff = abs(val_1 - val_2)
677677
abs_check = abs_diff <= abs_tol
678678
rel_check = abs_diff <= min(abs(val_1 * rel_tol), abs(val_2 * rel_tol))
679-
680679
if abs_check or rel_check:
681680
diff.clear()
682681

0 commit comments

Comments
 (0)