Skip to content

Commit 7347d13

Browse files
MykolaGolubyevMykolaGolubyev
authored
matchers: option to force assertion error contain full details (#1589)
1 parent c62b70d commit 7347d13

12 files changed

Lines changed: 137 additions & 45 deletions

File tree

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/cfg/WebTauConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ public class WebTauConfig implements PrettyPrintable {
5353

5454
private final ConfigValue verbosityLevel = declare("verbosityLevel", "output verbosity level. " +
5555
"0 - no output; 1 - test names; 2 - first level steps; etc", () -> Integer.MAX_VALUE);
56+
private final ConfigValue fullAssertionError = declare("fullAssertionError",
57+
"Always include detailed message in assertion error (even when console output is present)", () -> false);
5658
private final ConfigValue fullStackTrace = declare("fullStackTrace", "print full stack trace to console",
5759
() -> false);
5860
private final ConfigValue disableConsoleOverallReport = declare("disableConsoleOverallReport", "do not print failed tests, overall summary and path to the generated report at the end", () -> false);
@@ -161,6 +163,14 @@ public void setVerbosityLevel(int level) {
161163
verbosityLevel.setAndReport("manual", level);
162164
}
163165

166+
public boolean isFullAssertionError() {
167+
return fullAssertionError.getAsBoolean();
168+
}
169+
170+
public void setFullAssertionError(boolean isFullAssertionError) {
171+
fullAssertionError.setAndReport("manual", isFullAssertionError);
172+
}
173+
164174
public boolean getFullStackTrace() {
165175
return fullStackTrace.getAsBoolean();
166176
}
@@ -471,6 +481,7 @@ private Map<String, ConfigValue> enumerateRegisteredConfigValues() {
471481
url,
472482
httpProxy,
473483
verbosityLevel,
484+
fullAssertionError,
474485
fullStackTrace,
475486
disableConsoleOverallReport,
476487
tableVerticalSeparator,

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/console/StringNoAnsiConsoleOutput.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright 2024 webtau maintainers
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.testingisdocumenting.webtau.console;
18+
19+
import org.testingisdocumenting.webtau.console.ansi.IgnoreAnsiString;
20+
21+
import java.util.ArrayList;
22+
import java.util.List;
23+
24+
public class StringNoAnsiConsoleOutput implements ConsoleOutput {
25+
private final List<String> out = new ArrayList<>();
26+
private final List<String> err = new ArrayList<>();
27+
28+
@Override
29+
public void out(Object... styleOrValues) {
30+
out.add(new IgnoreAnsiString(styleOrValues).toString());
31+
}
32+
33+
@Override
34+
public void err(Object... styleOrValues) {
35+
err.add(new IgnoreAnsiString(styleOrValues).toString());
36+
}
37+
38+
public String getOut() {
39+
return String.join("\n", out);
40+
}
41+
42+
public String getErr() {
43+
return String.join("\n", err);
44+
}
45+
}

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/data/render/PrettyPrinter.java‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
package org.testingisdocumenting.webtau.data.render;
1818

1919
import org.testingisdocumenting.webtau.console.ConsoleOutput;
20+
import org.testingisdocumenting.webtau.console.StringNoAnsiConsoleOutput;
2021
import org.testingisdocumenting.webtau.console.ansi.AnsiAsStylesValuesListConsoleOutput;
2122
import org.testingisdocumenting.webtau.console.ansi.Color;
22-
import org.testingisdocumenting.webtau.console.ansi.IgnoreAnsiString;
2323
import org.testingisdocumenting.webtau.data.ValuePath;
2424
import org.testingisdocumenting.webtau.data.converters.ValueConverter;
2525
import org.testingisdocumenting.webtau.utils.ServiceLoaderUtils;
@@ -147,19 +147,10 @@ public static String renderAsTextWithoutColors(Object value) {
147147
}
148148

149149
public String renderAsTextWithoutColors() {
150-
List<String> lines = new ArrayList<>();
151-
renderToConsole(new ConsoleOutput() {
152-
@Override
153-
public void out(Object... styleOrValues) {
154-
lines.add(new IgnoreAnsiString(styleOrValues).toString());
155-
}
156-
157-
@Override
158-
public void err(Object... styleOrValues) {
159-
}
160-
});
150+
StringNoAnsiConsoleOutput output = new StringNoAnsiConsoleOutput();
151+
renderToConsole(output);
161152

162-
return String.join("\n", lines);
153+
return output.getOut();
163154
}
164155

165156
public int calcMaxWidth() {

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/reporter/ConsoleStepReporter.java‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.testingisdocumenting.webtau.reporter;
1919

20-
import org.testingisdocumenting.webtau.console.ConsoleOutputs;
20+
import org.testingisdocumenting.webtau.console.ConsoleOutput;
2121
import org.testingisdocumenting.webtau.console.ansi.AnsiConsoleUtils;
2222
import org.testingisdocumenting.webtau.console.ansi.Color;
2323
import org.testingisdocumenting.webtau.console.ansi.FontStyle;
@@ -33,10 +33,12 @@
3333
import static org.testingisdocumenting.webtau.WebTauCore.*;
3434

3535
public class ConsoleStepReporter implements StepReporter {
36+
private final ConsoleOutput output;
3637
private final TokenizedMessageToAnsiConverter toAnsiConverter;
3738
private final Supplier<Integer> verboseLevelSupplier;
3839

39-
public ConsoleStepReporter(TokenizedMessageToAnsiConverter toAnsiConverter, Supplier<Integer> verboseLevelSupplier) {
40+
public ConsoleStepReporter(ConsoleOutput output, TokenizedMessageToAnsiConverter toAnsiConverter, Supplier<Integer> verboseLevelSupplier) {
41+
this.output = output;
4042
this.toAnsiConverter = toAnsiConverter;
4143
this.verboseLevelSupplier = verboseLevelSupplier;
4244
}
@@ -56,6 +58,15 @@ public void onStepFailure(WebTauStep step) {
5658
executeIfWithinVerboseLevel(step, () -> printStepFailure(step, false));
5759
}
5860

61+
public void printStepFailure(WebTauStep step, boolean forceFailureDisplay) {
62+
if (step.getClassifier().equals(WebTauStepClassifiers.MATCHER)) {
63+
printStepFailureFailedStepMessageFirst(step, forceFailureDisplay);
64+
} else {
65+
printStepOutputs(step);
66+
printStepFailureWithoutOutput(step, forceFailureDisplay);
67+
}
68+
}
69+
5970
@Override
6071
public void onStepRepeatStart(WebTauStep step, int current, int total) {
6172
executeIfWithinVerboseLevel(step, () -> printStepRepeatStart(step, current, total));
@@ -86,7 +97,7 @@ public void printStepFailureWithoutOutput(WebTauStep step, boolean forceFailureD
8697
List<Object> prefix = Stream.concat(stepFailureBeginningStream(step), personaStream(step))
8798
.collect(Collectors.toList());
8899

89-
ConsoleOutputs.out(Stream.concat(Stream.concat(prefix.stream(),
100+
output.out(Stream.concat(Stream.concat(prefix.stream(),
90101
toAnsiConverter.convert(step.getValueConverter(), completionMessageToUse, AnsiConsoleUtils.calcEffectiveWidth(prefix)).stream()),
91102
timeTakenTokenStream(step)).toArray());
92103
}
@@ -98,14 +109,14 @@ public void printStepOutputs(WebTauStep step) {
98109

99110
PrettyPrinter printer = createInputOutputPrettyPrinter(step);
100111
step.getOutputsStream().forEach(output -> output.prettyPrint(printer));
101-
printer.renderToConsole(ConsoleOutputs.asCombinedConsoleOutput());
112+
printer.renderToConsole(output);
102113
}
103114

104115
private void printStepStart(WebTauStep step) {
105116
List<Object> prefix = Stream.concat(stepStartBeginningStream(step), personaStream(step))
106117
.collect(Collectors.toList());
107118

108-
ConsoleOutputs.out(
119+
output.out(
109120
Stream.concat(
110121
prefix.stream(),
111122
toAnsiConverter.convert(step.getValueConverter(), step.getInProgressMessage(), AnsiConsoleUtils.calcEffectiveWidth(prefix)).stream()
@@ -131,29 +142,20 @@ private void printStepSuccess(WebTauStep step) {
131142
List<Object> messagePrefix = Stream.concat(stepSuccessBeginningStream(step), personaStream(step))
132143
.collect(Collectors.toList());
133144

134-
ConsoleOutputs.out(Stream.concat(
145+
output.out(Stream.concat(
135146
Stream.concat(
136147
messagePrefix.stream(),
137148
toAnsiConverter.convert(step.getValueConverter(), completionMessageToUse, AnsiConsoleUtils.calcEffectiveWidth(messagePrefix)).stream()),
138149
timeTakenTokenStream(step)).toArray());
139150
}
140151

141-
private void printStepFailure(WebTauStep step, boolean forceFailureDisplay) {
142-
if (step.getClassifier().equals(WebTauStepClassifiers.MATCHER)) {
143-
printStepFailureFailedStepMessageFirst(step, forceFailureDisplay);
144-
} else {
145-
printStepOutputs(step);
146-
printStepFailureWithoutOutput(step, forceFailureDisplay);
147-
}
148-
}
149-
150152
private void printStepRepeatStart(WebTauStep step, int currentIdx, int total) {
151-
ConsoleOutputs.out(Stream.concat(stepStartBeginningStream(step),
153+
output.out(Stream.concat(stepStartBeginningStream(step),
152154
stepCurrentIdxOfTotalStream(currentIdx, total)).toArray());
153155
}
154156

155157
private void printStepRepeatSuccess(WebTauStep step, int currentIdx, int total) {
156-
ConsoleOutputs.out(Stream.concat(stepSuccessBeginningStream(step),
158+
output.out(Stream.concat(stepSuccessBeginningStream(step),
157159
Stream.concat(
158160
stepCurrentIdxOfTotalStream(currentIdx, total),
159161
timeTakenTokenStream(step))).toArray());
@@ -202,7 +204,7 @@ private void printStepInput(WebTauStep step) {
202204

203205
PrettyPrinter printer = createInputOutputPrettyPrinter(step);
204206
step.getInput().prettyPrint(printer);
205-
printer.renderToConsole(ConsoleOutputs.asCombinedConsoleOutput());
207+
printer.renderToConsole(output);
206208
}
207209

208210
private PrettyPrinter createInputOutputPrettyPrinter(WebTauStep step) {

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/reporter/StepReporters.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.testingisdocumenting.webtau.reporter;
1919

2020
import org.testingisdocumenting.webtau.cfg.WebTauConfig;
21+
import org.testingisdocumenting.webtau.console.ConsoleOutputs;
2122
import org.testingisdocumenting.webtau.utils.ServiceLoaderUtils;
2223

2324
import java.util.ArrayList;
@@ -29,7 +30,7 @@
2930

3031
public class StepReporters {
3132
public static final ConsoleStepReporter defaultStepReporter =
32-
new ConsoleStepReporter(TokenizedMessageToAnsiConverter.DEFAULT, () -> WebTauConfig.getCfg().getVerbosityLevel());
33+
new ConsoleStepReporter(ConsoleOutputs.asCombinedConsoleOutput(), TokenizedMessageToAnsiConverter.DEFAULT, () -> WebTauConfig.getCfg().getVerbosityLevel());
3334

3435
private static final List<StepReporter> reporters = Collections.synchronizedList(
3536
ServiceLoaderUtils.load(StepReporter.class));

‎webtau-core/src/main/java/org/testingisdocumenting/webtau/reporter/WebTauStep.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717

1818
package org.testingisdocumenting.webtau.reporter;
1919

20+
import org.testingisdocumenting.webtau.cfg.WebTauConfig;
2021
import org.testingisdocumenting.webtau.console.ConsoleOutputs;
22+
import org.testingisdocumenting.webtau.console.StringNoAnsiConsoleOutput;
2123
import org.testingisdocumenting.webtau.data.converters.ValueConverter;
2224
import org.testingisdocumenting.webtau.expectation.AssertionTokenizedError;
2325
import org.testingisdocumenting.webtau.persona.Persona;
@@ -447,7 +449,7 @@ private <R> R executeSingleRunWithAction(StepReportOptions stepReportOptions,
447449

448450
// to avoid full mismatch reports printing twice
449451
if (e instanceof AssertionTokenizedError) {
450-
throw new AssertionError(reduceMismatchedMessage(e.getMessage()));
452+
throw new AssertionError(reduceOrEnhanceAssertionMessage(e.getMessage()));
451453
} else {
452454
throw e;
453455
}
@@ -459,26 +461,37 @@ private <R> R executeSingleRunWithAction(StepReportOptions stepReportOptions,
459461
}
460462
}
461463

462-
private String reduceMismatchedMessage(String message) {
464+
private String reduceOrEnhanceAssertionMessage(String message) {
463465
// we throw the full message if the details are not rendered to the console
466+
// or config explicitly asks for full messages at all times
464467
if (!StepReporters.isConsoleStepReporterActive() ||
465468
!ConsoleOutputs.isTerminalConsoleOutputActive() ||
466-
!StepReporters.defaultStepReporter.isWithinVerboseLevel(this)) {
467-
return message;
469+
!StepReporters.defaultStepReporter.isWithinVerboseLevel(this) ||
470+
WebTauConfig.getCfg().isFullAssertionError()) {
471+
return renderStep(this);
468472
}
469473

470-
if (StringUtils.numberOfLines(message) == 1) {
474+
String seeMoreLabel = "see the failed assertion details above";
475+
if (message.equals(seeMoreLabel)) {
471476
return message;
472477
}
473478

474-
String seeMoreLabel = "see the failed assertion details above";
475-
if (message.equals(seeMoreLabel)) {
479+
if (StringUtils.numberOfLines(message) == 1) {
476480
return message;
477481
}
478482

479483
return seeMoreLabel;
480484
}
481485

486+
private String renderStep(WebTauStep step) {
487+
StringNoAnsiConsoleOutput output = new StringNoAnsiConsoleOutput();
488+
ConsoleStepReporter reporter = new ConsoleStepReporter(output,
489+
TokenizedMessageToAnsiConverter.DEFAULT,
490+
() -> Integer.MAX_VALUE);
491+
reporter.printStepFailure(step, true);
492+
return output.getOut();
493+
}
494+
482495
private <R> R executeMultipleRuns(StepReportOptions stepReportOptions) {
483496
WebTauStep repeatRoot = getCurrentStep();
484497
R result = executeSingleRunWithAction(stepReportOptions, multipleRunsActionWrapper(stepReportOptions));

‎webtau-core/src/test/groovy/org/testingisdocumenting/webtau/reporter/ConsoleStepReporterTest.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,8 @@ class ConsoleStepReporterTest implements ConsoleOutput {
311311

312312
private void expectReport(int verbosityLevel, String expectedReport, Closure code) {
313313
lines = new ArrayList<>()
314-
def stepReporter = new ConsoleStepReporter(TokenizedMessageToAnsiConverter.DEFAULT, () -> verbosityLevel)
314+
def stepReporter = new ConsoleStepReporter(ConsoleOutputs.asCombinedConsoleOutput(),
315+
TokenizedMessageToAnsiConverter.DEFAULT, () -> verbosityLevel)
315316

316317
try {
317318
StepReporters.add(stepReporter)

‎webtau-core/src/test/groovy/org/testingisdocumenting/webtau/reporter/WebTauStepTest.groovy‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,12 +223,33 @@ class WebTauStepTest {
223223

224224
code {
225225
step1.execute(SKIP_START)
226-
} should(throwException("error\ndetails"))
226+
} should(throwException(contain("X failed c1 action:\n" +
227+
" error\n" +
228+
" details")))
227229
} finally {
228230
WebTauConfig.getCfg().setVerbosityLevel(Integer.MAX_VALUE)
229231
}
230232
}
231233

234+
@Test
235+
void "should throw full exception details if explicitly forced via config"() {
236+
WebTauConfig.getCfg().setFullAssertionError(true)
237+
238+
try {
239+
def step1 = createStep("c1 action") {
240+
throw new AssertionTokenizedError(tokenizedMessage().error("error").newLine().action("details"))
241+
}
242+
243+
code {
244+
step1.execute(SKIP_START)
245+
} should(throwException(contain("X failed c1 action:\n" +
246+
" error\n" +
247+
" details")))
248+
} finally {
249+
WebTauConfig.getCfg().setFullAssertionError(false)
250+
}
251+
}
252+
232253
@Test
233254
void "should throw full exception details if console step reporter is not present"() {
234255
def output = new TestConsoleOutput()
@@ -240,7 +261,9 @@ class WebTauStepTest {
240261

241262
code {
242263
step1.execute(SKIP_START)
243-
} should(throwException("error\ndetails"))
264+
} should(throwException(contain("X failed c1 action:\n" +
265+
" error\n" +
266+
" details")))
244267
} finally {
245268
ConsoleOutputs.remove(output)
246269
}

‎webtau-docs/znai/release-notes/2.4/add-2024-04-09-force-assertion-full-details.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* Add: `fullAssertionError` to force `AssertionError` to include the full step information. Convenient when using CI and depend on its reporting.

‎webtau-groovy-app/src/main/groovy/org/testingisdocumenting/webtau/app/WebTauCliApp.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@ class WebTauCliApp implements TestListener, ReportGenerator {
236236
}
237237

238238
private static StepReporter createConsoleStepReporter() {
239-
return new ConsoleStepReporter(TokenizedMessageToAnsiConverter.DEFAULT, () -> cfg.getVerbosityLevel())
239+
return new ConsoleStepReporter(ConsoleOutputs.asCombinedConsoleOutput(),
240+
TokenizedMessageToAnsiConverter.DEFAULT, () -> cfg.getVerbosityLevel())
240241
}
241242
}

0 commit comments

Comments
 (0)