From b5ee7a0b9144698895e0f6fc8380dd9a3d00eef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Havl=C3=AD=C4=8Dek?= Date: Sat, 25 Jul 2026 08:32:34 +0200 Subject: [PATCH] add -testQuiet to report only failing tests Running a project with hundreds of test functions prints several lines per passing test, which buries the failures in CI. -testQuiet (short form -tq) buffers the output of the running test, discards it when the test passes and prints it in full when the test fails. The summary is always printed. Also repairs the interpreter output redirection this relies on, which never reached the tests' own output: - ProgramStateIO shadowed ProgramState.outStream and overrode the accessors without forwarding to the native providers. - ReflectionNativeProvider.setOutStream was empty, so OutputProvider kept writing to its default System.err. - The PrintStream wrapping the redirect had no autoflush, so its buffer was never emptied. As a result, output printed by tests during -runtests now goes to the stream RunTests prints to instead of System.err, as the redirection intended. --- .../de/peeeq/wurstio/CompilationProcess.java | 2 +- .../interpreter/ProgramStateIO.java | 10 -- .../ReflectionNativeProvider.java | 11 ++- .../providers/OutputProvider.java | 4 + .../languageserver/requests/RunTests.java | 70 +++++++++++-- .../java/de/peeeq/wurstscript/RunArgs.java | 33 ++++++- .../wurstscript/tests/RunTestsQuietTests.java | 97 +++++++++++++++++++ 7 files changed, 205 insertions(+), 22 deletions(-) create mode 100644 de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java index dc60e61fa..097065167 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java @@ -160,7 +160,7 @@ private void runTests(ImTranslator translator, WurstCompilerJassImpl compiler, i if (!runArgs.isCompactOutput()) { System.out.println("Running tests"); } - RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput()) { + RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput(), runArgs.isTestQuiet()) { @Override protected void print(String message) { out.print(message); diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java index 566d97fb1..7d3cc78ec 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java @@ -40,7 +40,6 @@ public class ProgramStateIO extends ProgramState { private final Map> createdObjectDefinitionIds = Maps.newLinkedHashMap(); private int id = 0; private final Map objDefinitions = Maps.newLinkedHashMap(); - private PrintStream outStream = System.err; private @Nullable WTS trigStrings = null; private final Optional mapFile; @@ -946,13 +945,4 @@ private Optional getObjectEditingOutputFolder() { return folder; } - @Override - public PrintStream getOutStream() { - return outStream; - } - - @Override - public void setOutStream(PrintStream os) { - outStream = os; - } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java index e25e93af5..a4cf28654 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java @@ -10,10 +10,14 @@ import java.io.PrintStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; public class ReflectionNativeProvider implements NativesProvider { private final HashMap methodMap = new HashMap<>(); + /** Providers that print to the interpreter output, so that redirecting it reaches them. */ + private final List outputProviders = new ArrayList<>(); public ReflectionNativeProvider(AbstractInterpreter interpreter) { addProvider(new AbilityProvider(interpreter)); @@ -51,6 +55,9 @@ public NativeJassFunction getFunctionPair(String funcName) { } private void addProvider(Provider provider) { + if (provider instanceof OutputProvider) { + outputProviders.add((OutputProvider) provider); + } for (Method method : provider.getClass().getMethods()) { Implements annotation = method.getAnnotation(Implements.class); if (annotation != null) { @@ -103,6 +110,8 @@ public ILconst invoke(String funcname, ILconst[] args) throws NoSuchNativeExcept @Override public void setOutStream(PrintStream outStream) { - + for (OutputProvider provider : outputProviders) { + provider.setOutStream(outStream); + } } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java index 4bccfaf72..083953019 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java @@ -18,6 +18,10 @@ public OutputProvider(AbstractInterpreter interpreter) { super(interpreter); } + public void setOutStream(PrintStream outStream) { + this.outStream = outStream; + } + public void DisplayTextToForce(IlConstHandle force, ILconstString msg) { outStream.println(msg.getVal()); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java index 61a62d37f..5e597cd2a 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java @@ -29,6 +29,7 @@ import org.eclipse.lsp4j.MessageType; import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.concurrent.*; @@ -47,6 +48,15 @@ public class RunTests extends UserRequest { private final int timeoutSeconds; private final Optional testFilter; private final boolean compactOutput; + private final boolean testQuiet; + + /** + * In quiet mode the output of the test currently running is collected here instead of + * being printed directly. It is flushed when the test fails and discarded when it passes. + * Synchronized, because a test that timed out keeps running on the test executor thread + * while this thread already flushes the buffer. + */ + private final StringBuffer pendingOutput = new StringBuffer(); private final List successTests = Lists.newArrayList(); private final List failTests = Lists.newArrayList(); @@ -102,6 +112,11 @@ public RunTests(Optional filename, int line, int column, Optional filename, int line, int column, Optional testName, int timeoutSeconds, Optional testFilter, boolean compactOutput) { + this(filename, line, column, testName, timeoutSeconds, testFilter, compactOutput, false); + } + + public RunTests(Optional filename, int line, int column, Optional testName, int timeoutSeconds, Optional testFilter, boolean compactOutput, boolean testQuiet) { + this.testQuiet = testQuiet; this.filename = filename.map(WFile::create); this.line = line; this.column = column; @@ -177,12 +192,16 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { + // compiletime output is relevant context for these errors, so keep it + flushPendingOutput(); for (CompileError compileError : gui.getErrorList()) { println(compactOutput ? compileError.toCompactString() : compileError.toString()); } println("There were some problem while running compiletime expressions and functions."); return new TestResult(0, 1); } + // output of successful compiletime functions does not belong to any test + discardPendingOutput(); WLogger.info("Ran compiletime functions"); @@ -232,7 +251,7 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { + flushPendingOutput(); StringBuilder sb = new StringBuilder(); int appendedErrors = 0; for (CompileError error : gui.getErrorList()) { @@ -285,15 +305,18 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { - println("" + (char) b); + testPrintln("" + (char) b); } } @Override public void write(byte[] b, int off, int len) throws IOException { if (!compactOutput) { - println(new String(b, off, len)); + testPrintln(new String(b, off, len, StandardCharsets.UTF_8)); } } }; - globalState.setOutStream(new PrintStream(os)); + // autoflush, so that output of a test is delivered while that test is still running + // and not left in the buffer of the PrintStream + globalState.setOutStream(new PrintStream(os, true, StandardCharsets.UTF_8)); } private String qualifiedTestName(ImFunction f) { @@ -388,6 +416,36 @@ protected void println(String message) { print(System.lineSeparator()); } + /** + * Prints output belonging to a single test. In quiet mode the output is buffered, so that + * it can be dropped if the test passes, or printed as usual if the test fails. + */ + private void testPrintln(String message) { + if (testQuiet) { + // appended in one call, so that a concurrent flush cannot split the line + pendingOutput.append(message + System.lineSeparator()); + } else { + println(message); + } + } + + /** Prints the buffered output of the current test, used when the test turned out to fail. */ + private void flushPendingOutput() { + String pending; + synchronized (pendingOutput) { + pending = pendingOutput.toString(); + pendingOutput.setLength(0); + } + if (!pending.isEmpty()) { + print(pending); + } + } + + /** Drops the buffered output of the current test, used when the test passed. */ + private void discardPendingOutput() { + pendingOutput.setLength(0); + } + protected void print(String message) { System.err.print(message); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java index d6cae57e9..1949e3273 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java @@ -58,6 +58,7 @@ public class RunArgs { private final RunOption optionHotStartmap; private final RunOption optionHotReload; private final RunOption optionTestTimeout; + private final RunOption optionTestQuiet; private final RunOption optionDevBuild; private int functionSplitLimit = 10000; @@ -82,17 +83,24 @@ public RunArgs with(String... additionalArgs) { private static class RunOption { final String name; + final @Nullable String alias; final String descr; final @Nullable Consumer argHandler; boolean isSet; RunOption(String name, String descr) { + this(name, null, descr); + } + + RunOption(String name, @Nullable String alias, String descr) { this.name = name; + this.alias = alias; this.descr = descr; this.argHandler = null; } RunOption(String name, String descr, Consumer argHandler2) { this.name = name; + this.alias = null; this.descr = descr; this.argHandler = argHandler2; } @@ -108,6 +116,8 @@ public RunArgs(String... args) { // interpreter optionRuntests = addOption("runtests", "Run all test functions found in the scripts."); optionTestTimeout = addOptionWithArg("testTimeout", "Timeout in seconds after which tests will be cancelled and considered failed, if they did not yet succeed.", arg -> testTimeout = Integer.parseInt(arg)); + optionTestQuiet = addOption("testQuiet", "tq", "Only report failing tests: output of passing tests is suppressed, " + + "while failing tests still print their full output, assertion message and stack trace."); addOptionWithArg("testFilter", "Only run tests whose qualified name (Package.function) contains this string (case-insensitive).", arg -> testFilter = arg); optionRunCompileTimeFunctions = addOption("runcompiletimefunctions", "Run all compiletime functions found in the scripts."); optionInjectCompiletimeObjects = addOption("injectobjects", "Injects the objects generated by compiletime functions into the map."); @@ -167,7 +177,7 @@ public RunArgs(String... args) { String a = args[i]; if (a.startsWith("-")) { for (RunOption o : options) { - if (("-" + o.name).equals(a)) { + if (("-" + o.name).equals(a) || (o.alias != null && ("-" + o.alias).equals(a))) { Consumer argHandler = o.argHandler; if (argHandler != null) { i++; @@ -208,11 +218,16 @@ public RunArgs(String... args) { } private boolean isDoubleArg(String arg, RunOption option) { - return (arg.contains(" ") && ("-" + option.name).equals(arg.substring(0, arg.indexOf(" ")))); + if (!arg.contains(" ")) { + return false; + } + String name = arg.substring(0, arg.indexOf(" ")); + return ("-" + option.name).equals(name) || (option.alias != null && ("-" + option.alias).equals(name)); } private boolean isEqualsArg(String arg, RunOption option) { - return arg.startsWith("-" + option.name + "="); + return arg.startsWith("-" + option.name + "=") + || (option.alias != null && arg.startsWith("-" + option.alias + "=")); } private RunOption addOption(String name, String descr) { @@ -221,6 +236,12 @@ private RunOption addOption(String name, String descr) { return opt; } + private RunOption addOption(String name, String alias, String descr) { + RunOption opt = new RunOption(name, alias, descr); + options.add(opt); + return opt; + } + private RunOption addOptionWithArg(String name, String descr, Consumer argHandler) { RunOption opt = new RunOption(name, descr, argHandler); options.add(opt); @@ -241,7 +262,7 @@ public void printHelpAndExit() { System.out.println("Options:"); System.out.println(); for (RunOption opt : options) { - System.out.println("-" + opt.name); + System.out.println(opt.alias == null ? "-" + opt.name : "-" + opt.name + ", -" + opt.alias); System.out.println(" " + opt.descr); System.out.println(); } @@ -403,6 +424,10 @@ public Optional getTestFilter() { return Optional.ofNullable(testFilter); } + public boolean isTestQuiet() { + return optionTestQuiet.isSet; + } + public boolean isMeasureTimes() { return optionMeasureTimes.isSet; } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java new file mode 100644 index 000000000..30d1ee46b --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java @@ -0,0 +1,97 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstio.WurstCompilerJassImpl; +import de.peeeq.wurstio.languageserver.requests.RunTests; +import de.peeeq.wurstscript.RunArgs; +import de.peeeq.wurstscript.ast.WurstModel; +import de.peeeq.wurstscript.gui.WurstGui; +import de.peeeq.wurstscript.gui.WurstGuiCliImpl; +import de.peeeq.wurstscript.jassIm.ImProg; +import de.peeeq.wurstscript.utils.Utils; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +/** + * Tests for the -testQuiet option, which suppresses the output of passing tests + * while keeping the full output of failing tests. + */ +public class RunTestsQuietTests extends WurstScriptTest { + + private static final String[] PROGRAM = { + "package test", + "native testFail(string msg)", + "native println(string msg)", + "@test function passingTest()", + " println(\"noise from the passing test\")", + "@test function failingTest()", + " println(\"context from the failing test\")", + " testFail(\"assertion message\")", + }; + + @Test + public void quietHidesPassingTests() { + String output = runTestsWith(true); + assertFalse(output.contains("passingTest"), output); + assertFalse(output.contains("noise from the passing test"), output); + assertFalse(output.contains("OK!"), output); + } + + @Test + public void quietKeepsFailingTests() { + String output = runTestsWith(true); + assertTrue(output.contains("failingTest"), output); + assertTrue(output.contains("context from the failing test"), output); + assertTrue(output.contains("FAILED assertion"), output); + assertTrue(output.contains("assertion message"), output); + } + + @Test + public void quietKeepsSummary() { + String output = runTestsWith(true); + assertTrue(output.contains("Tests succeeded: 1/2"), output); + assertTrue(output.contains("1 Tests have failed!"), output); + } + + @Test + public void defaultReportsPassingTests() { + String output = runTestsWith(false); + assertTrue(output.contains("passingTest"), output); + assertTrue(output.contains("noise from the passing test"), output); + assertTrue(output.contains("OK!"), output); + } + + @Test + public void quietFlagIsParsed() { + assertTrue(new RunArgs("-testQuiet").isTestQuiet()); + assertTrue(new RunArgs("-tq").isTestQuiet()); + assertFalse(new RunArgs("-runtests").isTestQuiet()); + } + + private String runTestsWith(boolean testQuiet) { + RunArgs runArgs = new RunArgs(); + WurstGui gui = new WurstGuiCliImpl(); + WurstCompilerJassImpl compiler = new WurstCompilerJassImpl(null, gui, null, runArgs); + WurstModel model = parseFiles(null, + Collections.singletonList(new CU("test", Utils.join(PROGRAM, "\n") + "\n")), false, compiler); + compiler.checkProg(model); + if (!gui.getErrorList().isEmpty()) { + throw gui.getErrorList().get(0); + } + ImProg imProg = compiler.translateProgToIm(model); + + StringBuilder output = new StringBuilder(); + RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), 20, Optional.empty(), false, testQuiet) { + @Override + protected void print(String message) { + output.append(message); + } + }; + runTests.runTests(compiler.getImTranslator(), imProg, Optional.empty(), Optional.empty()); + return output.toString(); + } +}