diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java index dc60e61fa..097065167 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java @@ -160,7 +160,7 @@ private void runTests(ImTranslator translator, WurstCompilerJassImpl compiler, i if (!runArgs.isCompactOutput()) { System.out.println("Running tests"); } - RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput()) { + RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput(), runArgs.isTestQuiet()) { @Override protected void print(String message) { out.print(message); diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java index 566d97fb1..7d3cc78ec 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java @@ -40,7 +40,6 @@ public class ProgramStateIO extends ProgramState { private final Map> createdObjectDefinitionIds = Maps.newLinkedHashMap(); private int id = 0; private final Map objDefinitions = Maps.newLinkedHashMap(); - private PrintStream outStream = System.err; private @Nullable WTS trigStrings = null; private final Optional mapFile; @@ -946,13 +945,4 @@ private Optional getObjectEditingOutputFolder() { return folder; } - @Override - public PrintStream getOutStream() { - return outStream; - } - - @Override - public void setOutStream(PrintStream os) { - outStream = os; - } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java index e25e93af5..a4cf28654 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java @@ -10,10 +10,14 @@ import java.io.PrintStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; public class ReflectionNativeProvider implements NativesProvider { private final HashMap methodMap = new HashMap<>(); + /** Providers that print to the interpreter output, so that redirecting it reaches them. */ + private final List outputProviders = new ArrayList<>(); public ReflectionNativeProvider(AbstractInterpreter interpreter) { addProvider(new AbilityProvider(interpreter)); @@ -51,6 +55,9 @@ public NativeJassFunction getFunctionPair(String funcName) { } private void addProvider(Provider provider) { + if (provider instanceof OutputProvider) { + outputProviders.add((OutputProvider) provider); + } for (Method method : provider.getClass().getMethods()) { Implements annotation = method.getAnnotation(Implements.class); if (annotation != null) { @@ -103,6 +110,8 @@ public ILconst invoke(String funcname, ILconst[] args) throws NoSuchNativeExcept @Override public void setOutStream(PrintStream outStream) { - + for (OutputProvider provider : outputProviders) { + provider.setOutStream(outStream); + } } } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java index 4bccfaf72..083953019 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java @@ -18,6 +18,10 @@ public OutputProvider(AbstractInterpreter interpreter) { super(interpreter); } + public void setOutStream(PrintStream outStream) { + this.outStream = outStream; + } + public void DisplayTextToForce(IlConstHandle force, ILconstString msg) { outStream.println(msg.getVal()); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java index 61a62d37f..5e597cd2a 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java @@ -29,6 +29,7 @@ import org.eclipse.lsp4j.MessageType; import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.concurrent.*; @@ -47,6 +48,15 @@ public class RunTests extends UserRequest { private final int timeoutSeconds; private final Optional testFilter; private final boolean compactOutput; + private final boolean testQuiet; + + /** + * In quiet mode the output of the test currently running is collected here instead of + * being printed directly. It is flushed when the test fails and discarded when it passes. + * Synchronized, because a test that timed out keeps running on the test executor thread + * while this thread already flushes the buffer. + */ + private final StringBuffer pendingOutput = new StringBuffer(); private final List successTests = Lists.newArrayList(); private final List failTests = Lists.newArrayList(); @@ -102,6 +112,11 @@ public RunTests(Optional filename, int line, int column, Optional filename, int line, int column, Optional testName, int timeoutSeconds, Optional testFilter, boolean compactOutput) { + this(filename, line, column, testName, timeoutSeconds, testFilter, compactOutput, false); + } + + public RunTests(Optional filename, int line, int column, Optional testName, int timeoutSeconds, Optional testFilter, boolean compactOutput, boolean testQuiet) { + this.testQuiet = testQuiet; this.filename = filename.map(WFile::create); this.line = line; this.column = column; @@ -177,12 +192,16 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { + // compiletime output is relevant context for these errors, so keep it + flushPendingOutput(); for (CompileError compileError : gui.getErrorList()) { println(compactOutput ? compileError.toCompactString() : compileError.toString()); } println("There were some problem while running compiletime expressions and functions."); return new TestResult(0, 1); } + // output of successful compiletime functions does not belong to any test + discardPendingOutput(); WLogger.info("Ran compiletime functions"); @@ -232,7 +251,7 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { + flushPendingOutput(); StringBuilder sb = new StringBuilder(); int appendedErrors = 0; for (CompileError error : gui.getErrorList()) { @@ -285,15 +305,18 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional 0) { - println("" + (char) b); + testPrintln("" + (char) b); } } @Override public void write(byte[] b, int off, int len) throws IOException { if (!compactOutput) { - println(new String(b, off, len)); + testPrintln(new String(b, off, len, StandardCharsets.UTF_8)); } } }; - globalState.setOutStream(new PrintStream(os)); + // autoflush, so that output of a test is delivered while that test is still running + // and not left in the buffer of the PrintStream + globalState.setOutStream(new PrintStream(os, true, StandardCharsets.UTF_8)); } private String qualifiedTestName(ImFunction f) { @@ -388,6 +416,36 @@ protected void println(String message) { print(System.lineSeparator()); } + /** + * Prints output belonging to a single test. In quiet mode the output is buffered, so that + * it can be dropped if the test passes, or printed as usual if the test fails. + */ + private void testPrintln(String message) { + if (testQuiet) { + // appended in one call, so that a concurrent flush cannot split the line + pendingOutput.append(message + System.lineSeparator()); + } else { + println(message); + } + } + + /** Prints the buffered output of the current test, used when the test turned out to fail. */ + private void flushPendingOutput() { + String pending; + synchronized (pendingOutput) { + pending = pendingOutput.toString(); + pendingOutput.setLength(0); + } + if (!pending.isEmpty()) { + print(pending); + } + } + + /** Drops the buffered output of the current test, used when the test passed. */ + private void discardPendingOutput() { + pendingOutput.setLength(0); + } + protected void print(String message) { System.err.print(message); } diff --git a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java index d6cae57e9..1949e3273 100644 --- a/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java +++ b/de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java @@ -58,6 +58,7 @@ public class RunArgs { private final RunOption optionHotStartmap; private final RunOption optionHotReload; private final RunOption optionTestTimeout; + private final RunOption optionTestQuiet; private final RunOption optionDevBuild; private int functionSplitLimit = 10000; @@ -82,17 +83,24 @@ public RunArgs with(String... additionalArgs) { private static class RunOption { final String name; + final @Nullable String alias; final String descr; final @Nullable Consumer argHandler; boolean isSet; RunOption(String name, String descr) { + this(name, null, descr); + } + + RunOption(String name, @Nullable String alias, String descr) { this.name = name; + this.alias = alias; this.descr = descr; this.argHandler = null; } RunOption(String name, String descr, Consumer argHandler2) { this.name = name; + this.alias = null; this.descr = descr; this.argHandler = argHandler2; } @@ -108,6 +116,8 @@ public RunArgs(String... args) { // interpreter optionRuntests = addOption("runtests", "Run all test functions found in the scripts."); optionTestTimeout = addOptionWithArg("testTimeout", "Timeout in seconds after which tests will be cancelled and considered failed, if they did not yet succeed.", arg -> testTimeout = Integer.parseInt(arg)); + optionTestQuiet = addOption("testQuiet", "tq", "Only report failing tests: output of passing tests is suppressed, " + + "while failing tests still print their full output, assertion message and stack trace."); addOptionWithArg("testFilter", "Only run tests whose qualified name (Package.function) contains this string (case-insensitive).", arg -> testFilter = arg); optionRunCompileTimeFunctions = addOption("runcompiletimefunctions", "Run all compiletime functions found in the scripts."); optionInjectCompiletimeObjects = addOption("injectobjects", "Injects the objects generated by compiletime functions into the map."); @@ -167,7 +177,7 @@ public RunArgs(String... args) { String a = args[i]; if (a.startsWith("-")) { for (RunOption o : options) { - if (("-" + o.name).equals(a)) { + if (("-" + o.name).equals(a) || (o.alias != null && ("-" + o.alias).equals(a))) { Consumer argHandler = o.argHandler; if (argHandler != null) { i++; @@ -208,11 +218,16 @@ public RunArgs(String... args) { } private boolean isDoubleArg(String arg, RunOption option) { - return (arg.contains(" ") && ("-" + option.name).equals(arg.substring(0, arg.indexOf(" ")))); + if (!arg.contains(" ")) { + return false; + } + String name = arg.substring(0, arg.indexOf(" ")); + return ("-" + option.name).equals(name) || (option.alias != null && ("-" + option.alias).equals(name)); } private boolean isEqualsArg(String arg, RunOption option) { - return arg.startsWith("-" + option.name + "="); + return arg.startsWith("-" + option.name + "=") + || (option.alias != null && arg.startsWith("-" + option.alias + "=")); } private RunOption addOption(String name, String descr) { @@ -221,6 +236,12 @@ private RunOption addOption(String name, String descr) { return opt; } + private RunOption addOption(String name, String alias, String descr) { + RunOption opt = new RunOption(name, alias, descr); + options.add(opt); + return opt; + } + private RunOption addOptionWithArg(String name, String descr, Consumer argHandler) { RunOption opt = new RunOption(name, descr, argHandler); options.add(opt); @@ -241,7 +262,7 @@ public void printHelpAndExit() { System.out.println("Options:"); System.out.println(); for (RunOption opt : options) { - System.out.println("-" + opt.name); + System.out.println(opt.alias == null ? "-" + opt.name : "-" + opt.name + ", -" + opt.alias); System.out.println(" " + opt.descr); System.out.println(); } @@ -403,6 +424,10 @@ public Optional getTestFilter() { return Optional.ofNullable(testFilter); } + public boolean isTestQuiet() { + return optionTestQuiet.isSet; + } + public boolean isMeasureTimes() { return optionMeasureTimes.isSet; } diff --git a/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java new file mode 100644 index 000000000..30d1ee46b --- /dev/null +++ b/de.peeeq.wurstscript/src/test/java/tests/wurstscript/tests/RunTestsQuietTests.java @@ -0,0 +1,97 @@ +package tests.wurstscript.tests; + +import de.peeeq.wurstio.WurstCompilerJassImpl; +import de.peeeq.wurstio.languageserver.requests.RunTests; +import de.peeeq.wurstscript.RunArgs; +import de.peeeq.wurstscript.ast.WurstModel; +import de.peeeq.wurstscript.gui.WurstGui; +import de.peeeq.wurstscript.gui.WurstGuiCliImpl; +import de.peeeq.wurstscript.jassIm.ImProg; +import de.peeeq.wurstscript.utils.Utils; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Optional; + +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +/** + * Tests for the -testQuiet option, which suppresses the output of passing tests + * while keeping the full output of failing tests. + */ +public class RunTestsQuietTests extends WurstScriptTest { + + private static final String[] PROGRAM = { + "package test", + "native testFail(string msg)", + "native println(string msg)", + "@test function passingTest()", + " println(\"noise from the passing test\")", + "@test function failingTest()", + " println(\"context from the failing test\")", + " testFail(\"assertion message\")", + }; + + @Test + public void quietHidesPassingTests() { + String output = runTestsWith(true); + assertFalse(output.contains("passingTest"), output); + assertFalse(output.contains("noise from the passing test"), output); + assertFalse(output.contains("OK!"), output); + } + + @Test + public void quietKeepsFailingTests() { + String output = runTestsWith(true); + assertTrue(output.contains("failingTest"), output); + assertTrue(output.contains("context from the failing test"), output); + assertTrue(output.contains("FAILED assertion"), output); + assertTrue(output.contains("assertion message"), output); + } + + @Test + public void quietKeepsSummary() { + String output = runTestsWith(true); + assertTrue(output.contains("Tests succeeded: 1/2"), output); + assertTrue(output.contains("1 Tests have failed!"), output); + } + + @Test + public void defaultReportsPassingTests() { + String output = runTestsWith(false); + assertTrue(output.contains("passingTest"), output); + assertTrue(output.contains("noise from the passing test"), output); + assertTrue(output.contains("OK!"), output); + } + + @Test + public void quietFlagIsParsed() { + assertTrue(new RunArgs("-testQuiet").isTestQuiet()); + assertTrue(new RunArgs("-tq").isTestQuiet()); + assertFalse(new RunArgs("-runtests").isTestQuiet()); + } + + private String runTestsWith(boolean testQuiet) { + RunArgs runArgs = new RunArgs(); + WurstGui gui = new WurstGuiCliImpl(); + WurstCompilerJassImpl compiler = new WurstCompilerJassImpl(null, gui, null, runArgs); + WurstModel model = parseFiles(null, + Collections.singletonList(new CU("test", Utils.join(PROGRAM, "\n") + "\n")), false, compiler); + compiler.checkProg(model); + if (!gui.getErrorList().isEmpty()) { + throw gui.getErrorList().get(0); + } + ImProg imProg = compiler.translateProgToIm(model); + + StringBuilder output = new StringBuilder(); + RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), 20, Optional.empty(), false, testQuiet) { + @Override + protected void print(String message) { + output.append(message); + } + }; + runTests.runTests(compiler.getImTranslator(), imProg, Optional.empty(), Optional.empty()); + return output.toString(); + } +}