Add -testQuiet to report only failing tests

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java

@@ -160,7 +160,7 @@ private void runTests(ImTranslator translator, WurstCompilerJassImpl compiler, i
         if (!runArgs.isCompactOutput()) {
             System.out.println("Running tests");
         }
-        RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput()) {
+        RunTests runTests = new RunTests(Optional.empty(), 0, 0, Optional.empty(), testTimeout, testFilter, runArgs.isCompactOutput(), runArgs.isTestQuiet()) {
             @Override
             protected void print(String message) {
                 out.print(message);

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java

@@ -40,7 +40,6 @@ public class ProgramStateIO extends ProgramState {
     private final Map<String, Set<String>> createdObjectDefinitionIds = Maps.newLinkedHashMap();
     private int id = 0;
     private final Map<String, ObjMod.Obj> objDefinitions = Maps.newLinkedHashMap();
-    private PrintStream outStream = System.err;
     private @Nullable WTS trigStrings = null;
     private final Optional<File> mapFile;
 
@@ -946,13 +945,4 @@ private Optional<File> getObjectEditingOutputFolder() {
         return folder;
     }
 
-    @Override
-    public PrintStream getOutStream() {
-        return outStream;
-    }
-
-    @Override
-    public void setOutStream(PrintStream os) {
-        outStream = os;
-    }
 }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/ReflectionNativeProvider.java

@@ -10,10 +10,14 @@
 import java.io.PrintStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 public class ReflectionNativeProvider implements NativesProvider {
     private final HashMap<String, NativeJassFunction> methodMap = new HashMap<>();
+    /** Providers that print to the interpreter output, so that redirecting it reaches them. */
+    private final List<OutputProvider> outputProviders = new ArrayList<>();
 
     public ReflectionNativeProvider(AbstractInterpreter interpreter) {
         addProvider(new AbilityProvider(interpreter));
@@ -51,6 +55,9 @@ public NativeJassFunction getFunctionPair(String funcName) {
     }
 
     private void addProvider(Provider provider) {
+        if (provider instanceof OutputProvider) {
+            outputProviders.add((OutputProvider) provider);
+        }
         for (Method method : provider.getClass().getMethods()) {
             Implements annotation = method.getAnnotation(Implements.class);
             if (annotation != null) {
@@ -103,6 +110,8 @@ public ILconst invoke(String funcname, ILconst[] args) throws NoSuchNativeExcept
 
     @Override
     public void setOutStream(PrintStream outStream) {
-
+        for (OutputProvider provider : outputProviders) {
+            provider.setOutStream(outStream);
+        }
     }
 }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/jassinterpreter/providers/OutputProvider.java

@@ -18,6 +18,10 @@ public OutputProvider(AbstractInterpreter interpreter) {
         super(interpreter);
     }
 
+    public void setOutStream(PrintStream outStream) {
+        this.outStream = outStream;
+    }
+
     public void DisplayTextToForce(IlConstHandle force, ILconstString msg) {
         outStream.println(msg.getVal());
     }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/languageserver/requests/RunTests.java

@@ -29,6 +29,7 @@
 import org.eclipse.lsp4j.MessageType;
 
 import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.*;
@@ -47,6 +48,15 @@ public class RunTests extends UserRequest<Object> {
     private final int timeoutSeconds;
     private final Optional<String> testFilter;
     private final boolean compactOutput;
+    private final boolean testQuiet;
+
+    /**
+     * In quiet mode the output of the test currently running is collected here instead of
+     * being printed directly. It is flushed when the test fails and discarded when it passes.
+     * Synchronized, because a test that timed out keeps running on the test executor thread
+     * while this thread already flushes the buffer.
+     */
+    private final StringBuffer pendingOutput = new StringBuffer();
 
     private final List<ImFunction> successTests = Lists.newArrayList();
     private final List<TestFailure> failTests = Lists.newArrayList();
@@ -102,6 +112,11 @@ public RunTests(Optional<String> filename, int line, int column, Optional<String
     }
 
     public RunTests(Optional<String> filename, int line, int column, Optional<String> testName, int timeoutSeconds, Optional<String> testFilter, boolean compactOutput) {
+        this(filename, line, column, testName, timeoutSeconds, testFilter, compactOutput, false);
+    }
+
+    public RunTests(Optional<String> filename, int line, int column, Optional<String> testName, int timeoutSeconds, Optional<String> testFilter, boolean compactOutput, boolean testQuiet) {
+        this.testQuiet = testQuiet;
         this.filename = filename.map(WFile::create);
         this.line = line;
         this.column = column;
@@ -177,12 +192,16 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional<Func
         cfr.run();
 
         if (gui.getErrorCount() > 0) {
+            // compiletime output is relevant context for these errors, so keep it
+            flushPendingOutput();
             for (CompileError compileError : gui.getErrorList()) {
                 println(compactOutput ? compileError.toCompactString() : compileError.toString());
             }
             println("There were some problem while running compiletime expressions and functions.");
             return new TestResult(0, 1);
         }
+        // output of successful compiletime functions does not belong to any test
+        discardPendingOutput();
 
         WLogger.info("Ran compiletime functions");
 
@@ -232,7 +251,7 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional<Func
                     String file = new File(source.getFile()).toPath().normalize().toString();
                     String message = "Running " + file + ":" + source.getLine() + " - " + f.getName() + "..";
                     if (!compactOutput) {
-                        println(message);
+                        testPrintln(message);
                     }
                     WLogger.info(message);
 
@@ -257,6 +276,7 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional<Func
                         }
 
                         if (gui.getErrorCount() > 0) {
+                            flushPendingOutput();
                             StringBuilder sb = new StringBuilder();
                             int appendedErrors = 0;
                             for (CompileError error : gui.getErrorList()) {
@@ -285,34 +305,40 @@ public TestResult runTests(ImTranslator translator, ImProg imProg, Optional<Func
                         } else {
                             successTests.add(f);
                             if (!compactOutput) {
-                                println("\tOK!");
+                                testPrintln("\tOK!");
                             }
+                            discardPendingOutput();
                         }
                     } catch (TestSuccessException e) {
                         successTests.add(f);
                         if (!compactOutput) {
-                            println("\tOK!");
+                            testPrintln("\tOK!");
                         }
+                        discardPendingOutput();
                     } catch (TestFailException e) {
+                        flushPendingOutput();
                         TestFailure failure = new TestFailure(f, interpreter.getStackFrames(), e.getMessage());
                         failTests.add(failure);
                         if (!compactOutput) {
                             println("\tFAILED assertion:");
                             println("\t" + failure.getMessageWithStackFrame());
                         }
                     } catch (TestTimeOutException e) {
+                        flushPendingOutput();
                         failTests.add(new TestFailure(f, interpreter.getStackFrames(), e.getMessage()));
                         if (!compactOutput) {
                             println("\tFAILED - TIMEOUT (This test did not complete in " + timeoutSeconds + " seconds, it might contain an endless loop)");
                             println(interpreter.getStackFrames().toString());
                         }
                     } catch (InterpreterException e) {
+                        flushPendingOutput();
                         TestFailure failure = new TestFailure(f, interpreter.getStackFrames(), e.getMessage());
                         failTests.add(failure);
                         if (!compactOutput) {
                             println("\t" + failure.getMessageWithStackFrame());
                         }
                     } catch (Throwable e) {
+                        flushPendingOutput();
                         failTests.add(new TestFailure(f, interpreter.getStackFrames(), e.toString()));
                         if (!compactOutput) {
                             println("\tFAILED with exception: " + e.getClass() + " " + e.getLocalizedMessage());
@@ -362,20 +388,22 @@ private void redirectInterpreterOutput(ProgramState globalState) {
             @Override
             public void write(int b) throws IOException {
                 if (!compactOutput && b > 0) {
-                    println("" + (char) b);
+                    testPrintln("" + (char) b);
                 }
             }
 
             @Override
             public void write(byte[] b, int off, int len) throws IOException {
                 if (!compactOutput) {
-                    println(new String(b, off, len));
+                    testPrintln(new String(b, off, len, StandardCharsets.UTF_8));
                 }
             }
 
 
         };
-        globalState.setOutStream(new PrintStream(os));
+        // autoflush, so that output of a test is delivered while that test is still running
+        // and not left in the buffer of the PrintStream
+        globalState.setOutStream(new PrintStream(os, true, StandardCharsets.UTF_8));
     }
 
     private String qualifiedTestName(ImFunction f) {
@@ -388,6 +416,36 @@ protected void println(String message) {
         print(System.lineSeparator());
     }
 
+    /**
+     * Prints output belonging to a single test. In quiet mode the output is buffered, so that
+     * it can be dropped if the test passes, or printed as usual if the test fails.
+     */
+    private void testPrintln(String message) {
+        if (testQuiet) {
+            // appended in one call, so that a concurrent flush cannot split the line
+            pendingOutput.append(message + System.lineSeparator());
+        } else {
+            println(message);
+        }
+    }
+
+    /** Prints the buffered output of the current test, used when the test turned out to fail. */
+    private void flushPendingOutput() {
+        String pending;
+        synchronized (pendingOutput) {
+            pending = pendingOutput.toString();
+            pendingOutput.setLength(0);
+        }
+        if (!pending.isEmpty()) {
+            print(pending);
+        }
+    }
+
+    /** Drops the buffered output of the current test, used when the test passed. */
+    private void discardPendingOutput() {
+        pendingOutput.setLength(0);
+    }
+
     protected void print(String message) {
         System.err.print(message);
     }

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/RunArgs.java

@@ -58,6 +58,7 @@ public class RunArgs {
     private final RunOption optionHotStartmap;
     private final RunOption optionHotReload;
     private final RunOption optionTestTimeout;
+    private final RunOption optionTestQuiet;
     private final RunOption optionDevBuild;
     private int functionSplitLimit = 10000;
 
@@ -82,17 +83,24 @@ public RunArgs with(String... additionalArgs) {
     private static class RunOption {
 
         final String name;
+        final @Nullable String alias;
         final String descr;
         final @Nullable Consumer<String> argHandler;
         boolean isSet;
         RunOption(String name, String descr) {
+            this(name, null, descr);
+        }
+
+        RunOption(String name, @Nullable String alias, String descr) {
             this.name = name;
+            this.alias = alias;
             this.descr = descr;
             this.argHandler = null;
         }
 
         RunOption(String name, String descr, Consumer<String> argHandler2) {
             this.name = name;
+            this.alias = null;
             this.descr = descr;
             this.argHandler = argHandler2;
         }
@@ -108,6 +116,8 @@ public RunArgs(String... args) {
         // interpreter
         optionRuntests = addOption("runtests", "Run all test functions found in the scripts.");
         optionTestTimeout = addOptionWithArg("testTimeout", "Timeout in seconds after which tests will be cancelled and considered failed, if they did not yet succeed.", arg -> testTimeout = Integer.parseInt(arg));
+        optionTestQuiet = addOption("testQuiet", "tq", "Only report failing tests: output of passing tests is suppressed, "
+                + "while failing tests still print their full output, assertion message and stack trace.");
         addOptionWithArg("testFilter", "Only run tests whose qualified name (Package.function) contains this string (case-insensitive).", arg -> testFilter = arg);
         optionRunCompileTimeFunctions = addOption("runcompiletimefunctions", "Run all compiletime functions found in the scripts.");
         optionInjectCompiletimeObjects = addOption("injectobjects", "Injects the objects generated by compiletime functions into the map.");
@@ -167,7 +177,7 @@ public RunArgs(String... args) {
             String a = args[i];
             if (a.startsWith("-")) {
                 for (RunOption o : options) {
-                    if (("-" + o.name).equals(a)) {
+                    if (("-" + o.name).equals(a) || (o.alias != null && ("-" + o.alias).equals(a))) {
                         Consumer<String> argHandler = o.argHandler;
                         if (argHandler != null) {
                             i++;
@@ -208,11 +218,16 @@ public RunArgs(String... args) {
     }
 
     private boolean isDoubleArg(String arg, RunOption option) {
-        return (arg.contains(" ") && ("-" + option.name).equals(arg.substring(0, arg.indexOf(" "))));
+        if (!arg.contains(" ")) {
+            return false;
+        }
+        String name = arg.substring(0, arg.indexOf(" "));
+        return ("-" + option.name).equals(name) || (option.alias != null && ("-" + option.alias).equals(name));
     }
 
     private boolean isEqualsArg(String arg, RunOption option) {
-        return arg.startsWith("-" + option.name + "=");
+        return arg.startsWith("-" + option.name + "=")
+            || (option.alias != null && arg.startsWith("-" + option.alias + "="));
     }
 
     private RunOption addOption(String name, String descr) {
@@ -221,6 +236,12 @@ private RunOption addOption(String name, String descr) {
         return opt;
     }
 
+    private RunOption addOption(String name, String alias, String descr) {
+        RunOption opt = new RunOption(name, alias, descr);
+        options.add(opt);
+        return opt;
+    }
+
     private RunOption addOptionWithArg(String name, String descr, Consumer<String> argHandler) {
         RunOption opt = new RunOption(name, descr, argHandler);
         options.add(opt);
@@ -241,7 +262,7 @@ public void printHelpAndExit() {
         System.out.println("Options:");
         System.out.println();
         for (RunOption opt : options) {
-            System.out.println("-" + opt.name);
+            System.out.println(opt.alias == null ? "-" + opt.name : "-" + opt.name + ", -" + opt.alias);
             System.out.println("	" + opt.descr);
             System.out.println();
         }
@@ -403,6 +424,10 @@ public Optional<String> getTestFilter() {
         return Optional.ofNullable(testFilter);
     }
 
+    public boolean isTestQuiet() {
+        return optionTestQuiet.isSet;
+    }
+
     public boolean isMeasureTimes() {
         return optionMeasureTimes.isSet;
     }