diff --git a/src/support/file.cpp b/src/support/file.cpp index 918ab29b2f6..830b34f98c4 100644 --- a/src/support/file.cpp +++ b/src/support/file.cpp @@ -126,6 +126,12 @@ wasm::Output::Output(const std::string& filename, Flags::BinaryOption binary) return buffer; }()) {} +void wasm::write_file(const std::string& filename, + const std::vector& bytes) { + wasm::Output out(filename, Flags::Binary); + out.write(bytes.data(), bytes.size()); +} + void wasm::copy_file(std::string input, std::string output) { std::ifstream src(wasm::Path::to_path(input), std::ios::binary); std::ofstream dst(wasm::Path::to_path(output), std::ios::binary); diff --git a/src/support/file.h b/src/support/file.h index 0726417285f..584f63cf125 100644 --- a/src/support/file.h +++ b/src/support/file.h @@ -69,6 +69,9 @@ class Output { std::ostream out; }; +// Write the bytes to the given file. +void write_file(const std::string& filename, const std::vector& bytes); + // Copies a file to another file void copy_file(std::string input, std::string output); diff --git a/src/tools/tool-options.h b/src/tools/tool-options.h index f2f062da303..7f2f57bac4b 100644 --- a/src/tools/tool-options.h +++ b/src/tools/tool-options.h @@ -275,6 +275,12 @@ struct ToolOptions : public Options { writer.write(wasm, filename); } + void + write(ModuleWriter& writer, Module& wasm, std::vector& output) const { + writer.setEmitModuleName(emitModuleNames); + writer.write(wasm, output); + } + virtual void addPassArg(const std::string& key, const std::string& value) { passOptions.arguments[key] = value; } diff --git a/src/tools/wasm-reduce/wasm-reduce.cpp b/src/tools/wasm-reduce/wasm-reduce.cpp index 2d916ceaaed..422686b8148 100644 --- a/src/tools/wasm-reduce/wasm-reduce.cpp +++ b/src/tools/wasm-reduce/wasm-reduce.cpp @@ -232,16 +232,18 @@ static std::unordered_set functionsWeTriedToRemove; // overlap. static size_t workingFileIndex = 0; -// Runs the external `wasm-opt` binary with `args` on `inPath` and writes the -// result to `outPath`. -static bool runWasmOpt(const std::string& inPath, - const std::string& outPath, +/// Runs the external `wasm-opt` binary with `args` on `inBytes` by writing to +// `testPath` and reading the result back into `outBytes`. +static bool runWasmOpt(const std::string& testPath, + const std::vector& inBytes, const std::string& args, + std::vector& outBytes, ProgramResult& result, bool verbose = false, std::string* outCmd = nullptr) { - std::string cmd = Path::getBinaryenBinaryTool("wasm-opt") + " " + inPath + - " -o " + outPath + " " + args; + write_file(testPath, inBytes); + std::string cmd = Path::getBinaryenBinaryTool("wasm-opt") + " " + testPath + + " -o " + testPath + " " + args; if (outCmd) { *outCmd = cmd; } @@ -249,7 +251,11 @@ static bool runWasmOpt(const std::string& inPath, std::cerr << "| trying pass command: " << cmd << "\n"; } result.getFromExecution(cmd); - return !result.failed(); + if (result.failed()) { + return false; + } + outBytes = read_file>(testPath, Flags::Binary); + return true; } struct Reducer @@ -258,8 +264,6 @@ struct Reducer bool binary, deNan, verbose, debugInfo; ToolOptions& toolOptions; - // test is the file we write to that the command will operate on - // working is the current temporary state, the reduction so far Reducer(std::string command, std::string test, std::string working, @@ -319,7 +323,8 @@ struct Reducer "--strip", "--remove-unused-types --closed-world", "--vacuum"}; - auto initialSize = file_size(working); + auto workingBytes = read_file>(working, Flags::Binary); + auto initialSize = workingBytes.size(); auto oldSize = initialSize; bool more = true; while (more) { @@ -333,16 +338,26 @@ struct Reducer args += " -S "; } ProgramResult result; + std::vector candidateBytes; std::string currCommand; - if (runWasmOpt(working, test, args, result, verbose, &currCommand)) { - auto newSize = file_size(test); + if (runWasmOpt(test, + workingBytes, + args, + candidateBytes, + result, + verbose, + &currCommand)) { + auto newSize = candidateBytes.size(); if (newSize < oldSize) { // the pass didn't fail, and the size looks smaller, so promising // see if it is still has the property we are preserving - if (ProgramResult(command) == expected) { + write_file(test, candidateBytes); + ProgramResult out(command); + if (out == expected) { std::cerr << "| command \"" << currCommand << "\" succeeded, reduced size to " << newSize << '\n'; - saveWorking(); + saveWorking(candidateBytes); + workingBytes = candidateBytes; more = true; oldSize = newSize; } @@ -356,16 +371,6 @@ struct Reducer return {initialSize, oldSize}; } - // Apply the test file to the working file, after we saw that it successfully - // reduced the testcase. - void saveWorking() { - copy_file(test, working); - - if (saveAllWorkingFiles) { - copy_file(working, working + '.' + std::to_string(workingFileIndex++)); - } - } - // does one pass of slow and destructive reduction. returns whether it // succeeded or not // the criterion here is a logical change in the program. this may actually @@ -402,7 +407,8 @@ struct Reducer ModuleReader reader; try { - reader.read(working, *module); + reader.readData(read_file>(working, Flags::Binary), + *module); } catch (ParseException& p) { p.dump(std::cerr); std::cerr << '\n'; @@ -423,6 +429,7 @@ struct Reducer std::unique_ptr builder; Index funcsSeen; uint64_t factor; + std::vector lastTestBytes; // write the module and see if the command still fails on it as expected bool writeAndTestReduction() { @@ -435,7 +442,9 @@ struct Reducer ModuleWriter writer(toolOptions.passOptions); writer.setBinary(binary); writer.setDebugInfo(debugInfo); - toolOptions.write(writer, *getModule(), test); + lastTestBytes.clear(); + toolOptions.write(writer, *getModule(), lastTestBytes); + write_file(test, lastTestBytes); // note that it is ok for the destructively-reduced module to be bigger // than the previous - each destructive reduction removes logical code, // and so is strictly better, even if the wasm binary format happens to @@ -497,9 +506,16 @@ struct Reducer return true; } + void saveWorking(const std::vector& bytes) { + write_file(working, bytes); + if (saveAllWorkingFiles) { + copy_file(working, working + '.' + std::to_string(workingFileIndex++)); + } + } + void noteReduction(size_t amount = 1) { reduced += amount; - saveWorking(); + saveWorking(lastTestBytes); } // tests a reduction on an arbitrary child @@ -1378,17 +1394,18 @@ static void checkDifferentBehaviorOnDifferentInputs(const std::string& test, "inputs (this " "verifies that the test file is used by the command)\n"; // Try it on an invalid input. - { - std::ofstream dst(test, std::ios::binary); - dst << "waka waka\n"; - } + std::string nonsenseStr = "waka waka\n"; + std::vector nonsense(nonsenseStr.begin(), nonsenseStr.end()); + write_file(test, nonsense); ProgramResult resultOnInvalid(command); if (resultOnInvalid == expected) { // Try it on a valid input. Module emptyModule; ModuleWriter writer(options.passOptions); writer.setBinary(true); - options.write(writer, emptyModule, test); + std::vector emptyBytes; + options.write(writer, emptyModule, emptyBytes); + write_file(test, emptyBytes); ProgramResult resultOnValid(command); if (resultOnValid == expected) { Fatal() @@ -1412,9 +1429,15 @@ static void checkCanonicalizedBinary(const std::string& working, args += " -S "; } ProgramResult readWrite; - if (!runWasmOpt(working, test, args, readWrite)) { + std::vector canonicalBytes; + if (!runWasmOpt(test, + read_file>(working, Flags::Binary), + args, + canonicalBytes, + readWrite)) { stopIfNotForced("failed to read and write the binary", readWrite, force); } else { + write_file(test, canonicalBytes); ProgramResult result(command); if (result != expected) { stopIfNotForced("running command on the canonicalized module should " @@ -1437,7 +1460,8 @@ static void reduceModule(const std::string& command, checkDifferentBehaviorOnDifferentInputs(test, command, force, options); checkCanonicalizedBinary(working, test, command, binary, force); - auto workingSize = file_size(working); + auto workingSize = + read_file>(working, Flags::Binary).size(); std::cerr << "|input size: " << workingSize << "\n"; std::cerr << "|starting reduction!\n"; @@ -1535,11 +1559,13 @@ static void reduceModule(const std::string& command, factor = std::max(uint64_t(1), factor / 4); } - std::cerr << "| destructive reduction led to size: " << file_size(working) + std::cerr << "| destructive reduction led to size: " + << read_file>(working, Flags::Binary).size() << '\n'; } - std::cerr << "|finished, final size: " << file_size(working) << "\n"; - copy_file(working, test); // just to avoid confusion + auto finalBytes = read_file>(working, Flags::Binary); + std::cerr << "|finished, final size: " << finalBytes.size() << "\n"; + write_file(test, finalBytes); // just to avoid confusion } // diff --git a/src/wasm-io.h b/src/wasm-io.h index 01d9462a5ca..30cda0f2da0 100644 --- a/src/wasm-io.h +++ b/src/wasm-io.h @@ -27,6 +27,8 @@ namespace wasm { +class BufferWithRandomAccess; + class ModuleIOBase { protected: bool debugInfo; @@ -63,6 +65,10 @@ class ModuleReader : public ModuleIOBase { // empty, read from stdin. void read(std::string filename, Module& wasm, std::string sourceMapFilename = ""); + // read text or binary from an in-memory buffer + void readData(const std::vector& input, + Module& wasm, + std::string sourceMapFilename = ""); // check whether a file is a wasm binary bool isBinaryFile(std::string filename); @@ -79,7 +85,7 @@ class ModuleReader : public ModuleIOBase { void readStdin(Module& wasm, std::string sourceMapFilename); - void readBinaryData(std::vector& input, + void readBinaryData(const std::vector& input, Module& wasm, std::string sourceMapFilename); }; @@ -117,6 +123,7 @@ class ModuleWriter : public ModuleIOBase { void writeText(Module& wasm, Output& output); void writeText(Module& wasm, std::string filename); // write binary + void writeBinary(Module& wasm, BufferWithRandomAccess& buffer); void writeBinary(Module& wasm, Output& output); void writeBinary(Module& wasm, std::string filename); // write text or binary, defaulting to binary unless setBinary(false), @@ -124,6 +131,7 @@ class ModuleWriter : public ModuleIOBase { // to stdout). void write(Module& wasm, Output& output); void write(Module& wasm, std::string filename); + void write(Module& wasm, std::vector& output); }; } // namespace wasm diff --git a/src/wasm/wasm-io.cpp b/src/wasm/wasm-io.cpp index 77fc57e57b7..15237f08c45 100644 --- a/src/wasm/wasm-io.cpp +++ b/src/wasm/wasm-io.cpp @@ -35,7 +35,7 @@ namespace wasm { #define DEBUG_TYPE "writer" static void readTextData(std::optional filename, - std::string& input, + std::string_view input, Module& wasm, IRProfile profile) { if (auto parsed = WATParser::parseModule(wasm, input, filename); @@ -50,7 +50,7 @@ void ModuleReader::readText(std::string filename, Module& wasm) { readTextData(filename, input, wasm, profile); } -void ModuleReader::readBinaryData(std::vector& input, +void ModuleReader::readBinaryData(const std::vector& input, Module& wasm, std::string sourceMapFilename) { std::vector sourceMapBuffer; @@ -109,21 +109,25 @@ void ModuleReader::read(std::string filename, } } -// TODO: reading into a vector then copying into a string is unnecessarily -// inefficient. It would be better to read just once into a stringstream. -void ModuleReader::readStdin(Module& wasm, std::string sourceMapFilename) { - std::vector input = read_stdin(); +void ModuleReader::readData(const std::vector& input, + Module& wasm, + std::string sourceMapFilename) { if (input.size() >= 4 && input[0] == '\0' && input[1] == 'a' && input[2] == 's' && input[3] == 'm') { readBinaryData(input, wasm, sourceMapFilename); } else { - std::ostringstream s; - s.write(input.data(), input.size()); - std::string input_str = s.str(); - readTextData(std::nullopt, input_str, wasm, profile); + readTextData(std::nullopt, + std::string_view(input.data(), input.size()), + wasm, + profile); } } +void ModuleReader::readStdin(Module& wasm, std::string sourceMapFilename) { + std::vector input = read_stdin(); + readData(input, wasm, sourceMapFilename); +} + #undef DEBUG_TYPE #define DEBUG_TYPE "writer" @@ -137,8 +141,7 @@ void ModuleWriter::writeText(Module& wasm, std::string filename) { writeText(wasm, output); } -void ModuleWriter::writeBinary(Module& wasm, Output& output) { - BufferWithRandomAccess buffer; +void ModuleWriter::writeBinary(Module& wasm, BufferWithRandomAccess& buffer) { WasmBinaryWriter writer(&wasm, buffer, options); // if debug info is used, then we want to emit the names section writer.setNamesSection(debugInfo); @@ -159,12 +162,17 @@ void ModuleWriter::writeBinary(Module& wasm, Output& output) { writer.setSymbolMap(symbolMap); } writer.write(); - buffer.writeTo(output); if (sourceMapStream) { sourceMapStream->close(); } } +void ModuleWriter::writeBinary(Module& wasm, Output& output) { + BufferWithRandomAccess buffer; + writeBinary(wasm, buffer); + buffer.writeTo(output); +} + void ModuleWriter::writeBinary(Module& wasm, std::string filename) { BYN_TRACE("writing binary to " << filename << "\n"); Output output(filename, Flags::Binary); @@ -187,4 +195,17 @@ void ModuleWriter::write(Module& wasm, std::string filename) { } } +void ModuleWriter::write(Module& wasm, std::vector& output) { + if (binary) { + BufferWithRandomAccess buffer; + writeBinary(wasm, buffer); + output.assign(buffer.begin(), buffer.end()); + } else { + std::ostringstream s; + s << wasm; + std::string str = s.str(); + output.assign(str.begin(), str.end()); + } +} + } // namespace wasm