From 4078769fcaa769314ae3f81179f9c3e1562787cf Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 18 Sep 2026 10:32:53 -0700 Subject: [PATCH 1/2] [NFC][wasm-reduce] Use byte vectors for I/O This refactoring prepares for a follow-on PR that will abstract how the module bytes are read from and written to the test and working files to support reducing Wasm modules embedded in JS files. --- src/tools/tool-options.h | 6 ++ src/tools/wasm-reduce/wasm-reduce.cpp | 106 +++++++++++++++++--------- src/wasm-io.h | 10 ++- src/wasm/wasm-io.cpp | 47 ++++++++---- 4 files changed, 118 insertions(+), 51 deletions(-) diff --git a/src/tools/tool-options.h b/src/tools/tool-options.h index f2f062da303..7f2f57bac4b 100644 --- a/src/tools/tool-options.h +++ b/src/tools/tool-options.h @@ -275,6 +275,12 @@ struct ToolOptions : public Options { writer.write(wasm, filename); } + void + write(ModuleWriter& writer, Module& wasm, std::vector& output) const { + writer.setEmitModuleName(emitModuleNames); + writer.write(wasm, output); + } + virtual void addPassArg(const std::string& key, const std::string& value) { passOptions.arguments[key] = value; } diff --git a/src/tools/wasm-reduce/wasm-reduce.cpp b/src/tools/wasm-reduce/wasm-reduce.cpp index 2d916ceaaed..d67566810ff 100644 --- a/src/tools/wasm-reduce/wasm-reduce.cpp +++ b/src/tools/wasm-reduce/wasm-reduce.cpp @@ -232,16 +232,24 @@ static std::unordered_set functionsWeTriedToRemove; // overlap. static size_t workingFileIndex = 0; -// Runs the external `wasm-opt` binary with `args` on `inPath` and writes the -// result to `outPath`. -static bool runWasmOpt(const std::string& inPath, - const std::string& outPath, +static void write_file(const std::string& filename, + const std::vector& bytes) { + Output out(filename, Flags::Binary); + out.write(bytes.data(), bytes.size()); +} + +/// Runs the external `wasm-opt` binary with `args` on `inBytes` by writing to +// `testPath` and reading the result back into `outBytes`. +static bool runWasmOpt(const std::string& testPath, + const std::vector& inBytes, const std::string& args, + std::vector& outBytes, ProgramResult& result, bool verbose = false, std::string* outCmd = nullptr) { - std::string cmd = Path::getBinaryenBinaryTool("wasm-opt") + " " + inPath + - " -o " + outPath + " " + args; + write_file(testPath, inBytes); + std::string cmd = Path::getBinaryenBinaryTool("wasm-opt") + " " + testPath + + " -o " + testPath + " " + args; if (outCmd) { *outCmd = cmd; } @@ -249,7 +257,11 @@ static bool runWasmOpt(const std::string& inPath, std::cerr << "| trying pass command: " << cmd << "\n"; } result.getFromExecution(cmd); - return !result.failed(); + if (result.failed()) { + return false; + } + outBytes = read_file>(testPath, Flags::Binary); + return true; } struct Reducer @@ -258,8 +270,6 @@ struct Reducer bool binary, deNan, verbose, debugInfo; ToolOptions& toolOptions; - // test is the file we write to that the command will operate on - // working is the current temporary state, the reduction so far Reducer(std::string command, std::string test, std::string working, @@ -319,7 +329,8 @@ struct Reducer "--strip", "--remove-unused-types --closed-world", "--vacuum"}; - auto initialSize = file_size(working); + auto workingBytes = read_file>(working, Flags::Binary); + auto initialSize = workingBytes.size(); auto oldSize = initialSize; bool more = true; while (more) { @@ -333,16 +344,26 @@ struct Reducer args += " -S "; } ProgramResult result; + std::vector candidateBytes; std::string currCommand; - if (runWasmOpt(working, test, args, result, verbose, &currCommand)) { - auto newSize = file_size(test); + if (runWasmOpt(test, + workingBytes, + args, + candidateBytes, + result, + verbose, + &currCommand)) { + auto newSize = candidateBytes.size(); if (newSize < oldSize) { // the pass didn't fail, and the size looks smaller, so promising // see if it is still has the property we are preserving - if (ProgramResult(command) == expected) { + write_file(test, candidateBytes); + ProgramResult out(command); + if (out == expected) { std::cerr << "| command \"" << currCommand << "\" succeeded, reduced size to " << newSize << '\n'; - saveWorking(); + saveWorking(candidateBytes); + workingBytes = candidateBytes; more = true; oldSize = newSize; } @@ -356,16 +377,6 @@ struct Reducer return {initialSize, oldSize}; } - // Apply the test file to the working file, after we saw that it successfully - // reduced the testcase. - void saveWorking() { - copy_file(test, working); - - if (saveAllWorkingFiles) { - copy_file(working, working + '.' + std::to_string(workingFileIndex++)); - } - } - // does one pass of slow and destructive reduction. returns whether it // succeeded or not // the criterion here is a logical change in the program. this may actually @@ -402,7 +413,8 @@ struct Reducer ModuleReader reader; try { - reader.read(working, *module); + reader.readData(read_file>(working, Flags::Binary), + *module); } catch (ParseException& p) { p.dump(std::cerr); std::cerr << '\n'; @@ -423,6 +435,7 @@ struct Reducer std::unique_ptr builder; Index funcsSeen; uint64_t factor; + std::vector lastTestBytes; // write the module and see if the command still fails on it as expected bool writeAndTestReduction() { @@ -435,7 +448,9 @@ struct Reducer ModuleWriter writer(toolOptions.passOptions); writer.setBinary(binary); writer.setDebugInfo(debugInfo); - toolOptions.write(writer, *getModule(), test); + lastTestBytes.clear(); + toolOptions.write(writer, *getModule(), lastTestBytes); + write_file(test, lastTestBytes); // note that it is ok for the destructively-reduced module to be bigger // than the previous - each destructive reduction removes logical code, // and so is strictly better, even if the wasm binary format happens to @@ -497,9 +512,16 @@ struct Reducer return true; } + void saveWorking(const std::vector& bytes) { + write_file(working, bytes); + if (saveAllWorkingFiles) { + copy_file(working, working + '.' + std::to_string(workingFileIndex++)); + } + } + void noteReduction(size_t amount = 1) { reduced += amount; - saveWorking(); + saveWorking(lastTestBytes); } // tests a reduction on an arbitrary child @@ -1378,17 +1400,18 @@ static void checkDifferentBehaviorOnDifferentInputs(const std::string& test, "inputs (this " "verifies that the test file is used by the command)\n"; // Try it on an invalid input. - { - std::ofstream dst(test, std::ios::binary); - dst << "waka waka\n"; - } + std::string nonsenseStr = "waka waka\n"; + std::vector nonsense(nonsenseStr.begin(), nonsenseStr.end()); + write_file(test, nonsense); ProgramResult resultOnInvalid(command); if (resultOnInvalid == expected) { // Try it on a valid input. Module emptyModule; ModuleWriter writer(options.passOptions); writer.setBinary(true); - options.write(writer, emptyModule, test); + std::vector emptyBytes; + options.write(writer, emptyModule, emptyBytes); + write_file(test, emptyBytes); ProgramResult resultOnValid(command); if (resultOnValid == expected) { Fatal() @@ -1412,9 +1435,15 @@ static void checkCanonicalizedBinary(const std::string& working, args += " -S "; } ProgramResult readWrite; - if (!runWasmOpt(working, test, args, readWrite)) { + std::vector canonicalBytes; + if (!runWasmOpt(test, + read_file>(working, Flags::Binary), + args, + canonicalBytes, + readWrite)) { stopIfNotForced("failed to read and write the binary", readWrite, force); } else { + write_file(test, canonicalBytes); ProgramResult result(command); if (result != expected) { stopIfNotForced("running command on the canonicalized module should " @@ -1437,7 +1466,8 @@ static void reduceModule(const std::string& command, checkDifferentBehaviorOnDifferentInputs(test, command, force, options); checkCanonicalizedBinary(working, test, command, binary, force); - auto workingSize = file_size(working); + auto workingSize = + read_file>(working, Flags::Binary).size(); std::cerr << "|input size: " << workingSize << "\n"; std::cerr << "|starting reduction!\n"; @@ -1535,11 +1565,13 @@ static void reduceModule(const std::string& command, factor = std::max(uint64_t(1), factor / 4); } - std::cerr << "| destructive reduction led to size: " << file_size(working) + std::cerr << "| destructive reduction led to size: " + << read_file>(working, Flags::Binary).size() << '\n'; } - std::cerr << "|finished, final size: " << file_size(working) << "\n"; - copy_file(working, test); // just to avoid confusion + auto finalBytes = read_file>(working, Flags::Binary); + std::cerr << "|finished, final size: " << finalBytes.size() << "\n"; + write_file(test, finalBytes); // just to avoid confusion } // diff --git a/src/wasm-io.h b/src/wasm-io.h index 01d9462a5ca..30cda0f2da0 100644 --- a/src/wasm-io.h +++ b/src/wasm-io.h @@ -27,6 +27,8 @@ namespace wasm { +class BufferWithRandomAccess; + class ModuleIOBase { protected: bool debugInfo; @@ -63,6 +65,10 @@ class ModuleReader : public ModuleIOBase { // empty, read from stdin. void read(std::string filename, Module& wasm, std::string sourceMapFilename = ""); + // read text or binary from an in-memory buffer + void readData(const std::vector& input, + Module& wasm, + std::string sourceMapFilename = ""); // check whether a file is a wasm binary bool isBinaryFile(std::string filename); @@ -79,7 +85,7 @@ class ModuleReader : public ModuleIOBase { void readStdin(Module& wasm, std::string sourceMapFilename); - void readBinaryData(std::vector& input, + void readBinaryData(const std::vector& input, Module& wasm, std::string sourceMapFilename); }; @@ -117,6 +123,7 @@ class ModuleWriter : public ModuleIOBase { void writeText(Module& wasm, Output& output); void writeText(Module& wasm, std::string filename); // write binary + void writeBinary(Module& wasm, BufferWithRandomAccess& buffer); void writeBinary(Module& wasm, Output& output); void writeBinary(Module& wasm, std::string filename); // write text or binary, defaulting to binary unless setBinary(false), @@ -124,6 +131,7 @@ class ModuleWriter : public ModuleIOBase { // to stdout). void write(Module& wasm, Output& output); void write(Module& wasm, std::string filename); + void write(Module& wasm, std::vector& output); }; } // namespace wasm diff --git a/src/wasm/wasm-io.cpp b/src/wasm/wasm-io.cpp index 77fc57e57b7..15237f08c45 100644 --- a/src/wasm/wasm-io.cpp +++ b/src/wasm/wasm-io.cpp @@ -35,7 +35,7 @@ namespace wasm { #define DEBUG_TYPE "writer" static void readTextData(std::optional filename, - std::string& input, + std::string_view input, Module& wasm, IRProfile profile) { if (auto parsed = WATParser::parseModule(wasm, input, filename); @@ -50,7 +50,7 @@ void ModuleReader::readText(std::string filename, Module& wasm) { readTextData(filename, input, wasm, profile); } -void ModuleReader::readBinaryData(std::vector& input, +void ModuleReader::readBinaryData(const std::vector& input, Module& wasm, std::string sourceMapFilename) { std::vector sourceMapBuffer; @@ -109,21 +109,25 @@ void ModuleReader::read(std::string filename, } } -// TODO: reading into a vector then copying into a string is unnecessarily -// inefficient. It would be better to read just once into a stringstream. -void ModuleReader::readStdin(Module& wasm, std::string sourceMapFilename) { - std::vector input = read_stdin(); +void ModuleReader::readData(const std::vector& input, + Module& wasm, + std::string sourceMapFilename) { if (input.size() >= 4 && input[0] == '\0' && input[1] == 'a' && input[2] == 's' && input[3] == 'm') { readBinaryData(input, wasm, sourceMapFilename); } else { - std::ostringstream s; - s.write(input.data(), input.size()); - std::string input_str = s.str(); - readTextData(std::nullopt, input_str, wasm, profile); + readTextData(std::nullopt, + std::string_view(input.data(), input.size()), + wasm, + profile); } } +void ModuleReader::readStdin(Module& wasm, std::string sourceMapFilename) { + std::vector input = read_stdin(); + readData(input, wasm, sourceMapFilename); +} + #undef DEBUG_TYPE #define DEBUG_TYPE "writer" @@ -137,8 +141,7 @@ void ModuleWriter::writeText(Module& wasm, std::string filename) { writeText(wasm, output); } -void ModuleWriter::writeBinary(Module& wasm, Output& output) { - BufferWithRandomAccess buffer; +void ModuleWriter::writeBinary(Module& wasm, BufferWithRandomAccess& buffer) { WasmBinaryWriter writer(&wasm, buffer, options); // if debug info is used, then we want to emit the names section writer.setNamesSection(debugInfo); @@ -159,12 +162,17 @@ void ModuleWriter::writeBinary(Module& wasm, Output& output) { writer.setSymbolMap(symbolMap); } writer.write(); - buffer.writeTo(output); if (sourceMapStream) { sourceMapStream->close(); } } +void ModuleWriter::writeBinary(Module& wasm, Output& output) { + BufferWithRandomAccess buffer; + writeBinary(wasm, buffer); + buffer.writeTo(output); +} + void ModuleWriter::writeBinary(Module& wasm, std::string filename) { BYN_TRACE("writing binary to " << filename << "\n"); Output output(filename, Flags::Binary); @@ -187,4 +195,17 @@ void ModuleWriter::write(Module& wasm, std::string filename) { } } +void ModuleWriter::write(Module& wasm, std::vector& output) { + if (binary) { + BufferWithRandomAccess buffer; + writeBinary(wasm, buffer); + output.assign(buffer.begin(), buffer.end()); + } else { + std::ostringstream s; + s << wasm; + std::string str = s.str(); + output.assign(str.begin(), str.end()); + } +} + } // namespace wasm From f8a36d8151be81face0892fde136ad9c2f7345fc Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Fri, 18 Sep 2026 14:47:54 -0700 Subject: [PATCH 2/2] move write_file to file.h/cpp --- src/support/file.cpp | 6 ++++++ src/support/file.h | 3 +++ src/tools/wasm-reduce/wasm-reduce.cpp | 6 ------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/support/file.cpp b/src/support/file.cpp index 918ab29b2f6..830b34f98c4 100644 --- a/src/support/file.cpp +++ b/src/support/file.cpp @@ -126,6 +126,12 @@ wasm::Output::Output(const std::string& filename, Flags::BinaryOption binary) return buffer; }()) {} +void wasm::write_file(const std::string& filename, + const std::vector& bytes) { + wasm::Output out(filename, Flags::Binary); + out.write(bytes.data(), bytes.size()); +} + void wasm::copy_file(std::string input, std::string output) { std::ifstream src(wasm::Path::to_path(input), std::ios::binary); std::ofstream dst(wasm::Path::to_path(output), std::ios::binary); diff --git a/src/support/file.h b/src/support/file.h index 0726417285f..584f63cf125 100644 --- a/src/support/file.h +++ b/src/support/file.h @@ -69,6 +69,9 @@ class Output { std::ostream out; }; +// Write the bytes to the given file. +void write_file(const std::string& filename, const std::vector& bytes); + // Copies a file to another file void copy_file(std::string input, std::string output); diff --git a/src/tools/wasm-reduce/wasm-reduce.cpp b/src/tools/wasm-reduce/wasm-reduce.cpp index d67566810ff..422686b8148 100644 --- a/src/tools/wasm-reduce/wasm-reduce.cpp +++ b/src/tools/wasm-reduce/wasm-reduce.cpp @@ -232,12 +232,6 @@ static std::unordered_set functionsWeTriedToRemove; // overlap. static size_t workingFileIndex = 0; -static void write_file(const std::string& filename, - const std::vector& bytes) { - Output out(filename, Flags::Binary); - out.write(bytes.data(), bytes.size()); -} - /// Runs the external `wasm-opt` binary with `args` on `inBytes` by writing to // `testPath` and reading the result back into `outBytes`. static bool runWasmOpt(const std::string& testPath,