Add support for wasm32-wasip3 components

Makefile

@@ -11,7 +11,7 @@ LOCAL_TOOLS := componentized-build-hello
 
 include build/build.mk
 
-test: test-sync
+test: test-sync check-components validate-components
 test-integration: test-hello test-hello-tool
 
 .PHONY: test-sync ## Test syncing the build into a repository, and its drift check

build/build.mk

@@ -18,9 +18,22 @@ export WKG_CONFIG_FILE := $(abspath .config/wasm-pkg/config.toml)
 .DEFAULT_GOAL := all
 
 # the rust target components are built for, `wasm32-wasip3` builds components that run std threads on
-# component model cooperative threads, see build/components.mk
+# component model cooperative threads, see build/components.mk. A component built with cargo may name
+# its own in its Cargo.toml, `COMPONENT_TARGET` given on the command line overrides it, e.g.
+#
+#   [package.metadata.componentized]
+#   target = "wasm32-wasip3"
 COMPONENT_TARGET ?= wasm32-unknown-unknown
 
+# read once while parsing, defines COMPONENT_TARGET_<component> for those that name one
+$(foreach file,$(wildcard components/*/Cargo.toml),$(eval COMPONENT_TARGET_$(word 2,$(subst /, ,$(file))) := $(shell sed -n '/^\[package\.metadata\.componentized\]/,/^\[/s/^target *= *"\(.*\)"/\1/p' $(file))))
+
+# the rust target a component is built for, e.g. `$(call component_target,trace)`
+component_target = $(if $(filter command line,$(origin COMPONENT_TARGET)),$(COMPONENT_TARGET),$(or $(COMPONENT_TARGET_$(1)),$(COMPONENT_TARGET)))
+
+# the targets the components built with cargo are built for
+COMPONENT_TARGETS = $(sort $(foreach file,$(wildcard components/*/Cargo.toml),$(call component_target,$(word 2,$(subst /, ,$(file))))))
+
 # the cli built from this repository, e.g. `constants`, released with an archive for each platform by
 # the CI workflow. Empty for a repository without a cli.
 CLI_BIN ?=

build/components.mk

@@ -12,37 +12,38 @@ COMPONENT_PROFILE ?= $(if $(shell grep -s '^\[profile\.component\]' Cargo.toml),
 COMPONENT_DEBUG_PROFILE ?= $(if $(shell grep -s '^\[profile\.component-debug\]' Cargo.toml),component-debug,dev)
 
 # the core module cargo built for a component with a profile, e.g. `$(call cargo_wasm,gate,dev)` for
-# target/wasm32-unknown-unknown/debug/gate.wasm
-cargo_wasm = target/$(COMPONENT_TARGET)/$(if $(filter dev,$(2)),debug,$(2))/$(subst -,_,$(1)).wasm
+# target/wasm32-unknown-unknown/debug/gate.wasm, `$(call cargo_wasm,gate,release,.opt)` for the
+# module wasm-opt optimized
+cargo_wasm = target/$(call component_target,$(1))/$(if $(filter dev,$(2)),debug,$(2))/$(subst -,_,$(1))$(3).wasm
 
 # the files the components cargo builds need besides the sources, e.g. a component the library
 # embeds, `CARGO_COMPONENT_DEPS := crates/componentized-static-config/constants-config.wasm`
 CARGO_COMPONENT_DEPS ?=
 
-ifeq ($(COMPONENT_TARGET),wasm32-wasip3)
+ifneq ($(filter wasm32-wasip3,$(COMPONENT_TARGETS)),)
 # wasm32-wasip3 runs std threads on component model cooperative threads, which needs a wasi-libc built
-# for them from wasi-sdk 34+'s experimental sysroot, installed with the tools or from WASI_SDK_PATH, and
-# a rust toolchain with cooperative thread support, 1.100+. `make components
-# COMPONENT_TARGET=wasm32-unknown-unknown` builds without.
-WASI_SYSROOT_LIB := $(WASI_SDK_PATH)/share/wasi-sysroot/experimental-coop-threads/lib/wasm32-wasip3
+# for them from wasi-sdk 34+'s experimental sysroot, installed with the tools or from WASI_SDK_PATH, or
+# built from WASI_LIBC_VERSION, see build/tools.mk, and a rust toolchain with cooperative thread
+# support, 1.100+. `make components COMPONENT_TARGET=wasm32-unknown-unknown` builds without.
+WASI_SYSROOT_LIB := $(WASI_SYSROOT)/lib/wasm32-wasip3
 # linked with wasm-component-ld, like the target's default, against the coop threads wasi-libc in
 # place of the libc vendored with std. Linked by clang, a cdylib is a shared library, not a component.
+# It links a core module, like wasm32-unknown-unknown, wasm-opt removes the code the release build
+# doesn't reach, and with it the imports only that code uses, before it's made a component.
 export CARGO_TARGET_WASM32_WASIP3_LINKER := $(WASI_SDK_PATH)/bin/wasm-component-ld
-export CARGO_TARGET_WASM32_WASIP3_RUSTFLAGS := -Clink-self-contained=n -Clink-arg=-L$(WASI_SYSROOT_LIB) -Clink-arg=$(WASI_SYSROOT_LIB)/crt1-reactor.o -Clink-arg=--export=cabi_realloc
-# turn the core module cargo built into a component, wasm32-wasip3 builds a component
-componentize = cp $(1) $(2)
-else
-componentize = $(WASM_TOOLS) component new $(1) -o $(2)
+export CARGO_TARGET_WASM32_WASIP3_RUSTFLAGS := -Clink-self-contained=n -Clink-arg=-L$(WASI_SYSROOT_LIB) -Clink-arg=$(WASI_SYSROOT_LIB)/crt1-reactor.o -Clink-arg=--export=cabi_realloc -Clink-arg=--skip-wit-component
 endif
 
+# turn the core module cargo built into a component
+componentize = $(WASM_TOOLS) component new $(1) -o $(2)
+
 # the release build of a component is optimized again for size with wasm-opt when it is pinned in
-# tools/Cargo.toml, `COMPONENT_OPTIMIZE=` builds without. Only a core module is optimized,
-# wasm32-wasip3 builds a component.
-COMPONENT_OPTIMIZE ?= $(if $(filter wasm-opt,$(TOOLS)),$(filter-out wasm32-wasip3,$(COMPONENT_TARGET)))
+# tools/Cargo.toml, `COMPONENT_OPTIMIZE=` builds without.
+COMPONENT_OPTIMIZE ?= $(filter wasm-opt,$(TOOLS))
 
-# the wasm features rust enables for wasm32-unknown-unknown, `rustc --print cfg --target
-# wasm32-unknown-unknown`. The stripped modules don't say which features they use, wasm-opt is
-# limited to these so it doesn't introduce instructions rust wouldn't emit.
+# the wasm features rust enables for wasm32-unknown-unknown and wasm32-wasip3, `rustc --print cfg
+# --target wasm32-unknown-unknown`. The stripped modules don't say which features they use, wasm-opt
+# is limited to these so it doesn't introduce instructions rust wouldn't emit.
 WASM_OPT_FEATURES ?= --enable-bulk-memory --enable-multivalue --enable-mutable-globals --enable-nontrapping-float-to-int --enable-reference-types --enable-sign-ext
 
 .PHONY: clean-components ## Remove the built components
@@ -52,11 +53,31 @@ clean-components: clean-wit
 .PHONY: components ## Build the interface and each component into target/components, `make components/<name>` builds one
 components: ${COMPONENTS_DIR}/interface.wasm $(foreach component,$(COMPONENTS),${COMPONENTS_DIR}/$(component)/$(component).wasm ${COMPONENTS_DIR}/$(component)/$(component).debug.wasm)
 
+.PHONY: check-components ## Check the release and debug build of each component have the same WIT, `make check-components/<name>` checks one
+check-components: $(foreach component,$(COMPONENTS),check-components/$(component))
+
+.PHONY: validate-components ## Validate the release and debug build of each component, `make validate-components/<name>` checks one
+validate-components: $(foreach component,$(COMPONENTS),validate-components/$(component))
+
 define BUILD_COMPONENT
 
 .PHONY: components/$1
 components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm
 
+# the same imports and exports, e.g. a debug build that isn't optimized may keep imports the release
+# build doesn't use, the order of the WIT may differ, see build/scripts/wit-canonical.sh
+.PHONY: check-components/$1
+check-components/$1: ${COMPONENTS_DIR}/$1/$1.wasm ${COMPONENTS_DIR}/$1/$1.debug.wasm build/scripts/wit-canonical.sh | $(call tool,wasm-tools)
+	@diff -u --label $1.wasm --label $1.debug.wasm \
+		<(WASM_TOOLS=$(WASM_TOOLS) build/scripts/wit-canonical.sh ${COMPONENTS_DIR}/$1/$1.wasm) \
+		<(WASM_TOOLS=$(WASM_TOOLS) build/scripts/wit-canonical.sh ${COMPONENTS_DIR}/$1/$1.debug.wasm) \
+		|| { echo "the WIT of the release and debug build of $1 differ" >&2; exit 1; }
+
+.PHONY: validate-components/$1
+validate-components/$1: ${COMPONENTS_DIR}/$1/$1.wasm | $(call tool,wasm-tools)
+	@$(WASM_TOOLS) validate --features all ${COMPONENTS_DIR}/$1/$1.wasm || { echo "$1.wasm is not a valid component"; exit 1; }
+	@$(WASM_TOOLS) validate --features all ${COMPONENTS_DIR}/$1/$1.debug.wasm || { echo "$1.debug.wasm is not a valid component"; exit 1; }
+
 ifneq ($(wildcard components/$1/wit/$1.constants.wit),)
 
 ${COMPONENTS_DIR}/$1/$1.wasm: components/$1/wit/deps ${COMPONENTS_DIR}/$1/README.md $(call tool_prerequisite,componentized-constants-cli)
@@ -95,17 +116,17 @@ ${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/RE
 # cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources
 else ifneq ($(wildcard components/$1/Cargo.toml),)
 
-${COMPONENTS_DIR}/$1/$1.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(COMPONENT_OPTIMIZE),$(call tool,wasm-opt)) $(WASI_SDK)
-	cargo build -p $1 --target $(COMPONENT_TARGET) --profile $(COMPONENT_PROFILE)
+${COMPONENTS_DIR}/$1/$1.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(COMPONENT_OPTIMIZE),$(call tool,wasm-opt)) $(if $(filter wasm32-wasip3,$(call component_target,$1)),$(WASI_SDK) $(WASI_LIBC))
+	cargo build -p $1 --target $(call component_target,$1) --profile $(COMPONENT_PROFILE)
 ifneq ($(COMPONENT_OPTIMIZE),)
-	$(WASM_OPT) -Oz $(WASM_OPT_FEATURES) $(call cargo_wasm,$1,$(COMPONENT_PROFILE)) -o $(call cargo_wasm,$1.opt,$(COMPONENT_PROFILE))
-	$(call componentize,$(call cargo_wasm,$1.opt,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm)
+	$(WASM_OPT) -Oz $(WASM_OPT_FEATURES) $(call cargo_wasm,$1,$(COMPONENT_PROFILE)) -o $(call cargo_wasm,$1,$(COMPONENT_PROFILE),.opt)
+	$(call componentize,$(call cargo_wasm,$1,$(COMPONENT_PROFILE),.opt),${COMPONENTS_DIR}/$1/$1.wasm)
 else
 	$(call componentize,$(call cargo_wasm,$1,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm)
 endif
 
-${COMPONENTS_DIR}/$1/$1.debug.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(WASI_SDK)
-	cargo build -p $1 --target $(COMPONENT_TARGET) --profile $(COMPONENT_DEBUG_PROFILE)
+${COMPONENTS_DIR}/$1/$1.debug.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(filter wasm32-wasip3,$(call component_target,$1)),$(WASI_SDK) $(WASI_LIBC))
+	cargo build -p $1 --target $(call component_target,$1) --profile $(COMPONENT_DEBUG_PROFILE)
 	$(call componentize,$(call cargo_wasm,$1,$(COMPONENT_DEBUG_PROFILE)),${COMPONENTS_DIR}/$1/$1.debug.wasm)
 
 endif

build/scripts/wit-canonical.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Print the WIT of a component in a canonical form, to compare the WIT of two components, e.g. the
+# release and debug build of a component.
+#
+#   WASM_TOOLS=<wasm-tools> scripts/wit-canonical.sh <component.wasm>
+#
+# `wasm-tools component wit` prints the items of a component's WIT in the order the component
+# defines them, which differs between builds of the same sources, e.g. a build optimized by
+# wasm-opt. Each line is printed with the items it's nested in, e.g.
+#
+#   package wasi:http@0.3.0 > interface types > resource request > get-method: func() -> method;
+#
+# the names a `use` imports are sorted, and the lines are sorted.
+
+set -euo pipefail
+
+component=$1
+wasm_tools=${WASM_TOOLS:-wasm-tools}
+
+"$wasm_tools" component wit "$component" | perl -ne '
+  s/^\s+|\s+$//g;
+  next unless length;
+  if ($_ eq "}") { pop @items; next }
+  s/\{([^{}]*)\}/"{" . join(", ", sort split(m{,\s*}, $1)) . "}"/e;
+  print join(" > ", @items, $_), "\n";
+  push @items, $1 if /^(.*?)\s*\{$/;
+' | LC_ALL=C sort

build/tools.mk

@@ -96,7 +96,7 @@ endef
 
 $(foreach name,$(LOCAL_TOOLS),$(eval $(call INSTALL_LOCAL_TOOL,$(name))))
 
-ifeq ($(COMPONENT_TARGET),wasm32-wasip3)
+ifneq ($(filter wasm32-wasip3,$(COMPONENT_TARGETS)),)
 
 # wasi-sdk, for its experimental sysroot with a wasi-libc built for component model cooperative
 # threads, see build/components.mk. Installed with the tools, set WASI_SDK_PATH to use an existing install.
@@ -114,4 +114,87 @@ $(WASI_SDK):
 	@rm -f $(@D)/wasi-sdk@*
 	@touch $@
 
+# a wasi-libc commit built for component model cooperative threads, in place of wasi-sdk's
+# experimental sysroot, for fixes not yet in a wasi-sdk release, e.g. `WASI_LIBC_VERSION :=
+# f6e9006376e6f8e8f91ab84fc956be4e345588bc` for WebAssembly/wasi-libc#884. Empty uses wasi-sdk's.
+WASI_LIBC_VERSION ?=
+# the WASI 0.3 version the wasi-libc built from WASI_LIBC_VERSION imports, in place of the one it's
+# generated for, e.g. `0.3.0` to match the WIT of the components. The WIT of the patch releases of
+# WASI 0.3 is the same but for docs, only the versions in the generated bindings are replaced.
+# Empty keeps wasi-libc's.
+WASI_LIBC_WASI_VERSION ?=
+
+ifeq ($(WASI_LIBC_VERSION),)
+
+WASI_SYSROOT := $(WASI_SDK_PATH)/share/wasi-sysroot/experimental-coop-threads
+WASI_LIBC :=
+
+else
+
+WASI_LIBC_BUILD := $(WASI_LIBC_VERSION)$(if $(WASI_LIBC_WASI_VERSION),-wasi-$(WASI_LIBC_WASI_VERSION))
+WASI_SYSROOT := $(abspath $(TOOLS_DIR))/wasi-libc-$(WASI_LIBC_BUILD)
+WASI_LIBC := $(TOOLS_DIR)/.installed/wasi-libc@$(WASI_LIBC_BUILD)
+
+# the bindings wasi-libc generates for WASI 0.3 from its WIT, checked in, and the version they're
+# generated for
+WASI_LIBC_WASIP3_BINDINGS := libc-bottom-half/headers/public/wasi/__generated_wasip3.h libc-bottom-half/sources/wasip3.c libc-bottom-half/sources/wasip3_component_type.o libc-bottom-half/crt/crt1-command.c
+wasi_libc_wasip3_version = $$(sed -n 's/^set(wasip3-version \(.*\))$$/\1/p' $(1)/CMakeLists.txt)
+
+# cmake and ninja build wasi-libc, installed with the tools
+CMAKE_VERSION := 4.4.4
+CMAKE_HOST := $(or $(word 2,$(subst :, ,$(filter $(shell rustc --print host-tuple):%,aarch64-apple-darwin:macos-universal x86_64-apple-darwin:macos-universal aarch64-unknown-linux-gnu:linux-aarch64 x86_64-unknown-linux-gnu:linux-x86_64))),unsupported-host)
+CMAKE_DIR := $(abspath $(TOOLS_DIR))/cmake-$(CMAKE_VERSION)
+# the macos archive holds an app bundle
+CMAKE := $(CMAKE_DIR)/$(if $(filter macos-%,$(CMAKE_HOST)),CMake.app/Contents/)bin/cmake
+CMAKE_INSTALLED := $(TOOLS_DIR)/.installed/cmake@$(CMAKE_VERSION)
+
+NINJA_VERSION := 1.13.2
+NINJA_HOST := $(or $(word 2,$(subst :, ,$(filter $(shell rustc --print host-tuple):%,aarch64-apple-darwin:mac x86_64-apple-darwin:mac aarch64-unknown-linux-gnu:linux-aarch64 x86_64-unknown-linux-gnu:linux))),unsupported-host)
+NINJA_DIR := $(abspath $(TOOLS_DIR))/ninja-$(NINJA_VERSION)
+NINJA := $(NINJA_DIR)/ninja
+NINJA_INSTALLED := $(TOOLS_DIR)/.installed/ninja@$(NINJA_VERSION)
+
+tools: $(WASI_LIBC)
+
+$(CMAKE_INSTALLED):
+	@rm -rf $(CMAKE_DIR)
+	@mkdir -p $(CMAKE_DIR) $(@D)
+	@set -o pipefail; curl -fsSL https://github.com/Kitware/CMake/releases/download/v$(CMAKE_VERSION)/cmake-$(CMAKE_VERSION)-$(CMAKE_HOST).tar.gz | tar xz --strip-components 1 -C $(CMAKE_DIR)
+	@rm -f $(@D)/cmake@*
+	@touch $@
+
+$(NINJA_INSTALLED):
+	@rm -rf $(NINJA_DIR)
+	@mkdir -p $(NINJA_DIR) $(@D)
+	@curl -fsSL https://github.com/ninja-build/ninja/releases/download/v$(NINJA_VERSION)/ninja-$(NINJA_HOST).zip -o $(NINJA_DIR)/ninja.zip
+	@unzip -q -o $(NINJA_DIR)/ninja.zip -d $(NINJA_DIR)
+	@rm $(NINJA_DIR)/ninja.zip
+	@rm -f $(@D)/ninja@*
+	@touch $@
+
+# built with wasi-sdk's clang, static libraries only, rust links libc statically and wasi-sdk's
+# compiler-rt builtins, which a shared libc links, aren't built for cooperative threads
+$(WASI_LIBC): | $(WASI_SDK) $(CMAKE_INSTALLED) $(NINJA_INSTALLED)
+	@rm -rf $(WASI_SYSROOT) $(WASI_SYSROOT)-src
+	@mkdir -p $(WASI_SYSROOT)-src $(@D)
+	@set -o pipefail; curl -fsSL https://github.com/WebAssembly/wasi-libc/archive/$(WASI_LIBC_VERSION).tar.gz | tar xz --strip-components 1 -C $(WASI_SYSROOT)-src
+ifneq ($(WASI_LIBC_WASI_VERSION),)
+	@# the type section of the bindings is binary, a version of the same length keeps it valid
+	@from=$(call wasi_libc_wasip3_version,$(WASI_SYSROOT)-src); to=$(WASI_LIBC_WASI_VERSION); \
+		if [ -z "$$from" ] || [ $${#from} -ne $${#to} ]; then echo "can't replace WASI $$from of wasi-libc with $$to" >&2; exit 1; fi; \
+		cd $(WASI_SYSROOT)-src && perl -0777 -pi -e "s/\@\Q$$from\E/\@$$to/g" $(WASI_LIBC_WASIP3_BINDINGS)
+endif
+	$(CMAKE) -S $(WASI_SYSROOT)-src -B $(WASI_SYSROOT)-src/build -G Ninja -DCMAKE_MAKE_PROGRAM=$(NINJA) \
+		-DCMAKE_C_COMPILER=$(WASI_SDK_PATH)/bin/clang -DCMAKE_AR=$(WASI_SDK_PATH)/bin/llvm-ar \
+		-DCMAKE_NM=$(WASI_SDK_PATH)/bin/llvm-nm -DCMAKE_RANLIB=$(WASI_SDK_PATH)/bin/llvm-ranlib \
+		-DTARGET_TRIPLE=wasm32-wasip3 -DENABLE_COOP_THREADS=ON -DBUILD_SHARED=OFF \
+		-DBUILTINS_LIB=$$(ls $(WASI_SDK_PATH)/lib/clang/*/lib/wasm32-unknown-wasip3/libclang_rt.builtins.a) \
+		-DCMAKE_INSTALL_PREFIX=$(WASI_SYSROOT)
+	$(NINJA) -C $(WASI_SYSROOT)-src/build install
+	@rm -rf $(WASI_SYSROOT)-src
+	@rm -f $(@D)/wasi-libc@*
+	@touch $@
+
+endif
+
 endif

tools/Cargo.toml

@@ -17,13 +17,15 @@ path = "lib.rs"
 [package.metadata.bins]
 componentized-constants-cli = ["constants"]
 wac-cli = ["wac"]
+wasm-opt = ["wasm-opt"]
 wasm-tools = ["wasm-tools"]
 wasmtime-cli = ["wasmtime"]
 wkg = ["wkg"]
 
 [dependencies]
 componentized-constants-cli = "=0.2.0"
 wac-cli = "=0.12.0"
+wasm-opt = "=0.116.1"
 wasm-tools = "=1.262.0"
 wasmtime-cli = "=50.0.0"
 wkg = "=0.16.1"