diff --git a/.github/workflows/ci-js-api.yml b/.github/workflows/ci-js-api.yml new file mode 100644 index 0000000000..2bef2752d7 --- /dev/null +++ b/.github/workflows/ci-js-api.yml @@ -0,0 +1,77 @@ +name: CI for Wasm JS API tests + +on: + push: + branches: [main] + paths: + - .github/workflows/ci-js-api.yml + - test/harness/testharness.js + - test/js-api/** + + pull_request: + branches: [main] + paths: + - .github/workflows/ci-js-api.yml + - test/harness/testharness.js + - test/js-api/** + + workflow_dispatch: + +permissions: + contents: read + +jobs: + js-api: + name: JS API (${{ matrix.browser }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - browser: firefox + os: ubuntu-24.04 + - browser: chrome + os: ubuntu-24.04 + - browser: safari + os: macos-15 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Install Selenium + run: python -m pip install selenium==4.53.0 + - name: Enable Safari Remote Automation + if: matrix.browser == 'safari' + run: sudo /usr/bin/safaridriver --enable + - name: Run JS API tests + env: + BROWSER: ${{ matrix.browser }} + run: python test/js-api/run.py --browser "$BROWSER" --json wasm-js-api.json + - name: Upload browser report + if: ${{ !cancelled() && hashFiles('wasm-js-api.json') != '' }} + uses: actions/upload-artifact@v4 + with: + name: wasm-js-api-${{ matrix.browser }} + path: wasm-js-api.json + if-no-files-found: error + - name: Summarize browser results + if: ${{ !cancelled() && hashFiles('wasm-js-api.json') != '' }} + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + from collections import Counter + import json + from pathlib import Path + + report = json.loads(Path('wasm-js-api.json').read_text(encoding='utf-8')) + counts = Counter(case['outcome'] for case in report['results']) + print(f"### Wasm JS API in {report['browser']} {report['version']}\n") + print('| Pass | Fail | Expected failure | Unexpected pass | Skip |') + print('| ---: | ---: | ---: | ---: | ---: |') + print('| ' + ' | '.join(str(counts[outcome]) for outcome in ('PASS', 'FAIL', 'XFAIL', 'XPASS', 'SKIP')) + ' |') + PY diff --git a/test/js-api/README.md b/test/js-api/README.md index ca30ca42f5..840ce414f2 100644 --- a/test/js-api/README.md +++ b/test/js-api/README.md @@ -10,6 +10,44 @@ The tests use the [testharness.js] library and the [multi-global tests] setup for smooth integrations with the web-platform-tests project and the existing test infrastructure in implementations. +## Running in browsers + +From the repository root, install Selenium in a Python virtual environment and +run the suite against an installed browser. + +```sh +python3 -m venv /tmp/wasm-js-api-venv +/tmp/wasm-js-api-venv/bin/python -m pip install selenium +/tmp/wasm-js-api-venv/bin/python test/js-api/run.py --browser firefox|chrome|safari +``` + +For Safari, enable Remote Automation once with `safaridriver --enable`. + +### Browser expectations + +Selenium runs load [`expectations.json`](expectations.json). + +```json +{ + "memory/buffer.any.js": [ + { + "browser": "safari", + "context": "dedicatedworker", + "subtest": "Branding", + "expect": "fail", + "reason": "Link to the issue tracking this failure" + } + ], + "table/grow.any.js": [ + { + "browser": "firefox", + "expect": "skip", + "reason": "Link to the issue tracking why this test cannot run" + } + ] +} +``` + ## Metadata All tests must have the `.any.js` extension. diff --git a/test/js-api/expectations.json b/test/js-api/expectations.json new file mode 100644 index 0000000000..057c5727fc --- /dev/null +++ b/test/js-api/expectations.json @@ -0,0 +1,454 @@ +{ + "exception/constructor.tentative.any.js": [ + { + "browser": "firefox", + "subtest": "length", + "expect": "fail", + "reason": "WebAssembly.Exception.length is 1 instead of 2." + }, + { + "browser": "safari", + "subtest": "length", + "expect": "fail", + "reason": "WebAssembly.Exception.length is 1 instead of 2." + } + ], + "exception/getArg.tentative.any.js": [ + { + "browser": "safari", + "subtest": "Getting out-of-range argument", + "expect": "fail", + "reason": "Exception.getArg throws RangeError instead of TypeError for an out-of-range argument." + } + ], + "global/value-get-set.any.js": [ + { + "browser": "firefox", + "subtest": "Calling setter without argument", + "expect": "fail", + "reason": "The WebAssembly.Global value setter rejects a call with no argument." + }, + { + "browser": "safari", + "subtest": "Calling setter without argument", + "expect": "fail", + "reason": "The WebAssembly.Global value setter rejects a call with no argument." + } + ], + "instance/constructor-caching.any.js": [ + { + "browser": "safari", + "context": "window", + "subtest": "instance/constructor-caching.any.js (window)", + "expect": "fail", + "reason": "The cached and original WebAssembly.Global objects have different identities." + }, + { + "browser": "safari", + "context": "dedicatedworker", + "subtest": "constructor-caching", + "expect": "fail", + "reason": "The cached and original WebAssembly.Global objects have different identities." + } + ], + "interface.any.js": [ + { + "browser": "firefox", + "subtest": "WebAssembly.Table.set: length", + "expect": "fail", + "reason": "WebAssembly.Table.prototype.set.length is 2 instead of 1." + } + ], + "memory/constructor-memory64.any.js": [ + { + "browser": "safari", + "subtest": "Initial value exceeds maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Zero initial (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Non-zero initial (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Memory with i64 address constructor", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Memory with string value for initial (i64)", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Memory with boolean value for initial (i64)", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + } + ], + "memory/constructor.any.js": [ + { + "browser": "safari", + "subtest": "Order of evaluation for descriptor", + "expect": "fail", + "reason": "Safari does not read the address property of the descriptor." + }, + { + "browser": "safari", + "subtest": "Zero initial", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Non-zero initial", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Stray argument", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Memory with address parameter omitted", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Memory with i32 address constructor", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Memory with string value for initial", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Memory with boolean value for initial", + "expect": "fail", + "reason": "Safari does not report the expected address type from Memory.type()." + }, + { + "browser": "safari", + "subtest": "Unknown memory address", + "expect": "fail", + "reason": "Safari accepts an unknown descriptor address instead of throwing TypeError." + } + ], + "memory/grow-memory64.any.js": [ + { + "browser": "safari", + "subtest": "Zero initial (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Non-zero initial (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Zero initial with respected maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Zero initial with respected maximum grown twice (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Zero initial growing too much (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: -1n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: 18446744073709551616n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: \"0x10000000000000000\"", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + } + ], + "memory/grow.any.js": [ + { + "browser": "firefox", + "subtest": "Growing shared memory does not detach old buffer", + "expect": "fail", + "reason": "The old buffer after shared memory growth is not frozen." + } + ], + "module/customSections.any.js": [ + { + "browser": "firefox", + "subtest": "Custom sections with U+FFFD", + "expect": "fail", + "reason": "A custom-section name lookup returns a match where the test expects none." + } + ], + "table/constructor-memory64.any.js": [ + { + "browser": "safari", + "subtest": "Initial value exceeds maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Basic (zero, i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Basic (non-zero, i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Table with i64 address constructor", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Table with string value for initial (i64)", + "expect": "fail", + "reason": "Safari reports table length as a number for an i64 table." + }, + { + "browser": "safari", + "subtest": "Table with boolean value for initial (i64)", + "expect": "fail", + "reason": "Safari reports table length as a number for an i64 table." + }, + { + "browser": "safari", + "subtest": "Table with string value for maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Table with boolean value for maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + } + ], + "table/constructor.any.js": [ + { + "browser": "safari", + "subtest": "Order of evaluation for descriptor", + "expect": "fail", + "reason": "Safari does not read the address property of the descriptor." + }, + { + "browser": "safari", + "subtest": "Unknown table address", + "expect": "fail", + "reason": "Safari accepts an unknown descriptor address instead of throwing TypeError." + } + ], + "table/get-set.any.js": [ + { + "browser": "chrome", + "subtest": "Arguments for anyfunc table set", + "expect": "fail", + "reason": "Table.set rejects an explicit undefined value for an anyfunc table." + }, + { + "browser": "firefox", + "subtest": "Arguments for anyfunc table set", + "expect": "fail", + "reason": "Table.set rejects an explicit undefined value for an anyfunc table." + }, + { + "browser": "safari", + "subtest": "Basic (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Growing (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Setting out-of-bounds (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Getting out-of-range argument (i64): -1n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Setting out-of-range argument (i64): -1n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Getting out-of-range argument (i64): 18446744073709551616n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Setting out-of-range argument (i64): 18446744073709551616n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Getting out-of-range argument (i64): \"0x10000000000000000\"", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Setting out-of-range argument (i64): \"0x10000000000000000\"", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Arguments for anyfunc table set", + "expect": "fail", + "reason": "Table.set rejects an explicit undefined value for an anyfunc table." + } + ], + "table/grow-memory64.any.js": [ + { + "browser": "chrome", + "subtest": "Basic i64", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "chrome", + "subtest": "Reached maximum (i64)", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "chrome", + "subtest": "Exceeded maximum (i64)", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "firefox", + "subtest": "Basic i64", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "firefox", + "subtest": "Reached maximum (i64)", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "firefox", + "subtest": "Exceeded maximum (i64)", + "expect": "fail", + "reason": "The test calls nulls(), which its declared helpers do not define." + }, + { + "browser": "safari", + "subtest": "Basic i64", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Reached maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Exceeded maximum (i64)", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: -1n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: 18446744073709551616n", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + }, + { + "browser": "safari", + "subtest": "Out-of-range i64 argument: \"0x10000000000000000\"", + "expect": "fail", + "reason": "Safari rejects the BigInt argument in this i64 memory or table operation." + } + ], + "table/grow.any.js": [ + { + "browser": "chrome", + "subtest": "Grow with undefined argument", + "expect": "fail", + "reason": "Table.grow rejects an explicit undefined initializer for an anyfunc table." + }, + { + "browser": "firefox", + "subtest": "Grow with undefined argument", + "expect": "fail", + "reason": "Table.grow rejects an explicit undefined initializer for an anyfunc table." + }, + { + "browser": "safari", + "subtest": "Grow with undefined argument", + "expect": "fail", + "reason": "Table.grow rejects an explicit undefined initializer for an anyfunc table." + } + ] +} diff --git a/test/js-api/run.py b/test/js-api/run.py new file mode 100755 index 0000000000..5cfd1e975c --- /dev/null +++ b/test/js-api/run.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +"""Run the Wasm JS API tests in a browser using Selenium.""" + +import argparse +import json +import re +from collections import Counter +from contextlib import nullcontext +from dataclasses import dataclass +from functools import cache +from html import escape +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from urllib.parse import quote, urljoin, urlsplit + +SUITE = Path(__file__).resolve().parent +MANIFEST = SUITE / "expectations.json" +PREFIX = "/wasm/jsapi/" +HARNESS = "/resources/testharness.js" +BROWSERS = ("chrome", "firefox", "safari") +BROWSER_GLOBALS = ("window", "dedicatedworker") +# Match the timeout values in testharness.js. +HARNESS_TIMEOUT_SECONDS = {"normal": 10, "long": 60} +WEBDRIVER_GRACE_SECONDS = 5 +# Manifest rules without a subtest name apply to the whole browser case. +WHOLE_CASE = None + +REPORTER = """\ +setup({output: false}); +globalThis.wasmTestResult = new Promise(resolve => { + let finished = false; + function report(result) { + if (finished) return; + finished = true; + document.getElementById("results").textContent = JSON.stringify(result, null, 2); + resolve(result); + } + + add_completion_callback((tests, harness) => { + const harnessStatuses = ["OK", "ERROR", "TIMEOUT", "PRECONDITION_FAILED"]; + const subtestStatuses = ["PASS", "FAIL", "TIMEOUT", "NOTRUN", "PRECONDITION_FAILED"]; + report({ + status: harnessStatuses[harness.status] ?? `UNKNOWN (${harness.status})`, + message: harness.message, + subtests: tests.map(test => ({ + name: test.name, + status: subtestStatuses[test.status] ?? `UNKNOWN (${test.status})`, + message: test.message, + })), + }); + }); + + // A missing classic script can leave the harness with only partial results. + addEventListener("error", event => { + if (event.target instanceof HTMLScriptElement) { + report({status: "ERROR", message: `Could not load ${event.target.src}`, subtests: []}); + } + }, true); +}); +""" + + +@dataclass(frozen=True) +class BrowserCase: + name: str + context: str + page_path: str + timeout_seconds: int + + +def test_metadata(path): + globals_ = set(BROWSER_GLOBALS) + helpers = [] + timeout = "normal" + with path.open(encoding="utf-8-sig") as source: + for line in source: + line = line.strip() + if line and not line.startswith("//"): + break + if not line.startswith("// META:"): + continue + match = re.fullmatch(r"// META:\s*(\w+)=(.+)", line) + if not match: + raise ValueError(f"{path.name}: invalid metadata: {line}") + key, value = match.groups() + if key == "global": + globals_ = {name.strip() for name in value.split(",")} + # Accept globals used by WPT even when this runner cannot launch them. + unknown = globals_ - {*BROWSER_GLOBALS, "jsshell", "shadowrealm"} + if unknown: + raise ValueError( + f"{path.name}: unsupported globals: {', '.join(sorted(unknown))}" + ) + elif key == "script": + helpers.append(value) + elif key == "timeout" and value == "long": + timeout = value + else: + raise ValueError(f"{path.name}: unsupported metadata: {line}") + return globals_, helpers, timeout + + +def discover_suite(): + sources = {path.relative_to(SUITE).as_posix(): path for path in SUITE.rglob("*.js")} + scripts = {PREFIX + quote(name): path for name, path in sources.items()} + scripts[HARNESS] = SUITE.parent / "harness" / "testharness.js" + tests = {name: path for name, path in sources.items() if name.endswith(".any.js")} + return scripts, tests + + +def generate_test_pages(scripts, tests, selected, contexts, metadata_for): + unknown = selected - tests.keys() + if unknown: + raise ValueError(f"Unknown tests: {', '.join(sorted(unknown))}") + pages = {} + cases = [] + for name in sorted(selected or tests): + url = PREFIX + quote(name) + stem = url.removesuffix(".js") + globals_, helpers, timeout_mode = metadata_for(tests[name]) + sources = [urljoin(url, helper) for helper in helpers] + for source in sources: + if source not in scripts: + raise ValueError(f"{name}: missing helper script {source}") + sources.append(url) + for context in contexts: + if context not in globals_: + continue + if context == "window": + page = stem + ".html" + body = "\n".join( + f'' for source in sources + ) + else: + page = stem + ".worker.html" + worker = stem + ".worker.js" + imports = ", ".join(map(json.dumps, [HARNESS, *sources])) + # The worker signals completion explicitly. Its parent reports the results. + pages[worker] = ( + "text/javascript", + f"importScripts({imports});\ndone();", + ) + body = f"" + pages[page] = ( + "text/html", + f""" + + +
Running {escape(name)} ({context})…
+
+
+{body}
+""",
+ )
+ cases.append(
+ BrowserCase(
+ name=name,
+ context=context,
+ page_path=page,
+ timeout_seconds=HARNESS_TIMEOUT_SECONDS[timeout_mode],
+ )
+ )
+ links = "\n".join(
+ f'