From 3ba70a67877eef57206faa749aeaeeb975519d56 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 21 Sep 2026 16:04:25 -0700 Subject: [PATCH 1/2] Use os.sched_getaffinity for parallelism On linux, os.sched_getaffinity(0) reports the number of cores available to the current process. Using it instead of raw os.cpu_count() will have better results when used in conjunction with e.g. `taskset` to limit parallel fuzzing or testing to a subset of the machine's cores. --- scripts/monitor_fuzz.py | 13 ++++++++++++- scripts/test/shared.py | 12 +++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/monitor_fuzz.py b/scripts/monitor_fuzz.py index 40b088ebd5a..437600a1d7a 100755 --- a/scripts/monitor_fuzz.py +++ b/scripts/monitor_fuzz.py @@ -31,6 +31,17 @@ import time +# Duplicated from test.shared (importing test.shared has unwanted side effects). +def cpu_count(): + # TODO: Use os.process_cpu_count for Python >= 3.13 + try: + # Available cores based on configured affinity (linux only) + return len(os.sched_getaffinity(0)) + except AttributeError: + # Fallback to number of logical cores on Mac/Windows. + return os.cpu_count() + + class FuzzMonitor: """Monitors fuzzer output stream, manages log files, and tracks state.""" @@ -164,7 +175,7 @@ def parse_args(): binaryen_root = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) default_log_dir = os.path.join(binaryen_root, 'out', 'test') - cores = os.cpu_count() or 1 + cores = cpu_count() or 1 parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '-j', diff --git a/scripts/test/shared.py b/scripts/test/shared.py index 0e65e159056..a154c3fac83 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -141,12 +141,22 @@ def red_stderr(): return red_output(file=sys.stderr) +def cpu_count(): + # TODO: Use os.process_cpu_count for Python >= 3.13 + try: + # Available cores based on configured affinity (linux only) + return len(os.sched_getaffinity(0)) + except AttributeError: + # Fallback to number of logical cores on Mac/Windows. + return os.cpu_count() + + def run_parallel_tests(run_one_test_func, tests, show_worker_count=True): global num_failures tests = list(tests) if not tests: return - worker_count = min(os.cpu_count() or 1, len(tests)) + worker_count = min(cpu_count() or 1, len(tests)) if show_worker_count: print(f"Running with {worker_count} workers") From 5cc064fcdc3fb04a6dcfd80d857c6b0ef7b85469 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Mon, 21 Sep 2026 16:50:19 -0700 Subject: [PATCH 2/2] adapt Emscripten's version --- scripts/monitor_fuzz.py | 21 ++++++++++++--------- scripts/test/shared.py | 22 +++++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/scripts/monitor_fuzz.py b/scripts/monitor_fuzz.py index 437600a1d7a..9f03304cc9f 100755 --- a/scripts/monitor_fuzz.py +++ b/scripts/monitor_fuzz.py @@ -32,14 +32,17 @@ # Duplicated from test.shared (importing test.shared has unwanted side effects). -def cpu_count(): - # TODO: Use os.process_cpu_count for Python >= 3.13 - try: - # Available cores based on configured affinity (linux only) - return len(os.sched_getaffinity(0)) - except AttributeError: - # Fallback to number of logical cores on Mac/Windows. - return os.cpu_count() +def get_num_cores(): + # Prefer `os.process_cpu_count` when available (3.13 and above) since it + # takes into account thread affinity. Fall back to `os.sched_getaffinity` + # where available and finally `os.cpu_count`, which should work everywhere. + if hasattr(os, 'process_cpu_count'): + cpu_count = os.process_cpu_count() + elif hasattr(os, 'sched_getaffinity'): + cpu_count = len(os.sched_getaffinity(0)) + else: + cpu_count = os.cpu_count() + return int(os.getenv('BINARYEN_CORES', cpu_count)) class FuzzMonitor: @@ -175,7 +178,7 @@ def parse_args(): binaryen_root = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) default_log_dir = os.path.join(binaryen_root, 'out', 'test') - cores = cpu_count() or 1 + cores = get_num_cores() or 1 parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '-j', diff --git a/scripts/test/shared.py b/scripts/test/shared.py index a154c3fac83..146b9e9c44a 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -141,14 +141,18 @@ def red_stderr(): return red_output(file=sys.stderr) -def cpu_count(): - # TODO: Use os.process_cpu_count for Python >= 3.13 - try: - # Available cores based on configured affinity (linux only) - return len(os.sched_getaffinity(0)) - except AttributeError: - # Fallback to number of logical cores on Mac/Windows. - return os.cpu_count() +# Adapted from Emscripten's tools/utils.py +def get_num_cores(): + # Prefer `os.process_cpu_count` when available (3.13 and above) since it + # takes into account thread affinity. Fall back to `os.sched_getaffinity` + # where available and finally `os.cpu_count`, which should work everywhere. + if hasattr(os, 'process_cpu_count'): + cpu_count = os.process_cpu_count() + elif hasattr(os, 'sched_getaffinity'): + cpu_count = len(os.sched_getaffinity(0)) + else: + cpu_count = os.cpu_count() + return int(os.getenv('BINARYEN_CORES', cpu_count)) def run_parallel_tests(run_one_test_func, tests, show_worker_count=True): @@ -156,7 +160,7 @@ def run_parallel_tests(run_one_test_func, tests, show_worker_count=True): tests = list(tests) if not tests: return - worker_count = min(cpu_count() or 1, len(tests)) + worker_count = min(get_num_cores() or 1, len(tests)) if show_worker_count: print(f"Running with {worker_count} workers")