Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
252 changes: 252 additions & 0 deletions .github/openclaw/qualification/engine-gc-progress-probe.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
// Verify that a passive collector progresses when its mutator starts waiting.
// Observe existing engine state without production hooks or state mutation.
#include <cmakeconfig.h>
#include <JavaScriptCore/APICast.h>
#include <JavaScriptCore/GCRequest.h>
#include <JavaScriptCore/Heap.h>
#include <JavaScriptCore/HeapHelperPool.h>
#include <JavaScriptCore/HeapInlines.h>
#include <JavaScriptCore/InitializeThreading.h>
#include <JavaScriptCore/JavaScript.h>
#include <JavaScriptCore/JSGlobalObject.h>
#include <JavaScriptCore/JSLock.h>
#include <JavaScriptCore/Options.h>
#include <JavaScriptCore/VM.h>
#include <wtf/Atomics.h>
#include <wtf/Condition.h>
#include <wtf/MainThread.h>
#include <wtf/ParkingLot.h>
#include <wtf/ParallelHelperPool.h>
#include <wtf/RefPtr.h>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdio>
#include <mutex>
#include <thread>
#include <utility>

namespace {

// Explicit-instantiation arguments are exempt from member access checks. These
// probe-only accessors expose observation addresses, never mutate heap state.
template<typename Tag, typename Tag::Type member>
struct ObserveMember {
friend typename Tag::Type observe(Tag) { return member; }
};

struct HeapCondition {
using Type = WTF::Condition JSC::Heap::*;
friend Type observe(HeapCondition);
};
template struct ObserveMember<HeapCondition, &JSC::Heap::m_markingConditionVariable>;

struct HeapWorldState {
using Type = WTF::Atomic<unsigned> JSC::Heap::*;
friend Type observe(HeapWorldState);
};
template struct ObserveMember<HeapWorldState, &JSC::Heap::m_worldState>;

struct ConditionWaitAddress {
using Type = WTF::Atomic<bool> WTF::Condition::*;
friend Type observe(ConditionWaitAddress);
};
template struct ObserveMember<ConditionWaitAddress, &WTF::Condition::m_hasWaiters>;

template<typename Tag, unsigned value>
struct ObserveConstant {
friend unsigned observe(Tag) { return value; }
};
struct MutatorWaitingBit { friend unsigned observe(MutatorWaitingBit); };
template struct ObserveConstant<MutatorWaitingBit, JSC::Heap::mutatorWaitingBit>;

class HeldHelpers {
public:
void enter()
{
std::unique_lock lock(m_mutex);
++m_entered;
m_changed.notify_all();
m_changed.wait(lock, [&] { return m_released; });
}

bool waitUntilEntered(unsigned expected)
{
std::unique_lock lock(m_mutex);
return m_changed.wait_for(lock, std::chrono::seconds(10), [&] { return m_entered == expected; });
}

void release()
{
std::lock_guard lock(m_mutex);
m_released = true;
m_changed.notify_all();
}

private:
std::mutex m_mutex;
std::condition_variable m_changed;
unsigned m_entered { 0 };
bool m_released { false };
};

struct Parked {
bool collector { false };
bool mutator { false };
uintptr_t collectorThread { 0 };
uintptr_t mutatorThread { 0 };
};

Parked inspectParking(const void* conditionAddress, const void* worldAddress, uintptr_t expectedMutator)
{
Parked result;
// The callback runs under ParkingLot locks: do not allocate, park, or log.
WTF::ParkingLot::forEach([&](uintptr_t thread, const void* address) {
if (address == conditionAddress) {
result.collector = true;
result.collectorThread = thread;
}
if (address == worldAddress && thread == expectedMutator) {
result.mutator = true;
result.mutatorThread = thread;
}
});
return result;
}

bool evaluate(JSGlobalContextRef context, const char* source, double* number = nullptr)
{
JSStringRef script = JSStringCreateWithUTF8CString(source);
JSValueRef exception = nullptr;
JSValueRef value = JSEvaluateScript(context, script, nullptr, nullptr, 1, &exception);
JSStringRelease(script);
if (exception || !value)
return false;
if (number)
*number = JSValueToNumber(context, value, &exception);
return !exception;
}

} // namespace

int main()
{
WTF::initializeMainThread();
JSC::initialize();
// Zero pause and utilization=1 force concurrent scheduling without an RNG
// outcome or allocation-time threshold. These are probe settings only.
if (!JSC::Options::setOptions("useConcurrentGC=true useStochasticMutatorScheduler=true minimumGCPauseMS=0 gcPauseScale=0 minimumMutatorUtilization=1 maximumMutatorUtilization=1 numberOfGCMarkers=2")) {
std::fprintf(stderr, "probe options rejected\n");
return 2;
}
JSGlobalContextRef context = JSGlobalContextCreate(nullptr);
if (!context || !evaluate(context, "globalThis.gcProgressRoots = Array.from({length: 32768}, (_, i) => ({index:i, child:{value:i}})); gcProgressRoots.length")) {
std::fprintf(stderr, "probe heap setup failed\n");
return 2;
}

auto& vm = toJS(context)->vm();
bool foundPassiveCollector = false;
bool foundLateWaitStall = false;
bool timedOut = false;
bool rootsSurvived = false;
bool completedBeforeHelperRelease = false;
unsigned helperCount = 0;
unsigned stalledWorldState = 0;
Parked stalledThreads;
const void* conditionAddress = nullptr;
const void* worldAddress = nullptr;
double elapsedMs = 0;
{
JSC::JSLockHolder lock(vm);
auto& heap = vm.heap;
// Settle setup collection while helpers are available. No prior cycle or
// unrelated VM is allowed to account for the later observed wait.
heap.collectNow(JSC::Sync, JSC::CollectionScope::Full);
auto& condition = heap.*observe(HeapCondition { });
auto& world = heap.*observe(HeapWorldState { });
conditionAddress = &(condition.*observe(ConditionWaitAddress { }));
worldAddress = &world;
const unsigned waitingBit = observe(MutatorWaitingBit { });
const uintptr_t mutatorParkingThread = WTF::ParkingLot::currentThreadID();

WTF::RefPtr<WTF::ParallelHelperPool> sharedPool = &JSC::heapHelperPool();
helperCount = sharedPool->numberOfThreads();
if (!helperCount) {
std::fprintf(stderr, "probe requires parallel GC helpers\n");
return 2;
}
HeldHelpers gate;
WTF::ParallelHelperClient blocker(std::move(sharedPool));
blocker.setFunction([&] { gate.enter(); });
if (!gate.waitUntilEntered(helperCount)) {
gate.release();
blocker.finish();
std::fprintf(stderr, "could not occupy every shared helper\n");
return 2;
}

std::atomic<bool> invokeWait { false };
std::atomic<bool> completed { false };
std::atomic<bool> giveUp { false };
auto begin = std::chrono::steady_clock::now();
std::thread observer([&] {
const auto deadline = begin + std::chrono::seconds(10);
while (std::chrono::steady_clock::now() < deadline) {
auto parked = inspectParking(conditionAddress, worldAddress, mutatorParkingThread);
if (!foundPassiveCollector && parked.collector && !(world.load() & waitingBit)) {
foundPassiveCollector = true;
invokeWait.store(true, std::memory_order_release);
}
if (completed.load(std::memory_order_acquire)) {
completedBeforeHelperRelease = true;
gate.release();
return;
}
if (foundPassiveCollector && parked.collector && parked.mutator && (world.load() & waitingBit)) {
// This is an observed wait cycle, not a timing inference.
foundLateWaitStall = true;
stalledThreads = parked;
stalledWorldState = world.load();
gate.release(); // Recovery is not counted as successful proof.
return;
}
std::this_thread::yield();
}
timedOut = true;
giveUp.store(true, std::memory_order_release);
gate.release();
});

heap.collectAsync(JSC::CollectionScope::Full);
// This thread retains real mutator heap access. Service the engine's
// stop handshake until the observer sees the collector actually parked.
while (!invokeWait.load(std::memory_order_acquire) && !giveUp.load(std::memory_order_acquire)) {
heap.stopIfNecessary();
heap.relinquishConn();
std::this_thread::yield();
}
if (invokeWait.load(std::memory_order_acquire)) {
heap.preventCollection();
heap.allowCollection();
completed.store(true, std::memory_order_release);
}
observer.join();
blocker.finish();
// Also settle the failure/control recovery before releasing the VM.
heap.collectNow(JSC::Sync, JSC::CollectionScope::Full);
double checksum = 0;
rootsSurvived = evaluate(context, "gcProgressRoots.length + gcProgressRoots[32767].child.value", &checksum)
&& checksum == 65535;
elapsedMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count();
// Final VM destruction must retain the API lock and its active atom string table.
JSGlobalContextRelease(context);
}
const bool passed = foundPassiveCollector && completedBeforeHelperRelease && !foundLateWaitStall && !timedOut && rootsSurvived;
std::printf("{\"passed\":%s,\"passiveCollectorObserved\":%s,\"lateWaitStallObserved\":%s,\"completedBeforeHelperRelease\":%s,\"timedOut\":%s,\"rootsSurvived\":%s,\"helperCount\":%u,\"worldStateAtStall\":%u,\"collectorParkingThreadId\":%llu,\"mutatorParkingThreadId\":%llu,\"conditionAddress\":\"%p\",\"worldAddress\":\"%p\",\"elapsedMs\":%.3f}\n",
passed ? "true" : "false", foundPassiveCollector ? "true" : "false", foundLateWaitStall ? "true" : "false",
completedBeforeHelperRelease ? "true" : "false", timedOut ? "true" : "false", rootsSurvived ? "true" : "false",
helperCount, stalledWorldState, static_cast<unsigned long long>(stalledThreads.collectorThread),
static_cast<unsigned long long>(stalledThreads.mutatorThread), conditionAddress, worldAddress, elapsedMs);
return passed ? 0 : foundLateWaitStall && rootsSurvived ? 1 : 2;
}
11 changes: 11 additions & 0 deletions .github/openclaw/qualify.sh
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ clang++-23 -std=c++23 -O2 -DNDEBUG -DBUILDING_WITH_CMAKE=1 -DHAVE_CONFIG_H=1 \
-Wl,--start-group "$WK"/lib/*.a -Wl,--end-group -lpthread -ldl -lm -latomic \
-o "$QUALIFICATION_DIR/engine-limit-probe" > "$QUALIFICATION_DIR/engine-limit-build.log" 2>&1
"$QUALIFICATION_DIR/engine-limit-probe" > "$QUALIFICATION_DIR/engine-limits.log" 2>&1
clang++-23 -std=c++23 -O2 -DNDEBUG -DBUILDING_WITH_CMAKE=1 -DHAVE_CONFIG_H=1 \
-I"$WK/include" -I"$WK/include/JavaScriptCore" -I"$WK/include/wtf" \
"$inputs/engine-gc-progress-probe.cpp" "$QUALIFICATION_DIR/bun/build/qualify-candidate/obj/vendor/mimalloc/src/static.c.o" \
-Wl,--start-group "$WK"/lib/*.a -Wl,--end-group -lpthread -ldl -lm -latomic \
-o "$QUALIFICATION_DIR/engine-gc-progress-probe" > "$QUALIFICATION_DIR/engine-gc-progress-build.log" 2>&1
"$QUALIFICATION_DIR/engine-gc-progress-probe" > "$QUALIFICATION_DIR/engine-gc-progress.json" 2> "$QUALIFICATION_DIR/engine-gc-progress.log"
"$candidate" "$inputs/module-context.mjs" matrix > "$QUALIFICATION_DIR/als-plugin.json"
"$candidate" "$inputs/module-context.mjs" matrix --native-hooks > "$QUALIFICATION_DIR/als-native.json"
python3 - "$QUALIFICATION_DIR" "$SOURCE_SHA" "$BUN_COMMIT" <<'PY'
Expand All @@ -95,7 +101,12 @@ for name,minimum in [('proxy',4),('resource-limits',16),('namespace',41),('array
passes=re.search(r'(\d+) pass',text)
assert passes and int(passes[1])>=minimum, name+' missing passing cases'
assert re.search(r'\b0 fail\b',text),name+' failures'
gc_progress=json.loads((root/'engine-gc-progress.json').read_text())
assert all(gc_progress[key] is True for key in ['passed','passiveCollectorObserved','completedBeforeHelperRelease','rootsSurvived']),'passive collector did not progress'
assert all(gc_progress[key] is False for key in ['lateWaitStallObserved','timedOut']),'passive collector stalled'
assert type(gc_progress['helperCount']) is int and gc_progress['helperCount']>0,'GC helper precondition missing'
(root/'gate.json').write_text(json.dumps({'passed':True,'source':sys.argv[2],'bun_commit':sys.argv[3],
'selected_files':44,'result_files':46,'als_variants_per_mode':14,'proxy_minimum':4,'resource_minimum':16,
'gc_progress':gc_progress,
'bun_adapter_sha256':hashlib.sha256((root/'bun-adapters.patch').read_bytes()).hexdigest()},indent=2)+'\n')
PY
2 changes: 1 addition & 1 deletion .github/openclaw/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ def publish(directory,source):
raise ValueError('publishing requires reviewed source on an explicitly permitted owned branch')
# Tag creation fails atomically if the name exists. Neither existing drafts nor releases are resumed.
api('-X','POST',f'repos/{REPO}/git/refs','-f','ref=refs/tags/'+tag,'-f','sha='+source)
body=f'OpenClaw-built WebKit at `{source}`. All selected lanes and Linux qualification passed.\n\nBatch 1: Segmenter surrogate boundaries; opt-in Proxy global prototypes; module-loader AsyncLocalStorage propagation; per-VM worker heap and stack budgets. Thanks @steipete and @robobun for the source changes and upstream work.\n\nSource and licenses: LICENSE-SOURCES.txt. Checksums: SHA256SUMS and manifest.json. Full build and test provenance: provenance.tar.gz.\n\nRollback by restoring the prior Bun manifest and WebKit pin together; prior releases remain available.\n'
body=f'OpenClaw-built WebKit at `{source}`. All selected lanes and Linux qualification passed.\n\nThis release wakes a passive garbage collector when its mutator starts waiting, so collection progresses while shared GC helpers are occupied elsewhere. Native qualification requires completion before helper release and survival of rooted objects.\n\nBatch 1: Segmenter surrogate boundaries; opt-in Proxy global prototypes; module-loader AsyncLocalStorage propagation; per-VM worker heap and stack budgets. Thanks @steipete and @robobun for the source changes and upstream work.\n\nSource and licenses: LICENSE-SOURCES.txt. Checksums: SHA256SUMS and manifest.json. Full build and test provenance: provenance.tar.gz.\n\nRollback by restoring the prior Bun manifest and WebKit pin together; prior releases remain available.\n'
draft=api('-X','POST',f'repos/{REPO}/releases','-f','tag_name='+tag,'-f','name='+tag,'-f','body='+body,'-F','draft=true')
release_id=draft['id']
execute('gh','release','upload',tag,'-R',REPO,*[str(p) for p in sorted(directory.iterdir())])
Expand Down
7 changes: 7 additions & 0 deletions Source/JavaScriptCore/heap/Heap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2849,6 +2849,13 @@ void Heap::waitForCollector(const Func& func)
}
}

if (!done) {
// The collector may already be waiting passively for helper threads. Pair its
// predicate and wait with this notification, without holding m_threadLock.
Locker locker { m_markingMutex };
m_markingConditionVariable.notifyAll();
}

// If we're in a stop-the-world scenario, we need to wait for that even if done is true.
unsigned oldState = m_worldState.load();
if (stopIfNecessarySlow(oldState))
Expand Down
7 changes: 7 additions & 0 deletions Source/JavaScriptCore/heap/SlotVisitor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,13 @@ SlotVisitor::SharedDrainResult SlotVisitor::drainInParallelPassively(MonotonicTi
return SharedDrainResult::Done;
}

// A synchronous collector wait can begin after the entry check. Keep making
// progress even when the process-wide helper pool is serving another heap.
if (m_heap.m_worldState.load() & Heap::mutatorWaitingBit) {
locker.unlockEarly();
return drainInParallel(timeout);
}

m_heap.m_markingConditionVariable.waitUntil(m_heap.m_markingMutex, timeout);
}
}
Expand Down