diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94efcc34d..e369d0859 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -451,7 +451,7 @@ jobs: working-directory: test/regression run: | failed="" - for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcArrayChunks IncrementalGcPacing IncrementalGcStartSweep IncrementalGcStats IncrementalGcFinishBoundary IncrementalGcVerifier IncrementalWindowsHeaders GcWeakMapChain GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do + for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcArrayChunks IncrementalGcDequeMoves IncrementalGcPacing IncrementalGcStartSweep IncrementalGcStats IncrementalGcFinishBoundary IncrementalGcVerifier IncrementalWindowsHeaders GcWeakMapChain GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do echo "::group::incremental gc: $test" # The lifecycle fixtures' hxml selects generational mode; compile # the same Main with incremental mode here instead. diff --git a/Changes.md b/Changes.md index d3372b98b..1fe9345e8 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed the incremental collector (HXCPP_GC_INCREMENTAL) freeing a thread object's arrays while the thread was being created, as the gc stress test's verifier caught on Windows. A slice could find the new object on the stack while its first array was being allocated, and the cycle could end before the constructor's barrier, with the arrays stored after that unscanned. The arrays are now made in a mutation scope, which the stop that finishes a cycle scans again * Added timedWait to hx::thread::ConditionVariable, which Haxe 5's sys.thread.Condition.timedWait calls: with the Haxe nightly, every program that used Condition failed to compile. It returns true once a signal or broadcast came and false when the time ran out, and it waits out the wakeups MSVC's timed waits give just before the deadline with nothing signalled * Fixed an exception from blit, setSize or resize under the cppia JIT ending the process instead of reaching the script's catch, on typed arrays and on Array: their native calls had no exception bridge. The script catches it, as it does interpreted and in compiled code (libhxcpp #404) * Fixed resize() and cpp.NativeArray.setSize() on an Array under the cppia JIT evaluating the array expression twice, so a call in it ran twice and the array it returned the second time was the one resized. The JIT evaluates it once (libhxcpp #411) @@ -19,6 +20,7 @@ libhxcpp 6.0.0 (unreleased) * Added HXCPP_PGO, profile-guided optimisation with MSVC, GCC and Clang: -D HXCPP_PGO=generate builds a program that counts what it runs, and -D HXCPP_PGO=use builds it again from what a run counted, with link-time optimisation. On a replication benchmark it took a further 4 to 29% off with MSVC, 19 to 43% with GCC and 12 to 46% with Clang. See docs/ProfileGuidedOptimisation.md * Fixed HXCPP_OPTIMIZE_LINK builds with Clang on Linux failing to link ("file format not recognized"): objects were compiled with -flto and linked without it, which GCC's linker plugin covered for, on one thread, and Clang's bitcode objects did not. Programs and shared libraries now link with -flto * Fixed an MSVC build with HXCPP_OPTIMIZE_LINK failing after any change (C1382, then LNK1257) until its objects were deleted: a group's PCH is rebuilt whenever one of its files compiles, and the link checks each object against the PCH it was compiled with. When the PCH is rebuilt under link-time code generation, the rest of the group is compiled against it too +* Fixed the incremental collector (HXCPP_GC_INCREMENTAL) freeing items still in a sys.thread.Deque of more than 256 items, as the gc stress test's verifier caught about once in 75 runs: a cycle scans a long array in chunks, and the runtime's Deque moves its items itself, when a pop compacts the queue and when a push at the front makes room, which could carry items no chunk had reached into chunks already scanned. Those moves now have the queue scanned again, as Array's own moves do (libhxcpp #424) * Fixed hxcpp.gc.Incremental.getStats() returning null for six of its twelve counters when read by name, such as active and requestedSlices: they were fixed fields out of hash order, which a read by name searches. Printing the object and Json showed them, as those use the field's own name (libhxcpp #372) * The incremental collector no longer stops for a full reclaim every 15 collections, which took 5 ms on a 120 MB heap and 17 ms on an 800 MB one. Slices sweep a share of the blocks each collection instead, a finished cycle counts rows and leaves the sweep to the zeroing thread and the allocator, and a slice looks at the clock after each large job (libhxcpp #372) * Fixed the incremental collector doing no incremental work for a program that allocates mostly array storage, such as Bytes: a slice was skipped inside a mutation scope and a cycle could not finish in one, so every collection was a full stop. Slices run there now, and the stop that finishes a cycle scans the holders of open scopes again (libhxcpp #372) diff --git a/src/hx/Thread.cpp b/src/hx/Thread.cpp index d7cfc35f5..b7f875746 100644 --- a/src/hx/Thread.cpp +++ b/src/hx/Thread.cpp @@ -50,7 +50,10 @@ struct Deque : public Array_obj } else if (head > 64 && head >= length - head) { - // Compact once the consumed prefix dominates - O(1) amortized + // Compact once the consumed prefix dominates - O(1) amortized. + // An incremental cycle may have scanned the slots the items move + // to and not those they come from, so it scans the queue again + HX_INCREMENTAL_MUTATION(this); int remaining = length - head; ::memmove(items, items + head, remaining * sizeof(Dynamic)); ::memset(items + remaining, 0, head * sizeof(Dynamic)); @@ -79,6 +82,8 @@ struct Deque : public Array_obj // building an n-deep queue that way was O(n^2) while holding the lock void openFront() { + // The items move, as for a compaction in popHead + HX_INCREMENTAL_MUTATION(this); int count = length; int gap = count >> 1; if (gap < 16) diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index 77e204196..771221f8e 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -20,6 +20,9 @@ #ifdef HXCPP_GC_INCREMENTAL #include #endif +#ifdef HXCPP_GC_INCREMENTAL_VERIFY +#include +#endif #ifdef __EMSCRIPTEN__ #include @@ -2245,6 +2248,11 @@ class MarkContext bool incrementalOracle = false; std::unordered_map oracleVisited; std::vector oracleQueue; + // How the oracle first reached each allocation: the object whose members + // it was walking, or the kind of root, for the report of an omission + Object *oracleParent = nullptr; + const char *oracleRoot = nullptr; + std::unordered_map > oracleFrom; #endif #ifdef HX_GC_VERIFY_GENERATIONAL @@ -2599,6 +2607,8 @@ bool IncrementalOracleMark(void *inPointer, bool inObject, MarkContext *inCtx) if (flags & HX_GC_CONST_ALLOC_BIT) return true; auto result = inCtx->oracleVisited.emplace(inPointer, inObject); + if (result.second) + inCtx->oracleFrom.emplace(inPointer, std::make_pair(inCtx->oracleParent, inCtx->oracleRoot)); // A backing allocation can also have been encountered through an object // pointer. In that case its outgoing members still need the object walk. if (inObject && (flags & IMMIX_ALLOC_IS_CONTAINER) && (result.second || !result.first->second)) @@ -6149,8 +6159,10 @@ class GlobalAllocator { hx::MarkContext oracle; oracle.incrementalOracle = true; + oracle.oracleRoot = "statics"; hx::MarkClassStatics(&oracle); { + oracle.oracleRoot = "roots"; std::lock_guard lock(*hx::sGCRootLock); for (hx::Object **slot : *hx::sgRootSet) if (*slot) @@ -6163,14 +6175,18 @@ class GlobalAllocator hx::MarkObjectAlloc(reinterpret_cast(ptr - entry.second), &oracle); } } + oracle.oracleRoot = "zombies"; for (int i = 0; i < hx::sZombieList.size(); i++) hx::MarkObjectAlloc(hx::sZombieList[i], &oracle); + oracle.oracleRoot = "stacks"; for (int i = 0; i < mLocalAllocs.size(); i++) MarkLocalAlloc(mLocalAllocs[i], &oracle); + oracle.oracleRoot = nullptr; while (!oracle.oracleQueue.empty()) { hx::Object *object = oracle.oracleQueue.back(); oracle.oracleQueue.pop_back(); + oracle.oracleParent = object; object->__Mark(&oracle); } for (const auto &entry : oracle.oracleVisited) @@ -6186,6 +6202,26 @@ class GlobalAllocator if (!valid) { fprintf(stderr, "Incremental GC omitted reachable %s %p flags=%08x epoch=%08x rows=%u first=%u vtable=%p\n", entry.second ? "object" : "allocation", entry.first, flags, hx::gMarkID, rows, rows ? *row : 0, entry.second ? *static_cast(entry.first) : nullptr); + // The way the oracle reached it, back to a root + void *at = entry.first; + for (int depth = 0; depth < 64; depth++) + { + auto from = oracle.oracleFrom.find(at); + if (from == oracle.oracleFrom.end()) + break; + hx::Object *parent = from->second.first; + if (!parent) + { + fprintf(stderr, " from %s\n", from->second.second ? from->second.second : "?"); + break; + } + unsigned int pflags = reinterpret_cast(parent)[-1]; + fprintf(stderr, " held by %p flags=%08x %s %s\n", parent, pflags, + (pflags & IMMIX_ALLOC_MARK_ID) == hx::gMarkID ? "marked" : "unmarked", + typeid(*parent).name()); + at = parent; + } + fprintf(stderr, " type %s\n", entry.second ? typeid(*static_cast(entry.first)).name() : "(allocation)"); fflush(stderr); abort(); } diff --git a/src/hx/thread/ThreadImpl.cpp b/src/hx/thread/ThreadImpl.cpp index 922a8b0fb..0c5e76106 100644 --- a/src/hx/thread/ThreadImpl.cpp +++ b/src/hx/thread/ThreadImpl.cpp @@ -161,25 +161,35 @@ int hx::thread::Thread_obj::id() } } +// An incremental slice can find the new object on this stack while one of +// these is being allocated, and the cycle can end at the next allocation. The +// mutation scope has that last stop scan the object again, so the arrays +// stored after it was found are kept too. +void hx::thread::ThreadImpl_obj::makeArrays() +{ + HX_INCREMENTAL_MUTATION(this); + scratch = Array(std::numeric_limits::max(), std::numeric_limits::max()); + slots = Array(0, 0); + localSlots = Array(0, 0); + localGenerations = Array(0, 0); +} + hx::thread::ThreadImpl_obj::ThreadImpl_obj(const int _id) - : scratch(std::numeric_limits::max(), std::numeric_limits::max()) - , slots(0, 0) - , localSlots(0, 0) - , localGenerations(0, 0) - , id(_id) - , native(new Native()) + : id(_id) + , native(nullptr) { + makeArrays(); + native = new Native(); HX_OBJ_WB_NEW_MARKED_OBJECT(this); } hx::thread::ThreadImpl_obj::ThreadImpl_obj(const int _id, Thread_obj::CreateFunction _job, CountingSemaphore _semaphore) - : scratch(std::numeric_limits::max(), std::numeric_limits::max()) - , slots(0, 0) - , localSlots(0, 0) - , localGenerations(0, 0) - , id(_id) - , native(new Native(_job, this, _semaphore)) + : id(_id) + , native(nullptr) { + // Before the thread starts, which reads them + makeArrays(); + native = new Native(_job, this, _semaphore); HX_OBJ_WB_NEW_MARKED_OBJECT(this); } diff --git a/src/hx/thread/ThreadImpl.hpp b/src/hx/thread/ThreadImpl.hpp index 9c7cd08a3..dfbcf3a28 100644 --- a/src/hx/thread/ThreadImpl.hpp +++ b/src/hx/thread/ThreadImpl.hpp @@ -28,6 +28,8 @@ namespace hx Array localSlots; Array localGenerations; + void makeArrays(); + public: friend class Scratch; diff --git a/test/regression/IncrementalGcDequeMoves/Main.hx b/test/regression/IncrementalGcDequeMoves/Main.hx new file mode 100644 index 000000000..7b88f1881 --- /dev/null +++ b/test/regression/IncrementalGcDequeMoves/Main.hx @@ -0,0 +1,110 @@ +import cpp.vm.Gc; +import sys.thread.Deque; +#if HXCPP_GC_INCREMENTAL +import hxcpp.gc.Incremental; +#end + +class Node { + public var value:Int; + public function new(value:Int) this.value = value; +} + +// A Deque is an array that moves its items itself: a pop compacts the queue +// once the consumed part dominates, and a push at the front opens room by +// moving the items up. An incremental cycle scans a long array in chunks, so +// an item moved into a chunk already scanned has to be found some other way. +class Main { + static inline var COUNT = 32768; + static var queue:Deque; + + static function fill() { + queue = new Deque(); + for (i in 0...COUNT) + queue.add(new Node(i)); + Gc.run(true); + } + + #if HXCPP_GC_INCREMENTAL + static function queuedChunks():Int { + return untyped __cpp__("__hxcpp_gc_incremental_test(9, nullptr)"); + } + + // Stops a cycle when markers have taken some of the queue's first batch of + // chunks and the rest wait, so the second half has not been queued yet + static function stopPartway() { + untyped __cpp__("__hxcpp_gc_incremental_test(0, nullptr)"); + var first = 0; + for (_ in 0...10000) { + Incremental.step(0.1); + var queued = queuedChunks(); + if (queued > first) + first = queued; + else if (queued > 0 && queued < first) + break; + } + var queued = queuedChunks(); + if (queued == 0 || queued >= first || !Incremental.isActive()) + throw "the queue was not partly scanned"; + untyped __cpp__("__hxcpp_gc_incremental_test(4, nullptr)"); + } + + static function finishCycle() { + var unfinished = true; + for (_ in 0...10000) { + unfinished = Incremental.step(0.1); + if (!unfinished) + break; + } + if (unfinished) + throw "cycle did not finish"; + } + #end + + static function drain(first:Int):Bool { + var valid = true; + var expected = first; + while (true) { + var node = queue.pop(false); + if (node == null) + break; + if (node.value != expected++) + valid = false; + } + return valid && expected == COUNT; + } + + static function main() { + Gc.enable(false); + + // Popping half the queue moves the second half, which no chunk has + // reached, down over chunks that are done + fill(); + #if HXCPP_GC_INCREMENTAL + stopPartway(); + #end + var valid = true; + for (i in 0...COUNT >> 1) + if (queue.pop(false).value != i) + valid = false; + #if HXCPP_GC_INCREMENTAL + finishCycle(); + #end + Gc.run(true); + Sys.println("items a pop moved survive: " + (drain(COUNT >> 1) && valid)); + + // Pushing at the front moves every item up, past chunks done already + fill(); + #if HXCPP_GC_INCREMENTAL + stopPartway(); + #end + queue.push(new Node(-1)); + #if HXCPP_GC_INCREMENTAL + finishCycle(); + #end + Gc.run(true); + valid = queue.pop(false).value == -1; + Sys.println("items a push at the front moved survive: " + (drain(0) && valid)); + + Gc.enable(true); + } +} diff --git a/test/regression/IncrementalGcDequeMoves/build.hxml b/test/regression/IncrementalGcDequeMoves/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/IncrementalGcDequeMoves/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/IncrementalGcDequeMoves/stdout.txt b/test/regression/IncrementalGcDequeMoves/stdout.txt new file mode 100644 index 000000000..e457fd26d --- /dev/null +++ b/test/regression/IncrementalGcDequeMoves/stdout.txt @@ -0,0 +1,2 @@ +items a pop moved survive: true +items a push at the front moved survive: true