From 4e676a392e860b9500815432600cb29a82bf9671 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:38:38 -0500 Subject: [PATCH 1/5] Fail and end the build cleanly on compile thread errors Two ways a parallel compile went wrong when something on a compile thread threw or exited: - When a compile failed, Compiler deleted the old object before calling setThreadError. If the delete threw, as it does for a read-only object or one another process has open on Windows, the error was never recorded, and the thread's catch only warned when no error had been recorded, so the build linked the old object and exited 0. The error is recorded first, the delete is guarded, and the catch records an error for anything thrown. - Tools.exit on a compile thread ran the exit hooks there, and ThreadPool's hook waits on the calling thread's own queue, which never fills, while the main thread waited for that thread: the build hung. HXCPP_EXIT_ON_ERROR on a compile error and any Log.error on a compile thread did this. On a compile thread, Tools.exit records the code and ends the job, and the main thread exits when the jobs have. HXCPP_EXIT_ON_ERROR is also read once the arguments and Options.txt are in, not only from the environment. Test: test/compile_errors/check.sh, run in CI, builds three projects that must fail: HXCPP_EXIT_ON_ERROR with a compile error and two optimisation tags on one file, which hung before, and on Windows a compile error over a read-only old object, which exited 0 before. All three fail within seconds now, on Windows x64. Fixes #471 --- .github/workflows/test.yml | 5 ++ Changes.md | 1 + test/compile_errors/check.sh | 71 ++++++++++++++++++++++++++++ test/compile_errors/error/Main.hx | 5 ++ test/compile_errors/error/build.hxml | 2 + test/compile_errors/tags/Main.hx | 6 +++ test/compile_errors/tags/build.hxml | 2 + tools/hxcpp/BuildTool.hx | 3 ++ tools/hxcpp/Compiler.hx | 11 ++++- tools/hxcpp/ThreadPool.hx | 27 +++++++++-- tools/hxcpp/Tools.hx | 9 ++++ 11 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 test/compile_errors/check.sh create mode 100644 test/compile_errors/error/Main.hx create mode 100644 test/compile_errors/error/build.hxml create mode 100644 test/compile_errors/tags/Main.hx create mode 100644 test/compile_errors/tags/build.hxml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 007a0b254..60b9e105c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -199,6 +199,11 @@ jobs: run: | haxelib run hxcpp xml/Build.xml -options options.txt test "$(./value | tr -d '\r')" = 42 + - name: build tool errors on compile threads + continue-on-error: ${{ inputs.warn_only }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + shell: bash + run: bash test/compile_errors/check.sh - name: build tool MSVC setup scripts with ! in the Visual Studio path continue-on-error: ${{ inputs.warn_only }} if: ${{ !cancelled() && steps.setup.outcome == 'success' && runner.os == 'Windows' }} diff --git a/Changes.md b/Changes.md index 54466e163..49dc1f9d9 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating diff --git a/test/compile_errors/check.sh b/test/compile_errors/check.sh new file mode 100644 index 000000000..f2808d632 --- /dev/null +++ b/test/compile_errors/check.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# libhxcpp #471: errors on the compile threads. +# - A failed compile whose old object could not be deleted, as when it is +# read-only, linked that object and exited 0. +# - Exiting from a compile thread, as HXCPP_EXIT_ON_ERROR does on a compile +# error, or an error the build tool reports there, hung the build. +# Each build here must fail, and within the time limit. +cd "$(dirname "$0")" +failed=0 + +# Runs a build in the background and fails it after $limit seconds +limit=600 +build() { + "$@" > build.log 2>&1 & + local pid=$! waited=0 + while kill -0 $pid 2>/dev/null; do + if [ $waited -ge $limit ]; then + kill $pid 2>/dev/null + echo "hung" + return 0 + fi + sleep 1 + waited=$((waited+1)) + done + wait $pid + echo "exit $?" +} + +expect_failure() { + local name=$1; shift + local result + result=$(build "$@") + if [ "$result" = "exit 0" ] || [ "$result" = "hung" ]; then + echo "::error::$name: $result, where it should fail" + tail -20 build.log + failed=1 + else + echo "$name: failed as it should ($result)" + fi +} + +(cd error && HXCPP_EXIT_ON_ERROR=1 expect_failure "HXCPP_EXIT_ON_ERROR in the environment" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1 +(cd error && expect_failure "-D HXCPP_EXIT_ON_ERROR" haxe build.hxml -D HXCPP_EXIT_ON_ERROR -D "$HXCPP_ARCH_FLAG") || failed=1 +(cd tags && expect_failure "two optimisation tags" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1 + +# Windows only: elsewhere a read-only file can still be deleted +if [ "$OS" = "Windows_NT" ]; then + ( + cd error + rm -rf ro && mkdir ro && cp build.hxml ro/ + cd ro + printf 'class Main {\n\tstatic function main() {\n\t\tSys.println("old code");\n\t}\n}\n' > Main.hx + haxe build.hxml -D "$HXCPP_ARCH_FLAG" > first.log 2>&1 || { echo "::error::first build failed"; cat first.log; exit 1; } + obj=$(ls bin/obj/*/*_Main.obj | head -1) + attrib +R "$(cygpath -w "$obj")" + # Newer than the object by more than the build tool's one second + sleep 2 + cp ../Main.hx Main.hx + expect_failure "a compile error over a read-only object" haxe build.hxml -D "$HXCPP_ARCH_FLAG" + # The old object must not be linked: the link itself may then fail or not + if grep -q 'Link:' build.log; then + echo "::error::a compile error over a read-only object: linked the old object" + failed=1 + fi + status=$failed + attrib -R "$(cygpath -w "$obj")" + exit $status + ) || failed=1 +fi + +exit $failed diff --git a/test/compile_errors/error/Main.hx b/test/compile_errors/error/Main.hx new file mode 100644 index 000000000..04e9c83f8 --- /dev/null +++ b/test/compile_errors/error/Main.hx @@ -0,0 +1,5 @@ +class Main { + static function main() { + untyped __cpp__("this_is_not_cpp();"); + } +} diff --git a/test/compile_errors/error/build.hxml b/test/compile_errors/error/build.hxml new file mode 100644 index 000000000..9553fac55 --- /dev/null +++ b/test/compile_errors/error/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-main Main diff --git a/test/compile_errors/tags/Main.hx b/test/compile_errors/tags/Main.hx new file mode 100644 index 000000000..766a93591 --- /dev/null +++ b/test/compile_errors/tags/Main.hx @@ -0,0 +1,6 @@ +@:fileXml("tags='haxe,static,optim-none,optim-size'") +class Main { + static function main() { + Sys.println("hi"); + } +} diff --git a/test/compile_errors/tags/build.hxml b/test/compile_errors/tags/build.hxml new file mode 100644 index 000000000..9553fac55 --- /dev/null +++ b/test/compile_errors/tags/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-main Main diff --git a/tools/hxcpp/BuildTool.hx b/tools/hxcpp/BuildTool.hx index e0148c1d9..37776920e 100644 --- a/tools/hxcpp/BuildTool.hx +++ b/tools/hxcpp/BuildTool.hx @@ -118,6 +118,9 @@ class BuildTool inIncludePath:Array, inDirtyList:Array ) { mDefines = inDefines; + // Read once the arguments and Options.txt are in, not only from the + // environment + exitOnThreadError = exitOnThreadError || inDefines.exists("HXCPP_EXIT_ON_ERROR"); mFileGroups = new FileGroups(); mCompiler = null; mStripper = null; diff --git a/tools/hxcpp/Compiler.hx b/tools/hxcpp/Compiler.hx index fa1689d18..261c9fae9 100644 --- a/tools/hxcpp/Compiler.hx +++ b/tools/hxcpp/Compiler.hx @@ -471,9 +471,16 @@ class Compiler cleanTmp(tmpFile); if (err!=0) { - if (FileSystem.exists(obj_name)) - FileSystem.deleteFile(obj_name); + // The error first: when the delete threw, as it does for a + // read-only object or one another process has open, it was + // lost, and the old object was linked (libhxcpp #471) BuildTool.setThreadError(err); + try + { + if (FileSystem.exists(obj_name)) + FileSystem.deleteFile(obj_name); + } + catch(e:Dynamic) { } } else inProgress.fileDone(); diff --git a/tools/hxcpp/ThreadPool.hx b/tools/hxcpp/ThreadPool.hx index 10e8c6247..33da2f616 100755 --- a/tools/hxcpp/ThreadPool.hx +++ b/tools/hxcpp/ThreadPool.hx @@ -1,22 +1,39 @@ #if haxe4 import sys.thread.Thread; import sys.thread.Mutex; + import sys.thread.Tls; #else #if neko import neko.vm.Thread; import neko.vm.Mutex; + import neko.vm.Tls; #else import cpp.vm.Thread; import cpp.vm.Mutex; + import cpp.vm.Tls; #end #end +// Thrown by Tools.exit on a compile thread, to end its job +class WorkerExit +{ + public function new() { } +} + #if cpp import cpp.AtomicInt; #end class ThreadPool { + static var worker = new Tls(); + + // Whether this is one of the compile threads + public static function onWorker():Bool + { + return worker.value == true; + } + var threads:Array; var mainThread:Thread; #if cpp @@ -43,6 +60,7 @@ class ThreadPool function threadLoop(threadId:Int) { + worker.value = true; while(true) { var job:Int->Void = Thread.readMessage(true); @@ -58,9 +76,12 @@ class ThreadPool } catch (error:Dynamic) { - if (BuildTool.threadExitCode!=0) - BuildTool.setThreadError(-1); - else + // Any error ends the build. As a warning, a compile that failed + // and then could not delete its old object went on to link that + // object and exit 0 (libhxcpp #471) + if (BuildTool.threadExitCode==0) + BuildTool.threadExitCode = -1; + if (!Std.isOfType(error, WorkerExit)) Log.warn("Error in compile thread: " + error); } diff --git a/tools/hxcpp/Tools.hx b/tools/hxcpp/Tools.hx index da8e5f466..9c402c2ca 100644 --- a/tools/hxcpp/Tools.hx +++ b/tools/hxcpp/Tools.hx @@ -6,6 +6,15 @@ class Tools { } public static function exit(exitCode:Int) { + // On a compile thread the hooks waited for this thread itself, since one + // waits for every compile thread to finish, so the build hung. The job + // ends here instead, and the main thread exits once the jobs have + // (libhxcpp #471) + if (ThreadPool.onWorker()) { + if (BuildTool.threadExitCode == 0) + BuildTool.threadExitCode = exitCode == 0 ? -1 : exitCode; + throw new ThreadPool.WorkerExit(); + } for (hook in onExit) { hook(exitCode); } From 614ef20f4fc3dfb44943c086197c4f66f5fa5747 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:41:14 -0500 Subject: [PATCH 2/5] Rebuild when a define that changes compile flags changes Without the compile cache, the copy of Options.txt the build tool keeps in the obj folder is all that tells it the compile flags changed, and filterOptions keeps only defines containing "hxcpp" and a few others. Defines the toolchains compile by, or choose files by, were dropped: switching one reused objects built without it. Adding CPPIA_NO_JIT to a -D scriptable build recompiled only Main.cpp and failed to link against the old Cppia.obj (unresolved hx::getJitType), and removing it linked a host without its JIT, saying nothing. The defines the toolchain files test or use in flags, and which no obj folder or other kept define already covers, are kept too: CPPIA_NO_JIT, CPPIA_JIT, ABI, objc, OBJC_ARC, STDLIBCPP, exe_link, O3, LEGACY_MACOSX_SDK, armv6, rpi, winrt, noM32, noM64 and no_shared_libs. Test: a -D scriptable build, then the same with -D CPPIA_NO_JIT, compiled the cppia sources again and linked, then without it again, on Windows x64. Fixes #470 --- Changes.md | 1 + tools/hxcpp/FileGroup.hx | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 49dc1f9d9..219d74f7c 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470) * Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now diff --git a/tools/hxcpp/FileGroup.hx b/tools/hxcpp/FileGroup.hx index 2d0a14aae..27d33b003 100644 --- a/tools/hxcpp/FileGroup.hx +++ b/tools/hxcpp/FileGroup.hx @@ -215,6 +215,13 @@ class FileGroup } } + // Defines that change how the toolchains compile, or what: without the + // cache, this copy of Options.txt is all that tells the objects are stale, + // so switching one of these reused objects built without it, and adding + // CPPIA_NO_JIT failed to link (libhxcpp #470) + static var compileDefines = ["cppia_no_jit", "cppia_jit", "abi", "objc", "objc_arc", "stdlibcpp", "exe_link", "o3", + "legacy_macosx_sdk", "armv6", "rpi", "winrt", "nom32", "nom64", "no_shared_libs"]; + public function filterOptions(contents:String) { // Old-style @@ -237,7 +244,7 @@ class FileGroup } else if (name.indexOf("hxcpp")>=0 || name=="scriptable" || name.indexOf("dll")>=0 || name=="no_console" || name.substr(0,8)=="android-" || name.substr(0,4)=="ndkv" || name=="toolchain" || name=="platform" || - name=="toolchain_version" || name=="android_ndk_root" ) + name=="toolchain_version" || name=="android_ndk_root" || compileDefines.indexOf(name)>=0 ) result.push(def); } From 4c1489b3b6bda99a3b519e186878ce64fd11014a Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:41:15 -0500 Subject: [PATCH 3/5] Fix per-file cxx_standard with a PCH, nvcc groups and NDKV20 and up - A file with its own cxx_standard, which docs/build_xml/Files.md documents, in a group with a precompiled header was given both the file's standard and the PCH built with the group's. MSVC refused (C2855) and clang refuses a PCH built with other language options. Such a file compiles without the PCH. - group.mTags=="nvcc"; compared instead of assigning, so an group kept the tags haxe,static and was given the host compiler's flags, which nvcc takes only through -Xcompiler. The tag is set. - Setup looked for NDKV6 to NDKV19 only, so -DNDKV25 was ignored and the newest NDK under ANDROID_NDK_DIR used. Any NDKV is read, the lowest if several are given. An NDK without meta/platforms.json then stopped the build with "Invalid field access : toString" while warning about it; the warning prints now. Test, on Windows x64: a file with cxx_standard='17' in a PCH group compiles and runs; a stand-in nvcc.exe that prints its arguments gets no host flags; with fake NDKs 25 and 28, -DNDKV25 picks 25 and -DNDKV28 picks 28. Fixes #472 --- Changes.md | 1 + tools/hxcpp/BuildTool.hx | 4 +++- tools/hxcpp/Compiler.hx | 4 +++- tools/hxcpp/Setup.hx | 18 ++++++++++++------ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Changes.md b/Changes.md index 219d74f7c..8ee8ce467 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed three build tool settings that did nothing or failed: a file with its own cxx_standard in a group with a precompiled header failed to compile (C2855 on MSVC), an group was given the host compiler's flags because its tag was compared instead of set, and -DNDKV20 and up were ignored, so the newest NDK was used instead. An NDK without meta/platforms.json no longer stops the build with "Invalid field access : toString" (libhxcpp #472) * Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470) * Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now diff --git a/tools/hxcpp/BuildTool.hx b/tools/hxcpp/BuildTool.hx index 37776920e..dc74fcc65 100644 --- a/tools/hxcpp/BuildTool.hx +++ b/tools/hxcpp/BuildTool.hx @@ -1217,8 +1217,10 @@ class BuildTool case "nvcc" : setupNvcc(); group.mNvcc = true; + // Set, not compared: the group kept the haxe tags, so it was + // given the host compiler's flags (libhxcpp #472) if (group.mTags=="haxe,static") - group.mTags=="nvcc"; + group.mTags="nvcc"; case "objprefix" : group.mObjPrefix = substitute(el.att.value); case "compilervalue" : diff --git a/tools/hxcpp/Compiler.hx b/tools/hxcpp/Compiler.hx index 261c9fae9..691ea224b 100644 --- a/tools/hxcpp/Compiler.hx +++ b/tools/hxcpp/Compiler.hx @@ -275,7 +275,9 @@ class Compiler args = args.concat(mCPPFlags); } - if (isRc || inFile.getTags()!=inFile.mGroup.getTags()) + // The precompiled header is built with the group's standard, and a file + // with its own cannot use it: MSVC and clang both refuse (libhxcpp #472) + if (isRc || inFile.getTags()!=inFile.mGroup.getTags() || inFile.mCxxStandard!=inFile.mGroup.mCxxStandard) allowPch = false; if (inFile.mGroup.isPrecompiled() && allowPch) diff --git a/tools/hxcpp/Setup.hx b/tools/hxcpp/Setup.hx index 759fa1a7b..f2d589e39 100644 --- a/tools/hxcpp/Setup.hx +++ b/tools/hxcpp/Setup.hx @@ -490,16 +490,22 @@ class Setup if (Log.verbose) Log.println(""); + // Any NDKV, the lowest if there are several. Only 6 to 19 were + // looked for, so a request for r20 or later was ignored, and the newest + // NDK found was used instead (libhxcpp #472) var ndkVersion = 0; - for (i in 6...20) + var requested = ~/^NDKV([0-9]+)$/; + for (key in defines.keys()) { - if (defines.exists("NDKV" + i)) + if (requested.match(key)) { - ndkVersion = i; - Log.info("", "\x1b[33;1mRequested Android NDK r" + i + "\x1b[0m"); - break; + var version = Std.parseInt(requested.matched(1)); + if (version!=null && version>=6 && (ndkVersion==0 || version = try { haxe.Json.parse(sys.io.File.getContent(platformsJson)).min; } catch (e:Dynamic) { - Log.warn("Unable to determine minimum supported Android platform: " + e.toString()); + Log.warn("Unable to determine minimum supported Android platform: " + Std.string(e)); null; }; From b21b16896798cb402de5959d85b7728fa7dd9ffb Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:42:51 -0500 Subject: [PATCH 4/5] Depend on every header hxcpp.h and the runtime include The build tool decides what to rebuild by comparing files against haxe-target.xml's depend lists, and the compile cache key hashes only the files listed. Headers hxcpp.h includes, directly or not, were missing from hxcpp-depends (hx/Functions.h, hx/FieldRef.h, hx/ErrorCodes.h, hx/Boot.h, hx/Undefine.h, cpp/CppInt32__.h, cpp/FastIterator.h, cpp/marshal/RootHandle.hpp and hx/Invoker.h), and headers the runtime's sources include were missing from its group (hx/DynamicImpl.h, hx/Memory.h, hx/NekoFunc.h, hx/CFFIJsPrime.h, hx/CFFINekoLoader.h, hx/TelemetryTracy.h and src/hx/FastMutex.h). So editing one, as an update of a haxelib git or dev checkout does, rebuilt nothing, and the cache relinked objects built before it. They are listed, and tools/check-depends.py, run in the Check headers job, fails when a header either reaches is missing again. Test: #error appended to include/hx/Functions.h now fails the next build, where it rebuilt nothing and exited 0 before; check-depends.py passes, and failed on main. Fixes #469 --- .github/workflows/main.yml | 5 +++ Changes.md | 1 + toolchain/haxe-target.xml | 17 ++++++++ tools/check-depends.py | 88 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 tools/check-depends.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b18bb090b..696d7fac7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,11 @@ jobs: include name: Check public headers + # Every header hxcpp.h and the runtime include must be in haxe-target.xml's + # depend lists, or editing it rebuilds nothing (libhxcpp #469) + - name: Check the build tool's depend lists + run: python3 tools/check-depends.py + check-gc-defines: name: Check GC defines runs-on: ubuntu-latest diff --git a/Changes.md b/Changes.md index 8ee8ce467..13102dbbb 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed an edit to some of the runtime's headers rebuilding nothing, as when a haxelib git or dev checkout updates: hxcpp.h includes headers, hx/Functions.h and cpp/FastIterator.h among them, that the build tool's depend lists did not name, so neither the objects compiled with them nor the compile cache were checked against them, and a stale precompiled header stopped clang builds. Every header hxcpp.h and the runtime include is listed now, and CI checks that it stays so (libhxcpp #469) * Fixed three build tool settings that did nothing or failed: a file with its own cxx_standard in a group with a precompiled header failed to compile (C2855 on MSVC), an group was given the host compiler's flags because its tag was compared instead of set, and -DNDKV20 and up were ignored, so the newest NDK was used instead. An NDK without meta/platforms.json no longer stops the build with "Invalid field access : toString" (libhxcpp #472) * Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470) * Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471) diff --git a/toolchain/haxe-target.xml b/toolchain/haxe-target.xml index da12bc909..4a17607e1 100644 --- a/toolchain/haxe-target.xml +++ b/toolchain/haxe-target.xml @@ -83,6 +83,16 @@ + + + + + + + + + +
@@ -189,6 +199,13 @@ + + + + + + + diff --git a/tools/check-depends.py b/tools/check-depends.py new file mode 100644 index 000000000..48c834931 --- /dev/null +++ b/tools/check-depends.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# Checks that toolchain/haxe-target.xml's depend lists name every header the +# runtime and generated code include, so editing one rebuilds what uses it. +# hxcpp-depends must hold everything hxcpp.h includes, directly or not, and the +# runtime group, with it, everything the runtime's sources include from this +# repository. A header missing there was never compared, so a change to it +# rebuilt nothing, and the compile cache reused objects built before it +# (libhxcpp #469). +# python3 tools/check-depends.py (from the repository root) +import os +import re +import sys + +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +include_dir = os.path.join(root, 'include') +target = open(os.path.join(root, 'toolchain', 'haxe-target.xml'), encoding='utf-8').read() +include_re = re.compile(r'^\s*#\s*include\s*[<"]([^>"]+)[>"]', re.M) + + +def group(name): + m = re.search(r']*>(.*?)' % re.escape(name), target, re.S) + if not m: + sys.exit('no files group %s in haxe-target.xml' % name) + return m.group(1) + + +def depends(body): + return {os.path.normpath(os.path.join(root, d.replace('${HXCPP}/', ''))) + for d in re.findall(r' found hx/Memory.h + directory, name = os.path.split(path) + return os.path.isfile(path) and name in os.listdir(directory) + + +def resolve(name, from_dir): + for base in (from_dir, include_dir): + path = os.path.normpath(os.path.join(base, name)) + if path.startswith(root + os.sep) and exists_exact(path): + return path + return None + + +def closure(starts): + seen = set() + todo = list(starts) + while todo: + path = todo.pop() + if path in seen: + continue + seen.add(path) + text = open(path, encoding='utf-8', errors='replace').read() + for name in include_re.findall(text): + found = resolve(name, os.path.dirname(path)) + # Third-party sources keep their own headers, outside these lists + if found and 'thirdparty' not in found: + todo.append(found) + return seen + + +rel = lambda p: os.path.relpath(p, root).replace(os.sep, '/') +failed = False + +common = depends(group('hxcpp-depends')) +need = closure([os.path.join(include_dir, 'hxcpp.h')]) +missing = sorted(rel(p) for p in need - common) +if missing: + failed = True + print('hxcpp-depends misses headers hxcpp.h includes:') + for m in missing: + print(' ' + m) + +runtime = group('runtime') +sources = [os.path.join(root, f) for f in re.findall(r' Date: Thu, 19 Nov 2026 09:56:10 -0500 Subject: [PATCH 5/5] Trim the compile cache at most once in ten minutes Before every build with the compile cache, the build tool stat'ed every file in it to keep it under HXCPP_CACHE_MB, even when the build had nothing to compile. With tens of thousands of files that took up to seconds a build. It trims when the cache's .trimmed stamp is older than ten minutes, and writes the stamp when it does. Measured with HXCPP_TIMES, a build with nothing to compile and a cache of 29792 files, three runs each on Windows x64: "setup cache" took 1138, 142 and 98 ms before, and 162 (the trim), 14 and 3 ms after. The other fixed cost in #473, running the MSVC setup script on every build (2.7 to 9.4 s in the same runs), is not changed here. Part of #473 --- Changes.md | 1 + tools/hxcpp/BuildTool.hx | 2 +- tools/hxcpp/CompileCache.hx | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 13102dbbb..2f04da150 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Builds with the compile cache no longer size the whole cache each time: with 29792 files that took 98 ms to over a second before every build, even one with nothing to compile, and takes 3 to 14 ms now. The cache is trimmed to HXCPP_CACHE_MB at most once in ten minutes (libhxcpp #473) * Fixed an edit to some of the runtime's headers rebuilding nothing, as when a haxelib git or dev checkout updates: hxcpp.h includes headers, hx/Functions.h and cpp/FastIterator.h among them, that the build tool's depend lists did not name, so neither the objects compiled with them nor the compile cache were checked against them, and a stale precompiled header stopped clang builds. Every header hxcpp.h and the runtime include is listed now, and CI checks that it stays so (libhxcpp #469) * Fixed three build tool settings that did nothing or failed: a file with its own cxx_standard in a group with a precompiled header failed to compile (C2855 on MSVC), an group was given the host compiler's flags because its tag was compared instead of set, and -DNDKV20 and up were ignored, so the newest NDK was used instead. An NDK without meta/platforms.json no longer stops the build with "Invalid field access : toString" (libhxcpp #472) * Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470) diff --git a/tools/hxcpp/BuildTool.hx b/tools/hxcpp/BuildTool.hx index dc74fcc65..2198179c2 100644 --- a/tools/hxcpp/BuildTool.hx +++ b/tools/hxcpp/BuildTool.hx @@ -311,7 +311,7 @@ class BuildTool if (cached) { var cacheSize = mDefines.exists("HXCPP_CACHE_MB") ? Std.parseInt( mDefines.get("HXCPP_CACHE_MB") ) : 1000; - if (cacheSize!=null && cacheSize>0) + if (cacheSize!=null && cacheSize>0 && CompileCache.dueForTrim()) CompileCache.clear(0,cacheSize,false,null); } diff --git a/tools/hxcpp/CompileCache.hx b/tools/hxcpp/CompileCache.hx index a1041a886..9392f3e1b 100644 --- a/tools/hxcpp/CompileCache.hx +++ b/tools/hxcpp/CompileCache.hx @@ -93,6 +93,22 @@ class CompileCache return dir + "/" + hash.substr(2) + inExt; } + // Sizing the cache stats every file in it, which took seconds once it held + // tens of thousands, and ran before every build, even one with nothing to + // compile (libhxcpp #473). Once in ten minutes keeps it near its limit + public static function dueForTrim():Bool + { + var stamp = compileCache + "/.trimmed"; + try + { + if (FileSystem.exists(stamp) && Date.now().getTime() - FileSystem.stat(stamp).mtime.getTime() < 10 * 60 * 1000.0) + return false; + sys.io.File.saveContent(stamp, ""); + } + catch(e:Dynamic) { } + return true; + } + public static function clear(inDays:Int,inMB:Int,inLogInfo:Bool,inProject:String) { try