diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b18bb090..696d7fac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,11 @@ jobs: include name: Check public headers + # Every header hxcpp.h and the runtime include must be in haxe-target.xml's + # depend lists, or editing it rebuilds nothing (libhxcpp #469) + - name: Check the build tool's depend lists + run: python3 tools/check-depends.py + check-gc-defines: name: Check GC defines runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 007a0b25..60b9e105 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -199,6 +199,11 @@ jobs: run: | haxelib run hxcpp xml/Build.xml -options options.txt test "$(./value | tr -d '\r')" = 42 + - name: build tool errors on compile threads + continue-on-error: ${{ inputs.warn_only }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + shell: bash + run: bash test/compile_errors/check.sh - name: build tool MSVC setup scripts with ! in the Visual Studio path continue-on-error: ${{ inputs.warn_only }} if: ${{ !cancelled() && steps.setup.outcome == 'success' && runner.os == 'Windows' }} diff --git a/Changes.md b/Changes.md index 54466e16..2f04da15 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,11 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Builds with the compile cache no longer size the whole cache each time: with 29792 files that took 98 ms to over a second before every build, even one with nothing to compile, and takes 3 to 14 ms now. The cache is trimmed to HXCPP_CACHE_MB at most once in ten minutes (libhxcpp #473) +* Fixed an edit to some of the runtime's headers rebuilding nothing, as when a haxelib git or dev checkout updates: hxcpp.h includes headers, hx/Functions.h and cpp/FastIterator.h among them, that the build tool's depend lists did not name, so neither the objects compiled with them nor the compile cache were checked against them, and a stale precompiled header stopped clang builds. Every header hxcpp.h and the runtime include is listed now, and CI checks that it stays so (libhxcpp #469) +* Fixed three build tool settings that did nothing or failed: a file with its own cxx_standard in a group with a precompiled header failed to compile (C2855 on MSVC), an group was given the host compiler's flags because its tag was compared instead of set, and -DNDKV20 and up were ignored, so the newest NDK was used instead. An NDK without meta/platforms.json no longer stops the build with "Invalid field access : toString" (libhxcpp #472) +* Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470) +* Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating diff --git a/test/compile_errors/check.sh b/test/compile_errors/check.sh new file mode 100644 index 00000000..f2808d63 --- /dev/null +++ b/test/compile_errors/check.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# libhxcpp #471: errors on the compile threads. +# - A failed compile whose old object could not be deleted, as when it is +# read-only, linked that object and exited 0. +# - Exiting from a compile thread, as HXCPP_EXIT_ON_ERROR does on a compile +# error, or an error the build tool reports there, hung the build. +# Each build here must fail, and within the time limit. +cd "$(dirname "$0")" +failed=0 + +# Runs a build in the background and fails it after $limit seconds +limit=600 +build() { + "$@" > build.log 2>&1 & + local pid=$! waited=0 + while kill -0 $pid 2>/dev/null; do + if [ $waited -ge $limit ]; then + kill $pid 2>/dev/null + echo "hung" + return 0 + fi + sleep 1 + waited=$((waited+1)) + done + wait $pid + echo "exit $?" +} + +expect_failure() { + local name=$1; shift + local result + result=$(build "$@") + if [ "$result" = "exit 0" ] || [ "$result" = "hung" ]; then + echo "::error::$name: $result, where it should fail" + tail -20 build.log + failed=1 + else + echo "$name: failed as it should ($result)" + fi +} + +(cd error && HXCPP_EXIT_ON_ERROR=1 expect_failure "HXCPP_EXIT_ON_ERROR in the environment" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1 +(cd error && expect_failure "-D HXCPP_EXIT_ON_ERROR" haxe build.hxml -D HXCPP_EXIT_ON_ERROR -D "$HXCPP_ARCH_FLAG") || failed=1 +(cd tags && expect_failure "two optimisation tags" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1 + +# Windows only: elsewhere a read-only file can still be deleted +if [ "$OS" = "Windows_NT" ]; then + ( + cd error + rm -rf ro && mkdir ro && cp build.hxml ro/ + cd ro + printf 'class Main {\n\tstatic function main() {\n\t\tSys.println("old code");\n\t}\n}\n' > Main.hx + haxe build.hxml -D "$HXCPP_ARCH_FLAG" > first.log 2>&1 || { echo "::error::first build failed"; cat first.log; exit 1; } + obj=$(ls bin/obj/*/*_Main.obj | head -1) + attrib +R "$(cygpath -w "$obj")" + # Newer than the object by more than the build tool's one second + sleep 2 + cp ../Main.hx Main.hx + expect_failure "a compile error over a read-only object" haxe build.hxml -D "$HXCPP_ARCH_FLAG" + # The old object must not be linked: the link itself may then fail or not + if grep -q 'Link:' build.log; then + echo "::error::a compile error over a read-only object: linked the old object" + failed=1 + fi + status=$failed + attrib -R "$(cygpath -w "$obj")" + exit $status + ) || failed=1 +fi + +exit $failed diff --git a/test/compile_errors/error/Main.hx b/test/compile_errors/error/Main.hx new file mode 100644 index 00000000..04e9c83f --- /dev/null +++ b/test/compile_errors/error/Main.hx @@ -0,0 +1,5 @@ +class Main { + static function main() { + untyped __cpp__("this_is_not_cpp();"); + } +} diff --git a/test/compile_errors/error/build.hxml b/test/compile_errors/error/build.hxml new file mode 100644 index 00000000..9553fac5 --- /dev/null +++ b/test/compile_errors/error/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-main Main diff --git a/test/compile_errors/tags/Main.hx b/test/compile_errors/tags/Main.hx new file mode 100644 index 00000000..766a9359 --- /dev/null +++ b/test/compile_errors/tags/Main.hx @@ -0,0 +1,6 @@ +@:fileXml("tags='haxe,static,optim-none,optim-size'") +class Main { + static function main() { + Sys.println("hi"); + } +} diff --git a/test/compile_errors/tags/build.hxml b/test/compile_errors/tags/build.hxml new file mode 100644 index 00000000..9553fac5 --- /dev/null +++ b/test/compile_errors/tags/build.hxml @@ -0,0 +1,2 @@ +-cpp bin +-main Main diff --git a/toolchain/haxe-target.xml b/toolchain/haxe-target.xml index da12bc90..4a17607e 100644 --- a/toolchain/haxe-target.xml +++ b/toolchain/haxe-target.xml @@ -83,6 +83,16 @@ + + + + + + + + + +
@@ -189,6 +199,13 @@ + + + + + + + diff --git a/tools/check-depends.py b/tools/check-depends.py new file mode 100644 index 00000000..48c83493 --- /dev/null +++ b/tools/check-depends.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# Checks that toolchain/haxe-target.xml's depend lists name every header the +# runtime and generated code include, so editing one rebuilds what uses it. +# hxcpp-depends must hold everything hxcpp.h includes, directly or not, and the +# runtime group, with it, everything the runtime's sources include from this +# repository. A header missing there was never compared, so a change to it +# rebuilt nothing, and the compile cache reused objects built before it +# (libhxcpp #469). +# python3 tools/check-depends.py (from the repository root) +import os +import re +import sys + +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +include_dir = os.path.join(root, 'include') +target = open(os.path.join(root, 'toolchain', 'haxe-target.xml'), encoding='utf-8').read() +include_re = re.compile(r'^\s*#\s*include\s*[<"]([^>"]+)[>"]', re.M) + + +def group(name): + m = re.search(r']*>(.*?)' % re.escape(name), target, re.S) + if not m: + sys.exit('no files group %s in haxe-target.xml' % name) + return m.group(1) + + +def depends(body): + return {os.path.normpath(os.path.join(root, d.replace('${HXCPP}/', ''))) + for d in re.findall(r' found hx/Memory.h + directory, name = os.path.split(path) + return os.path.isfile(path) and name in os.listdir(directory) + + +def resolve(name, from_dir): + for base in (from_dir, include_dir): + path = os.path.normpath(os.path.join(base, name)) + if path.startswith(root + os.sep) and exists_exact(path): + return path + return None + + +def closure(starts): + seen = set() + todo = list(starts) + while todo: + path = todo.pop() + if path in seen: + continue + seen.add(path) + text = open(path, encoding='utf-8', errors='replace').read() + for name in include_re.findall(text): + found = resolve(name, os.path.dirname(path)) + # Third-party sources keep their own headers, outside these lists + if found and 'thirdparty' not in found: + todo.append(found) + return seen + + +rel = lambda p: os.path.relpath(p, root).replace(os.sep, '/') +failed = False + +common = depends(group('hxcpp-depends')) +need = closure([os.path.join(include_dir, 'hxcpp.h')]) +missing = sorted(rel(p) for p in need - common) +if missing: + failed = True + print('hxcpp-depends misses headers hxcpp.h includes:') + for m in missing: + print(' ' + m) + +runtime = group('runtime') +sources = [os.path.join(root, f) for f in re.findall(r', inDirtyList:Array ) { mDefines = inDefines; + // Read once the arguments and Options.txt are in, not only from the + // environment + exitOnThreadError = exitOnThreadError || inDefines.exists("HXCPP_EXIT_ON_ERROR"); mFileGroups = new FileGroups(); mCompiler = null; mStripper = null; @@ -308,7 +311,7 @@ class BuildTool if (cached) { var cacheSize = mDefines.exists("HXCPP_CACHE_MB") ? Std.parseInt( mDefines.get("HXCPP_CACHE_MB") ) : 1000; - if (cacheSize!=null && cacheSize>0) + if (cacheSize!=null && cacheSize>0 && CompileCache.dueForTrim()) CompileCache.clear(0,cacheSize,false,null); } @@ -1214,8 +1217,10 @@ class BuildTool case "nvcc" : setupNvcc(); group.mNvcc = true; + // Set, not compared: the group kept the haxe tags, so it was + // given the host compiler's flags (libhxcpp #472) if (group.mTags=="haxe,static") - group.mTags=="nvcc"; + group.mTags="nvcc"; case "objprefix" : group.mObjPrefix = substitute(el.att.value); case "compilervalue" : diff --git a/tools/hxcpp/CompileCache.hx b/tools/hxcpp/CompileCache.hx index a1041a88..9392f3e1 100644 --- a/tools/hxcpp/CompileCache.hx +++ b/tools/hxcpp/CompileCache.hx @@ -93,6 +93,22 @@ class CompileCache return dir + "/" + hash.substr(2) + inExt; } + // Sizing the cache stats every file in it, which took seconds once it held + // tens of thousands, and ran before every build, even one with nothing to + // compile (libhxcpp #473). Once in ten minutes keeps it near its limit + public static function dueForTrim():Bool + { + var stamp = compileCache + "/.trimmed"; + try + { + if (FileSystem.exists(stamp) && Date.now().getTime() - FileSystem.stat(stamp).mtime.getTime() < 10 * 60 * 1000.0) + return false; + sys.io.File.saveContent(stamp, ""); + } + catch(e:Dynamic) { } + return true; + } + public static function clear(inDays:Int,inMB:Int,inLogInfo:Bool,inProject:String) { try diff --git a/tools/hxcpp/Compiler.hx b/tools/hxcpp/Compiler.hx index fa1689d1..691ea224 100644 --- a/tools/hxcpp/Compiler.hx +++ b/tools/hxcpp/Compiler.hx @@ -275,7 +275,9 @@ class Compiler args = args.concat(mCPPFlags); } - if (isRc || inFile.getTags()!=inFile.mGroup.getTags()) + // The precompiled header is built with the group's standard, and a file + // with its own cannot use it: MSVC and clang both refuse (libhxcpp #472) + if (isRc || inFile.getTags()!=inFile.mGroup.getTags() || inFile.mCxxStandard!=inFile.mGroup.mCxxStandard) allowPch = false; if (inFile.mGroup.isPrecompiled() && allowPch) @@ -471,9 +473,16 @@ class Compiler cleanTmp(tmpFile); if (err!=0) { - if (FileSystem.exists(obj_name)) - FileSystem.deleteFile(obj_name); + // The error first: when the delete threw, as it does for a + // read-only object or one another process has open, it was + // lost, and the old object was linked (libhxcpp #471) BuildTool.setThreadError(err); + try + { + if (FileSystem.exists(obj_name)) + FileSystem.deleteFile(obj_name); + } + catch(e:Dynamic) { } } else inProgress.fileDone(); diff --git a/tools/hxcpp/FileGroup.hx b/tools/hxcpp/FileGroup.hx index 2d0a14aa..27d33b00 100644 --- a/tools/hxcpp/FileGroup.hx +++ b/tools/hxcpp/FileGroup.hx @@ -215,6 +215,13 @@ class FileGroup } } + // Defines that change how the toolchains compile, or what: without the + // cache, this copy of Options.txt is all that tells the objects are stale, + // so switching one of these reused objects built without it, and adding + // CPPIA_NO_JIT failed to link (libhxcpp #470) + static var compileDefines = ["cppia_no_jit", "cppia_jit", "abi", "objc", "objc_arc", "stdlibcpp", "exe_link", "o3", + "legacy_macosx_sdk", "armv6", "rpi", "winrt", "nom32", "nom64", "no_shared_libs"]; + public function filterOptions(contents:String) { // Old-style @@ -237,7 +244,7 @@ class FileGroup } else if (name.indexOf("hxcpp")>=0 || name=="scriptable" || name.indexOf("dll")>=0 || name=="no_console" || name.substr(0,8)=="android-" || name.substr(0,4)=="ndkv" || name=="toolchain" || name=="platform" || - name=="toolchain_version" || name=="android_ndk_root" ) + name=="toolchain_version" || name=="android_ndk_root" || compileDefines.indexOf(name)>=0 ) result.push(def); } diff --git a/tools/hxcpp/Setup.hx b/tools/hxcpp/Setup.hx index 759fa1a7..f2d589e3 100644 --- a/tools/hxcpp/Setup.hx +++ b/tools/hxcpp/Setup.hx @@ -490,16 +490,22 @@ class Setup if (Log.verbose) Log.println(""); + // Any NDKV, the lowest if there are several. Only 6 to 19 were + // looked for, so a request for r20 or later was ignored, and the newest + // NDK found was used instead (libhxcpp #472) var ndkVersion = 0; - for (i in 6...20) + var requested = ~/^NDKV([0-9]+)$/; + for (key in defines.keys()) { - if (defines.exists("NDKV" + i)) + if (requested.match(key)) { - ndkVersion = i; - Log.info("", "\x1b[33;1mRequested Android NDK r" + i + "\x1b[0m"); - break; + var version = Std.parseInt(requested.matched(1)); + if (version!=null && version>=6 && (ndkVersion==0 || version = try { haxe.Json.parse(sys.io.File.getContent(platformsJson)).min; } catch (e:Dynamic) { - Log.warn("Unable to determine minimum supported Android platform: " + e.toString()); + Log.warn("Unable to determine minimum supported Android platform: " + Std.string(e)); null; }; diff --git a/tools/hxcpp/ThreadPool.hx b/tools/hxcpp/ThreadPool.hx index 10e8c624..33da2f61 100755 --- a/tools/hxcpp/ThreadPool.hx +++ b/tools/hxcpp/ThreadPool.hx @@ -1,22 +1,39 @@ #if haxe4 import sys.thread.Thread; import sys.thread.Mutex; + import sys.thread.Tls; #else #if neko import neko.vm.Thread; import neko.vm.Mutex; + import neko.vm.Tls; #else import cpp.vm.Thread; import cpp.vm.Mutex; + import cpp.vm.Tls; #end #end +// Thrown by Tools.exit on a compile thread, to end its job +class WorkerExit +{ + public function new() { } +} + #if cpp import cpp.AtomicInt; #end class ThreadPool { + static var worker = new Tls(); + + // Whether this is one of the compile threads + public static function onWorker():Bool + { + return worker.value == true; + } + var threads:Array; var mainThread:Thread; #if cpp @@ -43,6 +60,7 @@ class ThreadPool function threadLoop(threadId:Int) { + worker.value = true; while(true) { var job:Int->Void = Thread.readMessage(true); @@ -58,9 +76,12 @@ class ThreadPool } catch (error:Dynamic) { - if (BuildTool.threadExitCode!=0) - BuildTool.setThreadError(-1); - else + // Any error ends the build. As a warning, a compile that failed + // and then could not delete its old object went on to link that + // object and exit 0 (libhxcpp #471) + if (BuildTool.threadExitCode==0) + BuildTool.threadExitCode = -1; + if (!Std.isOfType(error, WorkerExit)) Log.warn("Error in compile thread: " + error); } diff --git a/tools/hxcpp/Tools.hx b/tools/hxcpp/Tools.hx index da8e5f46..9c402c2c 100644 --- a/tools/hxcpp/Tools.hx +++ b/tools/hxcpp/Tools.hx @@ -6,6 +6,15 @@ class Tools { } public static function exit(exitCode:Int) { + // On a compile thread the hooks waited for this thread itself, since one + // waits for every compile thread to finish, so the build hung. The job + // ends here instead, and the main thread exits once the jobs have + // (libhxcpp #471) + if (ThreadPool.onWorker()) { + if (BuildTool.threadExitCode == 0) + BuildTool.threadExitCode = exitCode == 0 ? -1 : exitCode; + throw new ThreadPool.WorkerExit(); + } for (hook in onExit) { hook(exitCode); }