Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ jobs:
include
name: Check public headers

# Every header hxcpp.h and the runtime include must be in haxe-target.xml's
# depend lists, or editing it rebuilds nothing (libhxcpp #469)
- name: Check the build tool's depend lists
run: python3 tools/check-depends.py

check-gc-defines:
name: Check GC defines
runs-on: ubuntu-latest
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ jobs:
run: |
haxelib run hxcpp xml/Build.xml -options options.txt
test "$(./value | tr -d '\r')" = 42
- name: build tool errors on compile threads
continue-on-error: ${{ inputs.warn_only }}
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
shell: bash
run: bash test/compile_errors/check.sh
- name: build tool MSVC setup scripts with ! in the Visual Studio path
continue-on-error: ${{ inputs.warn_only }}
if: ${{ !cancelled() && steps.setup.outcome == 'success' && runner.os == 'Windows' }}
Expand Down
5 changes: 5 additions & 0 deletions Changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@
libhxcpp 6.0.0 (unreleased)
------------------------------------------------------------

* Builds with the compile cache no longer size the whole cache each time: with 29792 files that took 98 ms to over a second before every build, even one with nothing to compile, and takes 3 to 14 ms now. The cache is trimmed to HXCPP_CACHE_MB at most once in ten minutes (libhxcpp #473)
* Fixed an edit to some of the runtime's headers rebuilding nothing, as when a haxelib git or dev checkout updates: hxcpp.h includes headers, hx/Functions.h and cpp/FastIterator.h among them, that the build tool's depend lists did not name, so neither the objects compiled with them nor the compile cache were checked against them, and a stale precompiled header stopped clang builds. Every header hxcpp.h and the runtime include is listed now, and CI checks that it stays so (libhxcpp #469)
* Fixed three build tool settings that did nothing or failed: a file with its own cxx_standard in a group with a precompiled header failed to compile (C2855 on MSVC), an <nvcc/> group was given the host compiler's flags because its tag was compared instead of set, and -DNDKV20 and up were ignored, so the newest NDK was used instead. An NDK without meta/platforms.json no longer stops the build with "Invalid field access : toString" (libhxcpp #472)
* Fixed the build tool reusing stale objects when a define that changes how files compile was switched without the compile cache: only defines containing hxcpp, and a few others, were compared, so adding -D CPPIA_NO_JIT to a scriptable build failed to link, removing it linked a host without its JIT, and ABI, OBJC_ARC, STDLIBCPP and the others like them did nothing until the objects changed. Each now rebuilds what it affects (libhxcpp #470)
* Fixed the build tool linking a stale object and exiting 0 when a compile failed and its old object could not be deleted, as when it is read-only or another process has it open, and hanging when a compile thread exited, as with HXCPP_EXIT_ON_ERROR on a compile error, or with an error such as two optimisation tags on one file. Both fail the build now, and HXCPP_EXIT_ON_ERROR works as a -D define, not only from the environment (libhxcpp #471)
* Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now
* Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now
* Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating
Expand Down
71 changes: 71 additions & 0 deletions test/compile_errors/check.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
# libhxcpp #471: errors on the compile threads.
# - A failed compile whose old object could not be deleted, as when it is
# read-only, linked that object and exited 0.
# - Exiting from a compile thread, as HXCPP_EXIT_ON_ERROR does on a compile
# error, or an error the build tool reports there, hung the build.
# Each build here must fail, and within the time limit.
cd "$(dirname "$0")"
failed=0

# Runs a build in the background and fails it after $limit seconds
limit=600
build() {
"$@" > build.log 2>&1 &
local pid=$! waited=0
while kill -0 $pid 2>/dev/null; do
if [ $waited -ge $limit ]; then
kill $pid 2>/dev/null
echo "hung"
return 0
fi
sleep 1
waited=$((waited+1))
done
wait $pid
echo "exit $?"
}

expect_failure() {
local name=$1; shift
local result
result=$(build "$@")
if [ "$result" = "exit 0" ] || [ "$result" = "hung" ]; then
echo "::error::$name: $result, where it should fail"
tail -20 build.log
failed=1
else
echo "$name: failed as it should ($result)"
fi
}

(cd error && HXCPP_EXIT_ON_ERROR=1 expect_failure "HXCPP_EXIT_ON_ERROR in the environment" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1
(cd error && expect_failure "-D HXCPP_EXIT_ON_ERROR" haxe build.hxml -D HXCPP_EXIT_ON_ERROR -D "$HXCPP_ARCH_FLAG") || failed=1
(cd tags && expect_failure "two optimisation tags" haxe build.hxml -D "$HXCPP_ARCH_FLAG") || failed=1

# Windows only: elsewhere a read-only file can still be deleted
if [ "$OS" = "Windows_NT" ]; then
(
cd error
rm -rf ro && mkdir ro && cp build.hxml ro/
cd ro
printf 'class Main {\n\tstatic function main() {\n\t\tSys.println("old code");\n\t}\n}\n' > Main.hx
haxe build.hxml -D "$HXCPP_ARCH_FLAG" > first.log 2>&1 || { echo "::error::first build failed"; cat first.log; exit 1; }
obj=$(ls bin/obj/*/*_Main.obj | head -1)
attrib +R "$(cygpath -w "$obj")"
# Newer than the object by more than the build tool's one second
sleep 2
cp ../Main.hx Main.hx
expect_failure "a compile error over a read-only object" haxe build.hxml -D "$HXCPP_ARCH_FLAG"
# The old object must not be linked: the link itself may then fail or not
if grep -q 'Link:' build.log; then
echo "::error::a compile error over a read-only object: linked the old object"
failed=1
fi
status=$failed
attrib -R "$(cygpath -w "$obj")"
exit $status
) || failed=1
fi

exit $failed
5 changes: 5 additions & 0 deletions test/compile_errors/error/Main.hx
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
class Main {
static function main() {
untyped __cpp__("this_is_not_cpp();");
}
}
2 changes: 2 additions & 0 deletions test/compile_errors/error/build.hxml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
-cpp bin
-main Main
6 changes: 6 additions & 0 deletions test/compile_errors/tags/Main.hx
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
@:fileXml("tags='haxe,static,optim-none,optim-size'")
class Main {
static function main() {
Sys.println("hi");
}
}
2 changes: 2 additions & 0 deletions test/compile_errors/tags/build.hxml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
-cpp bin
-main Main
17 changes: 17 additions & 0 deletions toolchain/haxe-target.xml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@
<depend name="${HXCPP}/include/hx/thread/Scratch.hpp"/>
<depend name="${HXCPP}/include/hx/atomic/AtomicInt.hpp"/>
<depend name="${HXCPP}/include/hx/atomic/AtomicObject.hpp"/>
<!-- Everything hxcpp.h includes, directly or not: tools/check-depends.py checks -->
<depend name="${HXCPP}/include/hx/Functions.h"/>
<depend name="${HXCPP}/include/hx/FieldRef.h"/>
<depend name="${HXCPP}/include/hx/ErrorCodes.h"/>
<depend name="${HXCPP}/include/hx/Boot.h"/>
<depend name="${HXCPP}/include/hx/Undefine.h"/>
<depend name="${HXCPP}/include/hx/Invoker.h"/>
<depend name="${HXCPP}/include/cpp/CppInt32__.h"/>
<depend name="${HXCPP}/include/cpp/FastIterator.h"/>
<depend name="${HXCPP}/include/cpp/marshal/RootHandle.hpp"/>
<section unless="HXCPP_LINK_NO_TOOL_DEPENDS" >
<depend name="${HXCPP}/toolchain/haxe-target.xml" dateOnly="true" />
<depend name="${HXCPP}/toolchain/${toolchain}-toolchain.xml" ifExists="${HXCPP}/toolchain/${toolchain}-toolchain.xml" dateOnly="true" />
Expand Down Expand Up @@ -189,6 +199,13 @@
<depend name="${HXCPP}/src/hx/gc/Incremental.h"/>
<depend name="${HXCPP}/src/hx/gc/IncrementalApi.h"/>
<depend name="${HXCPP}/src/hx/thread/ThreadImpl.hpp"/>
<depend name="${HXCPP}/include/hx/DynamicImpl.h"/>
<depend name="${HXCPP}/include/hx/Memory.h"/>
<depend name="${HXCPP}/include/hx/NekoFunc.h"/>
<depend name="${HXCPP}/include/hx/CFFIJsPrime.h"/>
<depend name="${HXCPP}/include/hx/CFFINekoLoader.h"/>
<depend name="${HXCPP}/include/hx/TelemetryTracy.h"/>
<depend name="${HXCPP}/src/hx/FastMutex.h"/>

<compilerflag value="-DHX_UNDEFINE_H" />

Expand Down
88 changes: 88 additions & 0 deletions tools/check-depends.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
# Checks that toolchain/haxe-target.xml's depend lists name every header the
# runtime and generated code include, so editing one rebuilds what uses it.
# hxcpp-depends must hold everything hxcpp.h includes, directly or not, and the
# runtime group, with it, everything the runtime's sources include from this
# repository. A header missing there was never compared, so a change to it
# rebuilt nothing, and the compile cache reused objects built before it
# (libhxcpp #469).
# python3 tools/check-depends.py (from the repository root)
import os
import re
import sys

root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
include_dir = os.path.join(root, 'include')
target = open(os.path.join(root, 'toolchain', 'haxe-target.xml'), encoding='utf-8').read()
include_re = re.compile(r'^\s*#\s*include\s*[<"]([^>"]+)[>"]', re.M)


def group(name):
m = re.search(r'<files id="%s"[^>]*>(.*?)</files>' % re.escape(name), target, re.S)
if not m:
sys.exit('no files group %s in haxe-target.xml' % name)
return m.group(1)


def depends(body):
return {os.path.normpath(os.path.join(root, d.replace('${HXCPP}/', '')))
for d in re.findall(r'<depend name="(\$\{HXCPP\}/[^"]+)"', body)}


def exists_exact(path):
# As on a case-sensitive file system: on Windows <memory.h> found hx/Memory.h
directory, name = os.path.split(path)
return os.path.isfile(path) and name in os.listdir(directory)


def resolve(name, from_dir):
for base in (from_dir, include_dir):
path = os.path.normpath(os.path.join(base, name))
if path.startswith(root + os.sep) and exists_exact(path):
return path
return None


def closure(starts):
seen = set()
todo = list(starts)
while todo:
path = todo.pop()
if path in seen:
continue
seen.add(path)
text = open(path, encoding='utf-8', errors='replace').read()
for name in include_re.findall(text):
found = resolve(name, os.path.dirname(path))
# Third-party sources keep their own headers, outside these lists
if found and 'thirdparty' not in found:
todo.append(found)
return seen


rel = lambda p: os.path.relpath(p, root).replace(os.sep, '/')
failed = False

common = depends(group('hxcpp-depends'))
need = closure([os.path.join(include_dir, 'hxcpp.h')])
missing = sorted(rel(p) for p in need - common)
if missing:
failed = True
print('hxcpp-depends misses headers hxcpp.h includes:')
for m in missing:
print(' ' + m)

runtime = group('runtime')
sources = [os.path.join(root, f) for f in re.findall(r'<file name\s*=\s*"([^"]+)"', runtime)]
sources = [s for s in sources if os.path.isfile(s)]
headers = {p for p in closure(sources) if not p.endswith(('.cpp', '.c', '.cc', '.mm', '.m'))}
missing = sorted(rel(p) for p in headers - common - depends(runtime) if not rel(p).startswith('project/'))
if missing:
failed = True
print('the runtime group misses headers its sources include:')
for m in missing:
print(' ' + m)

if failed:
sys.exit(1)
print('depend lists cover %d headers from hxcpp.h and %d from %d runtime sources' % (len(need), len(headers), len(sources)))
9 changes: 7 additions & 2 deletions tools/hxcpp/BuildTool.hx
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ class BuildTool
inIncludePath:Array<String>, inDirtyList:Array<String> )
{
mDefines = inDefines;
// Read once the arguments and Options.txt are in, not only from the
// environment
exitOnThreadError = exitOnThreadError || inDefines.exists("HXCPP_EXIT_ON_ERROR");
mFileGroups = new FileGroups();
mCompiler = null;
mStripper = null;
Expand Down Expand Up @@ -308,7 +311,7 @@ class BuildTool
if (cached)
{
var cacheSize = mDefines.exists("HXCPP_CACHE_MB") ? Std.parseInt( mDefines.get("HXCPP_CACHE_MB") ) : 1000;
if (cacheSize!=null && cacheSize>0)
if (cacheSize!=null && cacheSize>0 && CompileCache.dueForTrim())
CompileCache.clear(0,cacheSize,false,null);
}

Expand Down Expand Up @@ -1214,8 +1217,10 @@ class BuildTool
case "nvcc" :
setupNvcc();
group.mNvcc = true;
// Set, not compared: the group kept the haxe tags, so it was
// given the host compiler's flags (libhxcpp #472)
if (group.mTags=="haxe,static")
group.mTags=="nvcc";
group.mTags="nvcc";
case "objprefix" :
group.mObjPrefix = substitute(el.att.value);
case "compilervalue" :
Expand Down
16 changes: 16 additions & 0 deletions tools/hxcpp/CompileCache.hx
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,22 @@ class CompileCache
return dir + "/" + hash.substr(2) + inExt;
}

// Sizing the cache stats every file in it, which took seconds once it held
// tens of thousands, and ran before every build, even one with nothing to
// compile (libhxcpp #473). Once in ten minutes keeps it near its limit
public static function dueForTrim():Bool
{
var stamp = compileCache + "/.trimmed";
try
{
if (FileSystem.exists(stamp) && Date.now().getTime() - FileSystem.stat(stamp).mtime.getTime() < 10 * 60 * 1000.0)
return false;
sys.io.File.saveContent(stamp, "");
}
catch(e:Dynamic) { }
return true;
}

public static function clear(inDays:Int,inMB:Int,inLogInfo:Bool,inProject:String)
{
try
Expand Down
15 changes: 12 additions & 3 deletions tools/hxcpp/Compiler.hx
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,9 @@ class Compiler
args = args.concat(mCPPFlags);
}

if (isRc || inFile.getTags()!=inFile.mGroup.getTags())
// The precompiled header is built with the group's standard, and a file
// with its own cannot use it: MSVC and clang both refuse (libhxcpp #472)
if (isRc || inFile.getTags()!=inFile.mGroup.getTags() || inFile.mCxxStandard!=inFile.mGroup.mCxxStandard)
allowPch = false;

if (inFile.mGroup.isPrecompiled() && allowPch)
Expand Down Expand Up @@ -471,9 +473,16 @@ class Compiler
cleanTmp(tmpFile);
if (err!=0)
{
if (FileSystem.exists(obj_name))
FileSystem.deleteFile(obj_name);
// The error first: when the delete threw, as it does for a
// read-only object or one another process has open, it was
// lost, and the old object was linked (libhxcpp #471)
BuildTool.setThreadError(err);
try
{
if (FileSystem.exists(obj_name))
FileSystem.deleteFile(obj_name);
}
catch(e:Dynamic) { }
}
else
inProgress.fileDone();
Expand Down
9 changes: 8 additions & 1 deletion tools/hxcpp/FileGroup.hx
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,13 @@ class FileGroup
}
}

// Defines that change how the toolchains compile, or what: without the
// cache, this copy of Options.txt is all that tells the objects are stale,
// so switching one of these reused objects built without it, and adding
// CPPIA_NO_JIT failed to link (libhxcpp #470)
static var compileDefines = ["cppia_no_jit", "cppia_jit", "abi", "objc", "objc_arc", "stdlibcpp", "exe_link", "o3",
"legacy_macosx_sdk", "armv6", "rpi", "winrt", "nom32", "nom64", "no_shared_libs"];

public function filterOptions(contents:String)
{
// Old-style
Expand All @@ -237,7 +244,7 @@ class FileGroup
}
else if (name.indexOf("hxcpp")>=0 || name=="scriptable" || name.indexOf("dll")>=0 || name=="no_console" ||
name.substr(0,8)=="android-" || name.substr(0,4)=="ndkv" || name=="toolchain" || name=="platform" ||
name=="toolchain_version" || name=="android_ndk_root" )
name=="toolchain_version" || name=="android_ndk_root" || compileDefines.indexOf(name)>=0 )
result.push(def);
}

Expand Down
18 changes: 12 additions & 6 deletions tools/hxcpp/Setup.hx
Original file line number Diff line number Diff line change
Expand Up @@ -490,16 +490,22 @@ class Setup

if (Log.verbose) Log.println("");

// Any NDKV<n>, the lowest if there are several. Only 6 to 19 were
// looked for, so a request for r20 or later was ignored, and the newest
// NDK found was used instead (libhxcpp #472)
var ndkVersion = 0;
for (i in 6...20)
var requested = ~/^NDKV([0-9]+)$/;
for (key in defines.keys())
{
if (defines.exists("NDKV" + i))
if (requested.match(key))
{
ndkVersion = i;
Log.info("", "\x1b[33;1mRequested Android NDK r" + i + "\x1b[0m");
break;
var version = Std.parseInt(requested.matched(1));
if (version!=null && version>=6 && (ndkVersion==0 || version<ndkVersion))
ndkVersion = version;
}
}
if (ndkVersion!=0)
Log.info("", "\x1b[33;1mRequested Android NDK r" + ndkVersion + "\x1b[0m");

if (!defines.exists("ANDROID_NDK_ROOT") || ndkVersion!=0)
{
Expand Down Expand Up @@ -624,7 +630,7 @@ class Setup
var minPlatform:Null<Int> = try {
haxe.Json.parse(sys.io.File.getContent(platformsJson)).min;
} catch (e:Dynamic) {
Log.warn("Unable to determine minimum supported Android platform: " + e.toString());
Log.warn("Unable to determine minimum supported Android platform: " + Std.string(e));
null;
};

Expand Down
Loading
Loading