From 043f892c978917c055bbe0e72441cfbd64a181cc Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:19:20 -0500 Subject: [PATCH 1/3] Make cppia's interpreter and JIT agree on array builtins Compared with the same code built as C++, which both now match: JIT: - Array and Array compound assignment stored the right hand side alone, and ++, --, insert and toString had no JIT code at all, so they did nothing ("Unknown ArrayBuiltinAny"). They call helpers sharing the interpreter's arithmetic, which catch, and the jitted code checks after each. - A Bool pushed, unshifted or set into an Array was stored as an Int. It goes in boxed. - Bitwise compound operators on an Array> did nothing: the bitwise cases fell through to the element's type for lack of a break. - An Int array's element read out of range as a String was "null", not "0". It reads the element type's zero. - ++d[i] on a Dynamic used as a statement decremented: the test was crement==coPostInc || crement==coPostInc. Interpreter: - A typed indexOf or lastIndexOf used as a Float returned 0 without running. - join and toString used as a statement, and iterator and keyValueIterator, were skipped, receiver and element toString calls with them. Test: CppiaArrayEngines, twelve lines, each wrong in one engine before and right in both now, interpreted and under the JIT on Windows x64. Fixes #468 --- Changes.md | 1 + src/hx/cppia/ArrayBuiltin.cpp | 26 ++- src/hx/cppia/ArrayVirtual.cpp | 174 +++++++++++++------ src/hx/cppia/Cppia.cpp | 2 +- test/regression/CppiaArrayEngines/Main.hx | 13 ++ test/regression/CppiaArrayEngines/Script.hx | 115 ++++++++++++ test/regression/CppiaArrayEngines/build.hxml | 7 + test/regression/CppiaArrayEngines/stdout.txt | 26 +++ 8 files changed, 306 insertions(+), 58 deletions(-) create mode 100644 test/regression/CppiaArrayEngines/Main.hx create mode 100644 test/regression/CppiaArrayEngines/Script.hx create mode 100644 test/regression/CppiaArrayEngines/build.hxml create mode 100644 test/regression/CppiaArrayEngines/stdout.txt diff --git a/Changes.md b/Changes.md index 54466e163..3ae004887 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed cppia's interpreter and JIT giving different results for array builtins, both of them now matching C++: under the JIT, a[i] += x and the other compound assignments on an Array stored x alone, a[i]++, insert and toString on one did nothing, a Bool stored in an Array became an Int, bitwise compound assignments on an Array> did nothing, an Int array read out of range as a String gave "null", and ++d[i] on a Dynamic as a statement decremented. Interpreted, a typed indexOf or lastIndexOf used as a Float gave 0, and join, toString and the iterators used as a statement did not run (libhxcpp #468) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now * Fixed a change of link flags or libraries alone leaving the program as it was: the build tool linked again only when an object or library file had changed, so -D HXCPP_LINK_STACK_SIZE, HXCPP_LINK_NO_MEMORY_GROWTH, HXCPP_LINK_NO_EMBIND or HXCPP_LINK_NODERAWFS on an Emscripten build, which change nothing that is compiled, or a flag or library added to a target in Build.xml, did nothing until the objects changed. The flags, libraries and library paths are now part of what a link is checked against, so each program links once more after updating diff --git a/src/hx/cppia/ArrayBuiltin.cpp b/src/hx/cppia/ArrayBuiltin.cpp index 0ecacbf92..d997fd03b 100644 --- a/src/hx/cppia/ArrayBuiltin.cpp +++ b/src/hx/cppia/ArrayBuiltin.cpp @@ -288,6 +288,9 @@ struct ArraySetter : public ArrayBuiltinBase case aoShr: case aoUShr: rightHandType = etInt; + // Fell through to the element's type, so on an Array> + // these did nothing under the JIT + break; default: rightHandType = elemType; @@ -693,7 +696,8 @@ struct ArrayBuiltin : public ArrayBuiltinBase return result; } - if (FUNC==afPush) + // indexOf and lastIndexOf used as a Float returned 0 without running + if (FUNC==afPush || FUNC==afIndexOf || FUNC==afLastIndexOf) return runInt(ctx); return 0.0; } @@ -980,9 +984,14 @@ struct ArrayBuiltin : public ArrayBuiltinBase runInt(ctx); if (FUNC==afConcat || FUNC==afCopy || FUNC==afReverse || FUNC==afSplice || FUNC==afSlice || - FUNC==afMap || FUNC==afFilter) + FUNC==afMap || FUNC==afFilter || FUNC==afIterator || FUNC==afKeyValueIterator) runObject(ctx); + // Statements too: they run the receiver and each element's toString, + // which were skipped, as the JIT and the other targets do not + if (FUNC==afJoin || FUNC==afToString) + runString(ctx); + if (FUNC==afReverse) { Array_obj *thisVal = (Array_obj*)thisExpr->runObject(ctx); @@ -1550,8 +1559,17 @@ struct ArrayBuiltin : public ArrayBuiltinBase // Check length.. JumpId lengthOk = compiler->compare( cmpI_LESS, sJitTemp0.as(jtInt), sJitTemp1.star(jtInt, ArrayBase::lengthOffset()) ); - // Out of bounds - return null / zero - compiler->returnNull(inDest, destType); + // Out of bounds - the element type's zero, as the interpreter + // and the other targets read it: an Int array's read as a + // String was "null", not "0" + ExprType outElemType = (ExprType)ExprTypeOf::value; + if ((outElemType==etInt || outElemType==etFloat) && (destType==etString || destType==etObject)) + { + compiler->move(sJitTemp2.as(jtInt), (int)0); + compiler->convert(sJitTemp2.as(jtInt), etInt, inDest, destType, isBoolElem()); + } + else + compiler->returnNull(inDest, destType); writtenNull = compiler->jump(); diff --git a/src/hx/cppia/ArrayVirtual.cpp b/src/hx/cppia/ArrayVirtual.cpp index 46ba3fb6f..aa185eb3d 100644 --- a/src/hx/cppia/ArrayVirtual.cpp +++ b/src/hx/cppia/ArrayVirtual.cpp @@ -176,6 +176,41 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase } const char *getName() HXCPP_OVERRIDE { return gArrayFuncNames[FUNC]; } + // a[i] op= val and ++ and --, for the interpreter and the JIT alike + static Dynamic applyAssign(Dynamic orig, Dynamic val) + { + switch(OP) + { + case aoAdd: return orig + val; + case aoMult: return orig * val; + case aoDiv: return orig / val; + case aoSub: return orig - val; + case aoAnd: return orig->__ToInt() & val->__ToInt(); + case aoOr: return orig->__ToInt() | val->__ToInt(); + case aoXOr: return orig->__ToInt() ^ val->__ToInt(); + case aoShl: return orig->__ToInt() << val->__ToInt(); + case aoShr: return orig->__ToInt() >> val->__ToInt(); + case aoUShr: return hx::UShr(orig,val); + case aoMod: return orig % val; + default: return val; + } + } + + static Dynamic applyCrement(ArrayAnyImpl *thisVal, int i) + { + if (OP==coPreInc) + return thisVal->__SetItem(i, thisVal->__GetItem(i) + 1); + if (OP==coPreDec) + return thisVal->__SetItem(i, thisVal->__GetItem(i) - 1); + + Dynamic result = thisVal->__GetItem(i); + if (OP==coPostDec) + thisVal->__SetItem(i, thisVal->__GetItem(i) - 1); + if (OP==coPostInc) + thisVal->__SetItem(i, thisVal->__GetItem(i) + 1); + return result; + } + int runInt(CppiaCtx *ctx) HXCPP_OVERRIDE { if (FUNC==afPush) @@ -306,62 +341,14 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase Dynamic val = args[1]->runObject(ctx); BCR_CHECK; - switch(OP) - { - case aoSet: break; - - case aoAdd: - val = orig + val; - break; - case aoMult: - val = orig * val; - break; - case aoDiv: - val = orig / val; - break; - case aoSub: - val = orig - val; - break; - case aoAnd: - val = orig->__ToInt() & val->__ToInt(); - break; - case aoOr: - val = orig->__ToInt() | val->__ToInt(); - break; - case aoXOr: - val = orig->__ToInt() ^ val->__ToInt(); - break; - case aoShl: - val = orig->__ToInt() << val->__ToInt(); - break; - case aoShr: - val = orig->__ToInt() >> val->__ToInt(); - break; - case aoUShr: - val = hx::UShr(orig,val); - break; - case aoMod: - val = orig % val; - break; - default: ; - } + val = applyAssign(orig, val); return thisVal->__SetItem(i,val).mPtr; } if (FUNC==af__crement) { int i = args[0]->runInt(ctx); BCR_CHECK; - if (OP==coPreInc) - return thisVal->__SetItem(i, thisVal->__GetItem(i) + 1).mPtr; - if (OP==coPreDec) - return thisVal->__SetItem(i, thisVal->__GetItem(i) - 1).mPtr; - - Dynamic result = thisVal->__GetItem(i); - if (OP==coPostDec) - thisVal->__SetItem(i, thisVal->__GetItem(i) - 1); - if (OP==coPostInc) - thisVal->__SetItem(i, thisVal->__GetItem(i) + 1); - return result.mPtr; + return applyCrement(thisVal, i).mPtr; } if (FUNC==afPop) @@ -666,6 +653,49 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase } + // Compound assignment, ++ and --, insert and toString, which the JIT did + // not compile: a[i] op= val stored val alone, and the others did nothing + // (libhxcpp #468). They work as the interpreter does + static hx::Object * SLJIT_CALL runAssign(ArrayAnyImpl *inArray, int inIndex, hx::Object *inValue) + { + TRY_NATIVE + Dynamic orig = inArray->__GetItem(inIndex); + if (!orig.mPtr) + NullReference("Object", false); + return inArray->__SetItem(inIndex, applyAssign(orig, Dynamic(inValue))).mPtr; + CATCH_NATIVE + return 0; + } + + static hx::Object * SLJIT_CALL runCrement(ArrayAnyImpl *inArray, int inIndex) + { + TRY_NATIVE + return applyCrement(inArray, inIndex).mPtr; + CATCH_NATIVE + return 0; + } + + static void SLJIT_CALL runInsert(ArrayAnyImpl *inArray, hx::Object *inPos, hx::Object *inValue) + { + TRY_NATIVE + inArray->CALL(insert)(Dynamic(inPos), Dynamic(inValue)); + CATCH_NATIVE + } + + static void SLJIT_CALL runToString(ArrayAnyImpl *inArray, String *outValue) + { + TRY_NATIVE + JitStore(outValue, inArray->toString()); + CATCH_NATIVE + } + + // A Bool goes in boxed, as a Bool: the Int paths stored it as an Int + static ExprType anyStoreType(CppiaExpr *inValue) + { + ExprType type = inValue->getType(); + return type==etInt && inValue->isBoolInt() ? etObject : type; + } + void genCode(CppiaCompiler *compiler, const JitVal &inDest, ExprType destType) HXCPP_OVERRIDE { JitTemp thisVal(compiler,etObject); @@ -826,7 +856,16 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase JitTemp index(compiler,etInt); args[0]->genCode(compiler, index, etInt); - switch(args[1]->getType()) + if (OP!=aoSet) + { + args[1]->genCode(compiler, sJitArg2.as(jtPointer), etObject); + compiler->callNative( (void *)runAssign, thisVal, index, sJitArg2.as(jtPointer)); + compiler->checkException(); + compiler->convertReturnReg(etObject, inDest, destType); + break; + } + + switch(anyStoreType(args[1])) { case etInt: args[1]->genCode(compiler, sJitArg2.as(jtInt), etInt); @@ -863,7 +902,7 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase case afPush: { - switch(args[0]->getType()) + switch(anyStoreType(args[0])) { case etInt: args[0]->genCode(compiler, sJitArg1.as(jtInt), etInt); @@ -896,7 +935,7 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase case afUnshift: { - switch(args[0]->getType()) + switch(anyStoreType(args[0])) { case etInt: args[0]->genCode(compiler, sJitArg1.as(jtInt), etInt); @@ -928,6 +967,35 @@ struct ArrayBuiltinAny : public ArrayBuiltinBase + case af__crement: + { + JitTemp index(compiler,etInt); + args[0]->genCode(compiler, index, etInt); + compiler->callNative( (void *)runCrement, thisVal, index); + compiler->checkException(); + compiler->convertReturnReg(etObject, inDest, destType); + } + break; + + case afInsert: + { + JitTemp pos(compiler,jtPointer); + args[0]->genCode(compiler, pos, etObject); + args[1]->genCode(compiler, sJitArg2.as(jtPointer), etObject); + compiler->callNative( (void *)runInsert, thisVal, pos, sJitArg2.as(jtPointer)); + compiler->checkException(); + } + break; + + case afToString: + { + JitTemp result(compiler,jtString); + compiler->callNative( (void *)runToString, thisVal, result); + compiler->checkException(); + compiler->convert(result, etString, inDest, destType); + } + break; + default: compiler->traceStrings("Unknown ArrayBuiltinAny:", getName() ); } diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index 3094c88b0..b8f88d0b2 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -5829,7 +5829,7 @@ struct DynamicArrayI : public CppiaDynamicExpr index->genCode(compiler, sJitTemp1, etInt); if (destType==etVoid || destType==etNull) { - compiler->callNative( crement==coPostInc || crement==coPostInc ? (void *)incItem : (void *)decItem, + compiler->callNative( crement==coPostInc || crement==coPreInc ? (void *)incItem : (void *)decItem, obj, sJitTemp1.as(jtInt) ); } else diff --git a/test/regression/CppiaArrayEngines/Main.hx b/test/regression/CppiaArrayEngines/Main.hx new file mode 100644 index 000000000..e1b4cc7a2 --- /dev/null +++ b/test/regression/CppiaArrayEngines/Main.hx @@ -0,0 +1,13 @@ +// Runs the script interpreted and then under the JIT +function main() { + var script = haxe.io.Path.join([haxe.io.Path.directory(Sys.programPath()), "script.cppia"]); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + Sys.println(jit ? "jit" : "interpreted"); + try { + cpp.cppia.Host.runFile(script); + } catch (e:Dynamic) { + Sys.println("error: " + e); + } + } +} diff --git a/test/regression/CppiaArrayEngines/Script.hx b/test/regression/CppiaArrayEngines/Script.hx new file mode 100644 index 000000000..15ae93a14 --- /dev/null +++ b/test/regression/CppiaArrayEngines/Script.hx @@ -0,0 +1,115 @@ +// libhxcpp #468: array builtins where the cppia interpreter and JIT disagreed. +// Each line is printed by both, and is what the other targets print. +class Obj { + public function new() {} + + public function toString() { + Script.calls++; + return "obj"; + } +} + +class Script { + public static var calls = 0; + static var got = 0; + + static function main() { + anyCompound(); + anyCrement(); + anyOthers(); + anyBools(); + nullIntBits(); + nullIntArith(); + searchAsFloat(); + statements(); + outOfRange(); + dynamicCrement(); + } + + static function anyCompound() { + var a:Array = [1, 2, 3, 4, 5]; + a[0] += 10; + a[1] *= 5; + a[2] |= 8; + a[3] -= 1; + a[4] <<= 2; + Sys.println("Array compound: " + a); + } + + static function anyCrement() { + var a:Array = [1, 2, 3]; + a[0]++; + var x = a[1]--; + var y = ++a[2]; + Sys.println("Array crement: " + a + " x=" + x + " y=" + y); + } + + static function anyOthers() { + var a:Array = [1, 2]; + a.insert(0, "z"); + Sys.println("Array insert: " + a); + var s:String = a.toString(); + Sys.println("Array toString: " + s); + } + + static function anyBools() { + var flag = true; + var b:Array = []; + b.push(flag); + b.unshift(false); + b[2] = true; + Sys.println("Array bools: " + b + " " + [for (x in b) Std.isOfType(x, Bool)]); + } + + static function nullIntBits() { + var n:Array> = [1, 2, 3, 16, 16, 7]; + n[0] |= 4; + n[1] <<= 2; + n[2] ^= 1; + n[3] >>= 2; + n[4] >>>= 1; + n[5] &= 3; + Sys.println("Array> bits: " + n); + } + + static function nullIntArith() { + var n:Array> = [1, 2, 3]; + n[0] += 4; + n[1] *= 3; + n[2] -= 1; + Sys.println("Array> arith: " + n); + } + + static function searchAsFloat() { + var a = [1, 2, 3]; + var f = 0.5 + a.indexOf(3); + var g = 0.5 + a.lastIndexOf(1); + var h:Float = a.indexOf(2); + Sys.println("indexOf as a Float: " + f + " " + g + " " + h); + } + + static function arr():Array { + got++; + return [new Obj()]; + } + + static function statements() { + arr().join(","); + Sys.println("join as a statement: receiver evaluated " + got + ", toString calls " + calls); + arr().toString(); + Sys.println("toString as a statement: receiver evaluated " + got + ", toString calls " + calls); + } + + static function outOfRange() { + var ints = [1, 2]; + var floats = [1.5]; + Sys.println("out of range as a String: " + ints[5] + " " + floats[3]); + } + + static function dynamicCrement() { + var d:Dynamic = [1, 2]; + ++d[0]; + d[1]++; + Sys.println("++ on a Dynamic array as a statement: " + d); + } +} diff --git a/test/regression/CppiaArrayEngines/build.hxml b/test/regression/CppiaArrayEngines/build.hxml new file mode 100644 index 000000000..d20d36bd4 --- /dev/null +++ b/test/regression/CppiaArrayEngines/build.hxml @@ -0,0 +1,7 @@ +--main Script +-cppia bin/script.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaArrayEngines/stdout.txt b/test/regression/CppiaArrayEngines/stdout.txt new file mode 100644 index 000000000..c321ee9e7 --- /dev/null +++ b/test/regression/CppiaArrayEngines/stdout.txt @@ -0,0 +1,26 @@ +interpreted +Array compound: [11,10,11,3,20] +Array crement: [2,1,4] x=2 y=4 +Array insert: [z,1,2] +Array toString: [z,1,2] +Array bools: [false,true,true] [true,true,true] +Array> bits: [5,8,2,4,8,3] +Array> arith: [5,6,2] +indexOf as a Float: 2.5 0.5 1 +join as a statement: receiver evaluated 1, toString calls 1 +toString as a statement: receiver evaluated 2, toString calls 2 +out of range as a String: 0 0 +++ on a Dynamic array as a statement: [2,3] +jit +Array compound: [11,10,11,3,20] +Array crement: [2,1,4] x=2 y=4 +Array insert: [z,1,2] +Array toString: [z,1,2] +Array bools: [false,true,true] [true,true,true] +Array> bits: [5,8,2,4,8,3] +Array> arith: [5,6,2] +indexOf as a Float: 2.5 0.5 1 +join as a statement: receiver evaluated 1, toString calls 1 +toString as a statement: receiver evaluated 2, toString calls 2 +out of range as a String: 0 0 +++ on a Dynamic array as a statement: [2,3] From d6d3e1dfc3b6a531c1fc06b5c18d7b925d8da994 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 09:27:41 -0500 Subject: [PATCH 2/3] Refuse the malformed cppia files that crashed the host SECURITY.md promises a load error for a malformed .cppia file, but these crashed the host, interpreted, under the JIT, or both: - MAIN in a module with no types: ScriptCallable::link read the type of its return, 0, from an empty table. Return type ids are checked. - An unknown script enum constructor left a null value that was dereferenced, and a call with the wrong number of arguments wrote past the enum value under the JIT. Both fail to link. - Two classes with one type id: the second overwrote the first's and wrote outside the vtable made for it. It fails to load. - A string constant id, and the text format's file id, were read unchecked, and the string table grew to them: by 100000000 entries, 1.4 GB, for a file id, and the JIT's addresses of its strings were left dangling. They are checked as other string ids are. - An enum parameter index past the value read past it. A script enum's constructors bound it at link, and otherwise the interpreter and the JIT check the value's own parameter count. - break or continue outside a loop made sljit assert under the JIT, ending the process. The function fails to compile instead, checked on the pass that emits code, since the measuring pass has no labels. - A super constructor call to a script class other than the super class ran that constructor on an instance laid out for this one. It fails to link, after the check for a class with no constructor. CppiaMalformedConstructors' control, such a call from a static main, is refused now too. Test: CppiaMalformedLoads, ten such files, loaded and run interpreted and with the JIT compiling as it loads. Before, the host crashed on the first; now each fails to load or runs safely. The cppia suite and the 60 Cppia regression tests pass, interpreted and under the JIT, on Windows x64. Fixes #467 --- Changes.md | 1 + src/hx/cppia/Cppia.cpp | 49 +++++++++++++++++-- src/hx/cppia/Cppia.h | 2 + src/hx/cppia/CppiaClasses.cpp | 20 ++++++++ src/hx/cppia/CppiaCompiler.cpp | 7 +++ src/hx/cppia/CppiaFunction.cpp | 4 ++ src/hx/cppia/CppiaModule.cpp | 16 +++++- src/hx/cppia/CppiaStream.h | 5 +- .../CppiaMalformedConstructors/stdout.txt | 4 +- test/regression/CppiaMalformedLoads/Files.hx | 15 ++++++ test/regression/CppiaMalformedLoads/Main.hx | 28 +++++++++++ .../regression/CppiaMalformedLoads/build.hxml | 4 ++ .../regression/CppiaMalformedLoads/stdout.txt | 26 ++++++++++ test/regression/emscripten-skip.txt | 1 + 14 files changed, 170 insertions(+), 12 deletions(-) create mode 100644 test/regression/CppiaMalformedLoads/Files.hx create mode 100644 test/regression/CppiaMalformedLoads/Main.hx create mode 100644 test/regression/CppiaMalformedLoads/build.hxml create mode 100644 test/regression/CppiaMalformedLoads/stdout.txt diff --git a/Changes.md b/Changes.md index 3ae004887..ecbb00d29 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed malformed cppia files crashing the host instead of failing to load: a MAIN in a module with no types, an unknown script enum constructor or one called with the wrong number of arguments, two classes with one type id, a string constant or file id past the string table, which grew the table to it, an enum parameter index past the value, break or continue outside a loop under the JIT, and a super constructor call to a class other than the super class. Each fails to load, or is read safely (libhxcpp #467) * Fixed cppia's interpreter and JIT giving different results for array builtins, both of them now matching C++: under the JIT, a[i] += x and the other compound assignments on an Array stored x alone, a[i]++, insert and toString on one did nothing, a Bool stored in an Array became an Int, bitwise compound assignments on an Array> did nothing, an Int array read out of range as a String gave "null", and ++d[i] on a Dynamic as a statement decremented. Interpreted, a typed indexOf or lastIndexOf used as a Float gave 0, and join, toString and the iterators used as a statement did not run (libhxcpp #468) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now * Fixed HXCPP_GC_INCREMENTAL failing to compile ("Incremental GC requires regenerated C++") with libhxcpp installed from a release, 6.0.0-rc.2 among them: the release workflow rewrote extraParams.hxml with the version and dropped its other line, the macro that turns on the generated write barriers the incremental collector needs. Only the version lines are rewritten now diff --git a/src/hx/cppia/Cppia.cpp b/src/hx/cppia/Cppia.cpp index b8f88d0b2..7c0cdafb3 100644 --- a/src/hx/cppia/Cppia.cpp +++ b/src/hx/cppia/Cppia.cpp @@ -3536,6 +3536,10 @@ struct CallMember : public CppiaExpr // malformed file calls one where neither has any if (!type->cppiaClass->newFunc) throw "Super constructor call on a class with no constructor"; + // A class's constructor calls its own super class's. Any other script + // class's ran on an instance laid out for this one, past its end + if (inModule.linkingClass && classId!=inModule.linkingClass->superId) + throw "Super constructor call to a class that is not the super class"; //CPPIALOG("Using cppia super %p %p\n", type->cppiaClass->newFunc, type->cppiaClass->newFunc->funExpr); CppiaExpr *replace = new CallFunExpr( this, 0, (ScriptCallable*)type->cppiaClass->newFunc->funExpr, args, true ); replace->link(inModule); @@ -6301,11 +6305,13 @@ struct EnumIExpr : public CppiaDynamicExpr CppiaExpr *object; int index; int classId; + // Whether link bounded index by the enum's constructors + bool indexChecked; - EnumIExpr(CppiaStream &stream) + EnumIExpr(CppiaStream &stream) : indexChecked(false) { - classId = stream.getInt(); + classId = stream.getTypeId(); index = stream.getInt(); object = createCppiaExpr(stream); } @@ -6313,6 +6319,18 @@ struct EnumIExpr : public CppiaDynamicExpr const char *getName() HXCPP_OVERRIDE { return "EnumIExpr"; } CppiaExpr *link(CppiaModule &inModule) HXCPP_OVERRIDE { + // An index from the file read past the enum value. A script enum's + // largest constructor bounds it here, for the JIT, which reads the + // parameter directly; the interpreter checks the value's own count + if (index<0) + throw "Bad enum index"; + CppiaClassInfo *info = inModule.types[classId]->cppiaClass; + if (info && info->isEnum) + { + if (index>=info->mostEnumArgs()) + throw "Bad enum index"; + indexChecked = true; + } object = object->link(inModule); return this; } @@ -6321,15 +6339,36 @@ struct EnumIExpr : public CppiaDynamicExpr { hx::Object *obj = object->runObject(ctx); BCR_CHECK; - return static_cast(obj)->_hx_getParamI(index).mPtr; + EnumBase_obj *value = static_cast(obj); + if (index>=value->_hx_getParamCount()) + return 0; + return value->_hx_getParamI(index).mPtr; } #ifdef CPPIA_JIT + static hx::Object * SLJIT_CALL paramChecked(hx::Object *inValue, int inIndex) + { + EnumBase_obj *value = static_cast(inValue); + if (!value || inIndex>=value->_hx_getParamCount()) + return 0; + return value->_hx_getParamI(inIndex).mPtr; + } + void genCode(CppiaCompiler *compiler, const JitVal &inDest, ExprType destType) HXCPP_OVERRIDE { object->genCode(compiler, sJitTemp0, etObject); - int offset = sizeof( EnumBase_obj ) + index*sizeof(cpp::Variant); - compiler->genVariantValueTemp0(offset, inDest, destType); + if (indexChecked) + { + int offset = sizeof( EnumBase_obj ) + index*sizeof(cpp::Variant); + compiler->genVariantValueTemp0(offset, inDest, destType); + } + else + { + // A host enum's constructors are not known here, so its value's own + // count bounds the read, as in the interpreter + compiler->callNative( (void *)paramChecked, sJitTemp0.as(jtPointer), JitVal(index) ); + compiler->convertReturnReg(etObject, inDest, destType); + } } #endif }; diff --git a/src/hx/cppia/Cppia.h b/src/hx/cppia/Cppia.h index 4eb9e2483..ca52f0b5f 100644 --- a/src/hx/cppia/Cppia.h +++ b/src/hx/cppia/Cppia.h @@ -844,6 +844,8 @@ class CppiaClassInfo : public CppiaOwned inline ScriptCallable *findStaticSetter(const String &inName) { return findFunction(staticSetters,inName); } CppiaEnumConstructor *findEnum(int inFieldId); + // The most arguments any of an enum's constructors takes + int mostEnumArgs(); ExprType findFunctionType(CppiaModule &inModule, int inName); CppiaFunction *findVTableFunction(int inId); int findFunctionSlot(int inName); diff --git a/src/hx/cppia/CppiaClasses.cpp b/src/hx/cppia/CppiaClasses.cpp index 3b30bbfdc..881aa903c 100644 --- a/src/hx/cppia/CppiaClasses.cpp +++ b/src/hx/cppia/CppiaClasses.cpp @@ -408,6 +408,13 @@ struct EnumField : public CppiaDynamicExpr if (!type->cppiaClass->isEnum) throw "Field of non-enum"; value = type->cppiaClass->findEnum(fieldId); + // Was a null value, dereferenced when used or compiled + if (!value) + throw "Unknown enum constructor"; + // A call with arguments to a constructor that takes others; none + // is the constructor as a function + if (args.size() && args.size()!=value->args.size()) + throw "Bad enum arg count"; } else { @@ -850,6 +857,15 @@ void CppiaClassInfo::dump() dumpVars("Static dyns",staticDynamicFunctions); } +int CppiaClassInfo::mostEnumArgs() +{ + size_t most = 0; + for(size_t i=0;iargs.size()>most) + most = enumConstructors[i]->args.size(); + return (int)most; +} + CppiaEnumConstructor *CppiaClassInfo::findEnum(int inFieldId) { for(int i=0;icppiaClass) + throw "Duplicate class"; cppia.types[typeId]->cppiaClass = this; } else diff --git a/src/hx/cppia/CppiaCompiler.cpp b/src/hx/cppia/CppiaCompiler.cpp index 7fad214f5..51942dfb0 100644 --- a/src/hx/cppia/CppiaCompiler.cpp +++ b/src/hx/cppia/CppiaCompiler.cpp @@ -470,13 +470,20 @@ class CppiaJitCompiler : public CppiaCompiler return oldPos; } + // Outside a loop there is nowhere to go, and sljit asserted, ending the + // process. Haxe emits neither there, so only a malformed file has one. The + // first pass, which only measures, has no labels, so it cannot tell void addContinue() HXCPP_OVERRIDE { + if (compiler && !continuePos) + setError("Continue outside a loop"); jump(continuePos); } void addBreak() HXCPP_OVERRIDE { + if (compiler && !continuePos) + setError("Break outside a loop"); allBreaks.push( jump(0) ); } diff --git a/src/hx/cppia/CppiaFunction.cpp b/src/hx/cppia/CppiaFunction.cpp index 714150d0f..ddb41a830 100644 --- a/src/hx/cppia/CppiaFunction.cpp +++ b/src/hx/cppia/CppiaFunction.cpp @@ -169,6 +169,10 @@ CppiaExpr *ScriptCallable::link(CppiaModule &inModule) inModule.layout = &layout; data = &inModule; + // A module's main, which names no type, uses the first, which a malformed + // module may not have + if (returnTypeId<0 || returnTypeId>=(int)inModule.types.size()) + throw "Bad type id"; returnType = inModule.types[ returnTypeId ]->expressionType; isBoolReturn = inModule.types[ returnTypeId ]->haxeClass==ClassOf(); layout.returnType = returnType; diff --git a/src/hx/cppia/CppiaModule.cpp b/src/hx/cppia/CppiaModule.cpp index f8be08f8f..2bd8e9666 100644 --- a/src/hx/cppia/CppiaModule.cpp +++ b/src/hx/cppia/CppiaModule.cpp @@ -421,7 +421,9 @@ void CppiaConst::fromStream(CppiaStream &stream) else if (tok[0]=='s') { type = cString; - ival = stream.getInt(); + // Checked: a bad id grew the string table to it, leaving the addresses + // of its strings that the JIT had built into code dangling + ival = stream.getStringId(); } else if (tok=="NULL") type = cNull; @@ -760,6 +762,18 @@ int CppiaStream::getStringId() return id; } +// The text format names each expression's file by a string id. A bad one grew +// the string table to it when the file name was read: 1.4 GB for 100000000 +int CppiaStream::getFileId() +{ + if (binary) + return 0; + int id = getInt(); + if (id<0 || (id>0 && id>=module->strings->length)) + throw "Bad file id"; + return id; +} + CppiaLoadedModule LoadCppia(const unsigned char *inData, int inDataLength) { CppiaModule *cppiaPtr = new CppiaModule(); diff --git a/src/hx/cppia/CppiaStream.h b/src/hx/cppia/CppiaStream.h index ec0a49ce4..97e08b6bb 100644 --- a/src/hx/cppia/CppiaStream.h +++ b/src/hx/cppia/CppiaStream.h @@ -72,10 +72,7 @@ struct CppiaStream { return binary ? 0 : getInt(); } - int getFileId() - { - return binary ? 0 : getInt(); - } + int getFileId(); void skipChar() { if (data>=max) diff --git a/test/regression/CppiaMalformedConstructors/stdout.txt b/test/regression/CppiaMalformedConstructors/stdout.txt index 4b4b161e4..6c1a36c4f 100644 --- a/test/regression/CppiaMalformedConstructors/stdout.txt +++ b/test/regression/CppiaMalformedConstructors/stdout.txt @@ -4,7 +4,7 @@ super call into a class with no constructor threw: Super constructor call on a c super call into an interface threw: Super constructor call on a class with no constructor super call into a constructor that is a return threw: Constructor is not a function super call into an interface with a constructor threw: Constructor is not a function -super call into a constructor loaded and ran +super call into a constructor threw: Super constructor call to a class that is not the super class new of a constructor that is a return threw: Constructor is not a function new of an interface threw: Could not link new Bar new of a class with no constructor loaded and ran @@ -18,7 +18,7 @@ super call into a class with no constructor threw: Super constructor call on a c super call into an interface threw: Super constructor call on a class with no constructor super call into a constructor that is a return threw: Constructor is not a function super call into an interface with a constructor threw: Constructor is not a function -super call into a constructor loaded and ran +super call into a constructor threw: Super constructor call to a class that is not the super class new of a constructor that is a return threw: Constructor is not a function new of an interface threw: Could not link new Bar new of a class with no constructor loaded and ran diff --git a/test/regression/CppiaMalformedLoads/Files.hx b/test/regression/CppiaMalformedLoads/Files.hx new file mode 100644 index 000000000..040e9da2a --- /dev/null +++ b/test/regression/CppiaMalformedLoads/Files.hx @@ -0,0 +1,15 @@ +// The malformed files, from the cases in libhxcpp #467, base64 encoded +class Files { + public static final all:Array<{name:String, data:String}> = [ + {name: "MAIN with no types", data: "Q1BQSUEgMSAxIGEgMCAwIE1BSU4gMCAwIE5VTEwgUkVTT1VSQ0VTIDAg"}, + {name: "unknown enum constructor", data: "Q1BQSUIKMTQKMCAKMSBBCjEgQgoxIHgKNCBtYWluCjEgYQoxIGIKNyBfaHhfdG1wCjUgdHJhY2UKOCBfaHhfdG1wMQo2IHN0cmluZwoxICAKNiBTY3JpcHQKOSBTY3JpcHQuaHgKOQowIAoxIEUKMyBpbnQKNiBTY3JpcHQKNCB2b2lkCjcgRHluYW1pYwo4IGhheGUuTG9nCjYgU3RyaW5nCjMgU3RkCjIKSQECAQACAQMCAEQDAAABAQEABAQABwQACQUqASwF/o8WAAEBIAEDKgEsBv6QFgABARYBAgE5ASoBLAf+8iQABQUcBggqASwJ/vQkAAcHZREICgEC/o8WOwsPAgL+8iRlAv70JBEICgEC/pAWMA0KDARLEQMEAE0A"}, + {name: "enum constructor with the wrong argument count", data: "Q1BQSUIKMTQKMCAKMSBBCjEgQgoxIHgKNCBtYWluCjEgYQoxIGIKNyBfaHhfdG1wCjUgdHJhY2UKOCBfaHhfdG1wMQo2IHN0cmluZwoxICAKNiBTY3JpcHQKOSBTY3JpcHQuaHgKOQowIAoxIEUKMyBpbnQKNiBTY3JpcHQKNCB2b2lkCjcgRHluYW1pYwo4IGhheGUuTG9nCjYgU3RyaW5nCjMgU3RkCjIKSQECAQACAQMCAEQDAAABAQEABAQABwQACQUqASwF/o8WAAEBIAEBKgEsBv6QFgABARYBAwE5ASoBLAf+8iQABQUcBggqASwJ/vQkAAcHZREICgEC/o8WOwsPAgL+8iRlAv70JBEICgEC/pAWMA0KDARLEQMEAE0A"}, + {name: "two classes with one type id", data: "Q1BQSUIKMjgKMCAKMyBuZXcKNSBoZWxsbwozIGdldAoxIGsKMyBzZXQKMSB2CjYgZXhpc3RzCjYgcmVtb3ZlCjQga2V5cwo4IGl0ZXJhdG9yCjE2IGtleVZhbHVlSXRlcmF0b3IKNCBjb3B5CjggdG9TdHJpbmcKNiBNYXBpc2gKNSBjbGVhcgo0IG1haW4KMSBtCjUgdHJhY2UKOSBtYWluIHJhbiAKMSBhCjYgU2NyaXB0CjkgU2NyaXB0Lmh4CjUgYXJyYXkKNyBjdXJyZW50CjcgaGFzTmV4dAo2IGxlbmd0aAo0IG5leHQKMTYKMCAKNSBQbGFpbgo0IHZvaWQKMyBpbnQKNiBNYXBpc2gKOSBoYXhlLklNYXAKNyBEeW5hbWljCjYgU3RyaW5nCjQgYm9vbAoyOCBoYXhlLml0ZXJhdG9ycy5BcnJheUl0ZXJhdG9yCjEyIEFycmF5LlN0cmluZwo5IEFycmF5LmludAo0IG51bGwKNiBTY3JpcHQKOCBoYXhlLkxvZwo5IEFycmF5LkFueQo0CkQEAAACAQEAAQEABwIACQABAAACAwAHAwAJAS8DOQFEBAABBQsBAQABBAAHAgAJAAEAAAMGAQQABwcGAQT+jxYABwAJAS8DOQEBAAAFAgIEAAcGAAMHAgIE/pAWAAcABv6RFgADAAkAAQAABwgBBAAHBwgBBP6SFgAHAAkBLwg9AQAACAgBBAAHBwgBBP6TFgAHAAkBLwg8AQAACQYABwYACQEvCS0JAQQXCgABAAAKBgAHBgAJAS8JLQkBBBcLAAEAAAsGAAcGAAkBLww+AQAADAUABwUACQEvBD8BAAANBwAHBwAJAS8HOw4BAAAPAgAHAgAJAEQNAAABAQEAEAIABwIACQIqASwR/pgWAAQELQQADwIcDhJlOxMSBAMBAv6YFjsUMBYlFRBECQAABQEBAAEJARcADwcCARf+3xUADwAJAg4dCRg5AA4dCRcC/t8VAgBFRQAXDwACAEVFABgDAAEAABkIAAcIAAkBLwhvHQkYHg8aHQkXAQAAGwYABwYACQEvBiIPHQkXJB0JGEsRDRAATQA="}, + {name: "a string constant past the string table", data: "Q1BQSUIKMjMKMCAKMSBmCjEgcwozIGFiYwoxIGcKMyBrZXkKNCBtYWluCjUgdHJhY2UKNiBTY3JpcHQKOSBTY3JpcHQuaHgKMyBydW4KNCBmaWxsCjQgbHBhZAoxIEEKNCBrZWVwCjIgX2cKMSBpCjMgbmV3CjUgYXJyYXkKNyBjdXJyZW50CjcgaGFzTmV4dAo2IGxlbmd0aAo0IG5leHQKMTMKMCAKNiBTY3JpcHQKNiBTdHJpbmcKNyBEeW5hbWljCjQgdm9pZAo4IGhheGUuTG9nCjkgY3BwLnZtLkdjCjExIFN0cmluZ1Rvb2xzCjEyIEFycmF5LlN0cmluZwozIGludAoyOCBoYXhlLml0ZXJhdG9ycy5BcnJheUl0ZXJhdG9yCjkgQXJyYXkuQW55CjQgYm9vbAoyCkQBAAADAQEAAQIBAgECBwIBAv6PFgACATv+6AMJAS8CAv6PFgEBAAQDAAcDAAkBLwMxAQU5AQEBAAYEAAcEAAkHDwIcBQcRAQQAMAkLCAYPAhwFBxEBAQE+MAkMCAYRBgoBPSoBLAv+uRYAAgIRBwwDOwA7DTlAKgEsDv66FgAICBcIAAkCKgEsD/68FgAJCTkAMwFvAv68Fjn/QA0DAAkCKgEsEP67FgAJCSQC/rwWDiIIAv66FngC/rsWOf6IE2UC/rkWAv67Fg8CHAUHEQEEADAJEggGRAoAAAUBAQARCgESAAsHBAES/t8VAAsACQIOHQoTOQAOHQoSAv7fFQIARUUAEgsAAgBFRQATCQABAAAUDAAHDAAJAS8Mbx0KEx4LFR0KEgEAABYDAAcDAAkBLwMiCx0KEiQdChNLEQEGAE0A"}, + {name: "an enum parameter index past its constructors", data: "Q1BQSUIKOQowIAoxIEIKMSB4CjQgbWFpbgoxIGUKNSB0cmFjZQoyIHg9CjYgU2NyaXB0CjkgU2NyaXB0Lmh4CjcKMCAKMSBFCjMgaW50CjYgU2NyaXB0CjQgdm9pZAo3IER5bmFtaWMKOCBoYXhlLkxvZwoyCkkBAQEBAgIARAMAAAEBAQADBAAHBAAJAioBLAT+jxYAAQEWAQEBOQEJAioBLAL+mRYAAgI1Bf+ghgEAAv6PFg8CHAYFZTsGAv6ZFjAICgcDSxEDAwBNAA=="}, + {name: "a file id past the string table", data: "Q1BQSUEgMSAxIGEgMSAwICAwIE1BSU4gMTAwMDAwMDAwIDAgTlVMTCBSRVNPVVJDRVMgMCA="}, + {name: "break outside a loop", data: "Q1BQSUIKNgowIAoxIGYKNCBtYWluCjUgdHJhY2UKNiBTY3JpcHQKOSBTY3JpcHQuaHgKNQowIAo2IFNjcmlwdAo0IGJvb2wKNCB2b2lkCjggaGF4ZS5Mb2cKMQpEAQAAAgEBAAECAAcCAAkBLwIKAQEAAgMABwMACQEPAhwEAxEBAQAwBQcEAksRAQIATQA="}, + {name: "continue outside a loop", data: "Q1BQSUIKNgowIAoxIGYKNCBtYWluCjUgdHJhY2UKNiBTY3JpcHQKOSBTY3JpcHQuaHgKNQowIAo2IFNjcmlwdAo0IGJvb2wKNCB2b2lkCjggaGF4ZS5Mb2cKMQpEAQAAAgEBAAECAAcCAAkBLwILAQEAAgMABwMACQEPAhwEAxEBAQAwBQcEAksRAQIATQA="}, + {name: "a super call to a class that is not the super class", data: "Q1BQSUIKMzYKMCAKMyBuZXcKMSBhCjEgMQoxIGIKMSAyCjEgYwoxIDMKMSBkCjEgNAo0IG1haW4KNCBpbnRzCjIgX2cKMSBpCjQgcHVzaAozIF9nMQoyIGkxCjMgcnVuCjQgb2JqcwozIF9nMgoyIGkyCjMgYmFkCjMgX2czCjIgaTMKMyBfZzQKMyBfZzUKNiBsZW5ndGgKMSB2CjUgdHJhY2UKMTYgY29ycnVwdGVkIGludHM6IAo2IFNjcmlwdAo5IFNjcmlwdC5oeAo1IGFycmF5CjcgY3VycmVudAo3IGhhc05leHQKNCBuZXh0CjE1CjAgCjQgQmFzZQo0IHZvaWQKNiBTdHJpbmcKNyBEZXJpdmVkCjYgU2NyaXB0CjEyIEFycmF5Lk9iamVjdAozIGludAo5IEFycmF5LmludAo5IGNwcC52bS5HYwo4IGhheGUuTG9nCjI4IGhheGUuaXRlcmF0b3JzLkFycmF5SXRlcmF0b3IKOSBBcnJheS5BbnkKNCBib29sCjcgRHluYW1pYwo0CkQBAAAFAQEAAQEABwIACQQOHQECOwMOHQEEOwUOHQEGOwcOHQEIOwkCAEVFAAIDAAIARUUABAMAAgBFRQAGAwACAEVFAAgDAEQEAAABAQEAAQQABwIACQEVAQBEBQAAAQEBAAoCAAcCAAkJKgEsC/6PFgAGBhcGAAkCKgEsDP6RFgAHBzkAMwFvAv6RFjn/oIYBAAkCKgEsDf6QFgAHByQC/pEWEgYOAQL+jxYXCAQC/pAWAv6QFgL+kBYC/pAWCQIqASwP/pMWAAcHOQAzAW8C/pMWOf+ghgEACQIqASwQ/pIWAAcHJAL+kxYYanh1Av6SFjkDOQI5AAkBDiIGAv6PFgL+khY+EQkRAT0qASwS/pUWAAYGFwYACQIqASwT/pcWAAcHOQAzAW8C/pcWOf5QwwkCKgEsFP6WFgAHByQC/pcWEgYOAQL+lRYtBAAqASwV/pgWAAcHOQAJAioBLBb+nRYABwc5ADMBbwL+nRY5/6CGAQAJAioBLBf+mRYABwckAv6dFhgNIgYC/o8WAv6ZFgkDKgEsGP6bFgAHBzkAKgEsGf6cFgAICCIGAv6PFgL+mRYzAW8C/psWHggaAv6cFgkDKgEsG/6aFgAHByIIAv6cFgL+mxYjAv6bFhhrAv6aFgL+mRYJASMC/pgWDwIcChxlOx0C/pgWMB8oHgpECwAABQEBAAELASAADAcCASD+3xUADAAJAg4dCyE5AA4dCyAC/t8VAgBFRQAgDAACAEVFACEHAAEAACINAAcNAAkBLw1vHQshHgwaHQsgAQAAIw4ABw4ACQEvDiIMHQsgJB0LIUsRBQoATQA="}, + ]; +} diff --git a/test/regression/CppiaMalformedLoads/Main.hx b/test/regression/CppiaMalformedLoads/Main.hx new file mode 100644 index 000000000..662c35c33 --- /dev/null +++ b/test/regression/CppiaMalformedLoads/Main.hx @@ -0,0 +1,28 @@ +import haxe.crypto.Base64; + +// libhxcpp #467: malformed cppia files that crashed the host instead of +// failing to load. SECURITY.md promises a load error. Each is loaded and run +// interpreted, then under the JIT compiling every function as it loads. +class Main { + static function main() { + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + untyped __cpp__("::hx::gLazyJit = false"); + Sys.println(jit ? "jit" : "interpreted"); + for (f in Files.all) { + var result = try { + var module = cpp.cppia.Module.fromData(Base64.decode(f.data).getData()); + module.boot(); + module.run(); + "ran"; + } catch (e:Dynamic) { + "failed to load"; + } + Sys.println(' ${f.name}: $result'); + } + } + // Referenced so --dce no keeps what the scripts link against + var keep:Array = [haxe.ds.StringMap, haxe.ds.IntMap, haxe.ds.ObjectMap, haxe.io.Bytes, haxe.Timer, cpp.vm.Gc, Lambda, StringTools, + EReg, Reflect, Type, Std, Math, Date, StringBuf, Sys, haxe.iterators.ArrayIterator]; + } +} diff --git a/test/regression/CppiaMalformedLoads/build.hxml b/test/regression/CppiaMalformedLoads/build.hxml new file mode 100644 index 000000000..5bfa2fac9 --- /dev/null +++ b/test/regression/CppiaMalformedLoads/build.hxml @@ -0,0 +1,4 @@ +-m Main +--cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaMalformedLoads/stdout.txt b/test/regression/CppiaMalformedLoads/stdout.txt new file mode 100644 index 000000000..ffe5aced6 --- /dev/null +++ b/test/regression/CppiaMalformedLoads/stdout.txt @@ -0,0 +1,26 @@ +interpreted + MAIN with no types: failed to load + unknown enum constructor: failed to load + enum constructor with the wrong argument count: failed to load + two classes with one type id: failed to load + a string constant past the string table: failed to load +Script.hx:10: x=0 + an enum parameter index past its constructors: ran + a file id past the string table: failed to load +Script.hx:7: false + break outside a loop: ran +Script.hx:7: false + continue outside a loop: ran + a super call to a class that is not the super class: failed to load +jit + MAIN with no types: failed to load + unknown enum constructor: failed to load + enum constructor with the wrong argument count: failed to load + two classes with one type id: failed to load + a string constant past the string table: failed to load +Script.hx:10: x=0 + an enum parameter index past its constructors: ran + a file id past the string table: failed to load + break outside a loop: failed to load + continue outside a loop: failed to load + a super call to a class that is not the super class: failed to load diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index 6342a8d49..a8c043dcd 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -22,6 +22,7 @@ CppiaHostMismatch: its host is the first --next section, which -D emscripten doe CppiaJitBuiltins: threads CppiaJitThrowNull: its host is the first --next section, which -D emscripten does not reach CppiaLazyJit: threads, and Emscripten has no cppia JIT +CppiaMalformedLoads: compares the cppia JIT, which Emscripten has not CppiaModuleFree: threads CppiaModuleRetained: threads CppiaNullDynamicArray: its host is the first --next section, which -D emscripten does not reach, and it runs the script under the JIT too, and Emscripten has no cppia JIT From 441d3ac91b8608ebc56851de503344c38ce567c8 Mon Sep 17 00:00:00 2001 From: Chris Speciale Date: Thu, 19 Nov 2026 10:19:26 -0500 Subject: [PATCH 3/3] Move instances in adoptInstances only when old and new code agree adoptInstances compared only field names, storage types and offsets, and docs/CppiaModuleRelease.md said methods could differ in any way. But a call's vtable slot is fixed when its code links, so old code still running, which the docs say finishes normally, called a moved instance by the old slots: with a method added first, foo.b() ran the new a(), and foo.a() reached x(), whose Int was read as a String, crashing the interpreter and giving the JIT a String of garbage. And a moved class's field could hold an instance of a class that stayed, since its layout changed, which new code then read with the new layout, past the object. Now: - methods, their super classes' included, must have the same names in the same slots, and so must the script interfaces the class implements; - fields compare their type names, so a retyped field within one storage type, Array to Array, keeps its instances, as the docs said it did; - a class with a field that can hold an instance of a class that stays stays too, to a fixed point. The report says which rule kept a class, and the docs describe the rule. Test: CppiaAdoptLayouts, the two cases, interpreted and under the JIT. Before, the first crashed the interpreter and the second read past the object; now neither class moves and old code keeps its own results. CppiaAdoptInstances, whose classes change only method bodies, still moves them, on Windows x64. Fixes #464 --- Changes.md | 1 + docs/CppiaModuleRelease.md | 4 +- src/hx/cppia/CppiaModule.cpp | 71 ++++++++++++++++++- test/regression/CppiaAdoptLayouts/Main.hx | 40 +++++++++++ test/regression/CppiaAdoptLayouts/build.hxml | 20 ++++++ .../CppiaAdoptLayouts/fields/new/Script.hx | 21 ++++++ .../CppiaAdoptLayouts/fields/old/Script.hx | 22 ++++++ .../CppiaAdoptLayouts/methods/new/Script.hx | 13 ++++ .../CppiaAdoptLayouts/methods/old/Script.hx | 21 ++++++ test/regression/CppiaAdoptLayouts/stdout.txt | 24 +++++++ test/regression/emscripten-skip.txt | 1 + 11 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 test/regression/CppiaAdoptLayouts/Main.hx create mode 100644 test/regression/CppiaAdoptLayouts/build.hxml create mode 100644 test/regression/CppiaAdoptLayouts/fields/new/Script.hx create mode 100644 test/regression/CppiaAdoptLayouts/fields/old/Script.hx create mode 100644 test/regression/CppiaAdoptLayouts/methods/new/Script.hx create mode 100644 test/regression/CppiaAdoptLayouts/methods/old/Script.hx create mode 100644 test/regression/CppiaAdoptLayouts/stdout.txt diff --git a/Changes.md b/Changes.md index ecbb00d29..0fea2b483 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed cppia's adoptInstances moving instances that code then used with the wrong layout: a class whose methods changed order or number moved, so old code still running called its moved instances by the old vtable slots, reaching another method or crashing, and a class moved while a field could hold an instance of a class that stayed, which new code read past its end. Such classes keep their instances now, and the report says why: methods must be the same, in the same slots, and so must the script interfaces a class implements, fields compare their types by name, and a class stays with the classes its fields can hold (libhxcpp #464) * Fixed malformed cppia files crashing the host instead of failing to load: a MAIN in a module with no types, an unknown script enum constructor or one called with the wrong number of arguments, two classes with one type id, a string constant or file id past the string table, which grew the table to it, an enum parameter index past the value, break or continue outside a loop under the JIT, and a super constructor call to a class other than the super class. Each fails to load, or is read safely (libhxcpp #467) * Fixed cppia's interpreter and JIT giving different results for array builtins, both of them now matching C++: under the JIT, a[i] += x and the other compound assignments on an Array stored x alone, a[i]++, insert and toString on one did nothing, a Bool stored in an Array became an Int, bitwise compound assignments on an Array> did nothing, an Int array read out of range as a String gave "null", and ++d[i] on a Dynamic as a statement decremented. Interpreted, a typed indexOf or lastIndexOf used as a Float gave 0, and join, toString and the iterators used as a statement did not run (libhxcpp #468) * Fixed a Haxe 4 haxe.atomic.AtomicInt kept in a field reading freed memory after a collection: Haxe 4's AtomicInt on C++ is a cpp.Pointer into a one-element array nothing else holds, and the collector ignored cpp.Pointer fields, so it freed the array (HaxeFoundation/hxcpp#1225; Haxe 5 changed AtomicInt instead, in HaxeFoundation/haxe#12236). A cpp.Pointer field, or a pointer boxed in a Dynamic, now keeps what it points into in the GC heap, as a pointer on the stack does, and pins it under HXCPP_GC_MOVING. Of 10,000 atomics in fields, 9,999 read freed memory after a collection and none do now diff --git a/docs/CppiaModuleRelease.md b/docs/CppiaModuleRelease.md index 42c17d271..6a304cac7 100644 --- a/docs/CppiaModuleRelease.md +++ b/docs/CppiaModuleRelease.md @@ -82,9 +82,9 @@ untyped previous.release(); - `name`, the class's name. - `moved`, how many of its instances moved. -- `reason`, null when they could move, otherwise why not, naming the first field that differs. +- `reason`, null when they could move, otherwise why not, naming the first field or method that differs, or the field that can hold an instance that stays. -The rule is strict, as an instance stays where it is, at its size. Both classes must extend the same host class, through super classes of the same names, implement the same interfaces, and have the same fields, dynamic functions included, with the same names, storage types and offsets. Methods can differ in any way, so editing method bodies, the usual case, is covered. A class that adds, removes, retypes or reorders a field keeps its instances, which keep the old module loaded until they go. +The rule is strict, as an instance stays where it is, at its size, and old code still running calls it by the slots its calls linked with. Both classes must extend the same host class, through super classes of the same names, implement the same interfaces, and have the same fields, dynamic functions included, with the same names, types and offsets. They must have the same methods, their super classes' included, in the same order, so that each keeps its slot, and so must the script interfaces they implement. Method bodies can differ in any way, so editing them, the usual case, is covered. A class that adds, removes, retypes or reorders a field or a method keeps its instances, which keep the old module loaded until they go. So does a class with a field that can hold an instance of a class that stays, since the new code would read that instance with the new layout. What does not move: diff --git a/src/hx/cppia/CppiaModule.cpp b/src/hx/cppia/CppiaModule.cpp index 2bd8e9666..b370672a8 100644 --- a/src/hx/cppia/CppiaModule.cpp +++ b/src/hx/cppia/CppiaModule.cpp @@ -1322,12 +1322,42 @@ static std::string slotDifference(const char *inWhat, CppiaClassInfo *inFrom, st return std::string(inWhat) + " " + name + " is not in the new class"; if (name!=inTo->cppia.identStr(inNew[i]->nameId)) return std::string(inWhat) + " " + name + " is not where it was"; - if (inOld[i]->storeType!=inNew[i]->storeType || inOld[i]->offset!=inNew[i]->offset) + if (inOld[i]->storeType!=inNew[i]->storeType || inOld[i]->offset!=inNew[i]->offset || + typeName(inFrom->cppia.types[inOld[i]->typeId])!=typeName(inTo->cppia.types[inNew[i]->typeId])) return std::string(inWhat) + " " + name + " changed its type"; } return std::string(); } +// The first method of a class, its super classes' included, that is not in the +// same vtable slot in both. A call's slot is fixed when its code links, so old +// code still running would call a moved instance by its old slots, reaching +// another method or reading its return as another type (libhxcpp #464) +static std::string methodDifference(CppiaClassInfo *inFrom, CppiaClassInfo *inTo) +{ + if (inFrom->memberFunctions.size()!=inTo->memberFunctions.size()) + return "its methods are not the same"; + for(size_t i=0;imemberFunctions.size();i++) + if (inFrom->memberFunctions[i]->name!=inTo->memberFunctions[i]->name || + inFrom->memberFunctions[i]->vtableSlot!=inTo->memberFunctions[i]->vtableSlot) + return "method " + inFrom->memberFunctions[i]->name + " is not where it was"; + return std::string(); +} + +// The script interfaces a class implements, by name, with their class info +static std::map scriptInterfaces(CppiaClassInfo *inInfo) +{ + std::map result; + for(CppiaClassInfo *info=inInfo; info; info=info->superType ? info->superType->cppiaClass : 0) + for(size_t i=0;iimplements.size();i++) + { + TypeData *type = info->cppia.types[info->implements[i]]; + if (type->cppiaClass) + result[typeName(type)] = type->cppiaClass; + } + return result; +} + // Why instances of inFrom cannot become instances of inTo, or empty when they can static std::string incompatibility(CppiaClassInfo *inFrom, CppiaClassInfo *inTo) { @@ -1352,6 +1382,17 @@ static std::string incompatibility(CppiaClassInfo *inFrom, CppiaClassInfo *inTo) if (inFrom->classSize!=inTo->classSize || inFrom->extraData!=inTo->extraData || inFrom->dynamicMapOffset!=inTo->dynamicMapOffset || inFrom->containsPointers!=inTo->containsPointers) return "its instances are laid out otherwise"; + std::string method = methodDifference(inFrom, inTo); + if (!method.empty()) + return method; + std::map fromInterfaces = scriptInterfaces(inFrom); + std::map toInterfaces = scriptInterfaces(inTo); + for(std::map::iterator i=fromInterfaces.begin(); i!=fromInterfaces.end(); ++i) + { + std::map::iterator other = toInterfaces.find(i->first); + if (other==toInterfaces.end() || !methodDifference(i->second, other->second).empty()) + return "interface " + i->first + " changed its methods"; + } return std::string(); } @@ -1398,6 +1439,34 @@ static Array adoptInstances(CppiaModule *inNext, CppiaModule *inPreviou } } + // New code reads a moved instance's fields with the new classes' layouts, + // so a field that can hold an instance staying in the old class, whose + // layout changed, would be read wrongly: the class holding it stays too, + // until nothing changes (libhxcpp #464) + for(bool changed=true; changed; ) + { + changed = false; + for(std::map::iterator i=adoptions.begin(); i!=adoptions.end(); ++i) + { + if (!i->second.to) + continue; + std::vector &vars = i->first->memberVars; + for(size_t v=0;vfirst->cppia.types[vars[v]->typeId]->cppiaClass; + std::map::iterator heldAdoption = held ? adoptions.find(held) : adoptions.end(); + if (heldAdoption!=adoptions.end() && !heldAdoption->second.to) + { + i->second.to = 0; + i->second.reason = std::string("field ") + i->first->cppia.identStr(vars[v]->nameId) + " can hold a " + + held->name + ", which stays in the old class"; + changed = true; + break; + } + } + } + } + sAdoptions = &adoptions; censusWithVisitor(adoptVisit); sAdoptions = 0; diff --git a/test/regression/CppiaAdoptLayouts/Main.hx b/test/regression/CppiaAdoptLayouts/Main.hx new file mode 100644 index 000000000..d55581f76 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/Main.hx @@ -0,0 +1,40 @@ +import haxe.io.Path; + +// libhxcpp #464. adoptInstances moved instances whose methods were not in the +// same vtable slots, so old code still running called the wrong method, or +// read its return as another type, and moved instances whose fields held +// instances that stayed, which new code read with the new layout. Neither +// moves now, and the report says why. +class Main { + static function scenario(dir:String, name:String) { + var load = (which:String) -> cpp.cppia.Module.fromData(sys.io.File.getBytes(Path.join([dir, '$name-$which.cppia'])).getData()); + var old = load("old"); + old.boot(); + old.run(); + var oldCls = old.resolveClass("Script"); + Sys.println(" before: " + Reflect.callMethod(oldCls, Reflect.field(oldCls, "before"), [])); + var next = load("new"); + next.boot(); + var report:Array = untyped next.adoptInstances(old); + for (r in report) + Sys.println(" adopt " + r.name + ": moved " + r.moved + ", " + r.reason); + Sys.println(" after, old code: " + Reflect.callMethod(oldCls, Reflect.field(oldCls, "after"), [])); + cpp.vm.Gc.run(true); + } + + static function main() { + var dir = Path.directory(Sys.programPath()); + for (jit in [false, true]) { + cpp.cppia.Host.enableJit(jit); + Sys.println(jit ? "jit" : "interpreted"); + for (name in ["methods", "fields"]) { + Sys.println(name); + try { + scenario(dir, name); + } catch (e:Dynamic) { + Sys.println(" error: " + e); + } + } + } + } +} diff --git a/test/regression/CppiaAdoptLayouts/build.hxml b/test/regression/CppiaAdoptLayouts/build.hxml new file mode 100644 index 000000000..1ab45b696 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/build.hxml @@ -0,0 +1,20 @@ +-cp methods/old +--main Script +-cppia bin/methods-old.cppia +--next +-cp methods/new +--main Script +-cppia bin/methods-new.cppia +--next +-cp fields/old +--main Script +-cppia bin/fields-old.cppia +--next +-cp fields/new +--main Script +-cppia bin/fields-new.cppia +--next +--main Main +-cpp bin +-D scriptable +--dce no diff --git a/test/regression/CppiaAdoptLayouts/fields/new/Script.hx b/test/regression/CppiaAdoptLayouts/fields/new/Script.hx new file mode 100644 index 000000000..209c21145 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/fields/new/Script.hx @@ -0,0 +1,21 @@ +// New generation: Parent is laid out the same, so its instances move, but Child +// gained a String before n, so its instances stay old. The moved Parent's c is +// still an old Child, which the new code reads with the new Child's layout +class Child { + public var s:String = "new child"; + public var n:Int = 2; + public function new() {} +} + +class Parent { + public var c:Child; + public function new() c = new Child(); + public function get():Int { + Sys.println(" new code reads c.s -> " + (c.s == null ? "null" : "length " + c.s.length)); + return c.n; + } +} + +class Script { + static function main() {} +} diff --git a/test/regression/CppiaAdoptLayouts/fields/old/Script.hx b/test/regression/CppiaAdoptLayouts/fields/old/Script.hx new file mode 100644 index 000000000..f4e8cbdc9 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/fields/old/Script.hx @@ -0,0 +1,22 @@ +// Old generation: Child holds an Int only +class Child { + public var n:Int = 1; + public function new() {} +} + +class Parent { + public var c:Child; + public function new() c = new Child(); + public function get():Int return c.n; +} + +class Script { + public static var p:Parent; + public static function before():String return "p.get() = " + p.get(); + // Old code handing the moved Parent to new code, as a host callback would + public static function after():String { + var get = Reflect.field(p, "get"); + return "p.get() = " + Reflect.callMethod(p, get, []); + } + static function main() p = new Parent(); +} diff --git a/test/regression/CppiaAdoptLayouts/methods/new/Script.hx b/test/regression/CppiaAdoptLayouts/methods/new/Script.hx new file mode 100644 index 000000000..5dc5469b7 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/methods/new/Script.hx @@ -0,0 +1,13 @@ +// New generation: the same fields, so Foo's instances move, but a method added +// first shifts a and b one vtable slot on ("Methods can differ in any way") +class Foo { + public var v:Int = 7; + public function new() {} + public function x():Int return 12345; + public function a():String return "new a " + v; + public function b():String return "new b " + v; +} + +class Script { + static function main() {} +} diff --git a/test/regression/CppiaAdoptLayouts/methods/old/Script.hx b/test/regression/CppiaAdoptLayouts/methods/old/Script.hx new file mode 100644 index 000000000..bde858046 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/methods/old/Script.hx @@ -0,0 +1,21 @@ +// Old generation: Foo.a and Foo.b take the first two script vtable slots +class Foo { + public var v:Int = 7; + public function new() {} + public function a():String return "old a " + v; + public function b():String return "old b " + v; +} + +class Script { + public static var foo:Foo; + // Old code calling methods on an instance that adoptInstances moves + public static function before():String return foo.b() + " / " + foo.a(); + public static function after():String { + var s = foo.b(); + Sys.println(" old code foo.b() -> " + s); + var t = foo.a(); + Sys.println(" old code foo.a() -> length " + t.length); + return s + " / " + t; + } + static function main() foo = new Foo(); +} diff --git a/test/regression/CppiaAdoptLayouts/stdout.txt b/test/regression/CppiaAdoptLayouts/stdout.txt new file mode 100644 index 000000000..0f0befda9 --- /dev/null +++ b/test/regression/CppiaAdoptLayouts/stdout.txt @@ -0,0 +1,24 @@ +interpreted +methods + before: old b 7 / old a 7 + adopt Foo: moved 0, its methods are not the same + old code foo.b() -> old b 7 + old code foo.a() -> length 7 + after, old code: old b 7 / old a 7 +fields + before: p.get() = 1 + adopt Child: moved 0, field n is not where it was + adopt Parent: moved 0, field c can hold a Child, which stays in the old class + after, old code: p.get() = 1 +jit +methods + before: old b 7 / old a 7 + adopt Foo: moved 0, its methods are not the same + old code foo.b() -> old b 7 + old code foo.a() -> length 7 + after, old code: old b 7 / old a 7 +fields + before: p.get() = 1 + adopt Child: moved 0, field n is not where it was + adopt Parent: moved 0, field c can hold a Child, which stays in the old class + after, old code: p.get() = 1 diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index a8c043dcd..76c27c1d1 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -14,6 +14,7 @@ CffiPrimeUnicodeFallback: loads an ndll it builds, and Emscripten links statical CffiQuietMissingPrimitive: loads an ndll it builds, and Emscripten links statically ConditionTimedWait: threads CppiaAdoptInstances: threads (std::thread, which needs a pthreads build) +CppiaAdoptLayouts: compares the cppia JIT, which Emscripten has not CppiaArrayAccess: its host is the first --next section, which -D emscripten does not reach CppiaBreakpoints: threads CppiaHostArgs: processes