diff --git a/machine-tests/src/test/java/run/endive/testing/MachinesTest.java b/machine-tests/src/test/java/run/endive/testing/MachinesTest.java index ac27a2e13..76f935e24 100644 --- a/machine-tests/src/test/java/run/endive/testing/MachinesTest.java +++ b/machine-tests/src/test/java/run/endive/testing/MachinesTest.java @@ -390,6 +390,30 @@ public void shouldCallIndirectCrossModuleTypeIndex( assertEquals(42, instance.export("call-other").apply()[0]); } + @ParameterizedTest(name = "{0}") + @MethodSource("crossModuleMachineFactories") + public void shouldCallTableInitialiserInItsDefiningInstance( + String name, + Function callerFactory, + Function definerFactory) { + var store = new Store(); + + var definer = + Instance.builder(loadModule("compiled/table_init_owner-export.wat.wasm")) + .withMachineFactory(definerFactory) + .build(); + store.register("test", definer); + + var caller = + Instance.builder(loadModule("compiled/table_init_owner-import.wat.wasm")) + .withImportValues(store.toImportValues()) + .withMachineFactory(callerFactory) + .build(); + + assertEquals(42, caller.export("call").apply(1)[0]); + assertEquals(definer, definer.table(0).instance(1)); + } + @Test public void tailcallReturnCallAot() { var instance = diff --git a/redline/api/src/main/java/run/endive/redline/experimental/api/NativeCodeSerializer.java b/redline/api/src/main/java/run/endive/redline/experimental/api/NativeCodeSerializer.java index 8cd467e48..57f70f39a 100644 --- a/redline/api/src/main/java/run/endive/redline/experimental/api/NativeCodeSerializer.java +++ b/redline/api/src/main/java/run/endive/redline/experimental/api/NativeCodeSerializer.java @@ -30,7 +30,7 @@ public final class NativeCodeSerializer { private static final int MAGIC = 0x434C344A; // "CL4J" - private static final int VERSION = 3; + private static final int VERSION = 4; private NativeCodeSerializer() {} diff --git a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java index 9209feca8..38103d04f 100644 --- a/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java +++ b/redline/api/src/main/java/run/endive/redline/experimental/api/internal/CtxBuffer.java @@ -9,9 +9,9 @@ * 0 i64 funcTablePtr Pointer to function pointer table * 8 i64 trampolinePtr Upcall stub for CALL_INDIRECT fallback * 16 i32 trapCode Trap code written by native pre-checks - * 20 i32 typeId CALL_INDIRECT: expected type index - * 24 i32 tableIdx CALL_INDIRECT: table index - * 28 i32 elemIdx CALL_INDIRECT: table element index + * 20 i32 typeId Foreign CALL_INDIRECT: expected type index + * 24 i32 tableIdx Foreign CALL_INDIRECT: table index + * 28 i32 elemIdx Foreign CALL_INDIRECT: table element index * 32 i32 argCount Number of call arguments * 36 i32 memGrowDelta Page count delta for memory.grow * 40 i64 argsPtr Pointer to separate args buffer @@ -20,6 +20,8 @@ * 64 i64 memsetPtr Pointer to libc memset * 72 i64 interruptFlag Non-zero = interrupt requested * 80 i64 memoryPagesPtr Shared memory: pointer to its current page count (i32) + * 88 i64 ownerId Unique id of this instance, held by the entries it owns + * 96 i64 tableWritersPtr Pointer to an i32 per table, set once it keeps this instance * 200 i64 globalsPtr Pointer to globals buffer * 208 i64 memGrowPtr Upcall stub for memory.grow * 216 i32 memoryPages Current memory page count @@ -54,6 +56,8 @@ private CtxBuffer() {} public static final int MEMSET_PTR = 64; public static final int INTERRUPT_FLAG = 72; public static final int MEMORY_PAGES_PTR = 80; + public static final int OWNER_ID = 88; + public static final int TABLE_WRITERS_PTR = 96; public static final int GLOBALS_PTR = 200; public static final int MEM_GROW_PTR = 208; @@ -86,11 +90,14 @@ private CtxBuffer() {} public static final int TABLE_SIZE_OFFSET = 0; public static final int TABLE_MAX_OFFSET = 4; public static final int TABLE_ENTRIES_OFFSET = 8; - public static final int TABLE_ENTRY_SIZE = 16; + public static final int TABLE_ENTRY_SIZE = 24; public static final int ENTRY_TYPE_IDX_OFFSET = 0; public static final int ENTRY_FUNC_ID_OFFSET = 4; public static final int ENTRY_FUNC_PTR_OFFSET = 8; + // Owner id of the native instance defining the function, or 0 if there is none + public static final int ENTRY_OWNER_ID_OFFSET = 16; + public static int argOffset(int i) { return 8 * i; } diff --git a/redline/api/src/test/java/run/endive/redline/experimental/api/NativeCodeSerializerTest.java b/redline/api/src/test/java/run/endive/redline/experimental/api/NativeCodeSerializerTest.java index 8b92e6c70..ed99e280b 100644 --- a/redline/api/src/test/java/run/endive/redline/experimental/api/NativeCodeSerializerTest.java +++ b/redline/api/src/test/java/run/endive/redline/experimental/api/NativeCodeSerializerTest.java @@ -54,15 +54,18 @@ public void roundTripKeepsTheLayout() throws IOException { } @Test - public void aVersionOneFileIsRejected() throws IOException { - var out = new ByteArrayOutputStream(); - var dos = new DataOutputStream(out); - dos.writeInt(0x434C344A); - dos.writeInt(1); - dos.writeInt(0); + public void anOlderVersionIsRejected() throws IOException { + // Version 3 code reads 16-byte table entries + for (int version = 1; version <= 3; version++) { + var out = new ByteArrayOutputStream(); + var dos = new DataOutputStream(out); + dos.writeInt(0x434C344A); + dos.writeInt(version); + dos.writeInt(0); - var e = assertThrows(IOException.class, () -> deserialize(out.toByteArray())); - assertTrue(e.getMessage().contains("version"), e.getMessage()); + var e = assertThrows(IOException.class, () -> deserialize(out.toByteArray())); + assertTrue(e.getMessage().contains("version"), e.getMessage()); + } } @Test diff --git a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java index aadda55be..f863dbb0a 100644 --- a/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java +++ b/redline/compiler/src/main/java/run/endive/redline/experimental/compiler/internal/NativeEmitters.java @@ -796,7 +796,25 @@ static void emitCall(EmitContext ctx, AnnotatedInstruction ins) { reloadMemBase(ctx); } + /** Emits call_indirect. See {@link #emitIndirectCall}. */ static void emitCallIndirect(EmitContext ctx, AnnotatedInstruction ins) { + emitIndirectCall(ctx, ins, false); + } + + /** + * Emits call_indirect, or return_call_indirect when {@code tail} is set. + * + *

A table entry's function pointer, type index and expected context + * belong to the instance that owns the function. The entry is called + * directly only from that instance's context. Any other entry goes through + * the runner, which checks the signature by structure and runs the call in + * the owning instance. + * + * @param ctx the emit context of the calling function + * @param ins the call_indirect or return_call_indirect instruction + * @param tail whether the call replaces the calling function's frame + */ + private static void emitIndirectCall(EmitContext ctx, AnnotatedInstruction ins, boolean tail) { int typeId = (int) ins.operands()[0]; int tableIdx = (int) ins.operands()[1]; FunctionType targetType = (FunctionType) ctx.module.typeSection().getType(typeId); @@ -813,12 +831,8 @@ static void emitCallIndirect(EmitContext ctx, AnnotatedInstruction ins) { var b = ctx.bridge.exports(); int zero = b.emitIconst32(0); - int ctxPtr = b.useVar(ctx.ctxPtrVar); - // 1. Load tablePtr from TABLE_PTRS[tableIdx] - int tablePtrsPtr = b.emitLoadI64(ctxPtr, zero, CtxBuffer.TABLE_PTRS); - int tableOffset = b.emitIconst32(tableIdx * 8); - int tablePtr = b.emitLoadI64(tablePtrsPtr, tableOffset, 0); + int tablePtr = loadTablePtr(ctx, tableIdx); // 2. Bounds check: elemIdx >= table.size → trap "undefined element" int tableSize = b.emitLoadI32(tablePtr, zero, CtxBuffer.TABLE_SIZE_OFFSET); @@ -829,33 +843,50 @@ static void emitCallIndirect(EmitContext ctx, AnnotatedInstruction ins) { fillTrapBlock(ctx, trapOobBlock, CtxBuffer.TRAP_UNDEFINED_ELEMENT); b.switchToBlock(afterOobBlock); - // 3. Calculate entry offset: elemIdx * 16 (TABLE_ENTRY_SIZE) + // 3. Calculate entry offset: elemIdx * TABLE_ENTRY_SIZE int entryOffset = b.emitImul(elemIdx, b.emitIconst32(CtxBuffer.TABLE_ENTRY_SIZE)); - // 4. Load funcPtr from entry: tablePtr[ENTRIES_OFFSET + entryOffset + FUNC_PTR_OFFSET] - int funcPtr = + // 4. Write args to argsBuffer (read by import stubs, and by the runner + // for a foreign entry) + int argsPtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.ARGS_PTR); + b.emitStoreI32( + b.useVar(ctx.ctxPtrVar), zero, b.emitIconst32(argCount), CtxBuffer.ARG_COUNT); + for (int i = 0; i < argCount; i++) { + int widened = ctx.widenToI64(argVals[i], targetType.params().get(i)); + b.emitStoreI64(argsPtr, zero, widened, CtxBuffer.argOffset(i)); + } + + // 5. Call an entry this instance owns directly; any other, null included, via the runner + int ownerId = b.emitLoadI64( tablePtr, entryOffset, - CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); + CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_OWNER_ID_OFFSET); + int selfId = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.OWNER_ID); + int isLocal = b.emitIcmp(0, ownerId, selfId); // EQ - // 5. Null check: funcPtr == 0 → trap - int zero64 = b.emitIconst64(0, 0); - int isNull = b.emitIcmp(0, funcPtr, zero64); // EQ - int trapNullBlock = b.createBlock(); - int afterNullBlock = b.createBlock(); - b.emitBrif(isNull, trapNullBlock, afterNullBlock); - fillTrapBlock(ctx, trapNullBlock, CtxBuffer.TRAP_UNINITIALIZED_ELEMENT); - b.switchToBlock(afterNullBlock); - - // 6. Type check: entry.typeIdx == expectedCanonicalType + boolean calleeMultiReturn = targetType.returns().size() > 1; + boolean singleReturn = targetType.returns().size() == 1; + int localBlock = b.createBlock(); + int foreignBlock = b.createBlock(); + int mergeBlock = tail ? -1 : b.createBlock(); + int mergeParam = + !tail && singleReturn + ? b.appendBlockParam( + mergeBlock, + EmitContext.valTypeToBridgeType(targetType.returns().get(0))) + : -1; + b.emitBrif(isLocal, localBlock, foreignBlock); + + // 6. Local entry: type check against the canonical type, then call + // funcPtr + b.switchToBlock(localBlock); int actualType = b.emitLoadI32( tablePtr, entryOffset, CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_TYPE_IDX_OFFSET); - int canonicalTypeId = ctx.canonicalTypeMap[typeId]; - int expectedType = b.emitIconst32(canonicalTypeId); + int expectedType = b.emitIconst32(ctx.canonicalTypeMap[typeId]); int typeMismatch = b.emitIcmp(1, b.emitUextendI64(actualType), b.emitUextendI64(expectedType)); // NE int trapTypeBlock = b.createBlock(); @@ -864,46 +895,81 @@ static void emitCallIndirect(EmitContext ctx, AnnotatedInstruction ins) { fillTrapBlock(ctx, trapTypeBlock, CtxBuffer.TRAP_INDIRECT_CALL_TYPE_MISMATCH); b.switchToBlock(afterTypeBlock); - // 7. Determine if callee might be multi-return (>1 return) - boolean calleeMultiReturn = targetType.returns().size() > 1; - - // Write args to argsBuffer (for import stubs that read from it) - int argsPtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.ARGS_PTR); - int zero2 = b.emitIconst32(0); - b.emitStoreI32( - b.useVar(ctx.ctxPtrVar), zero2, b.emitIconst32(argCount), CtxBuffer.ARG_COUNT); + int funcPtr = + b.emitLoadI64( + tablePtr, + entryOffset, + CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); + // A function the image left uncompiled has no address + int noCode = b.emitIcmp(0, funcPtr, b.emitIconst64(0, 0)); // EQ + int trapNoCodeBlock = b.createBlock(); + int afterNoCodeBlock = b.createBlock(); + b.emitBrif(noCode, trapNoCodeBlock, afterNoCodeBlock); + fillTrapBlock(ctx, trapNoCodeBlock, CtxBuffer.TRAP_UNINITIALIZED_ELEMENT); + b.switchToBlock(afterNoCodeBlock); + int sigRef = + calleeMultiReturn + ? ctx.getOrCreateMultiReturnSigRef(targetType) + : ctx.getOrCreateSigRef(targetType); + b.pushCallArg(b.useVar(ctx.memBaseVar)); + b.pushCallArg(b.useVar(ctx.ctxPtrVar)); for (int i = 0; i < argCount; i++) { - int widened = ctx.widenToI64(argVals[i], targetType.params().get(i)); - b.emitStoreI64(argsPtr, zero2, widened, CtxBuffer.argOffset(i)); + b.pushCallArg(argVals[i]); } - - // 8. Build SigRef and call (funcPtr loaded directly from table entry) - int sigRef; - if (calleeMultiReturn) { - sigRef = ctx.getOrCreateMultiReturnSigRef(targetType); + if (tail) { + b.emitReturnCallIndirect(sigRef, funcPtr); } else { - sigRef = ctx.getOrCreateSigRef(targetType); + int rawResult = b.emitCallIndirect(sigRef, funcPtr); + if (singleReturn) { + b.emitJumpWithArg(mergeBlock, rawResult); + } else { + b.emitJump(mergeBlock); + } } - b.pushCallArg(b.useVar(ctx.memBaseVar)); + // 7. Foreign entry: the runner checks the type and calls the owning + // instance. A single result comes back as raw bits, several are left + // in argsBuffer. + b.switchToBlock(foreignBlock); + b.emitStoreI32(b.useVar(ctx.ctxPtrVar), zero, b.emitIconst32(typeId), CtxBuffer.TYPE_ID); + b.emitStoreI32( + b.useVar(ctx.ctxPtrVar), zero, b.emitIconst32(tableIdx), CtxBuffer.TABLE_IDX); + b.emitStoreI32(b.useVar(ctx.ctxPtrVar), zero, elemIdx, CtxBuffer.ELEM_IDX); + int trampolinePtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.TRAMPOLINE_PTR); b.pushCallArg(b.useVar(ctx.ctxPtrVar)); - for (int i = 0; i < argCount; i++) { - b.pushCallArg(argVals[i]); + int raw = b.emitCallIndirect(ctx.getOrCreateTrampolineSigRef(), trampolinePtr); + // Not a real tail call: a chain of them across instances grows the stack + if (tail) { + emitTrapCheck(ctx); + if (singleReturn) { + b.emitReturn(ctx.narrowFromI64ForType(raw, targetType.returns().get(0))); + } else if (calleeMultiReturn) { + // Multi-value results are already in argsBuffer, where this + // function returns them + b.emitReturn(b.emitIconst64(0, 0)); + } else { + b.emitReturnVoid(); + } + return; + } + if (singleReturn) { + b.emitJumpWithArg( + mergeBlock, ctx.narrowFromI64ForType(raw, targetType.returns().get(0))); + } else { + b.emitJump(mergeBlock); } - int rawResult = b.emitCallIndirect(sigRef, funcPtr); + // 8. Handle results + b.switchToBlock(mergeBlock); emitTrapCheck(ctx); - - // 9. Handle results if (calleeMultiReturn) { - int zero3 = b.emitIconst32(0); - int argsPtr2 = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero3, CtxBuffer.ARGS_PTR); + int argsPtr2 = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.ARGS_PTR); for (int i = 0; i < targetType.returns().size(); i++) { - int raw = b.emitLoadI64(argsPtr2, zero3, CtxBuffer.argOffset(i)); - ctx.valueStack.push(ctx.narrowFromI64ForType(raw, targetType.returns().get(i))); + int value = b.emitLoadI64(argsPtr2, zero, CtxBuffer.argOffset(i)); + ctx.valueStack.push(ctx.narrowFromI64ForType(value, targetType.returns().get(i))); } - } else if (!targetType.returns().isEmpty()) { - ctx.valueStack.push(rawResult); + } else if (singleReturn) { + ctx.valueStack.push(mergeParam); } // Reload memBase — callee may have called memory.grow @@ -967,91 +1033,9 @@ static void emitReturnCall(EmitContext ctx, AnnotatedInstruction ins) { ctx.bridge.exports().emitReturnCallIndirect(sigRef, funcPtr); } + /** Emits return_call_indirect. See {@link #emitIndirectCall}. */ static void emitReturnCallIndirect(EmitContext ctx, AnnotatedInstruction ins) { - int typeId = (int) ins.operands()[0]; - int tableIdx = (int) ins.operands()[1]; - FunctionType targetType = (FunctionType) ctx.module.typeSection().getType(typeId); - - int elemIdx = ctx.valueStack.pop(); - - int argCount = targetType.params().size(); - int[] argVals = new int[argCount]; - for (int i = argCount - 1; i >= 0; i--) { - argVals[i] = ctx.valueStack.pop(); - } - - var b = ctx.bridge.exports(); - int zero = b.emitIconst32(0); - int ctxPtr = b.useVar(ctx.ctxPtrVar); - - int tablePtrsPtr = b.emitLoadI64(ctxPtr, zero, CtxBuffer.TABLE_PTRS); - int tableOffset = b.emitIconst32(tableIdx * 8); - int tablePtr = b.emitLoadI64(tablePtrsPtr, tableOffset, 0); - - int tableSize = b.emitLoadI32(tablePtr, zero, CtxBuffer.TABLE_SIZE_OFFSET); - int oobCheck = b.emitIcmp(9, b.emitUextendI64(elemIdx), b.emitUextendI64(tableSize)); - int trapOobBlock = b.createBlock(); - int afterOobBlock = b.createBlock(); - b.emitBrif(oobCheck, trapOobBlock, afterOobBlock); - fillTrapBlock(ctx, trapOobBlock, CtxBuffer.TRAP_UNDEFINED_ELEMENT); - b.switchToBlock(afterOobBlock); - - int entryOffset = b.emitImul(elemIdx, b.emitIconst32(CtxBuffer.TABLE_ENTRY_SIZE)); - - int funcPtr = - b.emitLoadI64( - tablePtr, - entryOffset, - CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); - - int zero64 = b.emitIconst64(0, 0); - int isNull = b.emitIcmp(0, funcPtr, zero64); - int trapNullBlock = b.createBlock(); - int afterNullBlock = b.createBlock(); - b.emitBrif(isNull, trapNullBlock, afterNullBlock); - fillTrapBlock(ctx, trapNullBlock, CtxBuffer.TRAP_UNINITIALIZED_ELEMENT); - b.switchToBlock(afterNullBlock); - - int actualType = - b.emitLoadI32( - tablePtr, - entryOffset, - CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_TYPE_IDX_OFFSET); - int canonicalTypeId = ctx.canonicalTypeMap[typeId]; - int expectedType = b.emitIconst32(canonicalTypeId); - int typeMismatch = - b.emitIcmp(1, b.emitUextendI64(actualType), b.emitUextendI64(expectedType)); - int trapTypeBlock = b.createBlock(); - int afterTypeBlock = b.createBlock(); - b.emitBrif(typeMismatch, trapTypeBlock, afterTypeBlock); - fillTrapBlock(ctx, trapTypeBlock, CtxBuffer.TRAP_INDIRECT_CALL_TYPE_MISMATCH); - b.switchToBlock(afterTypeBlock); - - boolean calleeMultiReturn = targetType.returns().size() > 1; - - int argsPtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.ARGS_PTR); - int zero2 = b.emitIconst32(0); - b.emitStoreI32( - b.useVar(ctx.ctxPtrVar), zero2, b.emitIconst32(argCount), CtxBuffer.ARG_COUNT); - for (int i = 0; i < argCount; i++) { - int widened = ctx.widenToI64(argVals[i], targetType.params().get(i)); - b.emitStoreI64(argsPtr, zero2, widened, CtxBuffer.argOffset(i)); - } - - int sigRef; - if (calleeMultiReturn) { - sigRef = ctx.getOrCreateMultiReturnSigRef(targetType); - } else { - sigRef = ctx.getOrCreateSigRef(targetType); - } - - b.pushCallArg(b.useVar(ctx.memBaseVar)); - b.pushCallArg(b.useVar(ctx.ctxPtrVar)); - for (int i = 0; i < argCount; i++) { - b.pushCallArg(argVals[i]); - } - - b.emitReturnCallIndirect(sigRef, funcPtr); + emitIndirectCall(ctx, ins, true); } // --- Table operations (fully native, no trampoline) --- @@ -1100,7 +1084,7 @@ static void emitTableGet(EmitContext ctx, AnnotatedInstruction ins) { fillTrapBlock(ctx, trapBlock, CtxBuffer.TRAP_TABLE_OOB); b.switchToBlock(okBlock); - // Load funcId from 16-byte entry: entry.funcId at offset +4 + // Load funcId from the entry int entryOffset = b.emitImul(index, b.emitIconst32(CtxBuffer.TABLE_ENTRY_SIZE)); int ref = b.emitLoadI32( @@ -1130,11 +1114,11 @@ static void emitTableSet(EmitContext ctx, AnnotatedInstruction ins) { fillTrapBlock(ctx, trapBlock, CtxBuffer.TRAP_TABLE_OOB); b.switchToBlock(okBlock); - // Calculate entry offset for 16-byte entries int entryOffset = b.emitImul(index, b.emitIconst32(CtxBuffer.TABLE_ENTRY_SIZE)); int entryTypeOff = CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_TYPE_IDX_OFFSET; int entryFuncIdOff = CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_FUNC_ID_OFFSET; int entryFuncPtrOff = CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_FUNC_PTR_OFFSET; + int entryOwnerOff = CtxBuffer.TABLE_ENTRIES_OFFSET + CtxBuffer.ENTRY_OWNER_ID_OFFSET; // Check if value is REF_NULL (-1) int refNull = b.emitIconst32(-1); @@ -1150,6 +1134,7 @@ static void emitTableSet(EmitContext ctx, AnnotatedInstruction ins) { b.emitStoreI32(tablePtr, entryOffset, b.emitIconst32(0), entryTypeOff); b.emitStoreI32(tablePtr, entryOffset, refNull, entryFuncIdOff); b.emitStoreI64(tablePtr, entryOffset, b.emitIconst64(0, 0), entryFuncPtrOff); + b.emitStoreI64(tablePtr, entryOffset, b.emitIconst64(0, 0), entryOwnerOff); b.emitJump(mergeBlock); // Non-null path @@ -1159,8 +1144,10 @@ static void emitTableSet(EmitContext ctx, AnnotatedInstruction ins) { b.emitStoreI32(tablePtr, entryOffset, b.emitIconst32(0), entryTypeOff); b.emitStoreI32(tablePtr, entryOffset, funcId, entryFuncIdOff); b.emitStoreI64(tablePtr, entryOffset, b.emitIconst64(0, 0), entryFuncPtrOff); + b.emitStoreI64(tablePtr, entryOffset, b.emitIconst64(0, 0), entryOwnerOff); } else { - // FuncRef: resolve funcId → funcPtr+typeIdx + // A funcref value carries no owner, so it is taken to be this instance's + emitRegisterTableWriter(ctx, tableIdx); int funcTablePtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.FUNC_TABLE_PTR); int funcPtrOffset = b.emitImul(funcId, b.emitIconst32(8)); @@ -1172,12 +1159,43 @@ static void emitTableSet(EmitContext ctx, AnnotatedInstruction ins) { b.emitStoreI32(tablePtr, entryOffset, typeIdx, entryTypeOff); b.emitStoreI32(tablePtr, entryOffset, funcId, entryFuncIdOff); b.emitStoreI64(tablePtr, entryOffset, funcPtr, entryFuncPtrOff); + int ownerId = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.OWNER_ID); + b.emitStoreI64(tablePtr, entryOffset, ownerId, entryOwnerOff); } b.emitJump(mergeBlock); b.switchToBlock(mergeBlock); } + /** Has the runner register this instance with the table before its first native write. */ + private static void emitRegisterTableWriter(EmitContext ctx, int tableIdx) { + var b = ctx.bridge.exports(); + int zero = b.emitIconst32(0); + int writersPtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.TABLE_WRITERS_PTR); + int registered = b.emitLoadI32(writersPtr, zero, tableIdx * 4); + int isNew = b.emitIcmp(0, registered, zero); // EQ + int registerBlock = b.createBlock(); + int afterBlock = b.createBlock(); + b.emitBrif(isNew, registerBlock, afterBlock); + + b.switchToBlock(registerBlock); + int argsPtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.ARGS_PTR); + b.emitStoreI64( + argsPtr, zero, b.emitUextendI64(b.emitIconst32(tableIdx)), CtxBuffer.argOffset(0)); + b.emitStoreI32( + b.useVar(ctx.ctxPtrVar), + zero, + b.emitIconst32(-6), // sentinel for table writer + CtxBuffer.ARG_COUNT); + int trampolinePtr = b.emitLoadI64(b.useVar(ctx.ctxPtrVar), zero, CtxBuffer.TRAMPOLINE_PTR); + b.pushCallArg(b.useVar(ctx.ctxPtrVar)); + b.emitCallIndirect(ctx.getOrCreateTrampolineSigRef(), trampolinePtr); + emitTrapCheck(ctx); + b.emitJump(afterBlock); + + b.switchToBlock(afterBlock); + } + static void emitTableSize(EmitContext ctx, AnnotatedInstruction ins) { int tableIdx = (int) ins.operands()[0]; var b = ctx.bridge.exports(); diff --git a/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/CrossInstanceCallIndirectTest.java b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/CrossInstanceCallIndirectTest.java new file mode 100644 index 000000000..c104bc17f --- /dev/null +++ b/redline/runner-jffi-tests/src/test/java/run/endive/redline/experimental/runner/jffi/internal/CrossInstanceCallIndirectTest.java @@ -0,0 +1,650 @@ +package run.endive.redline.experimental.runner.jffi.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; +import run.endive.redline.experimental.api.NativeCode; +import run.endive.redline.experimental.api.internal.RedlineTarget; +import run.endive.redline.experimental.compiler.internal.NativeCompiler; +import run.endive.redline.experimental.runner.jffi.JffiNativeMachineFactory; +import run.endive.runtime.HostFunction; +import run.endive.runtime.Instance; +import run.endive.runtime.Store; +import run.endive.testing.NativeInstanceBuilder; +import run.endive.tools.wasm.Wat2Wasm; +import run.endive.wasm.Parser; +import run.endive.wasm.WasmEngineException; +import run.endive.wasm.WasmModule; +import run.endive.wasm.types.FunctionType; +import run.endive.wasm.types.ValType; + +/** + * A function placed in a table runs in the instance that defined it, whichever + * instance calls it through the table. The spec suite can't catch a runner that + * gets this wrong, because its shared-table functions only return constants and + * behave the same in any instance. The functions here read their own global and + * memory, call their own import, and trap. + * + *

This is how wit-component lays out every component: a shim module's + * functions are call_indirects through a table that a fixup module fills with + * another instance's functions. + */ +public class CrossInstanceCallIndirectTest { + + /** + * Owns the functions. Its global and memory hold 42 and 1000, which the + * caller's do not. + */ + private static final String OWNER = + "(module\n" + + " (import \"host\" \"seven\" (func $host (result i32)))\n" + + " (memory 1)\n" + + " (global $g (mut i32) (i32.const 42))\n" + + " (table (export \"tab\") 6 funcref)\n" + + " (data (i32.const 16) \"\\e8\\03\\00\\00\")\n" + + " (func $readGlobal (result i32) global.get $g)\n" + + " (func $readMemory (result i32) (i32.load (i32.const 16)))\n" + + " (func $trap (result i32) unreachable)\n" + + " (func $pair (result i32 i32) global.get $g (i32.load (i32.const 16)))\n" + + " (func $bumpGlobal (result i32)\n" + + " (global.set $g (i32.add (global.get $g) (i32.const 1)))\n" + + " global.get $g)\n" + + " (elem (i32.const 0) $readGlobal $readMemory $host $trap $pair $bumpGlobal))\n"; + + /** + * Calls through the owner's table. Its own global and memory differ from + * the owner's, and its types are declared in a different order, so type + * indices differ between the two. + */ + private static final String CALLER = + "(module\n" + + " (type $byIndex (func (param i32) (result i32)))\n" + + " (type $i64 (func (result i64)))\n" + + " (type $pair (func (result i32 i32)))\n" + + " (type $i32 (func (result i32)))\n" + + " (import \"owner\" \"tab\" (table 6 funcref))\n" + + " (memory 1)\n" + + " (global $g (mut i32) (i32.const -1))\n" + + " (func (export \"call\") (type $byIndex)\n" + + " (call_indirect (type $i32) (local.get 0)))\n" + + " (func (export \"tailCall\") (type $byIndex)\n" + + " (return_call_indirect (type $i32) (local.get 0)))\n" + + " (func (export \"callPair\") (result i32)\n" + + " (call_indirect (type $pair) (i32.const 4))\n" + + " i32.sub)\n" + + " (func (export \"wrongType\") (result i64)\n" + + " (call_indirect (type $i64) (i32.const 0))))\n"; + + /** Defines an empty table and calls through it. */ + private static final String HOLDER = + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (table (export \"tab\") 2 4 funcref)\n" + + " (func (export \"call\") (param i32) (result i32)\n" + + " (call_indirect (type $i32) (local.get 0))))\n"; + + /** Puts its function, which returns 42, in the holder's table. */ + private static final String WRITER = + "(module\n" + + " (import \"holder\" \"tab\" (table 2 funcref))\n" + + " (global $g i32 (i32.const 42))\n" + + " (func $answer (result i32) global.get $g)\n" + + " (elem (i32.const 0) $answer))\n"; + + /** + * Calls through the holder's table. Its own function has the same index as + * the writer's but returns -1. + */ + private static final String READER = + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (import \"holder\" \"tab\" (table 2 funcref))\n" + + " (global $g i32 (i32.const -1))\n" + + " (func $own (result i32) global.get $g)\n" + + " (func (export \"call\") (param i32) (result i32)\n" + + " (call_indirect (type $i32) (local.get 0))))\n"; + + @Test + public void calleeReadsItsOwnGlobal() { + try (var linked = linkNative()) { + assertEquals( + 42, call(linked.caller, "call", 0), "the owner's global, not the caller's"); + } + } + + @Test + public void calleeReadsItsOwnMemory() { + try (var linked = linkNative()) { + assertEquals( + 1000, call(linked.caller, "call", 1), "the owner's memory, not the caller's"); + } + } + + @Test + public void calleeWritesItsOwnGlobal() { + try (var linked = linkNative()) { + assertEquals(43, call(linked.caller, "call", 5)); + assertEquals(44, call(linked.caller, "call", 5)); + assertEquals( + 44, (int) linked.owner.global(0).getValue(), "the write landed in the owner"); + } + } + + @Test + public void calleeCallsItsOwnImport() { + try (var linked = linkNative()) { + assertEquals(7, call(linked.caller, "call", 2)); + } + } + + @Test + public void tailCallRunsInTheOwner() { + try (var linked = linkNative()) { + assertEquals(42, call(linked.caller, "tailCall", 0)); + assertEquals(1000, call(linked.caller, "tailCall", 1)); + } + } + + @Test + public void multiValueResultsReachTheCaller() { + try (var linked = linkNative()) { + assertEquals(42 - 1000, (int) linked.caller.export("callPair").apply()[0]); + } + } + + @Test + public void calleeTrapStopsTheCaller() { + try (var linked = linkNative()) { + var e = assertThrows(WasmEngineException.class, () -> call(linked.caller, "call", 3)); + assertTrue(e.getMessage().contains("unreachable"), e.getMessage()); + } + } + + /** Compiled code caches the page count, so the caller must see a growth in the callee. */ + @Test + public void callerSeesMemoryTheCalleeGrew() { + var store = new Store(); + try (var owner = + nativeInstance( + "(module\n" + + " (memory (export \"mem\") 1)\n" + + " (table (export \"tab\") 1 funcref)\n" + + " (func $grow (result i32) (memory.grow (i32.const 1)))\n" + + " (elem (i32.const 0) $grow))\n", + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (import \"owner\" \"mem\" (memory 1))\n" + + " (import \"owner\" \"tab\" (table 1 funcref))\n" + + " (func (export \"growThenStore\") (result i32)\n" + + " (drop (call_indirect (type $i32) (i32.const 0)))\n" + + " (i32.store (i32.const 65536) (memory.size))\n" + + " (i32.load (i32.const 65536))))\n", + store)) { + assertEquals(2, (int) caller.export("growThenStore").apply()[0]); + } + } + } + + /** + * The callee gets what is left of the caller's stack budget, not a new one, + * or a chain of instances could outgrow the thread's stack. + */ + @Test + public void calleeSharesTheCallersStackBudget() { + var store = new Store(); + try (var owner = + nativeInstance( + "(module\n" + + " (global (export \"depth\") (mut i32) (i32.const 0))\n" + + " (table (export \"tab\") 1 funcref)\n" + + " (func $recurse (result i32)\n" + + " (global.set 0 (i32.add (global.get 0) (i32.const 1)))\n" + + " (call $recurse))\n" + + " (elem (i32.const 0) $recurse))\n", + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (import \"owner\" \"tab\" (table 1 funcref))\n" + + " (global (export \"depth\") (mut i32) (i32.const 0))\n" + + " (func $descend (export \"descend\") (param i32) (result i32)\n" + + " (global.set 0 (i32.add (global.get 0) (i32.const 1)))\n" + + " (if (result i32) (local.get 0)\n" + + " (then\n" + + " (call $descend (i32.sub (local.get 0) (i32.const 1))))\n" + + " (else (call_indirect (type $i32) (i32.const 0))))))\n", + store)) { + int callerBudget = depthReached(caller, -1, caller); + int fromTheTop = depthReached(caller, 0, owner); + int fromHalfway = depthReached(caller, callerBudget / 2, owner); + assertTrue( + fromHalfway < fromTheTop * 3 / 4, + fromHalfway + " levels from halfway, " + fromTheTop + " from the top"); + } + } + } + + @Test + public void signatureIsCheckedByStructure() { + try (var linked = linkNative()) { + var e = + assertThrows( + WasmEngineException.class, + () -> linked.caller.export("wrongType").apply()); + assertTrue(e.getMessage().contains("indirect call type mismatch"), e.getMessage()); + } + } + + /** As in the interpreter, a function whose declared type is a subtype matches. */ + @Test + public void signatureAcceptsADeclaredSubtype() { + var store = new Store(); + try (var owner = + nativeInstance( + "(module\n" + + " (type $super (sub (func (result funcref))))\n" + + " (type $sub (sub $super (func (result (ref func)))))\n" + + " (table (export \"tab\") 1 funcref)\n" + + " (func $f (type $sub) (ref.func $f))\n" + + " (elem (i32.const 0) $f))\n", + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + "(module\n" + + " (type $super (sub (func (result funcref))))\n" + + " (import \"owner\" \"tab\" (table 1 funcref))\n" + + " (func (export \"call\") (result i32)\n" + + " (ref.is_null (call_indirect (type $super) (i32.const" + + " 0)))))\n", + store)) { + assertEquals(0, (int) caller.export("call").apply()[0]); + } + } + } + + @Test + public void tableReportsTheOwnerOfEachEntry() { + try (var linked = linkNative()) { + assertSame(linked.owner, linked.owner.table(0).instance(0)); + } + } + + @Test + public void nullEntryTraps() { + try (var holder = nativeInstance(HOLDER, new Store())) { + var e = assertThrows(WasmEngineException.class, () -> call(holder, "call", 0)); + assertTrue(e.getMessage().contains("uninitialized element"), e.getMessage()); + } + } + + /** + * The table belongs to a native instance and an interpreted instance fills + * it with its own functions, so the entries have no native address to call. + */ + @Test + public void entriesFromAnInterpretedInstance() { + var store = new Store(); + try (var tableOwner = + nativeInstance( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (table (export \"tab\") 2 funcref)\n" + + " (global $g i32 (i32.const -1))\n" + + " (func $own (result i32) global.get $g)\n" + + " (elem (i32.const 1) $own)\n" + + " (func (export \"call\") (param i32) (result i32)\n" + + " (call_indirect (type $i32) (local.get 0))))\n", + store)) { + store.register("owner", tableOwner); + try (var filler = + interpretedInstance( + "(module\n" + + " (import \"owner\" \"tab\" (table 2 funcref))\n" + + " (global $g i32 (i32.const 42))\n" + + " (func $answer (result i32) global.get $g)\n" + + " (elem (i32.const 0) $answer))\n", + store)) { + assertEquals(42, call(tableOwner, "call", 0), "the interpreted filler's function"); + assertEquals(-1, call(tableOwner, "call", 1), "the table owner's own function"); + assertSame(filler, tableOwner.table(0).instance(0)); + } + } + } + + /** The interpreter learns the native owner of an entry from the table. */ + @Test + public void interpretedCallerRunsTheNativeOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store)) { + store.register("owner", owner); + try (var caller = interpretedInstance(CALLER, store)) { + assertEquals(42, call(caller, "call", 0)); + assertEquals(1000, call(caller, "call", 1)); + assertEquals(7, call(caller, "call", 2)); + } + } + } + + /** No one calls the defining instance, so only the table says it owns the entries. */ + @Test + public void tableInitialiserBelongsToItsDefiningInstance() { + var store = new Store(); + try (var definer = + nativeInstance( + "(module\n" + + " (global $g i32 (i32.const 42))\n" + + " (func $answer (result i32) global.get $g)\n" + + " (table (export \"tab\") 2 funcref (ref.func $answer)))\n", + store)) { + store.register("holder", definer); + try (var reader = nativeInstance(READER, store)) { + assertEquals(42, call(reader, "call", 1), "the definer's function"); + assertSame(definer, definer.table(0).instance(1)); + } + } + } + + /** Entries copied out of an imported table still run in their owner. */ + @Test + public void copiedEntriesKeepTheirNativeOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store)) { + store.register("owner", owner); + try (var copier = + nativeInstance( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (import \"owner\" \"tab\" (table $imported 6 funcref))\n" + + " (table $own 6 funcref)\n" + + " (func (export \"copy\")\n" + + " (table.copy $own $imported\n" + + " (i32.const 0) (i32.const 0) (i32.const 6)))\n" + + " (func (export \"call\") (param i32) (result i32)\n" + + " (call_indirect $own (type $i32) (local.get 0))))\n", + store)) { + copier.export("copy").apply(); + assertEquals(42, call(copier, "call", 0)); + assertEquals(1000, call(copier, "call", 1)); + assertSame(owner, copier.table(1).instance(0)); + } + } + } + + /** Copies within and across tables keep the owners an interpreted instance wrote. */ + @Test + public void copiedEntriesKeepTheirInterpretedOwner() { + var store = new Store(); + try (var tableOwner = + nativeInstance( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (table $a (export \"a\") 4 funcref)\n" + + " (table $b 4 funcref)\n" + + " (func (export \"copyAToB\")\n" + + " (table.copy $b $a (i32.const 0) (i32.const 0) (i32.const 4)))\n" + + " (func (export \"shiftA\")\n" + + " (table.copy $a $a (i32.const 1) (i32.const 0) (i32.const 3)))\n" + + " (func (export \"callA\") (param i32) (result i32)\n" + + " (call_indirect $a (type $i32) (local.get 0)))\n" + + " (func (export \"callB\") (param i32) (result i32)\n" + + " (call_indirect $b (type $i32) (local.get 0))))\n", + store)) { + store.register("owner", tableOwner); + try (var filler = + interpretedInstance( + "(module\n" + + " (import \"owner\" \"a\" (table 4 funcref))\n" + + " (global $g i32 (i32.const 42))\n" + + " (func $answer (result i32) global.get $g)\n" + + " (elem (i32.const 0) $answer))\n", + store)) { + tableOwner.export("copyAToB").apply(); + assertEquals(42, call(tableOwner, "callB", 0)); + assertSame(filler, tableOwner.table(1).instance(0)); + + tableOwner.export("shiftA").apply(); + assertEquals(42, call(tableOwner, "callA", 0)); + assertEquals(42, call(tableOwner, "callA", 1)); + assertSame(filler, tableOwner.table(0).instance(1)); + assertNull(tableOwner.table(0).instance(2), "shifted in from a null entry"); + } + } + } + + @Test + public void hostWritesRecordTheOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store); + var holder = nativeInstance(HOLDER, store)) { + var table = holder.table(0); + table.setRef(0, 1, owner); // $readGlobal + assertEquals(42, call(holder, "call", 0)); + assertEquals(2, table.grow(1, 2, owner)); // $readMemory + assertEquals(1000, call(holder, "call", 2)); + assertSame(owner, table.instance(2)); + } + } + + /** table.set, table.grow and table.fill run natively in the instance writing the entry. */ + @Test + public void nativeWritesByAnotherInstance() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + try (var setter = + nativeInstance( + "(module\n" + + " (import \"holder\" \"tab\" (table 2 funcref))\n" + + " (global $g i32 (i32.const 42))\n" + + " (func $answer (result i32) global.get $g)\n" + + " (elem declare func $answer)\n" + + " (func (export \"set\") (param i32)\n" + + " (table.set (local.get 0) (ref.func $answer)))\n" + + " (func (export \"grow\") (result i32)\n" + + " (table.grow (ref.func $answer) (i32.const 1)))\n" + + " (func (export \"fill\")\n" + + " (table.fill (i32.const 0) (ref.func $answer) (i32.const" + + " 2))))\n", + store)) { + setter.export("set").apply(0); + assertEquals(42, call(holder, "call", 0)); + assertSame(setter, holder.table(0).instance(0)); + + assertEquals(2, (int) setter.export("grow").apply()[0]); + assertEquals(42, call(holder, "call", 2)); + + setter.export("fill").apply(); + assertEquals(42, call(holder, "call", 1)); + } + } + } + + /** A new instance may get a closed owner's context address, and must not take its entries. */ + @Test + public void closedOwnerTrapsInsteadOfRunningAnotherInstance() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + var writer = nativeInstance(WRITER, store); + assertEquals(42, call(holder, "call", 0)); + writer.close(); + + try (var reader = nativeInstance(READER, store); + var interpretedReader = interpretedInstance(READER, store)) { + assertSame(writer, holder.table(0).instance(0)); + assertClosed(() -> call(reader, "call", 0)); + assertClosed(() -> call(holder, "call", 0)); + assertClosed(() -> call(interpretedReader, "call", 0)); + } + } + } + + /** The table drops closed instances that none of its entries refer to. */ + @Test + public void tableReleasesClosedWriters() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + for (int i = 0; i < 50; i++) { + nativeInstance(WRITER, store).close(); + } + var table = (JffiNativeTable) holder.table(0); + assertTrue(table.writerCount() <= 8, table.writerCount() + " writers kept"); + assertClosed(() -> call(holder, "call", 0)); + } + } + + /** An image may leave functions out, which a table must not jump to. */ + @Test + public void uncompiledFunctionIsNotCalled() { + var module = + parse( + "(module\n" + + " (type $i32 (func (result i32)))\n" + + " (table 2 funcref)\n" + + " (func $uncompiled (result i32) (i32.const 7))\n" + + " (elem (i32.const 0) $uncompiled)\n" + + " (func (export \"callElem\") (result i32)\n" + + " (call_indirect (type $i32) (i32.const 0)))\n" + + " (func (export \"setAndCall\") (result i32)\n" + + " (table.set (i32.const 1) (ref.func $uncompiled))\n" + + " (call_indirect (type $i32) (i32.const 1))))\n"); + try (var instance = + JffiNativeMachineFactory.builder(module) + .withPrecompiledCode(withoutFirstBody(compile(module))) + .build()) { + var e = + assertThrows( + WasmEngineException.class, () -> instance.export("callElem").apply()); + assertTrue(e.getMessage().contains("not compiled"), e.getMessage()); + e = + assertThrows( + WasmEngineException.class, () -> instance.export("setAndCall").apply()); + assertTrue(e.getMessage().contains("uninitialized element"), e.getMessage()); + } + } + + /** + * Has the caller descend {@code levels} before calling the owner, until + * the stack runs out. + * + * @return how many levels {@code counted} recursed + */ + private static int depthReached(Instance caller, int levels, Instance counted) { + var depth = counted.exports().global("depth"); + int before = (int) depth.getValue(); + var e = assertThrows(WasmEngineException.class, () -> call(caller, "descend", levels)); + assertTrue(e.getMessage().contains("call stack exhausted"), e.getMessage()); + return (int) depth.getValue() - before; + } + + private static void assertClosed(Runnable call) { + var e = assertThrows(WasmEngineException.class, call::run); + assertTrue(e.getMessage().contains("instance is closed"), e.getMessage()); + } + + /** The owner and caller instances, closed caller first. */ + private static final class Linked implements AutoCloseable { + final Instance owner; + final Instance caller; + + Linked(Instance owner, Instance caller) { + this.owner = owner; + this.caller = caller; + } + + @Override + public void close() { + caller.close(); + owner.close(); + } + } + + /** + * Instantiates the owner and the caller natively, with the caller importing + * the owner's table. + * + * @return the linked instances, to close after the test + */ + private static Linked linkNative() { + var store = storeWithHost(); + var owner = nativeInstance(OWNER, store); + store.register("owner", owner); + var caller = nativeInstance(CALLER, store); + return new Linked(owner, caller); + } + + /** A store with the owner's host import, which returns 7. */ + private static Store storeWithHost() { + var seven = + new HostFunction( + "host", + "seven", + FunctionType.of(List.of(), List.of(ValType.I32)), + (inst, args) -> new long[] {7}); + return new Store().addFunction(seven); + } + + /** + * Calls an export that takes a table index and returns an i32. + * + * @param instance the instance to call + * @param export the export's name + * @param index the table index to pass + * @return the result + */ + private static int call(Instance instance, String export, int index) { + return (int) instance.export(export).apply(index)[0]; + } + + /** Instantiates {@code wat} natively against the store's exports. */ + private static Instance nativeInstance(String wat, Store store) { + return NativeInstanceBuilder.builder(parse(wat)) + .withImportValues(store.toImportValues()) + .build(); + } + + /** Instantiates {@code wat} in the interpreter against the store's exports. */ + private static Instance interpretedInstance(String wat, Store store) { + return Instance.builder(parse(wat)).withImportValues(store.toImportValues()).build(); + } + + private static NativeCode compile(WasmModule module) { + return NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), module); + } + + /** Returns {@code code} with its first function body left uncompiled. */ + private static NativeCode withoutFirstBody(NativeCode code) { + int[] bodies = new int[code.functionBodyCount()]; + int[] entries = new int[code.functionBodyCount()]; + for (int i = 0; i < bodies.length; i++) { + bodies[i] = i == 0 ? NativeCode.NOT_COMPILED : code.bodyOffset(i); + entries[i] = i == 0 ? NativeCode.NOT_COMPILED : code.entryTrampolineOffset(i); + } + int[] importTrampolines = new int[code.importCount()]; + int[] importSlots = new int[code.importCount()]; + for (int i = 0; i < importTrampolines.length; i++) { + importTrampolines[i] = code.importTrampolineOffset(i); + importSlots[i] = code.importStubSlotOffset(i); + } + return new NativeCode( + code.triple(), code.image(), bodies, entries, importTrampolines, importSlots); + } + + /** Parses a module from WebAssembly text format. */ + private static WasmModule parse(String wat) { + return Parser.parse(Wat2Wasm.parse(wat)); + } +} diff --git a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java index 3ba3eceb3..6360e553d 100644 --- a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java +++ b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeMachine.java @@ -16,6 +16,7 @@ import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import run.endive.redline.experimental.api.NativeCode; import run.endive.redline.experimental.api.internal.CtxBuffer; import run.endive.redline.experimental.api.internal.InterruptWatchdog; @@ -86,6 +87,9 @@ public final class JffiNativeMachine implements Machine { MEMSET_ADDR = memset; } + // Never reused, unlike a context address, so a stale entry cannot match a new instance + private static final AtomicLong OWNER_IDS = new AtomicLong(); + private final Instance instance; private final Function[] entryTrampolines; // entry trampoline per func private final FunctionType[] funcTypes; // wasm FunctionType per func @@ -98,7 +102,9 @@ public final class JffiNativeMachine implements Machine { private final long globalsBufferAddr; private final long funcTypesArrayAddr; private final long funcTypesArraySize; // byte size + private final long ownerId = OWNER_IDS.incrementAndGet(); private long tablePtrsArrayAddr; + private long tableWritersAddr; private JffiNativeTable[] nativeTables; private boolean[] ownsTable; private boolean tablesInitialized; @@ -204,6 +210,7 @@ public JffiNativeMachine( MEM.putLong(ctxBufferAddr + CtxBuffer.FUNC_TYPES_PTR, funcTypesArrayAddr); MEM.putLong(ctxBufferAddr + CtxBuffer.MEMMOVE_PTR, MEMMOVE_ADDR); MEM.putLong(ctxBufferAddr + CtxBuffer.MEMSET_PTR, MEMSET_ADDR); + MEM.putLong(ctxBufferAddr + CtxBuffer.OWNER_ID, ownerId); byte[] image = code.image(); @@ -271,6 +278,20 @@ public JffiNativeMachine( this.ownsMemory = instance.imports().memoryCount() == 0; } + /** Returns the id that the table entries this machine owns hold. */ + long ownerId() { + return ownerId; + } + + boolean isClosed() { + return closed; + } + + /** Returns the instance this machine executes. */ + Instance instance() { + return instance; + } + @Override public void close() { if (closed) { @@ -291,6 +312,7 @@ public void close() { } if (tablePtrsArrayAddr != 0) { MEM.freeMemory(tablePtrsArrayAddr); + MEM.freeMemory(tableWritersAddr); } trampolineHandle.dispose(); memGrowHandle.dispose(); @@ -469,6 +491,14 @@ private Closure.Handle createTrampolineStub() { closure, Type.SINT64, new Type[] {Type.SINT64}, CallingConvention.DEFAULT); } + /** + * Entry point for compiled code that needs the runner: table operations, + * flagged by a negative argument count, and call_indirect through an entry + * another instance owns. + * + * @param ctxAddr the calling function's context + * @return the operation's result, or 0 if it recorded an exception + */ private long callIndirectTrampoline(long ctxAddr) { try { int argCount = MEM.getInt(ctxAddr + CtxBuffer.ARG_COUNT); @@ -477,13 +507,70 @@ private long callIndirectTrampoline(long ctxAddr) { if (argCount < 0) { return handleTableOperation(argCount); } - throw new WasmEngineException("Unexpected trampoline call: argCount " + argCount); + return callForeignEntry( + MEM.getInt(ctxAddr + CtxBuffer.TYPE_ID), + MEM.getInt(ctxAddr + CtxBuffer.TABLE_IDX), + MEM.getInt(ctxAddr + CtxBuffer.ELEM_IDX), + argCount); } catch (Throwable t) { recordHostException(t); return 0L; } } + /** + * Calls a table entry this instance does not own, in the instance that + * does. Compiled code has already checked the index. The entry's type + * index is the owner's, so the signature is checked here by structure. The + * call runs through the owner's machine, against the owner's memory, + * globals and tables. A trap there becomes an exception that unwinds this + * instance too. + * + * @param typeId the expected type, in this module's index space + * @param tableIdx the table, in this module's index space + * @param elemIdx the entry + * @param argCount the number of arguments in argsBuffer + * @return a single result as raw bits, or 0 with the results left in + * argsBuffer for a multi-value call + */ + private long callForeignEntry(int typeId, int tableIdx, int elemIdx, int argCount) { + JffiNativeTable table = nativeTables[tableIdx]; + int funcId = table.ref(elemIdx); + if (funcId == Value.REF_NULL_VALUE) { + throw new TrapException("uninitialized element"); + } + // An entry written with no owner runs in the caller, as in the interpreter + Instance owner = table.instance(elemIdx); + if (owner == null) { + owner = instance; + } + var expected = instance.type(typeId); + var actual = owner.type(owner.functionType(funcId)); + if (!FunctionType.matches(actual, expected, instance.module().typeSection())) { + throw new TrapException("indirect call type mismatch"); + } + long[] args = new long[argCount]; + for (int i = 0; i < argCount; i++) { + args[i] = MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(i)); + } + // One stack budget for the whole chain, or each instance would add its own + long stackLimit = MEM.getLong(ctxBufferAddr + CtxBuffer.STACK_LIMIT); + var ownerMachine = owner.getMachine(); + long[] results = + ownerMachine instanceof JffiNativeMachine + ? ((JffiNativeMachine) ownerMachine) + .callWithStackLimit(funcId, args, stackLimit) + : ownerMachine.call(funcId, args); + refreshMemoryPages(); + if (expected.returns().size() > 1) { + for (int i = 0; i < results.length; i++) { + MEM.putLong(argsBufferAddr + CtxBuffer.argOffset(i), results[i]); + } + return 0L; + } + return results == null || results.length == 0 ? 0L : results[0]; + } + private long handleTableOperation(int opCode) { switch (opCode) { case -1: @@ -491,12 +578,8 @@ private long handleTableOperation(int opCode) { int oldSize = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(0)); int newSize = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(1)); int fillValue = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(2)); - long tableAddr = MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(3)); int tblIdx = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(4)); - boolean externRef = nativeTables[tblIdx].isExternRef(); - for (int i = oldSize; i < newSize; i++) { - writeTableEntry(tableAddr, i, fillValue, externRef); - } + nativeTables[tblIdx].fill(oldSize, newSize, fillValue, instance); nativeTables[tblIdx].limits().grow(newSize - oldSize); break; } @@ -505,30 +588,18 @@ private long handleTableOperation(int opCode) { int offset = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(0)); int end = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(1)); int fillValue = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(2)); - long tableAddr = MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(3)); int tblIdx = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(4)); - boolean externRef = nativeTables[tblIdx].isExternRef(); - for (int i = offset; i < end; i++) { - writeTableEntry(tableAddr, i, fillValue, externRef); - } + nativeTables[tblIdx].fill(offset, end, fillValue, instance); break; } case -3: - { // table copy (16-byte entries) + { // table copy long srcAddr = MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(0)); long dstAddr = MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(1)); int srcOff = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(2)); int dstOff = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(3)); int size = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(4)); - if (dstOff <= srcOff) { - for (int i = 0; i < size; i++) { - copyTableEntry(srcAddr, srcOff + i, dstAddr, dstOff + i); - } - } else { - for (int i = size - 1; i >= 0; i--) { - copyTableEntry(srcAddr, srcOff + i, dstAddr, dstOff + i); - } - } + tableAt(dstAddr).copyFrom(tableAt(srcAddr), srcOff, dstOff, size); break; } case -4: @@ -548,6 +619,13 @@ private long handleTableOperation(int opCode) { instance.setElement(elemIdx, null); break; } + case -6: + { // first native table.set into a table + int tblIdx = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(0)); + nativeTables[tblIdx].registerWriter(this); + MEM.putInt(tableWritersAddr + (long) tblIdx * 4, 1); + break; + } case -8: { // memory.init int segmentId = (int) MEM.getLong(argsBufferAddr + CtxBuffer.argOffset(0)); @@ -592,43 +670,19 @@ private long handleTableOperation(int opCode) { return 0L; } - private void writeTableEntry(long tableAddr, int index, int funcId, boolean isExternRef) { - long base = - tableAddr - + CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) index * CtxBuffer.TABLE_ENTRY_SIZE; - if (funcId == Value.REF_NULL_VALUE) { - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, Value.REF_NULL_VALUE); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); - } else if (isExternRef) { - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); - } else { - long funcPtr = MEM.getLong(funcTableAddr + (long) funcId * 8); - int typeIdx = MEM.getInt(funcTypesArrayAddr + (long) funcId * 4); - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + /** + * Finds this instance's table whose native buffer is at {@code address}. + * + * @param address a table buffer address + * @return the table + */ + private JffiNativeTable tableAt(long address) { + for (JffiNativeTable table : nativeTables) { + if (table.nativeBufferAddress() == address) { + return table; + } } - } - - private static void copyTableEntry(long srcAddr, int srcIdx, long dstAddr, int dstIdx) { - long srcBase = - srcAddr - + CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) srcIdx * CtxBuffer.TABLE_ENTRY_SIZE; - long dstBase = - dstAddr - + CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) dstIdx * CtxBuffer.TABLE_ENTRY_SIZE; - int typeIdx = MEM.getInt(srcBase + CtxBuffer.ENTRY_TYPE_IDX_OFFSET); - int funcId = MEM.getInt(srcBase + CtxBuffer.ENTRY_FUNC_ID_OFFSET); - long funcPtr = MEM.getLong(srcBase + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); - MEM.putInt(dstBase + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - MEM.putInt(dstBase + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - MEM.putLong(dstBase + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + throw new WasmEngineException("no table at " + address); } // --- Memory grow upcall stub --- @@ -734,6 +788,7 @@ private void initializeNativeTables() { boolean[] owned = new boolean[tableCount]; this.ownsTable = owned; this.tablePtrsArrayAddr = MEM.allocateMemory((long) tableCount * 8, true); + this.tableWritersAddr = MEM.allocateMemory((long) tableCount * 4, true); for (int i = 0; i < tableCount; i++) { var table = instance.table(i); @@ -752,6 +807,7 @@ private void initializeNativeTables() { } MEM.putLong(ctxBufferAddr + CtxBuffer.TABLE_PTRS, tablePtrsArrayAddr); + MEM.putLong(ctxBufferAddr + CtxBuffer.TABLE_WRITERS_PTR, tableWritersAddr); } /** Package-private: used by JffiNativeTable to resolve funcId → funcPtr. */ @@ -896,6 +952,15 @@ private long invokeViaBufferWithTrampoline( @Override public long[] call(int funcId, long[] args) throws WasmEngineException { + return callWithStackLimit(funcId, args, 0L); + } + + /** Calls with the stack limit an outermost call starts from, 0 to anchor a new one. */ + private long[] callWithStackLimit(int funcId, long[] args, long stackLimit) { + if (closed) { + // Its native code and buffers are gone, but a table may still hold its functions + throw new WasmEngineException("instance is closed"); + } if (funcId < numImports) { // Host import — delegate directly var imprt = instance.imports().function(funcId); @@ -904,6 +969,9 @@ public long[] call(int funcId, long[] args) throws WasmEngineException { var funcType = funcTypes[funcId]; long funcAddr = MEM.getLong(funcTableAddr + (long) funcId * 8); + if (entryTrampolines[funcId] == null) { + throw new WasmEngineException("Function " + funcId + " not compiled"); + } try { boolean outermostCall = callDepth++ == 0; @@ -915,7 +983,7 @@ public long[] call(int funcId, long[] args) throws WasmEngineException { // against where the outer call began, or every level moves the // limit deeper and the guard stops firing. if (outermostCall) { - MEM.putLong(ctxBufferAddr + CtxBuffer.STACK_LIMIT, 0L); + MEM.putLong(ctxBufferAddr + CtxBuffer.STACK_LIMIT, stackLimit); } if (!memBaseInitialized) { diff --git a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeTable.java b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeTable.java index 765f38064..7addf0938 100644 --- a/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeTable.java +++ b/redline/runner-jffi/src/main/java/run/endive/redline/experimental/runner/jffi/internal/JffiNativeTable.java @@ -3,6 +3,7 @@ import static run.endive.wasm.types.Value.REF_NULL_VALUE; import com.kenai.jffi.MemoryIO; +import java.util.Arrays; import run.endive.redline.experimental.api.internal.CtxBuffer; import run.endive.runtime.Instance; import run.endive.runtime.TableInstance; @@ -16,12 +17,20 @@ * *

Layout: [size:i32 @ 0][max:i32 @ 4][entries... @ 8] * - *

Each entry is 16 bytes: + *

Each entry is 24 bytes: * 

  *   [0..4)   i32  canonicalTypeIdx
  *   [4..8)   i32  funcId
  *   [8..16)  i64  funcPtr (native address)
+ *   [16..24) i64  ownerId (id of the native instance defining the function)
  *
+ * + *

The type index, function id and pointer belong to the instance that owns + * the function, which is not necessarily the one using the table. Native code + * calls an entry directly only from the owner's instance. The table keeps every + * native instance whose id its entries hold, see {@link #writers}. An entry + * owned by an instance without native code has no pointer and no owner id; its + * owner is kept in {@link #owners} instead. */ public final class JffiNativeTable extends TableInstance { @@ -33,6 +42,19 @@ public final class JffiNativeTable extends TableInstance { private final boolean isExternRef; private boolean freed; + /** + * Owner of each entry written from Java without an owner id: one not + * yet resolved, or one owned by an instance without native code. It grows + * with the table instead of being allocated at capacity, which can be a + * million entries. + */ + private Instance[] owners = new Instance[0]; + + // Machines whose ids entries may hold, replaced on change so readers need no lock + private volatile JffiNativeMachine[] writers = new JffiNativeMachine[0]; + private int pruneAt = 8; + private volatile JffiNativeMachine lastWriter; + public JffiNativeTable(Table table, int initValue) { super(table, initValue); this.isExternRef = table.elementType().equals(ValType.ExternRef); @@ -55,36 +77,57 @@ public JffiNativeTable(Table table, int initValue) { } if (initValue != REF_NULL_VALUE) { - // The instance has no machine yet, so funcPtr cannot be resolved - // here. resolvePendingRefs fills it in once there is one. + // Instance records the owner right after; otherwise the first caller takes it for (int i = 0; i < initial; i++) { - writeUnresolvedEntry(i, initValue); + writeUnresolvedEntry(i, initValue, null); } } } - /** A funcref whose native address is not known yet. */ - private void writeUnresolvedEntry(int index, int value) { - long base = bufferAddress + entryBase(index); - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, value); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + /** + * Writes a funcref with no native address, either not resolved yet or owned + * by an instance without native code. + * + * @param index the entry to write + * @param value the function id, in {@code owner}'s index space + * @param owner the instance that defines the function, or {@code null} if + * unknown + */ + private void writeUnresolvedEntry(int index, int value, Instance owner) { + writeEntry(index, 0, value, 0L, 0L); + setOwner(index, owner); } /** - * Fills in the native address of entries written before the instance had a - * machine, which is the case for a table initialiser. + * Writes every field of entry {@code index}. + * + * @param index the entry to write + * @param typeIdx the owner's canonical type index for the function + * @param funcId the function id, in the owner's index space + * @param funcPtr the function's native address, or 0 + * @param ownerId the owner's id, or 0 */ + private void writeEntry(int index, int typeIdx, int funcId, long funcPtr, long ownerId) { + long base = bufferAddress + entryBase(index); + MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); + MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); + MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + MEM.putLong(base + CtxBuffer.ENTRY_OWNER_ID_OFFSET, ownerId); + } + + private long ownerId(int index) { + return MEM.getLong(bufferAddress + entryBase(index) + CtxBuffer.ENTRY_OWNER_ID_OFFSET); + } + + /** Resolves entries written with no owner, which only a host does, against {@code instance}. */ void resolvePendingRefs(Instance instance) { if (isExternRef) { return; } int sz = size(); for (int i = 0; i < sz; i++) { - long base = bufferAddress + entryBase(i); - int funcId = MEM.getInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET); - long funcPtr = MEM.getLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); - if (funcId != REF_NULL_VALUE && funcPtr == 0L) { + int funcId = MEM.getInt(bufferAddress + entryBase(i) + CtxBuffer.ENTRY_FUNC_ID_OFFSET); + if (funcId != REF_NULL_VALUE && ownerId(i) == 0L && recordedOwner(i) == null) { resolveFromInstance(i, funcId, instance); } } @@ -95,40 +138,207 @@ private long entryBase(int index) { } private void writeNullEntry(int index) { - long base = bufferAddress + entryBase(index); - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, REF_NULL_VALUE); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + writeEntry(index, 0, REF_NULL_VALUE, 0L, 0L); + setOwner(index, null); } - private void writeOpaqueEntry(int index, int value) { - long base = bufferAddress + entryBase(index); - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, value); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + /** + * Writes {@code funcId} with the native address, type index and id of + * {@code owner}'s function. + * + * @param index the entry to write + * @param funcId the function id, in {@code owner}'s index space + * @param owner the open native machine that defines the function + */ + private void writeResolvedEntry(int index, int funcId, JffiNativeMachine owner) { + long funcPtr = MEM.getLong(owner.getFuncTableAddress() + (long) funcId * 8); + if (funcPtr == 0L) { + // Not compiled, so only the owner's runner can report it + writeUnresolvedEntry(index, funcId, owner.instance()); + return; + } + registerWriter(owner); + int typeIdx = MEM.getInt(owner.getFuncTypesArrayAddress() + (long) funcId * 4); + writeEntry(index, typeIdx, funcId, funcPtr, owner.ownerId()); + setOwner(index, null); } - private void writeResolvedEntry(int index, int funcId, long ftAddr, long ftaAddr) { - long base = bufferAddress + entryBase(index); - long funcPtr = MEM.getLong(ftAddr + (long) funcId * 8); - int typeIdx = MEM.getInt(ftaAddr + (long) funcId * 4); - MEM.putInt(base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - MEM.putInt(base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - MEM.putLong(base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + /** + * Writes {@code funcId} as a function of {@code instance}. An open native + * instance gives the entry its address and id. Otherwise the entry keeps + * its owner here and is called through the runner. + * + * @param index the entry to write + * @param funcId the function id, in {@code instance}'s index space + * @param instance the instance that defines the function, or {@code null} + */ + private void resolveFromInstance(int index, int funcId, Instance instance) { + if (isExternRef) { + writeEntry(index, 0, funcId, 0L, 0L); + } else if (instance != null + && instance.getMachine() instanceof JffiNativeMachine + && !((JffiNativeMachine) instance.getMachine()).isClosed()) { + writeResolvedEntry(index, funcId, (JffiNativeMachine) instance.getMachine()); + } else { + writeUnresolvedEntry(index, funcId, instance); + } } - private boolean resolveFromInstance(int index, int funcId, Instance instance) { - if (isExternRef) { - writeOpaqueEntry(index, funcId); - return true; + /** + * Returns the owner recorded for entry {@code index}. + * + * @param index the entry + * @return the recorded owner, or {@code null} + */ + private Instance recordedOwner(int index) { + return index < owners.length ? owners[index] : null; + } + + /** + * Records the owner of entry {@code index}, growing {@link #owners} as + * needed. + * + * @param index the entry + * @param owner the owning instance, or {@code null} + */ + private void setOwner(int index, Instance owner) { + if (index >= owners.length) { + if (owner == null) { + return; + } + ensureOwners(index + 1); + } + owners[index] = owner; + } + + private void ensureOwners(int length) { + if (length > owners.length) { + owners = Arrays.copyOf(owners, Math.max(length, Math.max(size(), owners.length * 2))); + } + } + + /** Keeps {@code writer}, whose id entries of this table may now hold. */ + void registerWriter(JffiNativeMachine writer) { + if (lastWriter == writer) { + return; + } + synchronized (this) { + for (JffiNativeMachine w : writers) { + if (w == writer) { + lastWriter = writer; + return; + } + } + if (writers.length >= pruneAt) { + pruneWriters(); + pruneAt = Math.max(8, writers.length * 2); + } + JffiNativeMachine[] grown = Arrays.copyOf(writers, writers.length + 1); + grown[writers.length] = writer; + writers = grown; + lastWriter = writer; + } + } + + // Drops closed writers that no entry refers to any more + private void pruneWriters() { + JffiNativeMachine[] current = writers; + boolean[] keep = new boolean[current.length]; + boolean anyClosed = false; + for (int w = 0; w < current.length; w++) { + keep[w] = !current[w].isClosed(); + anyClosed |= !keep[w]; + } + if (!anyClosed) { + return; + } + int sz = size(); + for (int i = 0; i < sz; i++) { + long id = ownerId(i); + if (id != 0L) { + for (int w = 0; w < current.length; w++) { + if (current[w].ownerId() == id) { + keep[w] = true; + break; + } + } + } + } + JffiNativeMachine[] kept = new JffiNativeMachine[current.length]; + int count = 0; + for (int w = 0; w < current.length; w++) { + if (keep[w]) { + kept[count++] = current[w]; + } + } + writers = Arrays.copyOf(kept, count); + lastWriter = null; + } + + /** Returns the machine, possibly closed, with owner id {@code id}. */ + private JffiNativeMachine writer(long id) { + for (JffiNativeMachine w : writers) { + if (w.ownerId() == id) { + return w; + } + } + throw new WasmEngineException("table entry owner " + id + " is not registered"); + } + + /** Returns how many machines the table keeps for its entries. */ + int writerCount() { + return writers.length; + } + + /** Writes {@code funcId}, a function of {@code instance}, to entries [from, to). */ + void fill(int from, int to, int funcId, Instance instance) { + for (int i = from; i < to; i++) { + if (funcId == REF_NULL_VALUE) { + writeNullEntry(i); + } else { + resolveFromInstance(i, funcId, instance); + } + } + } + + /** Copies {@code count} entries and their owners from {@code src}, which may be this table. */ + void copyFrom(JffiNativeTable src, int srcIndex, int dstIndex, int count) { + if (dstIndex <= srcIndex) { + for (int i = 0; i < count; i++) { + copyEntry(src, srcIndex + i, dstIndex + i); + } + } else { + for (int i = count - 1; i >= 0; i--) { + copyEntry(src, srcIndex + i, dstIndex + i); + } + } + if (src != this) { + long lastId = 0L; + for (int i = dstIndex; i < dstIndex + count; i++) { + long id = ownerId(i); + if (id != 0L && id != lastId) { + registerWriter(src.writer(id)); + lastId = id; + } + } + } + int fromSrc = Math.max(0, Math.min(count, src.owners.length - srcIndex)); + if (fromSrc > 0) { + ensureOwners(dstIndex + fromSrc); + // Read src.owners after the resize, as src may be this table + System.arraycopy(src.owners, srcIndex, owners, dstIndex, fromSrc); } - if (instance != null && instance.getMachine() instanceof JffiNativeMachine) { - JffiNativeMachine nm = (JffiNativeMachine) instance.getMachine(); - writeResolvedEntry( - index, funcId, nm.getFuncTableAddress(), nm.getFuncTypesArrayAddress()); - return true; + int clearTo = Math.min(dstIndex + count, owners.length); + if (dstIndex + fromSrc < clearTo) { + Arrays.fill(owners, dstIndex + fromSrc, clearTo, null); } - return false; + } + + private void copyEntry(JffiNativeTable src, int srcIndex, int dstIndex) { + MEM.copyMemory( + src.bufferAddress + src.entryBase(srcIndex), + bufferAddress + entryBase(dstIndex), + CtxBuffer.TABLE_ENTRY_SIZE); } /** Get the native address of the table buffer, for passing to native code. */ @@ -191,11 +401,7 @@ public void setRef(int index, int value, Instance instance) { if (index < 0 || index >= size()) { throw new run.endive.wasm.UninstantiableException("out of bounds table access"); } - if (value == REF_NULL_VALUE) { - writeNullEntry(index); - } else if (!resolveFromInstance(index, value, instance)) { - writeUnresolvedEntry(index, value); - } + fill(index, index + 1, value, instance); } @Override @@ -206,23 +412,28 @@ public int grow(int delta, int value, Instance instance) { if (delta < 0 || newSize > max || newSize > capacity) { return -1; } - // Fill new slots - for (int i = oldSize; i < newSize; i++) { - if (value == REF_NULL_VALUE) { - writeNullEntry(i); - } else if (!resolveFromInstance(i, value, instance)) { - writeUnresolvedEntry(i, value); - } - } + fill(oldSize, newSize, value, instance); // Update size MEM.putInt(bufferAddress + CtxBuffer.TABLE_SIZE_OFFSET, newSize); limits().grow(delta); return oldSize; } + /** + * Returns the instance whose function entry {@code index} holds, even if it + * has since been closed. + * + * @param index the entry + * @return the owning instance, or {@code null} for a null entry or one + * written with no owner + */ @Override public Instance instance(int index) { - return null; + if (index < 0 || index >= size() || isExternRef || ref(index) == REF_NULL_VALUE) { + return null; + } + long id = ownerId(index); + return id != 0L ? writer(id).instance() : recordedOwner(index); } @Override diff --git a/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/CrossInstanceCallIndirectTest.java b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/CrossInstanceCallIndirectTest.java new file mode 100644 index 000000000..6d8ffab1f --- /dev/null +++ b/redline/runner-tests/src/test/java/run/endive/redline/experimental/runner/internal/CrossInstanceCallIndirectTest.java @@ -0,0 +1,677 @@ +package run.endive.redline.experimental.runner.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; +import run.endive.redline.experimental.api.NativeCode; +import run.endive.redline.experimental.api.internal.RedlineTarget; +import run.endive.redline.experimental.compiler.internal.NativeCompiler; +import run.endive.redline.experimental.runner.NativeMachineFactory; +import run.endive.runtime.HostFunction; +import run.endive.runtime.Instance; +import run.endive.runtime.Store; +import run.endive.testing.NativeInstanceBuilder; +import run.endive.tools.wasm.Wat2Wasm; +import run.endive.wasm.Parser; +import run.endive.wasm.WasmEngineException; +import run.endive.wasm.WasmModule; +import run.endive.wasm.types.FunctionType; +import run.endive.wasm.types.ValType; + +/** + * A function placed in a table runs in the instance that defined it, whichever + * instance calls it through the table. The spec suite can't catch a runner that + * gets this wrong, because its shared-table functions only return constants and + * behave the same in any instance. The functions here read their own global and + * memory, call their own import, and trap. + * + *

This is how wit-component lays out every component: a shim module's + * functions are call_indirects through a table that a fixup module fills with + * another instance's functions. + */ +public class CrossInstanceCallIndirectTest { + + /** + * Owns the functions. Its global and memory hold 42 and 1000, which the + * caller's do not. + */ + private static final String OWNER = + """ + (module + (import "host" "seven" (func $host (result i32))) + (memory 1) + (global $g (mut i32) (i32.const 42)) + (table (export "tab") 6 funcref) + (data (i32.const 16) "\\e8\\03\\00\\00") + (func $readGlobal (result i32) global.get $g) + (func $readMemory (result i32) (i32.load (i32.const 16))) + (func $trap (result i32) unreachable) + (func $pair (result i32 i32) global.get $g (i32.load (i32.const 16))) + (func $bumpGlobal (result i32) + (global.set $g (i32.add (global.get $g) (i32.const 1))) + global.get $g) + (elem (i32.const 0) $readGlobal $readMemory $host $trap $pair $bumpGlobal)) + """; + + /** + * Calls through the owner's table. Its own global and memory differ from + * the owner's, and its types are declared in a different order, so type + * indices differ between the two. + */ + private static final String CALLER = + """ + (module + (type $byIndex (func (param i32) (result i32))) + (type $i64 (func (result i64))) + (type $pair (func (result i32 i32))) + (type $i32 (func (result i32))) + (import "owner" "tab" (table 6 funcref)) + (memory 1) + (global $g (mut i32) (i32.const -1)) + (func (export "call") (type $byIndex) + (call_indirect (type $i32) (local.get 0))) + (func (export "tailCall") (type $byIndex) + (return_call_indirect (type $i32) (local.get 0))) + (func (export "callPair") (result i32) + (call_indirect (type $pair) (i32.const 4)) + i32.sub) + (func (export "wrongType") (result i64) + (call_indirect (type $i64) (i32.const 0)))) + """; + + /** Defines an empty table and calls through it. */ + private static final String HOLDER = + """ + (module + (type $i32 (func (result i32))) + (table (export "tab") 2 4 funcref) + (func (export "call") (param i32) (result i32) + (call_indirect (type $i32) (local.get 0)))) + """; + + /** Puts its function, which returns 42, in the holder's table. */ + private static final String WRITER = + """ + (module + (import "holder" "tab" (table 2 funcref)) + (global $g i32 (i32.const 42)) + (func $answer (result i32) global.get $g) + (elem (i32.const 0) $answer)) + """; + + /** + * Calls through the holder's table. Its own function has the same index as + * the writer's but returns -1. + */ + private static final String READER = + """ + (module + (type $i32 (func (result i32))) + (import "holder" "tab" (table 2 funcref)) + (global $g i32 (i32.const -1)) + (func $own (result i32) global.get $g) + (func (export "call") (param i32) (result i32) + (call_indirect (type $i32) (local.get 0)))) + """; + + @Test + public void calleeReadsItsOwnGlobal() { + try (var linked = linkNative()) { + assertEquals( + 42, call(linked.caller, "call", 0), "the owner's global, not the caller's"); + } + } + + @Test + public void calleeReadsItsOwnMemory() { + try (var linked = linkNative()) { + assertEquals( + 1000, call(linked.caller, "call", 1), "the owner's memory, not the caller's"); + } + } + + @Test + public void calleeWritesItsOwnGlobal() { + try (var linked = linkNative()) { + assertEquals(43, call(linked.caller, "call", 5)); + assertEquals(44, call(linked.caller, "call", 5)); + assertEquals( + 44, (int) linked.owner.global(0).getValue(), "the write landed in the owner"); + } + } + + @Test + public void calleeCallsItsOwnImport() { + try (var linked = linkNative()) { + assertEquals(7, call(linked.caller, "call", 2)); + } + } + + @Test + public void tailCallRunsInTheOwner() { + try (var linked = linkNative()) { + assertEquals(42, call(linked.caller, "tailCall", 0)); + assertEquals(1000, call(linked.caller, "tailCall", 1)); + } + } + + @Test + public void multiValueResultsReachTheCaller() { + try (var linked = linkNative()) { + assertEquals(42 - 1000, (int) linked.caller.export("callPair").apply()[0]); + } + } + + @Test + public void calleeTrapStopsTheCaller() { + try (var linked = linkNative()) { + var e = assertThrows(WasmEngineException.class, () -> call(linked.caller, "call", 3)); + assertTrue(e.getMessage().contains("unreachable"), e.getMessage()); + } + } + + /** Compiled code caches the page count, so the caller must see a growth in the callee. */ + @Test + public void callerSeesMemoryTheCalleeGrew() { + var store = new Store(); + try (var owner = + nativeInstance( + """ + (module + (memory (export "mem") 1) + (table (export "tab") 1 funcref) + (func $grow (result i32) (memory.grow (i32.const 1))) + (elem (i32.const 0) $grow)) + """, + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + """ + (module + (type $i32 (func (result i32))) + (import "owner" "mem" (memory 1)) + (import "owner" "tab" (table 1 funcref)) + (func (export "growThenStore") (result i32) + (drop (call_indirect (type $i32) (i32.const 0))) + (i32.store (i32.const 65536) (memory.size)) + (i32.load (i32.const 65536)))) + """, + store)) { + assertEquals(2, (int) caller.export("growThenStore").apply()[0]); + } + } + } + + /** + * The callee gets what is left of the caller's stack budget, not a new one, + * or a chain of instances could outgrow the thread's stack. + */ + @Test + public void calleeSharesTheCallersStackBudget() { + var store = new Store(); + try (var owner = + nativeInstance( + """ + (module + (global (export "depth") (mut i32) (i32.const 0)) + (table (export "tab") 1 funcref) + (func $recurse (result i32) + (global.set 0 (i32.add (global.get 0) (i32.const 1))) + (call $recurse)) + (elem (i32.const 0) $recurse)) + """, + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + """ + (module + (type $i32 (func (result i32))) + (import "owner" "tab" (table 1 funcref)) + (global (export "depth") (mut i32) (i32.const 0)) + (func $descend (export "descend") (param i32) (result i32) + (global.set 0 (i32.add (global.get 0) (i32.const 1))) + (if (result i32) (local.get 0) + (then (call $descend (i32.sub (local.get 0) (i32.const 1)))) + (else (call_indirect (type $i32) (i32.const 0)))))) + """, + store)) { + int callerBudget = depthReached(caller, -1, caller); + int fromTheTop = depthReached(caller, 0, owner); + int fromHalfway = depthReached(caller, callerBudget / 2, owner); + assertTrue( + fromHalfway < fromTheTop * 3 / 4, + fromHalfway + " levels from halfway, " + fromTheTop + " from the top"); + } + } + } + + @Test + public void signatureIsCheckedByStructure() { + try (var linked = linkNative()) { + var e = + assertThrows( + WasmEngineException.class, + () -> linked.caller.export("wrongType").apply()); + assertTrue(e.getMessage().contains("indirect call type mismatch"), e.getMessage()); + } + } + + /** As in the interpreter, a function whose declared type is a subtype matches. */ + @Test + public void signatureAcceptsADeclaredSubtype() { + var store = new Store(); + try (var owner = + nativeInstance( + """ + (module + (type $super (sub (func (result funcref)))) + (type $sub (sub $super (func (result (ref func))))) + (table (export "tab") 1 funcref) + (func $f (type $sub) (ref.func $f)) + (elem (i32.const 0) $f)) + """, + store)) { + store.register("owner", owner); + try (var caller = + nativeInstance( + """ + (module + (type $super (sub (func (result funcref)))) + (import "owner" "tab" (table 1 funcref)) + (func (export "call") (result i32) + (ref.is_null (call_indirect (type $super) (i32.const 0))))) + """, + store)) { + assertEquals(0, (int) caller.export("call").apply()[0]); + } + } + } + + @Test + public void tableReportsTheOwnerOfEachEntry() { + try (var linked = linkNative()) { + assertSame(linked.owner, linked.owner.table(0).instance(0)); + } + } + + @Test + public void nullEntryTraps() { + try (var holder = nativeInstance(HOLDER, new Store())) { + var e = assertThrows(WasmEngineException.class, () -> call(holder, "call", 0)); + assertTrue(e.getMessage().contains("uninitialized element"), e.getMessage()); + } + } + + /** + * The table belongs to a native instance and an interpreted instance fills + * it with its own functions, so the entries have no native address to call. + */ + @Test + public void entriesFromAnInterpretedInstance() { + var store = new Store(); + try (var tableOwner = + nativeInstance( + """ + (module + (type $i32 (func (result i32))) + (table (export "tab") 2 funcref) + (global $g i32 (i32.const -1)) + (func $own (result i32) global.get $g) + (elem (i32.const 1) $own) + (func (export "call") (param i32) (result i32) + (call_indirect (type $i32) (local.get 0)))) + """, + store)) { + store.register("owner", tableOwner); + try (var filler = + interpretedInstance( + """ + (module + (import "owner" "tab" (table 2 funcref)) + (global $g i32 (i32.const 42)) + (func $answer (result i32) global.get $g) + (elem (i32.const 0) $answer)) + """, + store)) { + assertEquals(42, call(tableOwner, "call", 0), "the interpreted filler's function"); + assertEquals(-1, call(tableOwner, "call", 1), "the table owner's own function"); + assertSame(filler, tableOwner.table(0).instance(0)); + } + } + } + + /** The interpreter learns the native owner of an entry from the table. */ + @Test + public void interpretedCallerRunsTheNativeOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store)) { + store.register("owner", owner); + try (var caller = interpretedInstance(CALLER, store)) { + assertEquals(42, call(caller, "call", 0)); + assertEquals(1000, call(caller, "call", 1)); + assertEquals(7, call(caller, "call", 2)); + } + } + } + + /** No one calls the defining instance, so only the table says it owns the entries. */ + @Test + public void tableInitialiserBelongsToItsDefiningInstance() { + var store = new Store(); + try (var definer = + nativeInstance( + """ + (module + (global $g i32 (i32.const 42)) + (func $answer (result i32) global.get $g) + (table (export "tab") 2 funcref (ref.func $answer))) + """, + store)) { + store.register("holder", definer); + try (var reader = nativeInstance(READER, store)) { + assertEquals(42, call(reader, "call", 1), "the definer's function"); + assertSame(definer, definer.table(0).instance(1)); + } + } + } + + /** Entries copied out of an imported table still run in their owner. */ + @Test + public void copiedEntriesKeepTheirNativeOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store)) { + store.register("owner", owner); + try (var copier = + nativeInstance( + """ + (module + (type $i32 (func (result i32))) + (import "owner" "tab" (table $imported 6 funcref)) + (table $own 6 funcref) + (func (export "copy") + (table.copy $own $imported + (i32.const 0) (i32.const 0) (i32.const 6))) + (func (export "call") (param i32) (result i32) + (call_indirect $own (type $i32) (local.get 0)))) + """, + store)) { + copier.export("copy").apply(); + assertEquals(42, call(copier, "call", 0)); + assertEquals(1000, call(copier, "call", 1)); + assertSame(owner, copier.table(1).instance(0)); + } + } + } + + /** Copies within and across tables keep the owners an interpreted instance wrote. */ + @Test + public void copiedEntriesKeepTheirInterpretedOwner() { + var store = new Store(); + try (var tableOwner = + nativeInstance( + """ + (module + (type $i32 (func (result i32))) + (table $a (export "a") 4 funcref) + (table $b 4 funcref) + (func (export "copyAToB") + (table.copy $b $a (i32.const 0) (i32.const 0) (i32.const 4))) + (func (export "shiftA") + (table.copy $a $a (i32.const 1) (i32.const 0) (i32.const 3))) + (func (export "callA") (param i32) (result i32) + (call_indirect $a (type $i32) (local.get 0))) + (func (export "callB") (param i32) (result i32) + (call_indirect $b (type $i32) (local.get 0)))) + """, + store)) { + store.register("owner", tableOwner); + try (var filler = + interpretedInstance( + """ + (module + (import "owner" "a" (table 4 funcref)) + (global $g i32 (i32.const 42)) + (func $answer (result i32) global.get $g) + (elem (i32.const 0) $answer)) + """, + store)) { + tableOwner.export("copyAToB").apply(); + assertEquals(42, call(tableOwner, "callB", 0)); + assertSame(filler, tableOwner.table(1).instance(0)); + + tableOwner.export("shiftA").apply(); + assertEquals(42, call(tableOwner, "callA", 0)); + assertEquals(42, call(tableOwner, "callA", 1)); + assertSame(filler, tableOwner.table(0).instance(1)); + assertNull(tableOwner.table(0).instance(2), "shifted in from a null entry"); + } + } + } + + @Test + public void hostWritesRecordTheOwner() { + var store = storeWithHost(); + try (var owner = nativeInstance(OWNER, store); + var holder = nativeInstance(HOLDER, store)) { + var table = holder.table(0); + table.setRef(0, 1, owner); // $readGlobal + assertEquals(42, call(holder, "call", 0)); + assertEquals(2, table.grow(1, 2, owner)); // $readMemory + assertEquals(1000, call(holder, "call", 2)); + assertSame(owner, table.instance(2)); + } + } + + /** table.set, table.grow and table.fill run natively in the instance writing the entry. */ + @Test + public void nativeWritesByAnotherInstance() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + try (var setter = + nativeInstance( + """ + (module + (import "holder" "tab" (table 2 funcref)) + (global $g i32 (i32.const 42)) + (func $answer (result i32) global.get $g) + (elem declare func $answer) + (func (export "set") (param i32) + (table.set (local.get 0) (ref.func $answer))) + (func (export "grow") (result i32) + (table.grow (ref.func $answer) (i32.const 1))) + (func (export "fill") + (table.fill (i32.const 0) (ref.func $answer) (i32.const 2)))) + """, + store)) { + setter.export("set").apply(0); + assertEquals(42, call(holder, "call", 0)); + assertSame(setter, holder.table(0).instance(0)); + + assertEquals(2, (int) setter.export("grow").apply()[0]); + assertEquals(42, call(holder, "call", 2)); + + setter.export("fill").apply(); + assertEquals(42, call(holder, "call", 1)); + } + } + } + + /** A new instance may get a closed owner's context address, and must not take its entries. */ + @Test + public void closedOwnerTrapsInsteadOfRunningAnotherInstance() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + var writer = nativeInstance(WRITER, store); + assertEquals(42, call(holder, "call", 0)); + writer.close(); + + try (var reader = nativeInstance(READER, store); + var interpretedReader = interpretedInstance(READER, store)) { + assertSame(writer, holder.table(0).instance(0)); + assertClosed(() -> call(reader, "call", 0)); + assertClosed(() -> call(holder, "call", 0)); + assertClosed(() -> call(interpretedReader, "call", 0)); + } + } + } + + /** The table drops closed instances that none of its entries refer to. */ + @Test + public void tableReleasesClosedWriters() { + var store = new Store(); + try (var holder = nativeInstance(HOLDER, store)) { + store.register("holder", holder); + for (int i = 0; i < 50; i++) { + nativeInstance(WRITER, store).close(); + } + var table = (NativeTable) holder.table(0); + assertTrue(table.writerCount() <= 8, table.writerCount() + " writers kept"); + assertClosed(() -> call(holder, "call", 0)); + } + } + + /** An image may leave functions out, which a table must not jump to. */ + @Test + public void uncompiledFunctionIsNotCalled() { + var module = + parse( + """ + (module + (type $i32 (func (result i32))) + (table 2 funcref) + (func $uncompiled (result i32) (i32.const 7)) + (elem (i32.const 0) $uncompiled) + (func (export "callElem") (result i32) + (call_indirect (type $i32) (i32.const 0))) + (func (export "setAndCall") (result i32) + (table.set (i32.const 1) (ref.func $uncompiled)) + (call_indirect (type $i32) (i32.const 1)))) + """); + try (var instance = + NativeMachineFactory.builder(module) + .withPrecompiledCode(withoutFirstBody(compile(module))) + .build()) { + var e = + assertThrows( + WasmEngineException.class, () -> instance.export("callElem").apply()); + assertTrue(e.getMessage().contains("not compiled"), e.getMessage()); + e = + assertThrows( + WasmEngineException.class, () -> instance.export("setAndCall").apply()); + assertTrue(e.getMessage().contains("uninitialized element"), e.getMessage()); + } + } + + /** + * Has the caller descend {@code levels} before calling the owner, until + * the stack runs out. + * + * @return how many levels {@code counted} recursed + */ + private static int depthReached(Instance caller, int levels, Instance counted) { + var depth = counted.exports().global("depth"); + int before = (int) depth.getValue(); + var e = assertThrows(WasmEngineException.class, () -> call(caller, "descend", levels)); + assertTrue(e.getMessage().contains("call stack exhausted"), e.getMessage()); + return (int) depth.getValue() - before; + } + + private static void assertClosed(Runnable call) { + var e = assertThrows(WasmEngineException.class, call::run); + assertTrue(e.getMessage().contains("instance is closed"), e.getMessage()); + } + + /** The owner and caller instances, closed caller first. */ + private record Linked(Instance owner, Instance caller) implements AutoCloseable { + @Override + public void close() { + caller.close(); + owner.close(); + } + } + + /** + * Instantiates the owner and the caller natively, with the caller importing + * the owner's table. + * + * @return the linked instances, to close after the test + */ + private static Linked linkNative() { + var store = storeWithHost(); + var owner = nativeInstance(OWNER, store); + store.register("owner", owner); + var caller = nativeInstance(CALLER, store); + return new Linked(owner, caller); + } + + /** A store with the owner's host import, which returns 7. */ + private static Store storeWithHost() { + var seven = + new HostFunction( + "host", + "seven", + FunctionType.of(List.of(), List.of(ValType.I32)), + (inst, args) -> new long[] {7}); + return new Store().addFunction(seven); + } + + /** + * Calls an export that takes a table index and returns an i32. + * + * @param instance the instance to call + * @param export the export's name + * @param index the table index to pass + * @return the result + */ + private static int call(Instance instance, String export, int index) { + return (int) instance.export(export).apply(index)[0]; + } + + /** Instantiates {@code wat} natively against the store's exports. */ + private static Instance nativeInstance(String wat, Store store) { + return NativeInstanceBuilder.builder(parse(wat)) + .withImportValues(store.toImportValues()) + .build(); + } + + /** Instantiates {@code wat} in the interpreter against the store's exports. */ + private static Instance interpretedInstance(String wat, Store store) { + return Instance.builder(parse(wat)).withImportValues(store.toImportValues()).build(); + } + + private static NativeCode compile(WasmModule module) { + return NativeCompiler.compile(RedlineTarget.detectHost().orElseThrow().triple(), module); + } + + /** Returns {@code code} with its first function body left uncompiled. */ + private static NativeCode withoutFirstBody(NativeCode code) { + int[] bodies = new int[code.functionBodyCount()]; + int[] entries = new int[code.functionBodyCount()]; + for (int i = 0; i < bodies.length; i++) { + bodies[i] = i == 0 ? NativeCode.NOT_COMPILED : code.bodyOffset(i); + entries[i] = i == 0 ? NativeCode.NOT_COMPILED : code.entryTrampolineOffset(i); + } + int[] importTrampolines = new int[code.importCount()]; + int[] importSlots = new int[code.importCount()]; + for (int i = 0; i < importTrampolines.length; i++) { + importTrampolines[i] = code.importTrampolineOffset(i); + importSlots[i] = code.importStubSlotOffset(i); + } + return new NativeCode( + code.triple(), code.image(), bodies, entries, importTrampolines, importSlots); + } + + /** Parses a module from WebAssembly text format. */ + private static WasmModule parse(String wat) { + return Parser.parse(Wat2Wasm.parse(wat)); + } +} diff --git a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java index 25df60753..de6c497c0 100644 --- a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java +++ b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeMachine.java @@ -1,5 +1,7 @@ package run.endive.redline.experimental.runner.internal; +import static java.util.Objects.requireNonNullElse; + import java.lang.foreign.Arena; import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; @@ -10,6 +12,7 @@ import java.lang.invoke.MethodType; import java.lang.ref.Reference; import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicLong; import run.endive.redline.experimental.api.NativeCode; import run.endive.redline.experimental.api.internal.CtxBuffer; import run.endive.redline.experimental.api.internal.InterruptWatchdog; @@ -44,6 +47,9 @@ public final class NativeMachine implements Machine { private static final int CTX_SIZE = CtxBuffer.CTX_SIZE; private static final long PAGE_SIZE = 4096; + // Never reused, unlike a context address, so a stale entry cannot match a new instance + private static final AtomicLong OWNER_IDS = new AtomicLong(); + // Conversion handles for adapting downcalls to uniform (MS, MS, long[]) → long. // These use Wasm bit-reinterpretation (not numeric casts). private static final MethodHandle LONG_TO_FLOAT; @@ -89,7 +95,9 @@ public final class NativeMachine implements Machine { private final MemorySegment argsBuffer; private final MemorySegment globalsBuffer; private final MemorySegment funcTypesArray; + private final long ownerId = OWNER_IDS.incrementAndGet(); private MemorySegment tablePtrsArray; + private MemorySegment tableWriters; private NativeTable[] nativeTables; private boolean tablesInitialized; private final int numImports; @@ -191,6 +199,7 @@ public NativeMachine( ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.FUNC_TYPES_PTR, funcTypesArray.address()); ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.MEMMOVE_PTR, PanamaExecutor.MEMMOVE_ADDR); ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.MEMSET_PTR, PanamaExecutor.MEMSET_ADDR); + ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.OWNER_ID, ownerId); byte[] image = code.image(); this.codeRegionSize = (Math.max(image.length, 1) + PAGE_SIZE - 1) & -PAGE_SIZE; @@ -240,6 +249,20 @@ public NativeMachine( this.ownsMemory = instance.imports().memoryCount() == 0; } + /** Returns the id that the table entries this machine owns hold. */ + long ownerId() { + return ownerId; + } + + boolean isClosed() { + return closed; + } + + /** Returns the instance this machine executes. */ + Instance instance() { + return instance; + } + @Override public void close() { if (closed) { @@ -512,10 +535,12 @@ private MemorySegment createTrampolineStub() { } /** - * Compiled code only reaches this with a table operation sentinel: it emits - * call_indirect inline and never writes TYPE_ID, TABLE_IDX or ELEM_IDX, so the - * call_indirect path this used to carry could only ever have dispatched on - * whatever those fields happened to hold. + * Entry point for compiled code that needs the runner: table operations, + * flagged by a negative argument count, and call_indirect through an entry + * another instance owns. + * + * @param ctxAddr the calling function's context + * @return the operation's result, or 0 if it recorded an exception */ @SuppressWarnings("unused") private long callIndirectTrampoline(long ctxAddr) { @@ -525,30 +550,73 @@ private long callIndirectTrampoline(long ctxAddr) { if (argCount < 0) { return handleTableOperation(argCount); } - throw new WasmEngineException("Unexpected trampoline call: argCount " + argCount); + return callForeignEntry( + ctx.get(ValueLayout.JAVA_INT, CtxBuffer.TYPE_ID), + ctx.get(ValueLayout.JAVA_INT, CtxBuffer.TABLE_IDX), + ctx.get(ValueLayout.JAVA_INT, CtxBuffer.ELEM_IDX), + argCount); } catch (Throwable t) { recordHostException(t); return 0L; } } + /** + * Calls a table entry this instance does not own, in the instance that + * does. Compiled code has already checked the index. The entry's type + * index is the owner's, so the signature is checked here by structure. The + * call runs through the owner's machine, against the owner's memory, + * globals and tables. A trap there becomes an exception that unwinds this + * instance too. + * + * @param typeId the expected type, in this module's index space + * @param tableIdx the table, in this module's index space + * @param elemIdx the entry + * @param argCount the number of arguments in argsBuffer + * @return a single result as raw bits, or 0 with the results left in + * argsBuffer for a multi-value call + */ + private long callForeignEntry(int typeId, int tableIdx, int elemIdx, int argCount) { + NativeTable table = nativeTables[tableIdx]; + int funcId = table.ref(elemIdx); + if (funcId == Value.REF_NULL_VALUE) { + throw new TrapException("uninitialized element"); + } + // An entry written with no owner runs in the caller, as in the interpreter + Instance owner = requireNonNullElse(table.instance(elemIdx), instance); + var expected = instance.type(typeId); + var actual = owner.type(owner.functionType(funcId)); + if (!FunctionType.matches(actual, expected, instance.module().typeSection())) { + throw new TrapException("indirect call type mismatch"); + } + long[] args = new long[argCount]; + for (int i = 0; i < argCount; i++) { + args[i] = argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(i)); + } + // One stack budget for the whole chain, or each instance would add its own + long stackLimit = ctxBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.STACK_LIMIT); + long[] results = + owner.getMachine() instanceof NativeMachine ownerMachine + ? ownerMachine.callWithStackLimit(funcId, args, stackLimit) + : owner.getMachine().call(funcId, args); + refreshMemoryPages(); + if (expected.returns().size() > 1) { + for (int i = 0; i < results.length; i++) { + argsBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(i), results[i]); + } + return 0L; + } + return results == null || results.length == 0 ? 0L : results[0]; + } + private long handleTableOperation(int opCode) { switch (opCode) { case -1 -> { // table grow fill int oldSize = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); int newSize = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(1)); int fillValue = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(2)); - long tableAddr = argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(3)); int tblIdx = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(4)); - boolean externRef = nativeTables[tblIdx].isExternRef(); - var tableBuf = - MemorySegment.ofAddress(tableAddr) - .reinterpret( - CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) newSize * CtxBuffer.TABLE_ENTRY_SIZE); - for (int i = oldSize; i < newSize; i++) { - writeTableEntry(tableBuf, i, fillValue, externRef); - } + nativeTables[tblIdx].fill(oldSize, newSize, fillValue, instance); // Update TableLimits so import validation sees the grown size nativeTables[tblIdx].limits().grow(newSize - oldSize); } @@ -556,45 +624,16 @@ private long handleTableOperation(int opCode) { int offset = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); int end = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(1)); int fillValue = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(2)); - long tableAddr = argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(3)); int tblIdx = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(4)); - boolean externRef = nativeTables[tblIdx].isExternRef(); - var tableBuf = - MemorySegment.ofAddress(tableAddr) - .reinterpret( - CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) end * CtxBuffer.TABLE_ENTRY_SIZE); - for (int i = offset; i < end; i++) { - writeTableEntry(tableBuf, i, fillValue, externRef); - } + nativeTables[tblIdx].fill(offset, end, fillValue, instance); } - case -3 -> { // table copy (16-byte entries) + case -3 -> { // table copy long srcAddr = argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); long dstAddr = argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(1)); int srcOff = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(2)); int dstOff = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(3)); int size = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(4)); - long entrySize = CtxBuffer.TABLE_ENTRY_SIZE; - var srcBuf = - MemorySegment.ofAddress(srcAddr) - .reinterpret( - CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) (srcOff + size) * entrySize); - var dstBuf = - MemorySegment.ofAddress(dstAddr) - .reinterpret( - CtxBuffer.TABLE_ENTRIES_OFFSET - + (long) (dstOff + size) * entrySize); - // Copy 16-byte entries with correct overlap handling - if (dstOff <= srcOff) { - for (int i = 0; i < size; i++) { - copyTableEntry(srcBuf, srcOff + i, dstBuf, dstOff + i); - } - } else { - for (int i = size - 1; i >= 0; i--) { - copyTableEntry(srcBuf, srcOff + i, dstBuf, dstOff + i); - } - } + tableAt(dstAddr).copyFrom(tableAt(srcAddr), srcOff, dstOff, size); } case -4 -> { // table init int tableIdx = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); @@ -611,6 +650,11 @@ private long handleTableOperation(int opCode) { int elemIdx = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); instance.setElement(elemIdx, null); } + case -6 -> { // first native table.set into a table + int tblIdx = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); + nativeTables[tblIdx].registerWriter(this); + tableWriters.set(ValueLayout.JAVA_INT, (long) tblIdx * 4, 1); + } case -8 -> { // memory.init int segmentId = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(0)); int dst = (int) argsBuffer.get(ValueLayout.JAVA_LONG, CtxBuffer.argOffset(1)); @@ -647,40 +691,19 @@ private long handleTableOperation(int opCode) { return 0L; } - private void writeTableEntry( - MemorySegment tableBuf, int index, int funcId, boolean isExternRef) { - long base = CtxBuffer.TABLE_ENTRIES_OFFSET + (long) index * CtxBuffer.TABLE_ENTRY_SIZE; - if (funcId == Value.REF_NULL_VALUE) { - tableBuf.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - tableBuf.set( - ValueLayout.JAVA_INT, - base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, - Value.REF_NULL_VALUE); - tableBuf.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); - } else if (isExternRef) { - tableBuf.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - tableBuf.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - tableBuf.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); - } else { - long funcPtr = funcTable.get(ValueLayout.JAVA_LONG, (long) funcId * 8); - int typeIdx = funcTypesArray.get(ValueLayout.JAVA_INT, (long) funcId * 4); - tableBuf.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - tableBuf.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - tableBuf.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + /** + * Finds this instance's table whose native buffer is at {@code address}. + * + * @param address a table buffer address + * @return the table + */ + private NativeTable tableAt(long address) { + for (NativeTable table : nativeTables) { + if (table.nativeBuffer().address() == address) { + return table; + } } - } - - /** Copy a single 16-byte table entry from src[srcIdx] to dst[dstIdx]. */ - private static void copyTableEntry( - MemorySegment src, int srcIdx, MemorySegment dst, int dstIdx) { - long srcBase = CtxBuffer.TABLE_ENTRIES_OFFSET + (long) srcIdx * CtxBuffer.TABLE_ENTRY_SIZE; - long dstBase = CtxBuffer.TABLE_ENTRIES_OFFSET + (long) dstIdx * CtxBuffer.TABLE_ENTRY_SIZE; - int typeIdx = src.get(ValueLayout.JAVA_INT, srcBase + CtxBuffer.ENTRY_TYPE_IDX_OFFSET); - int funcId = src.get(ValueLayout.JAVA_INT, srcBase + CtxBuffer.ENTRY_FUNC_ID_OFFSET); - long funcPtr = src.get(ValueLayout.JAVA_LONG, srcBase + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); - dst.set(ValueLayout.JAVA_INT, dstBase + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - dst.set(ValueLayout.JAVA_INT, dstBase + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - dst.set(ValueLayout.JAVA_LONG, dstBase + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + throw new WasmEngineException("no table at " + address); } // --- Memory grow upcall stub --- @@ -801,6 +824,7 @@ private void initializeNativeTables() { // Just collect references and build the pointer array. this.nativeTables = new NativeTable[tableCount]; this.tablePtrsArray = arena.allocate((long) tableCount * 8, 8); + this.tableWriters = arena.allocate((long) tableCount * 4, 4).fill((byte) 0); for (int i = 0; i < tableCount; i++) { var table = instance.table(i); @@ -816,6 +840,7 @@ private void initializeNativeTables() { } ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.TABLE_PTRS, tablePtrsArray.address()); + ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.TABLE_WRITERS_PTR, tableWriters.address()); } /** Package-private: used by NativeTable to resolve funcId → funcPtr across modules. */ @@ -907,6 +932,15 @@ private Class valTypeToJavaClass(ValType type) { @Override public long[] call(int funcId, long[] args) throws WasmEngineException { + return callWithStackLimit(funcId, args, 0L); + } + + /** Calls with the stack limit an outermost call starts from, 0 to anchor a new one. */ + private long[] callWithStackLimit(int funcId, long[] args, long stackLimit) { + if (closed) { + // Its native code and buffers are gone, but a table may still hold its functions + throw new WasmEngineException("instance is closed"); + } if (funcId < numImports) { // Host import — delegate directly var imprt = instance.imports().function(funcId); @@ -914,6 +948,9 @@ public long[] call(int funcId, long[] args) throws WasmEngineException { } var handle = downcalls[funcId]; + if (handle == null) { + throw new WasmEngineException("Function " + funcId + " not compiled"); + } try { boolean outermostCall = callDepth++ == 0; @@ -927,7 +964,7 @@ public long[] call(int funcId, long[] args) throws WasmEngineException { // against where the outer call began, or every level moves the // limit deeper and the guard stops firing. if (outermostCall) { - ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.STACK_LIMIT, 0L); + ctxBuffer.set(ValueLayout.JAVA_LONG, CtxBuffer.STACK_LIMIT, stackLimit); } if (!memBaseInitialized) { diff --git a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeTable.java b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeTable.java index 2831ec6a3..fe48d6566 100644 --- a/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeTable.java +++ b/redline/runner/src/main/java/run/endive/redline/experimental/runner/internal/NativeTable.java @@ -5,6 +5,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.util.Arrays; import run.endive.redline.experimental.api.internal.CtxBuffer; import run.endive.runtime.Instance; import run.endive.runtime.TableInstance; @@ -19,14 +20,22 @@ * *

Layout: [size:i32 @ 0][max:i32 @ 4][entries... @ 8] * - *

Each entry is 16 bytes: + *

Each entry is 24 bytes: * 

  *   [0..4)   i32  canonicalTypeIdx
  *   [4..8)   i32  funcId
  *   [8..16)  i64  funcPtr (native address)
+ *   [16..24) i64  ownerId (id of the native instance defining the function)
  *
* - *

NULL entry: funcId == REF_NULL_VALUE (-1), funcPtr == 0, typeIdx == 0. + *

NULL entry: funcId == REF_NULL_VALUE (-1), funcPtr == 0, typeIdx == 0, ownerId == 0. + * + *

The type index, function id and pointer belong to the instance that owns + * the function, which is not necessarily the one using the table. Native code + * calls an entry directly only from the owner's instance. The table keeps every + * native instance whose id its entries hold, see {@link #writers}. An entry + * owned by an instance without native code has no pointer and no owner id; its + * owner is kept in {@link #owners} instead. * *

The entries array is pre-allocated to the table's max capacity so that * {@code TABLE.GROW} only bumps the size field without reallocation. @@ -45,6 +54,19 @@ public final class NativeTable extends TableInstance { private final int capacity; private final boolean isExternRef; + /** + * Owner of each entry written from Java without an owner id: one not + * yet resolved, or one owned by an instance without native code. It grows + * with the table instead of being allocated at capacity, which can be a + * million entries. + */ + private Instance[] owners = new Instance[0]; + + // Machines whose ids entries may hold, replaced on change so readers need no lock + private volatile NativeMachine[] writers = new NativeMachine[0]; + private int pruneAt = 8; + private volatile NativeMachine lastWriter; + public NativeTable(Table table, int initValue, Arena arena) { super(table, initValue); this.isExternRef = table.elementType().equals(ValType.ExternRef); @@ -68,10 +90,9 @@ public NativeTable(Table table, int initValue, Arena arena) { } if (initValue != REF_NULL_VALUE) { - // The instance has no machine yet, so funcPtr cannot be resolved - // here. resolvePendingRefs fills it in once there is one. + // Instance records the owner right after; otherwise the first caller takes it for (int i = 0; i < initial; i++) { - writeUnresolvedEntry(i, initValue); + writeUnresolvedEntry(i, initValue, null); } } } @@ -81,71 +102,248 @@ private long entryBase(int index) { } private void writeNullEntry(int index) { - long base = entryBase(index); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, REF_NULL_VALUE); - buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + writeEntry(index, 0, REF_NULL_VALUE, 0L, 0L); + setOwner(index, null); } - /** A funcref whose native address is not known yet. */ - private void writeUnresolvedEntry(int index, int value) { - long base = entryBase(index); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, value); - buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + /** + * Writes a funcref with no native address, either not resolved yet or owned + * by an instance without native code. + * + * @param index the entry to write + * @param value the function id, in {@code owner}'s index space + * @param owner the instance that defines the function, or {@code null} if + * unknown + */ + private void writeUnresolvedEntry(int index, int value, Instance owner) { + writeEntry(index, 0, value, 0L, 0L); + setOwner(index, owner); } /** - * Fills in the native address of entries written before the instance had a - * machine, which is the case for a table initialiser. + * Writes every field of entry {@code index}. + * + * @param index the entry to write + * @param typeIdx the owner's canonical type index for the function + * @param funcId the function id, in the owner's index space + * @param funcPtr the function's native address, or 0 + * @param ownerId the owner's id, or 0 */ + private void writeEntry(int index, int typeIdx, int funcId, long funcPtr, long ownerId) { + long base = entryBase(index); + buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); + buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); + buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_OWNER_ID_OFFSET, ownerId); + } + + private long ownerId(int index) { + return buffer.get( + ValueLayout.JAVA_LONG, entryBase(index) + CtxBuffer.ENTRY_OWNER_ID_OFFSET); + } + + /** Resolves entries written with no owner, which only a host does, against {@code instance}. */ void resolvePendingRefs(Instance instance) { if (isExternRef) { return; } int sz = size(); for (int i = 0; i < sz; i++) { - long base = entryBase(i); - int funcId = buffer.get(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET); - long funcPtr = - buffer.get(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET); - if (funcId != REF_NULL_VALUE && funcPtr == 0L) { + int funcId = + buffer.get(ValueLayout.JAVA_INT, entryBase(i) + CtxBuffer.ENTRY_FUNC_ID_OFFSET); + if (funcId != REF_NULL_VALUE && ownerId(i) == 0L && recordedOwner(i) == null) { resolveFromInstance(i, funcId, instance); } } } - private void writeOpaqueEntry(int index, int value) { - long base = entryBase(index); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, 0); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, value); - buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, 0L); + /** + * Writes {@code funcId} with the native address, type index and id of + * {@code owner}'s function. + * + * @param index the entry to write + * @param funcId the function id, in {@code owner}'s index space + * @param owner the open native machine that defines the function + */ + private void writeResolvedEntry(int index, int funcId, NativeMachine owner) { + long funcPtr = owner.getFuncTable().get(ValueLayout.JAVA_LONG, (long) funcId * 8); + if (funcPtr == 0L) { + // Not compiled, so only the owner's runner can report it + writeUnresolvedEntry(index, funcId, owner.instance()); + return; + } + registerWriter(owner); + int typeIdx = owner.getFuncTypesArray().get(ValueLayout.JAVA_INT, (long) funcId * 4); + writeEntry(index, typeIdx, funcId, funcPtr, owner.ownerId()); + setOwner(index, null); } - private void writeResolvedEntry(int index, int funcId, MemorySegment ft, MemorySegment fta) { - long base = entryBase(index); - long funcPtr = ft.get(ValueLayout.JAVA_LONG, (long) funcId * 8); - int typeIdx = fta.get(ValueLayout.JAVA_INT, (long) funcId * 4); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_TYPE_IDX_OFFSET, typeIdx); - buffer.set(ValueLayout.JAVA_INT, base + CtxBuffer.ENTRY_FUNC_ID_OFFSET, funcId); - buffer.set(ValueLayout.JAVA_LONG, base + CtxBuffer.ENTRY_FUNC_PTR_OFFSET, funcPtr); + /** + * Writes {@code funcId} as a function of {@code instance}. An open native + * instance gives the entry its address and id. Otherwise the entry keeps + * its owner here and is called through the runner. + * + * @param index the entry to write + * @param funcId the function id, in {@code instance}'s index space + * @param instance the instance that defines the function, or {@code null} + */ + private void resolveFromInstance(int index, int funcId, Instance instance) { + if (isExternRef) { + writeEntry(index, 0, funcId, 0L, 0L); + } else if (instance != null + && instance.getMachine() instanceof NativeMachine nm + && !nm.isClosed()) { + writeResolvedEntry(index, funcId, nm); + } else { + writeUnresolvedEntry(index, funcId, instance); + } } /** - * Try to resolve funcId → funcPtr+typeIdx using the instance's NativeMachine. - * For externref tables, stores the value as-is without function resolution. - * Returns true if resolved, false if not (instance is null or not NativeMachine). + * Returns the owner recorded for entry {@code index}. + * + * @param index the entry + * @return the recorded owner, or {@code null} */ - private boolean resolveFromInstance(int index, int funcId, Instance instance) { - if (isExternRef) { - writeOpaqueEntry(index, funcId); - return true; + private Instance recordedOwner(int index) { + return index < owners.length ? owners[index] : null; + } + + /** + * Records the owner of entry {@code index}, growing {@link #owners} as + * needed. + * + * @param index the entry + * @param owner the owning instance, or {@code null} + */ + private void setOwner(int index, Instance owner) { + if (index >= owners.length) { + if (owner == null) { + return; + } + ensureOwners(index + 1); } - if (instance != null && instance.getMachine() instanceof NativeMachine nm) { - writeResolvedEntry(index, funcId, nm.getFuncTable(), nm.getFuncTypesArray()); - return true; + owners[index] = owner; + } + + private void ensureOwners(int length) { + if (length > owners.length) { + owners = Arrays.copyOf(owners, Math.max(length, Math.max(size(), owners.length * 2))); + } + } + + /** Keeps {@code writer}, whose id entries of this table may now hold. */ + void registerWriter(NativeMachine writer) { + if (lastWriter == writer) { + return; + } + synchronized (this) { + for (NativeMachine w : writers) { + if (w == writer) { + lastWriter = writer; + return; + } + } + if (writers.length >= pruneAt) { + pruneWriters(); + pruneAt = Math.max(8, writers.length * 2); + } + NativeMachine[] grown = Arrays.copyOf(writers, writers.length + 1); + grown[writers.length] = writer; + writers = grown; + lastWriter = writer; + } + } + + // Drops closed writers that no entry refers to any more + private void pruneWriters() { + NativeMachine[] current = writers; + boolean[] keep = new boolean[current.length]; + boolean anyClosed = false; + for (int w = 0; w < current.length; w++) { + keep[w] = !current[w].isClosed(); + anyClosed |= !keep[w]; + } + if (!anyClosed) { + return; + } + int sz = size(); + for (int i = 0; i < sz; i++) { + long id = ownerId(i); + if (id != 0L) { + for (int w = 0; w < current.length; w++) { + if (current[w].ownerId() == id) { + keep[w] = true; + break; + } + } + } + } + NativeMachine[] kept = new NativeMachine[current.length]; + int count = 0; + for (int w = 0; w < current.length; w++) { + if (keep[w]) { + kept[count++] = current[w]; + } + } + writers = Arrays.copyOf(kept, count); + lastWriter = null; + } + + /** Returns the machine, possibly closed, with owner id {@code id}. */ + private NativeMachine writer(long id) { + for (NativeMachine w : writers) { + if (w.ownerId() == id) { + return w; + } + } + throw new WasmEngineException("table entry owner " + id + " is not registered"); + } + + /** Returns how many machines the table keeps for its entries. */ + int writerCount() { + return writers.length; + } + + /** Writes {@code funcId}, a function of {@code instance}, to entries [from, to). */ + void fill(int from, int to, int funcId, Instance instance) { + for (int i = from; i < to; i++) { + if (funcId == REF_NULL_VALUE) { + writeNullEntry(i); + } else { + resolveFromInstance(i, funcId, instance); + } + } + } + + /** Copies {@code count} entries and their owners from {@code src}, which may be this table. */ + void copyFrom(NativeTable src, int srcIndex, int dstIndex, int count) { + MemorySegment.copy( + src.buffer, + src.entryBase(srcIndex), + buffer, + entryBase(dstIndex), + (long) count * CtxBuffer.TABLE_ENTRY_SIZE); + if (src != this) { + long lastId = 0L; + for (int i = dstIndex; i < dstIndex + count; i++) { + long id = ownerId(i); + if (id != 0L && id != lastId) { + registerWriter(src.writer(id)); + lastId = id; + } + } + } + int fromSrc = Math.max(0, Math.min(count, src.owners.length - srcIndex)); + if (fromSrc > 0) { + ensureOwners(dstIndex + fromSrc); + // Read src.owners after the resize, as src may be this table + System.arraycopy(src.owners, srcIndex, owners, dstIndex, fromSrc); + } + int clearTo = Math.min(dstIndex + count, owners.length); + if (dstIndex + fromSrc < clearTo) { + Arrays.fill(owners, dstIndex + fromSrc, clearTo, null); } - return false; } /** Get the native address of the table buffer, for passing to native code. */ @@ -195,11 +393,7 @@ public void setRef(int index, int value, Instance instance) { if (index < 0 || index >= size()) { throw new UninstantiableException("out of bounds table access"); } - if (value == REF_NULL_VALUE) { - writeNullEntry(index); - } else if (!resolveFromInstance(index, value, instance)) { - writeUnresolvedEntry(index, value); - } + fill(index, index + 1, value, instance); } @Override @@ -210,24 +404,28 @@ public int grow(int delta, int value, Instance instance) { if (delta < 0 || newSize > max || newSize > capacity) { return -1; } - // Fill new slots - for (int i = oldSize; i < newSize; i++) { - if (value == REF_NULL_VALUE) { - writeNullEntry(i); - } else if (!resolveFromInstance(i, value, instance)) { - writeUnresolvedEntry(i, value); - } - } + fill(oldSize, newSize, value, instance); // Update size buffer.set(ValueLayout.JAVA_INT, CtxBuffer.TABLE_SIZE_OFFSET, newSize); limits().grow(delta); return oldSize; } + /** + * Returns the instance whose function entry {@code index} holds, even if it + * has since been closed. + * + * @param index the entry + * @return the owning instance, or {@code null} for a null entry or one + * written with no owner + */ @Override public Instance instance(int index) { - // Single-module assumption — all entries belong to the same instance - return null; + if (index < 0 || index >= size() || isExternRef || ref(index) == REF_NULL_VALUE) { + return null; + } + long id = ownerId(index); + return id != 0L ? writer(id).instance() : recordedOwner(index); } @Override diff --git a/runtime/src/main/java/run/endive/runtime/Instance.java b/runtime/src/main/java/run/endive/runtime/Instance.java index 2a582a3f2..45004fa6a 100644 --- a/runtime/src/main/java/run/endive/runtime/Instance.java +++ b/runtime/src/main/java/run/endive/runtime/Instance.java @@ -146,6 +146,13 @@ static final class TailCallPending { for (int j = 0; j < tbl.size(); j++) { tbl.setObjRef(j, result.ref(), this); } + } else if (initValue != Value.REF_NULL_VALUE) { + // Record the owner, or a table importer would call it as its own function + var tbl = this.tables[i]; + var owner = computeConstantInstance(this, tables[i].initialize()); + for (int j = 0; j < tbl.size(); j++) { + tbl.setRef(j, initValue, owner); + } } } diff --git a/wasm-corpus/src/main/resources/compiled/table_init_owner-export.wat.wasm b/wasm-corpus/src/main/resources/compiled/table_init_owner-export.wat.wasm new file mode 100644 index 000000000..501750ee9 Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/table_init_owner-export.wat.wasm differ diff --git a/wasm-corpus/src/main/resources/compiled/table_init_owner-import.wat.wasm b/wasm-corpus/src/main/resources/compiled/table_init_owner-import.wat.wasm new file mode 100644 index 000000000..bcd7c2fff Binary files /dev/null and b/wasm-corpus/src/main/resources/compiled/table_init_owner-import.wat.wasm differ diff --git a/wasm-corpus/src/main/resources/wat/table_init_owner-export.wat b/wasm-corpus/src/main/resources/wat/table_init_owner-export.wat new file mode 100644 index 000000000..f28593a78 --- /dev/null +++ b/wasm-corpus/src/main/resources/wat/table_init_owner-export.wat @@ -0,0 +1,10 @@ +;; Every slot of the exported table starts out holding $answer. Nothing calls +;; this module, so only the table says whose function the slots hold. +(module + (global $g i32 (i32.const 42)) + + (func $answer (result i32) + global.get $g) + + (table (export "tab") 2 funcref (ref.func $answer)) +) diff --git a/wasm-corpus/src/main/resources/wat/table_init_owner-import.wat b/wasm-corpus/src/main/resources/wat/table_init_owner-import.wat new file mode 100644 index 000000000..84e2b631c --- /dev/null +++ b/wasm-corpus/src/main/resources/wat/table_init_owner-import.wat @@ -0,0 +1,15 @@ +;; Its own function has the same index as the exporter's $answer, but returns -1. +(module + (type $i32 (func (result i32))) + + (import "test" "tab" (table 2 funcref)) + + (global $g i32 (i32.const -1)) + + (func $own (result i32) + global.get $g) + + (func (export "call") (param i32) (result i32) + local.get 0 + call_indirect (type $i32)) +)