diff --git a/Readme.md b/Readme.md index 7feef542..04518dd7 100644 --- a/Readme.md +++ b/Readme.md @@ -88,6 +88,21 @@ classes and interfaces rather than opaque native handles (see without JNI glue ([test and demo](tests/integration/jvm_link_names/src/lib.rs)). +Java exposure is explicit: `pub` controls Rust visibility, while +`#[jvm_codegen::export]` on a public function, type, or module preserves its +Java-facing API. Marking a module includes its public descendants; marking a type +includes its public inherent methods. Register the tool with +`#![feature(register_tool)]` and `#![register_tool(jvm_codegen)]`. +For a library intended entirely for Java, add `#![feature(custom_inner_attributes)]` +and `#![jvm_codegen::export]` at the crate root, as in the examples below. +Mark the types Java constructs or inspects as well as the functions it calls. +Use concrete, nongeneric exports as Java entry points for generic Rust APIs. +Ordinary Rust dependencies remain internal: the compiler can flatten their +representations, merge their classes, and remove unused methods. Rebuild Rust +dependencies with the same backend; generated internal classes are not a stable +Java API. Importing existing Java classes with `jvm::class` or `jvm::interface` +does not require an export marker. + For example, one Rust API can expose an enum and accept both a JVM implementation of a Rust trait and a standard JVM lambda. Its result can then cross the Rust/Kotlin async bridge. The complete example is kept executable in @@ -96,6 +111,10 @@ cross the Rust/Kotlin async bridge. The complete example is kept executable in **Rust** ```rust +#![feature(register_tool, custom_inner_attributes)] +#![register_tool(jvm_codegen)] +#![jvm_codegen::export] + pub trait BatchObserver { fn accept(&mut self, processed: u32) -> bool; } @@ -247,8 +266,9 @@ Add `jvm = { package = "rcj", git = "https://github.com/IntegralPilot/rustc_code to `[dependencies]`. ```rust -#![feature(extern_types, register_tool)] +#![feature(extern_types, register_tool, custom_inner_attributes)] #![register_tool(jvm_codegen)] +#![jvm_codegen::export] #[jvm::class("java.time.LocalDate", rename_all = "camelCase")] impl JavaLocalDate { diff --git a/Tester.py b/Tester.py index 56c0e7a2..e99b3943 100644 --- a/Tester.py +++ b/Tester.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import zipfile from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -31,6 +32,13 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") +def normalize_internal_symbols(text: str) -> str: + """Private owner placement and Rust identity hashes are not debug semantics.""" + text = re.sub(r"\b[\w.$]+\.mono\.Mono(?:Bucket)?(?:_[0-9a-f]+)?\b", "private.Mono", text) + text = re.sub(r"\bMono(?:Bucket)?_[0-9a-f]+\b", "Mono", text) + return re.sub(r"(?:\$[0-9a-f]{16}|\$m\$*[0-9a-f]+)(?=\()", "", text) + + def write_failure(path: Path, proc) -> None: path.write_text( f"STDOUT:\n{proc.stdout}\n\nSTDERR:\n{proc.stderr}", @@ -69,8 +77,10 @@ def check_results(proc, test: TestCase, release: bool, logs: list[str]) -> bool: return True # so it works on Windows - expected = "".join(read(expected_path).strip().split()).replace("\\", "/") - actual = "".join(f"STDOUT:{proc.stdout.strip()}STDERR:{proc.stderr.strip()}".split()).replace("\\", "/") + expected = normalize_internal_symbols(read(expected_path)) + actual = normalize_internal_symbols(f"STDOUT:{proc.stdout.strip()}STDERR:{proc.stderr.strip()}") + expected = "".join(expected.strip().split()).replace("\\", "/") + actual = "".join(actual.split()).replace("\\", "/") if actual == expected: logs.append("|--- ✅ Output matches expected output!") @@ -350,6 +360,7 @@ def javap_debug_info(output: str, included_methods: set[str] | None = None) -> s source copies and reordering branches changes the executable line sequence. """ source: str | None = None + current_source: str | None = None methods: dict[str, list[str]] = {} local_rows: dict[str, set[str]] = {} current_method: str | None = None @@ -358,7 +369,7 @@ def javap_debug_info(output: str, included_methods: set[str] | None = None) -> s for line in output.splitlines(): stripped = line.strip() if stripped.startswith("Compiled from "): - source = stripped + current_source = stripped continue if ( line.startswith(" ") @@ -366,11 +377,14 @@ def javap_debug_info(output: str, included_methods: set[str] | None = None) -> s and "(" in stripped and stripped.endswith(";") ): + stripped = normalize_internal_symbols(stripped) current_method = ( stripped if included_methods is None or stripped in included_methods else None ) + if current_method is not None: + source = current_source table = None continue if stripped in {"LineNumberTable:", "LocalVariableTable:"}: @@ -440,9 +454,16 @@ def check_javap_debug_info( if not expected_path.exists(): return True - class_name = f"{test.artifact_name}.{test.artifact_name}" + class_names = [f"{test.artifact_name}.{test.artifact_name}"] + # Private Rust methods can move between classes. Debug checks must include their current owners. + with zipfile.ZipFile(jar) as archive: + class_names.extend(sorted( + name.removesuffix(".class").replace("/", ".") + for name in archive.namelist() + if "/mono/Mono" in name and name.endswith(".class") + )) logs.append("|--- 🔎 Checking JVM debug metadata with javap...") - proc = run_command(["javap", "-classpath", str(jar), "-c", "-l", "-p", class_name]) + proc = run_command(["javap", "-classpath", str(jar), "-c", "-l", "-p", *class_names]) if proc.returncode != 0: write_failure(test.directory / "javap-fail.generated", proc) logs.append(f"|---- ❌ javap exited with code {proc.returncode}") @@ -466,6 +487,26 @@ def check_javap_debug_info( return False +def check_scalar_paths(test: TestCase, jar: Path, release: bool, logs: list[str]) -> bool: + expected = test.directory / "scalar-paths.expected" + if not release or not expected.exists(): + return True + from test_codegen import scalar_path_errors + proc = run_command(["javap", "-classpath", str(jar), "-c", "-p", f"{test.artifact_name}.{test.artifact_name}"]) + if proc.returncode: + write_failure(test.directory / "javap-fail.generated", proc) + logs.append("|---- ❌ Could not inspect generated bytecode") + return False + errors = scalar_path_errors(proc.stdout, set(read(expected).split())) + if errors: + (test.directory / "scalar-paths-fail.generated").write_text("\n".join(errors) + "\n", encoding="utf-8") + logs.append("|---- ❌ Runtime work remains on a normal return path") + ci_diagnostic(logs, "\n".join(errors)) + return False + logs.append("|--- ✅ Scalar paths contain no runtime calls or object allocations") + return True + + def run_test( test: TestCase, release: bool, @@ -494,6 +535,8 @@ def run_test( return False, logs if not check_javap_debug_info(test, jar, release, logs): return False, logs + if not check_scalar_paths(test, jar, release, logs): + return False, logs logs.append("|--- ✅ Test passed!") return True, logs diff --git a/compiler-core/src/analysis/arrays.rs b/compiler-core/src/analysis/arrays.rs new file mode 100644 index 00000000..b1a51c7f --- /dev/null +++ b/compiler-core/src/analysis/arrays.rs @@ -0,0 +1,275 @@ +//! Prove that fresh primitive arrays have no decoded or encoded aliases. +//! Escape checks cover calls, storage, addresses and mixed joins. Local checks +//! also permit native writes before the first escape in a block. +use super::{NO_ORIGIN, origins::origins_with}; +use crate::ir::*; +use crate::opt::Live; + +pub(crate) fn native_array_accesses( + body: &Body, + types: &Types, + live: &Live, +) -> Vec> { + analyze(body, types, live).0 +} + +pub(crate) fn native_array_roots( + body: &Body, + types: &Types, + live: &Live, +) -> Vec> { + analyze(body, types, live).1 +} + +fn analyze( + body: &Body, + types: &Types, + live: &Live, +) -> (Vec>, Vec>) { + let candidates = body + .instructions + .iter() + .enumerate() + .filter_map(|(id, inst)| { + if !live.instructions[id] || !matches!(inst.op, Op::NewArray(_)) { + return None; + } + let value = inst.result?; + let Type::Array(element) = types.get(body.value_type(value))? else { + return None; + }; + StorageSlot::scalar(element, types).map(|_| (value, element)) + }) + .collect::>(); + if candidates.is_empty() { + return (Vec::new(), Vec::new()); + } + let mut roots = vec![NO_ORIGIN; body.values.len()]; + for (index, &(value, _)) in candidates.iter().enumerate() { + roots[value.index()] = index as u32; + } + let origins = origins_with( + body, + &roots, + |op| match op { + Op::Reinterpret(value) | Op::Refine(value) => Some(value), + Op::ViewRoot { + backing, + codec: None, + .. + } => Some(backing), + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } if body.methods[method.index()].owner == "org/rustlang/runtime/Pointer" + && matches!( + body.methods[method.index()].name.as_str(), + "scalarSliceRoot" | "locationSliceBacking" + ) => + { + Some(body.args[args.start as usize]) + } + _ => None, + }, + true, + ); + let mut escaped = vec![false; candidates.len()]; + for (id, inst) in body.instructions.iter().enumerate() { + if !live.instructions[id] { + continue; + } + inst.op.visit_uses(&body.args, |value| { + let origin = origins[value.index()]; + if origin == NO_ORIGIN { + return; + } + let element = candidates[origin as usize].1; + let safe = safe_use(body, types, *inst, value, element); + escaped[origin as usize] |= !safe; + }); + } + let mut escape = |value: ValueId| { + let origin = origins[value.index()]; + if origin != NO_ORIGIN { + escaped[origin as usize] = true; + } + }; + for edge in &body.edges { + for (&value, ¶m) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + if live.values[body.resolve(param).index()] + && origins[value.index()] != origins[param.index()] + { + escape(value); + } + } + } + for (index, block) in body.blocks.iter().enumerate() { + if live.blocks[index] { + block.terminator.unwrap().visit_uses(&mut escape); + } + } + // Permit native initialization before the first escape, including a later return. + // Restrict this proof to one block because other paths can expose the array. + let mut fresh = vec![None; body.blocks.len()]; + for block in &body.blocks { + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator { + if let Some(value) = body.instructions[inst.index()].result { + let root = roots[value.index()]; + if root != NO_ORIGIN { + fresh[body.edges[normal.index()].target.index()] = Some(root); + } + } + } + } + let mut available = vec![None; candidates.len()]; + let mut native = vec![None; body.instructions.len()]; + for (index, block) in body.blocks.iter().enumerate() { + if !live.blocks[index] { + continue; + } + let current = Some(BlockId::new(index)); + if let Some(root) = fresh[index] { + available[root as usize] = current; + } + let invoke = match block.terminator { + Some(Terminator::Invoke { inst, .. }) => Some(inst), + _ => None, + }; + for &id in block.instructions.iter().chain(invoke.iter()) { + if !live.instructions[id.index()] { + continue; + } + let inst = body.instructions[id.index()]; + if let Some((array, element)) = access(body, inst) { + let origin = origins[array.index()]; + if origin != NO_ORIGIN + && candidates[origin as usize].1 == element + && (!escaped[origin as usize] || available[origin as usize] == current) + { + native[id.index()] = Some(element); + } + } + inst.op.visit_uses(&body.args, |value| { + let origin = origins[value.index()]; + if origin != NO_ORIGIN + && !safe_use(body, types, inst, value, candidates[origin as usize].1) + { + available[origin as usize] = None; + } + }); + if let Some(value) = inst.result { + let root = roots[value.index()]; + if root != NO_ORIGIN { + available[root as usize] = current; + } + } + } + } + let values = origins + .iter() + .map(|&origin| { + (origin != NO_ORIGIN && !escaped[origin as usize]).then(|| candidates[origin as usize]) + }) + .collect(); + (native, values) +} + +fn access(body: &Body, inst: Inst) -> Option<(ValueId, TypeId)> { + Some(match inst.op { + Op::ArrayGet { array, .. } => (array, body.value_type(inst.result?)), + Op::ArraySet { array, value, .. } | Op::ArrayFill { array, value } => { + (array, body.value_type(value)) + } + Op::ViewGet(parts) => ( + body.args[parts.start as usize], + body.value_type(inst.result?), + ), + Op::ViewSet { parts, value } => (body.args[parts.start as usize], body.value_type(value)), + _ => return None, + }) +} + +fn safe_use(body: &Body, types: &Types, inst: Inst, value: ValueId, element: TypeId) -> bool { + match inst.op { + Op::ViewRoot { + backing, + size, + codec: None, + } => { + backing == value && StorageSlot::scalar(element, types).is_some_and(|s| s.size == size) + } + Op::LoadAddress(parts) => { + body.args[parts.start as usize] == value + && inst.result.is_some_and(|v| body.value_type(v) == element) + } + Op::StoreAddress { + parts, + value: stored, + } => body.args[parts.start as usize] == value && body.value_type(stored) == element, + Op::LocationEqual(parts) | Op::LocationCompare(parts) => { + body.args[parts.start as usize] == value || body.args[parts.start as usize + 2] == value + } + Op::LocationTag(parts) => body.args[parts.start as usize] == value, + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } => { + let method = &body.methods[method.index()]; + if method.owner == "org/rustlang/runtime/Pointer" { + match method.name.as_str() { + "nullableViewLocationTag" | "nullableLocationTag" => { + return body.args[args.start as usize] == value; + } + "offsetLocations" + | "offsetLocationsUnsigned" + | "byteOffsetLocations" + | "byteOffsetLocationsUnsigned" + | "sameLocation" => { + return body.args[args.start as usize] == value + || body.args[args.start as usize + 2] == value; + } + _ => {} + } + } + let size = match method.name.as_str() { + "scalarSliceRoot" if args.len == 2 => 1, + "locationSliceBacking" | "locationSliceOffset" if args.len == 3 => 2, + _ => return false, + }; + method.owner == "org/rustlang/runtime/Pointer" + && body.args[args.start as usize] == value + && body + .scalar_value(body.args[args.start as usize + size]) + .map(|s| s.bits()) + == StorageSlot::scalar(element, types).map(|s| u128::from(s.size)) + } + Op::Reinterpret(_) | Op::Refine(_) | Op::ArrayLength(_) => true, + Op::ArrayGet { array, .. } => { + array == value && inst.result.is_some_and(|r| body.value_type(r) == element) + } + Op::ArraySet { + array, + value: stored, + .. + } + | Op::ArrayFill { + array, + value: stored, + } => array == value && body.value_type(stored) == element, + Op::ViewGet(parts) => { + body.args[parts.start as usize] == value + && inst.result.is_some_and(|r| body.value_type(r) == element) + } + Op::ViewSet { + parts, + value: stored, + } => body.args[parts.start as usize] == value && body.value_type(stored) == element, + _ => false, + } +} diff --git a/compiler-core/src/analysis/arrays_tests.rs b/compiler-core/src/analysis/arrays_tests.rs new file mode 100644 index 00000000..d80b6c83 --- /dev/null +++ b/compiler-core/src/analysis/arrays_tests.rs @@ -0,0 +1,240 @@ +use super::native_array_accesses; +use crate::classfile::attributes::Instruction; +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +#[test] +fn initialization_is_native_only_until_the_first_escape() { + for unwind in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let array_ty = types.intern(Type::Array(int)); + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, int); + let one = b.constant(int, Scalar::integer(ScalarType::I32, 1).unwrap()); + let zero = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let array = if unwind { + let handler = b.create_block(); + let value = b + .invoke(Op::NewArray(one), Some(array_ty), handler) + .unwrap(); + let normal = b.current(); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + b.switch_to(normal); + value + } else { + b.emit(Op::NewArray(one), Some(array_ty)).unwrap() + }; + let before = b.body.instructions.len(); + b.emit( + Op::ArraySet { + array, + index: zero, + value: one, + native: false, + }, + None, + ); + let method = b.method(MethodRef { + owner: "Escape".into(), + name: "registerAlias".into(), + params: vec![array_ty], + returns: unit, + interface: false, + }); + let args = b.args([array]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + None, + ); + let after = b.body.instructions.len(); + let value = b + .emit( + Op::ArrayGet { + array, + index: zero, + native: false, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(value))); + let body = b.finish().unwrap(); + let native = native_array_accesses(&body, &types, &crate::opt::live(&body, &types)); + assert_eq!(native[before], Some(int)); + assert_eq!(native[after], None); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!(code.instructions.contains(&Instruction::Iastore)); + assert!(!code.instructions.contains(&Instruction::Iaload)); + } +} + +#[test] +fn private_slice_loop_uses_native_array_instructions() { + for escape in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let long = types.scalar(ScalarType::U64); + let array_ty = types.intern(Type::Array(int)); + let slice_ty = types.intern(Type::Slice(int)); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, int); + let limit = b.parameter(b.current(), int); + let one = b.constant(int, Scalar::integer(ScalarType::I32, 1).unwrap()); + let three = b.constant(int, Scalar::integer(ScalarType::I32, 3).unwrap()); + let array = b.emit(Op::NewArray(three), Some(array_ty)).unwrap(); + b.emit(Op::ArrayFill { array, value: one }, None); + let root = b.emit(Op::Reinterpret(array), Some(object)).unwrap(); + let length = b.constant(long, Scalar::integer(ScalarType::U64, 2).unwrap()); + let parts = b.args([root, one, length]); + let view = b.emit(Op::ViewPack(parts), Some(slice_ty)).unwrap(); + let index = b.variable(int); + b.define(index, one); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let iteration = b.read(index); + let condition = b + .emit( + Op::Binary { + op: BinaryOp::Lt, + left: iteration, + right: limit, + }, + Some(boolean), + ) + .unwrap(); + b.branch(condition, step, done); + b.switch_to(step); + b.emit( + Op::ArraySet { + native: false, + array: view, + index: one, + value: iteration, + }, + None, + ); + if escape { + let method = b.method(MethodRef { + owner: "Escape".into(), + name: "registerAlias".into(), + params: vec![object], + returns: unit, + interface: false, + }); + let args = b.args([root]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + None, + ); + } + let loaded = b + .emit( + Op::ArrayGet { + native: false, + array: view, + index: one, + }, + Some(int), + ) + .unwrap(); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: loaded, + right: one, + }, + Some(int), + ) + .unwrap(); + b.define(index, next); + b.jump(header, vec![]); + b.switch_to(done); + b.terminate(Terminator::Return(Some(iteration))); + let mut body = b.finish().unwrap(); + crate::opt::decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = crate::opt::live(&body, &types); + let native = native_array_accesses(&body, &types, &live); + for (id, inst) in body.instructions.iter().enumerate() { + if matches!(inst.op, Op::ViewGet(_) | Op::ViewSet { .. }) { + assert_eq!(native[id], (!escape).then_some(int)); + } + } + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, Instruction::New(_))) + ); + assert_eq!(code.instructions.contains(&Instruction::Iaload), !escape); + assert_eq!(code.instructions.contains(&Instruction::Iastore), !escape); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, Instruction::Getfield(_))) + ); + } +} + +#[test] +fn mixed_array_origins_escape_even_when_the_join_only_reads() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let array_ty = types.intern(Type::Array(int)); + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), boolean); + let input = b.parameter(b.current(), array_ty); + let one = b.constant(int, Scalar::integer(ScalarType::I32, 1).unwrap()); + let array = b.emit(Op::NewArray(one), Some(array_ty)).unwrap(); + let variable = b.variable(array_ty); + b.define(variable, array); + let other = b.create_block(); + let joined = b.create_block(); + b.branch(condition, other, joined); + b.switch_to(other); + b.define(variable, input); + b.jump(joined, vec![]); + b.switch_to(joined); + let alias = b.read(variable); + let zero = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let loaded = b + .emit( + Op::ArrayGet { + native: false, + array: alias, + index: zero, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(loaded))); + let body = b.finish().unwrap(); + let live = crate::opt::live(&body, &types); + assert!( + native_array_accesses(&body, &types, &live) + .iter() + .all(Option::is_none) + ); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!(!code.instructions.contains(&Instruction::Iaload)); +} diff --git a/compiler-core/src/analysis/mod.rs b/compiler-core/src/analysis/mod.rs new file mode 100644 index 00000000..2b43b87e --- /dev/null +++ b/compiler-core/src/analysis/mod.rs @@ -0,0 +1,9 @@ +//! Bounded facts shared by representation and storage optimizations. +mod origins; +pub(crate) use origins::{NO_ORIGIN, origins}; +mod users; +pub(crate) use users::ValueUsers; +mod arrays; +pub(crate) use arrays::{native_array_accesses, native_array_roots}; +#[cfg(test)] +mod arrays_tests; diff --git a/compiler-core/src/analysis/origins.rs b/compiler-core/src/analysis/origins.rs new file mode 100644 index 00000000..38b86e81 --- /dev/null +++ b/compiler-core/src/analysis/origins.rs @@ -0,0 +1,135 @@ +//! Allocation origins through identity annotations and equal-origin CFG joins. +use crate::ir::*; + +pub(crate) const NO_ORIGIN: u32 = u32::MAX; +const UNKNOWN: u32 = u32::MAX - 1; + +/// Track allocation identity through aliases and joins. +/// Only roots in an entry block without incoming edges can cross joins. +/// Conflicting inputs make joins unknown. Each value changes at most twice. +pub(crate) fn origins(body: &Body, roots: &[u32]) -> Vec { + origins_with( + body, + roots, + |op| match op { + Op::Reinterpret(source) | Op::Refine(source) => Some(source), + _ => None, + }, + false, + ) +} + +pub(crate) fn origins_with( + body: &Body, + roots: &[u32], + source_of: impl Fn(Op) -> Option, + ignore_uninit: bool, +) -> Vec { + let count = body.values.len(); + let mut stable = vec![ + false; + roots + .iter() + .filter(|&&r| r != NO_ORIGIN) + .max() + .map_or(0, |r| *r as usize + 1) + ]; + if !body.edges.iter().any(|edge| edge.target == body.entry) { + let entry = &body.blocks[body.entry.index()]; + let invoke = match entry.terminator { + Some(Terminator::Invoke { inst, .. }) => Some(inst), + _ => None, + }; + for &id in entry.instructions.iter().chain(invoke.iter()) { + if let Some(value) = body.instructions[id.index()].result { + let root = roots[value.index()]; + if root != NO_ORIGIN { + stable[root as usize] = true; + } + } + } + } + let mut state = roots.to_vec(); + let mut joins = vec![false; count]; + let mut undefined = vec![false; count]; + let mut users = super::ValueUsers::new(count); + for (index, value) in body.values.iter().enumerate() { + if roots[index] != NO_ORIGIN { + continue; + } + let source = match value.def { + ValueDef::Alias(source) => Some(source), + ValueDef::Inst(inst) => { + let op = body.instructions[inst.index()].op; + // An inactive enum payload can use any allocation. A real null cannot. + if ignore_uninit + && matches!(op, Op::Constant(id) if matches!(body.constants[id.index()], Constant::Uninit(_))) + { + undefined[index] = true; + state[index] = UNKNOWN; + } + source_of(op) + } + ValueDef::Param(block) if block != body.entry => { + joins[index] = true; + state[index] = UNKNOWN; + None + } + _ => None, + }; + if let Some(source) = source { + state[index] = UNKNOWN; + users.connect(source, index); + } + } + for edge in &body.edges { + for (&source, &target) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + if joins[target.index()] { + users.connect(source, target.index()); + } + } + } + let mut pending = (0..count) + .filter(|&i| state[i] != UNKNOWN) + .collect::>(); + for phase in 0..2 { + while let Some(source) = pending.pop() { + for target in users.users(source) { + let mut incoming = state[source]; + if joins[target] && incoming != NO_ORIGIN && !stable[incoming as usize] { + incoming = NO_ORIGIN; + } + let previous = state[target]; + let merged = if previous == UNKNOWN || previous == incoming { + incoming + } else { + NO_ORIGIN + }; + if merged != previous { + state[target] = merged; + pending.push(target); + } + } + } + if phase == 0 { + // A cycle without a defining root cannot prove allocation identity, + // even if another incoming edge has a known root. + for (index, value) in state.iter_mut().enumerate() { + if *value == UNKNOWN && !undefined[index] { + *value = NO_ORIGIN; + pending.push(index); + } + } + } + } + for origin in &mut state { + if *origin == UNKNOWN { + *origin = NO_ORIGIN; + } + } + state +} diff --git a/compiler-core/src/analysis/users.rs b/compiler-core/src/analysis/users.rs new file mode 100644 index 00000000..ea9c7bed --- /dev/null +++ b/compiler-core/src/analysis/users.rs @@ -0,0 +1,73 @@ +//! Store reverse value dependencies in two flat allocations. +//! Representation passes share this graph instead of per-value vectors. +use crate::ir::ValueId; + +const END: u32 = u32::MAX; + +pub(crate) struct ValueUsers { + heads: Vec, + edges: Vec<(u32, u32)>, +} + +impl ValueUsers { + pub fn new(count: usize) -> Self { + Self { + heads: vec![END; count], + edges: Vec::new(), + } + } + + pub fn connect(&mut self, source: ValueId, user: usize) { + let next = self.heads[source.index()]; + self.heads[source.index()] = u32::try_from(self.edges.len()).expect("too many value uses"); + self.edges + .push((next, u32::try_from(user).expect("too many values"))); + } + + pub fn users(&self, source: usize) -> impl Iterator + '_ { + let mut next = self.heads[source]; + std::iter::from_fn(move || { + if next == END { + return None; + } + let (following, user) = self.edges[next as usize]; + next = following; + Some(user as usize) + }) + } + + /// Reject a candidate if any input is ineligible. + /// Queue each rejected value once, including values in cycles. + pub fn close(&self, eligible: &mut [bool]) { + let mut pending = eligible + .iter() + .enumerate() + .filter_map(|(index, &yes)| (!yes).then_some(index as u32)) + .collect::>(); + while let Some(index) = pending.pop() { + for user in self.users(index as usize) { + if std::mem::replace(&mut eligible[user], false) { + pending.push(user as u32); + } + } + } + } + + /// Propagate proven identity aliases. Joins retain their own components. + /// They must not reuse the components of one input. + pub fn propagate(&self, eligible: &[bool], known: &mut [Option]) { + let mut pending = known + .iter() + .enumerate() + .filter_map(|(index, value)| value.is_some().then_some(index as u32)) + .collect::>(); + while let Some(index) = pending.pop() { + for user in self.users(index as usize) { + if eligible[user] && known[user].is_none() { + known[user] = known[index as usize]; + pending.push(user as u32); + } + } + } + } +} diff --git a/compiler-core/src/classfile/constant_pool.rs b/compiler-core/src/classfile/constant_pool.rs index b423cb7a..cd3c562e 100644 --- a/compiler-core/src/classfile/constant_pool.rs +++ b/compiler-core/src/classfile/constant_pool.rs @@ -9,6 +9,8 @@ pub struct InternedConstantPool { pool: ConstantPool<'static>, constants: HashMap, strings: HashMap, + resource_anchor: Option, + resources: Vec, } impl Default for InternedConstantPool { @@ -17,6 +19,8 @@ impl Default for InternedConstantPool { pool: ConstantPool::default(), constants: HashMap::default(), strings: HashMap::default(), + resource_anchor: None, + resources: Vec::new(), } } } @@ -30,7 +34,30 @@ impl Deref for InternedConstantPool { } impl InternedConstantPool { + pub fn set_resource_anchor(&mut self, owner: u16) { + self.resource_anchor = Some(owner); + } + + pub fn resource_anchor(&self) -> Option { + self.resource_anchor + } + + pub fn add_resource(&mut self, bytes: Vec) -> jvm::Result { + let resource = super::resources::Resource::new(bytes); + let name = self.add_name_string(&resource.name)?; + self.resources.push(resource); + Ok(name) + } + + pub fn take_resources(&mut self) -> Vec { + std::mem::take(&mut self.resources) + } + pub fn into_inner(self) -> ConstantPool<'static> { + assert!( + self.resources.is_empty(), + "binary constants were not emitted" + ); self.pool } @@ -84,20 +111,15 @@ impl InternedConstantPool { pub fn add_string>(&mut self, value: S) -> jvm::Result { let value = value.as_ref(); - let string_index = if value.starts_with(super::names::STRING_TAG) { - self.add_utf8(format!("{}{value}", super::names::LITERAL_STRING))? - } else { - self.add_utf8(value)? - }; + // Literal text must never become a reflection or class-liveness root. + // The final linker removes this tag without interpreting its payload. + let string_index = self.add_utf8(format!("{}{value}", super::names::LITERAL_STRING))?; self.add(Constant::String(string_index)) } /// Compiler-generated reflection names/descriptors need the same namespace /// relocation as class references. Rust string literals use `add_string`. pub fn add_name_string(&mut self, value: &str) -> jvm::Result { - if !value.contains(super::names::CRATE_MARKER) { - return self.add_string(value); - } let index = self.add_utf8(format!("{}{value}", super::names::NAME_STRING))?; self.add(Constant::String(index)) } diff --git a/compiler-core/src/classfile/mod.rs b/compiler-core/src/classfile/mod.rs index d3cf6cc2..8e32dc3e 100644 --- a/compiler-core/src/classfile/mod.rs +++ b/compiler-core/src/classfile/mod.rs @@ -5,6 +5,7 @@ pub mod encode; pub mod key; pub mod names; pub mod registry; +pub mod resources; pub mod summary; pub use ristretto_classfile::byte_reader::ByteReader; pub use ristretto_classfile::*; diff --git a/compiler-core/src/classfile/names.rs b/compiler-core/src/classfile/names.rs index 0a444fe8..17ae2bbe 100644 --- a/compiler-core/src/classfile/names.rs +++ b/compiler-core/src/classfile/names.rs @@ -5,6 +5,81 @@ pub const STRING_TAG: &str = "\u{1}rustc-jvm:"; pub const NAME_STRING: &str = "\u{1}rustc-jvm:name:"; pub const LITERAL_STRING: &str = "\u{1}rustc-jvm:literal:"; +/// Include the temporary linker tag in the JVM's modified UTF-8 limit. NUL +/// expands to two bytes and supplementary characters to a surrogate pair. +pub fn literal_fits(value: &str) -> bool { + let limit = u16::MAX as usize - LITERAL_STRING.len(); + value.len() <= limit / 2 + || value + .chars() + .map(|c| match c { + '\0' => 2, + c if c as u32 > 0xffff => 6, + c => c.len_utf8(), + }) + .sum::() + <= limit +} + +pub fn codec_owner(name: &str) -> bool { + name.rsplit('/') + .next() + .is_some_and(|name| name.starts_with("Codecs_")) +} + +pub fn codec_method_key(name: &str) -> Option<&str> { + let (operation, key) = name.split_once('$')?; + (matches!(operation, "e" | "w" | "d" | "a" | "b" | "s" | "c") + && key.len() == 16 + && key.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then_some(key) +} + +/// Parse packed memory codecs in pointer and view recipes. +/// Declaration, emission and linking share this dependency grammar. +pub fn codec_recipes(value: &str) -> impl Iterator { + value.lines().filter_map(|line| { + let line = line.strip_prefix(NAME_STRING).unwrap_or(line); + let mut parts = line.split('#'); + let (Some(owner), Some(key), Some(descriptor)) = (parts.next(), parts.next(), parts.next()) + else { + return None; + }; + let valid_size = parts.next().is_none_or(|size| size.parse::().is_ok()); + (valid_size + && parts.next().is_none() + && codec_owner(owner) + && key.len() == 16 + && key.bytes().all(|byte| byte.is_ascii_hexdigit()) + && !descriptor.is_empty()) + .then_some((line, key)) + }) +} + +#[cfg(test)] +mod codec_tests { + #[test] + fn nested_recipes_retain_every_exact_identity() { + let first = "pkg/Codecs_12#1234567890abcdef#Lpkg/Value;"; + let second = "pkg/Codecs_ab#abcdef1234567890#[B#8"; + let nested = format!( + "{}@slice-pointer\nview\n4\n{first}\n@raw-pointer\n{second}", + super::NAME_STRING + ); + assert_eq!( + super::codec_recipes(&nested) + .map(|(text, _)| text) + .collect::>(), + [first, second] + ); + assert!( + super::codec_recipes("pkg/Value#1234567890abcdef#I\npkg/Codecs_12#oops#I") + .next() + .is_none() + ); + } +} + pub fn is_crate_marker(bytes: &[u8]) -> bool { bytes.len() == CRATE_MARKER_LEN && bytes.starts_with(CRATE_MARKER.as_bytes()) diff --git a/compiler-core/src/classfile/resources.rs b/compiler-core/src/classfile/resources.rs new file mode 100644 index 00000000..668265df --- /dev/null +++ b/compiler-core/src/classfile/resources.rs @@ -0,0 +1,32 @@ +//! Binary constant payloads, independent of Rust allocation identity. +pub const BUNDLE_PREFIX: &str = "@resource:"; +pub const DIRECTORY: &str = "META-INF/rust-data/"; +pub const BLOCK_BYTES: usize = 64 * 1024; + +#[derive(Clone, Debug)] +pub struct Resource { + pub name: String, + pub bytes: Vec, +} + +impl Resource { + pub fn new(bytes: Vec) -> Self { + // Two byte hashes make names deterministic across hosts. + // The linker compares complete payloads and rejects hash collisions. + let mut a = 0xcbf29ce484222325u64; + let mut b = 0x84222325cbf29ce4u64; + for &byte in &bytes { + a = (a ^ u64::from(byte)).wrapping_mul(0x100000001b3); + b = (b ^ u64::from(byte)).wrapping_mul(0x9e3779b185ebca87); + } + Self { + name: format!("{DIRECTORY}{a:016x}{b:016x}"), + bytes, + } + } +} + +pub fn valid_name(name: &str) -> bool { + name.strip_prefix(DIRECTORY) + .is_some_and(|hash| hash.len() == 32 && hash.bytes().all(|b| b.is_ascii_hexdigit())) +} diff --git a/compiler-core/src/classfile/summary.rs b/compiler-core/src/classfile/summary.rs index 37757bd1..8e127c0e 100644 --- a/compiler-core/src/classfile/summary.rs +++ b/compiler-core/src/classfile/summary.rs @@ -1,12 +1,71 @@ -//! Read class identity and entry-point metadata without decoding method bodies. -//! Constant strings borrow the input; only the returned class name is allocated. +//! Borrowed dependency summaries for private holders and proven carrier helpers. +//! Ordinary classes retain their complete constant-pool dependencies. use super::JavaStr; use std::io; +mod code; +mod forwarder; #[derive(Debug, PartialEq, Eq)] pub struct Summary { pub name: String, pub has_main: bool, + pub private: bool, + pub opaque_reflection: bool, + pub method_demands: bool, + pub carrier: Option>, +} + +pub const PRIVATE_ATTRIBUTE: &str = "RustJvmPrivate"; +pub const CARRIER_ATTRIBUTE: &str = "RustJvmCarrier"; + +/// These namespaces have no observable Java method ownership. +/// Eligible classes must also have the private marker and no instance state. +pub fn method_owner(name: &str) -> bool { + name.contains("/mono/Mono_") + || name + .rsplit('/') + .next() + .is_some_and(|n| n.starts_with("Codecs_")) +} + +/// Static enum helpers do not participate in virtual dispatch. +/// Exact calls and runtime reflection names determine which helpers are live. +pub fn enum_helper(name: &str) -> bool { + matches!( + name, + "eq" | "variantIndex" + | "is_some" + | "is_none" + | "_unionDiscriminant" + | "_fromUnionDiscriminant" + | "_writeUnionStorage" + | "_readUnionStorage" + ) || name.starts_with("_rust_drop_fields$") +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct MethodKey<'a> { + pub owner: &'a [u8], + pub name: &'a [u8], + pub descriptor: &'a [u8], +} + +#[derive(Clone, Copy)] +pub enum Dependency<'a> { + /// Definition metadata is separate from dependency edges. + Definition { + public: bool, + forward: Option>, + /// Count definition bytes before constant-pool sharing. + /// Count each repeated fragment once and exclude unreachable methods. + bytes: usize, + }, + Class(&'a [u8]), + Text(&'a [u8]), + String(&'a [u8]), + Method(MethodKey<'a>), + /// A field or non-private method name is part of a nominal Java surface. + FixedMemberName(&'a [u8]), } #[derive(Clone, Copy)] @@ -14,16 +73,26 @@ enum Constant<'a> { Other, Utf8(&'a [u8]), Class(u16), + Text(u16), + String(u16), + Member(u16, u16, bool), + NameAndType(u16, u16), + Handle(u16), + Dynamic(u16, u16), +} +struct Method<'a> { + flags: u16, + name: &'a [u8], + descriptor: &'a [u8], + code: Option<&'a [u8]>, + supported: bool, } - struct Reader<'a> { bytes: &'a [u8], } - fn invalid() -> io::Error { io::Error::new(io::ErrorKind::InvalidData, "invalid JVM class metadata") } - impl<'a> Reader<'a> { fn take(&mut self, count: usize) -> io::Result<&'a [u8]> { let (head, tail) = self.bytes.split_at_checked(count).ok_or_else(invalid)?; @@ -39,29 +108,118 @@ impl<'a> Reader<'a> { fn u32(&mut self) -> io::Result { Ok(u32::from_be_bytes(self.take(4)?.try_into().unwrap())) } - fn attributes(&mut self) -> io::Result<()> { - for _ in 0..self.u16()? { - self.u16()?; - let count = usize::try_from(self.u32()?).map_err(|_| invalid())?; - self.take(count)?; +} +struct Pool<'a> { + constants: Vec>, + bootstrap: Vec>, +} +impl<'a> Pool<'a> { + fn utf8(&self, index: u16) -> io::Result<&'a [u8]> { + match self.constants.get(index as usize) { + Some(Constant::Utf8(bytes)) => Ok(bytes), + _ => Err(invalid()), + } + } + fn class(&self, index: u16) -> io::Result<&'a [u8]> { + match self.constants.get(index as usize) { + Some(Constant::Class(index)) => self.utf8(*index), + _ => Err(invalid()), + } + } + fn member(&self, owner: u16, index: u16) -> io::Result> { + let Some(Constant::NameAndType(name, descriptor)) = self.constants.get(index as usize) + else { + return Err(invalid()); + }; + Ok(MethodKey { + owner: self.class(owner)?, + name: self.utf8(*name)?, + descriptor: self.utf8(*descriptor)?, + }) + } + fn visit( + &self, + index: u16, + stamp: u32, + seen: &mut [u32], + visit: &mut impl FnMut(Dependency<'a>), + ) -> io::Result<()> { + let entry = seen.get_mut(index as usize).ok_or_else(invalid)?; + if *entry == stamp { + return Ok(()); + } + *entry = stamp; + match self.constants.get(index as usize).ok_or_else(invalid)? { + Constant::Class(index) => visit(Dependency::Class(self.utf8(*index)?)), + Constant::Utf8(bytes) => visit(Dependency::Text(bytes)), + Constant::Text(index) => visit(Dependency::Text(self.utf8(*index)?)), + Constant::String(index) => visit(Dependency::String(self.utf8(*index)?)), + Constant::Member(owner, member, method) => { + let key = self.member(*owner, *member)?; + visit(Dependency::Class(key.owner)); + visit(Dependency::Text(key.descriptor)); + if *method { + visit(Dependency::Method(key)); + } else { + visit(Dependency::FixedMemberName(key.name)); + } + } + Constant::NameAndType(_, descriptor) => { + visit(Dependency::Text(self.utf8(*descriptor)?)) + } + Constant::Handle(index) => self.visit(*index, stamp, seen, visit)?, + Constant::Dynamic(bootstrap, signature) => { + if let Some(Constant::NameAndType(name, _)) = + self.constants.get(*signature as usize) + { + visit(Dependency::FixedMemberName(self.utf8(*name)?)); + } + self.visit(*signature, stamp, seen, visit)?; + for &index in self + .bootstrap + .get(*bootstrap as usize) + .ok_or_else(invalid)? + { + self.visit(index, stamp, seen, visit)?; + } + } + Constant::Other => {} } Ok(()) } } pub fn read(bytes: &[u8]) -> io::Result { + read_dependencies(bytes, |_| {}) +} +pub fn read_dependencies( + bytes: &[u8], + mut visit: impl FnMut(Dependency<'_>), +) -> io::Result { + read_demands(bytes, |_, dependency| visit(dependency)) +} + +/// Limit method scopes to private, stateless compiler namespaces. +/// Keep other fragments intact. Borrowed data remains local to this call. +pub fn read_demands<'a>( + bytes: &'a [u8], + mut visit: impl FnMut(Option>, Dependency<'a>), +) -> io::Result { let mut r = Reader { bytes }; if r.take(4)? != b"\xca\xfe\xba\xbe" { return Err(invalid()); } - r.take(4)?; // minor/major version - let count = usize::from(r.u16()?); - let mut constants = vec![Constant::Other; count]; + r.take(4)?; + let count = r.u16()? as usize; + let mut pool = Pool { + constants: vec![Constant::Other; count], + bootstrap: Vec::new(), + }; let mut index = 1; while index < count { - constants[index] = match r.u8()? { + pool.constants[index] = match r.u8()? { 1 => { - let len = usize::from(r.u16()?); + let len = r.u16()? as usize; Constant::Utf8(r.take(len)?) } 7 => Constant::Class(r.u16()?), @@ -77,52 +235,254 @@ pub fn read(bytes: &[u8]) -> io::Result { } Constant::Other } - 8 | 16 | 19 | 20 => { + 8 => Constant::String(r.u16()?), + 16 => Constant::Text(r.u16()?), + 19 | 20 => { r.take(2)?; Constant::Other } - 9 | 10 | 11 | 12 | 17 | 18 => { - r.take(4)?; - Constant::Other - } + tag @ 9..=11 => Constant::Member(r.u16()?, r.u16()?, tag != 9), + 12 => Constant::NameAndType(r.u16()?, r.u16()?), + 17 | 18 => Constant::Dynamic(r.u16()?, r.u16()?), 15 => { - r.take(3)?; - Constant::Other + r.u8()?; + Constant::Handle(r.u16()?) } _ => return Err(invalid()), }; index += 1; } - let utf8 = |index: u16| match constants.get(usize::from(index)) { - Some(Constant::Utf8(bytes)) => Ok(*bytes), - _ => Err(invalid()), - }; - r.u16()?; // access flags - let Some(Constant::Class(name_index)) = constants.get(usize::from(r.u16()?)) else { - return Err(invalid()); - }; - let name = JavaStr::from_mutf8(utf8(*name_index)?) + let flags = r.u16()?; + let owner = pool.class(r.u16()?)?; + let name = JavaStr::from_mutf8(owner) .map_err(|_| invalid())? .to_rust_string(); - r.u16()?; // superclass - let interface_count = usize::from(r.u16()?); - r.take(interface_count * 2)?; - for _ in 0..r.u16()? { - r.take(6)?; // flags, name, descriptor - r.attributes()?; + let superclass = r.u16()?; + let interfaces = (0..r.u16()?) + .map(|_| r.u16()) + .collect::>>()?; + let fields = r.u16()?; + let mut field_descriptors = Vec::with_capacity(fields as usize); + let mut plain_fields = true; + for _ in 0..fields { + plain_fields &= r.u16()? & 0x0008 == 0; + visit(None, Dependency::FixedMemberName(pool.utf8(r.u16()?)?)); + field_descriptors.push(pool.utf8(r.u16()?)?); + let attributes = r.u16()?; + plain_fields &= attributes == 0; + for _ in 0..attributes { + r.u16()?; + let length = r.u32()? as usize; + r.take(length)?; + } } + let mut methods = Vec::new(); let mut has_main = false; + let mut opaque_reflection = false; for _ in 0..r.u16()? { let flags = r.u16()?; - let name = utf8(r.u16()?)?; - let descriptor = utf8(r.u16()?)?; + opaque_reflection |= flags & 0x0100 != 0; + let name = pool.utf8(r.u16()?)?; + let descriptor = pool.utf8(r.u16()?)?; has_main |= flags & 0x0009 == 0x0009 && name == b"main" && descriptor == b"([Ljava/lang/String;)V"; - r.attributes()?; + let mut method = Method { + flags, + name, + descriptor, + code: None, + supported: true, + }; + for _ in 0..r.u16()? { + let attribute = pool.utf8(r.u16()?)?; + let len = r.u32()? as usize; + let data = r.take(len)?; + match attribute { + b"Code" => method.code = Some(data), + b"MethodParameters" => {} + _ => method.supported = false, + } + } + methods.push(method); + } + let mut private = false; + let mut carrier = None; + let mut supported = true; + let mut metadata_classes = Vec::new(); + for _ in 0..r.u16()? { + let attribute = pool.utf8(r.u16()?)?; + let len = r.u32()? as usize; + let payload = r.take(len)?; + match attribute { + b"RustJvmPrivate" if payload.is_empty() => private = true, + b"RustJvmCarrier" => carrier = Some(payload.to_vec()), + b"SourceFile" => {} + b"InnerClasses" => { + let mut nested = Reader { bytes: payload }; + for _ in 0..nested.u16()? { + for _ in 0..2 { + let index = nested.u16()?; + if index != 0 { + metadata_classes.push(pool.class(index)?); + } + } + nested.take(4)?; // simple name and access flags + } + if !nested.bytes.is_empty() { + return Err(invalid()); + } + } + b"BootstrapMethods" => { + let mut b = Reader { bytes: payload }; + for _ in 0..b.u16()? { + let handle = b.u16()?; + let mut arguments = vec![handle]; + for _ in 0..b.u16()? { + arguments.push(b.u16()?); + } + pool.bootstrap.push(arguments); + } + if !b.bytes.is_empty() { + return Err(invalid()); + } + } + _ => supported = false, + } } - r.attributes()?; if !r.bytes.is_empty() { return Err(invalid()); } - Ok(Summary { name, has_main }) + for constant in &pool.constants { + let Constant::Member(owner, member, true) = *constant else { + continue; + }; + let key = pool.member(owner, member)?; + // A symbolic owner may be a ClassLoader subclass, so do not require + // the exact platform owner for class-loading operations. + opaque_reflection |= matches!( + key.name, + b"forName" + | b"loadClass" + | b"findClass" + | b"defineClass" + | b"getResource" + | b"getResources" + | b"getResourceAsStream" + ) || (matches!( + key.owner, + b"java/lang/Class" | b"java/lang/invoke/MethodHandles$Lookup" + ) && matches!( + key.name, + b"findStatic" + | b"findVirtual" + | b"findConstructor" + | b"findGetter" + | b"findSetter" + | b"findStaticGetter" + | b"findStaticSetter" + | b"getMethod" + | b"getDeclaredMethod" + | b"getField" + | b"getDeclaredField" + | b"getConstructor" + | b"getDeclaredConstructor" + )); + } + let static_owner = method_owner(&name) + && flags & 0x0200 == 0 + && interfaces.is_empty() + && methods + .iter() + .all(|m| m.flags & 0x0008 != 0 && m.flags & 0x0520 == 0 && m.name != b""); + let private_interface = flags & 0x0200 != 0 && !method_owner(&name); + let private_carrier = carrier.is_some() && plain_fields; + let method_demands = private + && supported + && (((static_owner || private_interface) && fields == 0) || private_carrier) + && superclass != 0 + && pool.class(superclass)? == b"java/lang/Object" + && methods.iter().all(|m| m.supported); + let mut seen = vec![0; count]; + if !method_demands || !static_owner { + for method in &methods { + visit(None, Dependency::FixedMemberName(method.name)); + } + } + if method_demands { + visit(None, Dependency::Class(b"java/lang/Object")); + for name in metadata_classes { + visit(None, Dependency::Class(name)); + } + for &interface in &interfaces { + visit(None, Dependency::Class(pool.class(interface)?)); + } + for descriptor in field_descriptors { + visit(None, Dependency::Text(descriptor)); + } + for (index, method) in methods.iter().enumerate() { + // Pure carrier recipes prove this final equality helper. + // Its virtual calls name the exact owner. Copy and interface dispatch + // still require the containing class. + let carrier_eq = private_carrier + && method.flags & 0x0010 != 0 + && method.name == b"eq" + && method + .descriptor + .strip_prefix(b"(L") + .and_then(|s| s.strip_suffix(b";)Z")) + == Some(owner); + let scope = ((method.flags & 0x0008 != 0 + && method.flags & 0x0520 == 0 + && method.name != b"") + || carrier_eq) + .then_some(MethodKey { + owner, + name: method.name, + descriptor: method.descriptor, + }); + if scope.is_some() { + let forward = (static_owner && method.flags & 0x0001 != 0) + .then(|| { + method + .code + .and_then(|bytes| forwarder::target(bytes, method.descriptor, &pool)) + }) + .flatten(); + visit( + scope, + Dependency::Definition { + public: method.flags & 0x0001 != 0, + forward, + bytes: method.code.map_or(0, <[u8]>::len) + + method.name.len() + + method.descriptor.len() + + 32, + }, + ); + } + visit(scope, Dependency::Text(method.descriptor)); // defines even an empty body + let mut dependency = |d| visit(scope, d); + if let Some(bytes) = method.code { + let stamp = index as u32 + 1; + code::constants(bytes, |i| pool.visit(i, stamp, &mut seen, &mut dependency))?; + code::stack_maps( + bytes, + |i| pool.utf8(i), + |i| pool.visit(i, stamp, &mut seen, &mut dependency), + )?; + } + } + } else { + for index in 1..count { + pool.visit(index as u16, 1, &mut seen, &mut |d| visit(None, d))?; + } + } + Ok(Summary { + name, + has_main, + private, + opaque_reflection, + method_demands, + carrier: private.then_some(carrier).flatten(), + }) } diff --git a/compiler-core/src/classfile/summary/code.rs b/compiler-core/src/classfile/summary/code.rs new file mode 100644 index 00000000..4c038118 --- /dev/null +++ b/compiler-core/src/classfile/summary/code.rs @@ -0,0 +1,112 @@ +//! Visit constant indexes without retaining an instruction or stack-map graph. +use super::*; +use crate::classfile::{ByteReader, attributes::Instruction}; + +pub(super) fn constants( + bytes: &[u8], + mut visit: impl FnMut(u16) -> io::Result<()>, +) -> io::Result<()> { + let mut r = Reader { bytes }; + r.take(4)?; // stack/local bounds + let length = r.u32()? as usize; + let mut code = ByteReader::new(r.take(length)?); + while code.remaining() != 0 { + use Instruction::*; + let instruction = Instruction::from_bytes(&mut code).map_err(|_| invalid())?; + let index = match instruction { + Ldc(i) => u16::from(i), + Ldc_w(i) + | Ldc2_w(i) + | Getstatic(i) + | Putstatic(i) + | Getfield(i) + | Putfield(i) + | Invokevirtual(i) + | Invokespecial(i) + | Invokestatic(i) + | Invokeinterface(i, _) + | Invokedynamic(i) + | New(i) + | Anewarray(i) + | Checkcast(i) + | Instanceof(i) + | Multianewarray(i, _) => i, + _ => continue, + }; + visit(index)?; + } + for _ in 0..r.u16()? { + r.take(6)?; + let catch = r.u16()?; + if catch != 0 { + visit(catch)?; + } + } + Ok(()) +} + +pub(super) fn stack_maps<'a>( + bytes: &[u8], + utf8: impl Fn(u16) -> io::Result<&'a [u8]>, + mut visit: impl FnMut(u16) -> io::Result<()>, +) -> io::Result<()> { + let mut r = Reader { bytes }; + r.take(4)?; + let length = r.u32()? as usize; + r.take(length)?; + let catches = r.u16()? as usize; + r.take(catches * 8)?; + for _ in 0..r.u16()? { + let name = utf8(r.u16()?)?; + let length = r.u32()? as usize; + let data = r.take(length)?; + if name != b"StackMapTable" { + continue; + } + let mut frames = Reader { bytes: data }; + fn ty(r: &mut Reader<'_>, visit: &mut impl FnMut(u16) -> io::Result<()>) -> io::Result<()> { + match r.u8()? { + 0..=6 => {} + 7 => visit(r.u16()?)?, + 8 => { + r.u16()?; + } + _ => return Err(invalid()), + } + Ok(()) + } + for _ in 0..frames.u16()? { + match frames.u8()? { + 0..=63 => {} + 64..=127 => ty(&mut frames, &mut visit)?, + 247 => { + frames.u16()?; + ty(&mut frames, &mut visit)?; + } + 248..=251 => { + frames.u16()?; + } + frame @ 252..=254 => { + frames.u16()?; + for _ in 0..frame - 251 { + ty(&mut frames, &mut visit)?; + } + } + 255 => { + frames.u16()?; + for _ in 0..frames.u16()? { + ty(&mut frames, &mut visit)?; + } + for _ in 0..frames.u16()? { + ty(&mut frames, &mut visit)?; + } + } + _ => return Err(invalid()), + } + } + if !frames.bytes.is_empty() { + return Err(invalid()); + } + } + Ok(()) +} diff --git a/compiler-core/src/classfile/summary/forwarder.rs b/compiler-core/src/classfile/summary/forwarder.rs new file mode 100644 index 00000000..47a9355e --- /dev/null +++ b/compiler-core/src/classfile/summary/forwarder.rs @@ -0,0 +1,216 @@ +//! Prove exact static forwarding without a bytecode graph. +//! Permit local spills. Reject casts, branches, handlers, effects and reordered +//! arguments. Require identical caller and target descriptors. +use super::*; +use crate::classfile::{ByteReader, attributes::Instruction}; + +#[derive(Clone, Copy, PartialEq, Eq)] +struct Value { + source: u16, + kind: u8, +} + +fn signature(bytes: &[u8]) -> Option<(Vec<(usize, Value)>, u8)> { + let mut cursor = bytes.strip_prefix(b"(")?; + let mut params = Vec::new(); + let mut slot = 0; + while !cursor.starts_with(b")") { + let (&first, rest) = cursor.split_first()?; + cursor = rest; + let kind = match first { + b'Z' | b'B' | b'C' | b'S' | b'I' => 0, + b'J' => 1, + b'F' => 2, + b'D' => 3, + b'L' | b'[' => { + let mut element = first; + while element == b'[' { + let (&next, rest) = cursor.split_first()?; + element = next; + cursor = rest; + } + if element == b'L' { + let end = cursor.iter().position(|&b| b == b';')?; + cursor = &cursor[end + 1..]; + } else if !b"ZBCSIJFD".contains(&element) { + return None; + } + 4 + } + _ => return None, + }; + params.push(( + slot, + Value { + source: params.len() as u16, + kind, + }, + )); + slot += if kind == 1 || kind == 3 { 2 } else { 1 }; + if slot > 64 { + return None; + } + } + let result = match cursor.get(1)? { + b'Z' | b'B' | b'C' | b'S' | b'I' => 0, + b'J' => 1, + b'F' => 2, + b'D' => 3, + b'L' | b'[' => 4, + b'V' => 5, + _ => return None, + }; + Some((params, result)) +} + +pub(super) fn target<'a>( + bytes: &[u8], + descriptor: &[u8], + pool: &Pool<'a>, +) -> Option> { + let (params, result) = signature(descriptor)?; + let mut raw = Reader { bytes }; + raw.take(4).ok()?; + let length = raw.u32().ok()? as usize; + if length > 512 { + return None; + } + let bytes = raw.take(length).ok()?; + if raw.u16().ok()? != 0 { + return None; + } + let mut code = ByteReader::new(bytes); + let mut locals = [None; 64]; + for &(slot, value) in ¶ms { + locals[slot] = Some(value); + } + let mut stack = Vec::with_capacity(params.len() + 1); + let mut target = None; + while code.remaining() != 0 { + use Instruction::*; + let instruction = Instruction::from_bytes(&mut code).ok()?; + let load = match instruction { + Iload(i) => Some((i as usize, 0)), + Lload(i) => Some((i as usize, 1)), + Fload(i) => Some((i as usize, 2)), + Dload(i) => Some((i as usize, 3)), + Aload(i) => Some((i as usize, 4)), + Iload_0 => Some((0, 0)), + Iload_1 => Some((1, 0)), + Iload_2 => Some((2, 0)), + Iload_3 => Some((3, 0)), + Lload_0 => Some((0, 1)), + Lload_1 => Some((1, 1)), + Lload_2 => Some((2, 1)), + Lload_3 => Some((3, 1)), + Fload_0 => Some((0, 2)), + Fload_1 => Some((1, 2)), + Fload_2 => Some((2, 2)), + Fload_3 => Some((3, 2)), + Dload_0 => Some((0, 3)), + Dload_1 => Some((1, 3)), + Dload_2 => Some((2, 3)), + Dload_3 => Some((3, 3)), + Aload_0 => Some((0, 4)), + Aload_1 => Some((1, 4)), + Aload_2 => Some((2, 4)), + Aload_3 => Some((3, 4)), + _ => None, + }; + if let Some((slot, kind)) = load { + let value = (*locals.get(slot)?)?; + if value.kind != kind { + return None; + } + stack.push(value); + continue; + } + let store = match instruction { + Istore(i) => Some((i as usize, 0)), + Lstore(i) => Some((i as usize, 1)), + Fstore(i) => Some((i as usize, 2)), + Dstore(i) => Some((i as usize, 3)), + Astore(i) => Some((i as usize, 4)), + Istore_0 => Some((0, 0)), + Istore_1 => Some((1, 0)), + Istore_2 => Some((2, 0)), + Istore_3 => Some((3, 0)), + Lstore_0 => Some((0, 1)), + Lstore_1 => Some((1, 1)), + Lstore_2 => Some((2, 1)), + Lstore_3 => Some((3, 1)), + Fstore_0 => Some((0, 2)), + Fstore_1 => Some((1, 2)), + Fstore_2 => Some((2, 2)), + Fstore_3 => Some((3, 2)), + Dstore_0 => Some((0, 3)), + Dstore_1 => Some((1, 3)), + Dstore_2 => Some((2, 3)), + Dstore_3 => Some((3, 3)), + Astore_0 => Some((0, 4)), + Astore_1 => Some((1, 4)), + Astore_2 => Some((2, 4)), + Astore_3 => Some((3, 4)), + _ => None, + }; + if let Some((slot, kind)) = store { + let value = stack.pop()?; + if value.kind != kind { + return None; + } + *locals.get_mut(slot)? = Some(value); + continue; + } + match instruction { + Nop => {} + Goto(target) if usize::from(target) == code.position() => {} + Goto_w(target) if usize::try_from(target).ok() == Some(code.position()) => {} + Invokestatic(index) if target.is_none() => { + if !stack.iter().eq(params.iter().map(|(_, value)| value)) { + return None; + } + let Constant::Member(owner, member, true) = *pool.constants.get(index as usize)? + else { + return None; + }; + let callee = pool.member(owner, member).ok()?; + if callee.descriptor != descriptor { + return None; + } + target = Some(callee); + stack.clear(); + if result != 5 { + stack.push(Value { + source: u16::MAX, + kind: result, + }); + } + } + Ireturn | Lreturn | Freturn | Dreturn | Areturn | Return => { + let kind = match instruction { + Ireturn => 0, + Lreturn => 1, + Freturn => 2, + Dreturn => 3, + Areturn => 4, + _ => 5, + }; + if kind != result || code.remaining() != 0 { + return None; + } + if result != 5 + && stack.pop()? + != (Value { + source: u16::MAX, + kind: result, + }) + { + return None; + } + return if stack.is_empty() { target } else { None }; + } + _ => return None, + } + } + None +} diff --git a/compiler-core/src/ir/body.rs b/compiler-core/src/ir/body.rs index 22396348..86044610 100644 --- a/compiler-core/src/ir/body.rs +++ b/compiler-core/src/ir/body.rs @@ -25,6 +25,15 @@ pub enum CallKind { Indirect, } +/// Storage operations for the JVM platform allocator. +/// Custom Rust allocators retain their calls and effects. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum HeapOp { + Allocate, + Reallocate, + Deallocate, +} + /// A body-local call target. Signatures describe value-bearing JVM arguments; /// source-language unit arguments never enter the operand pool. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -42,13 +51,12 @@ pub struct FieldRef { pub name: String, pub ty: TypeId, pub is_static: bool, - /// Generated Rust pointer fields store a base plus two displacement fields. - pub relative_pointer: bool, } /// A field view preserves Rust byte layout and the runtime allocation identity. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct PointerProjection { + pub parent: Option, pub field: MemberId, pub offset: u64, pub size: u64, @@ -73,7 +81,7 @@ impl StorageSlot { }; let size = match scalar { Bool | I8 | U8 => 1, - I16 | U16 => 2, + I16 | U16 | F16 => 2, I32 | U32 | F32 => 4, I64 | U64 | F64 => 8, _ => return None, @@ -116,11 +124,40 @@ pub enum Op { Opaque(ValueId), Neg(ValueId), Cast(ValueId), + /// Refine a reference type after representation analysis proves the cast. + /// Emit the JVM cast only if a consumer needs the result. + Refine(ValueId), /// Representation adaptation at a JVM ABI boundary (boxing, views, casts). Adapt(ValueId), /// Same physical JVM carrier with a different semantic type annotation. Reinterpret(ValueId), NewArray(ValueId), + /// Primitive repeat initialization, retaining the scalar value unboxed. + ArrayFill { + array: ValueId, + value: ValueId, + }, + /// Fresh, naturally aligned scalar storage with exact initial contents. + /// Lower to a primitive array only if its address survives SSA promotion. + ScalarCell(ValueId), + /// Allocate [size, alignment], reallocate [root, offset, old size, + /// alignment, new size], or release [root, offset]. Return storage roots. + /// Keep address metadata separate. + Heap { + operation: HeapOp, + args: List, + }, + /// Select typed storage for a scalar field. + /// Use a boundary carrier if the root requires general memory access. + ProjectRoot { + address: List, + projection: ProjectionId, + }, + ProjectOffset { + root: ValueId, + base: ValueId, + offset: ValueId, + }, FunctionPointer { signature: MethodId, target: MethodId, @@ -132,6 +169,12 @@ pub enum Op { }, AddressOfSlot(SlotId), Load(ValueId), + /// An owned aggregate snapshot, distinct from a live decoded memory view. + LoadCopy(ValueId), + /// An owned Rust value snapshot, independent of subsequent source writes. + CopyValue(ValueId), + /// Complete mutations to a live decoded view using the original load owner. + Commit(ValueId), Store { pointer: ValueId, value: ValueId, @@ -145,11 +188,44 @@ pub enum Op { base: ValueId, projection: ProjectionId, }, + /// Copy an owned field without constructing an intermediate address. + /// Keep the read and snapshot together, including inside an Invoke. + LoadFieldCopy { + base: ValueId, + projection: ProjectionId, + }, StoreField { base: ValueId, projection: ProjectionId, value: ValueId, }, + LoadFieldPart { + base: ValueId, + projection: ProjectionId, + index: u8, + }, + /// Read a field through a storage root and displacement. + /// The index selects one physical component of a borrowed field. + LoadStorageField { + address: List, + projection: ProjectionId, + index: Option, + }, + LoadStorageFieldCopy { + address: List, + projection: ProjectionId, + }, + /// [root, displacement, value/components] for a projected field write. + StoreStorageField { + args: List, + projection: ProjectionId, + split: bool, + }, + StoreFieldParts { + base: ValueId, + projection: ProjectionId, + parts: List, + }, Offset { pointer: ValueId, offset: ValueId, @@ -178,24 +254,146 @@ pub enum Op { ArrayGet { array: ValueId, index: ValueId, + /// Compiler-owned ABI and outline scratch, inaccessible to Rust code. + native: bool, + }, + /// Read an independent value without creating a borrowed element view. + ArrayGetCopy { + array: ValueId, + index: ValueId, }, ArraySet { array: ValueId, index: ValueId, value: ValueId, + /// Same unaliased scratch-storage contract as ArrayGet::native. + native: bool, + }, + /// Slice access through [backing, start, index], without a view carrier. + ViewGet(List), + ViewGetCopy(List), + ViewSet { + parts: List, + value: ValueId, }, /// Logical Rust length, including zero-sized slices larger than JVM arrays. Length(ValueId), - /// Immutable fat-pointer carrier; data retains allocation identity and view. + /// Address with physical components [storage root, byte offset]. + AddressPack(List), + /// Scalar [payload, discriminant], boxed only at an opaque boundary. + TaggedPack(List), + TaggedPart { + value: ValueId, + index: u8, + }, + /// Exact Rust view layout belongs to the location, not its JVM carrier. + RetypeAddress { + pointer: ValueId, + size: u32, + codec: Option, + }, + /// A component location whose pointee layout is exact at this use. + TypedAddressPack { + parts: List, + size: u32, + codec: Option, + }, + /// An owned aggregate load with a statically known view layout. + LoadTypedCopy { + parts: List, + size: u32, + codec: Option, + }, + /// Parts are [root, displacement, optional JVM class name]. + /// Class and interface results request their type when no name is supplied. + /// Object and array results use getObject's null target. + /// No later consumer can commit the borrowed view's binding. + LoadTyped { + parts: List, + size: u32, + codec: Option, + }, + /// [root, displacement, value] with an exact source-language layout. + StoreTyped { + parts: List, + size: u32, + codec: Option, + }, + AddressEqual { + left: ValueId, + right: ValueId, + }, + LocationEqual(List), + /// Compare unsigned data addresses and return a signed comparison code. + AddressCompare { + left: ValueId, + right: ValueId, + }, + LocationCompare(List), + /// Discriminant of a valid null-niche pointer enum (None = 0, Some = 1). + AddressTag(ValueId), + LocationTag(List), + AddressPart { + address: ValueId, + index: u8, + }, + LoadAddress(List), + LoadAddressCopy(List), + /// [source root, source offset, destination root, destination offset, bytes]. + CopyStorage { + parts: List, + // Intern exact layouts instead of enlarging every instruction. + layouts: [TypeId; 2], + nonoverlapping: bool, + }, + StoreAddress { + parts: List, + value: ValueId, + }, + /// The authoritative primitive array backing an addressable scalar local. + SlotRoot(SlotId), View { data: ValueId, length: ValueId, }, + /// Boundary materialization from [backing, start, logical length]. + ViewPack(List), + /// Read one physical component without constructing an address. + ViewPart { + view: ValueId, + index: u8, + }, ViewData { view: ValueId, size: u32, codec: Option, }, + /// Extract an address from [backing, start, logical length]. + /// No view object is needed, including at memory boundaries. + ViewAddress { + parts: List, + size: u32, + codec: Option, + }, + /// Normalize a nonzero-sized slice backing to retain its exact element layout. + /// Keep the element displacement separate. + ViewRoot { + backing: ValueId, + size: u32, + codec: Option, + }, + /// Extract a view backing or element start from a scalar location. + /// Do not allocate a thin-pointer carrier. + AddressViewPart { + address: ValueId, + index: u8, + }, + TypedAddressViewPart { + parts: List, + size: u32, + codec: Option, + index: u8, + }, } impl Op { @@ -203,16 +401,32 @@ impl Op { /// scalar computations and literal loads do not. pub fn may_throw(self, body: &Body, types: &Types) -> bool { match self { - Self::Constant(id) => matches!(body.constants[id.index()], Constant::External { .. }), + Self::Constant(id) => matches!( + body.constants[id.index()], + Constant::External { pure: false, .. } + ), + // Native scratch is initialized and accessed within its declared bounds. + // Unused reads have no Rust-visible effect. + Self::ArrayGet { native: true, .. } => false, Self::Nop | Self::Exception + | Self::AddressTag(_) + | Self::LocationTag(_) | Self::Reinterpret(_) + | Self::TaggedPack(_) + | Self::TaggedPart { .. } + | Self::Refine(_) | Self::Not(_) | Self::Neg(_) | Self::Bit { .. } | Self::Overflow { .. } => false, // A typed field address has no source-language effects until used. - Self::Project { .. } => false, + Self::Project { .. } + | Self::ProjectRoot { .. } + | Self::ProjectOffset { .. } + | Self::RetypeAddress { size: 1.., .. } + | Self::ViewRoot { .. } + | Self::TypedAddressPack { .. } => false, Self::Binary { op: BinaryOp::Div | BinaryOp::Rem, left, @@ -229,32 +443,69 @@ impl Op { } pub fn visit_uses(self, args: &[ValueId], mut visit: impl FnMut(ValueId)) { match self { - Self::Binary { left, right, .. } => { + Self::ProjectRoot { address, .. } => { + for &value in &args[address.range()] { + visit(value); + } + } + Self::ProjectOffset { root, base, offset } => { + visit(root); + visit(base); + visit(offset); + } + Self::AddressViewPart { address, .. } => visit(address), + Self::ViewRoot { backing, .. } => visit(backing), + Self::RetypeAddress { pointer, .. } => visit(pointer), + Self::Binary { left, right, .. } + | Self::AddressEqual { left, right } + | Self::AddressCompare { left, right } => { visit(left); visit(right); } - Self::Not(v) + Self::TaggedPart { value: v, .. } + | Self::AddressTag(v) + | Self::Not(v) | Self::Opaque(v) | Self::Bit { value: v, .. } | Self::Neg(v) | Self::Cast(v) + | Self::Refine(v) | Self::Adapt(v) | Self::Reinterpret(v) | Self::NewArray(v) + | Self::ScalarCell(v) | Self::ArrayLength(v) | Self::Load(v) + | Self::CopyValue(v) + | Self::LoadCopy(v) + | Self::Commit(v) | Self::Length(v) => visit(v), Self::StoreSlot { value, .. } | Self::SetStatic { value, .. } => visit(value), - Self::Store { pointer, value } => { + Self::Store { pointer, value } + | Self::ArrayFill { + array: pointer, + value, + } => { visit(pointer); visit(value); } - Self::Project { base, .. } | Self::LoadField { base, .. } => visit(base), + Self::Project { base, .. } + | Self::LoadField { base, .. } + | Self::LoadFieldCopy { base, .. } + | Self::LoadFieldPart { base, .. } => visit(base), + Self::StoreFieldParts { base, parts, .. } => { + visit(base); + for &value in &args[parts.range()] { + visit(value); + } + } Self::StoreField { base, value, .. } => { visit(base); visit(value); } - Self::ViewData { view, .. } => visit(view), + Self::ViewData { view, .. } + | Self::ViewPart { view, .. } + | Self::AddressPart { address: view, .. } => visit(view), Self::View { data, length } => { visit(data); visit(length); @@ -265,17 +516,45 @@ impl Op { visit(pointer); visit(offset); } - Self::Call { args: list, .. } | Self::Overflow { args: list, .. } => { + Self::Call { args: list, .. } + | Self::Heap { args: list, .. } + | Self::StoreStorageField { args: list, .. } + | Self::LoadStorageField { address: list, .. } + | Self::LoadStorageFieldCopy { address: list, .. } + | Self::Overflow { args: list, .. } + | Self::TaggedPack(list) + | Self::ViewPack(list) + | Self::ViewGet(list) + | Self::ViewGetCopy(list) + | Self::AddressPack(list) + | Self::LoadAddress(list) + | Self::LoadAddressCopy(list) + | Self::CopyStorage { parts: list, .. } + | Self::LoadTypedCopy { parts: list, .. } + | Self::LoadTyped { parts: list, .. } + | Self::StoreTyped { parts: list, .. } + | Self::TypedAddressPack { parts: list, .. } + | Self::TypedAddressViewPart { parts: list, .. } + | Self::LocationTag(list) + | Self::LocationEqual(list) + | Self::LocationCompare(list) + | Self::ViewAddress { parts: list, .. } => { for &arg in &args[list.range()] { visit(arg); } } + Self::StoreAddress { parts, value } | Self::ViewSet { parts, value } => { + for &part in &args[parts.range()] { + visit(part); + } + visit(value); + } Self::GetField { object, .. } => visit(object), Self::SetField { object, value, .. } => { visit(object); visit(value); } - Self::ArrayGet { array, index } => { + Self::ArrayGet { array, index, .. } | Self::ArrayGetCopy { array, index } => { visit(array); visit(index); } @@ -283,6 +562,7 @@ impl Op { array, index, value, + .. } => { visit(array); visit(index); @@ -293,6 +573,7 @@ impl Op { | Self::Exception | Self::LoadSlot(_) | Self::AddressOfSlot(_) + | Self::SlotRoot(_) | Self::GetStatic(_) | Self::FunctionPointer { .. } => {} } @@ -318,6 +599,8 @@ pub enum Constant { External { index: u32, ty: TypeId, + /// A literal with no initialization or runtime helper effects. + pure: bool, }, } @@ -447,7 +730,7 @@ impl Body { } value } - pub(super) fn resolve_mut(&mut self, mut value: ValueId) -> ValueId { + pub(crate) fn resolve_mut(&mut self, mut value: ValueId) -> ValueId { let root = self.resolve(value); while let ValueDef::Alias(next) = self.values[value.index()].def { self.values[value.index()].def = ValueDef::Alias(root); diff --git a/compiler-core/src/ir/fold.rs b/compiler-core/src/ir/fold.rs index 9599a89c..3ecd7b58 100644 --- a/compiler-core/src/ir/fold.rs +++ b/compiler-core/src/ir/fold.rs @@ -1,41 +1,45 @@ use super::*; use crate::scalar::{BinaryFold, Scalar, fold_binary}; -impl Builder<'_> { - pub(super) fn scalar_value(&self, value: ValueId) -> Option { - let value = self.body.resolve(value); - let ValueDef::Inst(inst) = self.body.values[value.index()].def else { +pub(crate) enum Folded { + Value(ValueId), + Constant(Scalar), +} + +impl Body { + pub(crate) fn scalar_value(&self, value: ValueId) -> Option { + let value = self.resolve(value); + let ValueDef::Inst(inst) = self.values[value.index()].def else { return None; }; - let Op::Constant(constant) = self.body.instructions[inst.index()].op else { + let Op::Constant(constant) = self.instructions[inst.index()].op else { return None; }; - let Constant::Scalar(scalar) = self.body.constants[constant.index()] else { + let Constant::Scalar(scalar) = self.constants[constant.index()] else { return None; }; Some(scalar) } - pub(super) fn fold(&mut self, op: Op, result: TypeId) -> Option { + pub(crate) fn fold(&self, types: &Types, op: Op, result: TypeId) -> Option { if let Op::Length(view) = op { - let view = self.body.resolve(view); - if let ValueDef::Inst(inst) = self.body.values[view.index()].def - && let Op::View { length, .. } = self.body.instructions[inst.index()].op - && self.body.value_type(length) == result + let view = self.resolve(view); + if let ValueDef::Inst(inst) = self.values[view.index()].def + && let Op::View { length, .. } = self.instructions[inst.index()].op + && self.value_type(length) == result { - return Some(length); + return Some(Folded::Value(length)); } } - let Some(Type::Scalar(result_ty)) = self.types.get(result) else { + let Some(Type::Scalar(result_ty)) = types.get(result) else { return None; }; let constant = match op { Op::Binary { op, left, right } => { - let Some(Type::Scalar(left_ty)) = self.types.get(self.body.value_type(left)) else { + let Some(Type::Scalar(left_ty)) = types.get(self.value_type(left)) else { return None; }; - let Some(Type::Scalar(right_ty)) = self.types.get(self.body.value_type(right)) - else { + let Some(Type::Scalar(right_ty)) = types.get(self.value_type(right)) else { return None; }; let result = fold_binary( @@ -44,33 +48,57 @@ impl Builder<'_> { right_ty, self.scalar_value(left), self.scalar_value(right), - || self.body.resolve(left) == self.body.resolve(right), + || self.resolve(left) == self.resolve(right), )?; match result { - BinaryFold::Left if result_ty == left_ty => return Some(left), - BinaryFold::Right if result_ty == right_ty => return Some(right), + BinaryFold::Left if result_ty == left_ty => return Some(Folded::Value(left)), + BinaryFold::Right if result_ty == right_ty => { + return Some(Folded::Value(right)); + } BinaryFold::Constant(value) => value, _ => return None, } } + Op::Reinterpret(value) => { + // Full JVM integer-width signedness is only an annotation. + // Narrow int carriers can have different extension semantics. + let value = self.scalar_value(value)?; + let (width, _) = value.ty().integer()?; + if !matches!(width, 32 | 64) || result_ty.integer()?.0 != width { + return None; + } + value.cast(result_ty)? + } Op::Not(value) => self.scalar_value(value)?.not()?, Op::Bit { op, value } => self.scalar_value(value)?.bit(op)?, Op::Overflow { op, args } => { - let args = &self.body.args[args.range()]; + let args = &self.args[args.range()]; let a = self.scalar_value(args[0])?; let b = self.scalar_value(args[1])?; Scalar::boolean(a.overflows(op, b)?) } Op::Neg(value) => self.scalar_value(value)?.neg()?, Op::Cast(value) => { - if self.body.value_type(value) == result { - return Some(value); + if self.value_type(value) == result { + return Some(Folded::Value(value)); } self.scalar_value(value)?.cast(result_ty)? } _ => return None, }; - (constant.ty() == result_ty).then(|| self.constant(result, constant)) + (constant.ty() == result_ty).then_some(Folded::Constant(constant)) + } +} + +impl Builder<'_> { + pub(super) fn scalar_value(&self, value: ValueId) -> Option { + self.body.scalar_value(value) + } + pub(super) fn fold(&mut self, op: Op, result: TypeId) -> Option { + match self.body.fold(self.types, op, result)? { + Folded::Value(value) => Some(value), + Folded::Constant(value) => Some(self.constant(result, value)), + } } } diff --git a/compiler-core/src/ir/ids.rs b/compiler-core/src/ir/ids.rs index faeed842..5e88a514 100644 --- a/compiler-core/src/ir/ids.rs +++ b/compiler-core/src/ir/ids.rs @@ -23,6 +23,7 @@ ids!( EdgeId, VariableId, TypeId, + LayoutId, SymbolId, ConstId, SlotId, diff --git a/compiler-core/src/ir/mod.rs b/compiler-core/src/ir/mod.rs index 9194851a..725da86e 100644 --- a/compiler-core/src/ir/mod.rs +++ b/compiler-core/src/ir/mod.rs @@ -4,6 +4,7 @@ pub use debug::*; mod body; mod builder; mod fold; +pub(crate) use fold::Folded; mod ids; mod parameters; mod remap; @@ -22,3 +23,6 @@ mod tests; #[cfg(test)] mod storage_tests; + +mod shape; +pub use shape::ComponentShape; diff --git a/compiler-core/src/ir/remap.rs b/compiler-core/src/ir/remap.rs index 97afb8a3..9f7e7f65 100644 --- a/compiler-core/src/ir/remap.rs +++ b/compiler-core/src/ir/remap.rs @@ -14,6 +14,39 @@ impl Op { pub fn remap(self, map: &mut impl Remap) -> Self { use Op::*; match self { + AddressViewPart { address, index } => AddressViewPart { + address: map.value(address), + index, + }, + RetypeAddress { + pointer, + size, + codec, + } => RetypeAddress { + pointer: map.value(pointer), + size, + codec, + }, + TypedAddressPack { parts, size, codec } => TypedAddressPack { + parts: map.args(parts), + size, + codec, + }, + LoadTypedCopy { parts, size, codec } => LoadTypedCopy { + parts: map.args(parts), + size, + codec, + }, + LoadTyped { parts, size, codec } => LoadTyped { + parts: map.args(parts), + size, + codec, + }, + StoreTyped { parts, size, codec } => StoreTyped { + parts: map.args(parts), + size, + codec, + }, Nop => Nop, Constant(c) => Constant(map.constant(c)), Exception => Exception, @@ -34,12 +67,33 @@ impl Op { }, Opaque(v) => Opaque(map.value(v)), Cast(v) => Cast(map.value(v)), + Refine(v) => Refine(map.value(v)), Adapt(v) => Adapt(map.value(v)), Reinterpret(v) => Reinterpret(map.value(v)), NewArray(v) => NewArray(map.value(v)), + ScalarCell(v) => ScalarCell(map.value(v)), + Heap { operation, args } => Heap { + operation, + args: map.args(args), + }, + ProjectRoot { + address, + projection, + } => ProjectRoot { + address: map.args(address), + projection: map.projection(projection), + }, + ProjectOffset { root, base, offset } => ProjectOffset { + root: map.value(root), + base: map.value(base), + offset: map.value(offset), + }, ArrayLength(v) => ArrayLength(map.value(v)), Length(v) => Length(map.value(v)), Load(v) => Load(map.value(v)), + LoadCopy(v) => LoadCopy(map.value(v)), + CopyValue(v) => CopyValue(map.value(v)), + Commit(v) => Commit(map.value(v)), FunctionPointer { signature, target } => FunctionPointer { signature: map.method(signature), target: map.method(target), @@ -62,6 +116,53 @@ impl Op { base: map.value(base), projection: map.projection(projection), }, + LoadFieldCopy { base, projection } => LoadFieldCopy { + base: map.value(base), + projection: map.projection(projection), + }, + LoadStorageFieldCopy { + address, + projection, + } => LoadStorageFieldCopy { + address: map.args(address), + projection: map.projection(projection), + }, + LoadStorageField { + address, + projection, + index, + } => LoadStorageField { + address: map.args(address), + projection: map.projection(projection), + index, + }, + StoreStorageField { + args, + projection, + split, + } => StoreStorageField { + args: map.args(args), + projection: map.projection(projection), + split, + }, + LoadFieldPart { + base, + projection, + index, + } => LoadFieldPart { + base: map.value(base), + projection: map.projection(projection), + index, + }, + StoreFieldParts { + base, + projection, + parts, + } => StoreFieldParts { + base: map.value(base), + projection: map.projection(projection), + parts: map.args(parts), + }, StoreField { base, projection, @@ -105,7 +206,16 @@ impl Op { field: map.field(field), value: map.value(value), }, - ArrayGet { array, index } => ArrayGet { + ArrayGet { + array, + index, + native, + } => ArrayGet { + native, + array: map.value(array), + index: map.value(index), + }, + ArrayGetCopy { array, index } => ArrayGetCopy { array: map.value(array), index: map.value(index), }, @@ -113,7 +223,9 @@ impl Op { array, index, value, + native, } => ArraySet { + native, array: map.value(array), index: map.value(index), value: map.value(value), @@ -122,11 +234,89 @@ impl Op { data: map.value(data), length: map.value(length), }, + ViewPack(parts) => ViewPack(map.args(parts)), + TaggedPack(parts) => TaggedPack(map.args(parts)), + TaggedPart { value, index } => TaggedPart { + value: map.value(value), + index, + }, + ArrayFill { array, value } => ArrayFill { + array: map.value(array), + value: map.value(value), + }, + ViewGet(parts) => ViewGet(map.args(parts)), + ViewGetCopy(parts) => ViewGetCopy(map.args(parts)), + ViewSet { parts, value } => ViewSet { + parts: map.args(parts), + value: map.value(value), + }, + AddressPack(parts) => AddressPack(map.args(parts)), + AddressEqual { left, right } => AddressEqual { + left: map.value(left), + right: map.value(right), + }, + AddressCompare { left, right } => AddressCompare { + left: map.value(left), + right: map.value(right), + }, + AddressTag(value) => AddressTag(map.value(value)), + LocationTag(parts) => LocationTag(map.args(parts)), + LocationEqual(parts) => LocationEqual(map.args(parts)), + LocationCompare(parts) => LocationCompare(map.args(parts)), + LoadAddress(parts) => LoadAddress(map.args(parts)), + LoadAddressCopy(parts) => LoadAddressCopy(map.args(parts)), + CopyStorage { + parts, + layouts, + nonoverlapping, + } => CopyStorage { + parts: map.args(parts), + layouts, + nonoverlapping, + }, + StoreAddress { parts, value } => StoreAddress { + parts: map.args(parts), + value: map.value(value), + }, + AddressPart { address, index } => AddressPart { + address: map.value(address), + index, + }, + SlotRoot(slot) => SlotRoot(map.slot(slot)), + ViewPart { view, index } => ViewPart { + view: map.value(view), + index, + }, ViewData { view, size, codec } => ViewData { view: map.value(view), size, codec, }, + TypedAddressViewPart { + parts, + size, + codec, + index, + } => TypedAddressViewPart { + parts: map.args(parts), + size, + codec, + index, + }, + ViewAddress { parts, size, codec } => ViewAddress { + parts: map.args(parts), + size, + codec, + }, + ViewRoot { + backing, + size, + codec, + } => ViewRoot { + backing: map.value(backing), + size, + codec, + }, } } } diff --git a/compiler-core/src/ir/shape.rs b/compiler-core/src/ir/shape.rs new file mode 100644 index 00000000..aac33418 --- /dev/null +++ b/compiler-core/src/ir/shape.rs @@ -0,0 +1,114 @@ +//! Describe physical components while retaining semantic pointee types. +//! Emit a boundary carrier only when a consumer needs one object. +use super::*; +use crate::scalar::ScalarType; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ComponentShape { + View, + TaggedI64, + Address, + /// This root retains the allocation's runtime view layout. + /// A bare primitive array cannot replace it, unlike scalar address roots. + StorageAddress, +} + +impl ComponentShape { + pub fn view_carrier(types: &Types, ty: TypeId) -> bool { + match types.get(ty) { + Some(Type::Slice(_) | Type::Str) => true, + Some(Type::Class(name)) => matches!( + types.symbol_name(name), + Some("org/rustlang/runtime/SliceView" | "org/rustlang/runtime/Utf8View") + ), + _ => false, + } + } + /// Runtime helpers can erase the semantic pointee to a carrier class. + /// This annotation change does not require a carrier allocation. + pub fn accepts_annotation(self, types: &Types, ty: TypeId) -> bool { + if Self::of(types, ty) == Some(self) { + return true; + } + let Some(Type::Class(name)) = types.get(ty) else { + return false; + }; + match types.symbol_name(name) { + Some("java/lang/Object") => true, + Some("org/rustlang/runtime/Pointer") => self.is_address(), + Some("org/rustlang/runtime/SliceView" | "org/rustlang/runtime/Utf8View") => { + self == Self::View + } + _ => false, + } + } + pub fn len(self) -> usize { + match self { + Self::View => 3, + Self::TaggedI64 => 2, + Self::Address | Self::StorageAddress => 2, + } + } + pub fn of(types: &Types, ty: TypeId) -> Option { + match types.get(ty)? { + Type::Slice(_) | Type::Str => Some(Self::View), + Type::TaggedI64 => Some(Self::TaggedI64), + Type::Pointer(inner) if StorageSlot::scalar(inner, types).is_some() => { + Some(Self::Address) + } + Type::Pointer(_) => Some(Self::StorageAddress), + _ => None, + } + } + pub fn parts(self, types: &mut Types) -> impl ExactSizeIterator + use<> { + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let (parts, count) = match self { + Self::View => ( + [ + object, + types.scalar(ScalarType::I32), + types.scalar(ScalarType::U64), + ], + 3, + ), + Self::TaggedI64 => { + let long = types.scalar(ScalarType::I64); + ([long, long, object], 2) + } + Self::Address | Self::StorageAddress => { + ([object, types.scalar(ScalarType::I64), object], 2) + } + }; + parts.into_iter().take(count) + } + pub fn slots(self) -> usize { + match self { + Self::View | Self::TaggedI64 => 4, + Self::Address | Self::StorageAddress => 3, + } + } + pub fn pack(self, parts: List) -> Op { + match self { + Self::View => Op::ViewPack(parts), + Self::TaggedI64 => Op::TaggedPack(parts), + Self::Address | Self::StorageAddress => Op::AddressPack(parts), + } + } + pub fn part(self, value: ValueId, index: u8) -> Op { + match self { + Self::View => Op::ViewPart { view: value, index }, + Self::TaggedI64 => Op::TaggedPart { value, index }, + Self::Address | Self::StorageAddress => Op::AddressPart { + address: value, + index, + }, + } + } + pub fn is_borrowed(self) -> bool { + self != Self::TaggedI64 + } + pub fn is_address(self) -> bool { + matches!(self, Self::Address | Self::StorageAddress) + } +} diff --git a/compiler-core/src/ir/types.rs b/compiler-core/src/ir/types.rs index fcf5659a..b73b40e6 100644 --- a/compiler-core/src/ir/types.rs +++ b/compiler-core/src/ir/types.rs @@ -1,4 +1,4 @@ -use super::{SymbolId, TypeId}; +use super::{LayoutId, SymbolId, TypeId}; use crate::scalar::ScalarType; use rustc_hash::FxHashMap; use std::sync::Arc; @@ -8,6 +8,8 @@ pub enum Type { Unit, /// Body-local identity for an uninspected pointee. Never a JVM value. Opaque(u32), + /// Exact source-language layout of an address pointee. Never a JVM value. + Layout(LayoutId), Scalar(ScalarType), Class(SymbolId), Interface(SymbolId), @@ -15,6 +17,8 @@ pub enum Type { Array(TypeId), Slice(TypeId), Str, + /// A 64-bit payload and independent discriminant. + TaggedI64, } impl Type { @@ -22,7 +26,7 @@ impl Type { pub fn carrier(self) -> u8 { use ScalarType::*; match self { - Self::Unit | Self::Opaque(_) => 0, + Self::Unit | Self::Opaque(_) | Self::Layout(_) => 0, Self::Scalar(Bool | I8 | U8 | I16 | U16 | I32 | U32 | Char | F16) => 1, Self::Scalar(I64 | U64) => 2, Self::Scalar(F32) => 3, @@ -32,16 +36,74 @@ impl Type { } } +/// Shared exact pointee layout, kept out of the eight-byte ordinary type entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct AddressLayout { + pub value: TypeId, + pub size: u32, + pub codec: Option, +} + #[derive(Default, Debug, Clone)] pub struct Types { base: Option>, values: Vec, ids: FxHashMap, + layouts: Vec, + layout_ids: FxHashMap, symbols: Vec>, symbol_ids: FxHashMap, SymbolId>, } impl Types { + pub fn layout(&mut self, layout: AddressLayout) -> TypeId { + let id = if let Some(&id) = self + .layout_ids + .get(&layout) + .or_else(|| self.base.as_ref().and_then(|b| b.layout_ids.get(&layout))) + { + id + } else { + let id = LayoutId::new(self.base_layouts() + self.layouts.len()); + self.layouts.push(layout); + self.layout_ids.insert(layout, id); + id + }; + self.intern(Type::Layout(id)) + } + fn base_layouts(&self) -> usize { + self.base.as_ref().map_or(0, |b| b.layouts.len()) + } + pub fn get_layout(&self, id: LayoutId) -> AddressLayout { + let base = self.base_layouts(); + if id.index() < base { + self.base.as_ref().unwrap().layouts[id.index()] + } else { + self.layouts[id.index() - base] + } + } + pub fn pointee(&self, pointer: TypeId) -> Option { + let Type::Pointer(inner) = self.get(pointer)? else { + return None; + }; + Some(match self.get(inner)? { + Type::Layout(id) => self.get_layout(id).value, + _ => inner, + }) + } + pub fn address_layout(&self, pointer: TypeId) -> Option<(u32, Option)> { + let Type::Pointer(inner) = self.get(pointer)? else { + return None; + }; + match self.get(inner)? { + Type::Layout(id) => { + let layout = self.get_layout(id); + Some((layout.size, layout.codec)) + } + _ => None, + } + } + /// Extend an immutable common vocabulary without copying its tables. Only /// one shared layer is allowed, so lookup cost cannot grow across bodies. pub fn with_base(base: Arc) -> Self { @@ -110,6 +172,12 @@ impl Types { self.symbol_ids.insert(name, id); id } + pub fn find_symbol(&self, name: &str) -> Option { + self.symbol_ids + .get(name) + .or_else(|| self.base.as_ref().and_then(|b| b.symbol_ids.get(name))) + .copied() + } pub fn symbol_name(&self, symbol: SymbolId) -> Option<&str> { let base = self.base_symbols(); if symbol.index() < base { @@ -127,7 +195,14 @@ impl Types { impl PartialEq for Types { fn eq(&self, other: &Self) -> bool { self.len() == other.len() - && (0..self.len()).all(|i| self.get(TypeId::new(i)) == other.get(TypeId::new(i))) + && (0..self.len()).all(|i| { + match (self.get(TypeId::new(i)), other.get(TypeId::new(i))) { + (Some(Type::Layout(a)), Some(Type::Layout(b))) => { + self.get_layout(a) == other.get_layout(b) + } + (a, b) => a == b, + } + }) && self.base_symbols() + self.symbols.len() == other.base_symbols() + other.symbols.len() && (0..self.base_symbols() + self.symbols.len()) @@ -139,7 +214,13 @@ impl std::hash::Hash for Types { fn hash(&self, state: &mut H) { self.len().hash(state); for i in 0..self.len() { - self.get(TypeId::new(i)).unwrap().hash(state); + let ty = self.get(TypeId::new(i)).unwrap(); + if let Type::Layout(id) = ty { + std::mem::discriminant(&ty).hash(state); + self.get_layout(id).hash(state); + } else { + ty.hash(state); + } } (self.base_symbols() + self.symbols.len()).hash(state); for i in 0..self.base_symbols() + self.symbols.len() { @@ -147,3 +228,28 @@ impl std::hash::Hash for Types { } } } + +#[cfg(test)] +mod layout_tests { + use super::*; + #[test] + fn layouts_are_interned_across_shared_vocabularies_without_growing_types() { + assert_eq!(std::mem::size_of::(), 8); + let mut base = Types::default(); + let value = base.scalar(ScalarType::I64); + let first = AddressLayout { + value, + size: 8, + codec: None, + }; + let ty = base.layout(first); + let mut child = Types::with_base(Arc::new(base)); + assert_eq!(child.layout(first), ty); + assert!(!child.has_additions()); + let different = child.layout(AddressLayout { size: 16, ..first }); + assert_ne!(different, ty); + let pointer = child.intern(Type::Pointer(different)); + assert_eq!(child.pointee(pointer), Some(value)); + assert_eq!(child.address_layout(pointer), Some((16, None))); + } +} diff --git a/compiler-core/src/ir/verify/mod.rs b/compiler-core/src/ir/verify/mod.rs index 4ed6ef87..cec60be2 100644 --- a/compiler-core/src/ir/verify/mod.rs +++ b/compiler-core/src/ir/verify/mod.rs @@ -36,14 +36,14 @@ pub fn verify_with_debug( check!( types .get(body.return_type) - .is_some_and(|t| !matches!(t, Type::Opaque(_))), + .is_some_and(|t| !matches!(t, Type::Opaque(_) | Type::Layout(_))), "invalid return type" ); for value in &body.values { check!( types .get(value.ty) - .is_some_and(|t| !matches!(t, Type::Opaque(_))), + .is_some_and(|t| !matches!(t, Type::Opaque(_) | Type::Layout(_))), "invalid value type" ); } @@ -67,16 +67,11 @@ pub fn verify_with_debug( Some(Type::Class(symbol) | Type::Interface(symbol)) => { check!(types.symbol_name(symbol).is_some(), "invalid storage class") } - Some(Type::Pointer(_) | Type::Slice(_) | Type::Str) => {} + Some(Type::Pointer(_) | Type::Slice(_) | Type::Str | Type::TaggedI64) => {} _ => return Err(VerifyError("unsupported storage type".into())), } } for field in &body.fields { - check!( - !field.relative_pointer - || (!field.is_static && matches!(types.get(field.ty), Some(Type::Pointer(_)))), - "invalid relative pointer field" - ); check!( matches!(types.get(field.owner), Some(Type::Class(symbol) | Type::Interface(symbol)) if types.symbol_name(symbol).is_some()), "invalid field owner" @@ -84,7 +79,7 @@ pub fn verify_with_debug( check!( types .get(field.ty) - .is_some_and(|t| !matches!(t, Type::Unit | Type::Opaque(_))), + .is_some_and(|t| !matches!(t, Type::Unit | Type::Opaque(_) | Type::Layout(_))), "invalid field type" ); } @@ -191,11 +186,36 @@ pub fn verify_with_debug( check!(method.index() < body.methods.len(), "invalid call target"); check!(args.range().end <= body.args.len(), "invalid operand list"); } - Op::Overflow { args, .. } => { + Op::Overflow { args, .. } + | Op::Heap { args, .. } + | Op::LoadStorageField { address: args, .. } + | Op::LoadStorageFieldCopy { address: args, .. } + | Op::StoreStorageField { args, .. } + | Op::AddressPack(args) + | Op::LoadAddress(args) + | Op::LoadAddressCopy(args) + | Op::CopyStorage { parts: args, .. } + | Op::LoadTypedCopy { parts: args, .. } + | Op::LoadTyped { parts: args, .. } + | Op::StoreTyped { parts: args, .. } + | Op::TypedAddressPack { parts: args, .. } + | Op::LocationTag(args) + | Op::LocationEqual(args) + | Op::LocationCompare(args) + | Op::TaggedPack(args) + | Op::ViewPack(args) + | Op::ViewGet(args) + | Op::ViewSet { parts: args, .. } + | Op::ViewAddress { parts: args, .. } + | Op::StoreAddress { parts: args, .. } + | Op::StoreFieldParts { parts: args, .. } => { check!(args.range().end <= body.args.len(), "invalid operand list") } Op::Constant(id) => check!(id.index() < body.constants.len(), "invalid constant"), - Op::LoadSlot(id) | Op::AddressOfSlot(id) | Op::StoreSlot { slot: id, .. } => { + Op::LoadSlot(id) + | Op::AddressOfSlot(id) + | Op::StoreSlot { slot: id, .. } + | Op::SlotRoot(id) => { check!(id.index() < body.slots.len(), "invalid storage slot") } _ => {} @@ -259,7 +279,9 @@ pub fn verify_with_debug( for local in &debug.locals { match *local { DebugLocal::Value(ty) => check!( - types.get(ty).is_some_and(|t| !matches!(t, Type::Opaque(_))), + types + .get(ty) + .is_some_and(|t| !matches!(t, Type::Opaque(_) | Type::Layout(_))), "invalid debug local type" ), DebugLocal::Storage(slot) => { diff --git a/compiler-core/src/ir/verify/types.rs b/compiler-core/src/ir/verify/types.rs index b7a6776f..dc910528 100644 --- a/compiler-core/src/ir/verify/types.rs +++ b/compiler-core/src/ir/verify/types.rs @@ -5,6 +5,176 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() let result = inst.result.map(|v| body.value_type(v)); let ty = |v| body.value_type(v); match inst.op { + Op::CopyStorage { parts, layouts, .. } => { + let parts = body + .args + .get(parts.range()) + .ok_or_else(|| VerifyError("invalid copy components".into()))?; + check!( + parts.len() == 5 && result.is_none(), + "invalid memory copy operands" + ); + for (i, layout) in layouts.into_iter().enumerate() { + let Some(Type::Layout(layout)) = types.get(layout) else { + return Err(VerifyError("copy needs an exact storage layout".into())); + }; + let AddressLayout { size, codec, .. } = types.get_layout(layout); + check!( + types + .get(ty(parts[i * 2])) + .is_some_and(|t| t.carrier() == 5) + && types.get(ty(parts[i * 2 + 1])) == Some(Type::Scalar(ScalarType::I64)), + "invalid copy location" + ); + check!( + size <= i32::MAX as u32 && codec.is_none_or(|s| types.symbol_name(s).is_some()), + "invalid copy layout" + ); + } + check!( + matches!( + types.get(ty(parts[4])), + Some(Type::Scalar( + ScalarType::I32 | ScalarType::U32 | ScalarType::I64 | ScalarType::U64 + )) + ), + "copy length needs an integer" + ); + } + Op::ViewRoot { + backing, + size, + codec, + } => check!( + types.get(ty(backing)).is_some_and(|t| t.carrier() == 5) + && result.is_some_and(|t| matches!(types.get(t), Some(Type::Class(s)) + if types.symbol_name(s) == Some("java/lang/Object"))) + && (1..=i32::MAX as u32).contains(&size) + && codec.is_none_or(|s| types.symbol_name(s).is_some()), + "invalid slice storage root" + ), + Op::Heap { operation, args } => { + let args = &body.args[args.range()]; + let (count, root, returns) = match operation { + HeapOp::Allocate => (2, false, true), + HeapOp::Reallocate => (5, true, true), + HeapOp::Deallocate => (2, true, false), + }; + check!(args.len() == count, "invalid heap operation arity"); + check!( + !root || types.get(ty(args[0])).is_some_and(|t| t.carrier() == 5), + "heap operation requires a storage root" + ); + check!( + args[usize::from(root)..] + .iter() + .all(|&arg| types.get(ty(arg)) == Some(Type::Scalar(ScalarType::I64))), + "heap sizes, alignments and offsets must be i64" + ); + check!( + if returns { + result.is_some_and(|ty| { + matches!(types.get(ty), Some(Type::Class(name)) + if types.symbol_name(name) == Some("java/lang/Object")) + }) + } else { + result.is_none() + }, + "invalid heap operation result" + ); + } + Op::CopyValue(value) => check!( + result == Some(ty(value)) && types.get(ty(value)).is_some_and(|t| t.carrier() == 5), + "copy requires the same reference value type" + ), + Op::TaggedPack(parts) => { + let parts = &body.args[parts.range()]; + check!( + parts.len() == 2 + && parts + .iter() + .all(|&v| types.get(ty(v)) == Some(Type::Scalar(ScalarType::I64))) + && result.is_some_and(|t| types.get(t) == Some(Type::TaggedI64)), + "invalid tagged scalar components" + ); + } + Op::TaggedPart { value, index } => { + check!( + index < 2 + && types.get(ty(value)) == Some(Type::TaggedI64) + && result.is_some_and(|t| types.get(t) == Some(Type::Scalar(ScalarType::I64))), + "invalid tagged scalar projection" + ); + } + Op::LoadStorageField { + address, + projection, + .. + } + | Op::LoadStorageFieldCopy { + address, + projection, + } + | Op::StoreStorageField { + args: address, + projection, + .. + } => { + let parts = body + .args + .get(address.range()) + .ok_or_else(|| VerifyError("invalid storage address".into()))?; + check!( + parts.len() >= 2 + && types.get(ty(parts[0])).is_some_and(|t| t.carrier() == 5) + && types.get(ty(parts[1])) == Some(Type::Scalar(ScalarType::I64)), + "invalid storage address components" + ); + let projection = body + .projections + .get(projection.index()) + .ok_or_else(|| VerifyError("invalid storage projection".into()))?; + let field = body + .fields + .get(projection.field.index()) + .ok_or_else(|| VerifyError("invalid storage field".into()))?; + check!( + !field.is_static && matches!(types.get(field.owner), Some(Type::Class(_))), + "storage projection needs a concrete owner" + ); + match inst.op { + Op::LoadStorageFieldCopy { .. } => check!( + parts.len() == 2 + && result == Some(field.ty) + && types.get(field.ty).is_some_and(|t| t.carrier() == 5), + "invalid owned storage field result" + ), + Op::LoadStorageField { index, .. } => check!( + parts.len() == 2 + && result.is_some_and(|result| match index { + Some(index) => + borrowed_part_matches(types, field.ty, index as usize, result), + None => result == field.ty, + }), + "invalid storage field result" + ), + Op::StoreStorageField { split, .. } => check!( + result.is_none() + && if split { + ComponentShape::of(types, field.ty) + .is_some_and(|s| s.len() == parts.len() - 2) + && parts[2..] + .iter() + .enumerate() + .all(|(i, &p)| borrowed_part_matches(types, field.ty, i, ty(p))) + } else { + parts.len() == 3 && ty(parts[2]) == field.ty + }, + "invalid storage field values" + ), + _ => unreachable!(), + } + } Op::Nop => check!(result.is_none(), "nop produces a value"), Op::ArrayLength(value) => check!( result.and_then(|t| types.get(t)) == Some(Type::Scalar(ScalarType::I32)) @@ -14,6 +184,13 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() ), "array length requires an array or view and int result" ), + Op::Refine(value) => { + check!( + result.and_then(|r| types.get(r)).map(Type::carrier) == Some(5) + && types.get(ty(value)).map(Type::carrier) == Some(5), + "reference refinement requires reference operands" + ); + } Op::Reinterpret(value) => { check!( result.and_then(|r| types.get(r)).map(Type::carrier) @@ -31,6 +208,38 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() "invalid ABI adaptation" ); } + Op::ScalarCell(value) => { + check!( + StorageSlot::scalar(ty(value), types).is_some() + && result.and_then(|r| types.get(r)) == Some(Type::Pointer(ty(value))), + "scalar storage requires an exact scalar pointee" + ); + } + Op::ProjectRoot { + address, + projection, + } => { + let values = &body.args[address.range()]; + check!( + values.len() == 2 + && types.get(ty(values[0])).is_some_and(|t| t.carrier() == 5) + && types.get(ty(values[1])) == Some(Type::Scalar(ScalarType::I64)) + && projection.index() < body.projections.len() + && result + .and_then(|r| types.get(r)) + .is_some_and(|t| t.carrier() == 5), + "invalid typed field location" + ); + } + Op::ProjectOffset { root, base, offset } => { + check!( + types.get(ty(root)).is_some_and(|t| t.carrier() == 5) + && types.get(ty(base)).is_some_and(|t| t.carrier() == 5) + && types.get(ty(offset)) == Some(Type::Scalar(ScalarType::I64)) + && result == Some(ty(offset)), + "invalid typed field displacement" + ); + } Op::NewArray(size) => { check!( types.get(ty(size)) == Some(Type::Scalar(ScalarType::I32)), @@ -41,6 +250,14 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() "array allocation requires array result" ); } + Op::ArrayFill { array, value } => { + check!( + result.is_none() + && types.get(ty(array)) == Some(Type::Array(ty(value))) + && StorageSlot::scalar(ty(value), types).is_some(), + "primitive array fill type mismatch" + ); + } Op::FunctionPointer { signature, target } => { let signature = body .methods @@ -251,8 +468,61 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() } } Op::Opaque(value) => check!(result == Some(ty(value)), "opaque value type mismatch"), + Op::AddressTag(value) => { + check!( + matches!(types.get(ty(value)), Some(Type::Pointer(_))) + && result.and_then(|ty| types.get(ty)) == Some(Type::Scalar(ScalarType::I64)), + "nullable tag needs pointer operand and i64 result" + ); + } + Op::AddressEqual { left, right } | Op::AddressCompare { left, right } => { + let output = if matches!(inst.op, Op::AddressCompare { .. }) { + ScalarType::I32 + } else { + ScalarType::Bool + }; + check!( + matches!(types.get(ty(left)), Some(Type::Pointer(_))) + && matches!(types.get(ty(right)), Some(Type::Pointer(_))) + && result.and_then(|ty| types.get(ty)) == Some(Type::Scalar(output)), + "invalid address comparison" + ); + } + Op::LocationEqual(parts) | Op::LocationCompare(parts) => { + let output = if matches!(inst.op, Op::LocationCompare(_)) { + ScalarType::I32 + } else { + ScalarType::Bool + }; + let values = body + .args + .get(parts.range()) + .ok_or_else(|| VerifyError("invalid location operands".into()))?; + check!( + values.len() == 4 + && result.and_then(|ty| types.get(ty)) == Some(Type::Scalar(output)), + "invalid location comparison" + ); + for (index, &value) in values.iter().enumerate() { + check!( + if index % 2 == 0 { + matches!(types.get(ty(value)), Some(Type::Class(s)) if types.symbol_name(s) == Some("java/lang/Object")) + } else { + types.get(ty(value)) == Some(Type::Scalar(ScalarType::I64)) + }, + "location component mismatch" + ); + } + } Op::Project { base, projection } | Op::LoadField { base, projection } + | Op::LoadFieldCopy { base, projection } + | Op::LoadFieldPart { + base, projection, .. + } + | Op::StoreFieldParts { + base, projection, .. + } | Op::StoreField { base, projection, .. } => { @@ -264,19 +534,56 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() .fields .get(projection.field.index()) .ok_or_else(|| VerifyError("invalid projection field".into()))?; + let mut owner = field.owner; + let mut parent = projection.parent; + let mut depth = 0; + while let Some(id) = parent { + depth += 1; + check!(depth <= 32, "projection path cycle"); + let previous = body + .projections + .get(id.index()) + .ok_or_else(|| VerifyError("invalid projection parent".into()))?; + let member = body + .fields + .get(previous.field.index()) + .ok_or_else(|| VerifyError("invalid projection parent field".into()))?; + check!( + !member.is_static && member.ty == owner, + "projection path type mismatch" + ); + owner = member.owner; + parent = previous.parent; + } check!( - !field.is_static && types.get(ty(base)) == Some(Type::Pointer(field.owner)), + !field.is_static && types.pointee(ty(base)) == Some(owner), "projection owner mismatch" ); - if !matches!(inst.op, Op::Project { .. }) { + check!( + projection.parent.is_none() + || matches!( + inst.op, + Op::LoadField { .. } | Op::LoadFieldCopy { .. } | Op::LoadFieldPart { .. } + ), + "projection paths require field reads" + ); + if !matches!(inst.op, Op::Project { .. } | Op::LoadFieldCopy { .. }) { check!( - matches!(types.get(field.ty), Some(Type::Scalar(_))), - "promoted field access requires a scalar field" + matches!( + types.get(field.ty), + Some(Type::Scalar(_) | Type::Pointer(_) | Type::Slice(_) | Type::Str) + ), + "promoted field access requires a scalar or pointer field" ); } match inst.op { + Op::LoadFieldCopy { .. } => check!( + result == Some(field.ty) + && types.get(field.ty).is_some_and(|t| t.carrier() == 5), + "owned field load type mismatch" + ), Op::Project { .. } => check!( - result.and_then(|id| types.get(id)) == Some(Type::Pointer(field.ty)), + result.and_then(|id| types.pointee(id)) == Some(field.ty), "projection result mismatch" ), Op::LoadField { .. } => { @@ -286,6 +593,41 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() result.is_none() && ty(value) == field.ty, "field store type mismatch" ), + Op::LoadFieldPart { index, .. } => { + check!( + result.is_some_and(|part| borrowed_part_matches( + types, + field.ty, + index as usize, + part + )), + "invalid split field load" + ); + } + Op::StoreFieldParts { parts, .. } => { + let values = body + .args + .get(parts.range()) + .ok_or_else(|| VerifyError("invalid split field operands".into()))?; + check!( + result.is_none() + && ComponentShape::of(types, field.ty) + .is_some_and(|shape| shape.len() == values.len()), + "invalid split field store" + ); + check!( + values + .iter() + .enumerate() + .all(|(index, &value)| borrowed_part_matches( + types, + field.ty, + index, + ty(value) + )), + "invalid split field components" + ); + } _ => unreachable!(), } check!( @@ -327,6 +669,193 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() "view length needs usize" ); } + Op::RetypeAddress { + pointer, + size, + codec, + } => { + check!( + matches!(types.get(ty(pointer)), Some(Type::Pointer(_))) + && matches!(result.and_then(|t| types.get(t)), Some(Type::Pointer(_))), + "retyped address needs pointer operands" + ); + check!( + size <= i32::MAX as u32 && codec.is_none_or(|id| types.symbol_name(id).is_some()), + "invalid address layout" + ); + } + Op::LocationTag(parts) + | Op::AddressPack(parts) + | Op::LoadAddress(parts) + | Op::LoadAddressCopy(parts) + | Op::LoadTypedCopy { parts, .. } + | Op::LoadTyped { parts, .. } + | Op::StoreTyped { parts, .. } + | Op::TypedAddressPack { parts, .. } + | Op::StoreAddress { parts, .. } => { + let parts = body + .args + .get(parts.range()) + .ok_or_else(|| VerifyError("invalid address components".into()))?; + check!( + match inst.op { + Op::LoadTyped { .. } => matches!(parts.len(), 2 | 3), + Op::StoreTyped { .. } => parts.len() == 3, + _ => parts.len() == 2, + }, + "address component count" + ); + check!( + types.get(ty(parts[0])).is_some_and(|t| t.carrier() == 5), + "address root needs a reference" + ); + check!( + types.get(ty(parts[1])) == Some(Type::Scalar(ScalarType::I64)), + "address offset needs signed long" + ); + if let Op::LoadTypedCopy { size, codec, .. } + | Op::LoadTyped { size, codec, .. } + | Op::TypedAddressPack { size, codec, .. } + | Op::StoreTyped { size, codec, .. } = inst.op + { + check!( + size > 0 + && size <= i32::MAX as u32 + && codec.is_none_or(|id| types.symbol_name(id).is_some()), + "invalid typed address layout" + ); + } + match inst.op { + Op::LocationTag(_) => check!( + result.and_then(|ty| types.get(ty)) == Some(Type::Scalar(ScalarType::I64)), + "nullable tag needs i64 result" + ), + Op::AddressPack(_) | Op::TypedAddressPack { .. } => check!( + result + .and_then(|t| ComponentShape::of(types, t)) + .is_some_and(ComponentShape::is_address), + "address pack needs pointer type" + ), + Op::LoadAddressCopy(_) | Op::LoadTypedCopy { .. } => check!( + result.is_some_and(|t| types.get(t).is_some_and(|t| t.carrier() == 5)), + "owned address load needs a reference carrier" + ), + Op::LoadTyped { .. } => { + check!( + result.is_some_and(|t| types.get(t).is_some_and(|t| t.carrier() == 5)), + "typed borrowed load needs a reference carrier" + ); + if let Some(&target) = parts.get(2) { + check!( + matches!(types.get(ty(target)), Some(Type::Class(name)) + if types.symbol_name(name) == Some("java/lang/String")), + "typed borrowed load needs a class name" + ); + } + } + Op::LoadAddress(_) => check!( + result.is_some_and(|t| StorageSlot::scalar(t, types).is_some() + || types.get(t).is_some_and(|t| t.carrier() == 5)), + "address load needs a stored value" + ), + Op::StoreTyped { .. } => check!( + result.is_none() && types.get(ty(parts[2])).is_some_and(|t| t.carrier() == 5), + "typed store needs an aggregate value" + ), + Op::StoreAddress { value, .. } => check!( + result.is_none() + && (StorageSlot::scalar(ty(value), types).is_some() + || types.get(ty(value)).is_some_and(|t| t.carrier() == 5)), + "address store needs a stored value" + ), + _ => unreachable!(), + } + } + Op::AddressPart { address, index } => { + check!( + ComponentShape::of(types, ty(address)).is_some_and(ComponentShape::is_address), + "address part needs pointer type" + ); + check!( + match index { + 0 => result + .and_then(|t| types.get(t)) + .is_some_and(|t| t.carrier() == 5), + 1 => result.and_then(|t| types.get(t)) == Some(Type::Scalar(ScalarType::I64)), + _ => false, + }, + "invalid address component" + ); + } + Op::SlotRoot(slot) => { + check!( + body.slots + .get(slot.index()) + .and_then(|s| StorageSlot::scalar(s.ty, types)) + .is_some(), + "slot root needs primitive storage" + ); + check!( + result + .and_then(|t| types.get(t)) + .is_some_and(|t| t.carrier() == 5), + "slot root needs reference result" + ); + } + Op::ViewPack(parts) | Op::ViewAddress { parts, .. } => { + let parts = body + .args + .get(parts.range()) + .ok_or_else(|| VerifyError("invalid view components".into()))?; + check!(parts.len() == 3, "view needs backing, start, and length"); + check!( + types.get(ty(parts[0])).is_some_and(|ty| ty.carrier() == 5), + "view backing needs a reference" + ); + check!( + types.get(ty(parts[1])) == Some(Type::Scalar(ScalarType::I32)), + "view start needs an int" + ); + check!( + types.get(ty(parts[2])) == Some(Type::Scalar(ScalarType::U64)), + "view length needs usize" + ); + if let Op::ViewPack(_) = inst.op { + check!( + result.is_some_and(|t| ComponentShape::view_carrier(types, t)), + "view pack result" + ); + return Ok(()); + } + let Op::ViewAddress { size, codec, .. } = inst.op else { + unreachable!() + }; + check!( + matches!(result.and_then(|ty| types.get(ty)), Some(Type::Pointer(_))), + "view address needs a pointer" + ); + check!( + size <= i32::MAX as u32 && codec.is_none_or(|id| types.symbol_name(id).is_some()), + "invalid view address layout" + ); + } + Op::ViewPart { view, index } => { + check!( + ComponentShape::view_carrier(types, ty(view)), + "view part source" + ); + check!( + match index { + 0 => result + .and_then(|t| types.get(t)) + .is_some_and(|t| t.carrier() == 5), + 1 => result.and_then(|t| types.get(t)) == Some(Type::Scalar(ScalarType::I32)), + 2 => result.and_then(|t| types.get(t)) == Some(Type::Scalar(ScalarType::U64)), + _ => false, + }, + "view part result" + ); + } Op::ViewData { view, size, codec } => { let element = view_element(types.get(ty(view)), types); check!( @@ -339,6 +868,44 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() "invalid view element layout" ); } + Op::AddressViewPart { address, index } => { + check!( + ComponentShape::of(types, ty(address)).is_some_and(ComponentShape::is_address), + "view part needs an address" + ); + check!( + match (index, result.and_then(|t| types.get(t))) { + (0, Some(Type::Class(s))) => types.symbol_name(s) == Some("java/lang/Object"), + (1, Some(Type::Scalar(ScalarType::I32))) => true, + _ => false, + }, + "invalid address view component" + ); + } + Op::TypedAddressViewPart { + parts, + size, + codec, + index, + } => { + let parts = &body.args[parts.range()]; + check!(parts.len() == 2, "invalid typed slice location arity"); + check!( + types.get(ty(parts[0])).is_some_and(|t| t.carrier() == 5) + && types.get(ty(parts[1])) == Some(Type::Scalar(ScalarType::I64)) + && (1..=i32::MAX as u32).contains(&size) + && codec.is_none_or(|s| types.symbol_name(s).is_some()), + "invalid typed slice location" + ); + check!( + match (index, result.and_then(|t| types.get(t))) { + (0, Some(Type::Class(s))) => types.symbol_name(s) == Some("java/lang/Object"), + (1, Some(Type::Scalar(ScalarType::I32))) => true, + _ => false, + }, + "invalid typed slice component" + ); + } Op::Cast(value) => { check!(result.is_some(), "cast has no result"); if matches!(types.get(ty(value)), Some(Type::Scalar(_))) { @@ -356,12 +923,25 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() result.and_then(|ty| types.get(ty)) == Some(Type::Pointer(body.slots[slot.index()].ty)), "slot address type mismatch" ), + Op::Commit(pointer) => check!( + result.is_none() && matches!(types.get(ty(pointer)), Some(Type::Pointer(_))), + "view commit needs its address owner" + ), + Op::LoadCopy(pointer) => check!( + result.is_some_and(|t| types.get(t).is_some_and(|t| t.carrier() == 5)) + && types.pointee(ty(pointer)) == result, + "owned pointer load type mismatch" + ), Op::Load(pointer) => check!( - result.is_some() && types.get(ty(pointer)) == result.map(Type::Pointer), - "pointer load type mismatch" + result.is_some() && types.pointee(ty(pointer)) == result, + "pointer load type mismatch: address {:?} ({:?}), result {:?} ({:?})", + pointer, + types.get(ty(pointer)), + result, + result.and_then(|ty| types.get(ty)) ), Op::Store { pointer, value } => check!( - result.is_none() && types.get(ty(pointer)) == Some(Type::Pointer(ty(value))), + result.is_none() && types.pointee(ty(pointer)) == Some(ty(value)), "pointer store type mismatch" ), Op::StoreSlot { slot, value } => { @@ -395,8 +975,41 @@ pub(super) fn verify_types(inst: &Inst, body: &Body, types: &Types) -> Result<() check!(result == Some(member.ty), "field load type mismatch"); } } - Op::ArraySet { .. } => check!(result.is_none(), "store produces value"), - _ => {} + Op::ViewGet(parts) | Op::ViewGetCopy(parts) | Op::ViewSet { parts, .. } => { + let parts = &body.args[parts.range()]; + check!( + parts.len() == 3 + && types.get(ty(parts[0])).is_some_and(|t| t.carrier() == 5) + && parts[1..] + .iter() + .all(|&p| types.get(ty(p)) == Some(Type::Scalar(ScalarType::I32))), + "slice access requires backing, start and index" + ); + check!( + matches!(inst.op, Op::ViewGet(_) | Op::ViewGetCopy(_)) == result.is_some(), + "slice access result mismatch" + ); + } + Op::ArrayGet { array, index, .. } + | Op::ArrayGetCopy { array, index } + | Op::ArraySet { array, index, .. } => { + check!( + !matches!( + inst.op, + Op::ArrayGet { native: true, .. } | Op::ArraySet { native: true, .. } + ) || matches!(types.get(ty(array)), Some(Type::Array(_))), + "native scratch access requires a JVM array" + ); + check!( + types.get(ty(index)) == Some(Type::Scalar(ScalarType::I32)), + "array index requires JVM int" + ); + check!( + matches!(inst.op, Op::ArrayGet { .. } | Op::ArrayGetCopy { .. }) + == result.is_some(), + "array access result mismatch" + ); + } } Ok(()) } @@ -408,3 +1021,18 @@ fn view_element(ty: Option, types: &Types) -> Option { _ => None, } } + +fn borrowed_part_matches(types: &Types, logical: TypeId, index: usize, part: TypeId) -> bool { + match (ComponentShape::of(types, logical), index, types.get(part)) { + (Some(ComponentShape::TaggedI64), 0..=1, Some(Type::Scalar(ScalarType::I64))) => true, + (Some(_), 0, Some(Type::Class(s))) => types.symbol_name(s) == Some("java/lang/Object"), + ( + Some(ComponentShape::Address | ComponentShape::StorageAddress), + 1, + Some(Type::Scalar(ScalarType::I64)), + ) + | (Some(ComponentShape::View), 1, Some(Type::Scalar(ScalarType::I32))) + | (Some(ComponentShape::View), 2, Some(Type::Scalar(ScalarType::U64))) => true, + _ => false, + } +} diff --git a/compiler-core/src/jvm/abi.rs b/compiler-core/src/jvm/abi.rs index 0acb87e3..883ba21d 100644 --- a/compiler-core/src/jvm/abi.rs +++ b/compiler-core/src/jvm/abi.rs @@ -1,15 +1,117 @@ //! JVM names shared by body selection and generated representation schemas. pub const SLICE_VIEW_CLASS: &str = "org/rustlang/runtime/SliceView"; pub const UTF8_VIEW_CLASS: &str = "org/rustlang/runtime/Utf8View"; +pub const TAGGED_LONG_CLASS: &str = "org/rustlang/runtime/TaggedLong"; pub const POINTER_CLASS: &str = "org/rustlang/runtime/Pointer"; -pub const RELATIVE_POINTER_METHOD_SUFFIX: &str = "$relative"; -pub const RELATIVE_POINTER_ELEMENT_OFFSET_SUFFIX: &str = "$rcj$elementOffset"; -pub const RELATIVE_POINTER_BYTE_OFFSET_SUFFIX: &str = "$rcj$byteOffset"; -pub fn relative_pointer_element_offset_field(field: &str) -> String { - format!("{field}{RELATIVE_POINTER_ELEMENT_OFFSET_SUFFIX}") +/// Reconstruction plans for addresses of stored borrows. These are ABI tags. +/// Ordinary scalar plans retain their 1/2/4/8-byte strides. +pub const STORED_VIEW: u32 = 64; +pub const STORED_ADDRESS: u32 = 128; + +pub fn address_plan(types: &crate::ir::Types, pointee: crate::ir::TypeId) -> u32 { + use crate::ir::{StorageSlot, Type}; + match types.get(pointee) { + Some(Type::Slice(_) | Type::Str) => STORED_VIEW, + Some(Type::Pointer(_)) => STORED_ADDRESS, + _ => StorageSlot::scalar(pointee, types).map_or(0, |slot| slot.size), + } +} + +/// Consume an Object root and long displacement. +/// Construct a boundary carrier only when a consumer needs one reference. +pub fn materialize_address( + cp: &mut crate::classfile::constant_pool::InternedConstantPool, + code: &mut Vec, + plan: u32, +) -> crate::classfile::Result<()> { + use crate::classfile::attributes::Instruction; + code.push(crate::jvm::constants::get_int_const_instr(cp, plan as i32)); + let owner = cp.add_class(POINTER_CLASS)?; + code.push(Instruction::Invokestatic(cp.add_method_ref( + owner, + "addressFromParts", + "(Ljava/lang/Object;JI)Lorg/rustlang/runtime/Pointer;", + )?)); + Ok(()) } -pub fn relative_pointer_byte_offset_field(field: &str) -> String { - format!("{field}{RELATIVE_POINTER_BYTE_OFFSET_SUFFIX}") +pub fn materialize_typed_address( + cp: &mut crate::classfile::constant_pool::InternedConstantPool, + code: &mut Vec, + size: u32, + codec: Option<&str>, +) -> crate::classfile::Result<()> { + use crate::classfile::attributes::Instruction; + code.push(crate::jvm::constants::get_int_const_instr(cp, size as i32)); + code.push(if let Some(codec) = codec { + Instruction::Ldc_w(cp.add_name_string(codec)?) + } else { + Instruction::Aconst_null + }); + let owner = cp.add_class(POINTER_CLASS)?; + code.push(Instruction::Invokestatic(cp.add_method_ref( + owner, + "fromTypedStorageLocation", + "(Ljava/lang/Object;JILjava/lang/String;)Lorg/rustlang/runtime/Pointer;", + )?)); + Ok(()) +} + +pub const VIEW_PARTS: [(&str, &str); 3] = [ + ("array", "Ljava/lang/Object;"), + ("offset", "I"), + ("rustLength", "J"), +]; + +/// Use null for an absent optional borrow at boundaries. +/// All adapters use the same default component slots for this niche. +pub fn view_part_access( + cp: &mut crate::classfile::constant_pool::InternedConstantPool, + index: usize, +) -> crate::classfile::Result { + let (name, descriptor) = VIEW_PARTS[index]; + let owner = cp.add_class(SLICE_VIEW_CLASS)?; + Ok(crate::classfile::attributes::Instruction::Invokestatic( + cp.add_method_ref( + owner, + format!("$part${name}"), + format!("(L{SLICE_VIEW_CLASS};){descriptor}"), + )?, + )) +} +/// Synthetic displacement field paired with a scalar-address root field. +pub fn tagged_field_names(name: &str) -> [String; 2] { + [name.into(), format!("$rust$t${name}")] +} +pub fn address_field_name(name: &str, size: u32) -> String { + format!("$rust${size}${name}") +} +/// Exact layouts are metadata on the carrier schema, not on every borrow. +pub fn typed_address_field_name(name: &str, size: u32) -> String { + format!("$rust$a{size}${name}") +} +pub fn address_codec_field_name(name: &str) -> String { + format!("$rust$c${name}") +} +pub fn address_displacement_name( + types: &crate::ir::Types, + pointer: crate::ir::TypeId, + name: &str, +) -> String { + if let Some((size, _)) = types.address_layout(pointer) { + typed_address_field_name(name, size) + } else { + let Some(crate::ir::Type::Pointer(inner)) = types.get(pointer) else { + unreachable!() + }; + address_field_name(name, address_plan(types, inner)) + } +} +pub fn view_field_names(name: &str, utf8: bool) -> [String; 3] { + [ + name.into(), + format!("$rust${}${name}", if utf8 { "u" } else { "s" }), + format!("$rust$l${name}"), + ] } diff --git a/compiler-core/src/jvm/frames/tests.rs b/compiler-core/src/jvm/frames/tests.rs index c08136c7..b4fddc10 100644 --- a/compiler-core/src/jvm/frames/tests.rs +++ b/compiler-core/src/jvm/frames/tests.rs @@ -89,3 +89,47 @@ fn definite_assignment_accepts_a_store_on_every_path() { let loads = locals_loaded_before_definite_store(&instructions, &[], 2, "test", &[]).unwrap(); assert!(loads.is_empty()); } + +#[test] +fn duplicate_two_words_under_one_preserves_categories() { + use FrameValue::*; + for (input, expected) in [ + (vec![Null, Long], vec![Long, Null, Long]), + ( + vec![Null, Integer, Float], + vec![Integer, Float, Null, Integer, Float], + ), + ] { + let mut state = FrameState::new(Vec::new(), 0); + for value in input { + state.push(value); + } + super::transfer::transfer_instruction( + 0, + &Instruction::Dup2_x1, + &mut state, + &[], + &ConstantPool::default(), + "dup2_x1", + &mut SignatureCache::default(), + ) + .unwrap(); + assert_eq!(state.stack, expected); + assert_eq!(state.stack_words, 5); + } + let mut state = FrameState::new(Vec::new(), 0); + state.push(Long); + state.push(Long); + assert!( + super::transfer::transfer_instruction( + 0, + &Instruction::Dup2_x1, + &mut state, + &[], + &ConstantPool::default(), + "dup2_x1", + &mut SignatureCache::default() + ) + .is_err() + ); +} diff --git a/compiler-core/src/jvm/frames/transfer.rs b/compiler-core/src/jvm/frames/transfer.rs index 1592aefe..33cd114c 100644 --- a/compiler-core/src/jvm/frames/transfer.rs +++ b/compiler-core/src/jvm/frames/transfer.rs @@ -233,7 +233,24 @@ pub(super) fn transfer_instruction( state.push(value1); } } - I::Dup_x2 | I::Dup2_x1 | I::Dup2_x2 => { + I::Dup2_x1 => { + let first = state.pop(context, instruction_index)?; + if first.is_category2() { + let below = state.pop_category1(context, instruction_index)?; + state.push(first.clone()); + state.push(below); + state.push(first); + } else { + let second = state.pop_category1(context, instruction_index)?; + let below = state.pop_category1(context, instruction_index)?; + state.push(second.clone()); + state.push(first.clone()); + state.push(below); + state.push(second); + state.push(first); + } + } + I::Dup_x2 | I::Dup2_x2 => { return Err(jvm::Error::VerificationError { context: context.to_string(), message: format!( diff --git a/compiler-core/src/jvm/select/addresses.rs b/compiler-core/src/jvm/select/addresses.rs new file mode 100644 index 00000000..1e78cabb --- /dev/null +++ b/compiler-core/src/jvm/select/addresses.rs @@ -0,0 +1,453 @@ +//! Physical address components and scalar memory operations. +use super::*; + +impl Selector<'_> { + pub(super) fn address(&mut self, inst: Inst) -> jvm::Result { + match inst.op { + Op::CopyStorage { + parts, + layouts, + nonoverlapping, + } => { + let parts = &self.body.args[parts.range()]; + for (i, layout) in layouts.into_iter().enumerate() { + let Some(Type::Layout(layout)) = self.types.get(layout) else { + return Err(error("copy needs an exact storage layout")); + }; + let AddressLayout { size, codec, .. } = self.types.get_layout(layout); + self.load(parts[i * 2])?; + self.load(parts[i * 2 + 1])?; + self.address_layout(size, codec)?; + } + self.load(parts[4])?; + if self + .types + .get(self.body.value_type(parts[4])) + .unwrap() + .carrier() + == 0 + { + self.assembly.code.push(Instruction::I2l); + } + self.assembly.code.push(if nonoverlapping { + Instruction::Iconst_1 + } else { + Instruction::Iconst_0 + }); + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref(owner, "copyStorage", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/Object;JILjava/lang/String;JZ)V")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::TypedAddressViewPart { + parts, + size, + codec, + index, + } => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.address_layout(size, codec)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let (name, result) = if index == 0 { + ("typedLocationSliceBacking", "Ljava/lang/Object;") + } else { + ("typedLocationSliceOffset", "I") + }; + let method = self.cp.add_method_ref( + owner, + name, + format!("(Ljava/lang/Object;JILjava/lang/String;){result}"), + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::RetypeAddress { + pointer, + size, + codec, + } => { + self.load(pointer)?; + self.address_layout(size, codec)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "retype", + "(ILjava/lang/String;)Lorg/rustlang/runtime/Pointer;", + )?; + self.assembly.code.push(Instruction::Invokevirtual(method)); + } + Op::TypedAddressPack { parts, size, codec } => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.address_layout(size, codec)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "fromTypedStorageLocation", + "(Ljava/lang/Object;JILjava/lang/String;)Lorg/rustlang/runtime/Pointer;", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::LoadTypedCopy { parts, size, codec } => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.address_layout(size, codec)?; + let ty = self.body.value_type(inst.result.unwrap()); + let name = self.address_target(ty)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "loadTypedStorageCopy", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;", + )?; + let class = self.cp.add_class(&name)?; + self.assembly.code.extend([ + Instruction::Invokestatic(method), + Instruction::Checkcast(class), + ]); + } + Op::LoadTyped { parts, size, codec } => { + let parts = &self.body.args[parts.range()]; + self.load(parts[0])?; + self.load(parts[1])?; + self.address_layout(size, codec)?; + if let Some(&target) = parts.get(2) { + self.load(target)?; + } else if let Some(Type::Class(name) | Type::Interface(name)) = + self.types.get(self.body.value_type(inst.result.unwrap())) + && self.types.symbol_name(name) != Some("java/lang/Object") + { + self.assembly.code.push(Instruction::Ldc_w( + self.cp + .add_name_string(self.types.symbol_name(name).unwrap())?, + )); + } else { + self.assembly.code.push(Instruction::Aconst_null); + } + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "loadTypedStorage", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + let mut descriptor = String::new(); + representation::descriptor( + self.types, + self.body.value_type(inst.result.unwrap()), + &mut descriptor, + )?; + if descriptor != "Ljava/lang/Object;" { + let target = descriptor + .strip_prefix('L') + .and_then(|s| s.strip_suffix(';')) + .unwrap_or(&descriptor); + self.assembly + .code + .push(Instruction::Checkcast(self.cp.add_class(target)?)); + } + } + Op::StoreTyped { parts, size, codec } => { + let parts = &self.body.args[parts.range()]; + self.load(parts[0])?; + self.load(parts[1])?; + self.address_layout(size, codec)?; + self.load(parts[2])?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "storeTypedStorage", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/Object;)V", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::ProjectRoot { + address, + projection, + } => { + if self.body.projections[projection.index()].parent.is_some() { + self.project_scalar_path(address, projection)?; + return Ok(true); + } + for &part in &self.body.args[address.range()] { + self.load(part)?; + } + self.project_field_arguments(projection)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let field = + &self.body.fields[self.body.projections[projection.index()].field.index()]; + let name = if ComponentShape::of(self.types, field.ty) + .is_some_and(ComponentShape::is_borrowed) + { + "storageBorrowedFieldRoot" + } else { + "storageFieldRoot" + }; + let method = self.cp.add_method_ref(owner, name, + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJLjava/lang/String;)Ljava/lang/Object;")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::ProjectOffset { root, base, offset } => { + self.load(root)?; + self.load(base)?; + self.load(offset)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "storageFieldOffset", + "(Ljava/lang/Object;Ljava/lang/Object;J)J", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::AddressTag(value) => { + self.load(value)?; + self.assembly.code.push(Instruction::Lconst_0); + self.location_tag()?; + } + Op::LocationTag(parts) => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.location_tag()?; + } + Op::AddressViewPart { address, index } => { + self.load(address)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let (name, descriptor) = if index == 0 { + ("sliceBackingArray", "()Ljava/lang/Object;") + } else { + ("sliceElementOffset", "()I") + }; + let method = self.cp.add_method_ref(owner, name, descriptor)?; + self.assembly.code.push(Instruction::Invokevirtual(method)); + } + Op::AddressEqual { left, right } | Op::AddressCompare { left, right } => { + self.load(left)?; + self.assembly.code.push(Instruction::Lconst_0); + self.load(right)?; + self.assembly.code.push(Instruction::Lconst_0); + self.location_comparison(matches!(inst.op, Op::AddressCompare { .. }))?; + } + Op::LocationEqual(parts) | Op::LocationCompare(parts) => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.location_comparison(matches!(inst.op, Op::LocationCompare(_)))?; + } + Op::AddressPart { address, index: 0 } => self.load(address)?, + Op::AddressPart { index: 1, .. } => self.assembly.code.push(Instruction::Lconst_0), + Op::SlotRoot(slot) => self + .assembly + .code + .push(Kind::Reference.load(self.storage[slot.index()])), + Op::AddressPack(parts) => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + let Some(Type::Pointer(inner)) = + self.types.get(self.body.value_type(inst.result.unwrap())) + else { + return Err(error("address pack needs pointer type")); + }; + self.materialize_address(inner)?; + } + Op::LoadAddressCopy(parts) => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.read_object_address(self.body.value_type(inst.result.unwrap()), true)?; + } + Op::LoadAddress(parts) => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.read_address(self.body.value_type(inst.result.unwrap()))?; + } + Op::StoreAddress { parts, value } => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.load(value)?; + self.write_address(self.body.value_type(value))?; + } + _ => return Ok(false), + } + Ok(true) + } + pub(super) fn materialize_address(&mut self, ty: TypeId) -> jvm::Result<()> { + if let Some(Type::Layout(id)) = self.types.get(ty) { + let AddressLayout { size, codec, .. } = self.types.get_layout(id); + return super::super::abi::materialize_typed_address( + self.cp, + &mut self.assembly.code, + size, + codec.map(|s| self.types.symbol_name(s).unwrap()), + ); + } + super::super::abi::materialize_address( + self.cp, + &mut self.assembly.code, + super::super::abi::address_plan(self.types, ty), + ) + } + fn location_tag(&mut self) -> jvm::Result<()> { + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = + self.cp + .add_method_ref(owner, "nullableLocationTag", "(Ljava/lang/Object;J)J")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + Ok(()) + } + fn location_comparison(&mut self, ordering: bool) -> jvm::Result<()> { + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + if ordering { + "compareLocations" + } else { + "sameLocation" + }, + if ordering { + "(Ljava/lang/Object;JLjava/lang/Object;J)I" + } else { + "(Ljava/lang/Object;JLjava/lang/Object;J)Z" + }, + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + Ok(()) + } + pub(super) fn read_address(&mut self, ty: TypeId) -> jvm::Result<()> { + if self.types.get(ty).is_some_and(|t| t.carrier() == 5) { + self.read_object_address(ty, false)?; + return Ok(()); + } + let size = StorageSlot::scalar(ty, self.types) + .ok_or_else(|| error("scalar load layout"))? + .size; + self.assembly + .code + .push(get_int_const_instr(self.cp, size as i32)); + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = + self.cp + .add_method_ref(owner, "loadLocationBits", "(Ljava/lang/Object;JI)J")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + self.bits_to_scalar(ty) + } + pub(super) fn bits_to_scalar(&mut self, ty: TypeId) -> jvm::Result<()> { + match self.types.get(ty) { + Some(Type::Scalar(ScalarType::F64)) => { + self.address_float("java/lang/Double", "longBitsToDouble", "(J)D")? + } + Some(Type::Scalar(ScalarType::I64 | ScalarType::U64)) => {} + Some(Type::Scalar(scalar)) => { + self.assembly.code.push(Instruction::L2i); + if scalar == ScalarType::F32 { + self.address_float("java/lang/Float", "intBitsToFloat", "(I)F")?; + } + } + _ => return Err(error("non-scalar address load")), + } + Ok(()) + } + pub(super) fn read_object_address(&mut self, ty: TypeId, owned: bool) -> jvm::Result<()> { + let name = self.address_target(ty)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + if owned { + "loadStorageCopy" + } else if matches!(self.types.get(ty), Some(Type::Array(_))) { + "loadStorageArray" + } else { + "loadStorageLocation" + }, + "(Ljava/lang/Object;JLjava/lang/String;)Ljava/lang/Object;", + )?; + let class = self.cp.add_class(&name)?; + self.assembly.code.extend([ + Instruction::Invokestatic(method), + Instruction::Checkcast(class), + ]); + Ok(()) + } + pub(super) fn address_layout(&mut self, size: u32, codec: Option) -> jvm::Result<()> { + self.assembly + .code + .push(get_int_const_instr(self.cp, size as i32)); + self.assembly.code.push(match codec { + Some(id) => Instruction::Ldc_w( + self.cp + .add_name_string(self.types.symbol_name(id).unwrap())?, + ), + None => Instruction::Aconst_null, + }); + Ok(()) + } + pub(super) fn address_target(&mut self, ty: TypeId) -> jvm::Result { + let mut descriptor = String::new(); + representation::descriptor(self.types, ty, &mut descriptor)?; + let name = descriptor + .strip_prefix('L') + .and_then(|s| s.strip_suffix(';')) + .unwrap_or(&descriptor); + self.assembly + .code + .push(if matches!(self.types.get(ty), Some(Type::Pointer(_))) { + Instruction::Aconst_null + } else { + Instruction::Ldc_w(self.cp.add_name_string(name)?) + }); + Ok(name.into()) + } + pub(super) fn write_address(&mut self, ty: TypeId) -> jvm::Result<()> { + if self.types.get(ty).is_some_and(|t| t.carrier() == 5) { + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "storeStorageLocation", + "(Ljava/lang/Object;JLjava/lang/Object;)V", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + return Ok(()); + } + let size = StorageSlot::scalar(ty, self.types) + .ok_or_else(|| error("scalar store layout"))? + .size; + self.scalar_to_bits(ty)?; + self.assembly + .code + .push(get_int_const_instr(self.cp, size as i32)); + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = + self.cp + .add_method_ref(owner, "storeLocationBits", "(Ljava/lang/Object;JJI)V")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + Ok(()) + } + pub(super) fn scalar_to_bits(&mut self, ty: TypeId) -> jvm::Result<()> { + match self.types.get(ty) { + Some(Type::Scalar(ScalarType::F64)) => { + self.address_float("java/lang/Double", "doubleToRawLongBits", "(D)J")? + } + Some(Type::Scalar(ScalarType::I64 | ScalarType::U64)) => {} + Some(Type::Scalar(scalar)) => { + if scalar == ScalarType::F32 { + self.address_float("java/lang/Float", "floatToRawIntBits", "(F)I")?; + } + self.assembly.code.push(Instruction::I2l); + } + _ => return Err(error("non-scalar address store")), + } + Ok(()) + } + fn address_float(&mut self, class: &str, name: &str, descriptor: &str) -> jvm::Result<()> { + let owner = self.cp.add_class(class)?; + let method = self.cp.add_method_ref(owner, name, descriptor)?; + self.assembly.code.push(Instruction::Invokestatic(method)); + Ok(()) + } +} diff --git a/compiler-core/src/jvm/select/allocate.rs b/compiler-core/src/jvm/select/allocate.rs index 8d288c97..79275443 100644 --- a/compiler-core/src/jvm/select/allocate.rs +++ b/compiler-core/src/jvm/select/allocate.rs @@ -18,7 +18,6 @@ pub(super) fn allocate( types: &Types, live: &crate::opt::Live, forwarded: &[bool], - relative_pointer_abi: bool, debug: Option<&DebugInfo>, order: &[BlockId], ) -> jvm::Result { @@ -35,14 +34,7 @@ pub(super) fn allocate( .count .checked_add(value_kind(body, types, param)?.width()) .ok_or_else(|| error("JVM local limit"))?; - if relative_pointer_abi - && matches!(types.get(body.value_type(param)), Some(Type::Pointer(_))) - { - result.count = result - .count - .checked_add(4) - .ok_or_else(|| error("JVM parameter limit"))?; - } + active.push(Reverse((intervals[param.index()].1, param))); } let mut order = (0..body.values.len()) @@ -270,7 +262,6 @@ mod tests { &types, &crate::opt::live(&body, &types), &vec![false; body.values.len()], - false, None, &body.layout(), ) diff --git a/compiler-core/src/jvm/select/arrays.rs b/compiler-core/src/jvm/select/arrays.rs index 409523d4..31eb5340 100644 --- a/compiler-core/src/jvm/select/arrays.rs +++ b/compiler-core/src/jvm/select/arrays.rs @@ -1,7 +1,37 @@ //! Array operations preserve lazy repeat copies and pointer-backed slice views. use super::*; +#[derive(Clone, Copy, PartialEq, Eq)] +enum Access { + Native, + Array, + View, + Owned, +} + impl Selector<'_> { - pub(super) fn array(&mut self, inst: Inst) -> jvm::Result { + pub(super) fn array(&mut self, id: InstId, inst: Inst) -> jvm::Result { + if let Op::ArrayFill { array, value } = inst.op { + let element = self.body.value_type(value); + let native = self.native_array_accesses.get(id.index()) == Some(&Some(element)); + self.load(array)?; + self.argument(value)?; + let mut descriptor = String::from("("); + representation::descriptor(self.types, self.body.value_type(array), &mut descriptor)?; + representation::descriptor(self.types, element, &mut descriptor)?; + descriptor.push_str(")V"); + let owner = self.cp.add_class(if native { + "java/util/Arrays" + } else { + POINTER_CLASS + })?; + let target = self.cp.add_method_ref( + owner, + if native { "fill" } else { "fillArray" }, + descriptor, + )?; + self.assembly.code.push(Instruction::Invokestatic(target)); + return Ok(true); + } if let Op::ArrayLength(array) = inst.op { self.load(array)?; let op = if matches!( @@ -16,13 +46,56 @@ impl Selector<'_> { self.assembly.code.push(op); return Ok(true); } - let (array, index, value) = match inst.op { - Op::ArrayGet { array, index } => (array, index, None), + if let Op::ViewGet(parts) | Op::ViewGetCopy(parts) | Op::ViewSet { parts, .. } = inst.op { + let [backing, start, index] = self.body.args[parts.range()] else { + return Err(error("slice access components")); + }; + let value = match inst.op { + Op::ViewSet { value, .. } => Some(value), + _ => None, + }; + let element = self.body.value_type(value.or(inst.result).unwrap()); + let native = self.native_array_accesses.get(id.index()) == Some(&Some(element)); + self.load(backing)?; + if native { + let mut descriptor = String::from("["); + representation::descriptor(self.types, element, &mut descriptor)?; + self.assembly + .code + .push(Instruction::Checkcast(self.cp.add_class(&descriptor)?)); + } + self.load(start)?; + self.load(index)?; + self.assembly.code.push(Instruction::Iadd); + if let Some(value) = value { + self.argument(value)?; + } + self.array_access( + element, + value.is_some(), + if matches!(inst.op, Op::ViewGetCopy(_)) { + Access::Owned + } else if native { + Access::Native + } else { + Access::View + }, + )?; + return Ok(true); + } + let (array, index, value, native) = match inst.op { + Op::ArrayGet { + array, + index, + native, + } => (array, index, None, native), + Op::ArrayGetCopy { array, index } => (array, index, None, false), Op::ArraySet { array, index, value, - } => (array, index, Some(value)), + native, + } => (array, index, Some(value), native), _ => return Ok(false), }; let representation = self.types.get(self.body.value_type(array)); @@ -45,6 +118,9 @@ impl Selector<'_> { if let Some(value) = value { self.argument(value)?; self.write_memory(element)?; + } else if matches!(inst.op, Op::ArrayGetCopy { .. }) { + self.assembly.code.push(Instruction::Lconst_0); + self.read_object_address(element, true)?; } else { self.read_memory(element)?; } @@ -72,7 +148,66 @@ impl Selector<'_> { if let Some(value) = value { self.argument(value)?; } + let native = native || self.native_array_accesses.get(id.index()) == Some(&Some(element)); + let primitive = matches!(self.types.get(element), Some(Type::Scalar(_))); + // A raw pointer can expose this array to a decoded aggregate alias later. + // Only a whole-lifetime proof can remove coherence checks. + self.array_access( + element, + value.is_some(), + if matches!(inst.op, Op::ArrayGetCopy { .. }) { + Access::Owned + } else if native { + Access::Native + } else if view || primitive { + Access::View + } else { + Access::Array + }, + )?; + Ok(true) + } + + fn array_access(&mut self, element: TypeId, store: bool, access: Access) -> jvm::Result<()> { use ScalarType::*; + if access == Access::Owned { + if let Some(bootstrap) = &mut self.bootstrap { + let owner = self.cp.add_class("org/rustlang/runtime/OwnedReads")?; + let method = self.cp.add_method_ref( + owner, + "bootstrap", + concat!( + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;", + "Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;" + ), + )?; + let bootstrap_method_ref = self + .cp + .add_method_handle(jvm::ReferenceKind::InvokeStatic, method)?; + let index = u16::try_from(bootstrap.len())?; + bootstrap.push(jvm::attributes::BootstrapMethod { + bootstrap_method_ref, + arguments: vec![], + }); + let mut descriptor = String::from("(Ljava/lang/Object;I)"); + representation::descriptor(self.types, element, &mut descriptor)?; + let site = self.cp.add_invoke_dynamic(index, "read", descriptor)?; + self.assembly.code.push(Instruction::Invokedynamic(site)); + return Ok(()); + } + let name = self.address_target(element)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "sliceGetObjectCopy", + "(Ljava/lang/Object;ILjava/lang/String;)Ljava/lang/Object;", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + self.assembly + .code + .push(Instruction::Checkcast(self.cp.add_class(&name)?)); + return Ok(()); + } let (suffix, descriptor, read, write) = match self.types.get(element) { Some(Type::Scalar(Bool)) => ("Boolean", "Z", Instruction::Baload, Instruction::Bastore), Some(Type::Scalar(I8 | U8)) => ("I8", "B", Instruction::Baload, Instruction::Bastore), @@ -97,24 +232,21 @@ impl Selector<'_> { Instruction::Aastore, ), }; - if view || (suffix == "Object" && value.is_none()) { + if access == Access::View || (access == Access::Array && suffix == "Object" && !store) { let owner = self.cp.add_class(POINTER_CLASS)?; - let name = if view { - format!( - "slice{}{suffix}", - if value.is_some() { "Set" } else { "Get" } - ) + let name = if access == Access::View { + format!("slice{}{suffix}", if store { "Set" } else { "Get" }) } else { "arrayGetObject".into() }; - let signature = if value.is_some() { + let signature = if store { format!("(Ljava/lang/Object;I{descriptor})V") } else { format!("(Ljava/lang/Object;I){descriptor}") }; let method = self.cp.add_method_ref(owner, name, signature)?; self.assembly.code.push(Instruction::Invokestatic(method)); - if suffix == "Object" && value.is_none() { + if suffix == "Object" && !store { let mut name = String::new(); representation::descriptor(self.types, element, &mut name)?; let name = name @@ -126,10 +258,8 @@ impl Selector<'_> { .push(Instruction::Checkcast(self.cp.add_class(name)?)); } } else { - self.assembly - .code - .push(if value.is_some() { write } else { read }); + self.assembly.code.push(if store { write } else { read }); } - Ok(true) + Ok(()) } } diff --git a/compiler-core/src/jvm/select/assemble.rs b/compiler-core/src/jvm/select/assemble.rs index a33343b1..a083eca4 100644 --- a/compiler-core/src/jvm/select/assemble.rs +++ b/compiler-core/src/jvm/select/assemble.rs @@ -79,8 +79,11 @@ impl Assembly { _ => unreachable!(), } } + let mut candidates = Vec::new(); for (index, label) in self.fixups { let target = self.labels[label.0].ok_or_else(|| error("unbound JVM label"))?; + let short = target == index + 2 + && matches!(self.code.get(index + 1), Some(Instruction::Goto_w(_))); let target = u16::try_from(target)?; match &mut self.code[index] { Instruction::Goto_w(t) => *t = i32::from(target), @@ -88,9 +91,73 @@ impl Assembly { *instruction = crate::jvm::flow::set_conditional_branch_target(instruction, target) .ok_or_else(|| error("invalid symbolic branch"))?; + if short { + candidates.push(index); + } } } } + if candidates.is_empty() { + return Ok(self.code); + } + let offsets = crate::jvm::encoding::instruction_byte_offsets(&self.code)?; + let mut targeted = vec![false; self.code.len()]; + for position in self.labels.into_iter().flatten() { + if position < targeted.len() { + targeted[position] = true; + } + } + for index in candidates { + if targeted[index + 1] { + continue; + } + let Instruction::Goto_w(target) = self.code[index + 1] else { + continue; + }; + let target = u16::try_from(target)?; + if i16::try_from(offsets[usize::from(target)] as i64 - offsets[index] as i64).is_ok() { + self.code[index] = + crate::jvm::flow::invert_conditional_branch(&self.code[index], target).unwrap(); + // Retain instruction indices and switch alignment for metadata. + self.code[index + 1] = Instruction::Nop; + } + } Ok(self.code) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shorten_only_unshared_conditional_trampolines_within_range() { + for (distance, shared, short) in + [(12, false, true), (40000, false, false), (12, true, false)] + { + let mut a = Assembly::default(); + let skip = a.label(); + let target = a.label(); + a.branch(Instruction::Ifeq(0), skip); + if shared { + let shared = a.label(); + a.bind(shared); + } + a.branch(Instruction::Goto_w(0), target); + a.bind(skip); + a.code.resize(distance, Instruction::Nop); + a.bind(target); + a.code.push(Instruction::Return); + let code = a.finish().unwrap(); + assert_eq!( + code[0], + if short { + Instruction::Ifne(distance as u16) + } else { + Instruction::Ifeq(2) + } + ); + assert_eq!(matches!(code[1], Instruction::Nop), short); + } + } +} diff --git a/compiler-core/src/jvm/select/calls.rs b/compiler-core/src/jvm/select/calls.rs index 948f6a18..5a1cf81d 100644 --- a/compiler-core/src/jvm/select/calls.rs +++ b/compiler-core/src/jvm/select/calls.rs @@ -26,6 +26,19 @@ impl Selector<'_> { for &arg in &self.body.args[args.range()] { self.argument(arg)?; } + if matches!(kind, CallKind::RustStatic | CallKind::JvmStatic) + && method.owner == POINTER_CLASS + && method.name == "field" + && signature + == "(Ljava/lang/Object;Ljava/lang/String;JLjava/lang/String;)Lorg/rustlang/runtime/Pointer;" + && self.runtime_call_site( + "org/rustlang/runtime/FieldProjections", + "field", + &signature, + )? + { + return Ok(()); + } self.assembly.code.push(match kind { CallKind::RustStatic | CallKind::JvmStatic => Instruction::Invokestatic(target), CallKind::Constructor => Instruction::Invokespecial(target), @@ -42,4 +55,35 @@ impl Selector<'_> { }); Ok(()) } + + fn runtime_call_site( + &mut self, + owner: &str, + name: &str, + descriptor: &str, + ) -> jvm::Result { + let Some(bootstrap) = &mut self.bootstrap else { + return Ok(false); + }; + let owner = self.cp.add_class(owner)?; + let method = self.cp.add_method_ref( + owner, + "bootstrap", + concat!( + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;", + "Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;" + ), + )?; + let bootstrap_method_ref = self + .cp + .add_method_handle(jvm::ReferenceKind::InvokeStatic, method)?; + let index = u16::try_from(bootstrap.len())?; + bootstrap.push(jvm::attributes::BootstrapMethod { + bootstrap_method_ref, + arguments: vec![], + }); + let site = self.cp.add_invoke_dynamic(index, name, descriptor)?; + self.assembly.code.push(Instruction::Invokedynamic(site)); + Ok(true) + } } diff --git a/compiler-core/src/jvm/select/debug_tests.rs b/compiler-core/src/jvm/select/debug_tests.rs index 981d9bb8..72499201 100644 --- a/compiler-core/src/jvm/select/debug_tests.rs +++ b/compiler-core/src/jvm/select/debug_tests.rs @@ -46,6 +46,15 @@ fn debug_mirrors_keep_dead_values_and_hide_uninitialized_join_bindings() { value: seven, }, ); + // Keep executable work after initializing the conditional binding. + // An empty edge has no bytecode range for its debug information. + debug.push( + &b, + DebugChange::Set { + local: 1, + value: parameter, + }, + ); b.jump(join, vec![]); b.switch_to(join); debug.push(&b, DebugChange::Scope(0)); diff --git a/compiler-core/src/jvm/select/fields.rs b/compiler-core/src/jvm/select/fields.rs index bb840018..cae2cfe6 100644 --- a/compiler-core/src/jvm/select/fields.rs +++ b/compiler-core/src/jvm/select/fields.rs @@ -1,66 +1,359 @@ -//! Select promoted field pointers without reflective runtime field cells. +//! Select projected fields through authoritative typed storage when available. use super::*; +mod bytes; + impl Selector<'_> { pub(super) fn field_memory(&mut self, inst: Inst) -> jvm::Result { - let (base, projection, value) = match inst.op { - Op::LoadField { base, projection } => (base, projection, None), + let address = match inst.op { + Op::LoadStorageField { address, .. } | Op::LoadStorageFieldCopy { address, .. } => { + Some(&self.body.args[address.range()]) + } + Op::StoreStorageField { args, .. } => Some(&self.body.args[args.range()][..2]), + _ => None, + }; + let owned = matches!( + inst.op, + Op::LoadFieldCopy { .. } | Op::LoadStorageFieldCopy { .. } + ); + let (base, projection, value, part, components) = match inst.op { + Op::LoadStorageFieldCopy { projection, .. } => { + (address.unwrap()[0], projection, None, None, None) + } + Op::LoadStorageField { + projection, index, .. + } => (address.unwrap()[0], projection, None, index, None), + Op::StoreStorageField { + args, + projection, + split, + } => { + let values = List { + start: args.start + 2, + len: args.len - 2, + }; + ( + address.unwrap()[0], + projection, + (!split).then(|| self.body.args[values.range().start]), + None, + split.then_some(values), + ) + } + Op::LoadField { base, projection } | Op::LoadFieldCopy { base, projection } => { + (base, projection, None, None, None) + } Op::StoreField { base, projection, value, - } => (base, projection, Some(value)), + } => (base, projection, Some(value), None, None), + Op::LoadFieldPart { + base, + projection, + index, + } => (base, projection, None, Some(index), None), + Op::StoreFieldParts { + base, + projection, + parts, + } => (base, projection, None, None, Some(parts)), _ => return Ok(false), }; let field = &self.body.fields[self.body.projections[projection.index()].field.index()]; + let mut path = Vec::new(); + let mut parent = self.body.projections[projection.index()].parent; + while let Some(id) = parent { + path.push(id); + parent = self.body.projections[id.index()].parent; + } + path.reverse(); + let root_owner = path.first().map_or(field.owner, |p| { + self.body.fields[self.body.projections[p.index()].field.index()].owner + }); + let Some(Type::Class(root_symbol)) = self.types.get(root_owner) else { + return Err(error("field path requires a concrete root")); + }; + let root_class = self + .cp + .add_class(self.types.symbol_name(root_symbol).unwrap())?; let Some(Type::Class(owner)) = self.types.get(field.owner) else { return Err(error("field memory requires a concrete owner")); }; let owner = self.cp.add_class(self.types.symbol_name(owner).unwrap())?; - let mut descriptor = String::new(); - representation::descriptor(self.types, field.ty, &mut descriptor)?; - let member = self.cp.add_field_ref(owner, &field.name, &descriptor)?; let pointer = self.cp.add_class(POINTER_CLASS)?; let direct = self.cp.add_method_ref( pointer, "directAggregate", "(Ljava/lang/Class;)Ljava/lang/Object;", )?; + let split = part.is_some() || components.is_some(); + let physical = if split { + let names = match self.types.get(field.ty) { + Some(Type::TaggedI64) => super::super::abi::tagged_field_names(&field.name) + .into_iter() + .zip(["J", "J"]) + .collect(), + Some(Type::Pointer(_)) => { + vec![ + (field.name.clone(), "Ljava/lang/Object;"), + ( + super::super::abi::address_displacement_name( + self.types, + field.ty, + &field.name, + ), + "J", + ), + ] + } + Some(Type::Slice(_) | Type::Str) => super::super::abi::view_field_names( + &field.name, + matches!(self.types.get(field.ty), Some(Type::Str)), + ) + .into_iter() + .zip(["Ljava/lang/Object;", "I", "J"]) + .collect(), + _ => return Err(error("invalid split field")), + }; + Some( + names + .into_iter() + .map(|(name, ty)| self.cp.add_field_ref(owner, &name, ty)) + .collect::>>()?, + ) + } else { + None + }; + let member = if let Some(physical) = &physical { + physical[part.unwrap_or(0) as usize] + } else { + let mut descriptor = String::new(); + representation::descriptor(self.types, field.ty, &mut descriptor)?; + self.cp.add_field_ref(owner, &field.name, &descriptor)? + }; let slow = self.assembly.label(); let done = self.assembly.label(); - // Keep the pointer for committing the live carrier, or projecting just - // this field when storage is raw or only partially initialized. + let store = value.is_some() || components.is_some(); self.load(base)?; - self.assembly.code.extend([ - Instruction::Dup, - Instruction::Ldc_w(owner), - Instruction::Invokevirtual(direct), - Instruction::Dup, - ]); + // Adjacent field reads share one resolved aggregate object. + // Mutations, opaque calls and control-flow boundaries invalidate it. + let cached = !store && !owned; + let key = ( + self.body.resolve(base), + address.map(|a| self.body.resolve(a[1])), + root_owner, + ); + if cached && self.aggregate_cache == Some(key) { + self.assembly + .code + .push(Kind::Reference.load(self.aggregate_slot.unwrap())); + } else { + self.assembly.code.push(Instruction::Dup); + if let Some(parts) = address { + self.load(parts[1])?; + self.assembly.code.push(Instruction::Ldc_w(root_class)); + let resolve = self.cp.add_method_ref( + pointer, + "directStorageAggregate", + "(Ljava/lang/Object;JLjava/lang/Class;)Ljava/lang/Object;", + )?; + self.assembly.code.push(Instruction::Invokestatic(resolve)); + } else { + self.assembly.code.extend([ + Instruction::Ldc_w(root_class), + Instruction::Invokevirtual(direct), + ]); + } + if cached { + let slot = if let Some(slot) = self.aggregate_slot { + slot + } else { + let slot = self.next_slot; + self.next_slot = self + .next_slot + .checked_add(1) + .ok_or_else(|| error("JVM local limit"))?; + self.aggregate_slot = Some(slot); + slot + }; + self.assembly + .code + .extend([Instruction::Dup, Kind::Reference.store(slot)]); + self.aggregate_cache = Some(key); + } + } + self.assembly.code.push(Instruction::Dup); self.assembly.branch(Instruction::Ifnull(0), slow); - if value.is_none() { + if !path.is_empty() { + self.assembly.code.push(Instruction::Checkcast(root_class)); + } + for &id in &path { + let member = &self.body.fields[self.body.projections[id.index()].field.index()]; + let Some(Type::Class(owner)) = self.types.get(member.owner) else { + return Err(error("field path requires a concrete owner")); + }; + let owner = self.cp.add_class(self.types.symbol_name(owner).unwrap())?; + let mut descriptor = String::new(); + representation::descriptor(self.types, member.ty, &mut descriptor)?; + let member = self.cp.add_field_ref(owner, &member.name, &descriptor)?; + self.assembly + .code + .extend([Instruction::Getfield(member), Instruction::Dup]); + self.assembly.branch(Instruction::Ifnull(0), slow); + } + if !store { self.assembly .code .extend([Instruction::Swap, Instruction::Pop]); } self.assembly.code.push(Instruction::Checkcast(owner)); - if let Some(value) = value { + if let Some(parts) = components { + let values = &self.body.args[parts.range()]; + for (index, &value) in values.iter().enumerate() { + if index + 1 < values.len() { + self.assembly.code.push(Instruction::Dup); + } + self.load(value)?; + self.assembly + .code + .push(Instruction::Putfield(physical.as_ref().unwrap()[index])); + } + } else if let Some(value) = value { self.argument(value)?; self.assembly.code.push(Instruction::Putfield(member)); - let commit = self.cp.add_method_ref(pointer, "commitMemoryView", "()V")?; - self.assembly.code.push(Instruction::Invokevirtual(commit)); } else { self.assembly.code.push(Instruction::Getfield(member)); + if owned { + self.copy_value(field.ty)?; + } + } + if store { + if let Some(parts) = address { + self.load(parts[1])?; + let commit = self.cp.add_method_ref( + pointer, + "commitStorageLocation", + "(Ljava/lang/Object;J)V", + )?; + self.assembly.code.push(Instruction::Invokestatic(commit)); + } else { + let commit = self.cp.add_method_ref(pointer, "commitMemoryView", "()V")?; + self.assembly.code.push(Instruction::Invokevirtual(commit)); + } } self.assembly.branch(Instruction::Goto_w(0), done); self.assembly.bind(slow); self.assembly.code.push(Instruction::Pop); + if part == Some(1) && matches!(self.types.get(field.ty), Some(Type::Pointer(_))) { + // The fallback root contains the full address. Its displacement is zero. + self.assembly + .code + .extend([Instruction::Pop, Instruction::Lconst_0]); + self.assembly.bind(done); + return Ok(true); + } + if !path.is_empty() && !owned && !store { + self.nested_byte_field(address.map(|parts| parts[1]), projection, part, done)?; + } + for &id in &path { + self.project_field(id)?; + } + if owned { + if let Some(parts) = address { + self.load(parts[1])?; + } else { + self.assembly.code.push(Instruction::Lconst_0); + } + self.project_field_arguments(projection)?; + let target = self.address_target(field.ty)?; + let method = self.cp.add_method_ref(pointer, "loadStorageFieldCopy", + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJLjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;")?; + let target = self.cp.add_class(target)?; + self.assembly.code.extend([ + Instruction::Invokestatic(method), + Instruction::Checkcast(target), + ]); + self.assembly.bind(done); + return Ok(true); + } + // Keep managed access direct. Share byte and fallback dispatch in the + // runtime because larger generated methods can prevent JVM inlining. + if let Some(value) = value + && matches!( + self.types.get(field.ty), + Some(Type::Class(_) | Type::Array(_)) + ) + { + if let Some(parts) = address { + self.load(parts[1])?; + } else { + self.assembly.code.push(Instruction::Lconst_0); + } + self.project_field_arguments(projection)?; + self.argument(value)?; + let method = self.cp.add_method_ref(pointer, "storeStorageField", + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJLjava/lang/String;Ljava/lang/Object;)V")?; + self.assembly.code.push(Instruction::Invokestatic(method)); + self.assembly.bind(done); + return Ok(true); + } + if !split && self.scalar_field(address.map(|parts| parts[1]), projection, value)? { + self.assembly.bind(done); + return Ok(true); + } + if let Some(parts) = address { + self.load(parts[1])?; + let materialize = self.cp.add_method_ref( + pointer, + "fromStorageLocation", + "(Ljava/lang/Object;J)Lorg/rustlang/runtime/Pointer;", + )?; + self.assembly + .code + .push(Instruction::Invokestatic(materialize)); + } self.project_field(projection)?; - if let Some(value) = value { + if let Some(parts) = components { + if let Some(Type::Pointer(inner)) = self.types.get(field.ty) { + for &value in &self.body.args[parts.range()] { + self.load(value)?; + } + self.materialize_address(inner)?; + } else if matches!(self.types.get(field.ty), Some(Type::TaggedI64)) { + self.materialize_tagged(parts)?; + } else { + self.materialize_view(field.ty, parts)?; + } + self.write_memory(field.ty)?; + } else if let Some(value) = value { self.argument(value)?; self.write_memory(field.ty)?; } else { self.read_memory(field.ty)?; + if let Some(index) = part { + if matches!(self.types.get(field.ty), Some(Type::Pointer(_))) { + debug_assert_eq!(index, 0); + } else if matches!(self.types.get(field.ty), Some(Type::TaggedI64)) { + let owner = self.cp.add_class(super::super::abi::TAGGED_LONG_CLASS)?; + self.assembly + .code + .push(Instruction::Invokestatic(self.cp.add_method_ref( + owner, + if index == 0 { "value" } else { "tag" }, + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?)); + } else { + let slice = self.cp.add_class(representation::SLICE_VIEW_CLASS)?; + let (name, ty) = [ + ("array", "Ljava/lang/Object;"), + ("offset", "I"), + ("rustLength", "J"), + ][index as usize]; + let member = self.cp.add_field_ref(slice, name, ty)?; + self.assembly.code.push(Instruction::Getfield(member)); + } + } } self.assembly.bind(done); Ok(true) diff --git a/compiler-core/src/jvm/select/fields/bytes.rs b/compiler-core/src/jvm/select/fields/bytes.rs new file mode 100644 index 00000000..ba9de5c5 --- /dev/null +++ b/compiler-core/src/jvm/select/fields/bytes.rs @@ -0,0 +1,187 @@ +//! Scalar field fallback with byte-storage dispatch shared in the runtime. +use super::*; + +impl Selector<'_> { + pub(super) fn nested_byte_field( + &mut self, + offset: Option, + projection: ProjectionId, + part: Option, + done: Label, + ) -> jvm::Result<()> { + let layout = &self.body.projections[projection.index()]; + let ty = self.body.fields[layout.field.index()].ty; + let scalar = part.is_none() + && layout.codec.is_none() + && StorageSlot::scalar(ty, self.types) + .is_some_and(|s| u64::from(s.size) == layout.size); + let pointer = matches!(self.types.get(ty), Some(Type::Pointer(_))) + && part.is_none_or(|part| part == 0) + && (1..=i32::MAX as u64).contains(&layout.size); + if !scalar && !pointer { + return Ok(()); + } + let mut displacement = 0_u64; + let mut next = Some(projection); + while let Some(id) = next { + let field = &self.body.projections[id.index()]; + let Some(sum) = displacement.checked_add(field.offset) else { + return Ok(()); + }; + displacement = sum; + next = field.parent; + } + let Ok(displacement) = i64::try_from(displacement) else { + return Ok(()); + }; + let runtime = self.cp.add_class(POINTER_CLASS)?; + let bytes = + self.cp + .add_method_ref(runtime, "hasPlainByteStorage", "(Ljava/lang/Object;)Z")?; + let fallback = self.assembly.label(); + self.assembly + .code + .extend([Instruction::Dup, Instruction::Invokestatic(bytes)]); + self.assembly.branch(Instruction::Ifeq(0), fallback); + self.assembly + .code + .push(get_long_const_instr(self.cp, displacement)); + if let Some(offset) = offset { + self.load(offset)?; + let math = self.cp.add_class("java/lang/Math")?; + let add = self.cp.add_method_ref(math, "addExact", "(JJ)J")?; + self.assembly.code.push(Instruction::Invokestatic(add)); + } + if scalar { + self.read_address(ty)?; + if self.types.get(ty) == Some(Type::Scalar(ScalarType::F16)) { + self.assembly.code.push(Instruction::I2s); + } + } else { + self.assembly + .code + .push(get_int_const_instr(self.cp, layout.size as i32)); + self.assembly.code.push(match &layout.codec { + Some(codec) => Instruction::Ldc_w(self.cp.add_name_string(codec)?), + None => Instruction::Aconst_null, + }); + self.assembly + .code + .push(Instruction::Ldc_w(self.cp.add_name_string(POINTER_CLASS)?)); + let load = self.cp.add_method_ref( + runtime, + "loadTypedStorage", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;", + )?; + self.assembly.code.extend([ + Instruction::Invokestatic(load), + Instruction::Checkcast(runtime), + ]); + } + self.assembly.branch(Instruction::Goto_w(0), done); + self.assembly.bind(fallback); + Ok(()) + } + + pub(in crate::jvm::select) fn project_scalar_path( + &mut self, + address: List, + projection: ProjectionId, + ) -> jvm::Result<()> { + let parts = &self.body.args[address.range()]; + let pointer = self.cp.add_class(POINTER_CLASS)?; + let bytes = + self.cp + .add_method_ref(pointer, "hasPlainByteStorage", "(Ljava/lang/Object;)Z")?; + let done = self.assembly.label(); + self.load(parts[0])?; + self.assembly + .code + .extend([Instruction::Dup, Instruction::Invokestatic(bytes)]); + self.assembly.branch(Instruction::Ifne(0), done); + self.load(parts[1])?; + let materialize = self.cp.add_method_ref( + pointer, + "fromStorageLocation", + "(Ljava/lang/Object;J)Lorg/rustlang/runtime/Pointer;", + )?; + self.assembly + .code + .push(Instruction::Invokestatic(materialize)); + let mut path = vec![projection]; + while let Some(parent) = self.body.projections[path[path.len() - 1].index()].parent { + path.push(parent); + } + for projection in path.into_iter().rev() { + self.project_field(projection)?; + } + self.assembly.bind(done); + Ok(()) + } + + /// Consume the root already on the stack, preserving stack forwarding. + pub(super) fn scalar_field( + &mut self, + offset: Option, + projection: ProjectionId, + value: Option, + ) -> jvm::Result { + let layout = &self.body.projections[projection.index()]; + let field = &self.body.fields[layout.field.index()]; + let ty = field.ty; + if layout.codec.is_some() + || !StorageSlot::scalar(ty, self.types) + .is_some_and(|s| (1..=8).contains(&s.size) && u64::from(s.size) == layout.size) + { + return Ok(false); + } + let Some(Type::Class(owner)) = self.types.get(field.owner) else { + return Err(error("scalar field requires a concrete owner")); + }; + if let Some(offset) = offset { + self.load(offset)?; + } else { + self.assembly.code.push(Instruction::Lconst_0); + } + self.assembly.code.extend([ + Instruction::Ldc_w( + self.cp + .add_name_string(self.types.symbol_name(owner).unwrap())?, + ), + Instruction::Ldc_w(self.cp.add_string(&field.name)?), + get_long_const_instr(self.cp, layout.offset as i64), + ]); + if let Some(value) = value { + self.argument(value)?; + self.scalar_to_bits(ty)?; + } + self.assembly + .code + .push(get_int_const_instr(self.cp, layout.size as i32)); + let pointer = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + pointer, + if value.is_some() { + "storeScalarField" + } else { + "loadScalarField" + }, + if value.is_some() { + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJI)V" + } else { + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JI)J" + }, + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + if value.is_none() { + self.bits_to_scalar(ty)?; + // Integer result normalization is shared with all other IR loads. + match self.types.get(ty) { + Some(Type::Scalar(ScalarType::F16)) => self.assembly.code.push(Instruction::I2s), + Some(Type::Scalar(ScalarType::Char)) => self.assembly.code.push(Instruction::I2c), + _ => {} + } + } + Ok(true) + } +} diff --git a/compiler-core/src/jvm/select/forward.rs b/compiler-core/src/jvm/select/forward.rs index 09852e38..02eeac44 100644 --- a/compiler-core/src/jvm/select/forward.rs +++ b/compiler-core/src/jvm/select/forward.rs @@ -38,6 +38,7 @@ pub(super) fn values(body: &Body, live: &crate::opt::Live, debug: Option<&DebugI match block.terminator.unwrap() { Terminator::Return(value) => value, Terminator::Branch { condition, .. } => Some(condition), + Terminator::Switch { value, cases, .. } if cases.len == 1 => Some(value), Terminator::Throw { value, unwind: None, @@ -57,24 +58,25 @@ fn first_operand(body: &Body, op: Op) -> Option { Op::Neg(value) | Op::Not(value) | Op::Cast(value) + | Op::Refine(value) | Op::Reinterpret(value) | Op::Adapt(value) | Op::NewArray(value) | Op::ArrayLength(value) | Op::Opaque(value) | Op::Load(value) + | Op::LoadCopy(value) | Op::Length(value) | Op::SetStatic { value, .. } => Some(value), - Op::Project { base, .. } | Op::LoadField { base, .. } | Op::StoreField { base, .. } => { - Some(base) - } + Op::Project { base, .. } + | Op::LoadField { base, .. } + | Op::LoadFieldCopy { base, .. } + | Op::StoreField { base, .. } + | Op::LoadFieldPart { base, .. } + | Op::StoreFieldParts { base, .. } => Some(base), Op::Offset { pointer, .. } | Op::Store { pointer, .. } => Some(pointer), Op::ViewData { view, .. } => Some(view), - Op::GetField { object, field } | Op::SetField { object, field, .. } - if !body.fields[field.index()].relative_pointer => - { - Some(object) - } + Op::GetField { object, .. } | Op::SetField { object, .. } => Some(object), Op::Call { kind, args, .. } if kind != CallKind::Constructor => { body.args[args.range()].first().copied() } @@ -98,17 +100,9 @@ mod tests { let result = b.constant(int, Scalar::integer(ScalarType::I32, 7).unwrap()); b.terminate(Terminator::Return(Some(result))); let body = b.finish().unwrap(); - let code = compile_with_options( - &body, - &types, - &mut Default::default(), - Options { - relative_pointer_abi: true, - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(code.max_locals, 6); // Pointer + two offsets + byte ABI slots. + let code = compile_with_options(&body, &types, &mut Default::default(), Options::default()) + .unwrap(); + assert_eq!(code.max_locals, 2); // Pointer and byte ABI slots. assert_eq!( code.instructions, vec![Instruction::Bipush(7), Instruction::Ireturn] diff --git a/compiler-core/src/jvm/select/general.rs b/compiler-core/src/jvm/select/general.rs index c063d52f..de3b2371 100644 --- a/compiler-core/src/jvm/select/general.rs +++ b/compiler-core/src/jvm/select/general.rs @@ -5,6 +5,21 @@ use jvm::attributes::{ArrayType, BootstrapMethod}; impl Selector<'_> { pub(super) fn general(&mut self, inst: Inst) -> jvm::Result { match inst.op { + Op::Heap { operation, args } => { + let (name, descriptor) = match operation { + HeapOp::Allocate => ("allocate", "(JJ)Ljava/lang/Object;"), + HeapOp::Reallocate => { + ("reallocate", "(Ljava/lang/Object;JJJJ)Ljava/lang/Object;") + } + HeapOp::Deallocate => ("deallocate", "(Ljava/lang/Object;J)V"), + }; + for &value in &self.body.args[args.range()] { + self.load(value)?; + } + let class = self.cp.add_class("org/rustlang/runtime/Heap")?; + let method = self.cp.add_method_ref(class, name, descriptor)?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } Op::Nop => {} Op::Exception => self.assembly.code.push( Kind::Reference.load( diff --git a/compiler-core/src/jvm/select/memory.rs b/compiler-core/src/jvm/select/memory.rs index 90d27001..46f0d942 100644 --- a/compiler-core/src/jvm/select/memory.rs +++ b/compiler-core/src/jvm/select/memory.rs @@ -9,35 +9,6 @@ fn scalar(types: &Types, ty: TypeId) -> jvm::Result { } impl Selector<'_> { - pub(super) fn materialize_parameters(&mut self) -> jvm::Result<()> { - for ¶m in &self.body.blocks[self.body.entry.index()].params { - // The ABI root reserves the slot; only additional uses need an - // actual pointer value. Unused arguments require no materialization. - if self.live.uses[param.index()] == 1 - || !matches!( - self.types.get(self.body.value_type(param)), - Some(Type::Pointer(_)) - ) - { - continue; - } - let slot = self.slot(param); - let owner = self.cp.add_class(POINTER_CLASS)?; - let method = self.cp.add_method_ref( - owner, - "materializeRelative", - "(Lorg/rustlang/runtime/Pointer;JJ)Lorg/rustlang/runtime/Pointer;", - )?; - self.assembly.code.extend([ - Kind::Reference.load(slot), - Kind::Long.load(slot + 1), - Kind::Long.load(slot + 3), - Instruction::Invokestatic(method), - Kind::Reference.store(slot), - ]); - } - Ok(()) - } pub(super) fn initialize_storage(&mut self) -> jvm::Result<()> { for storage in &self.body.slots { let owner = self.cp.add_class(POINTER_CLASS)?; @@ -46,7 +17,7 @@ impl Selector<'_> { let code = match scalar { Bool => 4, I8 | U8 => 8, - I16 => 9, + I16 | F16 => 9, U16 => 5, I32 | U32 => 10, I64 | U64 => 11, @@ -56,17 +27,10 @@ impl Selector<'_> { }; let array = jvm::attributes::ArrayType::from_bytes(&mut jvm::ByteReader::new(&[code]))?; - self.assembly.code.extend([ - Instruction::Iconst_1, - Instruction::Newarray(array), - Instruction::Iconst_0, - get_int_const_instr(self.cp, storage.size as i32), - ]); - self.cp.add_method_ref( - owner, - "array", - "(Ljava/lang/Object;II)Lorg/rustlang/runtime/Pointer;", - )? + self.assembly + .code + .extend([Instruction::Iconst_1, Instruction::Newarray(array)]); + None } else { self.assembly.code.push(Instruction::Aconst_null); self.assembly @@ -83,11 +47,11 @@ impl Selector<'_> { self.assembly .code .push(get_int_const_instr(self.cp, storage.alignment as i32)); - self.cp.add_method_ref( + Some(self.cp.add_method_ref( owner, "cellAligned", "(Ljava/lang/Object;ILjava/lang/String;I)Lorg/rustlang/runtime/Pointer;", - )? + )?) }; let slot = self.next_slot; self.next_slot = self @@ -95,10 +59,10 @@ impl Selector<'_> { .checked_add(1) .ok_or_else(|| error("JVM storage slot limit"))?; self.storage.push(slot); - self.assembly.code.extend([ - Instruction::Invokestatic(target), - Kind::Reference.store(slot), - ]); + if let Some(target) = target { + self.assembly.code.push(Instruction::Invokestatic(target)); + } + self.assembly.code.push(Kind::Reference.store(slot)); } Ok(()) } @@ -108,6 +72,10 @@ impl Selector<'_> { return Ok(true); } match inst.op { + Op::CopyValue(value) => { + self.load(value)?; + self.copy_value(self.body.value_type(value))?; + } Op::Opaque(value) => self.load(value)?, Op::Project { base, projection } => { self.load(base)?; @@ -140,20 +108,37 @@ impl Selector<'_> { )?; self.assembly.code.push(Instruction::Invokestatic(method)); } - Op::AddressOfSlot(slot) => self - .assembly - .code - .push(Kind::Reference.load(self.storage[slot.index()])), + Op::AddressOfSlot(slot) => { + self.assembly + .code + .push(Kind::Reference.load(self.storage[slot.index()])); + let ty = self.body.slots[slot.index()].ty; + if StorageSlot::scalar(ty, self.types).is_some() { + self.assembly.code.push(Instruction::Lconst_0); + self.materialize_address(ty)?; + } + } Op::LoadSlot(slot) => { self.assembly .code .push(Kind::Reference.load(self.storage[slot.index()])); - self.read_memory(self.body.slots[slot.index()].ty)?; + let ty = self.body.slots[slot.index()].ty; + if StorageSlot::scalar(ty, self.types).is_some() { + self.assembly.code.push(Instruction::Lconst_0); + self.read_address(ty)?; + } else { + self.read_memory(ty)?; + } } Op::StoreSlot { slot, value } => { self.assembly .code .push(Kind::Reference.load(self.storage[slot.index()])); + let storage = &self.body.slots[slot.index()]; + let scalar = StorageSlot::scalar(storage.ty, self.types).is_some(); + if scalar { + self.assembly.code.push(Instruction::Lconst_0); + } self.argument(value)?; let storage = &self.body.slots[slot.index()]; if storage.size == 0 { @@ -164,10 +149,23 @@ impl Selector<'_> { "(Ljava/lang/Object;)V", )?; self.assembly.code.push(Instruction::Invokevirtual(init)); + } else if scalar { + self.write_address(storage.ty)?; } else { self.write_memory(storage.ty)?; } } + Op::Commit(pointer) => { + self.load(pointer)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref(owner, "commitMemoryView", "()V")?; + self.assembly.code.push(Instruction::Invokevirtual(method)); + } + Op::LoadCopy(pointer) => { + self.load(pointer)?; + self.assembly.code.push(Instruction::Lconst_0); + self.read_object_address(self.body.value_type(inst.result.unwrap()), true)?; + } Op::Load(pointer) => { self.load(pointer)?; self.read_memory(self.body.value_type(inst.result.unwrap()))?; @@ -193,13 +191,9 @@ impl Selector<'_> { else { return Err(error("unsupported reference cast")); }; - let to = scalar(self.types, pointee)?; - let bytes = match to { - ScalarType::Bool => 1, - ScalarType::F32 => 4, - ScalarType::F64 => 8, - _ => to.integer().ok_or_else(|| error("pointer view type"))?.0 / 8, - }; + let bytes = StorageSlot::scalar(pointee, self.types) + .ok_or_else(|| error("pointer view type"))? + .size; self.load(value)?; self.assembly.code.extend([ get_int_const_instr(self.cp, bytes as i32), @@ -218,8 +212,100 @@ impl Selector<'_> { Ok(true) } + pub(super) fn copy_value(&mut self, ty: TypeId) -> jvm::Result<()> { + if let Some(Type::Class(symbol)) = self.types.get(ty) + && let Some(name) = self.types.symbol_name(symbol) + && self.direct_copy.is_some_and(|copy| copy(name)) + { + let class = self.cp.add_class(name)?; + let method = self + .cp + .add_method_ref(class, "rustCopy", "()Ljava/lang/Object;")?; + let done = self.assembly.label(); + self.assembly.code.push(Instruction::Dup); + self.assembly.branch(Instruction::Ifnull(0), done); + self.assembly.code.extend([ + Instruction::Invokevirtual(method), + Instruction::Checkcast(class), + ]); + self.assembly.bind(done); + return Ok(()); + } + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "copyManagedValue", + "(Ljava/lang/Object;)Ljava/lang/Object;", + )?; + let mut name = String::new(); + descriptor(self.types, ty, &mut name)?; + let name = name + .strip_prefix('L') + .and_then(|s| s.strip_suffix(';')) + .unwrap_or(&name); + let class = self.cp.add_class(name)?; + self.assembly.code.extend([ + Instruction::Invokestatic(method), + Instruction::Checkcast(class), + ]); + Ok(()) + } + /// Project the pointer already on the operand stack. pub(super) fn project_field(&mut self, projection: ProjectionId) -> jvm::Result<()> { + if let Some(bootstrap) = &mut self.bootstrap { + let projection = &self.body.projections[projection.index()]; + let field = &self.body.fields[projection.field.index()]; + let Some(Type::Class(symbol)) = self.types.get(field.owner) else { + return Err(error("projection requires class layout")); + }; + let owner = self.cp.add_class("org/rustlang/runtime/FieldProjections")?; + let method = self.cp.add_method_ref( + owner, + "project", + concat!( + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;", + "Ljava/lang/invoke/MethodType;Ljava/lang/Class;Ljava/lang/String;JJ", + "Ljava/lang/String;)Ljava/lang/invoke/CallSite;" + ), + )?; + let bootstrap_method_ref = self + .cp + .add_method_handle(jvm::ReferenceKind::InvokeStatic, method)?; + let arguments = [ + self.cp.add_class(self.types.symbol_name(symbol).unwrap())?, + self.cp.add_string(&field.name)?, + self.cp.add_long(projection.offset as i64)?, + self.cp.add_long(projection.size as i64)?, + self.cp + .add_name_string(projection.codec.as_deref().unwrap_or(""))?, + ]; + if let Some(&site) = self.projection_sites.get(&arguments) { + self.assembly.code.push(Instruction::Invokedynamic(site)); + return Ok(()); + } + let index = u16::try_from(bootstrap.len())?; + bootstrap.push(jvm::attributes::BootstrapMethod { + bootstrap_method_ref, + arguments: arguments.to_vec(), + }); + let site = self.cp.add_invoke_dynamic( + index, + "project", + "(Lorg/rustlang/runtime/Pointer;)Lorg/rustlang/runtime/Pointer;", + )?; + self.projection_sites.insert(arguments, site); + self.assembly.code.push(Instruction::Invokedynamic(site)); + return Ok(()); + } + self.project_field_arguments(projection)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref(owner, "projectStructField", "(Ljava/lang/String;Ljava/lang/String;JJLjava/lang/String;)Lorg/rustlang/runtime/Pointer;")?; + self.assembly.code.push(Instruction::Invokevirtual(method)); + Ok(()) + } + + pub(super) fn project_field_arguments(&mut self, projection: ProjectionId) -> jvm::Result<()> { let projection = &self.body.projections[projection.index()]; let field = &self.body.fields[projection.field.index()]; let Some(Type::Class(symbol)) = self.types.get(field.owner) else { @@ -230,7 +316,7 @@ impl Selector<'_> { self.cp .add_name_string(self.types.symbol_name(symbol).unwrap())?, ), - Instruction::Ldc_w(self.cp.add_name_string(&field.name)?), + Instruction::Ldc_w(self.cp.add_string(&field.name)?), get_long_const_instr(self.cp, projection.offset as i64), get_long_const_instr(self.cp, projection.size as i64), match &projection.codec { @@ -238,12 +324,23 @@ impl Selector<'_> { None => Instruction::Aconst_null, }, ]); - let owner = self.cp.add_class(POINTER_CLASS)?; - let method = self.cp.add_method_ref(owner, "projectStructField", "(Ljava/lang/String;Ljava/lang/String;JJLjava/lang/String;)Lorg/rustlang/runtime/Pointer;")?; - self.assembly.code.push(Instruction::Invokevirtual(method)); Ok(()) } pub(super) fn read_memory(&mut self, ty: TypeId) -> jvm::Result<()> { + if matches!(self.types.get(ty), Some(Type::Array(_))) { + let mut descriptor = String::new(); + representation::descriptor(self.types, ty, &mut descriptor)?; + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self + .cp + .add_method_ref(owner, "getObject", "()Ljava/lang/Object;")?; + let class = self.cp.add_class(descriptor)?; + self.assembly.code.extend([ + Instruction::Invokevirtual(method), + Instruction::Checkcast(class), + ]); + return Ok(()); + } let class = match self.types.get(ty) { Some(Type::Class(symbol) | Type::Interface(symbol)) => { Some(self.types.symbol_name(symbol).unwrap()) @@ -251,13 +348,20 @@ impl Selector<'_> { Some(Type::Pointer(_)) => Some(POINTER_CLASS), Some(Type::Slice(_)) => Some(representation::SLICE_VIEW_CLASS), Some(Type::Str) => Some(representation::UTF8_VIEW_CLASS), + Some(Type::TaggedI64) => Some(super::super::abi::TAGGED_LONG_CLASS), _ => None, }; if let Some(class) = class { let owner = self.cp.add_class(POINTER_CLASS)?; let target = if matches!( self.types.get(ty), - Some(Type::Class(_) | Type::Interface(_) | Type::Slice(_) | Type::Str) + Some( + Type::Class(_) + | Type::Interface(_) + | Type::Slice(_) + | Type::Str + | Type::TaggedI64 + ) ) { self.assembly .code @@ -282,7 +386,7 @@ impl Selector<'_> { let (name, result) = match scalar(self.types, ty)? { Bool => ("getBoolean", "Z"), I8 | U8 => ("getI8", "B"), - I16 | U16 => ("getI16", "S"), + I16 | U16 | F16 => ("getI16", "S"), I32 | U32 => ("getI32", "I"), I64 | U64 => ("getI64", "J"), F32 => ("getF32", "F"), @@ -301,7 +405,13 @@ impl Selector<'_> { if matches!( self.types.get(ty), Some( - Type::Class(_) | Type::Interface(_) | Type::Pointer(_) | Type::Slice(_) | Type::Str + Type::Class(_) + | Type::Interface(_) + | Type::Pointer(_) + | Type::Slice(_) + | Type::Str + | Type::TaggedI64 + | Type::Array(_) ) ) { signature.push_str("Ljava/lang/Object;"); diff --git a/compiler-core/src/jvm/select/mod.rs b/compiler-core/src/jvm/select/mod.rs index fa9ad141..a43dce2b 100644 --- a/compiler-core/src/jvm/select/mod.rs +++ b/compiler-core/src/jvm/select/mod.rs @@ -1,4 +1,5 @@ //! Direct selection from compact SSA with interval-based JVM local reuse. +mod addresses; mod allocate; mod arrays; mod assemble; @@ -90,8 +91,8 @@ pub struct Options<'a> { /// verify; production selection otherwise consumes trusted internal IR. pub verify: bool, pub lines: Option<&'a SourceLines>, - pub relative_pointer_abi: bool, pub constants: Option<&'a dyn Constants>, + pub direct_copy: Option<&'a dyn Fn(&str) -> bool>, pub debug: Option<&'a DebugInfo>, pub bootstrap: Option<&'a mut Vec>, } @@ -105,6 +106,8 @@ struct Selector<'a> { slots: Vec>, forwarded: Vec, stack_value: Option, + branch_condition: Option, + pending_branch: Option, next_slot: u16, scratch_used: bool, exception_slot: Option, @@ -112,8 +115,13 @@ struct Selector<'a> { handlers: Vec<(Label, BlockId)>, copies: Vec<(u16, u16, Kind)>, live: crate::opt::Live, + native_array_accesses: Vec>, storage: Vec, + aggregate_cache: Option<(ValueId, Option, TypeId)>, + aggregate_slot: Option, + projection_sites: rustc_hash::FxHashMap<[u16; 5], u16>, constants: Option<&'a dyn Constants>, + direct_copy: Option<&'a dyn Fn(&str) -> bool>, bootstrap: Option<&'a mut Vec>, } @@ -135,8 +143,8 @@ pub fn compile_with_options( let Options { verify, lines, - relative_pointer_abi, constants, + direct_copy, debug, bootstrap, } = options; @@ -154,15 +162,8 @@ pub fn compile_with_options( crate::opt::live_with_roots(body, types, debug.into_iter().flat_map(|d| d.roots(body))); let order = body.layout(); let forwarded = forward::values(body, &live, debug); - let allocation = allocate::allocate( - body, - types, - &live, - &forwarded, - relative_pointer_abi, - debug, - &order, - )?; + let allocation = allocate::allocate(body, types, &live, &forwarded, debug, &order)?; + let native_array_accesses = crate::analysis::native_array_accesses(body, types, &live); let mut debug = debug.map(|d| debug::Debug::new(d, body)); let mut s = Selector { body, @@ -173,6 +174,8 @@ pub fn compile_with_options( slots: allocation.slots, forwarded, stack_value: None, + branch_condition: None, + pending_branch: None, next_slot: allocation.count, scratch_used: false, exception_slot: None, @@ -180,8 +183,13 @@ pub fn compile_with_options( handlers: Vec::new(), copies: Vec::new(), live, + native_array_accesses, storage: Vec::new(), + aggregate_cache: None, + aggregate_slot: None, + projection_sites: Default::default(), constants, + direct_copy, bootstrap, }; for _ in &body.blocks { @@ -207,12 +215,6 @@ pub fn compile_with_options( for ¶m in &body.blocks[body.entry.index()].params { let value = s.initial_value(param)?; frames::push_local_value(&mut initial, value); - if relative_pointer_abi - && matches!(types.get(body.value_type(param)), Some(Type::Pointer(_))) - { - frames::push_local_value(&mut initial, frames::FrameValue::Long); - frames::push_local_value(&mut initial, frames::FrameValue::Long); - } } if body.blocks.iter().any(|b| { matches!( @@ -233,16 +235,32 @@ pub fn compile_with_options( .checked_add(1) .ok_or_else(|| error("JVM local limit"))?; } - for block in order { + for (position, &block) in order.iter().enumerate() { + s.aggregate_cache = None; + s.branch_condition = match body.blocks[block.index()].terminator.unwrap() { + Terminator::Branch { condition, .. } => Some(body.resolve(condition)), + Terminator::Switch { value, cases, .. } + if cases.len == 1 + && types.get(body.value_type(value)) + == Some(Type::Scalar(ScalarType::Bool)) => + { + Some(body.resolve(value)) + } + _ => None, + }; + // Keep loop headers after method entry. Frame-offset conversion reserves + // offset zero for the implicit entry frame. Removing an entry jump must + // not create a branch target at zero. + if s.assembly.code.is_empty() && body.edges.iter().any(|edge| edge.target == block) { + s.assembly.code.push(Instruction::Nop); + } s.assembly.bind(s.blocks[block.index()]); if block == body.entry { s.initialize_storage()?; if let Some(debug) = &mut debug { debug.allocate(&mut s)?; } - if relative_pointer_abi { - s.materialize_parameters()?; - } + // JVM byte/short parameters are sign-extended by Java callers. for ¶m in &body.blocks[block.index()].params { if s.live.uses[param.index()] > 1 @@ -272,6 +290,7 @@ pub fn compile_with_options( .is_some_and(|event| event.position as usize == position) { let event = events.next().unwrap(); + s.aggregate_cache = None; let start = s.assembly.code.len(); debug.as_mut().unwrap().apply(&mut s, event.change)?; if lines.is_some() { @@ -295,8 +314,12 @@ pub fn compile_with_options( } } let start = s.assembly.code.len(); - s.terminator(body.blocks[block.index()].terminator.unwrap())?; + s.terminator( + body.blocks[block.index()].terminator.unwrap(), + order.get(position + 1).copied(), + )?; debug_assert!(s.stack_value.is_none()); + debug_assert!(s.pending_branch.is_none()); if let Some(debug) = &mut debug { debug.mark(start, s.assembly.code.len()); } @@ -434,29 +457,28 @@ impl Selector<'_> { Ok(()) } fn jump(&mut self, edge: EdgeId) -> jvm::Result<()> { + self.jump_to(edge, None) + } + fn jump_to(&mut self, edge: EdgeId, fallthrough: Option) -> jvm::Result<()> { self.copies(edge)?; - self.assembly.branch( - Instruction::Goto_w(0), - self.blocks[self.body.edges[edge.index()].target.index()], - ); + let target = self.body.edges[edge.index()].target; + if Some(target) != fallthrough { + self.assembly + .branch(Instruction::Goto_w(0), self.blocks[target.index()]); + } Ok(()) } - fn terminator(&mut self, term: Terminator) -> jvm::Result<()> { + fn terminator(&mut self, term: Terminator, fallthrough: Option) -> jvm::Result<()> { match term { - Terminator::Jump(edge) => self.jump(edge)?, + Terminator::Jump(edge) => self.jump_to(edge, fallthrough)?, Terminator::Branch { condition, yes, no } => { - let yes_label = self.assembly.label(); - self.load(condition)?; - self.assembly.branch(Instruction::Ifne(0), yes_label); - self.jump(no)?; - self.assembly.bind(yes_label); - self.jump(yes)?; + self.branch(condition, yes, no, fallthrough)?; } Terminator::Switch { value, cases, otherwise, - } => self.switch(value, cases, otherwise)?, + } => self.switch(value, cases, otherwise, fallthrough)?, Terminator::Return(Some(value)) => { self.argument(value)?; self.assembly.code.push(self.value_kind(value)?.return_op()); @@ -474,7 +496,7 @@ impl Selector<'_> { self.instruction(inst)?; let end = u16::try_from(self.assembly.code.len())?; self.protect(start, end, unwind); - self.jump(normal)?; + self.jump_to(normal, fallthrough)?; } Terminator::Rethrow => { self.assembly.code.extend([ diff --git a/compiler-core/src/jvm/select/object_tests.rs b/compiler-core/src/jvm/select/object_tests.rs index 25e85acc..cf3ba113 100644 --- a/compiler-core/src/jvm/select/object_tests.rs +++ b/compiler-core/src/jvm/select/object_tests.rs @@ -61,7 +61,6 @@ fn jvm_executes_constructors_fields_and_virtual_interface_calls() { name: "narrow".into(), ty: int, is_static: false, - relative_pointer: false, }); let old = b .emit( @@ -237,9 +236,9 @@ fn field_projection_verifies_both_ends_of_the_pointer_view() { name: "value".into(), ty: int, is_static: false, - relative_pointer: false, }); let projection = b.projection(PointerProjection { + parent: None, field, offset: 8, size: 4, @@ -277,3 +276,82 @@ fn field_projection_verifies_both_ends_of_the_pointer_view() { .contains("address space") ); } + +#[test] +fn exact_managed_copies_keep_null_and_unknown_types_on_their_original_paths() { + let mut types = Types::default(); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let mut b = Builder::new(&types, pair); + let source = b.parameter(b.current(), pair); + let copied = b.emit(Op::CopyValue(source), Some(pair)).unwrap(); + b.terminate(Terminator::Return(Some(copied))); + let body = b.finish().unwrap(); + for known in [false, true] { + let mut pool = Default::default(); + let code = compile_with_options( + &body, + &types, + &mut pool, + Options { + direct_copy: Some(&|owner| known && owner == "Pair"), + ..Default::default() + }, + ) + .unwrap(); + let owner = pool.add_class("Pair").unwrap(); + let copy = pool + .add_method_ref(owner, "rustCopy", "()Ljava/lang/Object;") + .unwrap(); + assert_eq!( + code.instructions + .contains(&Instruction::Invokevirtual(copy)), + known + ); + assert_eq!( + code.instructions + .iter() + .any(|i| matches!(i, Instruction::Ifnull(_))), + known + ); + } +} + +#[test] +fn owned_array_reads_keep_separate_typed_sites() { + let mut types = Types::default(); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let array_type = types.intern(Type::Array(pair)); + let int = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, pair); + let array = b.parameter(b.current(), array_type); + let index = b.parameter(b.current(), int); + b.emit(Op::ArrayGetCopy { array, index }, Some(pair)); + let result = b + .emit(Op::ArrayGetCopy { array, index }, Some(pair)) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let body = b.finish().unwrap(); + for dynamic in [false, true] { + let mut pool = Default::default(); + let mut bootstrap = Vec::new(); + let code = compile_with_options( + &body, + &types, + &mut pool, + Options { + bootstrap: dynamic.then_some(&mut bootstrap), + ..Default::default() + }, + ) + .unwrap(); + let sites = code + .instructions + .iter() + .filter(|i| matches!(i, Instruction::Invokedynamic(_))) + .count(); + assert_eq!(sites, if dynamic { 2 } else { 0 }); + assert_eq!(bootstrap.len(), if dynamic { 2 } else { 0 }); + } +} diff --git a/compiler-core/src/jvm/select/objects.rs b/compiler-core/src/jvm/select/objects.rs index a0114a7e..993dd137 100644 --- a/compiler-core/src/jvm/select/objects.rs +++ b/compiler-core/src/jvm/select/objects.rs @@ -1,11 +1,8 @@ -use super::super::abi::{ - relative_pointer_byte_offset_field, relative_pointer_element_offset_field, -}; use super::*; impl Selector<'_> { pub(super) fn object(&mut self, inst: Inst) -> jvm::Result { - if let Op::Cast(value) = inst.op { + if let Op::Cast(value) | Op::Refine(value) = inst.op { if self.value_kind(value)? == Kind::Reference && self.value_kind(inst.result.unwrap())? == Kind::Reference { @@ -63,38 +60,6 @@ impl Selector<'_> { _ => unreachable!(), }; self.assembly.code.push(op); - if field.relative_pointer { - let (object, store) = match inst.op { - Op::GetField { object, .. } => (object, false), - Op::SetField { object, .. } => (object, true), - _ => return Err(error("relative pointer fields require an instance")), - }; - for name in [ - relative_pointer_element_offset_field(&field.name), - relative_pointer_byte_offset_field(&field.name), - ] { - let offset = self.cp.add_field_ref(class, name, "J")?; - self.load(object)?; - if store { - self.assembly - .code - .extend([Instruction::Lconst_0, Instruction::Putfield(offset)]); - } else { - self.assembly.code.push(Instruction::Getfield(offset)); - } - } - if !store { - let owner = self.cp.add_class(POINTER_CLASS)?; - let materialize = self.cp.add_method_ref( - owner, - "materializeRelative", - "(Lorg/rustlang/runtime/Pointer;JJ)Lorg/rustlang/runtime/Pointer;", - )?; - self.assembly - .code - .push(Instruction::Invokestatic(materialize)); - } - } Ok(true) } } diff --git a/compiler-core/src/jvm/select/representation.rs b/compiler-core/src/jvm/select/representation.rs index 98f6106d..8e0e2945 100644 --- a/compiler-core/src/jvm/select/representation.rs +++ b/compiler-core/src/jvm/select/representation.rs @@ -1,6 +1,6 @@ use super::*; -pub use super::super::abi::{POINTER_CLASS, SLICE_VIEW_CLASS, UTF8_VIEW_CLASS}; +pub use super::super::abi::{POINTER_CLASS, SLICE_VIEW_CLASS, TAGGED_LONG_CLASS, UTF8_VIEW_CLASS}; pub(super) fn value_kind(types: &Types, ty: TypeId) -> jvm::Result { match types.get(ty) { @@ -11,7 +11,8 @@ pub(super) fn value_kind(types: &Types, ty: TypeId) -> jvm::Result { | Type::Interface(_) | Type::Array(_) | Type::Slice(_) - | Type::Str, + | Type::Str + | Type::TaggedI64, ) => Ok(Kind::Reference), _ => Err(error("unsupported JVM value representation")), } @@ -20,11 +21,12 @@ pub(super) fn value_kind(types: &Types, ty: TypeId) -> jvm::Result { pub(super) fn descriptor(types: &Types, ty: TypeId, output: &mut String) -> jvm::Result<()> { use ScalarType::*; match types.get(ty) { - Some(t @ (Type::Pointer(_) | Type::Slice(_) | Type::Str)) => { + Some(t @ (Type::Pointer(_) | Type::Slice(_) | Type::Str | Type::TaggedI64)) => { output.push('L'); output.push_str(match t { Type::Slice(_) => SLICE_VIEW_CLASS, Type::Str => UTF8_VIEW_CLASS, + Type::TaggedI64 => TAGGED_LONG_CLASS, _ => POINTER_CLASS, }); output.push(';'); diff --git a/compiler-core/src/jvm/select/scalar.rs b/compiler-core/src/jvm/select/scalar.rs index 22cf75bc..e94d5c10 100644 --- a/compiler-core/src/jvm/select/scalar.rs +++ b/compiler-core/src/jvm/select/scalar.rs @@ -3,6 +3,13 @@ use super::*; impl Selector<'_> { pub(super) fn instruction(&mut self, id: InstId) -> jvm::Result<()> { let inst = self.body.instructions[id.index()]; + if !matches!( + inst.op, + Op::LoadFieldPart { .. } | Op::LoadField { .. } | Op::LoadStorageField { .. } + ) && inst.op.may_throw(self.body, self.types) + { + self.aggregate_cache = None; + } if inst.result.is_some_and(|v| literal(self.body, v).is_some()) { return Ok(()); } @@ -25,7 +32,8 @@ impl Selector<'_> { return Ok(()); } if self.general(inst)? - || self.array(inst)? + || self.array(id, inst)? + || self.address(inst)? || self.memory(inst)? || self.object(inst)? || self.view(inst)? @@ -50,6 +58,10 @@ impl Selector<'_> { self.load(right)?; let left_ty = self.scalar_type(left)?; if op.is_comparison() { + if self.forwarded[result.index()] && self.branch_condition == Some(result) { + self.pending_branch = Some(self.comparison_branch(op, left_ty)?); + return Ok(()); + } self.comparison(op, left_ty)?; } else { self.binary(op, left_ty, self.scalar_type(right)?)?; @@ -149,9 +161,13 @@ impl Selector<'_> { self.assembly.code.push(I::L2i); } let (width, signed) = ty.integer().ok_or_else(|| error("non-integer shift"))?; - self.assembly - .code - .extend([get_int_const_instr(self.cp, (width - 1) as i32), I::Iand]); + // JVM int/long shifts already mask to 5/6 bits. Only narrower + // Rust integer widths require a stricter mask. + if width < 32 { + self.assembly + .code + .extend([get_int_const_instr(self.cp, (width - 1) as i32), I::Iand]); + } self.assembly.code.push(match (op, kind, signed) { (Shl, Kind::Int, _) => I::Ishl, (Shl, Kind::Long, _) => I::Lshl, @@ -206,7 +222,7 @@ impl Selector<'_> { )); Ok(()) } - pub(super) fn comparison(&mut self, op: BinaryOp, ty: ScalarType) -> jvm::Result<()> { + fn comparison_branch(&mut self, op: BinaryOp, ty: ScalarType) -> jvm::Result { use BinaryOp::*; use Instruction as I; let direct = kind(ty)? == Kind::Int && ty != ScalarType::U32; @@ -234,9 +250,7 @@ impl Selector<'_> { }); } } - let yes = self.assembly.label(); - let end = self.assembly.label(); - let branch = match (op, direct) { + Ok(match (op, direct) { (Eq, true) => I::If_icmpeq(0), (Ne, true) => I::If_icmpne(0), (Lt, true) => I::If_icmplt(0), @@ -250,7 +264,14 @@ impl Selector<'_> { (Gt, false) => I::Ifgt(0), (Ge, false) => I::Ifge(0), _ => unreachable!(), - }; + }) + } + + pub(super) fn comparison(&mut self, op: BinaryOp, ty: ScalarType) -> jvm::Result<()> { + use Instruction as I; + let branch = self.comparison_branch(op, ty)?; + let yes = self.assembly.label(); + let end = self.assembly.label(); self.assembly.branch(branch, yes); self.assembly.code.push(I::Iconst_0); self.assembly.branch(I::Goto_w(0), end); diff --git a/compiler-core/src/jvm/select/switches.rs b/compiler-core/src/jvm/select/switches.rs index 86fadec4..f0c161f5 100644 --- a/compiler-core/src/jvm/select/switches.rs +++ b/compiler-core/src/jvm/select/switches.rs @@ -1,17 +1,62 @@ use super::*; impl Selector<'_> { + pub(super) fn branch( + &mut self, + condition: ValueId, + yes: EdgeId, + no: EdgeId, + fallthrough: Option, + ) -> jvm::Result<()> { + // The first edge must skip the last edge's parameter copies. + let (invert, first, last) = if Some(self.body.edges[no.index()].target) == fallthrough { + (true, yes, no) + } else { + (false, no, yes) + }; + let last_label = self.assembly.label(); + let branch = if let Some(comparison) = self.pending_branch.take() { + if invert { + crate::jvm::flow::invert_conditional_branch(&comparison, 0).unwrap() + } else { + comparison + } + } else { + self.load(condition)?; + if invert { + Instruction::Ifeq(0) + } else { + Instruction::Ifne(0) + } + }; + self.assembly.branch(branch, last_label); + self.jump(first)?; + self.assembly.bind(last_label); + self.jump_to(last, fallthrough) + } + pub(super) fn switch( &mut self, value: ValueId, list: List, otherwise: EdgeId, + fallthrough: Option, ) -> jvm::Result<()> { let cases = &self.body.cases[list.range()]; if cases.is_empty() { return self.jump(otherwise); } let ty = self.scalar_type(value)?; + if let [(key, edge)] = cases + && ty == ScalarType::Bool + { + let (yes, no) = if key.bits() == 0 { + (otherwise, *edge) + } else { + (*edge, otherwise) + }; + return self.branch(value, yes, no, fallthrough); + } let labels: Vec<_> = cases.iter().map(|_| self.assembly.label()).collect(); if kind(ty)? == Kind::Int && cases.len() >= 3 { self.load(value)?; diff --git a/compiler-core/src/jvm/select/tests.rs b/compiler-core/src/jvm/select/tests.rs index 5ed1d3d0..378944e9 100644 --- a/compiler-core/src/jvm/select/tests.rs +++ b/compiler-core/src/jvm/select/tests.rs @@ -25,6 +25,117 @@ fn binary_body(types: &Types, ty: TypeId, ret: TypeId, op: BinaryOp) -> Body { b.finish().unwrap() } +fn comparison_branch( + types: &Types, + ty: TypeId, + boolean: TypeId, + op: BinaryOp, + switch: bool, +) -> Body { + let mut b = Builder::new(types, boolean); + let left = b.parameter(b.current(), ty); + let right = b.parameter(b.current(), ty); + let value = b + .emit(Op::Binary { op, left, right }, Some(boolean)) + .unwrap(); + let yes = b.create_block(); + let no = b.create_block(); + let yes_edge = b.edge(yes, vec![]); + let no_edge = b.edge(no, vec![]); + b.body.cases.push((Scalar::boolean(false), no_edge)); + b.terminate(if switch { + Terminator::Switch { + value, + cases: List { start: 0, len: 1 }, + otherwise: yes_edge, + } + } else { + Terminator::Branch { + condition: value, + yes: yes_edge, + no: no_edge, + } + }); + for (block, value) in [(yes, true), (no, false)] { + b.switch_to(block); + let result = b.constant(boolean, Scalar::boolean(value)); + b.terminate(Terminator::Return(Some(result))); + } + b.finish().unwrap() +} + +fn branch_arguments(types: &Types, int: TypeId, boolean: TypeId) -> Body { + let mut b = Builder::new(types, int); + let condition = b.parameter(b.current(), boolean); + let left = b.parameter(b.current(), int); + let right = b.parameter(b.current(), int); + let join = b.create_block(); + let value = b.parameter(join, int); + let yes = b.edge(join, vec![left]); + let no = b.edge(join, vec![right]); + b.terminate(Terminator::Branch { condition, yes, no }); + b.switch_to(join); + b.terminate(Terminator::Return(Some(value))); + b.finish().unwrap() +} + +fn entry_loop(types: &Types, int: TypeId, boolean: TypeId, array: TypeId) -> Body { + let mut b = Builder::new(types, int); + let count = b.parameter(b.current(), array); + let header = b.create_block(); + let update = b.create_block(); + let exit = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let zero = integer(&mut b, int, ScalarType::I32, 0); + let n = b + .emit( + Op::ArrayGet { + array: count, + index: zero, + native: true, + }, + Some(int), + ) + .unwrap(); + let condition = b + .emit( + Op::Binary { + op: BinaryOp::Gt, + left: n, + right: zero, + }, + Some(boolean), + ) + .unwrap(); + b.branch(condition, update, exit); + b.switch_to(update); + let one = integer(&mut b, int, ScalarType::I32, 1); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Sub, + left: n, + right: one, + }, + Some(int), + ) + .unwrap(); + b.emit( + Op::ArraySet { + array: count, + index: zero, + value: next, + native: true, + }, + None, + ); + b.jump(header, vec![]); + b.switch_to(exit); + b.terminate(Terminator::Return(Some(n))); + b.finish().unwrap() +} + fn swap_loop( types: &Types, ty: TypeId, @@ -267,8 +378,24 @@ fn jvm_verifies_and_executes_ssa_loops_parallel_copies_scalars_and_throw_points( let byte = types.scalar(ScalarType::U8); let short = types.scalar(ScalarType::I16); let signed_byte = types.scalar(ScalarType::I8); + let int_array = types.intern(Type::Array(int)); use crate::scalar::BitOp; let methods = [ + ( + "branchLtFloat", + "(FF)Z", + comparison_branch(&types, float, boolean, BinaryOp::Lt, false), + ), + ( + "switchGeFloat", + "(FF)Z", + comparison_branch(&types, float, boolean, BinaryOp::Ge, true), + ), + ( + "branchLtUnsigned", + "(JJ)Z", + comparison_branch(&types, ulong, boolean, BinaryOp::Lt, false), + ), ( "popByte", "(B)I", @@ -356,6 +483,16 @@ fn jvm_verifies_and_executes_ssa_loops_parallel_copies_scalars_and_throw_points( overflow_body(&types, ulong, boolean, BinaryOp::Mul), ), ("callHandler", "(II)I", call_with_handler(&types, int)), + ( + "entryLoop", + "([I)I", + entry_loop(&types, int, boolean, int_array), + ), + ( + "branchArguments", + "(ZII)I", + branch_arguments(&types, int, boolean), + ), ( "tableSwitch", "(I)I", @@ -402,6 +539,21 @@ fn jvm_verifies_and_executes_ssa_loops_parallel_copies_scalars_and_throw_points( "(SS)S", binary_body(&types, short, short, BinaryOp::Shr), ), + ( + "shiftInt", + "(II)I", + binary_body(&types, int, int, BinaryOp::Shl), + ), + ( + "shiftLong", + "(JJ)J", + binary_body(&types, long, long, BinaryOp::Shr), + ), + ( + "shiftUnsignedLong", + "(JJ)J", + binary_body(&types, ulong, ulong, BinaryOp::Shr), + ), ( "addByte", "(BB)B", @@ -455,6 +607,21 @@ fn jvm_verifies_and_executes_ssa_loops_parallel_copies_scalars_and_throw_points( .into_iter() .map(|(name, descriptor, body)| { let code = compile(&body, &types, &mut cp).unwrap(); + if name == "entryLoop" { + assert!( + code.instructions + .iter() + .any(|inst| matches!(inst, Instruction::Goto_w(1))) + ); + } + if matches!(name, "swap" | "swapWide" | "branchArguments") { + assert!(!code.instructions.iter().enumerate().any(|(i, inst)| { + matches!(inst, Instruction::Goto_w(target) if *target as usize == i + 1) + })); + } + if matches!(name, "shiftInt" | "shiftLong" | "shiftUnsignedLong") { + assert!(!code.instructions.contains(&Instruction::Iand)); + } Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, name_index: cp.add_utf8(name).unwrap(), @@ -520,6 +687,9 @@ public class Run { } long[] boundaries = {Long.MIN_VALUE, Long.MIN_VALUE+1, -3037000500L, -1, 0, 1, 3037000500L, Long.MAX_VALUE-1, Long.MAX_VALUE}; for (long a : boundaries) for (long b : boundaries) { + check(SsaFixture.shiftInt((int)a, (int)b) == ((int)a << (int)b)); + check(SsaFixture.shiftLong(a, b) == (a >> b)); + check(SsaFixture.shiftUnsignedLong(a, b) == (a >>> b)); boolean add=false, sub=false, mul=false; try { Math.addExact(a,b); } catch (ArithmeticException ex) { add=true; } try { Math.subtractExact(a,b); } catch (ArithmeticException ex) { sub=true; } @@ -537,6 +707,11 @@ public class Run { } check(SsaFixture.callHandler(-7, 2)==-4); check(SsaFixture.callHandler(123, 0)==123); + check(SsaFixture.branchArguments(true, 11, 29)==11); + check(SsaFixture.branchArguments(false, 11, 29)==29); + check(SsaFixture.entryLoop(new int[]{100})==0); + check(SsaFixture.entryLoop(new int[]{0})==0); + check(SsaFixture.entryLoop(new int[]{-1})==-1); for (int n=-100; n<100; n++) check(SsaFixture.tableSwitch(n)==(n>=-2 && n<=2 ? n+3 : -1)); check(SsaFixture.lookupSwitch(Integer.MIN_VALUE)==1); check(SsaFixture.lookupSwitch(0)==2); check(SsaFixture.lookupSwitch(Integer.MAX_VALUE)==3); @@ -556,6 +731,13 @@ public class Run { check(SsaFixture.divUnsigned(-1, 2)==Integer.divideUnsigned(-1,2)); check(!SsaFixture.ltUnsigned(-1, 0)); check(SsaFixture.ltUnsigned(0,-1)); check(!SsaFixture.ltFloat(Float.NaN, 1)); check(!SsaFixture.ltFloat(1,Float.NaN)); + float[] floats = {Float.NaN, Float.NEGATIVE_INFINITY, -1, -0.0f, 0.0f, 1, Float.POSITIVE_INFINITY}; + for (float a : floats) for (float b : floats) { + check(SsaFixture.branchLtFloat(a,b) == (a < b)); + check(SsaFixture.switchGeFloat(a,b) == (a >= b)); + } + for (long a : boundaries) for (long b : boundaries) + check(SsaFixture.branchLtUnsigned(a,b) == (Long.compareUnsigned(a,b) < 0)); check(!SsaFixture.geFloat(Float.NaN, 1)); check(SsaFixture.geFloat(-0.0f,0.0f)); check(SsaFixture.eqDouble(-0.0,0.0)); check(!SsaFixture.eqDouble(Double.NaN,Double.NaN)); check(Double.doubleToRawLongBits(SsaFixture.divDouble(-0.0,1.0))==Long.MIN_VALUE); @@ -582,6 +764,29 @@ public class Run { fs::remove_dir_all(directory).unwrap(); } +#[test] +fn empty_blocks_fall_through_without_emitting_branches() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, int); + let value = integer(&mut b, int, ScalarType::I32, 7); + for _ in 0..4 { + let block = b.create_block(); + b.jump(block, vec![]); + b.switch_to(block); + } + b.terminate(Terminator::Return(Some(value))); + let code = compile(&b.finish().unwrap(), &types, &mut Default::default()).unwrap(); + assert_eq!( + code.instructions, + vec![ + Instruction::Nop, + Instruction::Bipush(7), + Instruction::Ireturn + ] + ); +} + #[test] fn source_line_tables_follow_selected_instructions_and_terminators() { let mut types = Types::default(); @@ -697,9 +902,11 @@ fn representation_constants_and_opaque_values_retain_ordered_effects() { let int = types.scalar(ScalarType::I32); let unit = types.intern(Type::Unit); let mut b = Builder::new(&types, unit); - b.body - .constants - .push(Constant::External { index: 7, ty: int }); + b.body.constants.push(Constant::External { + index: 7, + ty: int, + pure: false, + }); let value = b.emit(Op::Constant(ConstId::new(0)), Some(int)).unwrap(); b.emit(Op::Opaque(value), Some(int)); b.terminate(Terminator::Return(None)); diff --git a/compiler-core/src/jvm/select/unwind_tests.rs b/compiler-core/src/jvm/select/unwind_tests.rs index 9dc2a470..70473ce4 100644 --- a/compiler-core/src/jvm/select/unwind_tests.rs +++ b/compiler-core/src/jvm/select/unwind_tests.rs @@ -13,6 +13,28 @@ fn jvm_preserves_throwable_identity_through_catch_and_rethrow() { let this_class = cp.add_class("UnwindSsa").unwrap(); let super_class = cp.add_class("java/lang/Object").unwrap(); let mut methods = Vec::new(); + // The JVM requires a terminal instruction on Rust-UB paths, + // including methods that return a value. + let mut b = Builder::new(&types, int); + b.terminate(Terminator::Unreachable); + let code = compile(&b.finish().unwrap(), &types, &mut cp).unwrap(); + assert_eq!( + code.instructions, + vec![Instruction::Aconst_null, Instruction::Athrow] + ); + methods.push(Method { + access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, + name_index: cp.add_utf8("unreachable").unwrap(), + descriptor_index: cp.add_utf8("()I").unwrap(), + attributes: vec![Attribute::Code { + name_index: cp.add_utf8("Code").unwrap(), + max_stack: code.max_stack, + max_locals: code.max_locals, + code: code.instructions, + exception_table: code.exceptions, + attributes: code.attributes, + }], + }); for rethrow in [false, true] { let mut b = Builder::new(&types, throwable); let original = b.parameter(b.current(), throwable); @@ -37,10 +59,13 @@ fn jvm_preserves_throwable_identity_through_catch_and_rethrow() { } else { Terminator::Return(Some(caught)) }); - let body = b.finish().unwrap(); + let mut body = b.finish().unwrap(); + crate::opt::remove_rethrows(&mut body, &types); let code = compile(&body, &types, &mut cp).unwrap(); - assert_eq!(code.exceptions.len(), 1); - assert!(code.exceptions[0].range_pc.start < code.exceptions[0].range_pc.end); + assert_eq!(code.exceptions.len(), usize::from(!rethrow)); + for entry in &code.exceptions { + assert!(entry.range_pc.start < entry.range_pc.end); + } methods.push(Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, name_index: cp @@ -81,6 +106,8 @@ fn jvm_preserves_throwable_identity_through_catch_and_rethrow() { r#" public class UnwindRun { public static void main(String[] args) throws Throwable { + try { UnwindSsa.unreachable(); throw new AssertionError("returned"); } + catch (NullPointerException expected) { } Throwable original = new IllegalArgumentException("identity"); if (UnwindSsa.capture(original) != original) throw new AssertionError("catch"); try { UnwindSsa.rethrow(original); throw new AssertionError("returned"); } diff --git a/compiler-core/src/jvm/select/view_tests.rs b/compiler-core/src/jvm/select/view_tests.rs index 599bc4c8..30eb0a76 100644 --- a/compiler-core/src/jvm/select/view_tests.rs +++ b/compiler-core/src/jvm/select/view_tests.rs @@ -47,7 +47,17 @@ fn executes_full_width_view_metadata_and_typed_data_extraction() { let mut invalid = body.clone(); invalid.instructions[1].op = Op::Length(data); assert!(verify(&invalid, &types).is_err()); + let mut body = body; + crate::opt::decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); let code = compile(&body, &types, &mut cp).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, Instruction::New(_))), + "local view allocated a carrier" + ); methods.push(Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, name_index: cp.add_utf8(name).unwrap(), @@ -131,11 +141,18 @@ public class Utf8View extends SliceView { package org.rustlang.runtime; public class Pointer { public static Pointer extracted; + public static long extractedLength; public static Pointer fromSlice(Object value, int size, String codec) { SliceView view = (SliceView)value; if (view.offset != 0 || codec != null || size != (view instanceof Utf8View ? 1 : 8)) throw new AssertionError("wrong extraction arguments"); - return extracted = (Pointer)view.array; + return fromSliceParts(view.array, view.offset, view.rustLength, size, codec); + } + public static Pointer fromSliceParts(Object data, int start, long length, int size, String codec) { + if (start != 0 || codec != null || (size != 1 && size != 8)) + throw new AssertionError("wrong component arguments"); + extractedLength = length; + return extracted = (Pointer)data; } }"#, ) @@ -146,9 +163,9 @@ public class ViewRun { public static void main(String[] args) { Pointer pointer = new Pointer(); for (long length : new long[] { 0, 1, Integer.MAX_VALUE, (1L << 40) + 23, Long.MAX_VALUE }) { - if (ViewSsa.slice(pointer, length) != length || Pointer.extracted != pointer) + if (ViewSsa.slice(pointer, length) != length || Pointer.extracted != pointer || Pointer.extractedLength != length) throw new AssertionError("slice " + length); - if (ViewSsa.string(pointer, length) != length || Pointer.extracted != pointer) + if (ViewSsa.string(pointer, length) != length || Pointer.extracted != pointer || Pointer.extractedLength != length) throw new AssertionError("string " + length); if (ViewSsa.read_slice(new SliceView(pointer, 3, length)) != length || ViewSsa.read_string(new Utf8View(pointer, 2, length)) != length) diff --git a/compiler-core/src/jvm/select/views.rs b/compiler-core/src/jvm/select/views.rs index 4c29ed3e..1116a2aa 100644 --- a/compiler-core/src/jvm/select/views.rs +++ b/compiler-core/src/jvm/select/views.rs @@ -5,6 +5,20 @@ use representation::{SLICE_VIEW_CLASS, UTF8_VIEW_CLASS}; impl Selector<'_> { pub(super) fn view(&mut self, inst: Inst) -> jvm::Result { match inst.op { + Op::TaggedPack(parts) => { + self.materialize_tagged(parts)?; + } + Op::TaggedPart { value, index } => { + self.load(value)?; + let owner = self.cp.add_class(super::super::abi::TAGGED_LONG_CLASS)?; + self.assembly + .code + .push(Instruction::Invokestatic(self.cp.add_method_ref( + owner, + if index == 0 { "value" } else { "tag" }, + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?)); + } Op::Length(view) => { self.load(view)?; let owner = self.cp.add_class(SLICE_VIEW_CLASS)?; @@ -30,6 +44,16 @@ impl Selector<'_> { self.load(length)?; self.assembly.code.push(Instruction::Invokespecial(init)); } + Op::ViewPart { view, index } => { + self.load(view)?; + self.assembly.code.push(super::super::abi::view_part_access( + self.cp, + index as usize, + )?); + } + Op::ViewPack(parts) => { + self.materialize_view(self.body.value_type(inst.result.unwrap()), parts)?; + } Op::ViewData { view, size, codec } => { self.load(view)?; self.assembly @@ -50,8 +74,93 @@ impl Selector<'_> { )?; self.assembly.code.push(Instruction::Invokestatic(method)); } + Op::ViewAddress { parts, size, codec } => { + for &part in &self.body.args[parts.range()] { + self.load(part)?; + } + self.assembly + .code + .push(get_int_const_instr(self.cp, size as i32)); + self.assembly.code.push(match codec { + Some(id) => Instruction::Ldc_w( + self.cp + .add_name_string(self.types.symbol_name(id).unwrap())?, + ), + None => Instruction::Aconst_null, + }); + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + "fromSliceParts", + "(Ljava/lang/Object;IJILjava/lang/String;)Lorg/rustlang/runtime/Pointer;", + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } + Op::ViewRoot { + backing, + size, + codec, + } => { + self.load(backing)?; + if codec.is_some() { + self.address_layout(size, codec)?; + } else { + self.assembly + .code + .push(get_int_const_instr(self.cp, size as i32)); + } + let owner = self.cp.add_class(POINTER_CLASS)?; + let method = self.cp.add_method_ref( + owner, + if codec.is_some() { + "sliceAddressRoot" + } else { + "scalarSliceRoot" + }, + if codec.is_some() { + "(Ljava/lang/Object;ILjava/lang/String;)Ljava/lang/Object;" + } else { + "(Ljava/lang/Object;I)Ljava/lang/Object;" + }, + )?; + self.assembly.code.push(Instruction::Invokestatic(method)); + } _ => return Ok(false), } Ok(true) } + pub(super) fn materialize_tagged(&mut self, parts: List) -> jvm::Result<()> { + let owner = self.cp.add_class(super::super::abi::TAGGED_LONG_CLASS)?; + self.assembly + .code + .extend([Instruction::New(owner), Instruction::Dup]); + for &value in &self.body.args[parts.range()] { + self.load(value)?; + } + self.assembly.code.push(Instruction::Invokespecial( + self.cp.add_method_ref(owner, "", "(JJ)V")?, + )); + Ok(()) + } + pub(super) fn materialize_view(&mut self, ty: TypeId, parts: List) -> jvm::Result<()> { + let owner = self + .cp + .add_class(if matches!(self.types.get(ty), Some(Type::Str)) + || matches!(self.types.get(ty), Some(Type::Class(s)) if self.types.symbol_name(s) == Some(UTF8_VIEW_CLASS)) { + UTF8_VIEW_CLASS + } else { + SLICE_VIEW_CLASS + })?; + let init = self + .cp + .add_method_ref(owner, "", "(Ljava/lang/Object;IJ)V")?; + self.assembly + .code + .extend([Instruction::New(owner), Instruction::Dup]); + for &value in &self.body.args[parts.range()] { + self.load(value)?; + } + self.assembly.code.push(Instruction::Invokespecial(init)); + Ok(()) + } } diff --git a/compiler-core/src/lib.rs b/compiler-core/src/lib.rs index f3b70949..d6905789 100644 --- a/compiler-core/src/lib.rs +++ b/compiler-core/src/lib.rs @@ -1,4 +1,5 @@ //! Rustc-independent IR and JVM compiler machinery. +pub mod analysis; pub mod ir; pub mod scalar; diff --git a/compiler-core/src/opt/address_intrinsics.rs b/compiler-core/src/opt/address_intrinsics.rs new file mode 100644 index 00000000..e4ea560b --- /dev/null +++ b/compiler-core/src/opt/address_intrinsics.rs @@ -0,0 +1,68 @@ +//! Use address components for intrinsics with explicit byte widths. +//! Runtime helpers retain synchronization and memory tracking. +use super::address_parts::address_parts; +use crate::ir::*; +use crate::scalar::ScalarType; + +fn arity(name: &str) -> Option { + Some(match name { + "atomicLoad" | "writeBytes" => 3, + "atomicStore" | "atomicExchange" | "atomicAdd" | "atomicSubtract" | "atomicAnd" + | "atomicNand" | "atomicOr" | "atomicXor" | "atomicMax" | "atomicMin" + | "atomicUnsignedMax" | "atomicUnsignedMin" => 4, + "atomicCompareExchange" => 6, + _ => return None, + }) +} + +pub fn lower_address_intrinsics(body: &mut Body, types: &mut Types) { + let mut component_types = None; + for index in 0..body.instructions.len() { + let Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } = body.instructions[index].op + else { + continue; + }; + let target = &body.methods[method.index()]; + if target.owner != "org/rustlang/runtime/Pointer" + || arity(&target.name) != Some(args.len as usize) + || target.params.len() != args.len as usize + || !matches!(types.get(target.params[0]), Some(Type::Pointer(_))) + { + continue; + } + if target.name == "writeBytes" + && !matches!( + types.get(target.params[2]), + Some(Type::Scalar(ScalarType::I64 | ScalarType::U64)) + ) + { + continue; + } + let values = &body.args[args.range()]; + let Some(address) = address_parts(body, types, values[0]) else { + continue; + }; + let mut new_args = body.args[address.parts.range()].to_vec(); + new_args.extend_from_slice(&values[1..]); + let mut target = target.clone(); + let [object, long] = *component_types.get_or_insert_with(|| { + let object = types.symbol("java/lang/Object"); + [ + types.intern(Type::Class(object)), + types.scalar(ScalarType::I64), + ] + }); + target.params.splice(..1, [object, long]); + let method = MethodId::new(body.methods.len()); + body.methods.push(target); + body.instructions[index].op = Op::Call { + method, + kind: CallKind::JvmStatic, + args: List::append(&mut body.args, new_args), + }; + } +} diff --git a/compiler-core/src/opt/address_intrinsics_tests.rs b/compiler-core/src/opt/address_intrinsics_tests.rs new file mode 100644 index 00000000..a7e253e5 --- /dev/null +++ b/compiler-core/src/opt/address_intrinsics_tests.rs @@ -0,0 +1,132 @@ +use super::*; +use crate::{ir::*, scalar::ScalarType}; + +#[test] +fn intrinsic_calls_keep_address_components_and_unwind_edges() { + for name in [ + "atomicLoad", + "atomicStore", + "atomicExchange", + "atomicAdd", + "atomicSubtract", + "atomicAnd", + "atomicNand", + "atomicOr", + "atomicXor", + "atomicMax", + "atomicMin", + "atomicUnsignedMax", + "atomicUnsignedMin", + "atomicCompareExchange", + "writeBytes", + ] { + for decomposed in [false, true] { + for typed in [false, true] { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let int = types.scalar(ScalarType::I32); + let void = types.intern(Type::Unit); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let pointer = types.intern(Type::Pointer(long)); + let result = if matches!(name, "atomicStore" | "writeBytes") { + void + } else { + long + }; + let mut b = Builder::new(&types, result); + let address = if decomposed { + let root = b.parameter(b.current(), object); + let offset = b.parameter(b.current(), long); + let parts = b.args([root, offset]); + b.emit( + if typed { + Op::TypedAddressPack { + parts, + size: 8, + codec: None, + } + } else { + Op::AddressPack(parts) + }, + Some(pointer), + ) + .unwrap() + } else { + b.parameter(b.current(), pointer) + }; + let mut params = vec![pointer]; + let mut values = vec![address]; + if name == "writeBytes" { + params.extend([int, long]); + values.extend([ + b.parameter(b.current(), int), + b.parameter(b.current(), long), + ]); + } else { + if name != "atomicLoad" { + params.push(long); + values.push(b.parameter(b.current(), long)); + } + if name == "atomicCompareExchange" { + params.push(long); + values.push(b.parameter(b.current(), long)); + } + for _ in 0..if name == "atomicCompareExchange" { + 3 + } else { + 2 + } { + params.push(int); + values.push(b.parameter(b.current(), int)); + } + } + let original_arity = params.len(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: name.into(), + params, + returns: result, + interface: false, + }); + let args = b.args(values); + let handler = b.create_block(); + let result = b.invoke( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + (result != void).then_some(result), + handler, + ); + b.terminate(Terminator::Return(result)); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + let mut body = b.finish().unwrap(); + lower_address_intrinsics(&mut body, &mut types); + verify(&body, &types).unwrap(); + let (id, method, args) = body + .instructions + .iter() + .enumerate() + .find_map(|(id, inst)| { + let Op::Call { method, args, .. } = inst.op else { + return None; + }; + Some((id, &body.methods[method.index()], args)) + }) + .unwrap(); + assert_eq!(method.name, name); + assert_eq!(args.len as usize, original_arity + usize::from(decomposed)); + if decomposed { + assert!(!live(&body, &types).values[address.index()]); + assert_eq!(&method.params[..2], &[object, long]); + } + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst.index() == id))); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } + } +} diff --git a/compiler-core/src/opt/address_observers.rs b/compiler-core/src/opt/address_observers.rs new file mode 100644 index 00000000..8148714c --- /dev/null +++ b/compiler-core/src/opt/address_observers.rs @@ -0,0 +1,135 @@ +//! Observe address components without a Pointer carrier. +//! Keep calls in place to retain provenance and alignment checks under their original handlers. +use super::address_parts::{AddressParts, address_parts}; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +const OWNER: &str = "org/rustlang/runtime/Pointer"; + +fn stride(address: &AddressParts, types: &Types) -> i64 { + if let Some((size, _)) = address.layout { + return i64::from(size); + } + match address.pointee.and_then(|ty| types.get(ty)) { + Some(Type::Pointer(_)) => 8, + Some(Type::Slice(_) | Type::Str) => 16, + // Resolve the root's layout in the runtime helper. + // Keep the original pointer operation's exception handler. + _ => -1, + } +} + +pub fn lower_address_observers(body: &mut Body, types: &mut Types, debug: Option<&mut DebugInfo>) { + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let long = types.scalar(ScalarType::I64); + let mut constants = FxHashMap::default(); + let mut prologue = Vec::new(); + let mut constant = |body: &mut Body, bits: i64| { + *constants.entry(bits).or_insert_with(|| { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I64, bits as u128).unwrap(), + )); + let id_inst = InstId::new(body.instructions.len()); + let result = ValueId::new(body.values.len()); + body.values.push(Value { + ty: long, + def: ValueDef::Inst(id_inst), + }); + body.instructions.push(Inst { + op: Op::Constant(id), + result: Some(result), + }); + prologue.push(id_inst); + result + }) + }; + for index in 0..body.instructions.len() { + let Op::Call { method, kind, args } = body.instructions[index].op else { + continue; + }; + let method = &body.methods[method.index()]; + if method.owner != OWNER + || !matches!(kind, CallKind::JvmStatic | CallKind::Virtual) + || !matches!( + types.get(method.returns), + Some(Type::Scalar(ScalarType::I64 | ScalarType::U64)) + ) + { + continue; + } + let name = match method.name.as_str() { + "addr" => "locationAddr", + "offset_from" | "offsetFrom" => "offsetLocations", + "offset_from_unsigned" => "offsetLocationsUnsigned", + "byte_offset_from" => "byteOffsetLocations", + "byte_offset_from_unsigned" => "byteOffsetLocationsUnsigned", + "align_offset" => "alignLocation", + _ => continue, + }; + let count = if name == "locationAddr" { 1 } else { 2 }; + if args.len as usize != count + || method.params.len() + usize::from(kind == CallKind::Virtual) != count + { + continue; + } + let returns = method.returns; + let args = body.args[args.range()].to_vec(); + let first = address_parts(body, types, args[0]); + let distance = name.contains("Offset") || name.starts_with("offset"); + let second = distance + .then(|| address_parts(body, types, args[1])) + .flatten(); + if first.is_none() || (distance && second.is_none()) { + continue; + } + if name == "alignLocation" && types.get(body.value_type(args[1])).unwrap().carrier() != 2 { + continue; + } + let step = first.as_ref().map_or(-1, |address| stride(address, types)); + let parts = |address: Option| -> [ValueId; 2] { + body.args[address.unwrap().parts.range()] + .try_into() + .unwrap() + }; + let mut values = parts(first).to_vec(); + let mut params = vec![object, long]; + if distance { + values.extend(parts(second)); + params.extend([object, long]); + } + if name.starts_with("offset") || name == "alignLocation" { + values.push(constant(body, step)); + params.push(long); + } + if name == "alignLocation" { + values.push(args[1]); + params.push(body.value_type(args[1])); + } + let method = MethodId::new(body.methods.len()); + body.methods.push(MethodRef { + owner: OWNER.into(), + name: name.into(), + params, + returns, + interface: false, + }); + body.instructions[index].op = Op::Call { + method, + kind: CallKind::JvmStatic, + args: List::append(&mut body.args, values), + }; + } + let count = prologue.len() as u32; + prologue.append(&mut body.blocks[body.entry.index()].instructions); + body.blocks[body.entry.index()].instructions = prologue; + if let Some(debug) = debug { + for event in &mut debug.events { + if event.block == body.entry { + event.position += count; + } + } + } +} diff --git a/compiler-core/src/opt/address_observers_tests.rs b/compiler-core/src/opt/address_observers_tests.rs new file mode 100644 index 00000000..80eac92e --- /dev/null +++ b/compiler-core/src/opt/address_observers_tests.rs @@ -0,0 +1,164 @@ +use super::*; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn address_observers_keep_components_and_exception_handlers() { + for name in [ + "addr", + "offset_from", + "offset_from_unsigned", + "byte_offset_from", + "byte_offset_from_unsigned", + "align_offset", + ] { + for typed in [false, true] { + for protected in [false, true] { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let int = types.scalar(ScalarType::I32); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let pointer = types.intern(Type::Pointer(int)); + let codec = types.symbol("test/Codec#pair#Ltest/Pair;"); + let mut b = Builder::new(&types, long); + let root = b.parameter(b.current(), object); + let offset = b.parameter(b.current(), long); + let parts = b.args([root, offset]); + let packed = b + .emit( + if typed { + Op::TypedAddressPack { + parts, + size: 16, + codec: Some(codec), + } + } else { + Op::AddressPack(parts) + }, + Some(pointer), + ) + .unwrap(); + let alignment = b.constant(long, Scalar::integer(ScalarType::I64, 8).unwrap()); + let (params, args) = match name { + "addr" => (vec![], vec![packed]), + "align_offset" => (vec![long], vec![packed, alignment]), + _ => (vec![pointer], vec![packed, packed]), + }; + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: name.into(), + params, + returns: long, + interface: false, + }); + let args = b.args(args); + let call = Op::Call { + method, + kind: CallKind::Virtual, + args, + }; + let (result, handler) = if protected { + let handler = b.create_block(); + (b.invoke(call, Some(long), handler).unwrap(), Some(handler)) + } else { + (b.emit(call, Some(long)).unwrap(), None) + }; + b.terminate(Terminator::Return(Some(result))); + if let Some(handler) = handler { + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + } + let mut body = b.finish().unwrap(); + lower_address_observers(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!(!live(&body, &types).values[packed.index()], "{name}"); + let (id, inst) = body.instructions.iter().enumerate().find(|(_, inst)| { + matches!(inst.op, Op::Call { method, .. } if body.methods[method.index()].name != name) + }).unwrap(); + let Op::Call { method, kind, args } = inst.op else { + unreachable!() + }; + assert_eq!(kind, CallKind::JvmStatic); + let selected = &body.methods[method.index()].name; + assert!( + selected.ends_with("Locations") + || selected.ends_with("LocationsUnsigned") + || selected == "alignLocation" + || selected == "locationAddr" + ); + if name.starts_with("offset") || name == "align_offset" { + let size = + body.args[args.start as usize + if name == "align_offset" { 2 } else { 4 }]; + assert_eq!( + body.scalar_value(size).unwrap().bits(), + if typed { 16 } else { 4 } + ); + } + if protected { + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst.index() == id))); + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } + } +} + +#[test] +fn dynamic_stride_comes_from_the_root() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let record = types.symbol("test/Record"); + let record = types.intern(Type::Class(record)); + let pointer = types.intern(Type::Pointer(record)); + let mut b = Builder::new(&types, long); + let root = b.parameter(b.current(), object); + let offset = b.parameter(b.current(), long); + let origin = b.parameter(b.current(), object); + let parts = b.args([root, offset]); + let packed = b.emit(Op::AddressPack(parts), Some(pointer)).unwrap(); + let origin_parts = b.args([origin, offset]); + let origin = b + .emit(Op::AddressPack(origin_parts), Some(pointer)) + .unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "offsetFrom".into(), + params: vec![pointer, pointer], + returns: long, + interface: false, + }); + let args = b.args([packed, origin]); + let result = b + .emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(long), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_address_observers(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!(!live(&body, &types).values[packed.index()]); + let Op::Call { args, .. } = body + .instructions + .iter() + .find(|inst| matches!(inst.op, Op::Call { .. })) + .unwrap() + .op + else { + unreachable!() + }; + let args = &body.args[args.range()]; + assert!(!live(&body, &types).values[origin.index()]); + assert_eq!(args[3], offset); + assert_eq!(body.scalar_value(args[4]).unwrap().bits() as i64, -1); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} diff --git a/compiler-core/src/opt/address_parts.rs b/compiler-core/src/opt/address_parts.rs new file mode 100644 index 00000000..41fb1c19 --- /dev/null +++ b/compiler-core/src/opt/address_parts.rs @@ -0,0 +1,70 @@ +//! Recover address components from representation lowering. +//! Read the defining pack's exact layout, even after JVM type erasure. +use crate::ir::*; + +pub(super) struct AddressParts { + pub value: ValueId, + pub parts: List, + pub pointee: Option, + pub layout: Option<(u32, Option)>, +} + +pub(super) fn address_parts(body: &Body, types: &Types, value: ValueId) -> Option { + find_parts(body, types, value, true) +} + +/// Scalar pointer casts replace the view width and codec. +/// They can ignore erased layout annotations, unlike ordinary ABI uses. +pub(super) fn scalar_cast_parts( + body: &Body, + types: &Types, + value: ValueId, +) -> Option { + find_parts(body, types, value, false) +} + +fn find_parts( + body: &Body, + types: &Types, + mut value: ValueId, + preserve_layout: bool, +) -> Option { + for _ in 0..32 { + value = body.resolve(value); + let ValueDef::Inst(id) = body.values[value.index()].def else { + return None; + }; + let ty = body.value_type(value); + let pointee = types.pointee(ty); + match body.instructions[id.index()].op { + Op::TypedAddressPack { parts, size, codec } => { + return Some(AddressParts { + value, + parts, + pointee, + layout: Some((size, codec)), + }); + } + Op::AddressPack(parts) => { + let layout = types + .address_layout(ty) + .or_else(|| StorageSlot::scalar(pointee?, types).map(|slot| (slot.size, None))); + return Some(AddressParts { + value, + parts, + pointee, + layout, + }); + } + Op::Refine(source) | Op::Reinterpret(source) + if !preserve_layout + || types.address_layout(ty) + == types.address_layout(body.value_type(source)) => + { + value = source + } + _ => return None, + } + } + None +} diff --git a/compiler-core/src/opt/addresses.rs b/compiler-core/src/opt/addresses.rs new file mode 100644 index 00000000..256e78e6 --- /dev/null +++ b/compiler-core/src/opt/addresses.rs @@ -0,0 +1,689 @@ +//! Keep scalar addresses as a storage root and byte displacement through loops +//! and calls. Unknown roots retain their full runtime provenance as one object. +use super::append_value as append; +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; +use rustc_hash::{FxHashMap, FxHashSet}; + +fn literal(body: &mut Body, ty: TypeId, bits: i64) -> (InstId, ValueId) { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I64, bits as u128).unwrap(), + )); + append(body, Op::Constant(id), ty) +} +fn parts( + body: &mut Body, + types: &Types, + value: ValueId, + known: &[Option<[ValueId; 2]>], + object: TypeId, + long: TypeId, + prefix: &mut Vec, +) -> [ValueId; 2] { + if let Some(parts) = known.get(value.index()).copied().flatten() { + return parts; + } + // Scalar projections can use an aggregate location's components. + // Their borrowed shapes do not need to match. + let mut source = value; + for _ in 0..64 { + source = body.resolve(source); + let ValueDef::Inst(id) = body.values[source.index()].def else { + break; + }; + match body.instructions[id.index()].op { + Op::AddressPack(parts) => return body.args[parts.range()].try_into().unwrap(), + Op::Reinterpret(value) | Op::Refine(value) + if types.address_layout(body.value_type(source)) + == types.address_layout(body.value_type(value)) => + { + source = value + } + _ => break, + } + } + let (cast, root) = append(body, Op::Reinterpret(value), object); + let (zero, offset) = literal(body, long, 0); + prefix.extend([cast, zero]); + [root, offset] +} + +pub fn decompose_addresses(body: &mut Body, types: &mut Types, mut debug: Option<&mut DebugInfo>) { + for shape in [ComponentShape::Address, ComponentShape::StorageAddress] { + if body + .values + .iter() + .any(|v| ComponentShape::of(types, v.ty) == Some(shape)) + { + decompose(body, types, debug.as_deref_mut(), shape); + } + } +} + +fn decompose( + body: &mut Body, + types: &mut Types, + debug: Option<&mut DebugInfo>, + shape: ComponentShape, +) { + if !body.instructions.iter().any(|i| { + matches!( + i.op, + Op::AddressPack(_) + | Op::AddressOfSlot(_) + | Op::Offset { .. } + | Op::AddressPart { .. } + | Op::AddressTag(_) + | Op::AddressEqual { .. } + | Op::AddressCompare { .. } + | Op::ViewAddress { .. } + | Op::AddressViewPart { .. } + | Op::Project { .. } + ) + }) { + return; + } + let count = body.values.len(); + let predecessors = body.predecessors(); + let mut eligible = vec![false; count]; + let mut users = crate::analysis::ValueUsers::new(count); + for index in 0..count { + let ty = body.values[index].ty; + let address = ComponentShape::of(types, ty) == Some(shape); + if !shape.accepts_annotation(types, ty) { + continue; + } + match body.values[index].def { + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::AddressPack(_) | Op::AddressOfSlot(_) | Op::Offset { .. } if address => { + eligible[index] = true + } + Op::Project { projection, .. } if address => { + let projection = &body.projections[projection.index()]; + let Type::Pointer(inner) = types.get(ty).unwrap() else { + unreachable!() + }; + eligible[index] = (projection.codec.is_none() + && StorageSlot::scalar(inner, types) + .is_some_and(|slot| u64::from(slot.size) == projection.size)) + || (ComponentShape::of(types, inner) + .is_some_and(ComponentShape::is_borrowed) + && matches!(projection.size, 8 | 16)); + } + Op::Constant(constant) + if matches!(body.constants[constant.index()], Constant::Uninit(_)) + || (address + && matches!(body.constants[constant.index()], Constant::Null(_))) => + { + eligible[index] = true + } + Op::ViewAddress { + size, codec: None, .. + } if address && shape == ComponentShape::Address => { + let Some(Type::Pointer(inner)) = types.get(ty) else { + unreachable!() + }; + eligible[index] = + StorageSlot::scalar(inner, types).is_some_and(|slot| slot.size == size); + } + Op::Cast(source) + if address + && shape == ComponentShape::Address + && matches!(types.get(body.value_type(source)), Some(Type::Pointer(_))) => + { + eligible[index] = true + } + Op::Reinterpret(source) | Op::Adapt(source) | Op::Refine(source) + if types.address_layout(ty) + == types.address_layout(body.value_type(source)) => + { + eligible[index] = true; + users.connect(source, index); + } + _ => {} + }, + ValueDef::Alias(source) => { + eligible[index] = true; + users.connect(source, index); + } + ValueDef::Param(block) if block != body.entry => { + eligible[index] = !predecessors[block.index()].is_empty(); + let position = body.blocks[block.index()] + .params + .iter() + .position(|&p| p.index() == index) + .unwrap(); + for &(_, edge) in &predecessors[block.index()] { + users.connect(body.edges[edge.index()].args[position], index); + } + } + _ => {} + } + } + users.close(&mut eligible); + let mut component_types = shape.parts(types); + let (object, long) = ( + component_types.next().unwrap(), + component_types.next().unwrap(), + ); + let mut known = vec![None::<[ValueId; 2]>; count]; + let mut roots = Vec::new(); + let mut joins = vec![Vec::new(); body.blocks.len()]; + for index in 0..count { + if !eligible[index] { + continue; + } + match body.values[index].def { + ValueDef::Param(block) => { + let parts = [object, long].map(|ty| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Param(block), + }); + body.blocks[block.index()].params.push(value); + value + }); + known[index] = Some(parts); + let position = body.blocks[block.index()] + .params + .iter() + .position(|&value| value.index() == index) + .unwrap(); + joins[block.index()].push((index, position)); + } + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::AddressPack(args) => { + known[index] = Some(body.args[args.range()].try_into().unwrap()) + } + Op::Constant(_) + | Op::AddressOfSlot(_) + | Op::Offset { .. } + | Op::Cast(_) + | Op::Project { .. } + | Op::ViewAddress { .. } => { + let a = append(body, Op::Nop, object); + let b = append(body, Op::Nop, long); + known[index] = Some([a.1, b.1]); + roots.push((id, a.0, b.0)); + } + _ => {} + }, + _ => {} + } + } + users.propagate(&eligible, &mut known); + let mut prefixes = FxHashMap::>::default(); + for (id, root, offset) in roots { + let original = body.instructions[id.index()]; + let mut prefix = Vec::new(); + let (root_op, offset_op) = match original.op { + Op::Project { base, projection } => { + let displacement = body.projections[projection.index()].offset; + let (base, projection, displacement) = if shape == ComponentShape::Address { + super::fields::fold_projection(body, types, base, projection).unwrap_or(( + base, + projection, + displacement, + )) + } else { + (base, projection, displacement) + }; + let source = parts(body, types, base, &known, object, long, &mut prefix); + let (constant, displacement) = literal(body, long, displacement as i64); + let (sum_inst, sum) = append( + body, + Op::Binary { + op: BinaryOp::Add, + left: source[1], + right: displacement, + }, + long, + ); + prefix.extend([constant, sum_inst]); + ( + Op::ProjectRoot { + address: List::append(&mut body.args, source), + projection, + }, + Op::ProjectOffset { + root: body.instructions[root.index()].result.unwrap(), + base: source[0], + offset: sum, + }, + ) + } + Op::Constant(constant) + if matches!(body.constants[constant.index()], Constant::Uninit(_)) => + { + let root = ConstId::new(body.constants.len()); + body.constants.push(Constant::Uninit(object)); + let offset = ConstId::new(body.constants.len()); + body.constants.push(Constant::Uninit(long)); + (Op::Constant(root), Op::Constant(offset)) + } + Op::Constant(_) => { + let null = ConstId::new(body.constants.len()); + body.constants.push(Constant::Null(object)); + let (zero, value) = literal(body, long, 0); + prefix.push(zero); + (Op::Constant(null), Op::Reinterpret(value)) + } + Op::ViewAddress { parts, size, .. } => { + let values = body.args[parts.range()].to_vec(); + let int = types.scalar(ScalarType::I32); + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, size.into()).unwrap(), + )); + let (size_inst, size_arg) = append(body, Op::Constant(constant), int); + let (cast, start) = append(body, Op::Cast(values[1]), long); + let (stride_inst, stride) = literal(body, long, size as i64); + prefix.extend([size_inst, cast, stride_inst]); + let method = MethodId::new(body.methods.len()); + body.methods.push(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "scalarSliceRoot".into(), + params: vec![object, int], + returns: object, + interface: false, + }); + ( + Op::Call { + method, + kind: CallKind::JvmStatic, + args: List::append(&mut body.args, [values[0], size_arg]), + }, + Op::Binary { + op: BinaryOp::Mul, + left: start, + right: stride, + }, + ) + } + Op::AddressOfSlot(slot) => { + let (zero, value) = literal(body, long, 0); + prefix.push(zero); + (Op::SlotRoot(slot), Op::Reinterpret(value)) + } + Op::Cast(value) => { + let source = super::address_parts::scalar_cast_parts(body, types, value) + .map(|address| body.args[address.parts.range()].try_into().unwrap()) + .unwrap_or_else(|| { + parts(body, types, value, &known, object, long, &mut prefix) + }); + (Op::Reinterpret(source[0]), Op::Reinterpret(source[1])) + } + Op::Offset { + pointer, + offset: delta, + bytes, + .. + } => { + let mut source = parts(body, types, pointer, &known, object, long, &mut prefix); + if let Some(Type::Pointer(inner)) = types.get(body.value_type(pointer)) + && ComponentShape::of(types, inner).is_some_and(ComponentShape::is_borrowed) + { + // Reference-to-borrow arithmetic needs the exact Rust stride. + // A fixed-array borrow uses one word. A slice borrow uses two. + // The enclosing storage size does not determine this stride. + let int = types.scalar(ScalarType::I32); + let plan = crate::jvm::abi::address_plan(types, inner); + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, plan.into()).unwrap(), + )); + let (id, plan) = append(body, Op::Constant(constant), int); + prefix.push(id); + let method = MethodId::new(body.methods.len()); + let carrier = types.symbol("org/rustlang/runtime/Pointer"); + let carrier = types.intern(Type::Class(carrier)); + body.methods.push(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "addressFromParts".into(), + params: vec![object, long, int], + returns: carrier, + interface: false, + }); + let args = List::append(&mut body.args, [source[0], source[1], plan]); + let (id, root) = append( + body, + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + carrier, + ); + let (erase, root) = append(body, Op::Reinterpret(root), object); + let (zero, displacement) = literal(body, long, 0); + prefix.extend([id, erase, zero]); + source = [root, displacement]; + } + let (cast, mut delta) = append(body, Op::Cast(delta), long); + prefix.push(cast); + if !bytes { + let Some(Type::Pointer(inner)) = types.get(body.value_type(pointer)) else { + unreachable!() + }; + let (constant, size) = + if let Some((size, _)) = types.address_layout(body.value_type(pointer)) { + literal(body, long, size as i64) + } else if let Some(slot) = StorageSlot::scalar(inner, types) { + literal(body, long, slot.size as i64) + } else if let ValueDef::Inst(id) = + body.values[body.resolve(pointer).index()].def + && let Op::RetypeAddress { size, .. } = body.instructions[id.index()].op + { + literal(body, long, size as i64) + } else { + let method = MethodId::new(body.methods.len()); + body.methods.push(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "locationStride".into(), + params: vec![object], + returns: long, + interface: false, + }); + let args = List::append(&mut body.args, [source[0]]); + append( + body, + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + long, + ) + }; + let (mul, scaled) = append( + body, + Op::Binary { + op: BinaryOp::Mul, + left: delta, + right: size, + }, + long, + ); + prefix.extend([constant, mul]); + delta = scaled; + } + ( + Op::Reinterpret(source[0]), + Op::Binary { + op: BinaryOp::Add, + left: source[1], + right: delta, + }, + ) + } + _ => unreachable!(), + }; + body.instructions[root.index()].op = root_op; + body.instructions[offset.index()].op = offset_op; + prefix.extend([root, offset]); + let values = known[original.result.unwrap().index()].unwrap(); + if ComponentShape::of(types, body.value_type(original.result.unwrap())) == Some(shape) { + body.instructions[id.index()].op = + Op::AddressPack(List::append(&mut body.args, values)); + } + prefixes.insert(id, prefix); + } + for (block, incoming) in predecessors.iter().enumerate() { + for &(_, edge) in incoming { + for &(_, position) in &joins[block] { + let input = body.edges[edge.index()].args[position]; + body.edges[edge.index()] + .args + .extend(known[input.index()].unwrap()); + } + } + } + // A decoded load and its later commit must use the same carrier. + // Separate carriers would lose the mutable view binding. + let bound_views = if shape == ComponentShape::StorageAddress { + body.instructions + .iter() + .filter_map(|inst| { + if let Op::Commit(pointer) = inst.op { + return known.get(pointer.index()).copied().flatten(); + } + let Op::Call { + method, + kind: CallKind::Virtual, + args, + } = inst.op + else { + return None; + }; + let method = &body.methods[method.index()]; + if method.owner != "org/rustlang/runtime/Pointer" + || method.name != "commitMemoryView" + { + return None; + } + known + .get(body.args[args.start as usize].index()) + .copied() + .flatten() + }) + .collect::>() + } else { + FxHashSet::default() + }; + for index in 0..body.instructions.len() { + let original = body.instructions[index]; + if shape == ComponentShape::StorageAddress { + if let Op::LoadFieldCopy { base, projection } = original.op + && let Some(parts) = known.get(base.index()).copied().flatten() + { + body.instructions[index].op = Op::LoadStorageFieldCopy { + address: List::append(&mut body.args, parts), + projection, + }; + continue; + } + let stored = match original.op { + Op::StoreField { + base, + projection, + value, + } => Some((base, projection, vec![value], false)), + Op::StoreFieldParts { + base, + projection, + parts, + } => Some((base, projection, body.args[parts.range()].to_vec(), true)), + _ => None, + }; + if let Some((base, projection, values, split)) = stored + && let Some(parts) = known.get(base.index()).copied().flatten() + { + body.instructions[index].op = Op::StoreStorageField { + args: List::append(&mut body.args, parts.into_iter().chain(values)), + projection, + split, + }; + continue; + } + let field = match original.op { + Op::LoadField { base, projection } => Some((base, projection, None)), + Op::LoadFieldPart { + base, + projection, + index, + } => Some((base, projection, Some(index))), + _ => None, + }; + if let Some((base, projection, index_part)) = field + && let Some(parts) = known.get(base.index()).copied().flatten() + { + body.instructions[index].op = Op::LoadStorageField { + address: List::append(&mut body.args, parts), + projection, + index: index_part, + }; + continue; + } + } + if let Op::AddressViewPart { + address, + index: part, + } = original.op + { + // Opaque helpers can retain a layout that differs from the pointer type, + // such as a DST tail or fixed-array cast. Normalize only proven scalar locations. + if shape != ComponentShape::Address + || known.get(address.index()).copied().flatten().is_none() + { + continue; + } + let mut prefix = Vec::new(); + let source = parts(body, types, address, &known, object, long, &mut prefix); + let Some(Type::Pointer(inner)) = types.get(body.value_type(address)) else { + unreachable!() + }; + let size = StorageSlot::scalar(inner, types).unwrap().size; + let int = types.scalar(ScalarType::I32); + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, size.into()).unwrap(), + )); + let (instruction, size) = append(body, Op::Constant(constant), int); + prefix.push(instruction); + let method = MethodId::new(body.methods.len()); + body.methods.push(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: if part == 0 { + "locationSliceBacking" + } else { + "locationSliceOffset" + } + .into(), + params: vec![object, long, int], + returns: body.value_type(original.result.unwrap()), + interface: false, + }); + body.instructions[index].op = Op::Call { + method, + kind: CallKind::JvmStatic, + args: List::append(&mut body.args, [source[0], source[1], size]), + }; + prefixes.insert(InstId::new(index), prefix); + continue; + } + if let Op::AddressTag(value) = original.op { + if ComponentShape::of(types, body.value_type(value)) != Some(shape) { + continue; + } + let mut prefix = Vec::new(); + let source = parts(body, types, value, &known, object, long, &mut prefix); + body.instructions[index].op = Op::LocationTag(List::append(&mut body.args, source)); + prefixes.insert(InstId::new(index), prefix); + continue; + } + if let Op::AddressEqual { left, right } | Op::AddressCompare { left, right } = original.op { + // Leave storage comparisons for the later storage-address pass. + // Boxing components here would hide their storage layout. + if [left, right] + .into_iter() + .any(|value| ComponentShape::of(types, body.value_type(value)) != Some(shape)) + { + continue; + } + let mut prefix = Vec::new(); + let left = parts(body, types, left, &known, object, long, &mut prefix); + let right = parts(body, types, right, &known, object, long, &mut prefix); + let parts = List::append(&mut body.args, left.into_iter().chain(right)); + body.instructions[index].op = if matches!(original.op, Op::AddressCompare { .. }) { + Op::LocationCompare(parts) + } else { + Op::LocationEqual(parts) + }; + prefixes.insert(InstId::new(index), prefix); + continue; + } + if let Op::Adapt(source) = original.op { + if let Some(result) = original.result + && let Some(parts) = known.get(result.index()).copied().flatten() + { + body.instructions[index].op = if shape == ComponentShape::StorageAddress { + // Keep the JVM type refinement even though the storage identity + // and layout are already known. + Op::Refine(source) + } else if ComponentShape::of(types, body.value_type(result)) == Some(shape) { + Op::AddressPack(List::append(&mut body.args, parts)) + } else { + Op::Reinterpret(source) + }; + continue; + } + } + let pointer = match original.op { + Op::Load(p) + | Op::LoadCopy(p) + | Op::Store { pointer: p, .. } + | Op::AddressPart { address: p, .. } => p, + _ => continue, + }; + let Some(parts) = known.get(pointer.index()).copied().flatten() else { + continue; + }; + body.instructions[index].op = match original.op { + Op::AddressPart { index, .. } => Op::Reinterpret(parts[index as usize]), + Op::LoadCopy(_) if shape == ComponentShape::StorageAddress => { + Op::LoadAddressCopy(List::append(&mut body.args, parts)) + } + Op::Load(_) + if (shape == ComponentShape::StorageAddress && !bound_views.contains(&parts)) + || StorageSlot::scalar(body.value_type(original.result.unwrap()), types) + .is_some() => + { + Op::LoadAddress(List::append(&mut body.args, parts)) + } + Op::Store { value, .. } + if shape == ComponentShape::StorageAddress + || StorageSlot::scalar(body.value_type(value), types).is_some() => + { + Op::StoreAddress { + parts: List::append(&mut body.args, parts), + value, + } + } + op => op, + }; + } + let mut positions = debug.as_ref().map(|_| Vec::new()); + for block in &mut body.blocks { + let previous = std::mem::take(&mut block.instructions); + let mut mapping = Vec::new(); + for id in previous { + if positions.is_some() { + mapping.push(block.instructions.len() as u32); + } + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(positions) = &mut positions { + mapping.push(block.instructions.len() as u32); + positions.push(mapping); + } + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + if matches!(body.instructions[inst.index()].op, Op::AddressPack(_)) { + block.instructions.push(inst); + block.terminator = Some(Terminator::Jump(normal)); + } + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/addresses_tests.rs b/compiler-core/src/opt/addresses_tests.rs new file mode 100644 index 00000000..3dcf43ff --- /dev/null +++ b/compiler-core/src/opt/addresses_tests.rs @@ -0,0 +1,612 @@ +use super::{decompose_addresses, lower_component_arguments}; +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +#[test] +fn borrowed_scalar_fields_keep_their_aggregate_root_across_calls() { + for nested in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let owner = types.symbol("Pair"); + let owner = types.intern(Type::Class(owner)); + let aggregate = types.intern(Type::Pointer(owner)); + let scalar = types.intern(Type::Pointer(int)); + let outer_name = types.symbol("Outer"); + let outer = types.intern(Type::Class(outer_name)); + let outer_pointer = types.intern(Type::Pointer(outer)); + let mut b = Builder::new(&types, int); + let mut root = b.parameter(b.current(), if nested { outer_pointer } else { aggregate }); + if nested { + let field = b.field(FieldRef { + owner: outer, + name: "pair".into(), + ty: owner, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 16, + size: 8, + codec: Some("PairCodec#pair#LPair;".into()), + }); + root = b + .emit( + Op::Project { + base: root, + projection, + }, + Some(aggregate), + ) + .unwrap(); + } + let field = b.field(FieldRef { + owner, + name: "second".into(), + ty: int, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 4, + size: 4, + codec: None, + }); + let pointer = b + .emit( + Op::Project { + base: root, + projection, + }, + Some(scalar), + ) + .unwrap(); + let method = b.method(MethodRef { + owner: "Kernel".into(), + name: "consume".into(), + params: vec![scalar], + returns: int, + interface: false, + }); + let args = b.args([pointer]); + let result = b + .emit( + Op::Call { + method, + kind: CallKind::RustStatic, + args, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(id, inst)| live.instructions[id] + && matches!(inst.op, Op::AddressPack(_) | Op::Project { .. })) + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::ProjectRoot { .. })) + ); + for inst in &body.instructions { + if let Op::ProjectRoot { projection, .. } = inst.op { + assert_eq!( + body.projections[projection.index()].parent.is_some(), + nested + ); + } + if let Op::ProjectOffset { offset, .. } = inst.op { + let ValueDef::Inst(id) = body.values[offset.index()].def else { + panic!() + }; + let Op::Binary { right, .. } = body.instructions[id.index()].op else { + panic!() + }; + let ValueDef::Inst(id) = body.values[right.index()].def else { + panic!() + }; + let Op::Constant(id) = body.instructions[id.index()].op else { + panic!() + }; + assert_eq!( + body.constants[id.index()], + Constant::Scalar( + Scalar::integer(ScalarType::I64, if nested { 20 } else { 4 }).unwrap() + ) + ); + } + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn thin_pointer_comparisons_use_components() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let owner = types.symbol("Item"); + let owner = types.intern(Type::Class(owner)); + for (pointee, ordering) in [int, owner] + .into_iter() + .flat_map(|p| [(p, false), (p, true)]) + { + let result_type = if ordering { int } else { boolean }; + let pointer = types.intern(Type::Pointer(pointee)); + let mut b = Builder::new(&types, result_type); + let left = b.parameter(b.current(), pointer); + let right = b.parameter(b.current(), pointer); + let equal = b + .emit( + if ordering { + Op::AddressCompare { left, right } + } else { + Op::AddressEqual { left, right } + }, + Some(result_type), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(equal))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!(body.instructions.iter().any(|i| if ordering { + matches!(i.op, Op::LocationCompare(_)) + } else { + matches!(i.op, Op::LocationEqual(_)) + })); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(index, i)| live.instructions[index] && matches!(i.op, Op::AddressPack(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn runtime_pointer_annotations_do_not_force_materialization() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(int)); + let runtime = types.symbol("org/rustlang/runtime/Pointer"); + let runtime = types.intern(Type::Class(runtime)); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let mut b = Builder::new(&types, boolean); + let left = b.parameter(b.current(), pointer); + let right = b.parameter(b.current(), pointer); + // Runtime helper signatures and source bindings both erase annotations. + let erased = b.emit(Op::Reinterpret(left), Some(runtime)).unwrap(); + let erased = b.emit(Op::Reinterpret(erased), Some(object)).unwrap(); + let recovered = b.emit(Op::Adapt(erased), Some(pointer)).unwrap(); + let equal = b + .emit( + Op::AddressEqual { + left: recovered, + right, + }, + Some(boolean), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(equal))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!(inst.op, Op::AddressPack(_) | Op::Adapt(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn scalar_reference_loop_has_no_materialized_addresses() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(int)); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let mut b = Builder::new(&types, int); + let start = b.parameter(b.current(), pointer); + let end = b.parameter(b.current(), long); + let address = b.variable(object); + let erased = b.emit(Op::Reinterpret(start), Some(object)).unwrap(); + b.define(address, erased); + let index = b.variable(long); + let zero = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + b.define(index, zero); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let i = b.read(index); + let current = b.read(address); + let current = b.emit(Op::Adapt(current), Some(pointer)).unwrap(); + let condition = b + .emit( + Op::Binary { + op: BinaryOp::Lt, + left: i, + right: end, + }, + Some(boolean), + ) + .unwrap(); + b.branch(condition, step, done); + b.switch_to(step); + let value = b.emit(Op::Load(current), Some(int)).unwrap(); + b.emit( + Op::Store { + pointer: current, + value, + }, + None, + ); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let next = b + .emit( + Op::Offset { + pointer: current, + offset: one, + bytes: false, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + let erased = b.emit(Op::Reinterpret(next), Some(object)).unwrap(); + b.define(address, erased); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: i, + right: one, + }, + Some(long), + ) + .unwrap(); + b.define(index, next); + b.jump(header, vec![]); + b.switch_to(done); + let value = b.emit(Op::Load(current), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + for (i, inst) in body.instructions.iter().enumerate() { + assert!( + !live.instructions[i] + || !matches!( + inst.op, + Op::AddressPack(_) | Op::Offset { .. } | Op::Load(_) | Op::Store { .. } + ) + ); + } + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, crate::classfile::attributes::Instruction::New(_))) + ); +} + +#[test] +fn aggregate_addresses_keep_layout_roots_through_calls_and_field_access() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let owner = types.symbol("Pair"); + let owner = types.intern(Type::Class(owner)); + let pointer = types.intern(Type::Pointer(owner)); + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, long); + let base = b.parameter(b.current(), pointer); + let offset = b.parameter(b.current(), long); + let field = b.field(FieldRef { + owner, + name: "value".into(), + ty: long, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 8, + codec: None, + }); + let address = b + .emit( + Op::Offset { + pointer: base, + offset, + bytes: false, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + let method = b.method(MethodRef { + owner: "test/Calls".into(), + name: "inspect".into(), + params: vec![pointer], + returns: unit, + interface: false, + }); + let args = b.args([address]); + b.emit( + Op::Call { + method, + kind: CallKind::RustStatic, + args, + }, + None, + ); + let value = b + .emit( + Op::LoadField { + base: address, + projection, + }, + Some(long), + ) + .unwrap(); + b.emit( + Op::StoreField { + base: address, + projection, + value, + }, + None, + ); + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::AddressPack(_) + | Op::AddressPart { .. } + | Op::Offset { .. } + | Op::LoadField { .. } + | Op::StoreField { .. } + )) + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LoadStorageField { .. })) + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::StoreStorageField { .. })) + ); + assert!(body.methods.iter().any(|m| m.name == "locationStride")); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn decoded_aggregate_writeback_retains_its_binding_owner() { + for (commit, owned) in [(false, false), (true, false), (false, true), (true, true)] { + let mut types = Types::default(); + let unit = types.intern(Type::Unit); + let long = types.scalar(ScalarType::I64); + let name = types.symbol("Pair"); + let object = types.intern(Type::Class(name)); + let pointer = types.intern(Type::Pointer(object)); + let name = types.symbol("java/lang/Object"); + let erased = types.intern(Type::Class(name)); + let mut b = Builder::new(&types, object); + let root = b.parameter(b.current(), pointer); + let displacement = b.parameter(b.current(), long); + let address = b + .emit( + Op::Offset { + pointer: root, + offset: displacement, + bytes: true, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + let erased_address = b.emit(Op::Reinterpret(address), Some(erased)).unwrap(); + let load_address = b.emit(Op::Adapt(erased_address), Some(pointer)).unwrap(); + let value = b + .emit( + if owned { + Op::LoadCopy(load_address) + } else { + Op::Load(load_address) + }, + Some(object), + ) + .unwrap(); + if commit { + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "commitMemoryView".into(), + params: vec![], + returns: unit, + interface: false, + }); + let commit_address = b.emit(Op::Adapt(erased_address), Some(pointer)).unwrap(); + let args = b.args([commit_address]); + b.emit( + Op::Call { + method, + kind: CallKind::Virtual, + args, + }, + None, + ); + } + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::Load(_))), + commit && !owned + ); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LoadAddress(_))), + !commit && !owned + ); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LoadAddressCopy(_))), + owned + ); + if commit && !owned { + let live = super::live(&body, &types); + assert_eq!( + body.instructions + .iter() + .enumerate() + .filter(|(i, inst)| { + live.instructions[*i] && matches!(inst.op, Op::AddressPack(_)) + }) + .count(), + 1, + "load and writeback must share one materialized owner" + ); + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn nullable_discriminants_do_not_materialize_borrowed_addresses() { + let mut types = Types::default(); + let scalar = types.scalar(ScalarType::I32); + let owner = types.symbol("Item"); + let owner = types.intern(Type::Class(owner)); + let long = types.scalar(ScalarType::I64); + for pointee in [scalar, owner] { + let pointer = types.intern(Type::Pointer(pointee)); + let mut b = Builder::new(&types, long); + let value = b.parameter(b.current(), pointer); + let tag = b.emit(Op::AddressTag(value), Some(long)).unwrap(); + b.terminate(Terminator::Return(Some(tag))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LocationTag(_))) + ); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!(inst.op, Op::AddressPack(_) | Op::AddressTag(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn inactive_and_nullable_return_joins_keep_address_components() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let owner = types.symbol("Item"); + let owner = types.intern(Type::Class(owner)); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + for (pointee, inactive) in [int, owner] + .into_iter() + .flat_map(|pointee| [(pointee, false), (pointee, true)]) + { + let pointer = types.intern(Type::Pointer(pointee)); + let mut b = Builder::new(&types, pointer); + let condition = b.parameter(b.current(), boolean); + let value = b.parameter(b.current(), pointer); + let result = b.variable(object); + let some = b.create_block(); + let none = b.create_block(); + let done = b.create_block(); + b.branch(condition, some, none); + b.switch_to(some); + let erased = b.emit(Op::Reinterpret(value), Some(object)).unwrap(); + b.define(result, erased); + b.jump(done, vec![]); + b.switch_to(none); + let constant = ConstId::new(b.body.constants.len()); + let ty = if inactive { object } else { pointer }; + b.body.constants.push(if inactive { + Constant::Uninit(ty) + } else { + Constant::Null(ty) + }); + let null = b.emit(Op::Constant(constant), Some(ty)).unwrap(); + let erased = b.emit(Op::Reinterpret(null), Some(object)).unwrap(); + b.define(result, erased); + b.jump(done, vec![]); + b.switch_to(done); + let result = b.read(result); + let result = b.emit(Op::Adapt(result), Some(pointer)).unwrap(); + let result = b.emit(Op::Refine(result), Some(pointer)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + super::lower_component_returns(&mut body, &mut types, true, |_| false, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!(!body.instructions.iter().enumerate().any(|(i, inst)| { + live.instructions[i] && matches!(inst.op, Op::AddressPack(_) | Op::AddressPart { .. }) + })); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} diff --git a/compiler-core/src/opt/aggregate_joins.rs b/compiler-core/src/opt/aggregate_joins.rs new file mode 100644 index 00000000..b419d60e --- /dev/null +++ b/compiler-core/src/opt/aggregate_joins.rs @@ -0,0 +1,226 @@ +//! Promote read-only carriers at joins as values, without allocation identity. +//! Join aliases and CFG inputs with union-find. Unknown producers, writes +//! and escaping uses reject the connected group. +use super::aggregates::Candidate; +use crate::ir::*; + +struct Groups { + parents: Vec, + ranks: Vec, +} +impl Groups { + fn root(&mut self, mut value: usize) -> usize { + while self.parents[value] != value { + self.parents[value] = self.parents[self.parents[value]]; + value = self.parents[value]; + } + value + } + fn join(&mut self, a: usize, b: usize) { + let a = self.root(a); + let b = self.root(b); + if a == b { + return; + } + let (a, b) = if self.ranks[a] < self.ranks[b] { + (b, a) + } else { + (a, b) + }; + self.parents[b] = a; + if self.ranks[a] == self.ranks[b] { + self.ranks[a] += 1; + } + } +} + +pub(super) fn promote( + body: &mut Body, + roots: &mut Vec, + candidates: &[Candidate], +) -> Vec { + let mut removed = vec![false; candidates.len()]; + if !body + .blocks + .iter() + .enumerate() + .any(|(index, block)| BlockId::new(index) != body.entry && !block.params.is_empty()) + { + return removed; + } + let count = body.values.len(); + let mut groups = Groups { + parents: (0..count).collect(), + ranks: vec![0; count], + }; + let mut users = crate::analysis::ValueUsers::new(count); + let mut known = vec![false; count]; + for (index, value) in body.values.iter().enumerate() { + if roots[index] != u32::MAX { + known[index] = true; + continue; + } + let source = match value.def { + ValueDef::Alias(source) => Some(source), + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::Reinterpret(source) | Op::Refine(source) => Some(source), + _ => None, + }, + ValueDef::Param(block) if block != body.entry => { + known[index] = true; + None + } + _ => None, + }; + if let Some(source) = source { + groups.join(index, source.index()); + users.connect(source, index); + known[index] = true; + } + } + for edge in &body.edges { + for (&source, &target) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + groups.join(source.index(), target.index()); + } + } + let group = (0..count).map(|i| groups.root(i)).collect::>(); + let mut layout = vec![None::; count]; + let mut valid = vec![true; count]; + for index in 0..count { + let group = group[index]; + valid[group] &= known[index]; + let root = roots[index]; + if root == u32::MAX { + continue; + } + let candidate = root as usize; + if candidates[candidate].fields.len() > 8 { + valid[group] = false; + } + if let Some(previous) = layout[group] { + valid[group] &= candidates[previous].fields == candidates[candidate].fields; + } else { + layout[group] = Some(candidate); + } + } + let position = |value: ValueId, field: MemberId| { + layout[group[value.index()]].and_then(|index| { + candidates[index] + .fields + .iter() + .position(|f| f == &body.fields[field.index()]) + }) + }; + for inst in &body.instructions { + inst.op.visit_uses(&body.args, |value| { + let allowed = match inst.op { + Op::Reinterpret(_) | Op::Refine(_) => true, + Op::GetField { object, field } => { + object == value && position(value, field).is_some() + } + _ => false, + }; + if !allowed { + valid[group[value.index()]] = false; + } + }); + } + for block in &body.blocks { + block + .terminator + .unwrap() + .visit_uses(|v| valid[group[v.index()]] = false); + } + let eligible = group + .iter() + .map(|&g| valid[g] && layout[g].is_some()) + .collect::>(); + if !eligible.iter().any(|&yes| yes) { + return removed; + } + let mut components = vec![None; count]; + let mut joins = Vec::new(); + for index in 0..count { + if !eligible[index] { + continue; + } + let root = roots[index]; + if root != u32::MAX { + components[index] = Some(candidates[root as usize].initial); + removed[root as usize] = true; + continue; + } + if let ValueDef::Param(block) = body.values[index].def { + let position = body.blocks[block.index()] + .params + .iter() + .position(|p| p.index() == index) + .unwrap(); + let fields = &candidates[layout[group[index]].unwrap()].fields; + let mut parts = Vec::with_capacity(fields.len()); + for field in fields { + let part = ValueId::new(body.values.len()); + body.values.push(Value { + ty: field.ty, + def: ValueDef::Param(block), + }); + body.blocks[block.index()].params.push(part); + parts.push(part); + } + components[index] = Some(List::append(&mut body.args, parts)); + joins.push((block, position)); + } + } + users.propagate(&eligible, &mut components); + let predecessors = body.predecessors(); + for (block, position) in joins { + for &(_, edge) in &predecessors[block.index()] { + let source = body.edges[edge.index()].args[position]; + let parts = components[source.index()].unwrap(); + body.edges[edge.index()] + .args + .extend_from_slice(&body.args[parts.range()]); + } + } + for inst in &mut body.instructions { + if let Some(value) = inst.result { + let root = roots[value.index()]; + if root != u32::MAX && removed[root as usize] { + let id = ConstId::new(body.constants.len()); + body.constants + .push(Constant::Null(body.values[value.index()].ty)); + inst.op = Op::Constant(id); + roots[value.index()] = u32::MAX; + continue; + } + } + if let Op::GetField { object, field } = inst.op + && eligible[object.index()] + { + let fields = &candidates[layout[group[object.index()]].unwrap()].fields; + let position = fields + .iter() + .position(|f| f == &body.fields[field.index()]) + .unwrap(); + let parts = components[object.index()].unwrap(); + inst.op = Op::Reinterpret(body.args[parts.range()][position]); + } + } + for block in &mut body.blocks { + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator + && matches!( + body.instructions[inst.index()].op, + Op::Constant(_) | Op::Reinterpret(_) + ) + { + block.instructions.push(inst); + block.terminator = Some(Terminator::Jump(normal)); + } + } + roots.resize(body.values.len(), u32::MAX); + removed +} diff --git a/compiler-core/src/opt/aggregates.rs b/compiler-core/src/opt/aggregates.rs new file mode 100644 index 00000000..c07e33a5 --- /dev/null +++ b/compiler-core/src/opt/aggregates.rs @@ -0,0 +1,194 @@ +//! Promote private generated carriers to SSA fields. +//! Require supplied constructor layouts. Arbitrary JVM constructors can run code. +use crate::ir::*; + +const NONE: u32 = u32::MAX; + +pub(super) struct Candidate { + pub(super) fields: Vec, + pub(super) initial: List, +} + +pub fn promote_aggregates( + mut body: Body, + types: &Types, + mut layout: impl FnMut(&str) -> Option>, +) -> Result { + super::owned_copies::reuse_readonly_copies(&mut body, types, &mut layout); + super::value_copies::expand_flat_copies(&mut body, types, &mut layout); + if !body.instructions.iter().any(|inst| { + matches!( + inst.op, + Op::Call { + kind: CallKind::Constructor, + .. + } + ) + }) { + return Ok(body); + } + let mut roots = vec![NONE; body.values.len()]; + let mut candidates = Vec::new(); + for inst in &body.instructions { + let Op::Call { + method, + kind: CallKind::Constructor, + args, + } = inst.op + else { + continue; + }; + let Some(fields) = layout(&body.methods[method.index()].owner) else { + continue; + }; + if fields.len() != args.len as usize + || fields + .iter() + .zip(&body.args[args.range()]) + .any(|(f, &v)| f.ty != body.value_type(v)) + { + continue; + } + roots[inst.result.unwrap().index()] = candidates.len() as u32; + candidates.push(Candidate { + fields, + initial: args, + }); + } + if candidates.is_empty() { + return Ok(body); + } + let mut escaped = super::aggregate_joins::promote(&mut body, &mut roots, &candidates); + let origins = crate::analysis::origins(&body, &roots); + let position = |index: usize, field: MemberId| { + candidates[index] + .fields + .iter() + .position(|f| f == &body.fields[field.index()]) + }; + for inst in &body.instructions { + inst.op.visit_uses(&body.args, |value| { + let index = origins[value.index()]; + if index == NONE { + return; + } + let index = index as usize; + let allowed = match inst.op { + Op::Reinterpret(_) | Op::Refine(_) => true, + Op::GetField { object, field } => { + object == value && position(index, field).is_some() + } + Op::SetField { + object, + field, + value: stored, + } => object == value && stored != value && position(index, field).is_some(), + _ => false, + }; + escaped[index] |= !allowed; + }); + } + let mut escape = |value: ValueId| { + let index = origins[value.index()]; + if index != NONE { + escaped[index as usize] = true; + } + }; + for edge in &body.edges { + for (&value, ¶m) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + if origins[value.index()] != origins[param.index()] { + escape(value); + } + } + } + for block in &body.blocks { + block.terminator.unwrap().visit_uses(&mut escape); + } + if escaped.iter().all(|&e| e) { + return Ok(body); + } + let mut builder = Builder::from_body(body, types); + let variables: Vec<_> = candidates + .iter() + .enumerate() + .map(|(index, candidate)| { + (!escaped[index]).then(|| { + candidate + .fields + .iter() + .map(|field| builder.variable(field.ty)) + .collect::>() + }) + }) + .collect(); + for block_index in 0..builder.body.blocks.len() { + builder.switch_to(BlockId::new(block_index)); + let instructions = builder.body.blocks[block_index].instructions.clone(); + let invoke = match builder.body.blocks[block_index].terminator.unwrap() { + Terminator::Invoke { inst, .. } => Some(inst), + _ => None, + }; + for id in instructions.into_iter().chain(invoke) { + let inst = builder.body.instructions[id.index()]; + if let Some(result) = inst.result { + let index = roots[result.index()]; + if index != NONE + && let Some(vars) = &variables[index as usize] + { + let initial = candidates[index as usize].initial; + for (&var, pos) in vars.iter().zip(initial.range()) { + builder.define(var, builder.body.args[pos]); + } + let constant = ConstId::new(builder.body.constants.len()); + builder + .body + .constants + .push(Constant::Null(builder.body.value_type(result))); + builder.body.instructions[id.index()].op = Op::Constant(constant); + continue; + } + } + let (object, field) = match inst.op { + Op::GetField { object, field } | Op::SetField { object, field, .. } => { + (object, field) + } + _ => continue, + }; + let index = origins[object.index()]; + if index == NONE { + continue; + } + let Some(vars) = &variables[index as usize] else { + continue; + }; + let position = candidates[index as usize] + .fields + .iter() + .position(|f| f == &builder.body.fields[field.index()]) + .unwrap(); + let variable = vars[position]; + builder.body.instructions[id.index()].op = match inst.op { + Op::GetField { .. } => Op::Reinterpret(builder.read(variable)), + Op::SetField { value, .. } => { + builder.define(variable, value); + Op::Nop + } + _ => unreachable!(), + }; + } + if let Some(Terminator::Invoke { inst, normal, .. }) = + builder.body.blocks[block_index].terminator + && !builder.body.instructions[inst.index()] + .op + .may_throw(&builder.body, types) + { + builder.body.blocks[block_index].instructions.push(inst); + builder.body.blocks[block_index].terminator = Some(Terminator::Jump(normal)); + } + } + builder.finish() +} diff --git a/compiler-core/src/opt/aggregates_tests.rs b/compiler-core/src/opt/aggregates_tests.rs new file mode 100644 index 00000000..0622820a --- /dev/null +++ b/compiler-core/src/opt/aggregates_tests.rs @@ -0,0 +1,700 @@ +use super::promote_aggregates; +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +#[test] +fn fresh_arrays_transfer_after_initialization_but_not_with_later_or_escaping_uses() { + for mode in ["last", "later", "escape", "loop", "invoke"] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let array = types.intern(Type::Array(int)); + let mut b = Builder::new(&types, array); + let repeat = b.parameter(b.current(), boolean); + let length = b.constant(int, Scalar::integer(ScalarType::I32, 2).unwrap()); + let zero = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let source = b.emit(Op::NewArray(length), Some(array)).unwrap(); + b.emit( + Op::ArraySet { + array: source, + index: zero, + value: length, + native: true, + }, + None, + ); + let value = b + .emit( + Op::ArrayGet { + array: source, + index: zero, + native: true, + }, + Some(int), + ) + .unwrap(); + if mode == "escape" { + let method = b.method(MethodRef { + owner: "Consumer".into(), + name: "retain".into(), + params: vec![array], + returns: int, + interface: false, + }); + let args = b.args([source]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(int), + ); + } + if mode == "loop" { + let header = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + } + let handler = (mode == "invoke").then(|| b.create_block()); + let copy = if let Some(handler) = handler { + b.invoke(Op::CopyValue(source), Some(array), handler) + .unwrap() + } else { + b.emit(Op::CopyValue(source), Some(array)).unwrap() + }; + if mode == "later" { + // Mutating the transferred result must not change the old source. + b.emit( + Op::ArraySet { + array: copy, + index: zero, + value: zero, + native: true, + }, + None, + ); + let later = b + .emit( + Op::ArrayGet { + array: source, + index: zero, + native: true, + }, + Some(int), + ) + .unwrap(); + b.emit( + Op::ArraySet { + array: copy, + index: zero, + value: later, + native: true, + }, + None, + ); + } else { + b.emit( + Op::ArraySet { + array: copy, + index: zero, + value, + native: true, + }, + None, + ); + } + if mode == "loop" { + let header = b.current(); + let done = b.create_block(); + b.branch(repeat, header, done); + b.switch_to(done); + } + b.terminate(Terminator::Return(Some(copy))); + if let Some(handler) = handler { + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + } + let body = promote_aggregates(b.finish().unwrap(), &types, |_| None).unwrap(); + verify(&body, &types).unwrap(); + let copies = body + .instructions + .iter() + .filter(|i| matches!(i.op, Op::CopyValue(_))) + .count(); + assert_eq!(copies, usize::from(mode != "last"), "{mode}"); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn redundant_array_copies_keep_escaping_and_loop_snapshots_distinct() { + for mode in ["local", "escape", "loop", "object"] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let element = if mode == "object" { object } else { int }; + let array = types.intern(Type::Array(element)); + let mut b = Builder::new(&types, array); + let input = b.parameter(b.current(), array); + let repeat = b.parameter(b.current(), boolean); + let first = b.emit(Op::CopyValue(input), Some(array)).unwrap(); + if mode == "escape" { + let method = b.method(MethodRef { + owner: "Consumer".into(), + name: "retain".into(), + params: vec![array], + returns: int, + interface: false, + }); + let args = b.args([first]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(int), + ); + } + if mode == "loop" { + let header = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + } + let second = b.emit(Op::CopyValue(first), Some(array)).unwrap(); + if mode == "loop" { + let header = b.current(); + let done = b.create_block(); + b.branch(repeat, header, done); + b.switch_to(done); + } + b.terminate(Terminator::Return(Some(second))); + let body = promote_aggregates(b.finish().unwrap(), &types, |_| None).unwrap(); + verify(&body, &types).unwrap(); + let copies = body + .instructions + .iter() + .filter(|i| matches!(i.op, Op::CopyValue(_))) + .count(); + assert_eq!(copies, if mode == "local" { 1 } else { 2 }, "{mode}"); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +fn construct( + b: &mut Builder<'_>, + owner: TypeId, + fields: &[FieldRef], + values: &[ValueId], +) -> ValueId { + let method = b.method(MethodRef { + owner: "Pair".into(), + name: "".into(), + params: fields.iter().map(|f| f.ty).collect(), + returns: TypeId::new(0), + interface: false, + }); + let args = b.args(values.iter().copied()); + b.emit( + Op::Call { + method, + kind: CallKind::Constructor, + args, + }, + Some(owner), + ) + .unwrap() +} + +#[test] +fn private_mutable_fields_become_loop_parameters() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let symbol = types.symbol("Pair"); + let owner = types.intern(Type::Class(symbol)); + let fields = ["count", "sum"].map(|name| FieldRef { + owner, + name: name.into(), + ty: int, + is_static: false, + }); + let mut b = Builder::new(&types, int); + let limit = b.parameter(b.current(), int); + let zero = b.constant(int, Scalar::integer(ScalarType::I64, 0).unwrap()); + let pair = construct(&mut b, owner, &fields, &[zero, zero]); + let count = b.field(fields[0].clone()); + let sum = b.field(fields[1].clone()); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let n = b + .emit( + Op::GetField { + object: pair, + field: count, + }, + Some(int), + ) + .unwrap(); + let test = b + .emit( + Op::Binary { + op: BinaryOp::Lt, + left: n, + right: limit, + }, + Some(boolean), + ) + .unwrap(); + b.branch(test, step, done); + b.switch_to(step); + let previous = b + .emit( + Op::GetField { + object: pair, + field: sum, + }, + Some(int), + ) + .unwrap(); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: n, + right: previous, + }, + Some(int), + ) + .unwrap(); + b.emit( + Op::SetField { + object: pair, + field: sum, + value: next, + }, + None, + ); + let one = b.constant(int, Scalar::integer(ScalarType::I64, 1).unwrap()); + let n = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: n, + right: one, + }, + Some(int), + ) + .unwrap(); + b.emit( + Op::SetField { + object: pair, + field: count, + value: n, + }, + None, + ); + b.jump(header, vec![]); + b.switch_to(done); + let result = b + .emit( + Op::GetField { + object: pair, + field: sum, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let body = promote_aggregates(b.finish().unwrap(), &types, |_| Some(fields.to_vec())).unwrap(); + verify(&body, &types).unwrap(); + assert_eq!(body.blocks[header.index()].params.len(), 2); + assert!(!body.instructions.iter().any(|inst| matches!( + inst.op, + Op::Call { .. } | Op::GetField { .. } | Op::SetField { .. } + ))); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|op| matches!(op, ristretto_classfile::attributes::Instruction::New(_))) + ); +} + +#[test] +fn keeps_returned_carriers_and_unknown_constructors() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let symbol = types.symbol("Pair"); + let owner = types.intern(Type::Class(symbol)); + let field = FieldRef { + owner, + name: "value".into(), + ty: int, + is_static: false, + }; + let mut b = Builder::new(&types, owner); + let value = b.parameter(b.current(), int); + let pair = construct(&mut b, owner, &[field.clone()], &[value]); + b.terminate(Terminator::Return(Some(pair))); + let body = b.finish().unwrap(); + assert_eq!( + body, + promote_aggregates(body.clone(), &types, |_| Some(vec![field.clone()])).unwrap() + ); + assert_eq!( + body, + promote_aggregates(body.clone(), &types, |_| None).unwrap() + ); +} + +#[test] +fn distinct_readonly_values_join_as_fields() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let fields = ["a", "b"].map(|name| FieldRef { + owner: pair, + name: name.into(), + ty: int, + is_static: false, + }); + for escaping in [false, true] { + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), boolean); + let unknown = escaping.then(|| b.parameter(b.current(), pair)); + let left = b.create_block(); + let right = b.create_block(); + let join = b.create_block(); + let merged = b.parameter(join, pair); + b.branch(condition, left, right); + b.switch_to(left); + let a = b.constant(int, Scalar::integer(ScalarType::I64, 13).unwrap()); + let c = b.constant(int, Scalar::integer(ScalarType::I64, 17).unwrap()); + let value = construct(&mut b, pair, &fields, &[a, c]); + b.jump(join, vec![value]); + b.switch_to(right); + let a = b.constant(int, Scalar::integer(ScalarType::I64, 19).unwrap()); + let c = b.constant(int, Scalar::integer(ScalarType::I64, 23).unwrap()); + let value = unknown.unwrap_or_else(|| construct(&mut b, pair, &fields, &[a, c])); + b.jump(join, vec![value]); + b.switch_to(join); + let field = b.field(fields[1].clone()); + let result = b + .emit( + Op::GetField { + object: merged, + field, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let body = + promote_aggregates(b.finish().unwrap(), &types, |_| Some(fields.to_vec())).unwrap(); + verify(&body, &types).unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + let constructed = code + .instructions + .iter() + .any(|i| matches!(i, ristretto_classfile::attributes::Instruction::New(_))); + assert_eq!( + constructed, escaping, + "an unknown input must keep its identity" + ); + if !escaping { + assert_eq!(body.blocks[join.index()].params.len(), 3); + assert!( + !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::GetField { .. } | Op::Call { .. })) + ); + } + } +} + +#[test] +fn new_loop_values_are_not_confused_with_previous_iterations() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let fields = ["a", "b"].map(|name| FieldRef { + owner: pair, + name: name.into(), + ty: int, + is_static: false, + }); + for mutate in [false, true] { + let mut b = Builder::new(&types, int); + let again = b.parameter(b.current(), boolean); + let zero = b.constant(int, Scalar::integer(ScalarType::I64, 0).unwrap()); + let initial = construct(&mut b, pair, &fields, &[zero, zero]); + let cursor = b.variable(pair); + b.define(cursor, initial); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let previous = b.read(cursor); + b.branch(again, step, done); + b.switch_to(step); + let field = b.field(fields[1].clone()); + let value = b + .emit( + Op::GetField { + object: previous, + field, + }, + Some(int), + ) + .unwrap(); + let one = b.constant(int, Scalar::integer(ScalarType::I64, 1).unwrap()); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: value, + right: one, + }, + Some(int), + ) + .unwrap(); + let new = construct(&mut b, pair, &fields, &[value, next]); + if mutate { + b.emit( + Op::SetField { + object: previous, + field, + value: one, + }, + None, + ); + } + b.define(cursor, new); + b.jump(header, vec![]); + b.switch_to(done); + let result = b + .emit( + Op::GetField { + object: previous, + field, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let body = + promote_aggregates(b.finish().unwrap(), &types, |_| Some(fields.to_vec())).unwrap(); + verify(&body, &types).unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + let constructed = code + .instructions + .iter() + .any(|i| matches!(i, ristretto_classfile::attributes::Instruction::New(_))); + assert_eq!( + constructed, mutate, + "aliased mutable identities must not become snapshots" + ); + } +} + +#[test] +fn flat_copy_snapshots_survive_source_mutation_without_carriers() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let symbol = types.symbol("Pair"); + let pair = types.intern(Type::Class(symbol)); + let fields = ["a", "b"].map(|name| FieldRef { + owner: pair, + name: name.into(), + ty: int, + is_static: false, + }); + let mut b = Builder::new(&types, int); + let seven = b.constant(int, Scalar::integer(ScalarType::I64, 7).unwrap()); + let nine = b.constant(int, Scalar::integer(ScalarType::I64, 9).unwrap()); + let original = construct(&mut b, pair, &fields, &[seven, nine]); + let copy = b.emit(Op::CopyValue(original), Some(pair)).unwrap(); + let second = b.emit(Op::CopyValue(copy), Some(pair)).unwrap(); + let field = b.field(fields[0].clone()); + b.emit( + Op::SetField { + object: original, + field, + value: nine, + }, + None, + ); + b.emit( + Op::SetField { + object: copy, + field, + value: nine, + }, + None, + ); + let result = b + .emit( + Op::GetField { + object: second, + field, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let body = promote_aggregates(b.finish().unwrap(), &types, |_| Some(fields.to_vec())).unwrap(); + verify(&body, &types).unwrap(); + assert!(!body.instructions.iter().any(|i| matches!( + i.op, + Op::CopyValue(_) | Op::Call { .. } | Op::GetField { .. } | Op::SetField { .. } + ))); + let Some(Terminator::Return(Some(result))) = body.blocks[body.entry.index()].terminator else { + panic!("return"); + }; + let mut snapshot = body.resolve(result); + while let ValueDef::Inst(id) = body.values[snapshot.index()].def { + let Op::Reinterpret(source) = body.instructions[id.index()].op else { + break; + }; + snapshot = body.resolve(source); + } + assert_eq!(snapshot, body.resolve(seven)); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, ristretto_classfile::attributes::Instruction::New(_))) + ); +} + +#[test] +fn nullable_or_nested_owned_copies_keep_the_general_operation() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let symbol = types.symbol("Pair"); + let pair = types.intern(Type::Class(symbol)); + for nested in [false, true] { + let field = FieldRef { + owner: pair, + name: "a".into(), + ty: if nested { pair } else { int }, + is_static: false, + }; + let mut b = Builder::new(&types, pair); + let input = b.parameter(b.current(), pair); + let source = if nested { + construct(&mut b, pair, &[field.clone()], &[input]) + } else { + input + }; + let copy = b.emit(Op::CopyValue(source), Some(pair)).unwrap(); + b.terminate(Terminator::Return(Some(copy))); + let body = + promote_aggregates(b.finish().unwrap(), &types, |_| Some(vec![field.clone()])).unwrap(); + verify(&body, &types).unwrap(); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::CopyValue(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn scalar_copy_reads_share_only_without_intervening_mutation_or_escape() { + for mode in ["plain", "store", "call", "escape", "branch"] { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let fields = vec![FieldRef { + owner: pair, + name: "a".into(), + ty: int, + is_static: false, + }]; + let mut b = Builder::new(&types, int); + let input = b.parameter(b.current(), pair); + let zero = b.constant(int, Scalar::integer(ScalarType::I64, 0).unwrap()); + construct(&mut b, pair, &fields, &[zero]); + let copy = b.emit(Op::CopyValue(input), Some(pair)).unwrap(); + let field = b.field(fields[0].clone()); + if mode == "store" { + b.emit( + Op::SetField { + object: input, + field, + value: zero, + }, + None, + ); + } + if mode == "call" || mode == "escape" { + let method = b.method(MethodRef { + owner: "Opaque".into(), + name: "write".into(), + params: vec![pair], + returns: int, + interface: false, + }); + let args = b.args([if mode == "escape" { copy } else { input }]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(int), + ); + } + if mode == "branch" { + let next = b.create_block(); + b.jump(next, vec![]); + b.switch_to(next); + } + let value = b + .emit( + Op::GetField { + object: copy, + field, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(value))); + let body = + promote_aggregates(b.finish().unwrap(), &types, |_| Some(fields.clone())).unwrap(); + verify(&body, &types).unwrap(); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::CopyValue(_))), + mode != "plain", + "{mode}" + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} diff --git a/compiler-core/src/opt/array_locations.rs b/compiler-core/src/opt/array_locations.rs new file mode 100644 index 00000000..562ae39d --- /dev/null +++ b/compiler-core/src/opt/array_locations.rs @@ -0,0 +1,268 @@ +//! Use JVM arrays for scalar locations whose storage never escapes. +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +pub fn lower_array_locations(body: &mut Body, types: &mut Types) -> bool { + let roots = crate::analysis::native_array_roots(body, types, &super::live(body, types)); + if roots.is_empty() { + return false; + } + let mut changed = false; + let long = types.scalar(ScalarType::I64); + let int = types.scalar(ScalarType::I32); + let mut prefixes = rustc_hash::FxHashMap::default(); + for index in 0..body.instructions.len() { + let inst = body.instructions[index]; + let observers = match inst.op { + Op::LocationEqual(args) | Op::LocationTag(args) => Some(( + args, + if matches!(inst.op, Op::LocationEqual(_)) { + "sameLocation" + } else { + "nullableLocationTag" + }, + )), + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } if body.methods[method.index()].owner == "org/rustlang/runtime/Pointer" => { + Some((args, body.methods[method.index()].name.as_str())) + } + _ => None, + }; + if let Some((args, name)) = observers { + let values = body.args[args.range()].to_vec(); + let root = values + .first() + .and_then(|v| roots.get(v.index())) + .copied() + .flatten(); + if let Some((allocation, element)) = root { + if matches!(name, "nullableLocationTag" | "nullableViewLocationTag") { + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I64, 1).unwrap(), + )); + body.instructions[index].op = Op::Constant(constant); + changed = true; + continue; + } + if matches!( + name, + "sameLocation" + | "offsetLocations" + | "offsetLocationsUnsigned" + | "byteOffsetLocations" + | "byteOffsetLocationsUnsigned" + ) && values.len() >= 4 + && roots + .get(values[2].index()) + .copied() + .flatten() + .is_some_and(|(other, _)| other == allocation) + { + let op = if name == "sameLocation" { + BinaryOp::Eq + } else { + BinaryOp::Sub + }; + let difference = Op::Binary { + op, + left: values[1], + right: values[3], + }; + if name.starts_with("offsetLocations") { + let size = StorageSlot::scalar(element, types).unwrap().size; + if values.len() != 5 + || body.scalar_value(values[4]).map(|s| s.bits()) + != Some(u128::from(size)) + { + continue; + } + let (prefix, delta) = super::append_value(body, difference, long); + prefixes.insert(InstId::new(index), vec![prefix]); + body.instructions[index].op = Op::Binary { + op: BinaryOp::Div, + left: delta, + right: values[4], + }; + } else { + body.instructions[index].op = difference; + } + changed = true; + continue; + } + } + } + let (root, offset, view_index, stored) = match inst.op { + Op::ViewRoot { + backing, + codec: None, + .. + } if roots.get(backing.index()).copied().flatten().is_some() => { + body.instructions[index].op = Op::Reinterpret(backing); + changed = true; + continue; + } + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } if body.methods[method.index()].owner == "org/rustlang/runtime/Pointer" => { + let values = &body.args[args.range()]; + let Some(&root) = values.first() else { + continue; + }; + if roots[root.index()].is_none() { + continue; + } + match body.methods[method.index()].name.as_str() { + "scalarSliceRoot" | "locationSliceBacking" => { + body.instructions[index].op = Op::Reinterpret(root); + changed = true; + continue; + } + "locationSliceOffset" => (root, values[1], None, None), + _ => continue, + } + } + Op::LoadAddress(parts) => ( + body.args[parts.start as usize], + body.args[parts.start as usize + 1], + None, + None, + ), + Op::StoreAddress { parts, value } => ( + body.args[parts.start as usize], + body.args[parts.start as usize + 1], + None, + Some(value), + ), + Op::ViewGet(parts) | Op::ViewSet { parts, .. } => { + let values = &body.args[parts.range()]; + ( + values[0], + values[1], + Some(values[2]), + if let Op::ViewSet { value, .. } = inst.op { + Some(value) + } else { + None + }, + ) + } + _ => continue, + }; + let Some((_, element)) = roots.get(root.index()).copied().flatten() else { + continue; + }; + let mut prefix = Vec::new(); + let position = if let Some(view_index) = view_index { + let (instruction, position) = super::append_value( + body, + Op::Binary { + op: BinaryOp::Add, + left: offset, + right: view_index, + }, + int, + ); + prefix.push(instruction); + position + } else { + let size = StorageSlot::scalar(element, types).unwrap().size; + if !aligned_offset(body, offset, size, 0) { + continue; + } + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I64, u128::from(size)).unwrap(), + )); + let (size_inst, stride) = super::append_value(body, Op::Constant(constant), long); + let (quotient_inst, quotient) = super::append_value( + body, + Op::Binary { + op: BinaryOp::Div, + left: offset, + right: stride, + }, + long, + ); + let (position_inst, position) = super::append_value(body, Op::Cast(quotient), int); + prefix.extend([size_inst, quotient_inst, position_inst]); + position + }; + let array = if matches!(inst.op, Op::Call { .. }) { + root + } else { + let ty = types.find(Type::Array(element)).unwrap(); + let (cast, array) = super::append_value(body, Op::Refine(root), ty); + prefix.push(cast); + array + }; + body.instructions[index].op = if matches!(inst.op, Op::Call { .. }) { + Op::Reinterpret(position) + } else if let Some(value) = stored { + Op::ArraySet { + array, + index: position, + value, + native: true, + } + } else { + Op::ArrayGet { + array, + index: position, + native: true, + } + }; + prefixes.insert(InstId::new(index), prefix); + changed = true; + } + for block in &mut body.blocks { + let previous = std::mem::take(&mut block.instructions); + for id in previous { + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(Terminator::Invoke { inst, .. }) = block.terminator + && let Some(prefix) = prefixes.remove(&inst) + { + block.instructions.extend(prefix); + } + } + changed +} + +fn aligned_offset(body: &Body, value: ValueId, size: u32, depth: usize) -> bool { + if size == 1 { + return true; + } + if let Some(value) = body.scalar_value(value) { + return value.bits() % u128::from(size) == 0; + } + if depth == 16 { + return false; + } + let ValueDef::Inst(id) = body.values[body.resolve(value).index()].def else { + return false; + }; + let aligned = |value| aligned_offset(body, value, size, depth + 1); + match body.instructions[id.index()].op { + Op::Reinterpret(value) | Op::Refine(value) => aligned(value), + Op::Binary { + op: BinaryOp::Mul, + left, + right, + } => aligned(left) || aligned(right), + Op::Binary { + op: BinaryOp::Add | BinaryOp::Sub, + left, + right, + } => aligned(left) && aligned(right), + _ => false, + } +} diff --git a/compiler-core/src/opt/array_locations_tests.rs b/compiler-core/src/opt/array_locations_tests.rs new file mode 100644 index 00000000..c6f6fc11 --- /dev/null +++ b/compiler-core/src/opt/array_locations_tests.rs @@ -0,0 +1,161 @@ +use super::lower_array_locations; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn only_private_arrays_use_native_locations() { + for (escape, byte_offset) in [(false, 4), (true, 4), (false, 1)] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::I64); + let array_ty = types.intern(Type::Array(int)); + let symbol = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(symbol)); + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, int); + let size = b.constant(int, Scalar::integer(ScalarType::I32, 4).unwrap()); + let offset = b.constant(long, Scalar::integer(ScalarType::I64, byte_offset).unwrap()); + let array = b.emit(Op::NewArray(size), Some(array_ty)).unwrap(); + let root = b + .emit( + Op::ViewRoot { + backing: array, + size: 4, + codec: None, + }, + Some(object), + ) + .unwrap(); + if escape { + let method = b.method(MethodRef { + owner: "Unknown".into(), + name: "borrow".into(), + params: vec![object], + returns: unit, + interface: false, + }); + let args = b.args([root]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + None, + ); + } + let parts = b.args([root, offset]); + b.emit(Op::StoreAddress { parts, value: size }, None); + let value = b.emit(Op::LoadAddress(parts), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + assert_eq!(lower_array_locations(&mut body, &mut types), !escape); + verify(&body, &types).unwrap(); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::ArrayGet { native: true, .. })), + !escape && byte_offset == 4 + ); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::ArraySet { native: true, .. })), + !escape && byte_offset == 4 + ); + } +} + +#[test] +fn pointer_differences_require_the_same_allocation() { + for shared in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::I64); + let array_ty = types.intern(Type::Array(int)); + let symbol = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(symbol)); + let mut b = Builder::new(&types, long); + let size = b.constant(int, Scalar::integer(ScalarType::I32, 4).unwrap()); + let stride = b.constant(long, Scalar::integer(ScalarType::I64, 4).unwrap()); + let zero = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + let left = b.emit(Op::NewArray(size), Some(array_ty)).unwrap(); + let right = if shared { + left + } else { + b.emit(Op::NewArray(size), Some(array_ty)).unwrap() + }; + let left = b.emit(Op::Reinterpret(left), Some(object)).unwrap(); + let right = b.emit(Op::Reinterpret(right), Some(object)).unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "offsetLocations".into(), + params: vec![object, long, object, long, long], + returns: long, + interface: false, + }); + let args = b.args([left, stride, right, zero, stride]); + let distance = b + .emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(long), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(distance))); + let mut body = b.finish().unwrap(); + assert_eq!(lower_array_locations(&mut body, &mut types), shared); + verify(&body, &types).unwrap(); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::Call { .. })), + !shared + ); + } +} + +#[test] +fn inactive_payloads_do_not_hide_array_origins_but_null_does() { + for uninit in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let long = types.scalar(ScalarType::I64); + let array_ty = types.intern(Type::Array(int)); + let symbol = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(symbol)); + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), boolean); + let one = b.constant(int, Scalar::integer(ScalarType::I32, 1).unwrap()); + let offset = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + let array = b.emit(Op::NewArray(one), Some(array_ty)).unwrap(); + let root = b.emit(Op::Reinterpret(array), Some(object)).unwrap(); + let constant = ConstId::new(b.body.constants.len()); + b.body.constants.push(if uninit { + Constant::Uninit(object) + } else { + Constant::Null(object) + }); + let empty = b.emit(Op::Constant(constant), Some(object)).unwrap(); + let some = b.create_block(); + let none = b.create_block(); + let join = b.create_block(); + let payload = b.parameter(join, object); + b.branch(condition, some, none); + b.switch_to(some); + b.jump(join, vec![root]); + b.switch_to(none); + b.jump(join, vec![empty]); + b.switch_to(join); + let parts = b.args([payload, offset]); + let loaded = b.emit(Op::LoadAddress(parts), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(loaded))); + let mut body = b.finish().unwrap(); + assert_eq!(lower_array_locations(&mut body, &mut types), uninit); + verify(&body, &types).unwrap(); + } +} diff --git a/compiler-core/src/opt/array_view_tests.rs b/compiler-core/src/opt/array_view_tests.rs new file mode 100644 index 00000000..cfd2ffd1 --- /dev/null +++ b/compiler-core/src/opt/array_view_tests.rs @@ -0,0 +1,166 @@ +use super::*; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn zero_sized_and_unsized_views_keep_their_metadata_carrier() { + for zero_sized in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let length = types.scalar(ScalarType::U64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let pointee = if zero_sized { + types.intern(Type::Opaque(0)) + } else { + let name = types.symbol("test/Dst"); + types.intern(Type::Interface(name)) + }; + let pointer = types.intern(Type::Pointer(pointee)); + let mut b = Builder::new(&types, pointer); + let backing = b.parameter(b.current(), object); + let start = b.parameter(b.current(), int); + let count = b.parameter(b.current(), length); + let parts = b.args([backing, start, count]); + let address = b + .emit( + Op::ViewAddress { + parts, + size: if zero_sized { 0 } else { 1 }, + codec: None, + }, + Some(pointer), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(address))); + let mut body = b.finish().unwrap(); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!( + body.instructions + .iter() + .any(|inst| matches!(inst.op, Op::ViewAddress { .. })) + ); + assert!(live(&body, &types).values[count.index()]); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn slice_to_fixed_array_to_scalar_slice_keeps_the_source_byte_displacement() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let int = types.scalar(ScalarType::I32); + let length = types.scalar(ScalarType::U64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let byte_pointer = types.intern(Type::Pointer(byte)); + let array = types.intern(Type::Array(byte)); + let codec = types.symbol("org/rustlang/runtime/ArrayMemoryCodec#array#[B#8"); + let layout = types.layout(AddressLayout { + value: array, + size: 8, + codec: Some(codec), + }); + let fixed_pointer = types.intern(Type::Pointer(layout)); + let erased_fixed = types.intern(Type::Pointer(array)); + let mut b = Builder::new(&types, byte); + let backing = b.parameter(b.current(), object); + let start = b.parameter(b.current(), int); + let count = b.constant(length, Scalar::integer(ScalarType::U64, 8).unwrap()); + let parts = b.args([backing, start, count]); + let data = b + .emit( + Op::ViewAddress { + parts, + size: 1, + codec: None, + }, + Some(byte_pointer), + ) + .unwrap(); + let fixed = b + .emit( + Op::RetypeAddress { + pointer: data, + size: 8, + codec: Some(codec), + }, + Some(fixed_pointer), + ) + .unwrap(); + let erased = b.emit(Op::Reinterpret(fixed), Some(erased_fixed)).unwrap(); + let bytes = b.emit(Op::Cast(erased), Some(byte_pointer)).unwrap(); + let root = b + .emit( + Op::AddressViewPart { + address: bytes, + index: 0, + }, + Some(object), + ) + .unwrap(); + let offset = b + .emit( + Op::AddressViewPart { + address: bytes, + index: 1, + }, + Some(int), + ) + .unwrap(); + let zero = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let parts = b.args([root, offset, zero]); + let first = b.emit(Op::ViewGet(parts), Some(byte)).unwrap(); + b.terminate(Terminator::Return(Some(first))); + let mut body = b.finish().unwrap(); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + simplify_components(&mut body, &types); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::AddressPack(_) + | Op::TypedAddressPack { .. } + | Op::RetypeAddress { .. } + | Op::ViewAddress { .. } + | Op::TypedAddressViewPart { .. } + )) + ); + assert!(body.instructions.iter().any(|inst| matches!( + inst.op, + Op::ViewRoot { + size: 1, + codec: None, + .. + } + ))); + // The eight-byte array layout must not scale the slice start by eight. + // Its folded byte displacement must equal i64(start). + let Op::Call { args, .. } = body + .instructions + .iter() + .find(|inst| { + matches!(inst.op, + Op::Call { method, .. } if body.methods[method.index()].name == "locationSliceOffset") + }) + .unwrap() + .op + else { + unreachable!() + }; + let displacement = body.resolve(body.args[args.start as usize + 1]); + let ValueDef::Inst(id) = body.values[displacement.index()].def else { + panic!() + }; + assert!(matches!(body.instructions[id.index()].op, Op::Cast(value) if value == start)); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} diff --git a/compiler-core/src/opt/borrowed_memory.rs b/compiler-core/src/opt/borrowed_memory.rs new file mode 100644 index 00000000..724fddf8 --- /dev/null +++ b/compiler-core/src/opt/borrowed_memory.rs @@ -0,0 +1,122 @@ +//! Use the component ABI for stored borrows, calls, returns and fields. +//! Run after local promotion so eliminated cells gain no runtime calls. +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +const OWNER: &str = "org/rustlang/runtime/Pointer"; + +pub fn borrowed_memory_method(method: &MethodRef) -> bool { + method.owner == OWNER + && matches!( + method.name.as_str(), + "loadBorrowedView" + | "loadBorrowedAddress" + | "storeBorrowedView" + | "storeBorrowedUtf8" + | "storeBorrowedAddress" + ) +} + +pub fn lower_borrowed_memory(body: &mut Body, types: &mut Types, debug: Option<&mut DebugInfo>) { + let mut methods = FxHashMap::default(); + let int = types.scalar(ScalarType::I32); + let unit = types.intern(Type::Unit); + let mut sizes = FxHashMap::default(); + let mut prologue = Vec::new(); + for index in 0..body.instructions.len() { + let instruction = body.instructions[index]; + let (pointer, value, ty) = match instruction.op { + Op::Load(pointer) | Op::LoadCopy(pointer) => { + (pointer, None, body.value_type(instruction.result.unwrap())) + } + Op::Store { pointer, value } => (pointer, Some(value), body.value_type(value)), + _ => continue, + }; + let Some(shape) = ComponentShape::of(types, ty) else { + continue; + }; + if shape == ComponentShape::TaggedI64 { + continue; + } + let parameter = body.value_type(pointer); + let mut params = vec![parameter]; + let mut args = vec![pointer]; + let (name, returns) = if let Some(value) = value { + params.push(ty); + args.push(value); + if shape == ComponentShape::View { + ( + if types.get(ty) == Some(Type::Str) { + "storeBorrowedUtf8" + } else { + "storeBorrowedView" + }, + unit, + ) + } else { + let size = match types.get(ty) { + Some(Type::Pointer(inner)) => crate::jvm::abi::address_plan(types, inner), + _ => unreachable!(), + }; + let constant = *sizes.entry(size).or_insert_with(|| { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, size.into()).unwrap(), + )); + let inst = InstId::new(body.instructions.len()); + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty: int, + def: ValueDef::Inst(inst), + }); + body.instructions.push(Inst { + op: Op::Constant(id), + result: Some(value), + }); + prologue.push(inst); + value + }); + params.push(int); + args.push(constant); + ("storeBorrowedAddress", unit) + } + } else { + ( + if shape == ComponentShape::View { + "loadBorrowedView" + } else { + "loadBorrowedAddress" + }, + ty, + ) + }; + let id = *methods + .entry((parameter, ty, value.is_some())) + .or_insert_with(|| { + body.methods.push(MethodRef { + owner: OWNER.into(), + name: name.into(), + params, + returns, + interface: false, + }); + body.methods.len() - 1 + }); + body.instructions[index].op = Op::Call { + method: MethodId::new(id), + kind: CallKind::JvmStatic, + args: List::append(&mut body.args, args), + }; + } + let count = prologue.len() as u32; + prologue.append(&mut body.blocks[body.entry.index()].instructions); + body.blocks[body.entry.index()].instructions = prologue; + if let Some(debug) = debug { + for event in &mut debug.events { + if event.block == body.entry { + event.position += count; + } + } + } +} diff --git a/compiler-core/src/opt/borrowed_memory_tests.rs b/compiler-core/src/opt/borrowed_memory_tests.rs new file mode 100644 index 00000000..c1372b78 --- /dev/null +++ b/compiler-core/src/opt/borrowed_memory_tests.rs @@ -0,0 +1,134 @@ +use super::*; +use crate::{ir::*, scalar::ScalarType}; + +#[test] +fn stored_borrows_cross_load_store_and_return_without_carriers() { + for view in 0..3 { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let borrowed = types.intern(match view { + 0 => Type::Pointer(byte), + 1 => Type::Slice(byte), + _ => Type::Str, + }); + let location = types.intern(Type::Pointer(borrowed)); + let mut builder = Builder::new(&types, borrowed); + let destination = builder.parameter(builder.current(), location); + let replacement = builder.parameter(builder.current(), borrowed); + let original = builder.emit(Op::Load(destination), Some(borrowed)).unwrap(); + builder.emit( + Op::Store { + pointer: destination, + value: replacement, + }, + None, + ); + builder.terminate(Terminator::Return(Some(original))); + let mut body = builder.finish().unwrap(); + lower_borrowed_memory(&mut body, &mut types, None); + lower_component_arguments(&mut body, &mut types, true, borrowed_memory_method, None); + lower_component_returns(&mut body, &mut types, true, borrowed_memory_method, None); + decompose_views(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + let mut calls = 0; + for (index, instruction) in body.instructions.iter().enumerate() { + if !live.instructions[index] { + continue; + } + assert!( + !matches!( + instruction.op, + Op::AddressPack(_) + | Op::ViewPack(_) + | Op::Load(_) + | Op::Store { .. } + | Op::NewArray(_) + ), + "{instruction:?}" + ); + if let Op::Call { method, .. } = instruction.op { + let method = &body.methods[method.index()]; + assert!( + method + .params + .iter() + .all(|&p| ComponentShape::of(&types, p).is_none()) + ); + assert!(ComponentShape::of(&types, method.returns).is_none()); + calls += 1; + } + } + assert_eq!(calls, 2); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn references_to_stored_borrows_keep_their_enclosing_owner() { + for view in [false, true] { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let borrowed = types.intern(if view { + Type::Slice(byte) + } else { + Type::Pointer(byte) + }); + let location = types.intern(Type::Pointer(borrowed)); + let name = types.symbol("Slot"); + let owner = types.intern(Type::Class(name)); + let aggregate = types.intern(Type::Pointer(owner)); + let mut builder = Builder::new(&types, borrowed); + let root = builder.parameter(builder.current(), aggregate); + let field = builder.field(FieldRef { + owner, + name: "borrow".into(), + ty: borrowed, + is_static: false, + }); + let projection = builder.projection(PointerProjection { + parent: None, + field, + offset: 8, + size: if view { 16 } else { 8 }, + codec: Some("layout".into()), + }); + let address = builder + .emit( + Op::Project { + base: root, + projection, + }, + Some(location), + ) + .unwrap(); + let value = builder.emit(Op::Load(address), Some(borrowed)).unwrap(); + builder.terminate(Terminator::Return(Some(value))); + let mut body = builder.finish().unwrap(); + lower_borrowed_memory(&mut body, &mut types, None); + lower_component_arguments(&mut body, &mut types, true, borrowed_memory_method, None); + lower_component_returns(&mut body, &mut types, true, borrowed_memory_method, None); + decompose_views(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::ProjectRoot { .. })) + ); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, instruction)| live.instructions[i] + && matches!( + instruction.op, + Op::Project { .. } | Op::AddressPack(_) | Op::ViewPack(_) + )) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} diff --git a/compiler-core/src/opt/cells.rs b/compiler-core/src/opt/cells.rs index 8a10d715..8f5fc9b8 100644 --- a/compiler-core/src/opt/cells.rs +++ b/compiler-core/src/opt/cells.rs @@ -1,45 +1,50 @@ //! Escape analysis and SSA promotion for private, explicitly initialized cells. use crate::ir::*; -const NONE: u32 = u32::MAX; +use crate::analysis::{NO_ORIGIN as NONE, origins}; fn cell_index(index: u32) -> Option { (index != NONE).then_some(index as usize) } -fn origins(body: &Body, roots: &[u32]) -> Vec { - let mut origins = roots.to_vec(); - let mut known = roots.iter().map(|&index| index != NONE).collect::>(); - let mut path = Vec::new(); - for index in 0..body.values.len() { - if known[index] { - continue; +struct Location { + cell: usize, + fields: Vec, +} + +fn location(body: &Body, types: &Types, origins: &[u32], mut value: ValueId) -> Option { + types.pointee(body.value_type(value))?; + let mut fields = Vec::new(); + for _ in 0..64 { + value = body.resolve(value); + if let Some(cell) = cell_index(origins[value.index()]) { + fields.reverse(); + return Some(Location { cell, fields }); } - let mut value = ValueId::new(index); - let root = loop { - path.push(value); - value = body.resolve(value); - if known[value.index()] { - break origins[value.index()]; - } - // Memoize negative results too. Marking before following a use - // bounds the walk even for malformed reinterpretation cycles. - known[value.index()] = true; - path.push(value); - let ValueDef::Inst(inst) = body.values[value.index()].def else { - break NONE; - }; - let Op::Reinterpret(input) = body.instructions[inst.index()].op else { - break NONE; - }; - value = input; + let ValueDef::Inst(id) = body.values[value.index()].def else { + return None; }; - for value in path.drain(..) { - known[value.index()] = true; - origins[value.index()] = root; + match body.instructions[id.index()].op { + Op::Project { base, projection } => { + let field = body.projections[projection.index()].field; + let member = &body.fields[field.index()]; + if types.pointee(body.value_type(base)) != Some(member.owner) + || types.pointee(body.value_type(value)) != Some(member.ty) + { + return None; + } + fields.push(field); + value = base; + } + Op::Reinterpret(source) | Op::Refine(source) + if body.value_type(source) == body.value_type(value) => + { + value = source + } + _ => return None, } } - origins + None } /// Each candidate is a fresh allocation and its typed initial contents. The @@ -60,37 +65,97 @@ pub fn promote_cells( roots[cell.index()] = u32::try_from(index).expect("too many private cells"); } let origins = origins(&body, &roots); + let locations = (0..body.values.len()) + .map(|i| location(&body, types, &origins, ValueId::new(i))) + .collect::>(); + let mut invoked = vec![false; body.instructions.len()]; + for block in &body.blocks { + if let Some(Terminator::Invoke { inst, .. }) = block.terminator { + invoked[inst.index()] = true; + } + } + // Field promotion can leave unused projections. + // They do not observe addresses and must not block SSA storage promotion. + let live = super::live(&body, types); let mut escaped = vec![false; cells.len()]; - for inst in &body.instructions { + for (id, inst) in body.instructions.iter().enumerate() { + if !live.instructions[id] { + continue; + } inst.op.visit_uses(&body.args, |value| { - let Some(index) = cell_index(origins[value.index()]) else { + let Some(location) = &locations[value.index()] else { return; }; - let pointee = body.value_type(cells[index].1); + let index = location.cell; + let pointee = location.fields.last().map_or_else( + || body.value_type(cells[index].1), + |field| body.fields[field.index()].ty, + ); + // Keep handlers when promotion would need extra field reads. + if invoked[id] + && (!location.fields.is_empty() || matches!(inst.op, Op::LoadFieldCopy { .. })) + { + escaped[index] = true; + return; + } let allowed = match inst.op { - Op::Reinterpret(_) => true, - Op::Load(pointer) => { + Op::Reinterpret(_) | Op::Refine(_) => { + inst.result.is_some_and(|v| locations[v.index()].is_some()) + } + Op::Load(pointer) | Op::LoadCopy(pointer) => { pointer == value && inst.result.is_some_and(|v| body.value_type(v) == pointee) } Op::Store { pointer, value: stored, } => pointer == value && stored != value && body.value_type(stored) == pointee, + Op::Project { base, projection } => { + base == value + && body.fields[body.projections[projection.index()].field.index()].owner + == pointee + && inst.result.is_some_and(|v| locations[v.index()].is_some()) + } + Op::LoadField { base, projection } | Op::LoadFieldCopy { base, projection } => { + base == value + && body.fields[body.projections[projection.index()].field.index()].owner + == pointee + } + Op::StoreField { + base, + projection, + value: stored, + } => { + base == value + && stored != value + && body.fields[body.projections[projection.index()].field.index()].owner + == pointee + } _ => false, }; escaped[index] |= !allowed; }); } - // Only trivial address joins have been resolved. A nontrivial address phi - // may select another allocation; retain its storage rather than guessing. + // Equal-origin joins retain one allocation. + // Mixed joins must retain each allocation's addressable storage. let mut escape = |value: ValueId| { - if let Some(index) = cell_index(origins[value.index()]) { - escaped[index] = true; + if let Some(location) = &locations[value.index()] { + escaped[location.cell] = true; } }; for edge in &body.edges { - for &value in &edge.args { - escape(value); + for (&value, ¶m) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + if live.values[body.resolve(param).index()] + && (origins[value.index()] != origins[param.index()] + || locations[value.index()] + .as_ref() + .is_some_and(|l| !l.fields.is_empty())) + { + escape(value); + } } } for block in &body.blocks { @@ -111,12 +176,15 @@ pub fn promote_cells( for block_index in 0..builder.body.blocks.len() { let block = BlockId::new(block_index); builder.switch_to(block); - let instructions = builder.body.blocks[block_index].instructions.clone(); + let instructions = std::mem::take(&mut builder.body.blocks[block_index].instructions); let invoke = match builder.body.blocks[block_index].terminator.unwrap() { Terminator::Invoke { inst, .. } => Some(inst), _ => None, }; for id in instructions.into_iter().chain(invoke) { + if Some(id) != invoke { + builder.body.blocks[block_index].instructions.push(id); + } let inst = builder.body.instructions[id.index()]; if let Some(index) = inst .result @@ -135,21 +203,89 @@ pub fn promote_cells( continue; } let pointer = match inst.op { - Op::Load(pointer) | Op::Store { pointer, .. } => pointer, + Op::Load(pointer) | Op::LoadCopy(pointer) | Op::Store { pointer, .. } => pointer, + Op::Project { base, .. } + | Op::LoadField { base, .. } + | Op::LoadFieldCopy { base, .. } + | Op::StoreField { base, .. } => base, _ => continue, }; - let Some(variable) = cell_index(origins[pointer.index()]).and_then(|i| variables[i]) - else { + let Some(location) = &locations[pointer.index()] else { continue; }; + let Some(variable) = variables[location.cell] else { + continue; + }; + if matches!(inst.op, Op::Project { .. }) { + let constant = ConstId::new(builder.body.constants.len()); + builder.body.constants.push(Constant::Null( + builder.body.value_type(inst.result.unwrap()), + )); + builder.body.instructions[id.index()].op = Op::Constant(constant); + continue; + } + let mut object = builder.read(variable); + let mut prefix = Vec::new(); + let (stored_field, fields) = if matches!(inst.op, Op::Store { .. }) { + location + .fields + .split_last() + .map_or((None, &location.fields[..]), |(field, parents)| { + (Some(*field), parents) + }) + } else { + (None, &location.fields[..]) + }; + for &field in fields { + let ty = builder.body.fields[field.index()].ty; + let (read, value) = + super::append_value(&mut builder.body, Op::GetField { object, field }, ty); + prefix.push(read); + object = value; + } builder.body.instructions[id.index()].op = match inst.op { - Op::Load(_) => Op::Reinterpret(builder.read(variable)), + Op::Load(_) => Op::Reinterpret(object), + Op::LoadCopy(_) => Op::CopyValue(object), Op::Store { value, .. } => { - builder.define(variable, value); - Op::Nop + if let Some(field) = stored_field { + Op::SetField { + object, + field, + value, + } + } else { + builder.define(variable, value); + Op::Nop + } } + Op::LoadField { projection, .. } => Op::GetField { + object, + field: builder.body.projections[projection.index()].field, + }, + Op::LoadFieldCopy { projection, .. } => { + let field = builder.body.projections[projection.index()].field; + let ty = builder.body.fields[field.index()].ty; + let (read, value) = + super::append_value(&mut builder.body, Op::GetField { object, field }, ty); + prefix.push(read); + Op::CopyValue(value) + } + Op::StoreField { + projection, value, .. + } => Op::SetField { + object, + field: builder.body.projections[projection.index()].field, + value, + }, _ => unreachable!(), }; + if !prefix.is_empty() { + assert_ne!(Some(id), invoke); + let block = &mut builder.body.blocks[block_index]; + block.instructions.pop(); + block.instructions.extend(prefix); + block.instructions.push(id); + } } if let Some(Terminator::Invoke { inst, normal, .. }) = builder.body.blocks[block_index].terminator diff --git a/compiler-core/src/opt/cells_tests.rs b/compiler-core/src/opt/cells_tests.rs index fe7e615f..28a2ebba 100644 --- a/compiler-core/src/opt/cells_tests.rs +++ b/compiler-core/src/opt/cells_tests.rs @@ -100,6 +100,32 @@ fn keeps_storage_when_its_address_escapes() { assert_eq!(body, before); } +#[test] +fn private_array_cells_disappear_but_owned_loads_still_copy() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let array = types.intern(Type::Array(byte)); + let pointer = types.intern(Type::Pointer(array)); + let mut b = Builder::new(&types, array); + let initial = b.parameter(b.current(), array); + let storage = cell(&mut b, array, pointer, initial); + let copied = b.emit(Op::LoadCopy(storage), Some(array)).unwrap(); + b.terminate(Terminator::Return(Some(copied))); + let body = promote_cells(b.finish().unwrap(), &types, &[(storage, initial)]).unwrap(); + verify(&body, &types).unwrap(); + assert!( + body.instructions + .iter() + .any(|i| i.op == Op::CopyValue(initial)) + ); + assert!( + !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::Call { .. } | Op::LoadCopy(_))) + ); +} + #[test] fn promoted_contents_reach_the_unwind_edge_at_the_throwing_call() { let mut types = Types::default(); @@ -214,3 +240,349 @@ fn follows_long_alias_chains_for_both_loads_and_escapes() { } } } + +#[test] +fn projected_borrow_follows_replacement_of_private_aggregate_storage() { + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let name = types.symbol("Counter"); + let object = types.intern(Type::Class(name)); + let pointer = types.intern(Type::Pointer(object)); + let field_pointer = types.intern(Type::Pointer(int)); + for escapes in [false, true] { + let mut b = Builder::new(&types, if escapes { field_pointer } else { int }); + let first = b.parameter(b.current(), object); + let second = b.parameter(b.current(), object); + let storage = cell(&mut b, object, pointer, first); + let field = b.field(FieldRef { + owner: object, + name: "count".into(), + ty: int, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 8, + codec: None, + }); + let borrow = b + .emit( + Op::Project { + base: storage, + projection, + }, + Some(field_pointer), + ) + .unwrap(); + let before = b.emit(Op::Load(borrow), Some(int)).unwrap(); + b.emit( + Op::Store { + pointer: storage, + value: second, + }, + None, + ); + b.emit( + Op::Store { + pointer: borrow, + value: before, + }, + None, + ); + let after = b.emit(Op::Load(borrow), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(if escapes { + borrow + } else { + after + }))); + let mut body = b.finish().unwrap(); + super::promote_fields(&mut body, &types); + let original = body.clone(); + let body = promote_cells(body, &types, &[(storage, first)]).unwrap(); + verify(&body, &types).unwrap(); + if escapes { + assert_eq!(body, original); + continue; + } + let read = |value: ValueId| { + let ValueDef::Inst(inst) = body.values[value.index()].def else { + panic!() + }; + body.instructions[inst.index()].op + }; + assert_eq!( + read(before), + Op::GetField { + object: first, + field + } + ); + assert_eq!( + read(after), + Op::GetField { + object: second, + field + } + ); + assert!(body.instructions.iter().any(|i| i.op + == Op::SetField { + object: second, + field, + value: before, + })); + let live = super::live(&body, &types); + assert!(!live.values[storage.index()] && !live.values[borrow.index()]); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn equal_allocation_origins_cross_control_flow_joins() { + for distinct in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(int)); + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), boolean); + let initial = b.constant(int, Scalar::integer(ScalarType::I64, 7).unwrap()); + let first = cell(&mut b, int, pointer, initial); + let second = if distinct { + cell(&mut b, int, pointer, initial) + } else { + first + }; + let left = b.create_block(); + let right = b.create_block(); + let join = b.create_block(); + let selected = b.parameter(join, pointer); + b.branch(condition, left, right); + b.switch_to(left); + let alias = b.emit(Op::Reinterpret(first), Some(pointer)).unwrap(); + b.jump(join, vec![alias]); + b.switch_to(right); + let alias = b.emit(Op::Refine(second), Some(pointer)).unwrap(); + b.jump(join, vec![alias]); + b.switch_to(join); + let next = b.constant(int, Scalar::integer(ScalarType::I64, 11).unwrap()); + b.emit( + Op::Store { + pointer: selected, + value: next, + }, + None, + ); + let result = b.emit(Op::Load(first), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut cells = vec![(first, initial)]; + if distinct { + cells.push((second, initial)); + } + let body = promote_cells(b.finish().unwrap(), &types, &cells).unwrap(); + verify(&body, &types).unwrap(); + let retains_storage = body + .instructions + .iter() + .any(|i| matches!(i.op, Op::Call { .. })); + assert_eq!(retains_storage, distinct); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} + +#[test] +fn nonentry_allocations_remain_conservative_at_joins() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(int)); + let mut b = Builder::new(&types, int); + let again = b.parameter(b.current(), boolean); + let initial = b.constant(int, Scalar::integer(ScalarType::I64, 7).unwrap()); + let allocate = b.create_block(); + let join = b.create_block(); + let repeat = b.create_block(); + let done = b.create_block(); + let selected = b.parameter(join, pointer); + b.jump(allocate, vec![]); + b.switch_to(allocate); + let storage = cell(&mut b, int, pointer, initial); + b.jump(join, vec![storage]); + b.switch_to(join); + let value = b.emit(Op::Load(selected), Some(int)).unwrap(); + b.branch(again, repeat, done); + b.switch_to(repeat); + let alias = b.emit(Op::Reinterpret(storage), Some(pointer)).unwrap(); + b.jump(join, vec![alias]); + b.switch_to(done); + b.terminate(Terminator::Return(Some(value))); + let body = promote_cells(b.finish().unwrap(), &types, &[(storage, initial)]).unwrap(); + // Only the entry region has a single-execution proof. + // This pass must retain allocations elsewhere, even if another proof could remove them. + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::Call { .. })) + ); + verify(&body, &types).unwrap(); +} + +#[test] +fn nested_fields_follow_replacement_and_keep_owned_reads() { + for mode in ["plain", "escape", "retype", "invoke"] { + let escapes = mode == "escape"; + let mut types = Types::default(); + types.intern(Type::Unit); + let int = types.scalar(ScalarType::I64); + let name = types.symbol("Inner"); + let inner = types.intern(Type::Class(name)); + let name = types.symbol("Outer"); + let outer = types.intern(Type::Class(name)); + let outer_ptr = types.intern(Type::Pointer(outer)); + let name = types.symbol("Other"); + let other = types.intern(Type::Class(name)); + let other_ptr = types.intern(Type::Pointer(other)); + let inner_ptr = types.intern(Type::Pointer(inner)); + let int_ptr = types.intern(Type::Pointer(int)); + let mut b = Builder::new(&types, if escapes { int_ptr } else { int }); + let first = b.parameter(b.current(), outer); + let second = b.parameter(b.current(), outer); + let storage = cell(&mut b, outer, outer_ptr, first); + let base = if mode == "retype" { + b.emit(Op::Reinterpret(storage), Some(other_ptr)).unwrap() + } else { + storage + }; + let inner_field = b.field(FieldRef { + owner: if mode == "retype" { other } else { outer }, + name: "inner".into(), + ty: inner, + is_static: false, + }); + let scalar_field = b.field(FieldRef { + owner: inner, + name: "scalar".into(), + ty: int, + is_static: false, + }); + let inner_projection = b.projection(PointerProjection { + parent: None, + field: inner_field, + offset: 0, + size: 8, + codec: Some("InnerCodec".into()), + }); + let scalar_projection = b.projection(PointerProjection { + parent: None, + field: scalar_field, + offset: 0, + size: 8, + codec: None, + }); + let nested = b + .emit( + Op::Project { + base, + projection: inner_projection, + }, + Some(inner_ptr), + ) + .unwrap(); + let borrow = b + .emit( + Op::Project { + base: nested, + projection: scalar_projection, + }, + Some(int_ptr), + ) + .unwrap(); + let snapshot = b.emit(Op::LoadCopy(nested), Some(inner)).unwrap(); + let snapshot_value = b + .emit( + Op::GetField { + object: snapshot, + field: scalar_field, + }, + Some(int), + ) + .unwrap(); + b.emit( + Op::Store { + pointer: storage, + value: second, + }, + None, + ); + b.emit( + Op::Store { + pointer: borrow, + value: snapshot_value, + }, + None, + ); + let handler = b.create_block(); + let after = if mode == "invoke" { + b.invoke(Op::Load(borrow), Some(int), handler).unwrap() + } else { + b.emit(Op::Load(borrow), Some(int)).unwrap() + }; + b.terminate(Terminator::Return(Some(if escapes { + borrow + } else { + after + }))); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + let mut body = b.finish().unwrap(); + super::promote_fields(&mut body, &types); + let original = body.clone(); + let body = promote_cells(body, &types, &[(storage, first)]).unwrap(); + verify(&body, &types).unwrap(); + if mode != "plain" { + assert_eq!(body, original, "{mode}"); + continue; + } + let operation = |value: ValueId| { + let ValueDef::Inst(id) = body.values[value.index()].def else { + panic!() + }; + body.instructions[id.index()].op + }; + let Op::CopyValue(copied) = operation(snapshot) else { + panic!() + }; + assert_eq!( + operation(copied), + Op::GetField { + object: first, + field: inner_field + } + ); + let Op::GetField { object, field } = operation(after) else { + panic!() + }; + assert_eq!(field, scalar_field); + assert_eq!( + operation(object), + Op::GetField { + object: second, + field: inner_field + } + ); + assert!(body.instructions.iter().any( + |i| matches!(i.op, Op::SetField { object, field, value } + if field == scalar_field && value == snapshot_value + && operation(object) == Op::GetField { object: second, field: inner_field }) + )); + let live = super::live(&body, &types); + assert!( + !live.values[storage.index()] + && !live.values[nested.index()] + && !live.values[borrow.index()] + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} diff --git a/compiler-core/src/opt/field_abi.rs b/compiler-core/src/opt/field_abi.rs new file mode 100644 index 00000000..7b3981ab --- /dev/null +++ b/compiler-core/src/opt/field_abi.rs @@ -0,0 +1,213 @@ +//! Split stored borrowed values using the same shapes as parameters and SSA joins. +use crate::ir::*; +use crate::jvm::abi::{address_displacement_name, view_field_names}; +use rustc_hash::FxHashMap; + +fn append( + body: &mut Body, + op: Op, + ty: Option, + prefix: &mut Vec, +) -> Option { + let id = InstId::new(body.instructions.len()); + let result = ty.map(|ty| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Inst(id), + }); + value + }); + body.instructions.push(Inst { op, result }); + prefix.push(id); + result +} + +pub fn lower_borrowed_fields( + body: &mut Body, + types: &mut Types, + split: impl Fn(&str) -> bool, + debug: Option<&mut DebugInfo>, +) { + let constructors = body + .methods + .iter() + .map(|method| method.name == "" && split(&method.owner)) + .collect::>(); + let fields = (0..body.fields.len()) + .map(|index| { + let field = body.fields[index].clone(); + let Type::Class(owner) = types.get(field.owner)? else { + return None; + }; + if field.is_static || !split(types.symbol_name(owner)?) { + return None; + } + let shape = ComponentShape::of(types, field.ty)?; + let names = match types.get(field.ty)? { + Type::TaggedI64 => crate::jvm::abi::tagged_field_names(&field.name).to_vec(), + Type::Pointer(_) => vec![ + field.name.clone(), + address_displacement_name(types, field.ty, &field.name), + ], + ty => view_field_names(&field.name, matches!(ty, Type::Str)).to_vec(), + }; + let members = names + .into_iter() + .zip(shape.parts(types)) + .map(|(name, ty)| { + let id = MemberId::new(body.fields.len()); + body.fields.push(FieldRef { + name, + ty, + ..field.clone() + }); + id + }) + .collect::>(); + Some((shape, members)) + }) + .collect::>(); + if fields.iter().all(Option::is_none) && !constructors.iter().any(|&v| v) { + return; + } + let mut prefixes = FxHashMap::default(); + for index in 0..body.instructions.len() { + let mut prefix = Vec::new(); + let original = body.instructions[index].op; + let replacement = match original { + Op::GetField { .. } + | Op::SetField { .. } + | Op::LoadField { .. } + | Op::StoreField { .. } => { + // Projected fields carry a projection ID instead of a member ID. + let member = match original { + Op::LoadField { projection, .. } | Op::StoreField { projection, .. } => { + body.projections[projection.index()].field + } + Op::GetField { field, .. } | Op::SetField { field, .. } => field, + _ => unreachable!(), + }; + let Some((shape, physical)) = fields.get(member.index()).and_then(Option::as_ref) + else { + continue; + }; + let mut values = Vec::new(); + for (index, &field) in physical.iter().enumerate() { + let ty = body.fields[field.index()].ty; + let op = match original { + Op::GetField { object, .. } => Op::GetField { object, field }, + Op::LoadField { base, projection } => Op::LoadFieldPart { + base, + projection, + index: index as u8, + }, + Op::SetField { value, .. } | Op::StoreField { value, .. } => { + shape.part(value, index as u8) + } + _ => unreachable!(), + }; + let value = append(body, op, Some(ty), &mut prefix).unwrap(); + values.push(value); + if let Op::SetField { object, .. } = original { + append( + body, + Op::SetField { + object, + field, + value, + }, + None, + &mut prefix, + ); + } + } + match original { + Op::SetField { .. } => Op::Nop, + Op::StoreField { + base, projection, .. + } => Op::StoreFieldParts { + base, + projection, + parts: List::append(&mut body.args, values), + }, + _ => shape.pack(List::append(&mut body.args, values)), + } + } + Op::Call { + method, + kind: CallKind::Constructor, + args, + } if constructors[method.index()] => { + let original = body.args[args.range()].to_vec(); + let params = body.methods[method.index()].params.clone(); + let mut values = Vec::new(); + for (value, ty) in original.into_iter().zip(params) { + if let Some(shape) = ComponentShape::of(types, ty) { + for (index, ty) in shape.parts(types).enumerate() { + values.push( + append(body, shape.part(value, index as u8), Some(ty), &mut prefix) + .unwrap(), + ); + } + } else { + values.push(value); + } + } + Op::Call { + method, + kind: CallKind::Constructor, + args: List::append(&mut body.args, values), + } + } + _ => continue, + }; + body.instructions[index].op = replacement; + prefixes.insert(InstId::new(index), prefix); + } + for (method, split) in body.methods.iter_mut().zip(constructors) { + if split { + let previous = std::mem::take(&mut method.params); + for ty in previous { + if let Some(shape) = ComponentShape::of(types, ty) { + method.params.extend(shape.parts(types)); + } else { + method.params.push(ty); + } + } + } + } + let mut positions = debug.as_ref().map(|_| Vec::new()); + for block in &mut body.blocks { + let previous = std::mem::take(&mut block.instructions); + let mut mapping = Vec::new(); + for id in previous { + mapping.push(block.instructions.len() as u32); + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + mapping.push(block.instructions.len() as u32); + if let Some(positions) = &mut positions { + positions.push(mapping); + } + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + if matches!( + body.instructions[inst.index()].op, + Op::Nop | Op::AddressPack(_) | Op::ViewPack(_) | Op::TaggedPack(_) + ) { + block.instructions.push(inst); + block.terminator = Some(Terminator::Jump(normal)); + } + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/field_abi_tests.rs b/compiler-core/src/opt/field_abi_tests.rs new file mode 100644 index 00000000..72802520 --- /dev/null +++ b/compiler-core/src/opt/field_abi_tests.rs @@ -0,0 +1,306 @@ +use crate::ir::*; +use crate::scalar::ScalarType; + +#[test] +fn uninitialized_view_fields_use_default_components() { + for utf8 in [false, true] { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let view = types.intern(if utf8 { Type::Str } else { Type::Slice(byte) }); + let name = types.symbol("test/Partial"); + let owner = types.intern(Type::Class(name)); + let void = types.intern(Type::Unit); + let mut b = Builder::new(&types, owner); + let null = ConstId::new(b.body.constants.len()); + b.body.constants.push(Constant::Null(view)); + let initial = b.emit(Op::Constant(null), Some(view)).unwrap(); + let constructor = b.method(MethodRef { + owner: "test/Partial".into(), + name: "".into(), + params: vec![view], + returns: void, + interface: false, + }); + let args = b.args([initial]); + let result = b + .emit( + Op::Call { + method: constructor, + kind: CallKind::Constructor, + args, + }, + Some(owner), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + super::lower_borrowed_fields(&mut body, &mut types, |_| true, None); + super::decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, crate::classfile::attributes::Instruction::Getfield(_))) + ); + } +} + +#[test] +fn stored_scalar_borrows_load_and_store_without_pointer_carriers() { + check_stored_borrow(false); + check_stored_borrow(true); +} + +#[test] +fn stored_views_have_no_live_view_carriers() { + for utf8 in [false, true] { + for indirect in [false, true] { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let len = types.scalar(ScalarType::U64); + let view = types.intern(if utf8 { Type::Str } else { Type::Slice(byte) }); + let name = types.symbol("test/Views"); + let owner = types.intern(Type::Class(name)); + let borrowed = types.intern(Type::Pointer(owner)); + let mut b = Builder::new(&types, len); + let object = b.parameter(b.current(), if indirect { borrowed } else { owner }); + let value = b.parameter(b.current(), view); + let field = b.field(FieldRef { + owner, + name: "value".into(), + ty: view, + is_static: false, + }); + let loaded = if indirect { + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 16, + codec: None, + }); + b.emit( + Op::StoreField { + base: object, + projection, + value, + }, + None, + ); + b.emit( + Op::LoadField { + base: object, + projection, + }, + Some(view), + ) + .unwrap() + } else { + b.emit( + Op::SetField { + object, + field, + value, + }, + None, + ); + b.emit(Op::GetField { object, field }, Some(view)).unwrap() + }; + let result = b.emit(Op::Length(loaded), Some(len)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + super::lower_component_arguments(&mut body, &mut types, true, |_| false, None); + super::lower_borrowed_fields(&mut body, &mut types, |_| true, None); + super::decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(index, i)| live.instructions[index] + && matches!(i.op, Op::ViewPack(_) | Op::ViewPart { .. } | Op::Length(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} + +fn check_stored_borrow(indirect: bool) { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let pointer = types.intern(Type::Pointer(int)); + let name = types.symbol("test/References"); + let owner = types.intern(Type::Class(name)); + let borrowed = types.intern(Type::Pointer(owner)); + let mut b = Builder::new(&types, int); + let object = b.parameter(b.current(), if indirect { borrowed } else { owner }); + let incoming = b.parameter(b.current(), pointer); + let field = b.field(FieldRef { + owner, + name: "value".into(), + ty: pointer, + is_static: false, + }); + let loaded = if indirect { + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 8, + codec: None, + }); + b.emit( + Op::StoreField { + base: object, + projection, + value: incoming, + }, + None, + ); + b.emit( + Op::LoadField { + base: object, + projection, + }, + Some(pointer), + ) + .unwrap() + } else { + b.emit( + Op::SetField { + object, + field, + value: incoming, + }, + None, + ); + b.emit(Op::GetField { object, field }, Some(pointer)) + .unwrap() + }; + let read = b.emit(Op::Load(loaded), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(read))); + let mut body = b.finish().unwrap(); + super::lower_component_arguments(&mut body, &mut types, true, |_| false, None); + super::lower_borrowed_fields( + &mut body, + &mut types, + |owner| owner == "test/References", + None, + ); + super::decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + for (index, inst) in body.instructions.iter().enumerate() { + if live.instructions[index] { + assert!(!matches!( + inst.op, + Op::AddressPack(_) | Op::AddressPart { .. } | Op::Load(_) + )); + if let Op::GetField { field, .. } | Op::SetField { field, .. } = inst.op { + assert_ne!(body.fields[field.index()].ty, pointer); + } + } + } + let mut pool = Default::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + let owner = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let read = pool + .add_method_ref(owner, "getObject", "()Ljava/lang/Object;") + .unwrap(); + assert_eq!( + code.instructions + .iter() + .filter(|i| **i == crate::classfile::attributes::Instruction::Invokevirtual(read)) + .count(), + usize::from(indirect), + "the displacement must not repeat the pointer read" + ); +} + +#[test] +fn split_field_components_share_resolution_only_before_the_next_effect() { + use crate::classfile::{ + Constant, attributes::Instruction, constant_pool::InternedConstantPool, + }; + for intervening_call in [false, true] { + let mut types = Types::default(); + let unit = types.intern(Type::Unit); + let byte = types.scalar(ScalarType::U8); + let view = types.intern(Type::Slice(byte)); + let owner_name = types.symbol("test/Views"); + let owner = types.intern(Type::Class(owner_name)); + let pointer = types.intern(Type::Pointer(owner)); + let parts = ComponentShape::View.parts(&mut types).collect::>(); + let mut b = Builder::new(&types, unit); + let base = b.parameter(b.current(), pointer); + let field = b.field(FieldRef { + owner, + name: "view".into(), + ty: view, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 16, + codec: None, + }); + for (index, &ty) in parts.iter().enumerate() { + if intervening_call && index == 1 { + let method = b.method(MethodRef { + owner: "test/External".into(), + name: "replaceStorage".into(), + params: vec![pointer], + returns: unit, + interface: false, + }); + let args = b.args([base]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + None, + ); + } + b.emit( + Op::LoadFieldPart { + base, + projection, + index: index as u8, + }, + Some(ty), + ); + } + b.terminate(Terminator::Return(None)); + let body = b.finish().unwrap(); + let mut cp = InternedConstantPool::default(); + let code = crate::jvm::select::compile(&body, &types, &mut cp).unwrap(); + let pool = cp.into_inner(); + let resolutions = code + .instructions + .iter() + .filter(|instruction| { + let Instruction::Invokevirtual(index) = instruction else { + return false; + }; + let Some(Constant::MethodRef { + name_and_type_index, + .. + }) = pool.get(*index) + else { + return false; + }; + let (name, _) = pool.try_get_name_and_type(*name_and_type_index).unwrap(); + pool.try_get_utf8(*name).unwrap() == "directAggregate" + }) + .count(); + assert_eq!(resolutions, if intervening_call { 2 } else { 1 }); + } +} diff --git a/compiler-core/src/opt/fields.rs b/compiler-core/src/opt/fields.rs index 958e3dc5..4d016380 100644 --- a/compiler-core/src/opt/fields.rs +++ b/compiler-core/src/opt/fields.rs @@ -1,16 +1,38 @@ //! Promote exact typed field projections to direct JVM field accesses. use crate::ir::*; -fn projection(body: &Body, mut value: ValueId) -> Option<(ValueId, ProjectionId)> { - // Reinterpretations change only the annotation, never allocation identity. - // Casts, offsets and nontrivial joins must retain the general pointer path. +fn projection_of( + body: &Body, + types: &Types, + mut value: ValueId, +) -> Option<(ValueId, ProjectionId)> { + // Annotations and exact-layout retypes preserve field storage identity. + // Casts, offsets and nontrivial joins require the general pointer path. + let mut layout: Option<(u32, Option)> = None; for _ in 0..64 { value = body.resolve(value); let ValueDef::Inst(id) = body.values[value.index()].def else { return None; }; match body.instructions[id.index()].op { - Op::Project { base, projection } => return Some((base, projection)), + Op::Project { base, projection } => { + let field = &body.projections[projection.index()]; + return layout + .is_none_or(|(size, codec)| { + field.size == u64::from(size) + && field.codec.as_deref() + == codec.and_then(|codec| types.symbol_name(codec)) + }) + .then_some((base, projection)); + } + Op::RetypeAddress { + pointer, + size: size @ 1.., + codec, + } if layout.is_none_or(|previous| previous == (size, codec)) => { + layout = Some((size, codec)); + value = pointer; + } Op::Reinterpret(source) => value = source, _ => return None, } @@ -50,15 +72,41 @@ pub fn promote_fields(body: &mut Body, types: &Types) { for index in 0..body.instructions.len() { let inst = body.instructions[index]; let (pointer, ty) = match inst.op { - Op::Load(pointer) => (pointer, body.value_type(inst.result.unwrap())), + Op::Load(pointer) | Op::LoadCopy(pointer) => { + (pointer, body.value_type(inst.result.unwrap())) + } Op::Store { pointer, value } => (pointer, body.value_type(value)), _ => continue, }; - let Some((base, projection)) = projection(body, pointer) else { + let Some((base, projection)) = projection_of(body, types, pointer) else { continue; }; let field = &body.fields[body.projections[projection.index()].field.index()]; - if field.ty != ty || !matches!(types.get(ty), Some(Type::Scalar(_))) { + if matches!(inst.op, Op::LoadCopy(_)) { + if field.ty == ty && matches!(types.get(ty), Some(Type::Class(_) | Type::Array(_))) { + body.instructions[index].op = Op::LoadFieldCopy { base, projection }; + } + continue; + } + if let Op::Store { value, .. } = inst.op + && field.ty == ty + && matches!(types.get(ty), Some(Type::Class(_) | Type::Array(_))) + { + body.instructions[index].op = Op::StoreField { + base, + projection, + value, + }; + continue; + } + // Pointer values are immutable carriers. + // Read and replace them through the same checked storage path. + if field.ty != ty + || !matches!( + types.get(ty), + Some(Type::Scalar(_) | Type::Pointer(_) | Type::Slice(_) | Type::Str) + ) + { continue; } body.instructions[index].op = match inst.op { @@ -72,3 +120,71 @@ pub fn promote_fields(body: &mut Body, types: &Types) { }; } } + +/// Follow managed field chains without materializing their intermediate addresses. +pub fn fold_field_paths(body: &mut Body, types: &Types) { + for index in 0..body.instructions.len() { + let (base, projection) = match body.instructions[index].op { + Op::LoadField { base, projection } + | Op::LoadFieldCopy { base, projection } + | Op::LoadFieldPart { + base, projection, .. + } => (base, projection), + _ => continue, + }; + let Some((base, projection, _)) = fold_projection(body, types, base, projection) else { + continue; + }; + body.instructions[index].op = match body.instructions[index].op { + Op::LoadField { .. } => Op::LoadField { base, projection }, + Op::LoadFieldCopy { .. } => Op::LoadFieldCopy { base, projection }, + Op::LoadFieldPart { index, .. } => Op::LoadFieldPart { + base, + projection, + index, + }, + _ => unreachable!(), + }; + } +} + +pub(super) fn fold_projection( + body: &mut Body, + types: &Types, + mut base: ValueId, + projection: ProjectionId, +) -> Option<(ValueId, ProjectionId, u64)> { + let mut path = vec![projection]; + let mut offset = body.projections[projection.index()].offset; + if body.projections[projection.index()].parent.is_some() { + return None; + } + for _ in 0..32 { + let Some((parent, projection)) = projection_of(body, types, base) else { + break; + }; + let layout = &body.projections[projection.index()]; + let member = &body.fields[layout.field.index()]; + let child = &body.fields[body.projections[path[path.len() - 1].index()].field.index()]; + if layout.parent.is_some() + || member.ty != child.owner + || types.pointee(body.value_type(parent)) != Some(member.owner) + { + break; + } + offset = offset.checked_add(layout.offset)?; + path.push(projection); + base = parent; + } + if path.len() < 2 { + return None; + } + let mut parent = path.pop(); + for id in path.into_iter().rev() { + let mut field = body.projections[id.index()].clone(); + field.parent = parent; + parent = Some(ProjectionId::new(body.projections.len())); + body.projections.push(field); + } + Some((base, parent.unwrap(), offset)) +} diff --git a/compiler-core/src/opt/fields_tests.rs b/compiler-core/src/opt/fields_tests.rs index f7e85ba4..3c7128b9 100644 --- a/compiler-core/src/opt/fields_tests.rs +++ b/compiler-core/src/opt/fields_tests.rs @@ -2,6 +2,93 @@ use super::fields::promote_fields; use crate::ir::*; use crate::scalar::{Scalar, ScalarType}; +#[test] +fn owned_field_copies_keep_exception_edges_and_eliminate_address_carriers() { + for components in [false, true] { + for protected in [false, true] { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let array = types.intern(Type::Array(byte)); + let array_pointer = types.intern(Type::Pointer(array)); + let owner = types.symbol("Pixel"); + let owner = types.intern(Type::Class(owner)); + let pointer = types.intern(Type::Pointer(owner)); + let mut b = Builder::new(&types, array); + let base = b.parameter(b.current(), pointer); + let field = b.field(FieldRef { + owner, + name: "rgba".into(), + ty: array, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 4, + size: 4, + codec: Some("org/rustlang/runtime/ArrayMemoryCodec#array#[B#4".into()), + }); + let address = b + .emit(Op::Project { base, projection }, Some(array_pointer)) + .unwrap(); + let value = if protected { + let handler = b.create_block(); + let value = b + .invoke(Op::LoadCopy(address), Some(array), handler) + .unwrap(); + b.terminate(Terminator::Return(Some(value))); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + value + } else { + let value = b.emit(Op::LoadCopy(address), Some(array)).unwrap(); + b.terminate(Terminator::Return(Some(value))); + value + }; + let mut body = b.finish().unwrap(); + promote_fields(&mut body, &types); + if components { + super::lower_component_arguments(&mut body, &mut types, true, |_| false, None); + super::decompose_addresses(&mut body, &mut types, None); + } + verify(&body, &types).unwrap(); + let ValueDef::Inst(copy) = body.values[value.index()].def else { + panic!() + }; + assert!(if components { + matches!( + body.instructions[copy.index()].op, + Op::LoadStorageFieldCopy { .. } + ) + } else { + matches!(body.instructions[copy.index()].op, Op::LoadFieldCopy { .. }) + }); + if protected { + assert!(body.blocks.iter().any(|b| matches!(b.terminator, + Some(Terminator::Invoke { inst, .. }) if inst == copy))); + } + let live = super::live(&body, &types); + assert!(!live.values[address.index()]); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!(inst.op, Op::Project { .. } | Op::AddressPack(_))) + ); + let mut pool = Default::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + let owner = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let load = pool.add_method_ref(owner, "loadStorageFieldCopy", + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJLjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;").unwrap(); + assert!(code.instructions.contains( + &crate::classfile::attributes::Instruction::Invokestatic(load) + )); + } + } +} + fn layout(types: &mut Types) -> (TypeId, TypeId, TypeId) { let scalar = types.scalar(ScalarType::I64); let owner = types.symbol("Counter"); @@ -23,9 +110,9 @@ fn project( name: "value".into(), ty: scalar, is_static: false, - relative_pointer: false, }); let projection = b.projection(PointerProjection { + parent: None, field, offset: 0, size: 8, @@ -141,3 +228,402 @@ fn rejects_field_loads_with_a_different_view_type() { .any(|i| matches!(i.op, Op::LoadField { .. })) ); } + +#[test] +fn retyped_borrowed_fields_promote_only_with_the_original_exact_layout() { + for mismatch in [0, 1, 2, 3] { + for protected in [false, true] { + let mut types = Types::default(); + let (_, address, _) = layout(&mut types); + let slot = types.intern(Type::Pointer(address)); + let owner = types.symbol("Iterator"); + let owner = types.intern(Type::Class(owner)); + let base_ty = types.intern(Type::Pointer(owner)); + let codec = types.symbol("@raw-pointer\n8\n\n"); + let other_codec = types.symbol("@raw-pointer\n4\n\n"); + let mut b = Builder::new(&types, address); + let base = b.parameter(b.current(), base_ty); + let field = b.field(FieldRef { + owner, + name: "end".into(), + ty: address, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 8, + size: 8, + codec: Some("@raw-pointer\n8\n\n".into()), + }); + let original = b + .emit(Op::Project { base, projection }, Some(slot)) + .unwrap(); + let retyped = b + .emit( + Op::RetypeAddress { + pointer: original, + size: if mismatch == 1 { + 4 + } else if mismatch == 3 { + 0 + } else { + 8 + }, + codec: Some(if mismatch == 2 { other_codec } else { codec }), + }, + Some(slot), + ) + .unwrap(); + let value = if protected { + let handler = b.create_block(); + let value = b.invoke(Op::Load(retyped), Some(address), handler).unwrap(); + let current = b.current(); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + b.switch_to(current); + value + } else { + b.emit(Op::Load(retyped), Some(address)).unwrap() + }; + b.emit( + Op::Store { + pointer: retyped, + value, + }, + None, + ); + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + promote_fields(&mut body, &types); + verify(&body, &types).unwrap(); + let ValueDef::Inst(load) = body.values[value.index()].def else { + panic!() + }; + assert_eq!( + matches!(body.instructions[load.index()].op, Op::LoadField { .. }), + mismatch == 0 + ); + assert_eq!( + body.instructions + .iter() + .any(|inst| matches!(inst.op, Op::StoreField { .. })), + mismatch == 0 + ); + if protected { + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst == load))); + } + if mismatch == 0 { + assert!(!super::live(&body, &types).values[retyped.index()]); + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} + +#[test] +fn field_storage_dispatch_preserves_forwarded_roots() { + for store in [false, true] { + let mut types = Types::default(); + let (base_ty, _, scalar) = layout(&mut types); + let Type::Pointer(owner) = types.get(base_ty).unwrap() else { + unreachable!() + }; + let mut b = Builder::new(&types, scalar); + let input = b.parameter(b.current(), base_ty); + let field = b.field(FieldRef { + owner, + name: "value".into(), + ty: scalar, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 8, + size: 8, + codec: None, + }); + let base = b.emit(Op::Opaque(input), Some(base_ty)).unwrap(); + let value = if store { + let value = b.constant(scalar, Scalar::integer(ScalarType::I64, 17).unwrap()); + b.emit( + Op::StoreField { + base, + projection, + value, + }, + None, + ); + value + } else { + b.emit(Op::LoadField { base, projection }, Some(scalar)) + .unwrap() + }; + b.terminate(Terminator::Return(Some(value))); + let body = b.finish().unwrap(); + let mut pool = crate::classfile::constant_pool::InternedConstantPool::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + let owner = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let direct = pool + .add_method_ref( + owner, + if store { + "storeScalarField" + } else { + "loadScalarField" + }, + if store { + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JJI)V" + } else { + "(Ljava/lang/Object;JLjava/lang/String;Ljava/lang/String;JI)J" + }, + ) + .unwrap(); + let project = pool.add_method_ref(owner, "projectStructField", + "(Ljava/lang/String;Ljava/lang/String;JJLjava/lang/String;)Lorg/rustlang/runtime/Pointer;", + ).unwrap(); + use crate::classfile::attributes::Instruction; + let access = code + .instructions + .iter() + .position(|i| *i == Instruction::Invokestatic(direct)) + .expect("byte storage must have a direct scalar access"); + assert!( + !code.instructions[..access].contains(&Instruction::Invokevirtual(project)), + "byte storage must not first construct a projected field pointer" + ); + } +} + +#[test] +fn aggregate_field_stores_reuse_storage_components_and_keep_handlers() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let array = types.intern(Type::Array(byte)); + let array_pointer = types.intern(Type::Pointer(array)); + let owner = types.symbol("Pixel"); + let owner = types.intern(Type::Class(owner)); + let pointer = types.intern(Type::Pointer(owner)); + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, unit); + let base = b.parameter(b.current(), pointer); + let value = b.parameter(b.current(), array); + let field = b.field(FieldRef { + owner, + name: "rgba".into(), + ty: array, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 4, + size: 4, + codec: Some("org/rustlang/runtime/ArrayMemoryCodec#array#[B#4".into()), + }); + let address = b + .emit(Op::Project { base, projection }, Some(array_pointer)) + .unwrap(); + let handler = b.create_block(); + b.invoke( + Op::Store { + pointer: address, + value, + }, + None, + handler, + ); + b.terminate(Terminator::Return(None)); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + let mut body = b.finish().unwrap(); + promote_fields(&mut body, &types); + super::lower_component_arguments(&mut body, &mut types, true, |_| false, None); + super::decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!(body.blocks.iter().any(|b| matches!(b.terminator, + Some(Terminator::Invoke { inst, .. }) if matches!(body.instructions[inst.index()].op, Op::StoreStorageField { .. })))); + assert!(!super::live(&body, &types).values[address.index()]); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn field_lookup_reuse_stops_at_mutations_and_calls() { + for barrier in ["none", "store", "call"] { + let mut types = Types::default(); + let (pointer, _, scalar) = layout(&mut types); + let Type::Pointer(owner) = types.get(pointer).unwrap() else { + unreachable!() + }; + let unit = types.intern(Type::Unit); + let mut b = Builder::new(&types, scalar); + let base = b.parameter(b.current(), pointer); + let field = b.field(FieldRef { + owner, + name: "value".into(), + ty: scalar, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 8, + size: 8, + codec: None, + }); + let first = b + .emit(Op::LoadField { base, projection }, Some(scalar)) + .unwrap(); + if barrier == "store" { + b.emit( + Op::StoreField { + base, + projection, + value: first, + }, + None, + ); + } else if barrier == "call" { + let method = b.method(MethodRef { + owner: "Consumer".into(), + name: "mutate".into(), + params: vec![pointer], + returns: unit, + interface: false, + }); + let args = b.args([base]); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + None, + ); + } + let second = b + .emit(Op::LoadField { base, projection }, Some(scalar)) + .unwrap(); + b.terminate(Terminator::Return(Some(second))); + let body = b.finish().unwrap(); + let mut pool = crate::classfile::constant_pool::InternedConstantPool::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + let owner = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let lookup = pool + .add_method_ref( + owner, + "directAggregate", + "(Ljava/lang/Class;)Ljava/lang/Object;", + ) + .unwrap(); + use crate::classfile::attributes::Instruction; + let lookups = code + .instructions + .iter() + .filter(|i| **i == Instruction::Invokevirtual(lookup)) + .count(); + assert_eq!( + lookups, + match barrier { + "store" => 3, + "call" => 2, + _ => 1, + }, + "{barrier}" + ); + } +} + +#[test] +fn nested_field_reads_keep_nullable_fallback_and_exception_handler() { + let mut types = Types::default(); + let scalar = types.scalar(ScalarType::I64); + let outer = types.symbol("Outer"); + let outer = types.intern(Type::Class(outer)); + let inner = types.symbol("Inner"); + let inner = types.intern(Type::Class(inner)); + let outer_ptr = types.intern(Type::Pointer(outer)); + let inner_ptr = types.intern(Type::Pointer(inner)); + let scalar_ptr = types.intern(Type::Pointer(scalar)); + let mut b = Builder::new(&types, scalar); + let root = b.parameter(b.current(), outer_ptr); + let field = b.field(FieldRef { + owner: outer, + name: "inner".into(), + ty: inner, + is_static: false, + }); + let parent = b.projection(PointerProjection { + parent: None, + field, + offset: 8, + size: 8, + codec: Some("InnerCodec".into()), + }); + let base = b + .emit( + Op::Project { + base: root, + projection: parent, + }, + Some(inner_ptr), + ) + .unwrap(); + let field = b.field(FieldRef { + owner: inner, + name: "value".into(), + ty: scalar, + is_static: false, + }); + let child = b.projection(PointerProjection { + parent: None, + field, + offset: 0, + size: 8, + codec: None, + }); + let address = b + .emit( + Op::Project { + base, + projection: child, + }, + Some(scalar_ptr), + ) + .unwrap(); + let handler = b.create_block(); + let value = b.invoke(Op::Load(address), Some(scalar), handler).unwrap(); + b.terminate(Terminator::Return(Some(value))); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + let mut body = b.finish().unwrap(); + promote_fields(&mut body, &types); + super::fold_field_paths(&mut body, &types); + verify(&body, &types).unwrap(); + assert!(!super::live(&body, &types).values[base.index()]); + let ValueDef::Inst(load) = body.values[value.index()].def else { + panic!() + }; + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst == load))); + let mut pool = Default::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + use crate::classfile::attributes::Instruction; + assert_eq!( + code.instructions + .iter() + .filter(|i| matches!(i, Instruction::Ifnull(_))) + .count(), + 2 + ); + let owner = pool.add_class("Outer").unwrap(); + let field = pool.add_field_ref(owner, "inner", "LInner;").unwrap(); + assert!(code.instructions.contains(&Instruction::Getfield(field))); + let runtime = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let load = pool + .add_method_ref(runtime, "loadLocationBits", "(Ljava/lang/Object;JI)J") + .unwrap(); + assert!(code.instructions.contains(&Instruction::Invokestatic(load))); +} diff --git a/compiler-core/src/opt/live.rs b/compiler-core/src/opt/live.rs index 7fa30186..4ca8c30b 100644 --- a/compiler-core/src/opt/live.rs +++ b/compiler-core/src/opt/live.rs @@ -85,7 +85,10 @@ pub fn live_with_roots( fn has_effects(op: Op, body: &Body, types: &Types) -> bool { // An unused fat-pointer carrier has no observable identity. - !matches!(op, Op::View { .. }) && op.may_throw(body, types) + !matches!( + op, + Op::View { .. } | Op::ViewPack(_) | Op::AddressPack(_) | Op::SlotRoot(_) + ) && op.may_throw(body, types) } #[cfg(test)] diff --git a/compiler-core/src/opt/memory_copies.rs b/compiler-core/src/opt/memory_copies.rs new file mode 100644 index 00000000..bd78ec3f --- /dev/null +++ b/compiler-core/src/opt/memory_copies.rs @@ -0,0 +1,53 @@ +//! Pass address components to byte-copy intrinsics. +//! The runtime handles overlap, alias tracking and encoded references. +use super::address_parts::address_parts; +use crate::ir::*; + +pub fn lower_memory_copies(body: &mut Body, types: &mut Types) { + for index in 0..body.instructions.len() { + let inst = body.instructions[index]; + let Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } = inst.op + else { + continue; + }; + let method = &body.methods[method.index()]; + if method.owner != "org/rustlang/runtime/Pointer" + || !matches!(method.name.as_str(), "copy" | "copyNonOverlapping") + || inst.result.is_some() + || args.len != 3 + { + continue; + } + let nonoverlapping = method.name == "copyNonOverlapping"; + let args = &body.args[args.range()]; + let count = args[2]; + let (Some(source), Some(destination)) = ( + address_parts(body, types, args[0]), + address_parts(body, types, args[1]), + ) else { + continue; + }; + let (Some((a, ac)), Some((b, bc))) = (source.layout, destination.layout) else { + continue; + }; + let layouts = [(args[0], a, ac), (args[1], b, bc)].map(|(value, size, codec)| { + types.layout(AddressLayout { + value: types.pointee(body.value_type(value)).unwrap(), + size, + codec, + }) + }); + let mut parts = body.args[source.parts.range()].to_vec(); + parts.extend_from_slice(&body.args[destination.parts.range()]); + parts.push(count); + body.instructions[index].op = Op::CopyStorage { + parts: List::append(&mut body.args, parts), + layouts, + nonoverlapping, + }; + } +} diff --git a/compiler-core/src/opt/memory_copies_tests.rs b/compiler-core/src/opt/memory_copies_tests.rs new file mode 100644 index 00000000..f8d34839 --- /dev/null +++ b/compiler-core/src/opt/memory_copies_tests.rs @@ -0,0 +1,100 @@ +use super::lower_memory_copies; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn exact_copy_components_keep_handlers_and_do_not_materialize_pointers() { + assert!( + std::mem::size_of::() <= 20, + "copy metadata must not inflate ordinary IR instructions" + ); + for typed in [false, true] { + for protected in [false, true] { + let mut types = Types::default(); + let unit = types.intern(Type::Unit); + let long = types.scalar(ScalarType::I64); + let int = types.scalar(ScalarType::I32); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let pointer = types.intern(Type::Pointer(int)); + let codec = types.symbol("test/Codec#value#[I"); + let mut b = Builder::new(&types, unit); + let root = b.parameter(b.current(), object); + let offset = b.constant(long, Scalar::integer(ScalarType::I64, 4).unwrap()); + let count = b.constant(long, Scalar::integer(ScalarType::I64, 16).unwrap()); + let parts = b.args([root, offset]); + let op = if typed { + Op::TypedAddressPack { + parts, + size: 8, + codec: Some(codec), + } + } else { + Op::AddressPack(parts) + }; + let source = b.emit(op, Some(pointer)).unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "copyNonOverlapping".into(), + params: vec![pointer, pointer, long], + returns: unit, + interface: false, + }); + let args = b.args([source, source, count]); + let call = Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }; + if protected { + let handler = b.create_block(); + b.invoke(call, None, handler); + b.terminate(Terminator::Return(None)); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + } else { + b.emit(call, None); + b.terminate(Terminator::Return(None)); + } + let mut body = b.finish().unwrap(); + lower_memory_copies(&mut body, &mut types); + verify(&body, &types).unwrap(); + let (index, op) = body + .instructions + .iter() + .enumerate() + .find(|(_, i)| matches!(i.op, Op::CopyStorage { .. })) + .unwrap(); + let Op::CopyStorage { + layouts, + nonoverlapping, + .. + } = op.op + else { + unreachable!() + }; + assert!(nonoverlapping); + let layouts = layouts.map(|ty| { + let Type::Layout(id) = types.get(ty).unwrap() else { + panic!() + }; + let layout = types.get_layout(id); + (layout.size, layout.codec) + }); + assert_eq!( + layouts, + if typed { + [(8, Some(codec)); 2] + } else { + [(4, None); 2] + } + ); + assert!(!super::live(&body, &types).values[source.index()]); + if protected { + assert!(body.blocks.iter().any(|b| matches!(b.terminator, + Some(Terminator::Invoke { inst, .. }) if inst.index() == index))); + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} diff --git a/compiler-core/src/opt/mod.rs b/compiler-core/src/opt/mod.rs index 6621eb4a..a78a8833 100644 --- a/compiler-core/src/opt/mod.rs +++ b/compiler-core/src/opt/mod.rs @@ -1,13 +1,118 @@ //! Analyses are built on demand and owned by one body compilation. +mod borrowed_memory; +mod field_abi; +pub use borrowed_memory::{borrowed_memory_method, lower_borrowed_memory}; +#[cfg(test)] +mod borrowed_memory_tests; mod fields; +pub use field_abi::lower_borrowed_fields; +#[cfg(test)] +mod field_abi_tests; mod live; -pub use fields::promote_fields; +pub use fields::{fold_field_paths, promote_fields}; #[cfg(test)] mod fields_tests; pub use live::{Live, live, live_with_roots}; mod cells; pub use cells::promote_cells; +mod storage; +pub use storage::lower_typed_storage; +#[cfg(test)] +mod storage_tests; +mod tagged; +pub use tagged::decompose_tagged; +mod array_locations; +#[cfg(test)] +mod array_locations_tests; +pub use array_locations::lower_array_locations; +mod views; +pub use views::decompose_views; +#[cfg(test)] +mod views_tests; #[cfg(test)] mod cells_tests; + +mod view_abi; +pub use view_abi::{component_argument_slots, lower_component_arguments}; + +mod addresses; +pub use addresses::decompose_addresses; + +#[cfg(test)] +mod addresses_tests; + +mod aggregate_joins; +mod aggregates; +mod owned_copies; +mod owned_reads; +mod value_copies; +pub use owned_reads::lower_owned_reads; +#[cfg(test)] +mod owned_reads_tests; +pub use aggregates::promote_aggregates; +#[cfg(test)] +mod aggregates_tests; + +mod return_abi; +pub use return_abi::lower_component_returns; + +#[cfg(test)] +mod return_abi_tests; + +mod address_parts; + +mod typed_addresses; +pub use typed_addresses::lower_typed_addresses; +mod view_roundtrips; +pub use view_roundtrips::fold_view_roundtrips; +#[cfg(test)] +mod typed_addresses_tests; + +mod typed_loads; +pub use typed_loads::lower_typed_loads; +#[cfg(test)] +mod typed_loads_tests; + +mod address_observers; +pub use address_observers::lower_address_observers; +#[cfg(test)] +mod address_observers_tests; + +mod address_intrinsics; +pub use address_intrinsics::lower_address_intrinsics; +#[cfg(test)] +mod address_intrinsics_tests; + +mod memory_copies; +pub use memory_copies::lower_memory_copies; +#[cfg(test)] +mod array_view_tests; +#[cfg(test)] +mod memory_copies_tests; + +mod rethrows; +pub use rethrows::remove_rethrows; +mod simplify; +mod unreachable; +pub use simplify::{simplify_components, simplify_control_flow}; + +fn append_value( + body: &mut crate::ir::Body, + op: crate::ir::Op, + ty: crate::ir::TypeId, +) -> (crate::ir::InstId, crate::ir::ValueId) { + use crate::ir::{Inst, InstId, Value, ValueDef, ValueId}; + let inst = InstId::new(body.instructions.len()); + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Inst(inst), + }); + body.instructions.push(Inst { + op, + result: Some(value), + }); + (inst, value) +} diff --git a/compiler-core/src/opt/owned_copies.rs b/compiler-core/src/opt/owned_copies.rs new file mode 100644 index 00000000..d0c3f2fb --- /dev/null +++ b/compiler-core/src/opt/owned_copies.rs @@ -0,0 +1,223 @@ +//! Reuse private scalar snapshots when neither identity can be changed or observed. +use crate::ir::*; +use rustc_hash::FxHashMap; + +pub(super) fn reuse_readonly_copies( + body: &mut Body, + types: &Types, + layout: &mut impl FnMut(&str) -> Option>, +) { + if !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::CopyValue(_))) + { + return; + } + let mut schemas = FxHashMap::default(); + let mut roots = vec![crate::analysis::NO_ORIGIN; body.values.len()]; + let mut values = Vec::new(); + for inst in &body.instructions { + if !matches!( + inst.op, + Op::CopyValue(_) + | Op::LoadCopy(_) + | Op::LoadFieldCopy { .. } + | Op::LoadStorageFieldCopy { .. } + | Op::LoadAddressCopy(_) + | Op::LoadTypedCopy { .. } + | Op::ArrayGetCopy { .. } + | Op::ViewGetCopy(_) + | Op::Call { + kind: CallKind::Constructor, + .. + } + ) { + continue; + } + let Some(value) = inst.result else { continue }; + let ty = body.value_type(value); + let Some(Type::Class(name)) = types.get(ty) else { + continue; + }; + let fields = schemas.entry(ty).or_insert_with(|| { + layout(types.symbol_name(name).unwrap()).filter(|fields| { + fields.iter().all(|f| { + f.owner == ty + && !f.is_static + && matches!(types.get(f.ty), Some(Type::Scalar(_))) + }) + }) + }); + if fields.is_some() { + roots[value.index()] = values.len() as u32; + values.push(value); + } + } + if values.is_empty() { + return; + } + let origins = crate::analysis::origins(body, &roots); + let mut readonly = vec![true; values.len()]; + for inst in &body.instructions { + inst.op.visit_uses(&body.args, |value| { + let origin = origins[value.index()]; + if origin == crate::analysis::NO_ORIGIN { + return; + } + let allowed = match inst.op { + Op::Reinterpret(_) | Op::Refine(_) => true, + Op::CopyValue(source) => { + body.value_type(source) == body.value_type(inst.result.unwrap()) + } + Op::GetField { object, field } => { + object == value + && schemas[&body.value_type(values[origin as usize])] + .as_ref() + .unwrap() + .contains(&body.fields[field.index()]) + } + _ => false, + }; + readonly[origin as usize] &= allowed; + }); + } + let mut escape = |value: ValueId| { + let origin = origins[value.index()]; + if origin != crate::analysis::NO_ORIGIN { + readonly[origin as usize] = false; + } + }; + for block in &body.blocks { + block.terminator.unwrap().visit_uses(&mut escape); + } + for edge in &body.edges { + for (&value, ¶m) in edge + .args + .iter() + .zip(&body.blocks[edge.target.index()].params) + { + if origins[value.index()] != origins[param.index()] { + escape(value); + } + } + } + for inst in &mut body.instructions { + let Op::CopyValue(source) = inst.op else { + continue; + }; + let a = origins[source.index()]; + let b = origins[inst.result.unwrap().index()]; + if a != crate::analysis::NO_ORIGIN + && b != crate::analysis::NO_ORIGIN + && readonly[a as usize] + && readonly[b as usize] + { + inst.op = Op::Reinterpret(source); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scalar::{Scalar, ScalarType}; + + #[test] + fn snapshots_share_across_blocks_only_while_both_remain_private_and_readonly() { + for array_read in [false, true] { + for mode in ["plain", "source_write", "copy_write", "escape", "opaque"] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I64); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let array = types.intern(Type::Array(pair)); + let index = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, int); + let input = b.parameter(b.current(), if array_read { array } else { pair }); + let zero = b.constant(int, Scalar::integer(ScalarType::I64, 0).unwrap()); + let field = FieldRef { + owner: pair, + name: "x".into(), + ty: int, + is_static: false, + }; + let member = b.field(field.clone()); + let source = if array_read { + let index = b.constant(index, Scalar::integer(ScalarType::I32, 0).unwrap()); + b.emit( + Op::ArrayGetCopy { + array: input, + index, + }, + Some(pair), + ) + .unwrap() + } else { + b.emit(Op::CopyValue(input), Some(pair)).unwrap() + }; + let copy = b.emit(Op::CopyValue(source), Some(pair)).unwrap(); + let ValueDef::Inst(copy_id) = b.body.values[copy.index()].def else { + panic!() + }; + let next = b.create_block(); + b.jump(next, vec![]); + b.switch_to(next); + if mode.ends_with("write") { + b.emit( + Op::SetField { + object: if mode == "copy_write" { copy } else { source }, + field: member, + value: zero, + }, + None, + ); + } + if mode == "escape" || mode == "opaque" { + let args = b.args(if mode == "escape" { vec![copy] } else { vec![] }); + let method = b.method(MethodRef { + owner: "Opaque".into(), + name: "call".into(), + params: if mode == "escape" { vec![pair] } else { vec![] }, + returns: int, + interface: false, + }); + b.emit( + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }, + Some(int), + ); + } + b.emit( + Op::GetField { + object: source, + field: member, + }, + Some(int), + ); + let result = b + .emit( + Op::GetField { + object: copy, + field: member, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + reuse_readonly_copies(&mut body, &types, &mut |_| Some(vec![field.clone()])); + verify(&body, &types).unwrap(); + assert_eq!( + matches!(body.instructions[copy_id.index()].op, Op::Reinterpret(_)), + mode == "plain" || mode == "opaque", + "{mode} array={array_read}" + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } + } +} diff --git a/compiler-core/src/opt/owned_reads.rs b/compiler-core/src/opt/owned_reads.rs new file mode 100644 index 00000000..4c0a9e72 --- /dev/null +++ b/compiler-core/src/opt/owned_reads.rs @@ -0,0 +1,102 @@ +//! Fuse owned reads and transfer private snapshots without crossing handlers. +use crate::ir::*; + +pub fn lower_owned_reads(body: &mut Body, types: &Types) { + if !body + .instructions + .iter() + .any(|inst| matches!(inst.op, Op::CopyValue(_))) + { + return; + } + let mut uses = vec![0_u8; body.values.len()]; + let mut count = |value| { + let count = &mut uses[body.resolve(value).index()]; + *count = count.saturating_add(1); + }; + for inst in &body.instructions { + inst.op.visit_uses(&body.args, &mut count); + } + for block in &body.blocks { + block.terminator.unwrap().visit_uses(&mut count); + } + for edge in &body.edges { + for &value in &edge.args { + count(value); + } + } + let mut positions = vec![None; body.instructions.len()]; + for (block, data) in body.blocks.iter().enumerate() { + for (position, id) in data.instructions.iter().enumerate() { + positions[id.index()] = Some((block, position, None)); + } + if let Some(Terminator::Invoke { inst, unwind, .. }) = data.terminator { + positions[inst.index()] = Some((block, data.instructions.len(), Some(unwind))); + } + } + for copy in (0..body.instructions.len()).rev() { + let Op::CopyValue(source) = body.instructions[copy].op else { + continue; + }; + let source = body.resolve(source); + if uses[source.index()] != 1 { + continue; + } + let ty = body.value_type(source); + if body.value_type(body.instructions[copy].result.unwrap()) != ty + || !matches!(types.get(ty), Some(Type::Class(_) | Type::Array(_))) + { + continue; + } + let ValueDef::Inst(read) = body.values[source.index()].def else { + continue; + }; + let op = match body.instructions[read.index()].op { + Op::ArrayGet { + array, + index, + native: false, + } => Op::ArrayGetCopy { array, index }, + Op::ViewGet(parts) => Op::ViewGetCopy(parts), + owned @ (Op::CopyValue(_) + | Op::LoadCopy(_) + | Op::LoadFieldCopy { .. } + | Op::LoadStorageFieldCopy { .. } + | Op::LoadAddressCopy(_) + | Op::LoadTypedCopy { .. } + | Op::ArrayGetCopy { .. } + | Op::ViewGetCopy(_)) => owned, + _ => continue, + }; + let (Some((from, start, read_handler)), Some((to, end, copy_handler))) = + (positions[read.index()], positions[copy]) + else { + continue; + }; + if read_handler.map(|e| &body.edges[e.index()]) + != copy_handler.map(|e| &body.edges[e.index()]) + { + continue; + } + let between = if from == to && start < end { + &body.blocks[to].instructions[start + 1..end] + } else if matches!(body.blocks[from].terminator, + Some(Terminator::Invoke { inst, normal, .. }) + if inst == read && body.edges[normal.index()].target.index() == to) + { + &body.blocks[to].instructions[..end] + } else { + continue; + }; + // A private snapshot cannot change through an intervening call. + if op != body.instructions[read.index()].op + && between + .iter() + .any(|id| body.instructions[id.index()].op.may_throw(body, types)) + { + continue; + } + body.instructions[read.index()].op = op; + body.instructions[copy].op = Op::Reinterpret(source); + } +} diff --git a/compiler-core/src/opt/owned_reads_tests.rs b/compiler-core/src/opt/owned_reads_tests.rs new file mode 100644 index 00000000..dd32557d --- /dev/null +++ b/compiler-core/src/opt/owned_reads_tests.rs @@ -0,0 +1,160 @@ +use super::lower_owned_reads; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn owned_elements_fuse_only_without_other_observers_effects_or_handlers() { + for mode in ["plain", "invoke", "effect", "observer", "handler"] { + let mut types = Types::default(); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let slice = types.intern(Type::Slice(pair)); + let int = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, pair); + let array = b.parameter(b.current(), slice); + let index = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let handler = b.create_block(); + let different = b.create_block(); + let op = Op::ArrayGet { + array, + index, + native: false, + }; + let invoked = matches!(mode, "invoke" | "handler"); + let value = if invoked { + b.invoke(op, Some(pair), handler).unwrap() + } else { + b.emit(op, Some(pair)).unwrap() + }; + if mode == "effect" { + b.emit( + Op::ArraySet { + array, + index, + value, + native: false, + }, + None, + ); + } + let copy = if invoked { + b.invoke( + Op::CopyValue(value), + Some(pair), + if mode == "handler" { + different + } else { + handler + }, + ) + .unwrap() + } else { + b.emit(Op::CopyValue(value), Some(pair)).unwrap() + }; + if mode == "observer" { + b.emit( + Op::ArraySet { + array, + index, + value, + native: false, + }, + None, + ); + } + b.terminate(Terminator::Return(Some(copy))); + for block in [handler, different] { + b.switch_to(block); + b.terminate(Terminator::Rethrow); + } + let mut body = b.finish().unwrap(); + lower_owned_reads(&mut body, &types); + verify(&body, &types).unwrap(); + assert_eq!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::ArrayGetCopy { .. })), + matches!(mode, "plain" | "invoke"), + "{mode}" + ); + } +} + +#[test] +fn transfer_a_sole_owned_snapshot_but_keep_observed_copies() { + for mode in ["plain", "effect", "observed"] { + let mut types = Types::default(); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let mut b = Builder::new(&types, pair); + let input = b.parameter(b.current(), pair); + let snapshot = b.emit(Op::CopyValue(input), Some(pair)).unwrap(); + let field = b.field(FieldRef { + owner: pair, + name: "next".into(), + ty: pair, + is_static: false, + }); + if mode == "effect" { + b.emit( + Op::SetField { + object: input, + field, + value: input, + }, + None, + ); + } + let copy = b.emit(Op::CopyValue(snapshot), Some(pair)).unwrap(); + if mode == "observed" { + b.emit( + Op::SetField { + object: input, + field, + value: snapshot, + }, + None, + ); + } + b.terminate(Terminator::Return(Some(copy))); + let mut body = b.finish().unwrap(); + lower_owned_reads(&mut body, &types); + verify(&body, &types).unwrap(); + let ValueDef::Inst(id) = body.values[copy.index()].def else { + panic!() + }; + assert_eq!( + body.instructions[id.index()].op, + if mode == "observed" { + Op::CopyValue(snapshot) + } else { + Op::Reinterpret(snapshot) + }, + "{mode}" + ); + } +} + +#[test] +fn keep_each_copy_when_the_consumer_is_in_a_loop() { + let mut types = Types::default(); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let boolean = types.scalar(ScalarType::Bool); + let mut b = Builder::new(&types, pair); + let input = b.parameter(b.current(), pair); + let again = b.parameter(b.current(), boolean); + let value = b.emit(Op::CopyValue(input), Some(pair)).unwrap(); + let repeat = b.create_block(); + b.jump(repeat, vec![]); + b.switch_to(repeat); + let copy = b.emit(Op::CopyValue(value), Some(pair)).unwrap(); + let done = b.create_block(); + b.branch(again, repeat, done); + b.switch_to(done); + b.terminate(Terminator::Return(Some(copy))); + let mut body = b.finish().unwrap(); + let original = body.clone(); + lower_owned_reads(&mut body, &types); + assert_eq!(body, original); +} diff --git a/compiler-core/src/opt/rethrows.rs b/compiler-core/src/opt/rethrows.rs new file mode 100644 index 00000000..d8047908 --- /dev/null +++ b/compiler-core/src/opt/rethrows.rs @@ -0,0 +1,195 @@ +//! Remove handlers that only pass the caught exception to the caller. +use crate::ir::*; +use rustc_hash::FxHashSet; + +pub fn remove_rethrows(body: &mut Body, types: &Types) { + let mut known = Vec::new(); + for index in 0..body.blocks.len() { + let Some(term) = body.blocks[index].terminator else { + continue; + }; + let unwind = match term { + Terminator::Invoke { unwind, .. } + | Terminator::Throw { + unwind: Some(unwind), + .. + } => unwind, + _ => continue, + }; + if known.is_empty() { + known.resize(body.blocks.len(), None); + } + let target = body.edges[unwind.index()].target; + let removable = *known[target.index()].get_or_insert_with(|| rethrows(body, types, target)); + if !removable { + continue; + } + body.blocks[index].terminator = Some(match term { + Terminator::Invoke { inst, normal, .. } => { + body.blocks[index].instructions.push(inst); + Terminator::Jump(normal) + } + Terminator::Throw { value, .. } => Terminator::Throw { + value, + unwind: None, + }, + _ => unreachable!(), + }); + } +} + +fn rethrows(body: &Body, types: &Types, mut block: BlockId) -> bool { + let mut caught = FxHashSet::default(); + // A cycle can do work or never return. Leave it unchanged. + for _ in 0..body.blocks.len().min(64) { + let data = &body.blocks[block.index()]; + for &id in &data.instructions { + let inst = body.instructions[id.index()]; + let identity = match inst.op { + Op::Nop => false, + Op::Constant(id) + if matches!( + body.constants[id.index()], + Constant::Scalar(_) | Constant::Null(_) | Constant::Uninit(_) + ) => + { + false + } + Op::Exception => true, + op @ (Op::Reinterpret(source) | Op::Refine(source) | Op::Adapt(source)) => { + let ty = body.value_type(inst.result.unwrap()); + let identity = caught.contains(&body.resolve(source)); + let safe_cast = identity + && matches!(types.get(ty), Some(Type::Class(name)) + if matches!(types.symbol_name(name), + Some("java/lang/Throwable" | "java/lang/Object"))); + if !safe_cast && (matches!(op, Op::Adapt(_)) || ty != body.value_type(source)) { + return false; + } + identity + } + _ => return false, + }; + if identity { + caught.insert(body.resolve(inst.result.unwrap())); + } + } + match data.terminator.unwrap() { + Terminator::Throw { + value, + unwind: None, + } => return caught.contains(&body.resolve(value)), + Terminator::Jump(edge) => { + let edge = &body.edges[edge.index()]; + let incoming = edge + .args + .iter() + .map(|&v| caught.contains(&body.resolve(v))) + .collect::>(); + for (¶m, identity) in + body.blocks[edge.target.index()].params.iter().zip(incoming) + { + if identity { + caught.insert(body.resolve(param)); + } else { + caught.remove(&body.resolve(param)); + } + } + block = edge.target; + } + _ => return false, + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scalar::{Scalar, ScalarType}; + + #[test] + fn remove_only_identity_unwinds_after_constant_cleanup_branches() { + for mode in [ + "rethrow", "upcast", "cleanup", "replace", "cast", "cycle", "return", + ] { + let mut types = Types::default(); + let name = types.symbol("java/lang/Throwable"); + let throwable = types.intern(Type::Class(name)); + let name = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(name)); + let name = types.symbol("java/lang/IllegalArgumentException"); + let narrow = types.intern(Type::Class(name)); + let int = types.scalar(ScalarType::I32); + let boolean = types.scalar(ScalarType::Bool); + let mut b = Builder::new(&types, int); + let other = b.parameter(b.current(), throwable); + let method = b.method(MethodRef { + owner: "Example".into(), + name: "effect".into(), + params: vec![], + returns: int, + interface: false, + }); + let args = b.args(vec![]); + let call = Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }; + let handler = b.create_block(); + let value = b.invoke(call, Some(int), handler).unwrap(); + b.terminate(Terminator::Return(Some(value))); + b.switch_to(handler); + let caught = b.emit(Op::Exception, Some(throwable)).unwrap(); + let cleanup = b.create_block(); + let exception = b.parameter(cleanup, throwable); + let dead = b.create_block(); + let condition = b.constant(boolean, Scalar::boolean(false)); + let yes = b.edge(dead, vec![]); + let no = b.edge(cleanup, vec![caught]); + b.terminate(Terminator::Branch { condition, yes, no }); + b.switch_to(dead); + b.emit(call, Some(int)); + b.terminate(Terminator::Unreachable); + b.switch_to(cleanup); + if mode == "cleanup" { + b.emit(call, Some(int)); + } + let exception = if mode == "cast" { + b.emit(Op::Refine(exception), Some(narrow)).unwrap() + } else if mode == "upcast" { + let erased = b.emit(Op::Adapt(exception), Some(object)).unwrap(); + b.emit(Op::Adapt(erased), Some(throwable)).unwrap() + } else if mode == "replace" { + other + } else { + exception + }; + if mode == "cycle" { + b.jump(cleanup, vec![exception]); + } else if mode == "return" { + let zero = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + b.terminate(Terminator::Return(Some(zero))); + } else { + b.terminate(Terminator::Throw { + value: exception, + unwind: None, + }); + } + let mut body = b.finish().unwrap(); + super::super::simplify_components(&mut body, &types); + remove_rethrows(&mut body, &types); + verify(&body, &types).unwrap(); + assert_eq!( + matches!( + body.blocks[body.entry.index()].terminator, + Some(Terminator::Jump(_)) + ), + mode == "rethrow" || mode == "upcast", + "{mode}" + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} diff --git a/compiler-core/src/opt/return_abi.rs b/compiler-core/src/opt/return_abi.rs new file mode 100644 index 00000000..3b8ff9d0 --- /dev/null +++ b/compiler-core/src/opt/return_abi.rs @@ -0,0 +1,304 @@ +//! Return borrowed roots directly. Write scalar metadata into caller-owned scratch. +//! Each frame reuses one scratch array and captures successful results into SSA. +//! Other frames and Rust code cannot access this scratch. +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +fn emit(body: &mut Body, op: Op, ty: Option, into: &mut Vec) -> Option { + let inst = InstId::new(body.instructions.len()); + let result = ty.map(|ty| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Inst(inst), + }); + value + }); + body.instructions.push(Inst { op, result }); + into.push(inst); + result +} +fn integer(body: &mut Body, ty: TypeId, bits: u128, into: &mut Vec) -> ValueId { + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, bits).unwrap(), + )); + emit(body, Op::Constant(constant), Some(ty), into).unwrap() +} + +/// Bypass scratch copies only for the last call that writes metadata. +/// Aliases do not prove this condition because a later call can overwrite scratch. +fn forwards_last_call( + body: &Body, + predecessors: &[Vec<(BlockId, EdgeId)>], + mut block: usize, + wanted: InstId, + first: &[InstId], + calls: &FxHashMap, +) -> bool { + let mut budget = 64usize; + for step in 0..16 { + let current = &body.blocks[block]; + let invoke = match current.terminator { + Some(Terminator::Invoke { inst, .. }) => Some(inst), + _ => None, + }; + let instructions = if step == 0 { + first + } else { + ¤t.instructions + }; + for inst in invoke.into_iter().chain(instructions.iter().rev().copied()) { + if inst == wanted { + return true; + } + if calls.contains_key(&inst) || budget == 0 { + return false; + } + budget -= 1; + } + let [incoming] = predecessors[block].as_slice() else { + return false; + }; + block = incoming.0.index(); + } + false +} + +pub fn lower_component_returns( + body: &mut Body, + types: &mut Types, + entry: bool, + select: impl Fn(&MethodRef) -> bool, + debug: Option<&mut DebugInfo>, +) { + let entry = entry + .then(|| ComponentShape::of(types, body.return_type)) + .flatten(); + let methods = body + .methods + .iter() + .map(|method| { + (select(method) + && super::component_argument_slots(types, method.params.iter().copied()) < 254) + .then(|| ComponentShape::of(types, method.returns)) + .flatten() + }) + .collect::>(); + let has_calls = body.instructions.iter().any(|inst| { + matches!(inst.op, + Op::Call { method, .. } if methods[method.index()].is_some()) + }); + if entry.is_none() && methods.iter().all(Option::is_none) { + return; + } + + let long = types.scalar(ScalarType::I64); + let int = types.scalar(ScalarType::I32); + let metadata = types.intern(Type::Array(long)); + if entry.is_none() && !has_calls { + // Function addresses and handles use the same physical descriptor, + // including targets that this body never calls directly. + for (method, shape) in body.methods.iter_mut().zip(methods) { + if let Some(shape) = shape { + method.params.push(metadata); + method.returns = shape.parts(types).next().unwrap(); + } + } + return; + } + + let mut prologue = Vec::new(); + let scratch = if entry.is_some() { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty: metadata, + def: ValueDef::Param(body.entry), + }); + body.blocks[body.entry.index()].params.push(value); + body.return_type = entry.unwrap().parts(types).next().unwrap(); + value + } else { + let length = integer(body, int, 2, &mut prologue); + emit(body, Op::NewArray(length), Some(metadata), &mut prologue).unwrap() + }; + let indices = [0, 1].map(|index| integer(body, int, index, &mut prologue)); + let count = body.instructions.len(); + let predecessors = body.predecessors(); + let mut call_results = FxHashMap::default(); + let mut calls = FxHashMap::default(); + let mut suffixes = FxHashMap::default(); + for index in 0..count { + let Inst { + op: Op::Call { method, kind, args }, + result, + } = body.instructions[index] + else { + continue; + }; + let Some(shape) = methods[method.index()] else { + continue; + }; + calls.insert(InstId::new(index), ()); + let mut arguments = body.args[args.range()].to_vec(); + arguments.push(scratch); + body.instructions[index].op = Op::Call { + method, + kind, + args: List::append(&mut body.args, arguments), + }; + if let Some(value) = result { + let root = ValueId::new(body.values.len()); + body.values.push(Value { + ty: shape.parts(types).next().unwrap(), + def: ValueDef::Inst(InstId::new(index)), + }); + body.instructions[index].result = Some(root); + call_results.insert(value, (InstId::new(index), root, shape)); + let mut suffix = Vec::new(); + let mut parts = vec![root]; + for (position, ty) in shape.parts(types).skip(1).enumerate() { + let value = emit( + body, + Op::ArrayGet { + native: true, + array: scratch, + index: indices[position], + }, + Some(long), + &mut suffix, + ) + .unwrap(); + let value = if ty == long { + value + } else { + emit(body, Op::Cast(value), Some(ty), &mut suffix).unwrap() + }; + parts.push(value); + } + let inst = InstId::new(body.instructions.len()); + body.values[value.index()].def = ValueDef::Inst(inst); + body.instructions.push(Inst { + op: shape.pack(List::append(&mut body.args, parts)), + result: Some(value), + }); + suffix.push(inst); + suffixes.insert(InstId::new(index), suffix); + } + } + for (method, shape) in body.methods.iter_mut().zip(methods) { + if let Some(shape) = shape { + method.params.push(metadata); + method.returns = shape.parts(types).next().unwrap(); + } + } + let original_blocks = body.blocks.len(); + let mut positions = debug.as_ref().map(|_| Vec::new()); + for index in 0..original_blocks { + let previous = std::mem::take(&mut body.blocks[index].instructions); + let mut instructions = Vec::new(); + if index == body.entry.index() { + instructions.append(&mut prologue); + } + let mut mapping = Vec::new(); + for id in previous { + mapping.push(instructions.len() as u32); + instructions.push(id); + if let Some(suffix) = suffixes.remove(&id) { + instructions.extend(suffix); + } + } + mapping.push(instructions.len() as u32); + if let Some(positions) = &mut positions { + positions.push(mapping); + } + match body.blocks[index].terminator { + Some(Terminator::Return(Some(value))) if entry.is_some() => { + let shape = entry.unwrap(); + let mut source = body.resolve(value); + for _ in 0..16 { + let ValueDef::Inst(inst) = body.values[source.index()].def else { + break; + }; + match body.instructions[inst.index()].op { + Op::Reinterpret(next) | Op::Refine(next) => source = body.resolve(next), + _ => break, + } + } + if let Some(&(call, root, returned)) = call_results.get(&source) + && shape == returned + && forwards_last_call(body, &predecessors, index, call, &instructions, &calls) + { + body.blocks[index].terminator = Some(Terminator::Return(Some(root))); + body.blocks[index].instructions = instructions; + continue; + } + let mut root = None; + for (position, ty) in shape.parts(types).enumerate() { + let part = emit( + body, + shape.part(value, position as u8), + Some(ty), + &mut instructions, + ) + .unwrap(); + if position == 0 { + root = Some(part); + continue; + } + let part = if ty == long { + part + } else { + emit(body, Op::Cast(part), Some(long), &mut instructions).unwrap() + }; + emit( + body, + Op::ArraySet { + native: true, + array: scratch, + index: indices[position - 1], + value: part, + }, + None, + &mut instructions, + ); + } + body.blocks[index].terminator = Some(Terminator::Return(root)); + } + Some(Terminator::Invoke { + inst, + normal, + unwind, + }) => { + if let Some(suffix) = suffixes.remove(&inst) { + // A throw leaves the caller's previous values intact. + // Only the successful edge can consume returned metadata. + let continuation = BlockId::new(body.blocks.len()); + body.blocks.push(Block { + params: Vec::new(), + instructions: suffix, + terminator: Some(Terminator::Jump(normal)), + }); + let edge = EdgeId::new(body.edges.len()); + body.edges.push(Edge { + target: continuation, + args: Vec::new(), + }); + body.blocks[index].terminator = Some(Terminator::Invoke { + inst, + normal: edge, + unwind, + }); + } + } + _ => {} + } + body.blocks[index].instructions = instructions; + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/return_abi_tests.rs b/compiler-core/src/opt/return_abi_tests.rs new file mode 100644 index 00000000..54723f19 --- /dev/null +++ b/compiler-core/src/opt/return_abi_tests.rs @@ -0,0 +1,154 @@ +use super::*; +use crate::ir::*; +use crate::scalar::ScalarType; + +#[test] +fn borrowed_return_components_cross_calls_and_unwind_edges() { + for unwind in [false, true] { + for returns_borrow in [false, true] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::U64); + let signed = types.scalar(ScalarType::I64); + let pointer = types.intern(Type::Pointer(int)); + let slice = types.intern(Type::Slice(int)); + let object = types.symbol("Pair"); + let object = types.intern(Type::Class(object)); + let storage = types.intern(Type::Pointer(object)); + let tagged = types.intern(Type::TaggedI64); + for borrowed in [pointer, slice, storage, tagged] { + let output = if returns_borrow { borrowed } else { long }; + let mut b = Builder::new(&types, output); + let input = b.parameter(b.current(), borrowed); + let method = b.method(MethodRef { + owner: "Leaf".into(), + name: "pick".into(), + params: vec![borrowed], + returns: borrowed, + interface: false, + }); + let args = b.args([input]); + let op = Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }; + let result = if unwind { + let handler = b.create_block(); + let value = b.invoke(op, Some(borrowed), handler).unwrap(); + let continuation = b.current(); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + b.switch_to(continuation); + value + } else { + b.emit(op, Some(borrowed)).unwrap() + }; + let result = if returns_borrow { + result + } else if borrowed == tagged { + let tag = b + .emit( + Op::TaggedPart { + value: result, + index: 1, + }, + Some(signed), + ) + .unwrap(); + b.emit(Op::Cast(tag), Some(long)).unwrap() + } else if borrowed == slice { + b.emit(Op::Length(result), Some(long)).unwrap() + } else { + let tag = b.emit(Op::AddressTag(result), Some(signed)).unwrap(); + b.emit(Op::Cast(tag), Some(long)).unwrap() + }; + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + lower_component_returns(&mut body, &mut types, true, |_| true, None); + verify(&body, &types).unwrap(); + decompose_tagged(&mut body, &types, None); + decompose_views(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!(!body.instructions.iter().enumerate().any( + |(index, inst)| live.instructions[index] + && matches!( + inst.op, + Op::AddressPack(_) | Op::ViewPack(_) | Op::TaggedPack(_) + ) + )); + assert_eq!( + body.instructions + .iter() + .filter(|i| matches!(i.op, Op::NewArray(_))) + .count(), + usize::from(!returns_borrow) + ); + assert_eq!( + body.blocks + .iter() + .filter(|b| matches!(b.terminator, Some(Terminator::Invoke { .. }))) + .count(), + usize::from(unwind) + ); + let code = + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + use crate::classfile::attributes::Instruction; + assert_eq!( + code.instructions.contains(&Instruction::Laload), + !returns_borrow + ); + assert_eq!(code.instructions.contains(&Instruction::Lastore), false); + } + } + } +} + +#[test] +fn earlier_return_metadata_survives_a_later_component_call() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let slice = types.intern(Type::Slice(byte)); + let mut b = Builder::new(&types, slice); + let input = b.parameter(b.current(), slice); + let method = b.method(MethodRef { + owner: "Leaf".into(), + name: "view".into(), + params: vec![slice], + returns: slice, + interface: false, + }); + let args = b.args([input]); + let op = Op::Call { + method, + kind: CallKind::RustStatic, + args, + }; + let first = b.emit(op, Some(slice)).unwrap(); + b.emit(op, Some(slice)); + b.terminate(Terminator::Return(Some(first))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + lower_component_returns(&mut body, &mut types, true, |_| true, None); + decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + use crate::classfile::attributes::Instruction; + assert_eq!( + code.instructions + .iter() + .filter(|i| **i == Instruction::Laload) + .count(), + 2 + ); + assert_eq!( + code.instructions + .iter() + .filter(|i| **i == Instruction::Lastore) + .count(), + 2 + ); +} diff --git a/compiler-core/src/opt/simplify.rs b/compiler-core/src/opt/simplify.rs new file mode 100644 index 00000000..ff18718c --- /dev/null +++ b/compiler-core/src/opt/simplify.rs @@ -0,0 +1,244 @@ +//! Fold arithmetic and aliases after representation lowering. +//! Each definition becomes an identity or constant at most twice. +//! Revisit only its consumers. Preserve source positions and instruction IDs. +use crate::ir::*; + +pub fn simplify_components(body: &mut Body, types: &Types) { + let mut users = crate::analysis::ValueUsers::new(body.values.len()); + let mut pending = Vec::new(); + let mut queued = vec![false; body.instructions.len()]; + for (index, inst) in body.instructions.iter().enumerate() { + if inst.result.is_some() + && matches!( + inst.op, + Op::Binary { .. } + | Op::Cast(_) + | Op::Reinterpret(_) + | Op::Neg(_) + | Op::Not(_) + | Op::Bit { .. } + | Op::Overflow { .. } + | Op::Length(_) + ) + { + inst.op + .visit_uses(&body.args, |v| users.connect(body.resolve(v), index)); + pending.push(index); + queued[index] = true; + } + } + // Visit definitions in order to avoid repeated work for straight-line code. + // The worklist also handles earlier users of appended components. + pending.reverse(); + while let Some(index) = pending.pop() { + queued[index] = false; + let inst = body.instructions[index]; + let Some(result) = inst.result else { + continue; + }; + let folded = match inst.op { + Op::Reinterpret(value) if body.value_type(value) == body.value_type(result) => Some( + body.scalar_value(value) + .map_or(Folded::Value(value), Folded::Constant), + ), + _ => body.fold(types, inst.op, body.value_type(result)), + }; + let Some(folded) = folded else { + continue; + }; + match folded { + Folded::Value(source) => { + let source = body.resolve(source); + if source == result { + continue; + } + let replacement = if let Some(value) = body.scalar_value(source) { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar(value)); + Op::Constant(id) + } else { + Op::Reinterpret(source) + }; + if inst.op == replacement { + continue; + } + // Retain the result until constant propagation ends. + // An appended source can become constant later and must reach these consumers. + body.instructions[index].op = replacement; + } + Folded::Constant(value) => { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar(value)); + body.instructions[index].op = Op::Constant(id); + } + } + for user in users.users(result.index()) { + if !std::mem::replace(&mut queued[user], true) { + pending.push(user); + } + } + } + simplify_control_flow(body); +} + +/// Builder joins can expose constants without adding representation instructions. +pub fn simplify_control_flow(body: &mut Body) { + // Remove exact identities only after the constant worklist has settled. + for index in 0..body.instructions.len() { + let inst = body.instructions[index]; + if let (Some(result), Op::Reinterpret(source)) = (inst.result, inst.op) + && body.value_type(source) == body.value_type(result) + { + let source = body.resolve_mut(source); + body.values[result.index()].def = ValueDef::Alias(source); + body.instructions[index] = Inst { + op: Op::Nop, + result: None, + }; + } + } + for index in 0..body.values.len() { + body.resolve_mut(ValueId::new(index)); + } + for index in 0..body.blocks.len() { + let terminator = body.blocks[index].terminator.unwrap(); + let replacement = match terminator { + Terminator::Invoke { inst, normal, .. } + if matches!(body.instructions[inst.index()].op, Op::Nop) + || matches!(body.instructions[inst.index()].op, Op::Constant(id) if matches!(body.constants[id.index()], Constant::Scalar(_))) => + { + body.blocks[index].instructions.push(inst); + Some(Terminator::Jump(normal)) + } + Terminator::Branch { condition, yes, no } => body + .scalar_value(condition) + .map(|value| Terminator::Jump(if value.bits() != 0 { yes } else { no })), + Terminator::Switch { + value, + cases, + otherwise, + } => body.scalar_value(value).map(|value| { + let edge = body.cases[cases.range()] + .iter() + .find_map(|&(key, edge)| (key == value).then_some(edge)) + .unwrap_or(otherwise); + Terminator::Jump(edge) + }), + _ => None, + }; + let replacement = replacement.or_else(|| super::unreachable::simplify(body, terminator)); + if let Some(replacement) = replacement { + body.blocks[index].terminator = Some(replacement); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scalar::{BinaryOp, Scalar, ScalarType}; + + #[test] + fn constants_reach_earlier_users_through_late_signedness_annotations() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let unsigned = types.scalar(ScalarType::U64); + let mut b = Builder::new(&types, long); + let x = b.parameter(b.current(), long); + let first = b.emit(Op::Reinterpret(x), Some(long)).unwrap(); + let result = b.emit(Op::Neg(first), Some(long)).unwrap(); + let late = b.emit(Op::Reinterpret(x), Some(unsigned)).unwrap(); + let a = b.constant(unsigned, Scalar::integer(ScalarType::U64, 8).unwrap()); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + let ValueDef::Inst(first_inst) = body.values[first.index()].def else { + panic!() + }; + let ValueDef::Inst(late_inst) = body.values[late.index()].def else { + panic!() + }; + body.instructions[first_inst.index()].op = Op::Reinterpret(late); + body.instructions[late_inst.index()].op = Op::Binary { + op: BinaryOp::Mul, + left: a, + right: a, + }; + // Definitions have newer IDs but occur before their users in the block. + body.blocks[body.entry.index()] + .instructions + .sort_by_key(|id| { + if *id == late_inst { + 0 + } else if *id == first_inst { + 1 + } else { + 2 + } + }); + simplify_components(&mut body, &types); + assert_eq!(body.scalar_value(result).unwrap().signed(), Some(-64)); + verify(&body, &types).unwrap(); + } + + // Representation passes append instructions after the builder's fold point. + // Propagate facts from these newer IDs to older users. + #[test] + fn folds_appended_components_without_dropping_traps_or_float_work() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let float = types.scalar(ScalarType::F64); + let mut b = Builder::new(&types, long); + let x = b.parameter(b.current(), long); + let f = b.parameter(b.current(), float); + let first = b.emit(Op::Reinterpret(x), Some(long)).unwrap(); + let second = b.emit(Op::Reinterpret(first), Some(long)).unwrap(); + let zero = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + let eight = b.constant(long, Scalar::integer(ScalarType::I64, 8).unwrap()); + let trap = b + .emit( + Op::Binary { + op: BinaryOp::Div, + left: zero, + right: x, + }, + Some(long), + ) + .unwrap(); + let float_zero = b.constant(float, Scalar::f64(0.0)); + let float_sum = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: f, + right: float_zero, + }, + Some(float), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(second))); + let mut body = b.finish().unwrap(); + let ValueDef::Inst(id) = body.values[first.index()].def else { + panic!() + }; + body.instructions[id.index()].op = Op::Binary { + op: BinaryOp::Mul, + left: eight, + right: eight, + }; + simplify_components(&mut body, &types); + assert_eq!(body.scalar_value(second).unwrap().bits(), 64); + assert!(body.scalar_value(trap).is_none()); + assert!(body.scalar_value(float_sum).is_none()); + verify(&body, &types).unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + code.instructions + .contains(&crate::classfile::attributes::Instruction::Ldiv) + ); + assert!( + !code + .instructions + .contains(&crate::classfile::attributes::Instruction::Lmul) + ); + } +} diff --git a/compiler-core/src/opt/storage.rs b/compiler-core/src/opt/storage.rs new file mode 100644 index 00000000..09a6c90e --- /dev/null +++ b/compiler-core/src/opt/storage.rs @@ -0,0 +1,203 @@ +//! Keep typed allocations as storage owners instead of eager address wrappers. +//! Private cells already use SSA. Escaping locations retain one owner, +//! including when a store replaces their contents. +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +/// The frontend checks positive size, alignment and exact initial contents. +/// It also requires a direct carrier with no DST metadata. +pub fn lower_typed_storage( + body: &mut Body, + types: &mut Types, + cells: &[(ValueId, ValueId)], + debug: Option<&mut DebugInfo>, +) { + if cells.is_empty() { + return; + } + let mut parts = ComponentShape::StorageAddress.parts(types); + let object = parts.next().unwrap(); + let long = parts.next().unwrap(); + let mut prefixes = FxHashMap::default(); + let mut suffixes = FxHashMap::default(); + for &(cell, _) in cells { + let ValueDef::Inst(id) = body.values[cell.index()].def else { + continue; + }; + let mut suffix = Vec::new(); + let (op, root_ty, initial) = match body.instructions[id.index()].op { + Op::ScalarCell(initial) => { + let int = types.scalar(ScalarType::I32); + let (one_inst, one) = integer(body, int, ScalarType::I32, 1); + prefixes.insert(id, one_inst); + ( + Op::NewArray(one), + types.intern(Type::Array(body.value_type(initial))), + Some(initial), + ) + } + Op::Call { + method, + kind: CallKind::JvmStatic, + args, + } => { + let mut method = body.methods[method.index()].clone(); + if method.owner != "org/rustlang/runtime/Pointer" { + continue; + } + let borrowed = match types.get(body.value_type(cell)) { + Some(Type::Pointer(inner)) => match types.get(inner) { + Some(Type::Pointer(_)) => Some(0), + Some(Type::Slice(_)) => Some(1), + Some(Type::Str) => Some(2), + _ => None, + }, + _ => None, + }; + method.name = match (method.name.as_str(), borrowed.is_some()) { + ("cell", false) => "storage", + ("cellAligned", false) => "storageAligned", + ("cell", true) => "borrowedStorage", + ("cellAligned", true) => "borrowedStorageAligned", + _ => continue, + } + .into(); + let args = if let Some(kind) = borrowed { + let int = types.scalar(ScalarType::I32); + let (inst, kind) = integer(body, int, ScalarType::I32, kind); + prefixes.insert(id, inst); + method.params.push(int); + let mut values = body.args[args.range()].to_vec(); + values.push(kind); + List::append(&mut body.args, values) + } else { + args + }; + method.returns = object; + let method_id = MethodId::new(body.methods.len()); + body.methods.push(method); + ( + Op::Call { + method: method_id, + kind: CallKind::JvmStatic, + args, + }, + object, + None, + ) + } + _ => continue, + }; + let root = ValueId::new(body.values.len()); + body.values.push(Value { + ty: root_ty, + def: ValueDef::Inst(id), + }); + body.instructions[id.index()] = Inst { + op, + result: Some(root), + }; + let root = if let Some(initial) = initial { + let int = types.scalar(ScalarType::I32); + let (index_inst, index) = integer(body, int, ScalarType::I32, 0); + suffix.push(index_inst); + suffix.push(InstId::new(body.instructions.len())); + body.instructions.push(Inst { + op: Op::ArraySet { + native: false, + array: root, + index, + value: initial, + }, + result: None, + }); + let cast = InstId::new(body.instructions.len()); + let erased = ValueId::new(body.values.len()); + body.values.push(Value { + ty: object, + def: ValueDef::Inst(cast), + }); + body.instructions.push(Inst { + op: Op::Reinterpret(root), + result: Some(erased), + }); + suffix.push(cast); + erased + } else { + root + }; + let (zero_inst, zero) = integer(body, long, ScalarType::I64, 0); + let pack = InstId::new(body.instructions.len()); + body.values[cell.index()].def = ValueDef::Inst(pack); + body.instructions.push(Inst { + op: Op::AddressPack(List::append(&mut body.args, [root, zero])), + result: Some(cell), + }); + suffix.extend([zero_inst, pack]); + suffixes.insert(id, suffix); + } + if suffixes.is_empty() { + return; + } + let mut positions = debug.as_ref().map(|_| Vec::new()); + for index in 0..body.blocks.len() { + let previous = std::mem::take(&mut body.blocks[index].instructions); + let mut mapping = Vec::new(); + for id in previous { + mapping.push(body.blocks[index].instructions.len() as u32); + if let Some(prefix) = prefixes.remove(&id) { + body.blocks[index].instructions.push(prefix); + } + body.blocks[index].instructions.push(id); + if let Some(suffix) = suffixes.remove(&id) { + body.blocks[index].instructions.extend(suffix); + } + } + mapping.push(body.blocks[index].instructions.len() as u32); + if let Some(positions) = &mut positions { + positions.push(mapping); + } + if let Some(Terminator::Invoke { + inst, + normal, + unwind, + }) = body.blocks[index].terminator + && let Some(suffix) = suffixes.remove(&inst) + { + if let Some(prefix) = prefixes.remove(&inst) { + body.blocks[index].instructions.push(prefix); + } + // Keep allocation in its original unwind region. + // Pack its result on the normal edge before copies consume the address. + let continuation = BlockId::new(body.blocks.len()); + body.blocks.push(Block { + params: Vec::new(), + instructions: suffix, + terminator: Some(Terminator::Jump(normal)), + }); + let edge = EdgeId::new(body.edges.len()); + body.edges.push(Edge { + target: continuation, + args: Vec::new(), + }); + body.blocks[index].terminator = Some(Terminator::Invoke { + inst, + normal: edge, + unwind, + }); + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} + +fn integer(body: &mut Body, ty: TypeId, scalar: ScalarType, value: u128) -> (InstId, ValueId) { + let constant = ConstId::new(body.constants.len()); + body.constants + .push(Constant::Scalar(Scalar::integer(scalar, value).unwrap())); + super::append_value(body, Op::Constant(constant), ty) +} diff --git a/compiler-core/src/opt/storage_tests.rs b/compiler-core/src/opt/storage_tests.rs new file mode 100644 index 00000000..84dd7330 --- /dev/null +++ b/compiler-core/src/opt/storage_tests.rs @@ -0,0 +1,153 @@ +use super::{decompose_addresses, live, lower_typed_storage}; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn scalar_storage_uses_one_primitive_array_without_boxing() { + for scalar in [ + ScalarType::Bool, + ScalarType::U8, + ScalarType::I16, + ScalarType::U16, + ScalarType::F16, + ScalarType::I32, + ScalarType::U32, + ScalarType::I64, + ScalarType::U64, + ScalarType::F32, + ScalarType::F64, + ] { + for unwind in [false, true] { + let mut types = Types::default(); + let scalar_ty = types.scalar(scalar); + let pointer = types.intern(Type::Pointer(scalar_ty)); + let mut b = Builder::new(&types, scalar_ty); + let initial = b.parameter(b.current(), scalar_ty); + let cell = if unwind { + let handler = b.create_block(); + let cell = b + .invoke(Op::ScalarCell(initial), Some(pointer), handler) + .unwrap(); + let continuation = b.current(); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + b.switch_to(continuation); + cell + } else { + b.emit(Op::ScalarCell(initial), Some(pointer)).unwrap() + }; + let result = b.emit(Op::Load(cell), Some(scalar_ty)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + let promoted = super::promote_cells(body.clone(), &types, &[(cell, initial)]).unwrap(); + let private = live(&promoted, &types); + assert!( + !promoted + .instructions + .iter() + .enumerate() + .any(|(id, i)| private.instructions[id] + && matches!(i.op, Op::ScalarCell(_) | Op::NewArray(_))) + ); + lower_typed_storage(&mut body, &mut types, &[(cell, initial)], None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let retained = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(id, i)| retained.instructions[id] + && matches!( + i.op, + Op::ScalarCell(_) | Op::AddressPack(_) | Op::Adapt(_) | Op::Call { .. } + )) + ); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert_eq!( + code.instructions + .iter() + .filter(|i| matches!(i, crate::classfile::attributes::Instruction::Newarray(_))) + .count(), + 1 + ); + } + } +} + +#[test] +fn typed_storage_keeps_its_owner_and_allocation_unwind_edge() { + for unwind in [false, true] { + let mut types = Types::default(); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let string = types.symbol("java/lang/String"); + let string = types.intern(Type::Class(string)); + let pointer = types.intern(Type::Pointer(object)); + let int = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, object); + let initial = b.parameter(b.current(), object); + let size = b.constant(int, Scalar::integer(ScalarType::I32, 8).unwrap()); + let codec = ConstId::new(b.body.constants.len()); + b.body.constants.push(Constant::Null(string)); + let codec = b.emit(Op::Constant(codec), Some(string)).unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "cellAligned".into(), + params: vec![object, int, string, int], + returns: pointer, + interface: false, + }); + let args = b.args([initial, size, codec, size]); + let op = Op::Call { + method, + kind: CallKind::JvmStatic, + args, + }; + let cell = if unwind { + let handler = b.create_block(); + let cell = b.invoke(op, Some(pointer), handler).unwrap(); + let continuation = b.current(); + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + b.switch_to(continuation); + cell + } else { + b.emit(op, Some(pointer)).unwrap() + }; + let result = b.emit(Op::Load(cell), Some(object)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + let mut debug = DebugInfo::default(); + debug.events.push(DebugEvent { + block: body.entry, + position: 2, + change: DebugChange::Scope(0), + line: Some(1), + }); + lower_typed_storage(&mut body, &mut types, &[(cell, initial)], Some(&mut debug)); + verify(&body, &types).unwrap(); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(id, inst)| live.instructions[id] + && matches!(inst.op, Op::AddressPack(_) | Op::Load(_))) + ); + assert_eq!( + body.blocks + .iter() + .filter(|b| matches!(b.terminator, Some(Terminator::Invoke { .. }))) + .count(), + usize::from(unwind) + ); + assert!(body.instructions.iter().any(|i| matches!(i.op, + Op::Call { method, .. } if body.methods[method.index()].name == "storageAligned"))); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } +} diff --git a/compiler-core/src/opt/tagged.rs b/compiler-core/src/opt/tagged.rs new file mode 100644 index 00000000..8568bc1f --- /dev/null +++ b/compiler-core/src/opt/tagged.rs @@ -0,0 +1,205 @@ +//! Keep tagged scalar products decomposed through joins and loop backedges. +use crate::ir::*; + +pub fn decompose_tagged(body: &mut Body, types: &Types, debug: Option<&mut DebugInfo>) { + if !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::TaggedPack(_) | Op::TaggedPart { .. })) + { + return; + } + let count = body.values.len(); + let mut eligible = vec![false; count]; + let mut users = crate::analysis::ValueUsers::new(count); + let predecessors = body.predecessors(); + for (index, value) in body.values.iter().enumerate() { + if types.get(value.ty) != Some(Type::TaggedI64) { + continue; + } + match value.def { + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::TaggedPack(_) => eligible[index] = true, + Op::Reinterpret(source) | Op::Adapt(source) | Op::Refine(source) => { + eligible[index] = true; + users.connect(source, index); + } + _ => {} + }, + ValueDef::Alias(source) => { + eligible[index] = true; + users.connect(source, index); + } + ValueDef::Param(block) if block != body.entry => { + eligible[index] = !predecessors[block.index()].is_empty(); + let position = body.blocks[block.index()] + .params + .iter() + .position(|v| v.index() == index) + .unwrap(); + for &(_, edge) in &predecessors[block.index()] { + users.connect(body.edges[edge.index()].args[position], index); + } + } + _ => {} + } + } + users.close(&mut eligible); + let long = types + .find(Type::Scalar(crate::scalar::ScalarType::I64)) + .unwrap(); + let mut components = vec![None::<[ValueId; 2]>; count]; + let mut joins = vec![Vec::new(); body.blocks.len()]; + for index in 0..count { + if !eligible[index] { + continue; + } + match body.values[index].def { + ValueDef::Inst(id) => { + if let Op::TaggedPack(parts) = body.instructions[id.index()].op { + components[index] = Some(body.args[parts.range()].try_into().unwrap()); + } + } + ValueDef::Param(block) => { + let parts = std::array::from_fn(|_| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty: long, + def: ValueDef::Param(block), + }); + value + }); + let position = body.blocks[block.index()] + .params + .iter() + .position(|v| v.index() == index) + .unwrap(); + joins[block.index()].push((index, position)); + components[index] = Some(parts); + } + _ => {} + } + } + users.propagate(&eligible, &mut components); + for (block, joins) in joins.iter_mut().enumerate() { + joins.sort_unstable_by_key(|&(_, position)| position); + let mut prologue = Vec::new(); + for &(index, position) in joins.iter().rev() { + let parts = components[index].unwrap(); + body.blocks[block] + .params + .splice(position..position + 1, parts); + for &(_, edge) in &predecessors[block] { + let input = body.edges[edge.index()].args[position]; + body.edges[edge.index()] + .args + .splice(position..position + 1, components[input.index()].unwrap()); + } + let inst = InstId::new(body.instructions.len()); + body.values[index].def = ValueDef::Inst(inst); + body.instructions.push(Inst { + op: Op::TaggedPack(List::append(&mut body.args, parts)), + result: Some(ValueId::new(index)), + }); + prologue.push(inst); + } + prologue.append(&mut body.blocks[block].instructions); + body.blocks[block].instructions = prologue; + } + for inst in &mut body.instructions { + if let Op::TaggedPart { value, index } = inst.op { + if let Some(Some(parts)) = components.get(value.index()) { + inst.op = Op::Reinterpret(parts[index as usize]); + } + } + } + if let Some(debug) = debug { + for event in &mut debug.events { + event.position += joins[event.block.index()].len() as u32; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scalar::{BinaryOp, Scalar, ScalarType}; + + #[test] + fn loop_joins_retain_scalars_without_a_boundary_carrier() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let tagged = types.intern(Type::TaggedI64); + let mut b = Builder::new(&types, long); + let limit = b.parameter(b.current(), long); + let zero = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let parts = b.args([zero, one]); + let first = b.emit(Op::TaggedPack(parts), Some(tagged)).unwrap(); + let value = b.variable(tagged); + b.define(value, first); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let pair = b.read(value); + let payload = b + .emit( + Op::TaggedPart { + value: pair, + index: 0, + }, + Some(long), + ) + .unwrap(); + let more = b + .emit( + Op::Binary { + op: BinaryOp::Lt, + left: payload, + right: limit, + }, + Some(boolean), + ) + .unwrap(); + b.branch(more, step, done); + b.switch_to(step); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: payload, + right: one, + }, + Some(long), + ) + .unwrap(); + let parts = b.args([next, one]); + let pair = b.emit(Op::TaggedPack(parts), Some(tagged)).unwrap(); + b.define(value, pair); + b.jump(header, vec![]); + b.switch_to(done); + b.terminate(Terminator::Return(Some(payload))); + let mut body = b.finish().unwrap(); + decompose_tagged(&mut body, &types, None); + verify(&body, &types).unwrap(); + let live = crate::opt::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(index, inst)| live.instructions[index] + && matches!(inst.op, Op::TaggedPack(_) | Op::TaggedPart { .. })) + ); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, crate::classfile::attributes::Instruction::New(_))) + ); + } +} diff --git a/compiler-core/src/opt/typed_addresses.rs b/compiler-core/src/opt/typed_addresses.rs new file mode 100644 index 00000000..d44e6c78 --- /dev/null +++ b/compiler-core/src/opt/typed_addresses.rs @@ -0,0 +1,470 @@ +//! Use an exact source-language view layout without allocating a cast address. +//! Facts are attached to operations, never inferred from shared JVM classes. +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +fn append(body: &mut Body, prefix: &mut Vec, op: Op, ty: TypeId) -> ValueId { + let (id, value) = super::append_value(body, op, ty); + prefix.push(id); + value +} +fn literal(body: &mut Body, prefix: &mut Vec, ty: TypeId, n: u32) -> ValueId { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I64, n.into()).unwrap(), + )); + append(body, prefix, Op::Constant(id), ty) +} + +pub fn lower_typed_addresses(body: &mut Body, types: &mut Types, debug: Option<&mut DebugInfo>) { + if !body.instructions.iter().any(|i| { + matches!( + i.op, + Op::RetypeAddress { size: 1.., .. } | Op::ViewAddress { size: 1.., .. } + ) + }) && !body + .values + .iter() + .any(|v| types.address_layout(v.ty).is_some()) + { + return; + } + // Join exact layouts across aliases and CFG edges. Each value starts unseen, + // gains one layout, can lose its root guarantee, then becomes unknown. + // Process backedges without whole-body scans or origin sets. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Fact { + Unseen, + // A rooted value retains the layout in its root. + // The component ABI can pass it without an exact type annotation. + Known { + size: u32, + codec: Option, + rooted: bool, + }, + Unknown, + } + let count = body.values.len(); + let predecessors = body.predecessors(); + let mut facts = vec![Fact::Unknown; count]; + let mut users = crate::analysis::ValueUsers::new(count); + for (index, value) in body.values.iter().enumerate() { + if !matches!(types.get(value.ty), Some(Type::Pointer(_))) { + continue; + } + match value.def { + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::AddressPack(_) if types.address_layout(value.ty).is_some() => { + let (size, codec) = types.address_layout(value.ty).unwrap(); + facts[index] = Fact::Known { + size, + codec, + rooted: false, + }; + } + Op::RetypeAddress { + pointer, + size: size @ 1.., + codec, + } => { + facts[index] = Fact::Known { + size, + codec, + rooted: true, + }; + users.connect(pointer, index); + } + Op::ViewAddress { + size: size @ 1.., + codec, + .. + } if codec.is_some() + || types + .pointee(value.ty) + .and_then(|ty| StorageSlot::scalar(ty, types)) + .is_some_and(|slot| slot.size == size) => + { + facts[index] = Fact::Known { + size, + codec, + rooted: true, + }; + } + Op::Offset { + pointer: source, .. + } + | Op::Refine(source) + | Op::Reinterpret(source) => { + facts[index] = Fact::Unseen; + users.connect(source, index); + } + _ => {} + }, + ValueDef::Alias(source) => { + facts[index] = Fact::Unseen; + users.connect(source, index); + } + ValueDef::Param(block) + if block != body.entry && !predecessors[block.index()].is_empty() => + { + facts[index] = Fact::Unseen; + let position = body.blocks[block.index()] + .params + .iter() + .position(|p| p.index() == index) + .unwrap(); + for &(_, edge) in &predecessors[block.index()] { + users.connect(body.edges[edge.index()].args[position], index); + } + } + _ => {} + } + } + let mut pending = (0..count) + .filter(|&i| facts[i] != Fact::Unseen) + .collect::>(); + for phase in 0..2 { + while let Some(source) = pending.pop() { + for target in users.users(source) { + let previous = facts[target]; + let incoming = facts[source]; + let retype = match body.values[target].def { + ValueDef::Inst(id) => { + matches!(body.instructions[id.index()].op, Op::RetypeAddress { .. }) + } + _ => false, + }; + let merged = match (previous, incoming) { + ( + Fact::Known { + size, + codec, + rooted, + }, + fact, + ) if retype => Fact::Known { + size, + codec, + rooted: rooted + && matches!(fact, Fact::Known { size: s, codec: c, rooted: true } if s == size && c == codec), + }, + (Fact::Unseen, fact) => fact, + ( + Fact::Known { + size: a, + codec: ac, + rooted: ar, + }, + Fact::Known { + size: b, + codec: bc, + rooted: br, + }, + ) if a == b && ac == bc => Fact::Known { + size: a, + codec: ac, + rooted: ar && br, + }, + _ => Fact::Unknown, + }; + if previous != merged { + facts[target] = merged; + pending.push(target); + } + } + } + if phase == 0 { + for (index, fact) in facts.iter_mut().enumerate() { + if *fact == Fact::Unseen { + *fact = Fact::Unknown; + pending.push(index); + } + } + } + } + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let long = types.scalar(ScalarType::I64); + let mut components = vec![None::<[ValueId; 2]>; count]; + let mut roots = Vec::new(); + let mut joins = vec![Vec::new(); body.blocks.len()]; + for index in 0..count { + if !matches!(facts[index], Fact::Known { .. }) { + continue; + } + match body.values[index].def { + ValueDef::Param(block) if block != body.entry => { + let position = body.blocks[block.index()] + .params + .iter() + .position(|p| p.index() == index) + .unwrap(); + components[index] = Some([object, long].map(|ty| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Param(block), + }); + body.blocks[block.index()].params.push(value); + value + })); + joins[block.index()].push((index, position)); + } + ValueDef::Inst(id) + if matches!(body.instructions[id.index()].op, Op::AddressPack(_)) => + { + let Op::AddressPack(parts) = body.instructions[id.index()].op else { + unreachable!() + }; + components[index] = Some(body.args[parts.range()].try_into().unwrap()); + } + ValueDef::Inst(id) + if matches!( + body.instructions[id.index()].op, + Op::RetypeAddress { .. } | Op::Offset { .. } | Op::ViewAddress { .. } + ) => + { + let mut pending = Vec::new(); + components[index] = + Some([object, long].map(|ty| append(body, &mut pending, Op::Nop, ty))); + roots.push((id, pending)); + } + _ => {} + } + } + let eligible = facts + .iter() + .map(|f| matches!(f, Fact::Known { .. })) + .collect::>(); + users.propagate(&eligible, &mut components); + let mut prefixes = FxHashMap::default(); + for (id, created) in roots { + let original = body.instructions[id.index()]; + let result = original.result.unwrap(); + let Fact::Known { + size, + codec, + rooted, + } = facts[result.index()] + else { + unreachable!(); + }; + let mut prefix = Vec::new(); + let source_parts = if let Op::ViewAddress { parts, .. } = original.op { + let values = &body.args[parts.range()]; + let (backing, start) = (values[0], values[1]); + let root = append( + body, + &mut prefix, + Op::ViewRoot { + backing, + size, + codec, + }, + object, + ); + let start = append(body, &mut prefix, Op::Cast(start), long); + let stride = literal(body, &mut prefix, long, size); + let offset = append( + body, + &mut prefix, + Op::Binary { + op: BinaryOp::Mul, + left: start, + right: stride, + }, + long, + ); + [root, offset] + } else { + let source = match original.op { + Op::RetypeAddress { pointer, .. } | Op::Offset { pointer, .. } => pointer, + Op::Refine(pointer) | Op::Reinterpret(pointer) => pointer, + _ => unreachable!(), + }; + // Repeated retypes can carry ZST or source-layout metadata. + // Keep the boundary unless the exact layouts match. + let source = body.resolve(source); + let source_parts = if matches!(facts.get(source.index()), + Some(&Fact::Known { size: s, codec: c, .. }) if s == size && c == codec) + { + components[source.index()] + } else { + None + }; + source_parts + .or_else(|| { + let parts = super::address_parts::address_parts(body, types, source)?; + let ValueDef::Inst(inst) = body.values[parts.value.index()].def else { + unreachable!() + }; + matches!(body.instructions[inst.index()].op, Op::AddressPack(_)) + .then(|| body.args[parts.parts.range()].try_into().unwrap()) + }) + .unwrap_or_else(|| { + [ + append(body, &mut prefix, Op::Reinterpret(source), object), + literal(body, &mut prefix, long, 0), + ] + }) + }; + let [root, mut offset] = source_parts; + if let Op::Offset { + offset: delta, + bytes, + .. + } = original.op + { + let mut delta = append(body, &mut prefix, Op::Cast(delta), long); + if !bytes { + let stride = literal(body, &mut prefix, long, size); + delta = append( + body, + &mut prefix, + Op::Binary { + op: BinaryOp::Mul, + left: delta, + right: stride, + }, + long, + ); + } + offset = append( + body, + &mut prefix, + Op::Binary { + op: BinaryOp::Add, + left: offset, + right: delta, + }, + long, + ); + } + body.instructions[created[0].index()].op = Op::Reinterpret(root); + body.instructions[created[1].index()].op = Op::Reinterpret(offset); + prefix.extend(created); + let parts = List::append(&mut body.args, components[result.index()].unwrap()); + body.instructions[id.index()].op = + if rooted || types.address_layout(body.value_type(result)) == Some((size, codec)) { + Op::AddressPack(parts) + } else { + Op::TypedAddressPack { parts, size, codec } + }; + prefixes.insert(id, prefix); + } + for (block, incoming) in predecessors.iter().enumerate() { + for &(_, edge) in incoming { + for &(_, position) in &joins[block] { + let input = body.edges[edge.index()].args[position]; + body.edges[edge.index()] + .args + .extend(components[input.index()].unwrap()); + } + } + } + for index in 0..body.instructions.len() { + let op = body.instructions[index].op; + if let Op::AddressViewPart { + address, + index: part, + } = op + && let Some(&Fact::Known { size, codec, .. }) = facts.get(address.index()) + && let Some(source) = components.get(address.index()).copied().flatten() + { + // Use ordinary scalar normalization for scalar components. + // It can retain a primitive backing array without a carrier. + if codec.is_none() + && ComponentShape::of(types, body.value_type(address)) + == Some(ComponentShape::Address) + { + continue; + } + body.instructions[index].op = Op::TypedAddressViewPart { + parts: List::append(&mut body.args, source), + size, + codec, + index: part, + }; + continue; + } + if let Op::AddressPart { + address, + index: part, + } = op + && (types.address_layout(body.value_type(address)).is_some() + || matches!( + facts.get(address.index()), + Some(Fact::Known { rooted: true, .. }) + )) + && let Some(parts) = components.get(address.index()).copied().flatten() + { + body.instructions[index].op = Op::Reinterpret(parts[part as usize]); + continue; + } + let pointer = match op { + Op::LoadCopy(pointer) => pointer, + Op::Store { pointer, value } + if types + .get(body.value_type(value)) + .is_some_and(|t| t.carrier() == 5) => + { + pointer + } + _ => continue, + }; + if let Some(&Fact::Known { size, codec, .. }) = facts.get(pointer.index()) + && let Some(parts) = components[pointer.index()] + { + body.instructions[index].op = match op { + Op::LoadCopy(_) => Op::LoadTypedCopy { + parts: List::append(&mut body.args, parts), + size, + codec, + }, + Op::Store { value, .. } => Op::StoreTyped { + parts: List::append(&mut body.args, [parts[0], parts[1], value]), + size, + codec, + }, + _ => unreachable!(), + }; + } + } + + let mut positions = debug.as_ref().map(|_| Vec::new()); + for block in &mut body.blocks { + let previous = std::mem::take(&mut block.instructions); + let mut mapping = Vec::new(); + for id in previous { + if positions.is_some() { + mapping.push(block.instructions.len() as u32); + } + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(positions) = &mut positions { + mapping.push(block.instructions.len() as u32); + positions.push(mapping); + } + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + if matches!( + body.instructions[inst.index()].op, + Op::TypedAddressPack { .. } | Op::AddressPack(_) + ) { + block.instructions.push(inst); + block.terminator = Some(Terminator::Jump(normal)); + } + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/typed_addresses_tests.rs b/compiler-core/src/opt/typed_addresses_tests.rs new file mode 100644 index 00000000..d80143ea --- /dev/null +++ b/compiler-core/src/opt/typed_addresses_tests.rs @@ -0,0 +1,453 @@ +use super::*; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; + +#[test] +fn aggregate_slice_addresses_cross_calls_returns_and_fields_as_components() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::I64); + let pair_name = types.symbol("Pair"); + let pair = types.intern(Type::Class(pair_name)); + let pointer = types.intern(Type::Pointer(pair)); + let slice = types.intern(Type::Slice(pair)); + let codec = types.symbol("PairCodec#pair#LPair;"); + let object_name = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object_name)); + let component_types = ComponentShape::View.parts(&mut types); + let mut b = Builder::new(&types, pointer); + let input = b.parameter(b.current(), slice); + let parts = component_types + .enumerate() + .map(|(index, ty)| { + b.emit( + Op::ViewPart { + view: input, + index: index as u8, + }, + Some(ty), + ) + .unwrap() + }) + .collect::>(); + let args = b.args(parts); + let first = b + .emit( + Op::ViewAddress { + parts: args, + size: 8, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let next = b + .emit( + Op::Offset { + pointer: first, + offset: one, + bytes: false, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + let next = b + .emit( + Op::RetypeAddress { + pointer: next, + size: 8, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let view_root = b + .emit( + Op::AddressViewPart { + address: next, + index: 0, + }, + Some(object), + ) + .unwrap(); + let view_start = b + .emit( + Op::AddressViewPart { + address: next, + index: 1, + }, + Some(int), + ) + .unwrap(); + let field = b.field(FieldRef { + owner: pair, + name: "second".into(), + ty: int, + is_static: false, + }); + let projection = b.projection(PointerProjection { + parent: None, + field, + offset: 4, + size: 4, + codec: None, + }); + let value = b + .emit( + Op::LoadField { + base: next, + projection, + }, + Some(int), + ) + .unwrap(); + let method = b.method(MethodRef { + owner: "Kernel".into(), + name: "consume".into(), + params: vec![pointer, int, object, int], + returns: pointer, + interface: false, + }); + let args = b.args([next, value, view_root, view_start]); + let result = b + .emit( + Op::Call { + method, + kind: CallKind::RustStatic, + args, + }, + Some(pointer), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + lower_component_returns(&mut body, &mut types, true, |_| true, None); + decompose_views(&mut body, &mut types, None); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + simplify_components(&mut body, &types); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::ViewAddress { .. } + | Op::Offset { .. } + | Op::AddressPack(_) + | Op::TypedAddressPack { .. } + | Op::LoadField { .. } + )) + ); + assert_eq!( + body.instructions + .iter() + .filter(|i| matches!(i.op, Op::ViewRoot { .. })) + .count(), + 1 + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LoadStorageField { .. })) + ); + assert!(!body.methods.iter().any(|m| m.name == "locationStride")); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn aggregate_copy_uses_static_layout_without_cast_or_offset_carriers() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let bytes = types.intern(Type::Pointer(byte)); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let pointer = types.intern(Type::Pointer(pair)); + let long = types.scalar(ScalarType::I64); + let codec = types.symbol("PairCodec#pair#LPair;"); + let mut b = Builder::new(&types, pair); + let source = b.parameter(b.current(), bytes); + let cast = b + .emit( + Op::RetypeAddress { + pointer: source, + size: 8, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let element = b + .emit( + Op::Offset { + pointer: cast, + offset: one, + bytes: false, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + let replacement = b.parameter(b.current(), pair); + b.emit( + Op::Store { + pointer: element, + value: replacement, + }, + None, + ); + let copy = b.emit(Op::LoadCopy(element), Some(pair)).unwrap(); + b.terminate(Terminator::Return(Some(copy))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::RetypeAddress { .. } | Op::Offset { .. } | Op::AddressPack(_) + )) + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::LoadTypedCopy { size: 8, .. })) + ); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::StoreTyped { size: 8, .. })) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn aggregate_cursor_keeps_components_through_backedges() { + use crate::scalar::BinaryOp; + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let bytes = types.intern(Type::Pointer(byte)); + let pair_name = types.symbol("Pair"); + let pair = types.intern(Type::Class(pair_name)); + let pointer = types.intern(Type::Pointer(pair)); + let long = types.scalar(ScalarType::I64); + let boolean = types.scalar(ScalarType::Bool); + let codec = types.symbol("PairCodec#pair#LPair;"); + let mut b = Builder::new(&types, pair); + let source = b.parameter(b.current(), bytes); + let end = b.parameter(b.current(), long); + let cast = b + .emit( + Op::RetypeAddress { + pointer: source, + size: 8, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let cursor = b.variable(pointer); + let position = b.variable(long); + b.define(cursor, cast); + let zero = b.constant(long, Scalar::integer(ScalarType::I64, 0).unwrap()); + b.define(position, zero); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let current = b.read(cursor); + let index = b.read(position); + let test = b + .emit( + Op::Binary { + op: BinaryOp::Lt, + left: index, + right: end, + }, + Some(boolean), + ) + .unwrap(); + b.branch(test, step, done); + b.switch_to(step); + b.emit(Op::LoadCopy(current), Some(pair)); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let next = b + .emit( + Op::Offset { + pointer: current, + offset: one, + bytes: false, + wrapping: true, + }, + Some(pointer), + ) + .unwrap(); + b.define(cursor, next); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Add, + left: index, + right: one, + }, + Some(long), + ) + .unwrap(); + b.define(position, next); + b.jump(header, vec![]); + b.switch_to(done); + let result = b.emit(Op::LoadCopy(current), Some(pair)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::RetypeAddress { .. } + | Op::Offset { .. } + | Op::AddressPack(_) + | Op::TypedAddressPack { .. } + | Op::LoadCopy(_) + )) + ); + assert_eq!( + body.instructions + .iter() + .filter(|i| matches!(i.op, Op::LoadTypedCopy { .. })) + .count(), + 2 + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn conflicting_pointee_layouts_keep_a_boundary_at_the_join() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let bytes = types.intern(Type::Pointer(byte)); + let name = types.symbol("Pair"); + let pair = types.intern(Type::Class(name)); + let pointer = types.intern(Type::Pointer(pair)); + let boolean = types.scalar(ScalarType::Bool); + let mut b = Builder::new(&types, pair); + let source = b.parameter(b.current(), bytes); + let condition = b.parameter(b.current(), boolean); + let result = b.variable(pointer); + let left = b.create_block(); + let right = b.create_block(); + let join = b.create_block(); + b.branch(condition, left, right); + for (block, size) in [(left, 8), (right, 16)] { + b.switch_to(block); + let value = b + .emit( + Op::RetypeAddress { + pointer: source, + size, + codec: None, + }, + Some(pointer), + ) + .unwrap(); + b.define(result, value); + b.jump(join, vec![]); + } + b.switch_to(join); + let pointer = b.read(result); + let value = b.emit(Op::LoadCopy(pointer), Some(pair)).unwrap(); + b.terminate(Terminator::Return(Some(value))); + let mut body = b.finish().unwrap(); + lower_typed_addresses(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!( + !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::LoadTypedCopy { .. })) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn exact_array_layout_survives_call_parameters_and_returns() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let array = types.intern(Type::Array(byte)); + let codec = types.symbol("ArrayCodec#eight#[B#8"); + let layout = types.layout(AddressLayout { + value: array, + size: 8, + codec: Some(codec), + }); + let pointer = types.intern(Type::Pointer(layout)); + let long = types.scalar(ScalarType::I64); + let mut b = Builder::new(&types, pointer); + let input = b.parameter(b.current(), pointer); + let one = b.constant(long, Scalar::integer(ScalarType::I64, 1).unwrap()); + let next = b + .emit( + Op::Offset { + pointer: input, + offset: one, + bytes: false, + wrapping: false, + }, + Some(pointer), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(next))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| true, None); + lower_component_returns(&mut body, &mut types, true, |_| true, None); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + super::simplify_components(&mut body, &types); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, instruction)| live.instructions[i] + && matches!( + instruction.op, + Op::AddressPack(_) + | Op::TypedAddressPack { .. } + | Op::RetypeAddress { .. } + | Op::Offset { .. } + )) + ); + assert!(!body.methods.iter().any(|m| m.name == "locationStride")); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .contains(&crate::classfile::attributes::Instruction::Lmul) + ); +} diff --git a/compiler-core/src/opt/typed_loads.rs b/compiler-core/src/opt/typed_loads.rs new file mode 100644 index 00000000..462bde6d --- /dev/null +++ b/compiler-core/src/opt/typed_loads.rs @@ -0,0 +1,103 @@ +//! Remove exact-layout carriers used only for one borrowed object read. +//! Shared carriers retain their binding for later commits. +use super::address_parts::address_parts; +use crate::ir::*; + +fn sole_view_consumer(body: &Body, uses: &[u8], mut value: ValueId) -> bool { + for _ in 0..32 { + value = body.resolve(value); + if uses[value.index()] != 1 { + return false; + } + let ValueDef::Inst(id) = body.values[value.index()].def else { + return false; + }; + match body.instructions[id.index()].op { + Op::AddressPack(_) | Op::TypedAddressPack { .. } => return true, + Op::Refine(source) | Op::Reinterpret(source) => value = source, + _ => return false, + } + } + false +} + +pub fn lower_typed_loads(body: &mut Body, types: &Types) { + let candidates = body + .instructions + .iter() + .enumerate() + .filter_map(|(index, inst)| { + let result = body.value_type(inst.result?); + if let Op::Load(pointer) = inst.op { + let eligible = match types.get(result) { + Some(Type::Class(name) | Type::Interface(name)) => { + types.symbol_name(name) != Some("java/lang/Object") + } + Some(Type::Array(_)) => true, + _ => false, + }; + return eligible.then_some((index, pointer, None)); + } + let Op::Call { + method, + kind: CallKind::Virtual, + args, + } = inst.op + else { + return None; + }; + let method = &body.methods[method.index()]; + let named = method.name == "getObjectAs" && args.len == 2; + // getObject's null target differs from getObjectAs(Object). + // Preserve this distinction when inferring load targets. + let untyped = method.name == "getObject" + && args.len == 1 + && matches!(types.get(result), Some(Type::Class(name)) + if types.symbol_name(name) == Some("java/lang/Object")); + (method.owner == "org/rustlang/runtime/Pointer" && (named || untyped)).then(|| { + ( + index, + body.args[args.start as usize], + named.then(|| body.args[args.start as usize + 1]), + ) + }) + }) + .collect::>(); + if candidates.is_empty() { + return; + } + let mut uses = vec![0_u8; body.values.len()]; + let mut use_value = |value: ValueId| { + let count = &mut uses[body.resolve(value).index()]; + *count = count.saturating_add(1); + }; + for inst in &body.instructions { + inst.op.visit_uses(&body.args, &mut use_value); + } + for block in &body.blocks { + block.terminator.unwrap().visit_uses(&mut use_value); + } + for edge in &body.edges { + for &value in &edge.args { + use_value(value); + } + } + for (index, receiver, target) in candidates { + if !sole_view_consumer(body, &uses, receiver) { + continue; + } + let Some(address) = address_parts(body, types, receiver) else { + continue; + }; + let Some((size, codec)) = address.layout else { + continue; + }; + let mut parts = body.args[address.parts.range()].to_vec(); + parts.extend(target); + body.instructions[index].op = Op::LoadTyped { + parts: List::append(&mut body.args, parts), + size, + codec, + }; + } +} diff --git a/compiler-core/src/opt/typed_loads_tests.rs b/compiler-core/src/opt/typed_loads_tests.rs new file mode 100644 index 00000000..550cb236 --- /dev/null +++ b/compiler-core/src/opt/typed_loads_tests.rs @@ -0,0 +1,266 @@ +use super::*; +use crate::ir::*; +use crate::scalar::ScalarType; + +#[test] +fn typed_borrow_loads_preserve_unwind_edges_and_shared_bindings() { + assert!(std::mem::size_of::() <= 20); + for (named, commit) in [(false, false), (true, false), (true, true)] { + for protected in [false, true] { + let (mut body, types, packed) = borrowed_load(named, protected, commit); + lower_typed_loads(&mut body, &types); + verify(&body, &types).unwrap(); + assert_eq!(live(&body, &types).values[packed.index()], commit); + let lowered = body + .instructions + .iter() + .enumerate() + .find(|(_, inst)| matches!(inst.op, Op::LoadTyped { .. })); + if commit { + assert!(lowered.is_none()); + } else { + let (id, inst) = lowered.unwrap(); + let Op::LoadTyped { parts, size, codec } = inst.op else { + unreachable!() + }; + assert_eq!(parts.len, if named { 3 } else { 2 }); + assert_eq!(size, 16); + assert_eq!( + types.symbol_name(codec.unwrap()), + Some("test/EnumCodec#enum#Ltest/Enum;") + ); + if protected { + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst.index() == id))); + } + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} + +#[test] +fn semantic_object_borrows_fuse_but_copies_and_shared_views_do_not() { + use crate::classfile::attributes::Instruction; + + for kind in 0..6 { + for (copied, shared) in [(false, false), (false, true), (true, false)] { + if copied && kind == 5 { + continue; + } + for protected in [false, true] { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let name = types.symbol("test/Variant"); + let payload = match kind { + 0 => types.intern(Type::Class(name)), + 1 => types.intern(Type::Interface(name)), + 2 => types.intern(Type::Array(long)), + 3 => object, + 4 => types.intern(Type::Pointer(long)), + _ => long, + }; + let pointer = types.intern(Type::Pointer(payload)); + let codec = types.symbol("test/EnumCodec#enum#Ltest/Enum;"); + let mut b = Builder::new(&types, payload); + let root = b.parameter(b.current(), object); + let offset = b.parameter(b.current(), long); + let parts = b.args([root, offset]); + let packed = b + .emit( + Op::TypedAddressPack { + parts, + size: 16, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let op = if copied { + Op::LoadCopy(packed) + } else { + Op::Load(packed) + }; + let handler = protected.then(|| b.create_block()); + let value = match handler { + Some(handler) => b.invoke(op, Some(payload), handler), + None => b.emit(op, Some(payload)), + } + .unwrap(); + if shared { + b.emit(Op::Commit(packed), None); + } + b.terminate(Terminator::Return(Some(value))); + if let Some(handler) = handler { + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + } + let mut body = b.finish().unwrap(); + lower_typed_loads(&mut body, &types); + verify(&body, &types).unwrap(); + let expected = kind < 3 && !copied && !shared; + assert_eq!(!live(&body, &types).values[packed.index()], expected); + let ValueDef::Inst(id) = body.values[value.index()].def else { + panic!() + }; + assert_eq!( + matches!(body.instructions[id.index()].op, Op::LoadTyped { .. }), + expected + ); + if protected { + assert!(body.blocks.iter().any(|block| matches!(block.terminator, + Some(Terminator::Invoke { inst, .. }) if inst == id))); + } + let mut pool = Default::default(); + let code = crate::jvm::select::compile(&body, &types, &mut pool).unwrap(); + let owner = pool.add_class("org/rustlang/runtime/Pointer").unwrap(); + let load = pool.add_method_ref(owner, "loadTypedStorage", + "(Ljava/lang/Object;JILjava/lang/String;Ljava/lang/String;)Ljava/lang/Object;").unwrap(); + assert_eq!( + code.instructions.contains(&Instruction::Invokestatic(load)), + expected + ); + if expected && kind < 2 { + let name = pool.add_name_string("test/Variant").unwrap(); + assert!(code.instructions.contains(&Instruction::Ldc_w(name))); + } + } + } + } +} + +#[test] +fn enum_downcasts_keep_the_entry_components_through_erased_annotations() { + let mut types = Types::default(); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let string = types.symbol("java/lang/String"); + let string = types.intern(Type::Class(string)); + let variant = types.symbol("test/Enum"); + let variant = types.intern(Type::Interface(variant)); + let pointer = types.intern(Type::Pointer(variant)); + let runtime = types.symbol("org/rustlang/runtime/Pointer"); + let runtime = types.intern(Type::Class(runtime)); + let codec = types.symbol("test/EnumCodec#enum#Ltest/Enum;"); + let mut b = Builder::new(&types, object); + let source = b.parameter(b.current(), pointer); + let target = b.parameter(b.current(), string); + let erased = b.emit(Op::Reinterpret(source), Some(runtime)).unwrap(); + let recovered = b.emit(Op::Reinterpret(erased), Some(pointer)).unwrap(); + let retyped = b + .emit( + Op::RetypeAddress { + pointer: recovered, + size: 16, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: "getObjectAs".into(), + params: vec![string], + returns: object, + interface: false, + }); + let args = b.args([retyped, target]); + let result = b + .emit( + Op::Call { + method, + kind: CallKind::Virtual, + args, + }, + Some(object), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + lower_component_arguments(&mut body, &mut types, true, |_| false, None); + lower_typed_addresses(&mut body, &mut types, None); + decompose_addresses(&mut body, &mut types, None); + lower_typed_loads(&mut body, &types); + verify(&body, &types).unwrap(); + let live = live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!( + inst.op, + Op::AddressPack(_) | Op::TypedAddressPack { .. } | Op::RetypeAddress { .. } + )) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +fn borrowed_load(named: bool, protected: bool, commit: bool) -> (Body, Types, ValueId) { + let mut types = Types::default(); + let long = types.scalar(ScalarType::I64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let string = types.symbol("java/lang/String"); + let string = types.intern(Type::Class(string)); + let variant = types.symbol("test/Enum"); + let variant = types.intern(Type::Interface(variant)); + let pointer = types.intern(Type::Pointer(variant)); + let runtime = types.symbol("org/rustlang/runtime/Pointer"); + let runtime = types.intern(Type::Class(runtime)); + let codec = types.symbol("test/EnumCodec#enum#Ltest/Enum;"); + let mut b = Builder::new(&types, object); + let root = b.parameter(b.current(), object); + let offset = b.parameter(b.current(), long); + let target = b.parameter(b.current(), string); + let parts = b.args([root, offset]); + let packed = b + .emit( + Op::TypedAddressPack { + parts, + size: 16, + codec: Some(codec), + }, + Some(pointer), + ) + .unwrap(); + let erased = b.emit(Op::Reinterpret(packed), Some(runtime)).unwrap(); + let method = b.method(MethodRef { + owner: "org/rustlang/runtime/Pointer".into(), + name: if named { "getObjectAs" } else { "getObject" }.into(), + params: if named { vec![string] } else { vec![] }, + returns: object, + interface: false, + }); + let args = b.args(if named { + vec![erased, target] + } else { + vec![erased] + }); + let call = Op::Call { + method, + kind: CallKind::Virtual, + args, + }; + let (result, handler) = if protected { + let handler = b.create_block(); + ( + b.invoke(call, Some(object), handler).unwrap(), + Some(handler), + ) + } else { + (b.emit(call, Some(object)).unwrap(), None) + }; + if commit { + b.emit(Op::Commit(packed), None); + } + b.terminate(Terminator::Return(Some(result))); + if let Some(handler) = handler { + b.switch_to(handler); + b.terminate(Terminator::Rethrow); + } + (b.finish().unwrap(), types, packed) +} diff --git a/compiler-core/src/opt/unreachable.rs b/compiler-core/src/opt/unreachable.rs new file mode 100644 index 00000000..af8ae65d --- /dev/null +++ b/compiler-core/src/opt/unreachable.rs @@ -0,0 +1,257 @@ +//! Bypass empty traps on undefined paths. +//! Keep preceding instructions because they can throw or unwind. +use crate::ir::*; + +fn impossible(body: &Body, edge: EdgeId) -> bool { + let block = &body.blocks[body.edges[edge.index()].target.index()]; + block.terminator == Some(Terminator::Unreachable) + && block + .instructions + .iter() + .all(|i| body.instructions[i.index()].op == Op::Nop) +} + +pub(super) fn simplify(body: &mut Body, term: Terminator) -> Option { + match term { + Terminator::Branch { yes, no, .. } => match (impossible(body, yes), impossible(body, no)) { + (true, true) => Some(Terminator::Unreachable), + (true, false) => Some(Terminator::Jump(no)), + (false, true) => Some(Terminator::Jump(yes)), + _ => None, + }, + Terminator::Switch { + value, + cases, + otherwise, + } => { + let targets = &body.cases[cases.range()]; + let default = if impossible(body, otherwise) { + let Some((_, edge)) = targets.iter().find(|(_, edge)| !impossible(body, *edge)) + else { + return Some(Terminator::Unreachable); + }; + *edge + } else { + otherwise + }; + if default == otherwise + && !targets + .iter() + .any(|&(_, edge)| edge == default || impossible(body, edge)) + { + return None; + } + let retained = targets + .iter() + .copied() + .filter(|&(_, edge)| edge != default && !impossible(body, edge)) + .collect::>(); + if retained.is_empty() { + return Some(Terminator::Jump(default)); + } + let cases = List { + start: body.cases.len().try_into().expect("switch pool capacity"), + len: retained.len().try_into().expect("switch capacity"), + }; + body.cases.extend(retained); + Some(Terminator::Switch { + value, + cases, + otherwise: default, + }) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scalar::{BinaryOp, Scalar, ScalarType}; + + #[test] + fn impossible_edges_disappear_without_losing_the_returned_value() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let bool_ = types.scalar(ScalarType::Bool); + for switch in [false, true] { + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), if switch { int } else { bool_ }); + let value = b.parameter(b.current(), int); + let valid = b.create_block(); + let result = b.parameter(valid, int); + let invalid = b.create_block(); + let yes = b.edge(valid, vec![value]); + let no = b.edge(invalid, vec![]); + let cases = List { start: 0, len: 1 }; + b.body + .cases + .push((Scalar::integer(ScalarType::I32, 17).unwrap(), yes)); + b.terminate(if switch { + Terminator::Switch { + value: condition, + cases, + otherwise: no, + } + } else { + Terminator::Branch { condition, yes, no } + }); + b.switch_to(valid); + b.terminate(Terminator::Return(Some(result))); + b.switch_to(invalid); + b.terminate(Terminator::Unreachable); + let mut body = b.finish().unwrap(); + super::super::simplify_components(&mut body, &types); + assert_eq!( + body.blocks[body.entry.index()].terminator, + Some(Terminator::Jump(yes)) + ); + assert_eq!(body.resolve(result), body.resolve(value)); + assert!(!body.reachable()[invalid.index()]); + verify(&body, &types).unwrap(); + } + } + + #[test] + fn work_before_unreachable_may_diverge_so_the_edge_must_remain() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let bool_ = types.scalar(ScalarType::Bool); + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), bool_); + let value = b.parameter(b.current(), int); + let divisor = b.parameter(b.current(), int); + let valid = b.create_block(); + let invalid = b.create_block(); + b.branch(condition, valid, invalid); + b.switch_to(valid); + b.terminate(Terminator::Return(Some(value))); + b.switch_to(invalid); + b.emit( + Op::Binary { + op: BinaryOp::Div, + left: value, + right: divisor, + }, + Some(int), + ); + b.terminate(Terminator::Unreachable); + let mut body = b.finish().unwrap(); + super::super::simplify_components(&mut body, &types); + assert!(matches!( + body.blocks[body.entry.index()].terminator, + Some(Terminator::Branch { .. }) + )); + assert!(body.reachable()[invalid.index()]); + verify(&body, &types).unwrap(); + } + #[test] + fn simplified_switch_preserves_distinct_arguments_on_the_jvm() { + use crate::classfile::{ + ClassAccessFlags, ClassFile, Method, MethodAccessFlags, Version, attributes::Attribute, + constant_pool::InternedConstantPool, + }; + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let mut b = Builder::new(&types, int); + let condition = b.parameter(b.current(), int); + let first = b.parameter(b.current(), int); + let second = b.parameter(b.current(), int); + let join = b.create_block(); + let result = b.parameter(join, int); + let invalid = b.create_block(); + let a = b.edge(join, vec![first]); + let c = b.edge(join, vec![second]); + let ub = b.edge(invalid, vec![]); + let invalid_case = b.edge(invalid, vec![]); + for (key, edge) in [(17, a), (23, c), (99, invalid_case)] { + b.body + .cases + .push((Scalar::integer(ScalarType::I32, key).unwrap(), edge)); + } + b.terminate(Terminator::Switch { + value: condition, + cases: List { start: 0, len: 3 }, + otherwise: ub, + }); + b.switch_to(join); + b.terminate(Terminator::Return(Some(result))); + b.switch_to(invalid); + b.terminate(Terminator::Unreachable); + let mut body = b.finish().unwrap(); + super::super::simplify_components(&mut body, &types); + let Some(Terminator::Switch { + cases, otherwise, .. + }) = body.blocks[body.entry.index()].terminator + else { + panic!("lost switch"); + }; + assert_eq!(cases.len, 1); + assert_eq!(otherwise, a); + assert_eq!(body.edges[a.index()].args, vec![first]); + assert_eq!(body.edges[c.index()].args, vec![second]); + assert!(!body.reachable()[invalid.index()]); + verify(&body, &types).unwrap(); + let mut cp = InternedConstantPool::default(); + let this_class = cp.add_class("SwitchUB").unwrap(); + let super_class = cp.add_class("java/lang/Object").unwrap(); + let code = crate::jvm::select::compile(&body, &types, &mut cp).unwrap(); + let method = Method { + access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, + name_index: cp.add_utf8("eval").unwrap(), + descriptor_index: cp.add_utf8("(III)I").unwrap(), + attributes: vec![Attribute::Code { + name_index: cp.add_utf8("Code").unwrap(), + max_stack: code.max_stack, + max_locals: code.max_locals, + code: code.instructions, + exception_table: code.exceptions, + attributes: code.attributes, + }], + }; + let class = ClassFile { + version: Version::Java8 { minor: 0 }, + constant_pool: cp.into_inner(), + access_flags: ClassAccessFlags::PUBLIC | ClassAccessFlags::SUPER, + this_class, + super_class, + methods: vec![method], + ..Default::default() + }; + let mut bytes = Vec::new(); + class.to_bytes(&mut bytes).unwrap(); + let dir = + std::env::temp_dir().join(format!("rcj-unreachable-switch-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("SwitchUB.class"), bytes).unwrap(); + std::fs::write( + dir.join("Run.java"), + r#" +public class Run { + public static void main(String[] args) { + for (int i = -100; i <= 100; ++i) { + if (SwitchUB.eval(17, i, i + 1) != i || SwitchUB.eval(23, i, i + 1) != i + 1) + throw new AssertionError("switch argument"); + } + } +}"#, + ) + .unwrap(); + for (program, args) in [ + ("javac", vec!["-cp", ".", "Run.java"]), + ("java", vec!["-Xverify:all", "-cp", ".", "Run"]), + ] { + let output = std::process::Command::new(program) + .args(args) + .current_dir(&dir) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/compiler-core/src/opt/value_copies.rs b/compiler-core/src/opt/value_copies.rs new file mode 100644 index 00000000..e17adf35 --- /dev/null +++ b/compiler-core/src/opt/value_copies.rs @@ -0,0 +1,357 @@ +//! Expose bounded scalar snapshots before aggregate promotion. +//! Supplied constructor schemas prove non-null values. Nullable boundaries +//! retain the original copy operation. +use crate::ir::*; +use rustc_hash::FxHashMap; + +/// Transfer a fresh primitive array at its last local use. +/// Earlier reads and writes are allowed. Escaping aliases and later uses are not. +/// Require allocation and transfer in the same block to prevent snapshot reuse across loop iterations. +fn reuse_array_snapshots(body: &mut Body, types: &Types) { + let mut candidates = Vec::new(); + for (index, inst) in body.instructions.iter().enumerate() { + let Op::CopyValue(source) = inst.op else { + continue; + }; + let source = body.resolve(source); + let ty = body.value_type(source); + let Some(Type::Array(element)) = types.get(ty) else { + continue; + }; + if body.value_type(inst.result.unwrap()) != ty + || !types + .get(element) + .is_some_and(|t| matches!(t, Type::Scalar(_)) && t.carrier() != 5) + { + continue; + } + let ValueDef::Inst(def) = body.values[source.index()].def else { + continue; + }; + if !matches!( + body.instructions[def.index()].op, + Op::NewArray(_) + | Op::CopyValue(_) + | Op::LoadCopy(_) + | Op::LoadFieldCopy { .. } + | Op::LoadStorageFieldCopy { .. } + | Op::LoadAddressCopy(_) + | Op::LoadTypedCopy { .. } + ) { + continue; + } + candidates.push((index, source, def)); + } + if candidates.is_empty() { + return; + } + let mut positions = vec![None; body.instructions.len()]; + for (block, data) in body.blocks.iter().enumerate() { + for (position, id) in data.instructions.iter().enumerate() { + positions[id.index()] = Some((block as u32, position as u32)); + } + } + let mut transfers = vec![None; body.values.len()]; + let mut invalid = vec![false; body.values.len()]; + for (index, source, def) in candidates { + let (Some((owner, _)), Some((block, _))) = (positions[def.index()], positions[index]) + else { + continue; + }; + if owner != block { + continue; + } + if transfers[source.index()] + .replace(InstId::new(index)) + .is_some() + { + invalid[source.index()] = true; + } + } + if transfers.iter().all(Option::is_none) { + return; + } + for (index, inst) in body.instructions.iter().enumerate() { + inst.op.visit_uses(&body.args, |value| { + let value = body.resolve(value); + let Some(copy) = transfers[value.index()] else { + return; + }; + if copy.index() == index { + return; + } + let local_array = match inst.op { + Op::ArrayGet { array, .. } + | Op::ArraySet { array, .. } + | Op::ArrayFill { array, .. } + | Op::ArrayLength(array) => body.resolve(array) == value, + _ => false, + }; + let before = match (positions[index], positions[copy.index()]) { + (Some((a, i)), Some((b, j))) => a == b && i < j, + _ => false, + }; + invalid[value.index()] |= !local_array || !before; + }); + } + let mut escape = |value| { + invalid[body.resolve(value).index()] = true; + }; + for block in &body.blocks { + if let Some(term) = block.terminator { + term.visit_uses(&mut escape); + } + } + for edge in &body.edges { + for &value in &edge.args { + escape(value); + } + } + for (source, copy) in transfers.into_iter().enumerate() { + if let Some(copy) = copy + && !invalid[source] + { + body.instructions[copy.index()].op = Op::Reinterpret(ValueId::new(source)); + } + } +} + +pub(super) fn expand_flat_copies( + body: &mut Body, + types: &Types, + layout: &mut impl FnMut(&str) -> Option>, +) { + if !body + .instructions + .iter() + .any(|i| matches!(i.op, Op::CopyValue(_))) + { + return; + } + reuse_array_snapshots(body, types); + let mut schemas = FxHashMap::)>::default(); + for inst in &body.instructions { + let Op::Call { + method, + kind: CallKind::Constructor, + .. + } = inst.op + else { + continue; + }; + let ty = body.value_type(inst.result.unwrap()); + if schemas.contains_key(&ty) { + continue; + } + let method_ref = &body.methods[method.index()]; + let Some(fields) = layout(&method_ref.owner) else { + continue; + }; + if fields.len() > 8 + || fields.len() != method_ref.params.len() + || !fields.iter().zip(&method_ref.params).all(|(field, &ty)| { + field.ty == ty && !field.is_static && matches!(types.get(ty), Some(Type::Scalar(_))) + }) + { + continue; + } + schemas.insert(ty, (method, fields)); + } + if schemas.is_empty() { + return; + } + reuse_scalar_snapshots(body, types, &schemas); + // Join non-null facts across all incoming values. + // Each value changes from unseen to non-null to unknown at most once. + let count = body.values.len(); + let mut facts = vec![2u8; count]; + let mut users = crate::analysis::ValueUsers::new(count); + let predecessors = body.predecessors(); + for (index, value) in body.values.iter().enumerate() { + if !schemas.contains_key(&value.ty) { + continue; + } + match value.def { + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::Call { + kind: CallKind::Constructor, + .. + } => facts[index] = 1, + Op::CopyValue(source) | Op::Reinterpret(source) | Op::Refine(source) + if body.value_type(source) == value.ty => + { + facts[index] = 0; + users.connect(source, index); + } + _ => {} + }, + ValueDef::Alias(source) => { + facts[index] = 0; + users.connect(source, index); + } + ValueDef::Param(block) + if block != body.entry && !predecessors[block.index()].is_empty() => + { + facts[index] = 0; + let position = body.blocks[block.index()] + .params + .iter() + .position(|v| v.index() == index) + .unwrap(); + for &(_, edge) in &predecessors[block.index()] { + users.connect(body.edges[edge.index()].args[position], index); + } + } + _ => {} + } + } + let mut pending = (0..count).filter(|&i| facts[i] != 0).collect::>(); + for phase in 0..2 { + while let Some(source) = pending.pop() { + for target in users.users(source) { + let next = facts[source].max(facts[target]); + if next != facts[target] { + facts[target] = next; + pending.push(target); + } + } + } + if phase == 0 { + for (i, fact) in facts.iter_mut().enumerate() { + if *fact == 0 { + *fact = 2; + pending.push(i); + } + } + } + } + let mut prefixes = FxHashMap::>::default(); + for index in 0..body.instructions.len() { + let Op::CopyValue(source) = body.instructions[index].op else { + continue; + }; + if facts[source.index()] != 1 { + continue; + } + let Some((method, fields)) = schemas.get(&body.value_type(source)) else { + continue; + }; + let mut prefix = Vec::with_capacity(fields.len()); + let args = fields + .iter() + .map(|field| { + let member = body + .fields + .iter() + .position(|f| f == field) + .map(MemberId::new) + .unwrap_or_else(|| { + let id = MemberId::new(body.fields.len()); + body.fields.push(field.clone()); + id + }); + let inst = InstId::new(body.instructions.len()); + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty: field.ty, + def: ValueDef::Inst(inst), + }); + body.instructions.push(Inst { + op: Op::GetField { + object: source, + field: member, + }, + result: Some(value), + }); + prefix.push(inst); + value + }) + .collect::>(); + body.instructions[index].op = Op::Call { + method: *method, + kind: CallKind::Constructor, + args: List::append(&mut body.args, args), + }; + prefixes.insert(InstId::new(index), prefix); + } + for block in &mut body.blocks { + let previous = std::mem::take(&mut block.instructions); + for id in previous { + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(Terminator::Invoke { inst, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + } + } +} + +// Scalar field reads can use the source while no intervening operation can change it. +fn reuse_scalar_snapshots( + body: &mut Body, + types: &Types, + schemas: &FxHashMap)>, +) { + let mut positions = vec![None; body.instructions.len()]; + for (block, data) in body.blocks.iter().enumerate() { + let mut epoch = 0; + for &id in &data.instructions { + positions[id.index()] = Some((block, epoch)); + let inst = body.instructions[id.index()]; + let read_only = match inst.op { + Op::CopyValue(source) => schemas.contains_key(&body.value_type(source)), + Op::GetField { field, .. } => schemas + .get(&body.fields[field.index()].owner) + .is_some_and(|(_, fields)| fields.contains(&body.fields[field.index()])), + _ => !inst.op.may_throw(body, types), + }; + if !read_only { + epoch += 1; + } + } + } + let mut candidates = vec![None; body.values.len()]; + for (index, inst) in body.instructions.iter().enumerate() { + if let Op::CopyValue(source) = inst.op { + if schemas.contains_key(&body.value_type(source)) && positions[index].is_some() { + candidates[inst.result.unwrap().index()] = Some(InstId::new(index)); + } + } + } + let mut valid = vec![true; body.values.len()]; + for (index, inst) in body.instructions.iter().enumerate() { + inst.op.visit_uses(&body.args, |value| { + let value = body.resolve(value); + if let Some(copy) = candidates[value.index()] { + valid[value.index()] &= positions[index] == positions[copy.index()] + && matches!(inst.op, Op::GetField { object, field } if body.resolve(object) == value + && schemas.get(&body.value_type(value)).is_some_and(|(_, fields)| fields.contains(&body.fields[field.index()]))); + } + }); + } + for block in &body.blocks { + block + .terminator + .unwrap() + .visit_uses(|value| valid[body.resolve(value).index()] = false); + } + for edge in &body.edges { + for &value in &edge.args { + valid[body.resolve(value).index()] = false; + } + } + for (value, copy) in candidates.into_iter().enumerate() { + if valid[value] { + if let Some(copy) = copy { + let Op::CopyValue(source) = body.instructions[copy.index()].op else { + unreachable!() + }; + body.instructions[copy.index()].op = Op::Reinterpret(source); + } + } + } +} diff --git a/compiler-core/src/opt/view_abi.rs b/compiler-core/src/opt/view_abi.rs new file mode 100644 index 00000000..43947cd0 --- /dev/null +++ b/compiler-core/src/opt/view_abi.rs @@ -0,0 +1,165 @@ +//! Flatten borrowed-view arguments before instruction selection. +//! Keep boundary values as virtual packs. SSA liveness determines whether a +//! consumer needs a JVM carrier. Retain only one body's physical operands. +use super::append_value as append; +use crate::ir::*; +use crate::scalar::ScalarType; +use rustc_hash::FxHashMap; + +pub fn component_argument_slots(types: &Types, params: impl IntoIterator) -> usize { + params + .into_iter() + .map(|ty| match types.get(ty).unwrap() { + _ if ComponentShape::of(types, ty).is_some() => { + ComponentShape::of(types, ty).unwrap().slots() + } + Type::Scalar(ScalarType::I64 | ScalarType::U64 | ScalarType::F64) => 2, + Type::Unit | Type::Opaque(_) | Type::Layout(_) => 0, + _ => 1, + }) + .sum() +} + +pub fn lower_component_arguments( + body: &mut Body, + types: &mut Types, + entry: bool, + select: impl Fn(&MethodRef) -> bool, + debug: Option<&mut DebugInfo>, +) { + let shape = |ty| ComponentShape::of(types, ty); + let methods = body + .methods + .iter() + .map(|method| { + (select(method) + && component_argument_slots(types, method.params.iter().copied()) + + usize::from(ComponentShape::of(types, method.returns).is_some()) + <= 254) + .then(|| { + method + .params + .iter() + .map(|&ty| shape(ty)) + .collect::>() + }) + .filter(|p| p.iter().any(Option::is_some)) + }) + .collect::>(); + let entry = entry + && body.blocks[body.entry.index()] + .params + .iter() + .any(|&p| shape(body.value_type(p)).is_some()); + if !entry && methods.iter().all(Option::is_none) { + return; + } + let mut prologue = Vec::new(); + if entry { + let params = std::mem::take(&mut body.blocks[body.entry.index()].params); + for value in params { + let Some(shape) = ComponentShape::of(types, body.value_type(value)) else { + body.blocks[body.entry.index()].params.push(value); + continue; + }; + let args = shape + .parts(types) + .into_iter() + .map(|ty| { + let value = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Param(body.entry), + }); + body.blocks[body.entry.index()].params.push(value); + value + }) + .collect::>(); + let id = InstId::new(body.instructions.len()); + body.values[value.index()].def = ValueDef::Inst(id); + let args = List::append(&mut body.args, args); + body.instructions.push(Inst { + op: shape.pack(args), + result: Some(value), + }); + prologue.push(id); + } + } + let mut prefixes = FxHashMap::>::default(); + let count = body.instructions.len(); + for index in 0..count { + let Op::Call { method, kind, args } = body.instructions[index].op else { + continue; + }; + let Some(expand) = &methods[method.index()] else { + continue; + }; + let receiver = usize::from(matches!(kind, CallKind::Virtual | CallKind::Interface)); + let original = body.args[args.range()].to_vec(); + let mut arguments = original[..receiver].to_vec(); + let mut prefix = Vec::new(); + for (&value, &expand) in original[receiver..].iter().zip(expand) { + if let Some(shape) = expand { + for (index, ty) in shape.parts(types).into_iter().enumerate() { + let (inst, value) = append(body, shape.part(value, index as u8), ty); + prefix.push(inst); + arguments.push(value); + } + } else { + arguments.push(value); + } + } + let args = List::append(&mut body.args, arguments); + body.instructions[index].op = Op::Call { method, kind, args }; + prefixes.insert(InstId::new(index), prefix); + } + for (method, expand) in body.methods.iter_mut().zip(methods) { + if let Some(expand) = expand { + let original = std::mem::take(&mut method.params); + for (ty, expand) in original.into_iter().zip(expand) { + if let Some(shape) = expand { + method.params.extend(shape.parts(types)); + } else { + method.params.push(ty); + } + } + } + } + let mut positions = debug + .as_ref() + .map(|_| Vec::with_capacity(body.blocks.len())); + for (index, block) in body.blocks.iter_mut().enumerate() { + let previous = std::mem::take(&mut block.instructions); + if index == body.entry.index() { + block.instructions.append(&mut prologue); + } + let mut mapping = positions + .as_ref() + .map(|_| Vec::with_capacity(previous.len() + 1)); + for id in previous { + if let Some(mapping) = &mut mapping { + mapping.push(block.instructions.len() as u32); + } + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(mapping) = &mut mapping { + mapping.push(block.instructions.len() as u32); + } + if let Some(Terminator::Invoke { inst, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + } + if let (Some(positions), Some(mapping)) = (&mut positions, mapping) { + positions.push(mapping); + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/view_roundtrips.rs b/compiler-core/src/opt/view_roundtrips.rs new file mode 100644 index 00000000..288906c1 --- /dev/null +++ b/compiler-core/src/opt/view_roundtrips.rs @@ -0,0 +1,188 @@ +//! Recover slice components from exact, bounded address conversions. +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +fn index_from_offset(body: &Body, types: &Types, offset: ValueId, size: u32) -> Option { + let mut offset = body.resolve(offset); + if let Some(value) = body.scalar_value(offset).and_then(Scalar::signed) { + if value % i128::from(size) != 0 { + return None; + } + let index = i32::try_from(value / i128::from(size)).ok()?; + return Some(Folded::Constant( + Scalar::integer(ScalarType::I32, index as u128).unwrap(), + )); + } + if size != 1 { + let ValueDef::Inst(id) = body.values[offset.index()].def else { + return None; + }; + let Op::Binary { + op: BinaryOp::Mul, + left, + right, + } = body.instructions[id.index()].op + else { + return None; + }; + offset = body.resolve( + if body.scalar_value(right).map(Scalar::bits) == Some(size.into()) { + left + } else if body.scalar_value(left).map(Scalar::bits) == Some(size.into()) { + right + } else { + return None; + }, + ); + } + let ValueDef::Inst(id) = body.values[offset.index()].def else { + return None; + }; + let Op::Cast(index) = body.instructions[id.index()].op else { + return None; + }; + (types.get(body.value_type(index)) == Some(Type::Scalar(ScalarType::I32))) + .then_some(Folded::Value(index)) +} + +pub fn fold_view_roundtrips(body: &mut Body, types: &Types) -> bool { + let mut changed = false; + for position in 0..body.instructions.len() { + let Op::TypedAddressViewPart { + parts, + size, + codec: Some(codec), + index, + } = body.instructions[position].op + else { + continue; + }; + if size == 0 || size > i32::MAX as u32 { + continue; + } + let args = &body.args[parts.range()]; + let root = body.resolve(args[0]); + let ValueDef::Inst(id) = body.values[root.index()].def else { + continue; + }; + if !matches!(body.instructions[id.index()].op, + Op::ViewRoot { size: actual, codec: Some(recipe), .. } + if actual == size && recipe == codec) + { + continue; + } + // Normalization fixes the root layout. The scaled i32 index cannot overflow i64. + let Some(start) = index_from_offset(body, types, args[1], size) else { + continue; + }; + body.instructions[position].op = match (index, start) { + (0, _) => Op::Reinterpret(root), + (1, Folded::Value(value)) => Op::Reinterpret(value), + (1, Folded::Constant(value)) => { + let id = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar(value)); + Op::Constant(id) + } + _ => continue, + }; + changed = true; + } + changed +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrips_require_exact_layout_and_a_bounded_element_index() { + for case in [ + "signed", + "unsigned", + "unaligned", + "oversized", + "codec", + "stride", + ] { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::I64); + let source_type = types.scalar(if case == "unsigned" { + ScalarType::U32 + } else { + ScalarType::I32 + }); + let name = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(name)); + let codec = types.symbol("PairCodec#pair#LPair;#16"); + let other = types.symbol("OtherCodec#pair#LPair;#16"); + let mut b = Builder::new(&types, int); + let backing = b.parameter(b.current(), object); + let start = b.parameter(b.current(), source_type); + let root = b + .emit( + Op::ViewRoot { + backing, + size: 16, + codec: Some(codec), + }, + Some(object), + ) + .unwrap(); + let wide = b.emit(Op::Cast(start), Some(long)).unwrap(); + let stride = b.constant( + long, + Scalar::integer(ScalarType::I64, if case == "stride" { 8 } else { 16 }).unwrap(), + ); + let offset = match case { + "unaligned" | "oversized" => b.constant( + long, + Scalar::integer( + ScalarType::I64, + if case == "unaligned" { + 1 + } else { + (i32::MAX as u128 + 1) * 16 + }, + ) + .unwrap(), + ), + _ => b + .emit( + Op::Binary { + op: BinaryOp::Mul, + left: wide, + right: stride, + }, + Some(long), + ) + .unwrap(), + }; + let parts = b.args([root, offset]); + let read = b + .emit( + Op::TypedAddressViewPart { + parts, + size: 16, + codec: Some(if case == "codec" { other } else { codec }), + index: 1, + }, + Some(int), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(read))); + let mut body = b.finish().unwrap(); + assert_eq!( + fold_view_roundtrips(&mut body, &types), + case == "signed", + "{case}" + ); + super::super::simplify_components(&mut body, &types); + verify(&body, &types).unwrap(); + if case == "signed" { + assert_eq!(body.resolve(read), start); + } + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + } + } +} diff --git a/compiler-core/src/opt/views.rs b/compiler-core/src/opt/views.rs new file mode 100644 index 00000000..1863e975 --- /dev/null +++ b/compiler-core/src/opt/views.rs @@ -0,0 +1,267 @@ +//! Decompose local views through SSA joins. +//! Data and length consumers use scalars. Opaque boundaries allocate carriers only when needed. +use super::append_value as append; +use crate::ir::*; +use crate::scalar::{Scalar, ScalarType}; +use rustc_hash::FxHashMap; + +pub fn decompose_views(body: &mut Body, types: &mut Types, debug: Option<&mut DebugInfo>) { + if !body.instructions.iter().any(|i| { + matches!( + i.op, + Op::View { .. } | Op::ViewPack(_) | Op::ViewPart { .. } + ) + }) { + return; + } + let count = body.values.len(); + let mut eligible = vec![false; count]; + let mut users = crate::analysis::ValueUsers::new(count); + let predecessors = body.predecessors(); + for index in 0..count { + let value = ValueId::new(index); + if !ComponentShape::View.accepts_annotation(types, body.value_type(value)) { + continue; + } + match body.values[index].def { + ValueDef::Inst(id) => match body.instructions[id.index()].op { + Op::View { .. } | Op::ViewPack(_) => eligible[index] = true, + Op::Constant(id) + if matches!( + body.constants[id.index()], + Constant::Null(_) | Constant::Uninit(_) + ) => + { + eligible[index] = true; + } + Op::Reinterpret(source) | Op::Adapt(source) | Op::Refine(source) => { + eligible[index] = true; + users.connect(source, index); + } + _ => {} + }, + ValueDef::Alias(source) => { + eligible[index] = true; + users.connect(source, index); + } + ValueDef::Param(block) if block != body.entry => { + eligible[index] = !predecessors[block.index()].is_empty(); + let position = body.blocks[block.index()] + .params + .iter() + .position(|&v| v == value) + .unwrap(); + for &(_, edge) in &predecessors[block.index()] { + users.connect(body.edges[edge.index()].args[position], index); + } + } + _ => {} + } + } + // Reject each node at most once, including cycles with unknown inputs. + // No repeated whole-body scan is needed. + users.close(&mut eligible); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let int = types.scalar(ScalarType::I32); + let length_type = types.scalar(ScalarType::U64); + let mut components = vec![None::<[ValueId; 3]>; count]; + let mut prefixes = FxHashMap::>::default(); + let mut joins = vec![Vec::new(); body.blocks.len()]; + for index in 0..count { + if !eligible[index] { + continue; + } + match body.values[index].def { + ValueDef::Param(block) => { + let mut parts = [ValueId::new(0); 3]; + for (part, ty) in parts.iter_mut().zip([object, int, length_type]) { + *part = ValueId::new(body.values.len()); + body.values.push(Value { + ty, + def: ValueDef::Param(block), + }); + body.blocks[block.index()].params.push(*part); + } + let position = body.blocks[block.index()] + .params + .iter() + .position(|&value| value.index() == index) + .unwrap(); + joins[block.index()].push((index, position)); + components[index] = Some(parts); + } + ValueDef::Inst(id) => { + // Partially initialized aggregates use null borrowed fields. + // Read their default component slots without dereferencing a view carrier. + if let Op::Constant(constant) = body.instructions[id.index()].op { + let uninit = matches!(body.constants[constant.index()], Constant::Uninit(_)); + let mut parts = [ValueId::new(0); 3]; + let mut prefix = Vec::new(); + for (index, (ty, constant)) in [ + (object, Constant::Null(object)), + ( + int, + Constant::Scalar(Scalar::integer(ScalarType::I32, 0).unwrap()), + ), + ( + length_type, + Constant::Scalar(Scalar::integer(ScalarType::U64, 0).unwrap()), + ), + ] + .into_iter() + .enumerate() + { + let constant_id = ConstId::new(body.constants.len()); + body.constants.push(if uninit { + Constant::Uninit(ty) + } else { + constant + }); + let (instruction, value) = append(body, Op::Constant(constant_id), ty); + prefix.push(instruction); + parts[index] = value; + } + prefixes.insert(id, prefix); + components[index] = Some(parts); + } + if let Op::ViewPack(parts) = body.instructions[id.index()].op { + components[index] = Some(body.args[parts.range()].try_into().unwrap()); + } + if let Op::View { data, length } = body.instructions[id.index()].op { + let (cast, data) = append(body, Op::Reinterpret(data), object); + let constant = ConstId::new(body.constants.len()); + body.constants.push(Constant::Scalar( + Scalar::integer(ScalarType::I32, 0).unwrap(), + )); + let (zero, start) = append(body, Op::Constant(constant), int); + prefixes.insert(id, vec![cast, zero]); + components[index] = Some([data, start, length]); + } + } + _ => {} + } + } + users.propagate(&eligible, &mut components); + for block in 0..body.blocks.len() { + // Visit parameters in the same order used when extending their target. + for &(_, edge) in &predecessors[block] { + for &(_, position) in &joins[block] { + let input = body.edges[edge.index()].args[position]; + body.edges[edge.index()] + .args + .extend(components[input.index()].unwrap()); + } + } + } + for inst in &mut body.instructions { + inst.op = match inst.op { + Op::Adapt(source) + if inst + .result + .and_then(|v| components.get(v.index()).copied().flatten()) + .is_some() => + { + let result = inst.result.unwrap(); + if matches!( + types.get(body.values[result.index()].ty), + Some(Type::Slice(_) | Type::Str) + ) { + Op::ViewPack(List::append( + &mut body.args, + components[result.index()].unwrap(), + )) + } else { + Op::Reinterpret(source) + } + } + Op::Length(view) if components.get(view.index()).is_some_and(Option::is_some) => { + Op::Reinterpret(components[view.index()].unwrap()[2]) + } + Op::ArrayLength(view) if components.get(view.index()).is_some_and(Option::is_some) => { + // SliceView.length contains the low 32 bits of the Rust length. + // Extract them without a view carrier. + Op::Cast(components[view.index()].unwrap()[2]) + } + Op::ViewPart { view, index } + if components.get(view.index()).is_some_and(Option::is_some) => + { + Op::Reinterpret(components[view.index()].unwrap()[index as usize]) + } + Op::ViewData { view, size, codec } + if components.get(view.index()).is_some_and(Option::is_some) => + { + let parts = List::append(&mut body.args, components[view.index()].unwrap()); + Op::ViewAddress { parts, size, codec } + } + Op::ArrayGet { + array, + index, + native: false, + } + | Op::ArrayGetCopy { array, index } + if components.get(array.index()).is_some_and(Option::is_some) => + { + let [root, start, _] = components[array.index()].unwrap(); + let parts = List::append(&mut body.args, [root, start, index]); + if matches!(inst.op, Op::ArrayGetCopy { .. }) { + Op::ViewGetCopy(parts) + } else { + Op::ViewGet(parts) + } + } + Op::ArraySet { + array, + index, + value, + native: false, + } if components.get(array.index()).is_some_and(Option::is_some) => { + let [root, start, _] = components[array.index()].unwrap(); + Op::ViewSet { + parts: List::append(&mut body.args, [root, start, index]), + value, + } + } + op => op, + }; + } + let mut positions = debug + .as_ref() + .map(|_| Vec::with_capacity(body.blocks.len())); + for block in &mut body.blocks { + let mut mapping = positions + .as_ref() + .map(|_| Vec::with_capacity(block.instructions.len() + 1)); + let previous = std::mem::take(&mut block.instructions); + for id in previous { + if let Some(mapping) = &mut mapping { + mapping.push(block.instructions.len() as u32); + } + if let Some(prefix) = prefixes.remove(&id) { + block.instructions.extend(prefix); + } + block.instructions.push(id); + } + if let Some(mapping) = &mut mapping { + mapping.push(block.instructions.len() as u32); + } + if let (Some(positions), Some(mapping)) = (&mut positions, mapping) { + positions.push(mapping); + } + if let Some(Terminator::Invoke { inst, normal, .. }) = block.terminator { + if let Some(prefix) = prefixes.remove(&inst) { + block.instructions.extend(prefix); + } + let op = body.instructions[inst.index()].op; + if matches!(op, Op::View { .. } | Op::ViewPack(_) | Op::Reinterpret(_)) { + block.instructions.push(inst); + block.terminator = Some(Terminator::Jump(normal)); + } + } + } + if let (Some(debug), Some(positions)) = (debug, positions) { + for event in &mut debug.events { + event.position = positions[event.block.index()][event.position as usize]; + } + } +} diff --git a/compiler-core/src/opt/views_tests.rs b/compiler-core/src/opt/views_tests.rs new file mode 100644 index 00000000..127b11e4 --- /dev/null +++ b/compiler-core/src/opt/views_tests.rs @@ -0,0 +1,215 @@ +use super::decompose_views; +use crate::ir::*; +use crate::scalar::{BinaryOp, Scalar, ScalarType}; + +#[test] +fn jvm_view_length_truncates_without_constructing_the_view() { + let mut types = Types::default(); + let int = types.scalar(ScalarType::I32); + let long = types.scalar(ScalarType::U64); + let object = types.symbol("java/lang/Object"); + let object = types.intern(Type::Class(object)); + let slice = types.intern(Type::Slice(int)); + let mut b = Builder::new(&types, int); + let root = b.parameter(b.current(), object); + let start = b.constant(int, Scalar::integer(ScalarType::I32, 0).unwrap()); + let length = b.constant( + long, + Scalar::integer(ScalarType::U64, 0x1_0000_0007).unwrap(), + ); + let parts = b.args([root, start, length]); + let view = b.emit(Op::ViewPack(parts), Some(slice)).unwrap(); + let result = b.emit(Op::ArrayLength(view), Some(int)).unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + decompose_views(&mut body, &mut types, None); + super::simplify_components(&mut body, &types); + verify(&body, &types).unwrap(); + assert_eq!(body.scalar_value(result).unwrap().bits(), 7); + let live = super::live(&body, &types); + assert!( + !body + .instructions + .iter() + .enumerate() + .any(|(i, inst)| live.instructions[i] + && matches!(inst.op, Op::ViewPack(_) | Op::ArrayLength(_))) + ); +} + +#[test] +fn view_components_cross_loops_without_a_carrier() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::U64); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(long)); + let slice = types.intern(Type::Slice(long)); + let runtime = types.symbol("org/rustlang/runtime/SliceView"); + let runtime = types.intern(Type::Class(runtime)); + let mut b = Builder::new(&types, pointer); + let data = b.parameter(b.current(), pointer); + let length = b.parameter(b.current(), long); + let initial = b.emit(Op::View { data, length }, Some(slice)).unwrap(); + let initial = b.emit(Op::Reinterpret(initial), Some(runtime)).unwrap(); + let initial = b.emit(Op::Adapt(initial), Some(slice)).unwrap(); + let value = b.variable(slice); + b.define(value, initial); + let header = b.create_block(); + let step = b.create_block(); + let done = b.create_block(); + b.jump(header, vec![]); + b.switch_to(header); + let view = b.read(value); + let length = b.emit(Op::Length(view), Some(long)).unwrap(); + let zero = b.constant(long, Scalar::integer(ScalarType::U64, 0).unwrap()); + let more = b + .emit( + Op::Binary { + op: BinaryOp::Gt, + left: length, + right: zero, + }, + Some(boolean), + ) + .unwrap(); + b.branch(more, step, done); + b.switch_to(step); + let one = b.constant(long, Scalar::integer(ScalarType::U64, 1).unwrap()); + let next = b + .emit( + Op::Binary { + op: BinaryOp::Sub, + left: length, + right: one, + }, + Some(long), + ) + .unwrap(); + let view = b + .emit(Op::View { data, length: next }, Some(slice)) + .unwrap(); + b.define(value, view); + b.jump(header, vec![]); + b.switch_to(done); + let view = b.read(value); + let pointer_value = b + .emit( + Op::ViewData { + view, + size: 8, + codec: None, + }, + Some(pointer), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(pointer_value))); + let mut body = b.finish().unwrap(); + decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + let live = super::live(&body, &types); + assert!( + body.instructions + .iter() + .enumerate() + .all(|(i, inst)| !live.instructions[i] + || !matches!( + inst.op, + Op::View { .. } + | Op::ViewPack(_) + | Op::Adapt(_) + | Op::Length(_) + | Op::ViewData { .. } + )) + ); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + assert!( + !code + .instructions + .iter() + .any(|i| matches!(i, crate::classfile::attributes::Instruction::New(_))) + ); +} + +#[test] +fn unknown_input_keeps_a_mixed_join_conservative() { + let mut types = Types::default(); + let long = types.scalar(ScalarType::U64); + let boolean = types.scalar(ScalarType::Bool); + let pointer = types.intern(Type::Pointer(long)); + let slice = types.intern(Type::Slice(long)); + let mut b = Builder::new(&types, long); + let input = b.parameter(b.current(), slice); + let condition = b.parameter(b.current(), boolean); + let data = b.parameter(b.current(), pointer); + let value = b.variable(slice); + b.define(value, input); + let create = b.create_block(); + let join = b.create_block(); + b.branch(condition, create, join); + b.switch_to(create); + let length = b.constant(long, Scalar::integer(ScalarType::U64, 17).unwrap()); + let constructed = b.emit(Op::View { data, length }, Some(slice)).unwrap(); + b.define(value, constructed); + b.jump(join, vec![]); + b.switch_to(join); + let joined = b.read(value); + let length = b.emit(Op::Length(joined), Some(long)).unwrap(); + b.terminate(Terminator::Return(Some(length))); + let mut body = b.finish().unwrap(); + decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert!( + body.instructions + .iter() + .any(|i| matches!(i.op, Op::Length(_))) + ); + crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); +} + +#[test] +fn component_calls_and_entries_need_no_view_objects() { + let mut types = Types::default(); + let byte = types.scalar(ScalarType::U8); + let length = types.scalar(ScalarType::U64); + let slice = types.intern(Type::Slice(byte)); + let mut b = Builder::new(&types, length); + let input = b.parameter(b.current(), slice); + let target = b.method(MethodRef { + owner: "Leaf".into(), + name: "len".into(), + params: vec![slice], + returns: length, + interface: false, + }); + let args = b.args([input]); + let result = b + .emit( + Op::Call { + method: target, + kind: CallKind::RustStatic, + args, + }, + Some(length), + ) + .unwrap(); + b.terminate(Terminator::Return(Some(result))); + let mut body = b.finish().unwrap(); + super::lower_component_arguments(&mut body, &mut types, true, |_| true, None); + decompose_views(&mut body, &mut types, None); + verify(&body, &types).unwrap(); + assert_eq!(body.blocks[body.entry.index()].params.len(), 3); + assert_eq!(body.methods[target.index()].params.len(), 3); + let code = crate::jvm::select::compile(&body, &types, &mut Default::default()).unwrap(); + use crate::classfile::attributes::Instruction; + assert!(!code.instructions.iter().any(|i| matches!( + i, + Instruction::New(_) | Instruction::Getfield(_) | Instruction::Checkcast(_) + ))); + assert_eq!( + code.instructions + .iter() + .filter(|i| matches!(i, Instruction::Invokestatic(_))) + .count(), + 1 + ); +} diff --git a/config.toml.template b/config.toml.template index 9fbdd937..1662c79f 100644 --- a/config.toml.template +++ b/config.toml.template @@ -3,8 +3,7 @@ rustflags = [ "-C", "link-args=../../../runtime/build/libs/runtime-0.1.0.jar", "-C", "strip=none", "-Z", "unstable-options", "--jobs-frontend=4", - "-Z", "share-generics=yes", - "-Z", "inline-mir=no" + "-Z", "share-generics=yes" ] # Rust panics cross JVM frames as exceptions and are caught by panic_unwind. diff --git a/java-linker/Cargo.lock b/java-linker/Cargo.lock index 674acec7..666d1b27 100644 --- a/java-linker/Cargo.lock +++ b/java-linker/Cargo.lock @@ -261,12 +261,13 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -342,6 +343,7 @@ dependencies = [ name = "java-linker" version = "0.1.0" dependencies = [ + "flate2", "jvm-compiler-core", "rayon", "ristretto_classfile", @@ -435,6 +437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -911,6 +914,12 @@ dependencies = [ "zstd", ] +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + [[package]] name = "zopfli" version = "0.8.1" diff --git a/java-linker/Cargo.toml b/java-linker/Cargo.toml index 6c03395a..977a7a27 100644 --- a/java-linker/Cargo.toml +++ b/java-linker/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] +flate2 = { version = "1.1.9", features = ["zlib-rs"] } jvm-compiler-core = { path = "../compiler-core" } rayon = "1.10.0" ristretto_classfile = "0.31.0" diff --git a/java-linker/src/aliases.rs b/java-linker/src/aliases.rs new file mode 100644 index 00000000..048d8cf6 --- /dev/null +++ b/java-linker/src/aliases.rs @@ -0,0 +1,67 @@ +//! Redirect direct calls and method handles through their shared constant-pool entries. +use crate::*; + +pub(crate) type Aliases = + HashMap>; + +pub(crate) fn redirect<'a>(pool: &mut ConstantPool<'a>, aliases: &Aliases) -> io::Result { + if aliases.is_empty() { + return Ok(false); + } + let error = |e| constant_pool_error("forwarding method relocation", e); + let mut replacements = Vec::new(); + for index in 1..=pool.len() as u16 { + let Some(Constant::MethodRef { + class_index, + name_and_type_index, + }) = pool.get(index) + else { + continue; + }; + let owner = pool.try_get_class(*class_index).map_err(error)?; + let Some(methods) = aliases.get(owner) else { + continue; + }; + let (name, descriptor) = pool + .try_get_name_and_type(*name_and_type_index) + .map_err(error)?; + let key = ( + pool.try_get_utf8(*name).map_err(error)?.to_owned(), + pool.try_get_utf8(*descriptor).map_err(error)?.to_owned(), + ); + if let Some((owner, name)) = methods.get(&key) { + replacements.push((index, *descriptor, owner.clone(), name.clone())); + } + } + if replacements.is_empty() { + return Ok(false); + } + let mut constants = constant_pool_index(pool); + for (index, descriptor_index, owner, name) in replacements { + let mut intern = |constant: Constant<'a>| -> io::Result { + let key = ConstantKey::from(&constant); + if let Some(&id) = constants.get(&key) { + return Ok(id); + } + let id = pool.add(constant).map_err(error)?; + constants.insert(key, id); + Ok(id) + }; + let owner = intern(Constant::Utf8(owner.into()))?; + let class_index = intern(Constant::Class(owner))?; + let name_index = intern(Constant::Utf8(name.into()))?; + let name_and_type_index = intern(Constant::NameAndType { + name_index, + descriptor_index, + })?; + pool.set( + index, + Constant::MethodRef { + class_index, + name_and_type_index, + }, + ) + .map_err(error)?; + } + Ok(true) +} diff --git a/java-linker/src/aliases_tests.rs b/java-linker/src/aliases_tests.rs new file mode 100644 index 00000000..c7dcd7d7 --- /dev/null +++ b/java-linker/src/aliases_tests.rs @@ -0,0 +1,282 @@ +use crate::test_support::{body, holder, write_classes}; +use crate::*; + +fn forward(class: &mut ClassFile<'static>, name: &str, owner: &str, target: &str) { + let owner = class.constant_pool.add_class(owner).unwrap(); + let target = class + .constant_pool + .add_method_ref(owner, target, "(I)I") + .unwrap(); + body( + class, + name, + "(I)I", + vec![ + Instruction::Iload_0, + Instruction::Invokestatic(target), + Instruction::Istore_0, + Instruction::Iload_0, + Instruction::Ireturn, + ], + ); +} + +#[test] +fn forwarder_proof_keeps_reflection_private_targets_initializers_cycles_and_missing_bodies() { + for mode in [ + "plain", + "reflection", + "private", + "initializer", + "cycle", + "missing", + ] { + let mut main = holder("test/Main", false); + let mut a = holder("test/mono/Mono_a", true); + let mut target = holder("test/mono/Mono_target", true); + let owner = main.constant_pool.add_class("test/mono/Mono_a").unwrap(); + main.constant_pool + .add_method_ref(owner, "forward", "(I)I") + .unwrap(); + let (owner, method) = if mode == "cycle" { + ("test/mono/Mono_a", "forward") + } else { + ("test/mono/Mono_target", "identity") + }; + forward(&mut a, "forward", owner, method); + body( + &mut target, + "identity", + "(I)I", + vec![Instruction::Iload_0, Instruction::Ireturn], + ); + if mode == "reflection" { + main.constant_pool.add_string("forward").unwrap(); + } + if mode == "private" { + target.methods[0] + .access_flags + .remove(ristretto_classfile::MethodAccessFlags::PUBLIC); + } + if mode == "initializer" { + body(&mut a, "", "()V", vec![Instruction::Return]); + } + let mut graph = reachability::Graph::default(); + for class in [main, a] { + graph + .scan(&serialize_class_file(&class).unwrap(), false) + .unwrap(); + } + if mode != "missing" { + graph + .scan(&serialize_class_file(&target).unwrap(), false) + .unwrap(); + } + let demands = graph.finish(true); + assert_eq!( + demands.aliases.len(), + usize::from(mode == "plain"), + "{mode}" + ); + } +} + +#[test] +fn forwarders_redirect_calls_and_handles_without_an_additional_jar_pass() { + let mut main = holder("test/Main", false); + let mut a = holder("test/mono/Mono_a", true); + let mut b = holder("test/mono/Mono_b", true); + let mut target = holder("test/mono/Mono_target", true); + forward(&mut a, "forward", "test/mono/Mono_b", "forward_again"); + forward(&mut b, "forward_again", "test/mono/Mono_target", "identity"); + body( + &mut target, + "identity", + "(I)I", + vec![Instruction::Iload_0, Instruction::Ireturn], + ); + let owner = main.constant_pool.add_class("test/mono/Mono_a").unwrap(); + let call = main + .constant_pool + .add_method_ref(owner, "forward", "(I)I") + .unwrap(); + let handle = main + .constant_pool + .add_method_handle(ristretto_classfile::ReferenceKind::InvokeStatic, call) + .unwrap(); + let handle_type = main + .constant_pool + .add_class("java/lang/invoke/MethodHandle") + .unwrap(); + let invoke = main + .constant_pool + .add_method_ref(handle_type, "invokeExact", "(I)I") + .unwrap(); + let system = main.constant_pool.add_class("java/lang/System").unwrap(); + let out = main + .constant_pool + .add_field_ref(system, "out", "Ljava/io/PrintStream;") + .unwrap(); + let print = main.constant_pool.add_class("java/io/PrintStream").unwrap(); + let print = main + .constant_pool + .add_method_ref(print, "println", "(I)V") + .unwrap(); + body( + &mut main, + "main", + "([Ljava/lang/String;)V", + vec![ + Instruction::Iconst_5, + Instruction::Invokestatic(call), + Instruction::Istore_0, + Instruction::Getstatic(out), + Instruction::Iload_0, + Instruction::Invokevirtual(print), + Instruction::Ldc_w(handle), + Instruction::Bipush(7), + Instruction::Invokevirtual(invoke), + Instruction::Istore_0, + Instruction::Getstatic(out), + Instruction::Iload_0, + Instruction::Invokevirtual(print), + Instruction::Return, + ], + ); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes(temp.path(), [main, a, b, target]); + let output = temp.path().join("test.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + let execution = std::process::Command::new("java") + .args(["-Xverify:all", "-cp", output.to_str().unwrap(), "test.Main"]) + .output() + .unwrap(); + assert!( + execution.status.success(), + "{}", + String::from_utf8_lossy(&execution.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&execution.stdout) + .lines() + .collect::>(), + ["5", "7"] + ); + let mut jar = ZipArchive::new(fs::File::open(output).unwrap()).unwrap(); + let mut methods = 0; + for i in 0..jar.len() { + let mut entry = jar.by_index(i).unwrap(); + if !entry.name().ends_with(".class") { + continue; + } + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).unwrap(); + methods += class_file_from_data(&bytes).unwrap().methods.len(); + } + assert_eq!(methods, 2, "only main and the actual implementation remain"); +} + +#[test] +fn forwarder_proof_tracks_argument_order_wide_slots_and_fallthrough() { + for (name, descriptor, mut loads, ret, accepted) in [ + ( + "ordered", + "(II)I", + vec![Instruction::Iload_0, Instruction::Iload_1], + Instruction::Ireturn, + true, + ), + ( + "reordered", + "(II)I", + vec![Instruction::Iload_1, Instruction::Iload_0], + Instruction::Ireturn, + false, + ), + ( + "wide", + "(JD)J", + vec![Instruction::Lload_0, Instruction::Dload_2], + Instruction::Lreturn, + true, + ), + ( + "duplicate", + "(II)I", + vec![Instruction::Iload_0, Instruction::Iload_0], + Instruction::Ireturn, + false, + ), + ( + "backward", + "(II)I", + vec![Instruction::Iload_0, Instruction::Iload_1], + Instruction::Ireturn, + false, + ), + ] { + let mut main = holder("test/Main", false); + let mut source = holder("test/mono/Mono_source", true); + let mut target = holder("test/mono/Mono_target", true); + let owner = main + .constant_pool + .add_class("test/mono/Mono_source") + .unwrap(); + main.constant_pool + .add_method_ref(owner, name, descriptor) + .unwrap(); + let owner = source + .constant_pool + .add_class("test/mono/Mono_target") + .unwrap(); + let call = source + .constant_pool + .add_method_ref(owner, "target", descriptor) + .unwrap(); + loads.extend([ + Instruction::Invokestatic(call), + Instruction::Goto(if name == "backward" { 0 } else { 4 }), + ret.clone(), + ]); + body(&mut source, name, descriptor, loads); + body( + &mut target, + "target", + descriptor, + vec![ + if descriptor == "(JD)J" { + Instruction::Lload_0 + } else { + Instruction::Iload_0 + }, + ret, + ], + ); + for class in [&mut source, &mut target] { + for method in &mut class.methods { + for attribute in &mut method.attributes { + if let Attribute::Code { + max_stack, + max_locals, + .. + } = attribute + { + *max_stack = 4; + *max_locals = 4; + } + } + } + } + let mut graph = reachability::Graph::default(); + for class in [main, source, target] { + graph + .scan(&serialize_class_file(&class).unwrap(), false) + .unwrap(); + } + assert_eq!( + graph.finish(true).aliases.len(), + usize::from(accepted), + "{name}" + ); + } +} diff --git a/java-linker/src/carriers.rs b/java-linker/src/carriers.rs new file mode 100644 index 00000000..41e875aa --- /dev/null +++ b/java-linker/src/carriers.rs @@ -0,0 +1,262 @@ +//! Share proven private storage shapes, including recursive shapes. Keep unproven class identities distinct. +use crate::*; + +/// Retain the live helpers from every alias before pruning. Otherwise, pruning can split equivalent classes. +pub(crate) fn live_representatives( + carriers: &[(&str, &[u8])], + dead_methods: &HashMap>, +) -> Vec { + let initial = representatives(carriers); + let mut dead_by_group = HashMap::>::default(); + for (i, (name, recipe)) in carriers.iter().enumerate() { + if !recipe.starts_with(b"carrier-v2;enum-v1;") { + continue; + } + let dead = dead_methods + .get(*name) + .map(|methods| { + methods + .iter() + .map(|(name, _)| name.clone()) + .collect::>() + }) + .unwrap_or_default(); + dead_by_group + .entry(initial[i]) + .and_modify(|common| common.retain(|method| dead.contains(method))) + .or_insert(dead); + } + let live = carriers + .iter() + .enumerate() + .map(|(i, (_, recipe))| { + dead_by_group + .get(&initial[i]) + .map_or(std::borrow::Cow::Borrowed(*recipe), |dead| { + live_recipe(recipe, dead) + }) + }) + .collect::>(); + if live + .iter() + .all(|recipe| matches!(recipe, std::borrow::Cow::Borrowed(_))) + { + return initial; + } + let recipes = carriers + .iter() + .zip(&live) + .map(|((name, _), recipe)| (*name, recipe.as_ref())) + .collect::>(); + representatives(&recipes) +} + +fn live_recipe<'a>(recipe: &'a [u8], dead: &HashSet) -> std::borrow::Cow<'a, [u8]> { + use std::borrow::Cow; + if dead.is_empty() || !recipe.starts_with(b"carrier-v2;enum-v1;") { + return Cow::Borrowed(recipe); + } + // Each enum method recipe includes its nested field recipes and class references. + let starts = recipe + .windows(8) + .enumerate() + .filter_map(|(i, w)| (w == b";method=").then_some(i + 1)) + .collect::>(); + let mut result = Vec::new(); + let mut cursor = 0; + for (n, &start) in starts.iter().enumerate() { + let name_start = start + 7; + let Some(length) = recipe[name_start..].iter().position(|&b| b == b';') else { + return Cow::Borrowed(recipe); + }; + let name = &recipe[name_start..name_start + length]; + if dead.iter().any(|method| method.as_bytes() == name) { + result.extend_from_slice(&recipe[cursor..start]); + cursor = starts.get(n + 1).copied().unwrap_or(recipe.len()); + } + } + if cursor == 0 { + return Cow::Borrowed(recipe); + } + result.extend_from_slice(&recipe[cursor..]); + Cow::Owned(result) +} + +pub(crate) fn representatives<'a>(carriers: &[(&'a str, &[u8])]) -> Vec { + let names = carriers + .iter() + .enumerate() + .map(|(i, (name, _))| (*name, i)) + .collect::>(); + // Intern fixed recipe bytes once. Refinement then hashes only partition numbers. + let mut templates = HashMap::default(); + let shapes = carriers + .iter() + .enumerate() + .map(|(i, (_, recipe))| { + let refs = references(recipe, &names); + let mut key = Vec::with_capacity(recipe.len()); + let mut start = 0; + for &(begin, end, _) in &refs { + key.extend_from_slice(&((begin - start) as u64).to_le_bytes()); + key.extend_from_slice(&recipe[start..begin]); + start = end; + } + key.extend_from_slice(&((recipe.len() - start) as u64).to_le_bytes()); + key.extend_from_slice(&recipe[start..]); + let template = *templates.entry(key).or_insert(i); + ( + template, + refs.into_iter() + .map(|(_, _, target)| target) + .collect::>(), + ) + }) + .collect::>(); + drop(templates); + let mut partitions = shapes + .iter() + .map(|(template, _)| *template) + .collect::>(); + loop { + let mut groups = HashMap::default(); + let next = shapes + .iter() + .enumerate() + .map(|(i, (template, refs))| { + let key = ( + *template, + refs.iter() + .map(|&target| partitions[target]) + .collect::>(), + ); + *groups.entry(key).or_insert(i) + }) + .collect::>(); + if next == partitions { + return next; + } + partitions = next; + } +} + +fn references(recipe: &[u8], names: &HashMap<&str, usize>) -> Vec<(usize, usize, usize)> { + // Length prefixes separate field names from descriptors. Rewrite only proven class references. + if !recipe.starts_with(b"carrier-v2;") { + return Vec::new(); + } + let mut result = Vec::new(); + let mut start = 0; + while start < recipe.len() { + let end = recipe[start..] + .iter() + .position(|&b| b == b';') + .map_or(recipe.len(), |i| start + i); + let part = &recipe[start..end]; + if let Some(colon) = part.iter().position(|&b| b == b':') { + if let Some(length) = std::str::from_utf8(&part[..colon]) + .ok() + .and_then(|n| n.parse::().ok()) + { + if let Some(descriptor) = (colon + 1) + .checked_add(length) + .and_then(|offset| part.get(offset..)) + { + let scalar = descriptor + .iter() + .position(|&b| b != b'[') + .unwrap_or(descriptor.len()); + if let Some(name) = descriptor.get(scalar..).and_then(|d| d.strip_prefix(b"L")) + { + if let Some(&target) = + std::str::from_utf8(name).ok().and_then(|n| names.get(n)) + { + result.push((end - name.len(), end, target)); + } + } + } + } + } + start = end + 1; + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn nested_and_recursive_storage_shares_but_nominal_boundaries_do_not() { + let carriers: Vec<(&str, &[u8])> = vec![ + ("a/Leaf", b"carrier-v2;eq;scalar;1:xI;1:xI;"), + ("b/Leaf", b"carrier-v2;eq;scalar;1:xI;1:xI;"), + ( + "a/Outer", + b"carrier-v2;eq;object:Class;1:xLa/Leaf;;1:xLa/Leaf;;", + ), + ( + "b/Outer", + b"carrier-v2;eq;object:Class;1:xLb/Leaf;;1:xLb/Leaf;;", + ), + ( + "a/Cycle", + b"carrier-v2;eq;object:Class;1:xLa/Cycle;;1:xLa/Cycle;;", + ), + ( + "b/Cycle", + b"carrier-v2;eq;object:Class;1:xLb/Cycle;;1:xLb/Cycle;;", + ), + ( + "a/Public", + b"carrier-v2;eq;object:Class;1:xLpublic/A;;1:xLpublic/A;;", + ), + ( + "b/Public", + b"carrier-v2;eq;object:Class;1:xLpublic/B;;1:xLpublic/B;;", + ), + ("a/OtherField", b"carrier-v2;eq;scalar;1:yI;1:yI;"), + ( + "a/OtherDispatch", + b"carrier-v2;eq;object:Enum;1:xLb/Leaf;;1:xLb/Leaf;;", + ), + ]; + assert_eq!( + representatives(&carriers), + vec![0, 0, 2, 2, 4, 4, 6, 7, 8, 9] + ); + // A class identity difference must propagate through multiple containing layers. + let carriers: Vec<(&str, &[u8])> = vec![ + ("a/Outer", b"carrier-v2;plain;1:xLa/Inner;;"), + ("b/Outer", b"carrier-v2;plain;1:xLb/Inner;;"), + ("a/Inner", b"carrier-v2;plain;1:xLpublic/A;;"), + ("b/Inner", b"carrier-v2;plain;1:xLpublic/B;;"), + ]; + assert_eq!(representatives(&carriers), vec![0, 1, 2, 3]); + } + + #[test] + fn callable_shapes_follow_storage_aliases_and_preserve_argument_order() { + let carriers: Vec<(&str, &[u8])> = vec![ + ("a/Value", b"carrier-v2;plain;1:xI;"), + ("b/Value", b"carrier-v2;plain;1:xI;"), + ( + "a/Call", + b"carrier-v2;function-v1;param;0:La/Value;;return;0:J;", + ), + ( + "b/Call", + b"carrier-v2;function-v1;param;0:Lb/Value;;return;0:J;", + ), + ( + "a/Reverse", + b"carrier-v2;function-v1;param;0:J;return;0:La/Value;;", + ), + ( + "a/Nominal", + b"carrier-v2;function-v1;param;0:Lpublic/Value;;return;0:J;", + ), + ]; + assert_eq!(representatives(&carriers), vec![0, 0, 2, 2, 4, 5]); + } +} diff --git a/java-linker/src/carriers_tests.rs b/java-linker/src/carriers_tests.rs new file mode 100644 index 00000000..7e10f13f --- /dev/null +++ b/java-linker/src/carriers_tests.rs @@ -0,0 +1,294 @@ +use crate::test_support::{body, holder, run_jar, write_classes}; +use crate::*; +use jvm_compiler_core::classfile::{names, summary}; + +#[test] +fn shared_callable_interfaces_keep_the_runtime_protocol_name() { + let mut main = holder("test/Main", false); + let mut classes = Vec::new(); + let mut code = Vec::new(); + for suffix in ["first", "second"] { + let name = format!("org/rustlang/runtime/FnPtr_{suffix}"); + let mut class = holder(&name, true); + class.access_flags = + ClassAccessFlags::PUBLIC | ClassAccessFlags::INTERFACE | ClassAccessFlags::ABSTRACT; + class.attributes.push(Attribute::Unknown { + name_index: class + .constant_pool + .add_utf8(summary::CARRIER_ATTRIBUTE) + .unwrap(), + info: b"carrier-v2;function-v1;param;0:I;return;0:I;".to_vec(), + }); + class.methods.push(ristretto_classfile::Method { + access_flags: ristretto_classfile::MethodAccessFlags::PUBLIC + | ristretto_classfile::MethodAccessFlags::ABSTRACT, + name_index: class.constant_pool.add_utf8("call").unwrap(), + descriptor_index: class.constant_pool.add_utf8("(I)I").unwrap(), + attributes: Vec::new(), + }); + code.extend([ + Instruction::Ldc_w(main.constant_pool.add_class(&name).unwrap()), + Instruction::Pop, + ]); + classes.push(class); + } + code.push(Instruction::Return); + body(&mut main, "main", "([Ljava/lang/String;)V", code); + classes.push(main); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes(temp.path(), classes); + let output = temp.path().join("shared.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + assert_eq!( + jar.file_names() + .filter(|name| name.starts_with("org/rustlang/runtime/FnPtr_")) + .count(), + 1 + ); + let mut bytes = Vec::new(); + jar.by_name("test/Main.class") + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let main = class_file_from_data(&bytes).unwrap(); + let references = main + .constant_pool + .iter() + .filter_map(|constant| { + if let Constant::Class(index) = constant { + let name = main.constant_pool.try_get_utf8(*index).unwrap().to_string(); + name.starts_with("org/rustlang/runtime/FnPtr_") + .then_some(name) + } else { + None + } + }) + .collect::>(); + assert_eq!(references.len(), 1); +} + +#[test] +fn matching_private_carriers_share_identity_and_relocate_only_symbolic_names() { + for barrier in [ + "none", + "dead-eq", + "fragment", + "library", + "reflection", + "collision", + ] { + let mut main = holder("test/Main", false); + let mut a = holder("test/CarrierA", true); + let mut b = holder("test/CarrierB", true); + for carrier in [&mut a, &mut b] { + carrier.attributes.push(Attribute::Unknown { + name_index: carrier + .constant_pool + .add_utf8(summary::CARRIER_ATTRIBUTE) + .unwrap(), + info: b"test-empty-carrier-v1".to_vec(), + }); + carrier.fields.push(ristretto_classfile::Field { + access_flags: ristretto_classfile::FieldAccessFlags::PUBLIC, + name_index: carrier.constant_pool.add_utf8("payload").unwrap(), + descriptor_index: carrier.constant_pool.add_utf8("Ltest/FieldType;").unwrap(), + field_type: ristretto_classfile::FieldType::Object("test/FieldType".into()), + attributes: Vec::new(), + }); + let init = carrier + .constant_pool + .add_method_ref(carrier.super_class, "", "()V") + .unwrap(); + body( + carrier, + "", + "()V", + vec![ + Instruction::Aload_0, + Instruction::Invokespecial(init), + Instruction::Return, + ], + ); + carrier.methods[0].access_flags = ristretto_classfile::MethodAccessFlags::PUBLIC; + let name = carrier + .constant_pool + .try_get_class(carrier.this_class) + .unwrap() + .to_string(); + body( + carrier, + "eq", + &format!("(L{name};)Z"), + vec![Instruction::Iconst_1, Instruction::Ireturn], + ); + carrier.methods[1].access_flags = ristretto_classfile::MethodAccessFlags::PUBLIC + | ristretto_classfile::MethodAccessFlags::FINAL; + if let Attribute::Code { max_locals, .. } = &mut carrier.methods[1].attributes[0] { + *max_locals = 2; + } + } + let mut code = Vec::new(); + for name in ["test/CarrierA", "test/CarrierB"] { + let class = main.constant_pool.add_class(name).unwrap(); + let init = main + .constant_pool + .add_method_ref(class, "", "()V") + .unwrap(); + code.extend([ + Instruction::New(class), + Instruction::Dup, + Instruction::Invokespecial(init), + ]); + if barrier == "none" && name == "test/CarrierB" { + let eq = main + .constant_pool + .add_method_ref(class, "eq", "(Ltest/CarrierB;)Z") + .unwrap(); + code.extend([Instruction::Dup, Instruction::Invokevirtual(eq)]); + } + code.push(Instruction::Pop); + } + main.constant_pool.add_string("test/CarrierB").unwrap(); + main.constant_pool + .add_string(format!("{}@zero-sized:test/CarrierB", names::NAME_STRING)) + .unwrap(); + if barrier == "reflection" { + let class = main.constant_pool.add_class("java/lang/Class").unwrap(); + main.constant_pool + .add_method_ref(class, "forName", "(Ljava/lang/String;)Ljava/lang/Class;") + .unwrap(); + } + code.push(Instruction::Return); + body(&mut main, "main", "([Ljava/lang/String;)V", code); + let mut classes = vec![main, a, b, holder("test/FieldType", true)]; + if barrier == "fragment" { + classes.push(holder("test/CarrierB", true)); + } + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes(temp.path(), classes); + let mut libraries = Vec::new(); + if matches!(barrier, "library" | "collision") { + let mut external = holder( + if barrier == "collision" { + "org/rustlang/shape/S0" + } else { + "external/Api" + }, + false, + ); + // A descriptor-only reference must pin the carrier too. + body( + &mut external, + "accept", + if barrier == "collision" { + "()V" + } else { + "(Ltest/CarrierB;)V" + }, + vec![Instruction::Return], + ); + let path = temp.path().join("api.jar"); + let mut jar = zip::ZipWriter::new(fs::File::create(&path).unwrap()); + jar.start_file( + format!("{}.class", external.class_name().unwrap()), + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + use std::io::Write; + jar.write_all(&serialize_class_file(&external).unwrap()) + .unwrap(); + jar.finish().unwrap(); + libraries.push(path.to_string_lossy().into_owned()); + } + let output = temp.path().join("shared.jar"); + pipeline::link(&paths, &[], &[], &libraries, output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + let mut bytes = Vec::new(); + jar.by_name("test/Main.class") + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + let allocations = class.methods[0] + .attributes + .iter() + .find_map(|a| match a { + Attribute::Code { code, .. } => Some( + code.iter() + .filter_map(|i| match i { + Instruction::New(i) => { + Some(class.constant_pool.try_get_class(*i).unwrap().to_string()) + } + _ => None, + }) + .collect::>(), + ), + _ => None, + }) + .unwrap(); + assert_eq!(allocations.len(), 2); + assert_eq!( + allocations[0] == allocations[1], + matches!(barrier, "none" | "dead-eq" | "collision"), + "{barrier}" + ); + if barrier == "library" { + assert_eq!( + allocations[1], "test/CarrierB", + "external descriptor must pin identity" + ); + } + if barrier == "reflection" { + assert_eq!(allocations, ["test/CarrierA", "test/CarrierB"]); + } + let mut carrier_bytes = Vec::new(); + jar.by_name(&format!("{}.class", allocations[0])) + .unwrap() + .read_to_end(&mut carrier_bytes) + .unwrap(); + let carrier = class_file_from_data(&carrier_bytes).unwrap(); + let dependency = carrier + .constant_pool + .try_get_utf8(carrier.fields[0].descriptor_index) + .unwrap() + .to_string(); + let dependency = dependency + .strip_prefix('L') + .unwrap() + .strip_suffix(';') + .unwrap(); + assert!( + jar.by_name(&format!("{dependency}.class")).is_ok(), + "field descriptor lost its dependency" + ); + assert_eq!( + carrier.methods.iter().any(|m| carrier + .constant_pool + .try_get_utf8(m.name_index) + .unwrap() + == "eq"), + matches!(barrier, "none" | "reflection"), + "{barrier}" + ); + let strings = class + .constant_pool + .iter() + .filter_map(|constant| match constant { + Constant::String(index) => Some( + class + .constant_pool + .try_get_utf8(*index) + .unwrap() + .to_string(), + ), + _ => None, + }) + .collect::>(); + assert!(strings.contains(&"test/CarrierB".to_owned())); + let expected = format!("@zero-sized:{}", allocations[1]); + assert!(strings.contains(&expected), "{strings:?}"); + } +} diff --git a/java-linker/src/compact.rs b/java-linker/src/compact.rs new file mode 100644 index 00000000..b6824072 --- /dev/null +++ b/java-linker/src/compact.rs @@ -0,0 +1,121 @@ +//! Rebuild supported class metadata and the pool reachable from retained members. +use crate::*; +use ristretto_classfile::MethodAccessFlags; + +pub(crate) fn class( + source: &ClassFile<'static>, + methods: &[usize], + owner: &str, +) -> io::Result { + let mut pool = ConstantPool::default(); + let this_class = pool + .add_class(owner) + .map_err(|e| constant_pool_error("split class name", e))?; + let superclass = source + .constant_pool + .try_get_class(source.super_class) + .map_err(|e| constant_pool_error("compact superclass", e))?; + let super_class = pool + .add_class(superclass.to_string()) + .map_err(|e| constant_pool_error("split superclass", e))?; + let mut constants = constant_pool_index(&pool); + let mut bootstrap = Vec::new(); + let indexes = ConstantImporter::new(source, &mut pool, &mut constants, &mut bootstrap); + let renamed = source + .class_name() + .map_err(|e| constant_pool_error("compact owner", e))? + != owner; + if renamed && !source.fields.is_empty() { + return Err(io::Error::other( + "field-bearing classes cannot move to another method holder", + )); + } + let fields = source + .fields + .iter() + .map(|field| { + let mut field = field.clone(); + field.name_index = indexes.remap(field.name_index)?; + field.descriptor_index = indexes.remap(field.descriptor_index)?; + for attribute in &mut field.attributes { + remap_attribute(attribute, &indexes)?; + } + Ok(field) + }) + .collect::>>()?; + let interfaces = source + .interfaces + .iter() + .map(|&i| indexes.remap(i)) + .collect::>>()?; + let methods = methods + .iter() + .map(|&i| { + let mut method = source.methods[i].clone(); + method.name_index = indexes.remap(method.name_index)?; + method.descriptor_index = indexes.remap(method.descriptor_index)?; + // Only split holders need wider access for sibling pieces. + if renamed { + method + .access_flags + .remove(MethodAccessFlags::PRIVATE | MethodAccessFlags::PROTECTED); + method.access_flags.insert(MethodAccessFlags::PUBLIC); + } + for attribute in &mut method.attributes { + remap_attribute(attribute, &indexes)?; + } + Ok(method) + }) + .collect::>>()?; + let mut attributes = Vec::new(); + for attribute in &source.attributes { + if let Attribute::Unknown { name_index, info } = attribute { + // Compiler demand markers have no embedded constant indexes. + if info.is_empty() { + attributes.push(Attribute::Unknown { + name_index: indexes.remap(*name_index)?, + info: Vec::new(), + }); + } + } else if let Attribute::SourceFile { + name_index, + source_file_index, + } = attribute + { + attributes.push(Attribute::SourceFile { + name_index: indexes.remap(*name_index)?, + source_file_index: indexes.remap(*source_file_index)?, + }); + } else if matches!(attribute, Attribute::InnerClasses { .. }) { + let mut attribute = attribute.clone(); + remap_attribute(&mut attribute, &indexes)?; + attributes.push(attribute); + } + } + drop(indexes); + if !bootstrap.is_empty() { + let name_index = pool + .add_utf8("BootstrapMethods") + .map_err(|e| constant_pool_error("split bootstrap name", e))?; + attributes.push(Attribute::BootstrapMethods { + name_index, + methods: bootstrap, + }); + } + let class = ClassFile { + version: source.version.clone(), + constant_pool: pool, + access_flags: source.access_flags, + this_class, + super_class, + methods, + fields, + interfaces, + attributes, + ..Default::default() + }; + Ok(ClassInfo { + jar_entry_name: format!("{owner}.class"), + data: serialize_class_file(&class)?, + }) +} diff --git a/java-linker/src/enum_carriers_tests.rs b/java-linker/src/enum_carriers_tests.rs new file mode 100644 index 00000000..9f206997 --- /dev/null +++ b/java-linker/src/enum_carriers_tests.rs @@ -0,0 +1,114 @@ +use crate::test_support::run_jar; +use crate::*; +use jvm_compiler_core::classfile::summary; + +#[test] +fn enum_sharing_retains_live_helper_unions_and_distinct_payloads_and_tags() { + let temp = tempfile::tempdir().unwrap(); + for live_equality in [false, true] { + let source = r#" +public class EnumRun { + public interface A { + long $rust$tag(); + static long _unionDiscriminant(A a) { return a == null ? 0 : a.$rust$tag(); } + static boolean eq(A a, A b) { return ((AV)a).value == ((AV)b).value; } + } + public interface B { + long $rust$tag(); + static long _unionDiscriminant(B a) { return a == null ? 0 : a.$rust$tag(); } + static boolean eq(B a, B b) { return ((BV)a).value == ((BV)b).value; } + } + public interface C { + long $rust$tag(); + static long _unionDiscriminant(C a) { return a == null ? 0 : a.$rust$tag(); } + static boolean eq(C a, C b) { return ((CV)a).value == ((CV)b).value; } + } + public static class AV implements A { + public int value; + public AV(int v) { value = v; } + public long $rust$tag() { return 7; } + } + public static class BV implements B { + public long value; + public BV(long v) { value = v; } + public long $rust$tag() { return -11; } + } + public static class CV implements C { + public int value; + public CV(int v) { value = v; } + public long $rust$tag() { return 7; } + } + public static void main(String[] args) { + AV a = new AV(19); BV b = new BV(23); CV c = new CV(29); + if (a.value != 19 || b.value != 23 || c.value != 29) throw new AssertionError("payload"); + if (A._unionDiscriminant(a) != 7 || B._unionDiscriminant(b) != -11 + || C._unionDiscriminant(c) != 7 || A._unionDiscriminant(null) != 0) + throw new AssertionError("tag dispatch"); + /* equality */ + } +} +"#.replace("/* equality */", if live_equality { + "if (!C.eq(c, new CV(29)) || C.eq(c, new CV(31))) throw new AssertionError(\"equality\");" + } else { "" }); + fs::write(temp.path().join("EnumRun.java"), source).unwrap(); + let compile = std::process::Command::new("javac") + .args(["--release", "8", "EnumRun.java"]) + .current_dir(temp.path()) + .output() + .unwrap(); + assert!( + compile.status.success(), + "{}", + String::from_utf8_lossy(&compile.stderr) + ); + let mut paths = Vec::new(); + for suffix in ["", "$A", "$B", "$C", "$AV", "$BV", "$CV"] { + let name = format!("EnumRun{suffix}"); + let path = temp.path().join(format!("{name}.class")); + let mut class = class_file_from_data(&fs::read(&path).unwrap()).unwrap(); + if !suffix.is_empty() { + let (owner, variant, scalar, tag) = match suffix { + "$A" | "$AV" => ("EnumRun$A", "EnumRun$AV", "I", 7), + "$B" | "$BV" => ("EnumRun$B", "EnumRun$BV", "J", -11), + _ => ("EnumRun$C", "EnumRun$CV", "I", 7), + }; + let recipe = if name == owner { + format!( + "carrier-v2;enum-v1;method=$rust$tag;abstract-tag;method=_unionDiscriminant;dispatch-tag;0:L{owner};;method=eq;eq;0:L{owner};;0:L{variant};;" + ) + } else { + format!( + "carrier-v2;variant-v1;V0;{tag};0:L{owner};;carrier-v2;plain;scalar;5:value{scalar};5:value{scalar};" + ) + }; + for (attribute, info) in [ + (summary::PRIVATE_ATTRIBUTE, Vec::new()), + (summary::CARRIER_ATTRIBUTE, recipe.into_bytes()), + ] { + class.attributes.push(Attribute::Unknown { + name_index: class.constant_pool.add_utf8(attribute).unwrap(), + info, + }); + } + fs::write(&path, serialize_class_file(&class).unwrap()).unwrap(); + } + paths.push(path.to_string_lossy().into_owned()); + } + let output = temp.path().join("linked.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(output).unwrap()).unwrap(); + let mut interfaces = 0; + for i in 0..jar.len() { + let mut entry = jar.by_index(i).unwrap(); + if !entry.name().ends_with(".class") { + continue; + } + let mut data = Vec::new(); + entry.read_to_end(&mut data).unwrap(); + let class = class_file_from_data(&data).unwrap(); + interfaces += usize::from(class.access_flags.contains(ClassAccessFlags::INTERFACE)); + } + assert_eq!(interfaces, if live_equality { 2 } else { 1 }); + } +} diff --git a/java-linker/src/inputs.rs b/java-linker/src/inputs.rs index 9d12e7f6..c67c6bcf 100644 --- a/java-linker/src/inputs.rs +++ b/java-linker/src/inputs.rs @@ -1,6 +1,5 @@ use crate::*; -use jvm_compiler_core::classfile::summary; use std::io::SeekFrom; #[derive(Clone, Copy, Debug)] @@ -13,8 +12,11 @@ pub(crate) struct Fragment { #[derive(Debug)] pub(crate) struct Group { pub(crate) name: String, + pub(crate) resource: bool, + pub(crate) demand: u32, pub(crate) fragments: Vec, pub(crate) bytes: usize, + pub(crate) carrier: Option>, } /// The linker retains byte ranges, not all input archives and class bodies. @@ -25,6 +27,16 @@ pub(crate) struct Index { pub(crate) mains: HashSet, positions: HashMap, scratch: Vec, + demands: reachability::Graph, + pub(crate) dead_methods: HashMap>, + pub(crate) pruned_resources: usize, + pub(crate) packable: HashMap, + pub(crate) share_carriers: bool, + pub(crate) aliases: crate::aliases::Aliases, + pub(crate) compact_carriers: bool, + pub(crate) private_classes: HashSet, + pub(crate) occupied_packages: HashSet, + pub(crate) short_methods: HashMap, Vec>, } fn invalid(message: impl Into) -> io::Error { @@ -84,22 +96,47 @@ impl Index { name: Option, ) -> io::Result<()> { let len = usize::try_from(len).map_err(|_| invalid("class exceeds host address space"))?; - reader.seek(SeekFrom::Start(offset))?; - self.scratch.resize(len, 0); - reader.read_exact(&mut self.scratch)?; - let summary = summary::read(&self.scratch)?; - if summary.has_main { - self.mains.insert(summary.name.clone()); - } - let name = name.unwrap_or(summary.name) + ".class"; + let resource = name + .as_deref() + .and_then(|n| n.strip_prefix(jvm_compiler_core::classfile::resources::BUNDLE_PREFIX)); + let (name, resource, demand, carrier) = if let Some(resource) = resource { + if !jvm_compiler_core::classfile::resources::valid_name(resource) { + return Err(invalid("invalid binary constant resource name")); + } + ( + resource.to_owned(), + true, + self.demands.resource(resource), + None, + ) + } else { + reader.seek(SeekFrom::Start(offset))?; + self.scratch.resize(len, 0); + reader.read_exact(&mut self.scratch)?; + let (summary, demand) = self.demands.scan(&self.scratch, false)?; + if summary.has_main { + self.mains.insert(summary.name.clone()); + } + ( + name.unwrap_or(summary.name) + ".class", + false, + demand, + summary.carrier, + ) + }; let next = self.groups.len(); let &mut position = self.positions.entry(name.clone()).or_insert(next); if position == next { self.groups.push(Group { name, + resource, + demand, fragments: Vec::new(), bytes: 0, + carrier, }); + } else if self.groups[position].carrier != carrier { + self.groups[position].carrier = None; } let group = &mut self.groups[position]; group.bytes = group @@ -110,6 +147,37 @@ impl Index { Ok(()) } + pub(crate) fn retain_demanded(&mut self, libraries: &[String]) -> io::Result { + if self.mains.is_empty() { + self.demands = reachability::Graph::default(); + return Ok(0); + } + self.demands.libraries(libraries)?; + let live = std::mem::take(&mut self.demands).finish(true); + let before_classes = self.groups.iter().filter(|g| !g.resource).count(); + let before_resources = self.groups.len() - before_classes; + self.groups.retain(|g| live.classes[g.demand as usize]); + self.dead_methods = live.dead_methods; + self.aliases = live.aliases; + self.packable = live.packable; + self.share_carriers = live.share_carriers; + self.compact_carriers = live.compact_carriers; + self.private_classes = live.private_classes; + self.occupied_packages = live.occupied_packages; + self.short_methods = live.short_methods; + for group in &mut self.groups { + if live + .pinned_classes + .contains(group.name.trim_end_matches(".class")) + { + group.carrier = None; + } + } + let after_classes = self.groups.iter().filter(|g| !g.resource).count(); + self.pruned_resources = before_resources - (self.groups.len() - after_classes); + Ok(before_classes - after_classes) + } + fn bundle( &mut self, reader: &mut (impl Read + Seek), @@ -133,6 +201,7 @@ impl Index { let name = String::from_utf8(name).map_err(|e| invalid(e.to_string()))?; self.record(reader, file, data_start, byte_len, Some(name))?; offset = data_end; + reader.seek(SeekFrom::Start(offset))?; } Ok(()) } diff --git a/java-linker/src/jar.rs b/java-linker/src/jar.rs index 70d3f570..b0efc7c2 100644 --- a/java-linker/src/jar.rs +++ b/java-linker/src/jar.rs @@ -8,7 +8,7 @@ pub(crate) struct Writer { fn options() -> SimpleFileOptions { SimpleFileOptions::default() .compression_method(CompressionMethod::DEFLATE) - .compression_level(Some(1)) + .compression_level(Some(9)) .unix_permissions(0o644) } @@ -29,8 +29,10 @@ impl Batch { let mut bytes = 0; for class in classes { let class = class?; - count += 1; - bytes += class.data.len(); + if class.jar_entry_name.ends_with(".class") { + count += 1; + bytes += class.data.len(); + } zip.start_file(class.jar_entry_name, options())?; zip.write_all(&class.data)?; } diff --git a/java-linker/src/literal_tests.rs b/java-linker/src/literal_tests.rs new file mode 100644 index 00000000..6856390c --- /dev/null +++ b/java-linker/src/literal_tests.rs @@ -0,0 +1,84 @@ +use crate::test_support::{body, holder, run_jar, write_classes}; +use crate::*; +use jvm_compiler_core::classfile::constant_pool::InternedConstantPool; + +#[test] +fn literal_text_does_not_retain_methods_classes_or_resources() { + let mut main = holder("test/Main", false); + let mut pool = InternedConstantPool::default(); + main.this_class = pool.add_class("test/Main").unwrap(); + main.super_class = pool.add_class("java/lang/Object").unwrap(); + main.attributes.clear(); + let system = pool.add_class("java/lang/System").unwrap(); + let stdout = pool + .add_field_ref(system, "out", "Ljava/io/PrintStream;") + .unwrap(); + let stream = pool.add_class("java/io/PrintStream").unwrap(); + let println = pool + .add_method_ref(stream, "println", "(Ljava/lang/String;)V") + .unwrap(); + let mut code = Vec::new(); + for literal in [ + "eq", + "test.Ghost", + "META-INF/rust-data/0123456789abcdef.bin", + ] { + code.extend([ + Instruction::Getstatic(stdout), + Instruction::Ldc_w(pool.add_string(literal).unwrap()), + Instruction::Invokevirtual(println), + ]); + } + // Actual compiler reflection names still retain their target method. + let kept = pool.add_name_string("kept").unwrap(); + code.extend([ + Instruction::Ldc_w(pool.add_name_string("test/mono/Mono_needed").unwrap()), + Instruction::Pop, + Instruction::Ldc_w(kept), + Instruction::Pop, + Instruction::Return, + ]); + main.constant_pool = pool.into_inner(); + body(&mut main, "main", "([Ljava/lang/String;)V", code); + let mut unused = holder("test/mono/Mono_unused", true); + body(&mut unused, "eq", "()V", vec![Instruction::Return]); + let mut needed = holder("test/mono/Mono_needed", true); + body(&mut needed, "kept", "()V", vec![Instruction::Return]); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes( + temp.path(), + [main, unused, needed, holder("test/Ghost", true)], + ); + let output = temp.path().join("literals.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + let result = run_jar(&output); + assert_eq!( + String::from_utf8(result.stdout) + .unwrap() + .replace("\r\n", "\n"), + "eq\ntest.Ghost\nMETA-INF/rust-data/0123456789abcdef.bin\n" + ); + let mut jar = ZipArchive::new(fs::File::open(output).unwrap()).unwrap(); + let names = jar + .file_names() + .filter(|name| name.ends_with(".class")) + .map(str::to_owned) + .collect::>(); + assert_eq!( + names.len(), + 2, + "only main and the reflection target are live" + ); + for name in names { + let mut bytes = Vec::new(); + jar.by_name(&name).unwrap().read_to_end(&mut bytes).unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + assert!( + class.methods.iter().all(|method| class + .constant_pool + .try_get_utf8(method.name_index) + .unwrap() + != "eq") + ); + } +} diff --git a/java-linker/src/main.rs b/java-linker/src/main.rs index e6c7ba46..42140432 100644 --- a/java-linker/src/main.rs +++ b/java-linker/src/main.rs @@ -23,6 +23,9 @@ mod remap; use remap::*; mod merge; use merge::*; +mod method_names; +#[cfg(test)] +mod method_names_tests; mod metrics; use metrics::*; mod args; @@ -30,15 +33,36 @@ use args::*; mod jar; #[cfg(test)] use jar::*; +mod aliases; +#[cfg(test)] +mod aliases_tests; +mod carriers; +#[cfg(test)] +mod carriers_tests; +mod compact; +#[cfg(test)] +mod enum_carriers_tests; mod inputs; -mod pipeline; +#[cfg(test)] +mod literal_tests; mod namespaces; #[cfg(test)] mod namespaces_tests; +mod packing; +mod pipeline; +mod private_names; +#[cfg(test)] +mod private_names_tests; +mod reachability; +#[cfg(test)] +mod reachability_tests; mod split; +mod split_bridges; #[cfg(test)] mod split_tests; #[cfg(test)] +mod test_support; +#[cfg(test)] mod tests; #[derive(Debug)] diff --git a/java-linker/src/merge.rs b/java-linker/src/merge.rs index b5894459..58fc6df2 100644 --- a/java-linker/src/merge.rs +++ b/java-linker/src/merge.rs @@ -259,11 +259,38 @@ pub(crate) fn merge_group(fragments: Vec) -> io::Result { .unwrap()) } +#[cfg(test)] pub(crate) fn merge_group_with_relocations( fragments: Vec, relocations: Option<&std::sync::Mutex>, ) -> io::Result> { - if fragments.len() == 1 { + merge_group_with_demands(fragments, relocations, None) +} + +fn prune_methods( + class: &mut ClassFile<'static>, + dead: &HashSet<(JavaString, JavaString)>, +) -> io::Result { + let keep = (0..class.methods.len()) + .map(|i| Ok(!dead.contains(&method_identity(class, i)?))) + .collect::>>()?; + let changed = keep.iter().any(|&keep| !keep); + if changed { + class.methods = std::mem::take(&mut class.methods) + .into_iter() + .zip(keep) + .filter_map(|(method, keep)| keep.then_some(method)) + .collect(); + } + Ok(changed) +} + +pub(crate) fn merge_group_with_demands( + fragments: Vec, + relocations: Option<&std::sync::Mutex>, + dead: Option<&HashSet<(JavaString, JavaString)>>, +) -> io::Result> { + if fragments.len() == 1 && dead.is_none() { return Ok(fragments); } @@ -288,17 +315,18 @@ pub(crate) fn merge_group_with_relocations( unique_fragments.push(fragment); content_hashes.entry(hash).or_default().push(index); } - if unique_fragments.len() == 1 { + if unique_fragments.len() == 1 && dead.is_none() { return Ok(unique_fragments); } // Parse every surviving fragment once. Previously the reorder // check parsed all fragments and the merge loop parsed them all a // second time. + let mut pruned = false; let mut parsed = unique_fragments .into_iter() .map(|fragment| { - let class_file = class_file_from_data(&fragment.data).map_err(|error| { + let mut class_file = class_file_from_data(&fragment.data).map_err(|error| { io::Error::new( error.kind(), format!( @@ -307,6 +335,9 @@ pub(crate) fn merge_group_with_relocations( ), ) })?; + if let Some(dead) = dead { + pruned |= prune_methods(&mut class_file, dead)?; + } Ok((fragment, class_file)) }) .collect::>>()?; @@ -329,7 +360,10 @@ pub(crate) fn merge_group_with_relocations( && split::eligible(&base) && parsed.iter().all(|(_, c)| split::eligible(c)) { - let original = class_file_from_data(&merged.data)?; + let mut original = class_file_from_data(&merged.data)?; + if let Some(dead) = dead { + prune_methods(&mut original, dead)?; + } return split::holders( std::iter::once(&original).chain(parsed.iter().map(|(_, c)| c)), &mut relocations.lock().unwrap(), @@ -345,6 +379,17 @@ pub(crate) fn merge_group_with_relocations( } } } + if pruned { + let owner = base + .class_name() + .map_err(|e| constant_pool_error("pruned owner", e))? + .to_string(); + return Ok(vec![split::compact( + &base, + &(0..base.methods.len()).collect::>(), + &owner, + )?]); + } if changed { merged.data = serialize_class_file(&base).map_err(|error| { io::Error::new( diff --git a/java-linker/src/method_names.rs b/java-linker/src/method_names.rs new file mode 100644 index 00000000..eccdcc7e --- /dev/null +++ b/java-linker/src/method_names.rs @@ -0,0 +1,94 @@ +//! Assign one short name per private method across shared constants and packed classes. +use crate::*; +use std::sync::Arc; + +pub(crate) fn mangled(name: &str) -> bool { + name.is_ascii() + && !name.bytes().any(|b| b.is_ascii_control()) + && !name.contains(['/', '.', '(', ')', ';', ':', '#', '[', ' ']) + && name.rsplit_once('$').is_some_and(|(_, suffix)| { + suffix.len() == 16 && suffix.bytes().all(|b| b.is_ascii_hexdigit()) + }) +} + +pub(crate) fn plan( + methods: impl Iterator, + spellings: &[Arc], + fixed: &HashSet, + reflective: &HashMap, + classes: impl Iterator>, +) -> HashMap, Vec> { + let mut candidates = HashSet::default(); + let mut pinned = fixed.clone(); + for (name, live, private) in methods { + if !private { + pinned.insert(name); + } else if live { + candidates.insert(name); + } + } + let mut names = candidates + .into_iter() + .filter(|name| !pinned.contains(name) && !reflective.contains_key(name)) + .map(|name| spellings[name as usize].as_ref()) + .filter(|name| mangled(name)) + .collect::>(); + // A default-package class can share a UTF8 entry with a member name. + for class in classes { + names.remove(class.as_ref()); + } + let mut names = names.into_iter().collect::>(); + names.sort_unstable(); + let mut prefix = "$m".to_owned(); + while spellings.iter().any(|s| s.contains(&prefix)) { + prefix.push('$'); + } + names + .into_iter() + .enumerate() + .map(|(i, name)| { + // Keep source function names readable in stack traces. + let readable = name.rsplit_once('$').unwrap().0; + ( + name.as_bytes().to_vec(), + format!("{readable}{prefix}{i:x}").into_bytes(), + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn nominal_unresolved_and_reflective_names_prevent_global_shortening() { + let names = [ + "local$0123456789abcdef", + "external$0123456789abcdef", + "reflective$0123456789abcdef", + "field$0123456789abcdef", + "class$0123456789abcdef", + "unicode😀$0123456789abcdef", + ] + .map(Arc::::from); + let methods = [ + (0, true, true), + (1, true, true), + (1, false, false), + (2, true, true), + (3, true, true), + (4, true, true), + (5, true, true), + ]; + let renamed = plan( + methods.into_iter(), + &names, + &HashSet::from_iter([3]), + &HashMap::from_iter([(2, 0)]), + [names[4].as_ref()].into_iter(), + ); + assert_eq!(renamed.len(), 1); + assert_eq!(renamed[names[0].as_bytes()], b"local$m0"); + } +} diff --git a/java-linker/src/method_names_tests.rs b/java-linker/src/method_names_tests.rs new file mode 100644 index 00000000..75bb39ea --- /dev/null +++ b/java-linker/src/method_names_tests.rs @@ -0,0 +1,184 @@ +use crate::test_support::{body, holder, run_jar, write_classes}; +use crate::*; +use jvm_compiler_core::classfile::names; + +#[test] +fn shortened_methods_preserve_handles_overloads_literals_and_nominal_names() { + let temp = tempfile::tempdir().unwrap(); + let value = "value$0123456789abcdef"; + let reflected = "reflected$0123456789abcdef"; + let identity = "identity$0123456789abcdef"; + let field = "field$0123456789abcdef"; + let published = "published$0123456789abcdef"; + let owner = "test/mono/Mono_names"; + fs::write(temp.path().join("NameLookup.java"), format!(r#" +package org.rustlang.runtime; +public class NameLookup {{ + public static int {field}; + public static void {published}() {{}} + public static void check(Class owner, String name) throws Exception {{ + if (!Integer.valueOf(3).equals(owner.getMethod(name).invoke(null))) throw new AssertionError(name); + }} + public static void identity(String name) throws Exception {{ + String[] parts = name.split("::", 2); + check(Class.forName(parts[0].replace('/', '.')), parts[1].split(":", 2)[0]); + }} +}} +"#)).unwrap(); + let javac = std::process::Command::new("javac") + .args(["--release", "8", "-d", ".", "NameLookup.java"]) + .current_dir(temp.path()) + .output() + .unwrap(); + assert!( + javac.status.success(), + "{}", + String::from_utf8_lossy(&javac.stderr) + ); + let mut helper = holder(owner, true); + for name in [value, reflected, identity, field, published] { + body( + &mut helper, + name, + "()I", + vec![Instruction::Iconst_3, Instruction::Ireturn], + ); + } + body( + &mut helper, + value, + "(I)I", + vec![Instruction::Iload_0, Instruction::Ireturn], + ); + // Reserve the default short-name prefix, including a method that dies. + body(&mut helper, "$m0", "()V", vec![Instruction::Return]); + let pool = &mut helper.constant_pool; + let system = pool.add_class("java/lang/System").unwrap(); + let out = pool + .add_field_ref(system, "out", "Ljava/io/PrintStream;") + .unwrap(); + let stream = pool.add_class("java/io/PrintStream").unwrap(); + let print = pool.add_method_ref(stream, "println", "(I)V").unwrap(); + let print_string = pool + .add_method_ref(stream, "println", "(Ljava/lang/String;)V") + .unwrap(); + let mut code = Vec::new(); + for name in [value, field, published] { + let call = pool.add_method_ref(helper.this_class, name, "()I").unwrap(); + code.extend([ + Instruction::Getstatic(out), + Instruction::Invokestatic(call), + Instruction::Invokevirtual(print), + ]); + } + let target = pool + .add_method_ref(helper.this_class, value, "(I)I") + .unwrap(); + let handle = pool + .add_method_handle(ristretto_classfile::ReferenceKind::InvokeStatic, target) + .unwrap(); + let mh = pool.add_class("java/lang/invoke/MethodHandle").unwrap(); + let invoke = pool.add_method_ref(mh, "invokeExact", "(I)I").unwrap(); + code.extend([ + Instruction::Getstatic(out), + Instruction::Ldc_w(handle), + Instruction::Iconst_5, + Instruction::Invokevirtual(invoke), + Instruction::Invokevirtual(print), + ]); + let literal = pool + .add_string(format!("{}{value}", names::LITERAL_STRING)) + .unwrap(); + code.extend([ + Instruction::Getstatic(out), + Instruction::Ldc_w(literal), + Instruction::Invokevirtual(print_string), + ]); + let lookup = pool.add_class("org/rustlang/runtime/NameLookup").unwrap(); + let check = pool + .add_method_ref(lookup, "check", "(Ljava/lang/Class;Ljava/lang/String;)V") + .unwrap(); + let reflected_string = pool.add_string(reflected).unwrap(); + code.extend([ + Instruction::Ldc_w(helper.this_class), + Instruction::Ldc_w(reflected_string), + Instruction::Invokestatic(check), + ]); + let check_identity = pool + .add_method_ref(lookup, "identity", "(Ljava/lang/String;)V") + .unwrap(); + let identity_string = pool.add_string(format!("{owner}::{identity}:()I")).unwrap(); + code.extend([ + Instruction::Ldc_w(identity_string), + Instruction::Invokestatic(check_identity), + Instruction::Return, + ]); + body(&mut helper, "run", "()V", code); + if let Attribute::Code { max_stack, .. } = helper + .methods + .last_mut() + .unwrap() + .attributes + .last_mut() + .unwrap() + { + *max_stack = 3; + } + let mut main = holder("test/Main", false); + let class = main.constant_pool.add_class(owner).unwrap(); + let run = main + .constant_pool + .add_method_ref(class, "run", "()V") + .unwrap(); + body( + &mut main, + "main", + "([Ljava/lang/String;)V", + vec![Instruction::Invokestatic(run), Instruction::Return], + ); + let mut paths = vec![ + temp.path() + .join("org/rustlang/runtime/NameLookup.class") + .to_string_lossy() + .into_owned(), + ]; + paths.extend(write_classes(temp.path(), [main, helper])); + let output = temp.path().join("names.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + let class_name = jar + .file_names() + .find(|name| name.contains("/mono/")) + .unwrap() + .to_owned(); + let mut bytes = Vec::new(); + jar.by_name(&class_name) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let linked = class_file_from_data(&bytes).unwrap(); + let methods = linked + .methods + .iter() + .map(|m| { + linked + .constant_pool + .try_get_utf8(m.name_index) + .unwrap() + .to_string() + }) + .collect::>(); + assert!(!methods.iter().any(|m| m == value), "{methods:?}"); + assert_eq!( + methods.iter().filter(|m| m.starts_with("value$m$")).count(), + 2 + ); + for name in [reflected, identity, field, published] { + assert!(methods.iter().any(|m| m == name), "{name}: {methods:?}"); + } + let run = run_jar(&output); + assert_eq!( + String::from_utf8(run.stdout).unwrap().replace("\r\n", "\n"), + format!("3\n3\n3\n5\n{value}\n") + ); +} diff --git a/java-linker/src/metrics.rs b/java-linker/src/metrics.rs index f2a49055..e58f590e 100644 --- a/java-linker/src/metrics.rs +++ b/java-linker/src/metrics.rs @@ -34,6 +34,12 @@ pub(crate) struct LinkerMetrics { pub(crate) duplicate_fragments: usize, pub(crate) top_duplicate_classes: Vec, pub(crate) merged_classes: usize, + pub(crate) pruned_classes: usize, + pub(crate) pruned_methods: usize, + pub(crate) forwarding_methods: usize, + pub(crate) pruned_resources: usize, + pub(crate) packed_owners: usize, + pub(crate) shared_carriers: usize, pub(crate) merged_class_bytes: usize, pub(crate) library_jars: usize, pub(crate) library_jar_bytes: u64, @@ -55,6 +61,12 @@ impl LinkerMetrics { duplicate_fragments: 0, top_duplicate_classes: Vec::new(), merged_classes: 0, + pruned_classes: 0, + pruned_methods: 0, + forwarding_methods: 0, + pruned_resources: 0, + packed_owners: 0, + shared_carriers: 0, merged_class_bytes: 0, library_jars: input_jar_files.len(), library_jar_bytes: input_jar_files @@ -88,7 +100,7 @@ impl LinkerMetrics { } pub(crate) fn from_index(index: &inputs::Index, libraries: &[String]) -> Self { - let groups = &index.groups; + let groups: Vec<_> = index.groups.iter().filter(|g| !g.resource).collect(); let mut duplicates = groups .iter() .filter(|g| g.fragments.len() > 1) @@ -108,6 +120,12 @@ impl LinkerMetrics { duplicate_fragments: groups.iter().map(|g| g.fragments.len() - 1).sum(), top_duplicate_classes: duplicates, merged_classes: 0, + pruned_classes: 0, + pruned_methods: 0, + forwarding_methods: 0, + pruned_resources: 0, + packed_owners: 0, + shared_carriers: 0, merged_class_bytes: 0, library_jars: libraries.len(), library_jar_bytes: libraries @@ -131,7 +149,7 @@ impl LinkerMetrics { let path = directory.join(format!("{}-linker.json", std::process::id())); let mut writer = BufWriter::new(fs::File::create(&path)?); writeln!(writer, "{{")?; - writeln!(writer, " \"schema_version\": 2,")?; + writeln!(writer, " \"schema_version\": 4,")?; writeln!(writer, " \"kind\": \"linker_work_metrics\",")?; writeln!(writer, " \"pid\": {},", std::process::id())?; writeln!(writer, " \"output_jar\": {},", json_string(output_jar))?; @@ -157,6 +175,16 @@ impl LinkerMetrics { self.duplicate_fragments )?; writeln!(writer, " \"merged_classes\": {},", self.merged_classes)?; + writeln!(writer, " \"pruned_classes\": {},", self.pruned_classes)?; + writeln!(writer, " \"pruned_methods\": {},", self.pruned_methods)?; + writeln!( + writer, + " \"forwarding_methods\": {},", + self.forwarding_methods + )?; + writeln!(writer, " \"pruned_resources\": {},", self.pruned_resources)?; + writeln!(writer, " \"packed_owners\": {},", self.packed_owners)?; + writeln!(writer, " \"shared_carriers\": {},", self.shared_carriers)?; writeln!( writer, " \"merged_class_bytes\": {},", diff --git a/java-linker/src/namespaces.rs b/java-linker/src/namespaces.rs index dda41f15..88f8f230 100644 --- a/java-linker/src/namespaces.rs +++ b/java-linker/src/namespaces.rs @@ -9,6 +9,10 @@ use ristretto_classfile::byte_reader::ByteReader; #[derive(Default)] pub(crate) struct Namespaces { removable: HashSet>, + packed: HashMap, + pub(crate) strip_proofs: bool, + pub(crate) aliases: crate::aliases::Aliases, + pub(crate) short_methods: HashMap, Vec>, } impl Namespaces { @@ -23,6 +27,10 @@ impl Namespaces { roots.entry(plain).or_default().insert(marker); } Self { + packed: HashMap::default(), + strip_proofs: false, + aliases: crate::aliases::Aliases::default(), + short_methods: HashMap::default(), removable: roots .into_values() .filter(|identities| identities.len() == 1) @@ -33,7 +41,67 @@ impl Namespaces { } } - fn rewrite(&self, bytes: &[u8]) -> Option> { + pub(crate) fn pack(&mut self, names: HashMap) { + self.packed = names; + } + + fn packed(&self, bytes: &[u8]) -> Option> { + if self.packed.is_empty() { + return None; + } + let mut output = Vec::new(); + let mut copied = 0; + let mut start = 0; + for end in 0..=bytes.len() { + if end < bytes.len() + && !matches!(bytes[end], b'(' | b')' | b';' | b'[' | b'#' | b':' | b'\n') + { + continue; + } + let token = &bytes[start..end]; + let text = std::str::from_utf8(token).ok(); + if let Some(text) = text { + let binary = text.contains('.'); + let normalized = if binary { + text.replace('.', "/") + } else { + text.into() + }; + let mut prefix = 0; + let mut target = self.packed.get(&normalized); + if target.is_none() { + let descriptor = normalized + .trim_start_matches(['B', 'C', 'D', 'F', 'I', 'J', 'S', 'Z', 'V']); + if let Some(name) = descriptor.strip_prefix('L') { + prefix = normalized.len() - name.len(); + target = self.packed.get(name); + } + } + if let Some(target) = target { + output.extend_from_slice(&bytes[copied..start + prefix]); + if binary { + output.extend_from_slice(target.replace('/', ".").as_bytes()); + } else { + output.extend_from_slice(target.as_bytes()); + } + copied = end; + } + } + start = end + 1; + } + if copied == 0 { + return None; + } + output.extend_from_slice(&bytes[copied..]); + Some(output) + } + + fn rewrite(&self, original: &[u8]) -> Option> { + if let Some(short) = self.short_methods.get(original) { + return Some(short.clone()); + } + let packed = self.packed(original); + let bytes = packed.as_deref().unwrap_or(original); let mut result = Vec::new(); let mut copied = 0; let mut scanned = 0; @@ -55,7 +123,7 @@ impl Namespaces { } } if copied == 0 { - return None; + return packed; } result.extend_from_slice(&bytes[copied..]); Some(result) @@ -73,7 +141,13 @@ impl Namespaces { reader.set_position(8); // magic and classfile version let mut pool = ConstantPool::from_bytes(&mut reader).map_err(error)?; let end = reader.position(); - let names = name_constants(&pool, &mut reader).map_err(error)?; + let mut changed = crate::aliases::redirect(&mut pool, &self.aliases)?; + let (names, class_attributes) = name_constants(&pool, &mut reader).map_err(error)?; + let tail = if self.strip_proofs { + strip_proofs(&pool, &class.data[class_attributes..]).map_err(error)? + } else { + None + }; let count = pool.len() as u16; let mut literals = HashMap::default(); for index in 1..=count { @@ -81,7 +155,6 @@ impl Namespaces { literals.entry(*value).or_insert_with(Vec::new).push(index); } } - let mut changed = false; for index in 1..=count { let Some(Constant::Utf8(value)) = pool.get(index) else { continue; @@ -95,6 +168,15 @@ impl Namespaces { .as_bytes() .strip_prefix(LITERAL_STRING.as_bytes()) .map(<[u8]>::to_vec) + .or_else(|| { + // Older static-owner recipes were untagged. + value + .as_bytes() + .windows(6) + .any(|s| s == b"/mono/" || s == b".mono.") + .then(|| self.packed(value.as_bytes())) + .flatten() + }) }; if let Some(bytes) = relocated { let value = JavaString::from_mutf8(bytes).map_err(error)?; @@ -124,15 +206,17 @@ impl Namespaces { .map_err(error)?; changed = true; } - if changed { + if changed || tail.is_some() { // Method bodies, stack maps and their constant indexes stay intact; // there is no second instruction decode/encode or whole-JAR pass. let mut bytes = Vec::with_capacity(class.data.len()); bytes.extend_from_slice(&class.data[..8]); pool.to_bytes(&mut bytes).map_err(error)?; - bytes.extend_from_slice(&class.data[end..]); + bytes.extend_from_slice(&class.data[end..class_attributes]); + bytes.extend_from_slice(tail.as_deref().unwrap_or(&class.data[class_attributes..])); class.data = bytes; - class.jar_entry_name = self.name(&class.jar_entry_name); + let name = class.jar_entry_name.trim_end_matches(".class"); + class.jar_entry_name = format!("{}.class", self.name(name)); } Ok(class) } @@ -164,7 +248,7 @@ impl Namespaces { fn name_constants( pool: &ConstantPool<'_>, reader: &mut ByteReader<'_>, -) -> ristretto_classfile::Result> { +) -> ristretto_classfile::Result<(HashSet, usize)> { let mut names = HashSet::default(); for constant in pool.iter() { match constant { @@ -193,8 +277,41 @@ fn name_constants( name_attributes(pool, reader, &mut names)?; } } + let attributes = reader.position(); name_attributes(pool, reader, &mut names)?; - Ok(names) + Ok((names, attributes)) +} + +/// Keep proof attributes for later library linking. Remove them from the final executable. +fn strip_proofs( + pool: &ConstantPool<'_>, + bytes: &[u8], +) -> ristretto_classfile::Result>> { + let mut reader = ByteReader::new(bytes); + let count = reader.read_u16()?; + let mut retained = count; + let mut output = Vec::new(); + let mut copied = 2; + for _ in 0..count { + let start = reader.position(); + let name = pool.try_get_utf8(reader.read_u16()?)?; + let length = reader.read_u32()? as usize; + reader.skip(length)?; + if matches!(name.as_bytes(), b"RustJvmPrivate" | b"RustJvmCarrier") { + if output.is_empty() { + output.extend_from_slice(&[0, 0]); + } + output.extend_from_slice(&bytes[copied..start]); + copied = reader.position(); + retained -= 1; + } + } + if retained == count { + return Ok(None); + } + output.extend_from_slice(&bytes[copied..]); + output[..2].copy_from_slice(&retained.to_be_bytes()); + Ok(Some(output)) } fn name_attributes( diff --git a/java-linker/src/namespaces_tests.rs b/java-linker/src/namespaces_tests.rs index e5ea8fd9..565938fb 100644 --- a/java-linker/src/namespaces_tests.rs +++ b/java-linker/src/namespaces_tests.rs @@ -32,7 +32,7 @@ fn names_and_reflection_strings_relocate_without_changing_literals_or_code() { let mut pool = InternedConstantPool::default(); let this_class = pool.add_class(&owner).unwrap(); let super_class = pool.add_class("java/lang/Object").unwrap(); - let literal = pool.add_string(&owner).unwrap(); // shares UTF8 with CONSTANT_Class + let literal = pool.add_string(&owner).unwrap(); let reflection = pool.add_name_string(&owner).unwrap(); let escaped_value = format!("{NAME_STRING}{owner}"); let escaped = pool.add_string(&escaped_value).unwrap(); @@ -124,3 +124,44 @@ fn crate_markers_are_not_nesting_separators() { assert_eq!(separators, [name.len() - "$Inner".len()]); assert_eq!(inner_name(&name), "Inner"); } + +#[test] +fn executable_proofs_are_removed_without_touching_other_metadata_or_library_proofs() { + let mut class = ClassFile::default(); + class.this_class = class.constant_pool.add_class("test/Storage").unwrap(); + class.super_class = class.constant_pool.add_class("java/lang/Object").unwrap(); + for (name, info) in [ + ("RustJvmPrivate", vec![]), + ("Unrelated", vec![4, 5]), + ("RustJvmCarrier", vec![1, 2, 3]), + ] { + class.attributes.push(Attribute::Unknown { + name_index: class.constant_pool.add_utf8(name).unwrap(), + info, + }); + } + let source = Attribute::SourceFile { + name_index: class.constant_pool.add_utf8("SourceFile").unwrap(), + source_file_index: class.constant_pool.add_utf8("storage.rs").unwrap(), + }; + class.attributes.push(source.clone()); + for strip_proofs in [false, true] { + let mut names = Namespaces::default(); + names.strip_proofs = strip_proofs; + let result = names + .class(ClassInfo { + jar_entry_name: "test/Storage.class".into(), + data: serialize_class_file(&class).unwrap(), + }) + .unwrap(); + let result = class_file_from_data(&result.data).unwrap(); + assert_eq!( + result.attributes, + if strip_proofs { + vec![class.attributes[1].clone(), source.clone()] + } else { + class.attributes.clone() + } + ); + } +} diff --git a/java-linker/src/packing.rs b/java-linker/src/packing.rs new file mode 100644 index 00000000..ac51d59e --- /dev/null +++ b/java-linker/src/packing.rs @@ -0,0 +1,212 @@ +//! Pack private stateless classes within fixed limits. Prefer classes that call each other. +use crate::*; + +const MAX_BYTES: usize = 2 * 1024 * 1024; +const MAX_METHODS: usize = 2048; +const MAX_OWNERS: usize = 256; + +#[derive(Default)] +pub(crate) struct Candidate { + pub(crate) methods: Vec<(u32, u32)>, + pub(crate) neighbors: Vec, + pub(crate) bytes: usize, +} + +pub(crate) struct Unit { + pub(crate) groups: Vec, + /// Input bytes include duplicate and dead fragments. Output size excludes them. + pub(crate) bytes: usize, +} + +#[derive(Default)] +pub(crate) struct Plan { + pub(crate) units: Vec, + pub(crate) names: HashMap, + pub(crate) shared_carriers: usize, +} + +fn owner(group: &inputs::Group) -> &str { + group.name.trim_end_matches(".class") +} + +fn package(name: &str) -> &str { + name.rsplit_once('/').map_or("", |(p, _)| p) +} + +impl Plan { + pub(crate) fn build(index: &mut inputs::Index) -> Self { + let candidates = std::mem::take(&mut index.packable); + let positions = index + .groups + .iter() + .enumerate() + .map(|(i, group)| (owner(group), i)) + .collect::>(); + let mut order = (0..index.groups.len()).collect::>(); + order.sort_unstable_by(|&a, &b| index.groups[a].name.cmp(&index.groups[b].name)); + let mut seen = vec![false; index.groups.len()]; + let mut plan = Self::default(); + if index.share_carriers { + let carriers = order + .iter() + .copied() + .filter(|&i| index.groups[i].carrier.is_some()) + .collect::>(); + let recipes = carriers + .iter() + .map(|&i| { + let group = &index.groups[i]; + (owner(group), group.carrier.as_deref().unwrap()) + }) + .collect::>(); + let representatives = + crate::carriers::live_representatives(&recipes, &index.dead_methods); + for (i, &position) in carriers.iter().enumerate() { + let group = &index.groups[position]; + let name = owner(group); + let representative = owner(&index.groups[carriers[representatives[i]]]); + if representative != name { + plan.names.insert(name.into(), representative.into()); + // Keep each alias's live helpers. Retain all overloads because aliased field types can change descriptors. + let dead = index + .dead_methods + .get(name) + .map(|methods| { + methods + .iter() + .map(|(name, _)| name.clone()) + .collect::>() + }) + .unwrap_or_default(); + if let Some(methods) = index.dead_methods.get_mut(representative) { + methods.retain(|(method, _)| dead.contains(method)); + } + plan.shared_carriers += 1; + seen[position] = true; + } + } + if index.compact_carriers { + let short = carriers + .iter() + .enumerate() + .filter(|(i, _)| representatives[*i] == *i) + .enumerate() + .map(|(i, (_, &position))| { + ( + owner(&index.groups[position]), + // Pointer and FunctionPointers identify callable carriers by this prefix. + if owner(&index.groups[position]) + .starts_with("org/rustlang/runtime/FnPtr_") + { + owner(&index.groups[position]).to_owned() + } else { + format!("org/rustlang/shape/S{i:x}") + }, + ) + }) + .collect::>(); + for &position in &carriers { + let name = owner(&index.groups[position]); + let representative = plan.names.get(name).map_or(name, String::as_str); + plan.names + .insert(name.into(), short[representative].clone()); + } + } + } + let mut targets = HashSet::default(); + for (position, &start) in order.iter().enumerate() { + if seen[start] { + continue; + } + seen[start] = true; + let source = &index.groups[start]; + let name = owner(source); + let mut unit = Unit { + groups: vec![start], + bytes: source.bytes, + }; + let Some(candidate) = candidates + .get(name) + .filter(|c| c.bytes < MAX_BYTES && c.methods.len() < MAX_METHODS) + else { + plan.units.push(unit); + continue; + }; + let mut methods = candidate.methods.iter().copied().collect::>(); + let mut emitted_bytes = candidate.bytes; + let mut pending = candidate + .neighbors + .iter() + .filter_map(|name| positions.get(name.as_str()).copied()) + .collect::>(); + // Limit each search to nearby classes in the same package. + pending.extend(order.iter().skip(position + 1).take(MAX_OWNERS).copied()); + for next in pending { + if unit.groups.len() == MAX_OWNERS { + break; + } + if seen[next] { + continue; + } + let group = &index.groups[next]; + let target = owner(group); + let Some(candidate) = candidates.get(target) else { + continue; + }; + if package(name) != package(target) + || emitted_bytes.saturating_add(candidate.bytes) > MAX_BYTES + || methods.len() + candidate.methods.len() > MAX_METHODS + || candidate.methods.iter().any(|m| methods.contains(m)) + { + continue; + } + methods.extend(candidate.methods.iter().copied()); + unit.groups.push(next); + unit.bytes += group.bytes; + emitted_bytes += candidate.bytes; + seen[next] = true; + } + if unit.groups.len() > 1 { + unit.groups + .sort_unstable_by(|&a, &b| index.groups[a].name.cmp(&index.groups[b].name)); + let mut hash = 0xcbf29ce484222325u64; + for &i in &unit.groups { + for byte in owner(&index.groups[i]).bytes().chain([0]) { + hash = (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3); + } + } + let target = if jvm_compiler_core::classfile::names::codec_owner(name) { + format!("{}/Codecs_pack_{hash:016x}", package(name)) + } else { + format!("{}/MonoBucket_{hash:016x}", package(name)) + }; + if positions.contains_key(target.as_str()) || !targets.insert(target.clone()) { + return Self::unpacked(index); + } + for &i in &unit.groups { + plan.names + .insert(owner(&index.groups[i]).into(), target.clone()); + } + } + plan.units.push(unit); + } + crate::private_names::compact(index, &mut plan.names); + plan + } + + fn unpacked(index: &inputs::Index) -> Self { + Self { + units: index + .groups + .iter() + .enumerate() + .map(|(i, g)| Unit { + groups: vec![i], + bytes: g.bytes, + }) + .collect(), + names: HashMap::default(), + shared_carriers: 0, + } + } +} diff --git a/java-linker/src/pipeline.rs b/java-linker/src/pipeline.rs index 6c9ad785..cbb0e586 100644 --- a/java-linker/src/pipeline.rs +++ b/java-linker/src/pipeline.rs @@ -12,7 +12,7 @@ pub(crate) fn link( libraries: &[String], output: &str, ) -> io::Result<()> { - let index = Index::collect(classes, bundles, archives)?; + let mut index = Index::collect(classes, bundles, archives)?; if index.groups.is_empty() && libraries.is_empty() { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -29,7 +29,38 @@ pub(crate) fn link( } let mut metrics = LinkerMetrics::enabled().then(|| LinkerMetrics::from_index(&index, libraries)); - let namespaces = namespaces::Namespaces::collect(index.groups.iter().map(|g| g.name.as_str())); + let mut namespaces = + namespaces::Namespaces::collect(index.groups.iter().map(|g| g.name.as_str())); + let pruned = index.retain_demanded(libraries)?; + if let Some(metrics) = &mut metrics { + metrics.pruned_classes = pruned; + metrics.forwarding_methods = index.aliases.values().map(HashMap::len).sum(); + metrics.pruned_resources = index.pruned_resources; + } + let plan = packing::Plan::build(&mut index); + // Release proof recipes before loading class bodies. + for group in &mut index.groups { + group.carrier = None; + } + if let Some(metrics) = &mut metrics { + metrics.packed_owners = plan.units.iter().map(|u| u.groups.len() - 1).sum(); + metrics.shared_carriers = plan.shared_carriers; + metrics.pruned_methods = plan + .units + .iter() + .flat_map(|unit| &unit.groups) + .filter_map(|&group| { + index + .dead_methods + .get(index.groups[group].name.trim_end_matches(".class")) + }) + .map(HashSet::len) + .sum(); + } + namespaces.pack(plan.names); + namespaces.aliases = std::mem::take(&mut index.aliases); + namespaces.short_methods = std::mem::take(&mut index.short_methods); + namespaces.strip_proofs = index.share_carriers; let output = Path::new(output); let parent = output .parent() @@ -47,11 +78,11 @@ pub(crate) fn link( let readers = Readers::open(&index)?; let relocations = std::sync::Mutex::new(split::Relocations::default()); let mut start = 0; - while start < index.groups.len() { + while start < plan.units.len() { let mut end = start; let mut bytes = 0usize; - while end < index.groups.len() && end - start < BATCH_CLASSES { - let next = index.groups[end].bytes; + while end < plan.units.len() && end - start < BATCH_CLASSES { + let next = plan.units[end].bytes; if end > start && next > BATCH_BYTES.saturating_sub(bytes) { break; } @@ -61,18 +92,12 @@ pub(crate) fn link( // A single oversized class group is indivisible. All other live input // bytes are bounded by BATCH_BYTES, independent of the whole program. let chunk = (end - start).div_ceil(rayon::current_num_threads()).max(1); - let merged = index.groups[start..end] + let merged = plan.units[start..end] .par_chunks(chunk) - .map(|groups| { - jar::Batch::encode(groups.iter().flat_map(|group| { - let result = readers.load(&index.paths, group).and_then(|fragments| { - merge_group_with_relocations(fragments, Some(&relocations)) - }); - match result { - Ok(classes) => classes - .into_iter() - .map(|class| namespaces.class(class)) - .collect::>(), + .map(|units| { + jar::Batch::encode(units.iter().flat_map(|unit| { + match merge_unit(unit, &index, &readers, &namespaces, &relocations) { + Ok(classes) => classes.into_iter().map(Ok).collect::>(), Err(error) => vec![Err(error)], } })) @@ -104,3 +129,43 @@ pub(crate) fn link( } Ok(()) } + +fn merge_unit( + unit: &packing::Unit, + index: &Index, + readers: &Readers, + namespaces: &namespaces::Namespaces, + relocations: &std::sync::Mutex, +) -> io::Result> { + let mut output = Vec::new(); + for &position in &unit.groups { + let group = &index.groups[position]; + let fragments = readers.load(&index.paths, group)?; + if group.resource { + let mut fragments = fragments.into_iter(); + let data = fragments.next().expect("resource has a fragment"); + if fragments.any(|fragment| fragment.data != data.data) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("conflicting binary constant {}", group.name), + )); + } + return Ok(vec![data]); + } + let classes = merge_group_with_demands( + fragments, + Some(relocations), + index + .dead_methods + .get(group.name.trim_end_matches(".class")), + )?; + for class in classes { + output.push(namespaces.class(class)?); + } + } + if unit.groups.len() > 1 { + merge_group_with_demands(output, Some(relocations), None) + } else { + Ok(output) + } +} diff --git a/java-linker/src/private_names.rs b/java-linker/src/private_names.rs new file mode 100644 index 00000000..43ac46f0 --- /dev/null +++ b/java-linker/src/private_names.rs @@ -0,0 +1,62 @@ +//! Shorten proven private class names. Relocate symbolic strings but preserve ordinary literals. +use crate::*; + +pub(crate) fn compact(index: &mut inputs::Index, names: &mut HashMap) { + let private = std::mem::take(&mut index.private_classes); + let occupied = std::mem::take(&mut index.occupied_packages); + if private.is_empty() { + return; + } + // Kotlin finds the companion by appending "$Body". Keep both class names unchanged. + let companions = index + .groups + .iter() + .filter_map(|group| group.name.strip_suffix("$Body.class")) + .collect::>(); + let mut prefix = "j".to_string(); + for i in 1.. { + if !occupied.contains(&prefix) { + break; + } + prefix = format!("j{i:x}"); + } + let eligible = |name: &str| { + private.contains(name) + && !companions.contains(name) + // Pointer's callable codec recognizes this runtime protocol. + && !name.starts_with("org/rustlang/runtime/FnPtr_") + }; + let mut targets = index + .groups + .iter() + .filter_map(|group| { + let name = group.name.trim_end_matches(".class"); + (!group.resource && eligible(name)) + .then(|| names.get(name).map_or(name, String::as_str).to_owned()) + }) + .collect::>(); + targets.sort_unstable(); + targets.dedup(); + let short = targets + .into_iter() + .enumerate() + .map(|(i, name)| { + let target = if jvm_compiler_core::classfile::names::codec_owner(&name) { + format!("{prefix}/Codecs_{i:x}") + } else if name.contains("/mono/Mono") { + // Preserve eligibility for exceptional constant-pool splitting. + format!("{prefix}/mono/Mono_{i:x}") + } else { + format!("{prefix}/C{i:x}") + }; + (name, target) + }) + .collect::>(); + for group in &index.groups { + let name = group.name.trim_end_matches(".class"); + if !group.resource && eligible(name) { + let target = names.get(name).map_or(name, String::as_str); + names.insert(name.to_owned(), short[target].clone()); + } + } +} diff --git a/java-linker/src/private_names_tests.rs b/java-linker/src/private_names_tests.rs new file mode 100644 index 00000000..8dd468f3 --- /dev/null +++ b/java-linker/src/private_names_tests.rs @@ -0,0 +1,145 @@ +use crate::test_support::{body, holder, run_jar, write_classes}; +use crate::*; +use jvm_compiler_core::classfile::names; + +#[test] +fn compact_names_avoid_external_names_and_preserve_reflective_codecs() { + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("Lookup.java"); + fs::write( + &source, + r##" +package org.rustlang.runtime; +public class Lookup { + public static void checkBody(Class carrier) throws Exception { + Class.forName(carrier.getName() + "$Body"); + } + public static void check(String recipe, int expected) throws Exception { + String[] parts = recipe.split("#"); + Object value = Class.forName(parts[0].replace('/', '.')) + .getMethod("e$" + parts[1]).invoke(null); + if (!Integer.valueOf(expected).equals(value)) throw new AssertionError(value); + } +} +"##, + ) + .unwrap(); + assert!( + std::process::Command::new("javac") + .arg("-d") + .arg(temp.path()) + .arg(&source) + .status() + .unwrap() + .success() + ); + let mut paths = vec![ + temp.path() + .join("org/rustlang/runtime/Lookup.class") + .to_string_lossy() + .into_owned(), + ]; + let mut main = holder("test/Main", false); + let lookup = main + .constant_pool + .add_class("org/rustlang/runtime/Lookup") + .unwrap(); + let check = main + .constant_pool + .add_method_ref(lookup, "check", "(Ljava/lang/String;I)V") + .unwrap(); + let mut code = Vec::new(); + let mut classes = Vec::new(); + for (owner, key, value) in [ + ( + "test/Codecs_first", + "0123456789abcdef", + Instruction::Iconst_1, + ), + ( + "test/Codecs_second", + "fedcba9876543210", + Instruction::Iconst_2, + ), + ] { + let mut codec = holder(owner, true); + body( + &mut codec, + &format!("e${key}"), + "()I", + vec![value.clone(), Instruction::Ireturn], + ); + classes.push(codec); + let recipe = main + .constant_pool + .add_string(format!("{}{owner}#{key}#I", names::NAME_STRING)) + .unwrap(); + code.extend([ + Instruction::Ldc_w(recipe), + value, + Instruction::Invokestatic(check), + ]); + } + // Occupied roots include identities whose crate marker will be removed. + classes.push(holder("j$crate0123456789abcdef$/Api", false)); + classes.push(holder("j1/Api", false)); + classes.push(holder("test/Future", true)); + classes.push(holder("test/Future$Body", false)); + let future = main.constant_pool.add_class("test/Future").unwrap(); + let check_body = main + .constant_pool + .add_method_ref(lookup, "checkBody", "(Ljava/lang/Class;)V") + .unwrap(); + code.extend([ + Instruction::Ldc_w(future), + Instruction::Invokestatic(check_body), + ]); + let callable = "org/rustlang/runtime/FnPtr_test"; + classes.push(holder(callable, true)); + let callable_class = main.constant_pool.add_class(callable).unwrap(); + code.extend([ + Instruction::Ldc_w(callable_class), + Instruction::Pop, + Instruction::Return, + ]); + // The same UTF8 entry is both a symbolic Class and an ordinary String. + main.constant_pool.add_class("test/Codecs_first").unwrap(); + main.constant_pool.add_string("test/Codecs_first").unwrap(); + body(&mut main, "main", "([Ljava/lang/String;)V", code); + classes.push(main); + paths.extend(write_classes(temp.path(), classes)); + let output = temp.path().join("compact.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(output).unwrap()).unwrap(); + for name in [ + "j/Api.class", + "j1/Api.class", + "org/rustlang/runtime/FnPtr_test.class", + "test/Main.class", + "test/Future.class", + "test/Future$Body.class", + ] { + assert!(jar.by_name(name).is_ok(), "stable identity lost: {name}"); + } + let codecs = jar + .file_names() + .filter(|name| name.contains("Codecs_")) + .collect::>(); + assert_eq!(codecs.len(), 1, "codec owners must pack"); + assert!(codecs[0].starts_with("j2/Codecs_"), "{codecs:?}"); + let mut bytes = Vec::new(); + jar.by_name("test/Main.class") + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let main = class_file_from_data(&bytes).unwrap(); + assert!( + main.constant_pool.iter().any(|c| match c { + Constant::String(i) => + main.constant_pool.try_get_utf8(*i).unwrap() == "test/Codecs_first", + _ => false, + }), + "ordinary literal must not follow class relocation" + ); +} diff --git a/java-linker/src/reachability.rs b/java-linker/src/reachability.rs new file mode 100644 index 00000000..24f60cb9 --- /dev/null +++ b/java-linker/src/reachability.rs @@ -0,0 +1,570 @@ +//! Trace class dependencies during input indexing. Remove only classes marked private by the compiler. +use crate::*; +use jvm_compiler_core::classfile::{JavaStr, names, summary}; +use std::borrow::Cow; + +fn text(bytes: &[u8]) -> Option> { + match std::str::from_utf8(bytes) { + Ok(text) => Some(Cow::Borrowed(text)), + Err(_) => JavaStr::from_mutf8(bytes) + .ok() + .map(|text| Cow::Owned(text.to_rust_string())), + } +} + +#[derive(Default)] +struct Node { + edges: Vec, + root: bool, + whole: bool, + defined_method: bool, + pinned: bool, + public_method: bool, + forward_seen: bool, + forward: Option, + bytes: usize, +} + +#[derive(Default)] +pub(crate) struct Graph { + names: HashMap, + nodes: Vec, + opaque: bool, + symbols: HashMap, u32>, + spellings: Vec>, + methods: HashMap<(u32, u32, u32), u32>, + reflective_names: HashMap, + fixed_names: HashSet, +} + +#[derive(Default)] +pub(crate) struct Demands { + pub(crate) share_carriers: bool, + pub(crate) compact_carriers: bool, + pub(crate) pinned_classes: HashSet, + pub(crate) classes: Vec, + pub(crate) dead_methods: HashMap>, + pub(crate) packable: HashMap, + pub(crate) aliases: crate::aliases::Aliases, + pub(crate) private_classes: HashSet, + pub(crate) occupied_packages: HashSet, + pub(crate) short_methods: HashMap, Vec>, +} + +impl Graph { + pub(crate) fn dependency(&mut self, dependency: summary::Dependency<'_>, edges: &mut Vec) { + match dependency { + summary::Dependency::Definition { .. } => {} + summary::Dependency::FixedMemberName(bytes) => { + if let Some(name) = text(bytes) { + let symbol = self.symbol(&name); + self.fixed_names.insert(symbol); + } + } + summary::Dependency::String(bytes) => { + if bytes.starts_with(names::LITERAL_STRING.as_bytes()) { + return; + } + self.references(bytes, edges); + if let Some(value) = text(bytes) { + let value = value.strip_prefix(names::NAME_STRING).unwrap_or(&value); + self.reflective_name(value, edges); + self.codec_recipe(value, edges); + } + } + summary::Dependency::Method(method) => { + if let Some(id) = self.method(method) { + edges.push(id); + } + } + summary::Dependency::Text(bytes) => { + self.references(bytes, edges); + // Annotation and debug metadata can require the original member name. + if let Some(name) = text(bytes) + && crate::method_names::mangled(&name) + { + let symbol = self.symbol(&name); + self.fixed_names.insert(symbol); + } + } + summary::Dependency::Class(name) => { + if let Some(name) = text(name) { + if name.starts_with('[') { + self.references(name.as_bytes(), edges); + } else { + edges.push(self.intern(&name)); + } + } + } + } + } + fn symbol(&mut self, name: &str) -> u32 { + if let Some(&id) = self.symbols.get(name) { + return id; + } + let id = self.spellings.len() as u32; + let name: std::sync::Arc = name.into(); + self.symbols.insert(name.clone(), id); + self.spellings.push(name); + id + } + fn method(&mut self, key: summary::MethodKey<'_>) -> Option { + let owner = text(key.owner)?; + let name = text(key.name)?; + if !summary::method_owner(&owner) && !summary::enum_helper(&name) { + let symbol = self.symbol(&name); + self.fixed_names.insert(symbol); + return None; + } + let descriptor = text(key.descriptor)?; + let owner = self.intern(&owner); + let key = (owner, self.symbol(&name), self.symbol(&descriptor)); + if let Some(&id) = self.methods.get(&key) { + return Some(id); + } + let id = self.nodes.len() as u32; + self.nodes.push(Node { + edges: vec![owner], + ..Node::default() + }); + self.methods.insert(key, id); + Some(id) + } + pub(crate) fn scan( + &mut self, + bytes: &[u8], + library: bool, + ) -> io::Result<(summary::Summary, u32)> { + let mut edges = Vec::new(); + let mut previous = None; + let mut source = None; + let mut outgoing = Vec::new(); + fn finish_method(nodes: &mut [Node], source: Option, outgoing: &mut Vec) { + if let Some(source) = source { + let node = &mut nodes[source as usize]; + node.defined_method = true; + // Combine repeated upstream method edges before they accumulate across input fragments. + node.edges.append(outgoing); + node.edges.sort_unstable(); + node.edges.dedup(); + } + } + let metadata = summary::read_demands(bytes, |scope, dependency| { + if library { + if let summary::Dependency::Class(name) = dependency { + if let Some(name) = text(name) { + let id = self.intern(&name); + self.nodes[id as usize].pinned = true; + } + } + } + if scope != previous { + finish_method(&mut self.nodes, source, &mut outgoing); + source = scope.and_then(|m| self.method(m)); + previous = scope; + } + if let ( + Some(source), + summary::Dependency::Definition { + public, + forward, + bytes, + }, + ) = (source, dependency) + { + let target = forward.and_then(|method| self.method(method)); + let node = &mut self.nodes[source as usize]; + node.bytes = node.bytes.max(bytes); + if node.forward_seen { + node.public_method &= public; + if node.forward != target { + node.forward = None; + } + } else { + node.forward_seen = true; + node.public_method = public; + node.forward = target; + } + } + if source.is_some() { + self.dependency(dependency, &mut outgoing); + } else { + self.dependency(dependency, &mut edges); + } + })?; + finish_method(&mut self.nodes, source, &mut outgoing); + let id = self.record(&metadata, edges, library); + Ok((metadata, id)) + } + fn intern(&mut self, name: &str) -> u32 { + if let Some(&id) = self.names.get(name) { + return id; + } + let id = u32::try_from(self.nodes.len()).expect("too many JVM dependency names"); + self.names.insert(name.into(), id); + self.nodes.push(Node::default()); + id + } + + pub(crate) fn resource(&mut self, name: &str) -> u32 { + self.intern(name) + } + + pub(crate) fn references(&mut self, bytes: &[u8], edges: &mut Vec) { + // Do not interpret user literals as compiler symbols. Unknown reflection disables pruning. + if bytes.starts_with(names::LITERAL_STRING.as_bytes()) { + return; + } + let Some(text) = text(bytes) else { + return; + }; + let text = text.strip_prefix(names::NAME_STRING).unwrap_or(&text); + let resource = text.trim_start_matches('/'); + if jvm_compiler_core::classfile::resources::valid_name(resource) { + edges.push(self.intern(resource)); + return; + } + let descriptor = + text.starts_with(['(', '[']) || (text.starts_with('L') && text.ends_with(';')); + for token in text.split(|c: char| { + c.is_whitespace() + || matches!( + c, + '(' | ')' | ';' | '[' | ']' | '<' | '>' | ':' | '#' | ',' | '\u{1}' + ) + }) { + if !token.contains('/') && !token.contains('.') { + if descriptor + && let Some(name) = token + .trim_start_matches(['B', 'C', 'D', 'F', 'I', 'J', 'S', 'Z', 'V']) + .strip_prefix('L') + { + edges.push(self.intern(name)); + } + continue; + } + // Descriptors, codec recipes, and class names share this token grammar. + let token = if token.contains('.') { + std::borrow::Cow::Owned(token.replace('.', "/")) + } else { + std::borrow::Cow::Borrowed(token) + }; + edges.push(self.intern(&token)); + // Skip primitive parameter types before a reference descriptor. + let descriptor = + token.trim_start_matches(['B', 'C', 'D', 'F', 'I', 'J', 'S', 'Z', 'V']); + if let Some(class) = descriptor.strip_prefix('L') { + edges.push(self.intern(class)); + } + } + } + + fn reflective_name(&mut self, name: &str, edges: &mut Vec) { + // Function-pointer adapters use owner::method:descriptor strings. Preserve their method names. + if let Some((owner, target)) = name.split_once("::") + && let Some((method, descriptor)) = target.split_once(':') + { + let symbol = self.symbol(method); + self.fixed_names.insert(symbol); + if let Some(id) = self.method(summary::MethodKey { + owner: owner.replace('.', "/").as_bytes(), + name: method.as_bytes(), + descriptor: descriptor.as_bytes(), + }) { + edges.push(id); + } + } + // Runtime protocols can pass owner, method, and descriptor as separate unmarked strings. + if name.is_empty() + || name.starts_with(names::STRING_TAG) + || name.contains(['/', '.', '(', ')', ';', '#', '[', '\0', ' ', '\n']) + { + return; + } + let symbol = self.symbol(name); + let id = *self.reflective_names.entry(symbol).or_insert_with(|| { + let id = self.nodes.len() as u32; + self.nodes.push(Node::default()); + id + }); + edges.push(id); + } + + fn codec_recipe(&mut self, value: &str, edges: &mut Vec) { + // The runtime resolves every hook, including hooks in nested codec recipes. + for (_, key) in names::codec_recipes(value) { + for prefix in ["e$", "d$", "a$", "w$", "b$", "s$", "c$"] { + self.reflective_name(&format!("{prefix}{key}"), edges); + } + } + } + + pub(crate) fn record( + &mut self, + summary: &summary::Summary, + mut edges: Vec, + library: bool, + ) -> u32 { + let id = self.intern(&summary.name); + // Unknown reflection and native code can reach any generated class. + self.opaque |= + summary.opaque_reflection && !summary.name.starts_with("org/rustlang/runtime/"); + if library { + for &edge in &edges { + self.nodes[edge as usize].pinned = true; + } + } + let node = &mut self.nodes[id as usize]; + node.root |= library || !summary.private || summary.has_main; + node.whole |= library || !summary.method_demands; + edges.extend_from_slice(&node.edges); + edges.sort_unstable(); + edges.dedup(); + node.edges = edges; + id + } + + pub(crate) fn libraries(&mut self, libraries: &[String]) -> io::Result<()> { + let mut bytes = Vec::new(); + for path in libraries { + let mut jar = ZipArchive::new(BufReader::new(fs::File::open(path)?))?; + for i in 0..jar.len() { + let mut entry = jar.by_index(i)?; + if !entry.name().ends_with(".class") { + continue; + } + bytes.clear(); + entry.read_to_end(&mut bytes)?; + match self.scan(&bytes, true) { + Ok(_) => {} + // Preserve unreadable library classes through the opaque-JAR copy path. + Err(_) => self.opaque = true, + } + } + } + Ok(()) + } + + pub(crate) fn finish(mut self, executable: bool) -> Demands { + if !executable || self.opaque { + return Demands { + classes: vec![true; self.nodes.len()], + ..Demands::default() + }; + } + for (&(owner, _, _), &method) in &self.methods { + if self.nodes[owner as usize].whole || self.nodes[owner as usize].pinned { + self.nodes[owner as usize].edges.push(method); + } + } + // Retain reflective methods only when their owner is live. Unknown reflection already disables pruning. + let mut reflective = HashMap::>::default(); + for (&(owner, name, _), &method) in &self.methods { + if let Some(&node) = self.reflective_names.get(&name) { + reflective.entry(node).or_default().push((owner, method)); + reflective.entry(owner).or_default().push((node, method)); + } + } + for node in &mut self.nodes { + node.edges.sort_unstable(); + node.edges.dedup(); + } + let mut live = vec![false; self.nodes.len()]; + let mut pending = self + .nodes + .iter() + .enumerate() + .filter_map(|(i, n)| n.root.then_some(i as u32)) + .collect::>(); + while let Some(id) = pending.pop() { + if !std::mem::replace(&mut live[id as usize], true) { + pending.extend_from_slice(&self.nodes[id as usize].edges); + if let Some(methods) = reflective.get(&id) { + pending.extend( + methods + .iter() + .filter_map(|&(other, method)| live[other as usize].then_some(method)), + ); + } + } + } + drop(reflective); + let owners = self + .names + .into_iter() + .map(|(name, id)| (id, name)) + .collect::>(); + // Reserve external package names even when their classes are unreachable. + let compact_carriers = !owners.values().any(|name| { + name.starts_with("org/rustlang/shape/") + || name.strip_prefix("org").is_some_and(|suffix| { + suffix + .as_bytes() + .get(..names::CRATE_MARKER_LEN) + .is_some_and(names::is_crate_marker) + && suffix[names::CRATE_MARKER_LEN..].starts_with("/rustlang/shape/") + }) + }); + let mut packable = HashMap::::default(); + let keys = self + .methods + .iter() + .map(|(&key, &id)| (id, key)) + .collect::>(); + let private_public = |id: u32| { + let Some(&(owner, _, _)) = keys.get(&id) else { + return false; + }; + let node = &self.nodes[owner as usize]; + self.nodes[id as usize].defined_method + && self.nodes[id as usize].public_method + && !node.whole + && !node.root + && !node.pinned + && owners[&owner].contains("/mono/Mono_") + }; + let removable = + |id: u32| private_public(id) && !self.reflective_names.contains_key(&keys[&id].1); + let mut alias_targets = HashMap::default(); + for &id in self.methods.values() { + if !live[id as usize] || !removable(id) { + continue; + } + let Some(mut target) = self.nodes[id as usize].forward else { + continue; + }; + for _ in 0..64 { + if target == id || !private_public(target) { + break; + } + if removable(target) + && let Some(next) = self.nodes[target as usize].forward + { + target = next; + } else { + alias_targets.insert(id, target); + break; + } + } + } + let mut aliases = crate::aliases::Aliases::default(); + for (&id, &target) in &alias_targets { + let (owner, name, descriptor) = keys[&id]; + let (target_owner, target_name, _) = keys[&target]; + aliases + .entry(JavaString::from(owners[&owner].as_str())) + .or_default() + .insert( + ( + JavaString::from(self.spellings[name as usize].as_ref()), + JavaString::from(self.spellings[descriptor as usize].as_ref()), + ), + ( + JavaString::from(owners[&target_owner].as_str()), + JavaString::from(self.spellings[target_name as usize].as_ref()), + ), + ); + } + for (&(owner, name, descriptor), &id) in &self.methods { + let node = &self.nodes[owner as usize]; + if !live[id as usize] + || !self.nodes[id as usize].defined_method + || node.whole + || node.root + || node.pinned + || !summary::method_owner(&owners[&owner]) + { + continue; + } + let candidate = packable.entry(owners[&owner].clone()).or_default(); + if !alias_targets.contains_key(&id) { + candidate.methods.push((name, descriptor)); + candidate.bytes = candidate + .bytes + .saturating_add(self.nodes[id as usize].bytes); + } + for edge in &self.nodes[id as usize].edges { + let target = keys.get(edge).map_or(edge, |key| &key.0); + if *target != owner { + if let Some(name) = owners.get(target) { + if summary::method_owner(name) { + candidate.neighbors.push(name.clone()); + } + } + } + } + } + for candidate in packable.values_mut() { + candidate.neighbors.sort_unstable(); + candidate.neighbors.dedup(); + } + let mut dead_methods = HashMap::>::default(); + let short_methods = crate::method_names::plan( + self.methods.iter().map(|(&(owner, name, _), &id)| { + let node = &self.nodes[owner as usize]; + ( + name, + live[id as usize] && !alias_targets.contains_key(&id), + self.nodes[id as usize].defined_method + && !node.whole + && !node.root + && !node.pinned + && owners[&owner].contains("/mono/Mono_"), + ) + }), + &self.spellings, + &self.fixed_names, + &self.reflective_names, + owners.values().map(String::as_str), + ); + for ((owner, name, descriptor), id) in self.methods { + if (!live[id as usize] || alias_targets.contains_key(&id)) + && self.nodes[id as usize].defined_method + { + dead_methods + .entry(owners[&owner].clone()) + .or_default() + .insert(( + JavaString::from(self.spellings[name as usize].as_ref()), + JavaString::from(self.spellings[descriptor as usize].as_ref()), + )); + } + } + Demands { + share_carriers: true, + compact_carriers, + pinned_classes: owners + .iter() + .filter_map(|(&id, name)| self.nodes[id as usize].pinned.then(|| name.clone())) + .collect(), + private_classes: owners + .iter() + .filter_map(|(&id, name)| { + let node = &self.nodes[id as usize]; + (live[id as usize] && !node.root && !node.pinned).then(|| name.clone()) + }) + .collect(), + occupied_packages: owners + .values() + .map(|name| { + name.split('/') + .next() + .unwrap() + .split(names::CRATE_MARKER) + .next() + .unwrap() + .to_owned() + }) + .collect(), + classes: live, + dead_methods, + packable, + aliases, + short_methods, + } + } + #[cfg(test)] + pub(crate) fn live(self, executable: bool) -> Vec { + self.finish(executable).classes + } +} diff --git a/java-linker/src/reachability_tests.rs b/java-linker/src/reachability_tests.rs new file mode 100644 index 00000000..23f882ca --- /dev/null +++ b/java-linker/src/reachability_tests.rs @@ -0,0 +1,831 @@ +use super::*; +use crate::test_support::{body, holder, run_jar, write_classes}; +use jvm_compiler_core::classfile::{names, summary}; + +fn summary(name: &str, private: bool) -> summary::Summary { + summary::Summary { + name: name.into(), + has_main: false, + private, + opaque_reflection: false, + method_demands: false, + carrier: None, + } +} + +#[test] +fn roots_follow_descriptors_recipes_and_cycles_without_retaining_dead_cycles() { + let mut graph = reachability::Graph::default(); + let mut edges = Vec::new(); + graph.references(b"(IJLtest/Value;[Ltest/Array;)Ltest/Return;", &mut edges); + graph.references( + format!( + "{}test/Codecs#0123456789abcdef#Ltest/Decoded;", + names::NAME_STRING + ) + .as_bytes(), + &mut edges, + ); + graph.references(b"test.BinaryName", &mut edges); + graph.record(&summary("test/Main", false), edges, false); + let live_ids = [ + "test/Value", + "test/Array", + "test/Return", + "test/Codecs", + "test/Decoded", + "test/BinaryName", + ] + .map(|name| { + let mut edges = Vec::new(); + graph.references(b"Ltest/Cycle;", &mut edges); + graph.record(&summary(name, true), edges, false) + }); + let mut cycle = Vec::new(); + graph.references(b"Ltest/Value;", &mut cycle); + let cycle = graph.record(&summary("test/Cycle", true), cycle, false); + let mut dead = Vec::new(); + graph.references(b"Ltest/DeadB;", &mut dead); + let dead_a = graph.record(&summary("test/DeadA", true), dead, false); + let mut dead = Vec::new(); + graph.references(b"Ltest/DeadA;", &mut dead); + let dead_b = graph.record(&summary("test/DeadB", true), dead, false); + let live = graph.live(true); + assert!( + live_ids + .into_iter() + .chain([cycle]) + .all(|id| live[id as usize]) + ); + assert!(!live[dead_a as usize] && !live[dead_b as usize]); +} + +#[test] +fn libraries_unknown_reflection_and_public_fragments_remain_conservative() { + for scenario in 0..4 { + let mut graph = reachability::Graph::default(); + let id = graph.record(&summary("test/Private", true), Vec::new(), scenario == 0); + match scenario { + 1 => { + let mut reflection = summary("user/Reflection", false); + reflection.opaque_reflection = true; + graph.record(&reflection, Vec::new(), false); + } + 2 => { + graph.record(&summary("test/Private", false), Vec::new(), false); + } + _ => {} + } + let demands = graph.finish(scenario != 3); + assert!(demands.classes[id as usize]); + assert!(demands.private_classes.is_empty()); + } +} + +#[test] +fn compact_storage_names_cannot_collide_after_crate_namespace_relocation() { + let mut graph = reachability::Graph::default(); + graph.record( + &summary("org$crate0123456789abcdef$/rustlang/shape/S0", false), + Vec::new(), + false, + ); + let demands = graph.finish(true); + assert!(demands.share_carriers); + assert!(!demands.compact_carriers); +} + +#[test] +fn private_marker_and_reflection_are_read_from_class_metadata() { + let mut class = ClassFile::default(); + class.this_class = class.constant_pool.add_class("test/Internal").unwrap(); + class.super_class = class.constant_pool.add_class("java/lang/Object").unwrap(); + let name_index = class + .constant_pool + .add_utf8(summary::PRIVATE_ATTRIBUTE) + .unwrap(); + class.attributes.push(Attribute::Unknown { + name_index, + info: Vec::new(), + }); + let owner = class.constant_pool.add_class("java/lang/Class").unwrap(); + class + .constant_pool + .add_method_ref(owner, "forName", "(Ljava/lang/String;)Ljava/lang/Class;") + .unwrap(); + let bytes = serialize_class_file(&class).unwrap(); + let summary = summary::read(&bytes).unwrap(); + assert!(summary.private && summary.opaque_reflection); +} + +#[test] +fn unused_static_cycles_and_their_types_disappear_but_handles_execute() { + let mut main = holder("test/Main", false); + let mut helpers = holder("test/mono/Mono_helpers", true); + let mut unused = holder("test/Unused", true); + let helper_owner = main + .constant_pool + .add_class("test/mono/Mono_helpers") + .unwrap(); + let call = main + .constant_pool + .add_method_ref(helper_owner, "live", "()V") + .unwrap(); + body( + &mut main, + "main", + "([Ljava/lang/String;)V", + vec![Instruction::Invokestatic(call), Instruction::Return], + ); + let handled = helpers + .constant_pool + .add_method_ref(helpers.this_class, "handled", "()V") + .unwrap(); + let handle = helpers + .constant_pool + .add_method_handle(ristretto_classfile::ReferenceKind::InvokeStatic, handled) + .unwrap(); + let mh = helpers + .constant_pool + .add_class("java/lang/invoke/MethodHandle") + .unwrap(); + let invoke = helpers + .constant_pool + .add_method_ref(mh, "invokeExact", "()V") + .unwrap(); + body( + &mut helpers, + "live", + "()V", + vec![ + Instruction::Ldc_w(handle), + Instruction::Invokevirtual(invoke), + Instruction::Return, + ], + ); + body(&mut helpers, "handled", "()V", vec![Instruction::Return]); + body( + &mut helpers, + "drop_glue$runtime", + "()V", + vec![Instruction::Return], + ); + // Runtime drop lookup uses a method name without a MethodRef. + main.constant_pool.add_string("drop_glue$runtime").unwrap(); + let a = helpers + .constant_pool + .add_method_ref(helpers.this_class, "deadA", "()V") + .unwrap(); + let b = helpers + .constant_pool + .add_method_ref(helpers.this_class, "deadB", "()V") + .unwrap(); + let dead_class = helpers.constant_pool.add_class("test/Unused").unwrap(); + body( + &mut helpers, + "deadA", + "()V", + vec![ + Instruction::Ldc_w(dead_class), + Instruction::Pop, + Instruction::Invokestatic(b), + Instruction::Return, + ], + ); + body( + &mut helpers, + "deadB", + "()V", + vec![Instruction::Invokestatic(a), Instruction::Return], + ); + body(&mut unused, "never", "()V", vec![Instruction::Return]); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes(temp.path(), [main, helpers, unused]); + let output = temp.path().join("output.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + assert!(jar.by_name("test/Unused.class").is_err()); + let mut bytes = Vec::new(); + let helper = jar + .file_names() + .find(|n| n.contains("/mono/Mono_")) + .unwrap() + .to_owned(); + jar.by_name(&helper) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + let names = (0..class.methods.len()) + .map(|i| method_identity(&class, i).unwrap().0.to_string()) + .collect::>(); + assert_eq!( + names, + [ + "live".to_owned(), + "handled".to_owned(), + "drop_glue$runtime".to_owned() + ] + .into_iter() + .collect() + ); + assert!( + !bytes + .windows(b"test/Unused".len()) + .any(|w| w == b"test/Unused") + ); +} + +#[test] +fn public_fragment_retains_all_methods_of_the_combined_owner() { + let mut private = holder("test/mono/Mono_mixed", true); + body(&mut private, "internal", "()V", vec![Instruction::Return]); + let public = holder("test/mono/Mono_mixed", false); + let mut graph = reachability::Graph::default(); + graph + .scan(&serialize_class_file(&private).unwrap(), false) + .unwrap(); + graph + .scan(&serialize_class_file(&public).unwrap(), false) + .unwrap(); + let demands = graph.finish(true); + assert!(demands.dead_methods.is_empty()); + assert!(!demands.private_classes.contains("test/mono/Mono_mixed")); +} + +#[test] +fn codec_recipe_retains_its_exact_method_family_without_other_recipes() { + let mut main = holder("test/Main", false); + let mut codecs = holder("test/Codecs_ab", true); + main.constant_pool + .add_string("test/Codecs_ab#abcdef0123456789#Ltest/Value;") + .unwrap(); + // Recipe text inside a user literal must not retain helpers. + main.constant_pool + .add_string(format!( + "{}example\ntest/Codecs_ab#ab0123456789cdef#Ltest/Value;", + jvm_compiler_core::classfile::names::LITERAL_STRING + )) + .unwrap(); + for key in ["abcdef0123456789", "ab0123456789cdef"] { + for prefix in ["e$", "d$", "a$", "w$", "b$", "s$", "c$"] { + body( + &mut codecs, + &format!("{prefix}{key}"), + "()V", + vec![Instruction::Return], + ); + } + } + let mut graph = reachability::Graph::default(); + graph + .scan(&serialize_class_file(&main).unwrap(), false) + .unwrap(); + let (metadata, _) = graph + .scan(&serialize_class_file(&codecs).unwrap(), false) + .unwrap(); + assert!(metadata.method_demands); + let demands = graph.finish(true); + let dead = &demands.dead_methods["test/Codecs_ab"]; + assert_eq!(dead.len(), 7); + assert!( + dead.iter() + .all(|(name, _)| name.to_string().ends_with("ab0123456789cdef")) + ); +} + +#[test] +fn binary_constant_resources_follow_live_code_and_dynamic_access_keeps_them() { + use jvm_compiler_core::classfile::resources::Resource; + let used = Resource::new(vec![1, 2, 3]); + let unused = Resource::new(vec![4, 5, 6]); + for dynamic in [false, true] { + let mut graph = reachability::Graph::default(); + let a = graph.resource(&used.name); + let b = graph.resource(&unused.name); + let mut main = holder("test/Main", false); + main.constant_pool + .add_string(format!( + "{}/{}", + jvm_compiler_core::classfile::names::NAME_STRING, + used.name + )) + .unwrap(); + main.constant_pool + .add_string(format!( + "{}{}", + jvm_compiler_core::classfile::names::LITERAL_STRING, + unused.name + )) + .unwrap(); + if dynamic { + let loader = main.constant_pool.add_class("test/CustomLoader").unwrap(); + main.constant_pool + .add_method_ref( + loader, + "getResourceAsStream", + "(Ljava/lang/String;)Ljava/io/InputStream;", + ) + .unwrap(); + } + graph + .scan(&serialize_class_file(&main).unwrap(), false) + .unwrap(); + let live = graph.finish(true); + assert!(live.classes[a as usize]); + assert_eq!(live.classes[b as usize], dynamic); + } +} + +#[test] +fn reflective_helpers_require_a_live_owner_and_follow_newly_discovered_owners() { + for reverse in [false, true] { + let mut main = holder("test/Main", false); + main.constant_pool.add_class("test/mono/Mono_a").unwrap(); + let mut runtime = holder("org/rustlang/runtime/Protocol", false); + runtime.constant_pool.add_string("hook").unwrap(); + let mut classes = vec![main, runtime]; + for (name, dependency) in [("a", Some("b")), ("b", Some("payload")), ("unused", None)] { + let mut class = holder(&format!("test/mono/Mono_{name}"), true); + let mut code = Vec::new(); + if let Some(dependency) = dependency { + code.extend([ + Instruction::Ldc_w( + class + .constant_pool + .add_class(format!("test/mono/Mono_{dependency}")) + .unwrap(), + ), + Instruction::Pop, + ]); + } + code.push(Instruction::Return); + body(&mut class, "hook", "()V", code); + classes.push(class); + } + classes.push(holder("test/mono/Mono_payload", true)); + if reverse { + classes.reverse(); + } + let mut graph = reachability::Graph::default(); + let mut ids = HashMap::default(); + for class in classes { + let (summary, id) = graph + .scan(&serialize_class_file(&class).unwrap(), false) + .unwrap(); + ids.insert(summary.name, id); + } + let demands = graph.finish(true); + for name in ["a", "b", "payload"] { + let owner = format!("test/mono/Mono_{name}"); + assert!(demands.classes[ids[&owner] as usize], "{owner}"); + assert!(!demands.dead_methods.contains_key(&owner)); + } + assert!(!demands.classes[ids["test/mono/Mono_unused"] as usize]); + } +} + +#[test] +fn private_holders_pack_after_pruning_and_calls_handles_and_names_follow() { + let mut main = holder("test/Main", false); + let mut a = holder("test/mono/Mono_a", true); + let mut b = holder("test/mono/Mono_b", true); + let mut c = holder("test/mono/Mono_c", true); + let a_class = main.constant_pool.add_class("test/mono/Mono_a").unwrap(); + let call = main + .constant_pool + .add_method_ref(a_class, "first", "()V") + .unwrap(); + body( + &mut main, + "main", + "([Ljava/lang/String;)V", + vec![Instruction::Invokestatic(call), Instruction::Return], + ); + let b_class = a.constant_pool.add_class("test/mono/Mono_b").unwrap(); + let call = a + .constant_pool + .add_method_ref(b_class, "second", "()V") + .unwrap(); + body( + &mut a, + "first", + "()V", + vec![Instruction::Invokestatic(call), Instruction::Return], + ); + let c_class = b.constant_pool.add_class("test/mono/Mono_c").unwrap(); + let call = b + .constant_pool + .add_method_ref(c_class, "third", "()V") + .unwrap(); + let handle = b + .constant_pool + .add_method_handle(ristretto_classfile::ReferenceKind::InvokeStatic, call) + .unwrap(); + let mh = b + .constant_pool + .add_class("java/lang/invoke/MethodHandle") + .unwrap(); + let invoke = b + .constant_pool + .add_method_ref(mh, "invokeExact", "()V") + .unwrap(); + body( + &mut b, + "second", + "()V", + vec![ + Instruction::Ldc_w(handle), + Instruction::Invokevirtual(invoke), + Instruction::Return, + ], + ); + body(&mut c, "third", "()V", vec![Instruction::Return]); + body(&mut c, "unused", "()V", vec![Instruction::Return]); + // Dead bodies and duplicate fragments affect input memory but not packing size. + for class in [&mut a, &mut b, &mut c] { + let mut dead = vec![Instruction::Nop; 40_000]; + dead.push(Instruction::Return); + body(class, "large_dead", "()V", dead); + } + for name in ["test/mono/Mono_a", "test.mono.Mono_b"] { + main.constant_pool + .add_string(format!("{}{name}", names::NAME_STRING)) + .unwrap(); + main.constant_pool + .add_string(format!("{}{name}", names::LITERAL_STRING)) + .unwrap(); + } + // Runtime-generated names without crate markers are relocated as well. + main.constant_pool.add_string("test/mono/Mono_c").unwrap(); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes( + temp.path(), + [main, a.clone(), a, b.clone(), b, c.clone(), c], + ); + let output = temp.path().join("packed.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + let names = jar + .file_names() + .filter(|n| n.ends_with(".class")) + .map(str::to_owned) + .collect::>(); + assert_eq!(names.len(), 2, "{names:?}"); + let packed = names.iter().find(|n| n.contains("/mono/Mono_")).unwrap(); + let mut bytes = Vec::new(); + jar.by_name(packed) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + assert_eq!( + class.methods.len(), + 2, + "the first method only forwards to the second" + ); + bytes.clear(); + jar.by_name("test/Main.class") + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + let strings = class + .constant_pool + .iter() + .filter_map(|c| match c { + Constant::String(i) => Some(class.constant_pool.try_get_utf8(*i).unwrap().to_string()), + _ => None, + }) + .collect::>(); + let target = packed.trim_end_matches(".class"); + assert!(strings.contains(&target.to_owned())); + assert!(strings.contains(&target.replace('/', "."))); + assert!(strings.contains(&"test/mono/Mono_a".to_owned())); + assert!(strings.contains(&"test.mono.Mono_b".to_owned())); +} + +#[test] +fn external_library_references_pin_private_method_owners() { + let mut helpers = holder("test/mono/Mono_a", true); + body(&mut helpers, "called", "()V", vec![Instruction::Return]); + let mut library = holder("javaapi/Entry", false); + let owner = library.constant_pool.add_class("test/mono/Mono_a").unwrap(); + library + .constant_pool + .add_method_ref(owner, "called", "()V") + .unwrap(); + let mut graph = reachability::Graph::default(); + graph + .scan(&serialize_class_file(&helpers).unwrap(), false) + .unwrap(); + graph + .scan(&serialize_class_file(&library).unwrap(), true) + .unwrap(); + let demands = graph.finish(true); + assert!(demands.packable.is_empty()); + assert!(!demands.private_classes.contains("test/mono/Mono_a")); +} + +#[test] +fn enum_static_helpers_are_demanded_without_erasing_interface_identity() { + let mut main = holder("test/Main", false); + let mut interface = holder("test/Choice", true); + interface.access_flags = + ClassAccessFlags::PUBLIC | ClassAccessFlags::INTERFACE | ClassAccessFlags::ABSTRACT; + interface.attributes.push(Attribute::InnerClasses { + name_index: interface.constant_pool.add_utf8("InnerClasses").unwrap(), + classes: vec![ristretto_classfile::attributes::InnerClass { + class_info_index: interface + .constant_pool + .add_class("test/ChoiceValue") + .unwrap(), + outer_class_info_index: interface.this_class, + name_index: interface.constant_pool.add_utf8("ChoiceValue").unwrap(), + access_flags: ristretto_classfile::attributes::NestedClassAccessFlags::PUBLIC + | ristretto_classfile::attributes::NestedClassAccessFlags::STATIC, + }], + }); + let mut implementation = holder("test/ChoiceValue", true); + let iface = implementation + .constant_pool + .add_class("test/Choice") + .unwrap(); + implementation.interfaces.push(iface); + let object = implementation + .constant_pool + .add_class("java/lang/Object") + .unwrap(); + let init = implementation + .constant_pool + .add_method_ref(object, "", "()V") + .unwrap(); + body( + &mut implementation, + "", + "()V", + vec![ + Instruction::Aload_0, + Instruction::Invokespecial(init), + Instruction::Return, + ], + ); + implementation.methods[0] + .access_flags + .remove(ristretto_classfile::MethodAccessFlags::STATIC); + body( + &mut interface, + "variantIndex", + "(Ltest/Choice;)I", + vec![Instruction::Iconst_0, Instruction::Ireturn], + ); + let dead = interface + .constant_pool + .add_class("test/DeadVariant") + .unwrap(); + body( + &mut interface, + "eq", + "()V", + vec![ + Instruction::Ldc_w(dead), + Instruction::Pop, + Instruction::Return, + ], + ); + let implementation_class = main.constant_pool.add_class("test/ChoiceValue").unwrap(); + let init = main + .constant_pool + .add_method_ref(implementation_class, "", "()V") + .unwrap(); + let iface = main.constant_pool.add_class("test/Choice").unwrap(); + let call = main + .constant_pool + .add_interface_method_ref(iface, "variantIndex", "(Ltest/Choice;)I") + .unwrap(); + body( + &mut main, + "main", + "([Ljava/lang/String;)V", + vec![ + Instruction::New(implementation_class), + Instruction::Dup, + Instruction::Invokespecial(init), + Instruction::Invokestatic(call), + Instruction::Pop, + Instruction::Return, + ], + ); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes( + temp.path(), + [ + main, + interface, + implementation, + holder("test/DeadVariant", true), + ], + ); + let output = temp.path().join("enums.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + run_jar(&output); + let mut jar = ZipArchive::new(fs::File::open(&output).unwrap()).unwrap(); + assert!(jar.by_name("test/DeadVariant.class").is_err()); + let mut bytes = Vec::new(); + jar.by_name("test/Main.class") + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let main = class_file_from_data(&bytes).unwrap(); + let interface = main + .constant_pool + .iter() + .find_map(|c| match c { + Constant::InterfaceMethodRef { class_index, .. } => Some( + main.constant_pool + .try_get_class(*class_index) + .unwrap() + .to_string(), + ), + _ => None, + }) + .unwrap(); + let implementation = main.methods[0] + .attributes + .iter() + .find_map(|a| match a { + Attribute::Code { code, .. } => code.iter().find_map(|i| match i { + Instruction::New(i) => { + Some(main.constant_pool.try_get_class(*i).unwrap().to_string()) + } + _ => None, + }), + _ => None, + }) + .unwrap(); + assert_ne!(interface, implementation); + bytes.clear(); + jar.by_name(&format!("{interface}.class")) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + let class = class_file_from_data(&bytes).unwrap(); + assert!(class.access_flags.contains(ClassAccessFlags::INTERFACE)); + assert_eq!(class.methods.len(), 1); + let nested = class + .attributes + .iter() + .find_map(|a| match a { + Attribute::InnerClasses { classes, .. } => Some(classes), + _ => None, + }) + .unwrap(); + assert_eq!(nested.len(), 1); + assert_eq!( + class + .constant_pool + .try_get_class(nested[0].class_info_index) + .unwrap(), + implementation.as_str() + ); + assert_eq!( + class + .constant_pool + .try_get_class(nested[0].outer_class_info_index) + .unwrap(), + interface.as_str() + ); +} + +#[test] +fn internal_static_roots_are_pruned_without_changing_live_initialization_cycles() { + use ristretto_classfile::{BaseType, Field, FieldAccessFlags, FieldType}; + fn static_owner(name: &str, private: bool, field: &str) -> ClassFile<'static> { + let mut c = holder(name, private); + c.fields.push(Field { + access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::STATIC, + name_index: c.constant_pool.add_utf8(field).unwrap(), + descriptor_index: c.constant_pool.add_utf8("I").unwrap(), + field_type: FieldType::Base(BaseType::Int), + attributes: vec![], + }); + c + } + let mut a = static_owner("test/A$Static", true, "value"); + let own = a + .constant_pool + .add_field_ref(a.this_class, "value", "I") + .unwrap(); + let other = a.constant_pool.add_class("test/B$Static").unwrap(); + let other = a.constant_pool.add_field_ref(other, "value", "I").unwrap(); + body( + &mut a, + "", + "()V", + vec![ + Instruction::Iconst_1, + Instruction::Putstatic(own), + Instruction::Getstatic(other), + Instruction::Getstatic(own), + Instruction::Iadd, + Instruction::Putstatic(own), + Instruction::Return, + ], + ); + let mut b = static_owner("test/B$Static", true, "value"); + let own = b + .constant_pool + .add_field_ref(b.this_class, "value", "I") + .unwrap(); + let other = b.constant_pool.add_class("test/A$Static").unwrap(); + let other = b.constant_pool.add_field_ref(other, "value", "I").unwrap(); + body( + &mut b, + "", + "()V", + vec![ + Instruction::Getstatic(other), + Instruction::Iconst_1, + Instruction::Iadd, + Instruction::Putstatic(own), + Instruction::Return, + ], + ); + let mut dead = static_owner("test/Dead$Static", true, "dead_initializer_marker"); + let unused = dead.constant_pool.add_class("test/UnusedPayload").unwrap(); + body( + &mut dead, + "", + "()V", + vec![ + Instruction::Ldc_w(unused), + Instruction::Pop, + Instruction::Return, + ], + ); + let mut payload = holder("test/UnusedPayload", true); + body( + &mut payload, + "unused_payload_marker", + "()V", + vec![Instruction::Return], + ); + let public = static_owner("test/Exported$Static", false, "exported_value"); + let mut main = holder("test/Main", false); + let system = main.constant_pool.add_class("java/lang/System").unwrap(); + let out = main + .constant_pool + .add_field_ref(system, "out", "Ljava/io/PrintStream;") + .unwrap(); + let stream = main.constant_pool.add_class("java/io/PrintStream").unwrap(); + let print = main + .constant_pool + .add_method_ref(stream, "println", "(I)V") + .unwrap(); + let mut code = Vec::new(); + for owner in ["test/A$Static", "test/B$Static"] { + let owner = main.constant_pool.add_class(owner).unwrap(); + let field = main + .constant_pool + .add_field_ref(owner, "value", "I") + .unwrap(); + code.extend([ + Instruction::Getstatic(out), + Instruction::Getstatic(field), + Instruction::Invokevirtual(print), + ]); + } + code.push(Instruction::Return); + body(&mut main, "main", "([Ljava/lang/String;)V", code); + let temp = tempfile::tempdir().unwrap(); + let paths = write_classes(temp.path(), [main, a, b, dead, payload, public]); + let jar = temp.path().join("statics.jar"); + pipeline::link(&paths, &[], &[], &[], jar.to_str().unwrap()).unwrap(); + let mut archive = ZipArchive::new(fs::File::open(&jar).unwrap()).unwrap(); + assert!( + archive + .file_names() + .any(|name| name == "test/Exported$Static.class") + ); + for index in 0..archive.len() { + let mut bytes = Vec::new(); + archive + .by_index(index) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + for marker in [ + b"dead_initializer_marker".as_slice(), + b"unused_payload_marker".as_slice(), + ] { + assert!(!bytes.windows(marker.len()).any(|window| window == marker)); + } + } + let run = run_jar(&jar); + assert_eq!( + String::from_utf8(run.stdout).unwrap().replace("\r\n", "\n"), + "3\n2\n" + ); +} diff --git a/java-linker/src/remap.rs b/java-linker/src/remap.rs index 48d2c760..f9ca2b43 100644 --- a/java-linker/src/remap.rs +++ b/java-linker/src/remap.rs @@ -116,28 +116,15 @@ pub(crate) fn remap_instruction( } pub(crate) fn instruction_byte_offsets(instructions: &[Instruction]) -> io::Result> { - let mut bytes = Cursor::new(Vec::new()); + // Branch operands are instruction indexes. Measure encoded sizes without treating those indexes as byte offsets. + let mut position = 0; let mut offsets = Vec::with_capacity(instructions.len() + 1); for instruction in instructions { - offsets.push(u16::try_from(bytes.position()).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "JVM method exceeds the bytecode offset limit", - ) - })?); - instruction.to_bytes(&mut bytes).map_err(|error| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("could not measure JVM instruction: {error}"), - ) - })?; + offsets + .push(u16::try_from(position).map_err(|e| constant_pool_error("bytecode offset", e))?); + position += jvm_compiler_core::jvm::encoding::instruction_size_at(instruction, position); } - offsets.push(u16::try_from(bytes.position()).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "JVM method exceeds the bytecode offset limit", - ) - })?); + offsets.push(u16::try_from(position).map_err(|e| constant_pool_error("bytecode size", e))?); Ok(offsets) } @@ -197,6 +184,23 @@ pub(crate) fn remap_attribute( indexes: &impl ConstantIndexes, ) -> io::Result<()> { match attribute { + Attribute::InnerClasses { + name_index, + classes, + } => { + *name_index = indexes.remap(*name_index)?; + for class in classes { + for index in [ + &mut class.class_info_index, + &mut class.outer_class_info_index, + &mut class.name_index, + ] { + if *index != 0 { + *index = indexes.remap(*index)?; + } + } + } + } Attribute::Code { name_index, code, @@ -205,12 +209,18 @@ pub(crate) fn remap_attribute( .. } => { *name_index = remapped_constant_index(*name_index, indexes)?; - let old_byte_offsets = instruction_byte_offsets(code)?; + let old_byte_offsets = attributes + .iter() + .any(|a| matches!(a, Attribute::LocalVariableTable { .. })) + .then(|| instruction_byte_offsets(code)) + .transpose()?; for instruction in code.iter_mut() { remap_instruction(instruction, indexes)?; } - let new_byte_offsets = instruction_byte_offsets(code)?; - remap_local_variable_ranges(attributes, &old_byte_offsets, &new_byte_offsets)?; + if let Some(old_byte_offsets) = old_byte_offsets { + let new_byte_offsets = instruction_byte_offsets(code)?; + remap_local_variable_ranges(attributes, &old_byte_offsets, &new_byte_offsets)?; + } for exception in exception_table { if exception.catch_type != 0 { exception.catch_type = remapped_constant_index(exception.catch_type, indexes)?; diff --git a/java-linker/src/split.rs b/java-linker/src/split.rs index 2c5bd656..805f0386 100644 --- a/java-linker/src/split.rs +++ b/java-linker/src/split.rs @@ -7,7 +7,9 @@ pub(crate) type Relocations = HashMap) -> bool { class.class_name().is_ok_and(|name| { let name = name.to_string(); - name.contains("/mono/Mono_") || name.contains("/mono/MonoBucket_") + name.contains("/mono/Mono_") + || name.contains("/mono/MonoBucket_") + || jvm_compiler_core::classfile::names::codec_owner(&name) }) && class.fields.is_empty() && class.interfaces.is_empty() && !class.access_flags.contains(ClassAccessFlags::INTERFACE) @@ -30,62 +32,7 @@ pub(crate) fn eligible(class: &ClassFile<'_>) -> bool { }) } -fn compact(source: &ClassFile<'static>, methods: &[usize], owner: &str) -> io::Result { - let mut pool = ConstantPool::default(); - let this_class = pool - .add_class(owner) - .map_err(|e| constant_pool_error("split class name", e))?; - let super_class = pool - .add_class("java/lang/Object") - .map_err(|e| constant_pool_error("split superclass", e))?; - let mut constants = constant_pool_index(&pool); - let mut bootstrap = Vec::new(); - let indexes = ConstantImporter::new(source, &mut pool, &mut constants, &mut bootstrap); - let methods = methods - .iter() - .map(|&i| { - let mut method = source.methods[i].clone(); - method.name_index = indexes.remap(method.name_index)?; - method.descriptor_index = indexes.remap(method.descriptor_index)?; - // References from sibling pieces must still be able to call helpers. - method - .access_flags - .remove(MethodAccessFlags::PRIVATE | MethodAccessFlags::PROTECTED); - method.access_flags.insert(MethodAccessFlags::PUBLIC); - for attribute in &mut method.attributes { - remap_attribute(attribute, &indexes)?; - } - Ok(method) - }) - .collect::>>()?; - drop(indexes); - let mut attributes = Vec::new(); - if !bootstrap.is_empty() { - let name_index = pool - .add_utf8("BootstrapMethods") - .map_err(|e| constant_pool_error("split bootstrap name", e))?; - attributes.push(Attribute::BootstrapMethods { - name_index, - methods: bootstrap, - }); - } - let class = ClassFile { - version: source.version.clone(), - constant_pool: pool, - access_flags: ClassAccessFlags::PUBLIC - | ClassAccessFlags::SUPER - | ClassAccessFlags::SYNTHETIC, - this_class, - super_class, - methods, - attributes, - ..Default::default() - }; - Ok(ClassInfo { - jar_entry_name: format!("{owner}.class"), - data: serialize_class_file(&class)?, - }) -} +pub(crate) use crate::compact::class as compact; pub(crate) fn holders<'a>( sources: impl IntoIterator>, @@ -143,6 +90,12 @@ pub(crate) fn holders<'a>( } } if let Some(owner) = owner { + if jvm_compiler_core::classfile::names::codec_owner(&owner.to_string()) { + // Keep reflective codec lookup on the original owner after moving method bodies. + let mut root = class_file_from_data(&output[0].data)?; + crate::split_bridges::retain_codec_surface(&mut root, &moves)?; + output[0].data = serialize_class_file(&root)?; + } relocations.insert(owner, moves); } Ok(output) diff --git a/java-linker/src/split_bridges.rs b/java-linker/src/split_bridges.rs new file mode 100644 index 00000000..0de32bf2 --- /dev/null +++ b/java-linker/src/split_bridges.rs @@ -0,0 +1,78 @@ +//! Preserve the runtime's reflective codec protocol after overflow splitting. +use crate::*; +use ristretto_classfile::{BaseType, FieldType, Method, MethodAccessFlags}; + +pub(crate) fn retain_codec_surface( + root: &mut ClassFile<'static>, + moves: &HashMap<(JavaString, JavaString), String>, +) -> io::Result<()> { + let error = |e| constant_pool_error("split codec bridge", e); + let mut methods = moves.iter().collect::>(); + methods.sort_unstable_by(|a, b| a.0.cmp(b.0)); + for ((name, descriptor), target) in methods { + let (params, result) = FieldType::parse_method_descriptor(descriptor).map_err(error)?; + let mut code = Vec::with_capacity(params.len() + 2); + let mut slots = 0_u16; + for param in params { + let (load, width) = match param { + FieldType::Base(BaseType::Long) => (Instruction::Lload(slots as u8), 2), + FieldType::Base(BaseType::Double) => (Instruction::Dload(slots as u8), 2), + FieldType::Base(BaseType::Float) => (Instruction::Fload(slots as u8), 1), + FieldType::Base(_) => (Instruction::Iload(slots as u8), 1), + FieldType::Object(_) | FieldType::Array(_) => (Instruction::Aload(slots as u8), 1), + }; + slots += width; + if slots > 255 { + return Err(io::Error::other("codec bridge exceeds JVM argument limit")); + } + code.push(load); + } + let owner = root.constant_pool.add_class(target).map_err(error)?; + let name_index = root + .constant_pool + .add(Constant::Utf8(name.clone().into())) + .map_err(error)?; + let descriptor_index = root + .constant_pool + .add(Constant::Utf8(descriptor.clone().into())) + .map_err(error)?; + let name_and_type_index = root + .constant_pool + .add(Constant::NameAndType { + name_index, + descriptor_index, + }) + .map_err(error)?; + let call = root + .constant_pool + .add(Constant::MethodRef { + class_index: owner, + name_and_type_index, + }) + .map_err(error)?; + code.push(Instruction::Invokestatic(call)); + let (ret, width) = match result { + None => (Instruction::Return, 0), + Some(FieldType::Base(BaseType::Long)) => (Instruction::Lreturn, 2), + Some(FieldType::Base(BaseType::Double)) => (Instruction::Dreturn, 2), + Some(FieldType::Base(BaseType::Float)) => (Instruction::Freturn, 1), + Some(FieldType::Base(_)) => (Instruction::Ireturn, 1), + Some(FieldType::Object(_) | FieldType::Array(_)) => (Instruction::Areturn, 1), + }; + code.push(ret); + root.methods.push(Method { + access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, + name_index, + descriptor_index, + attributes: vec![Attribute::Code { + name_index: root.constant_pool.add_utf8("Code").map_err(error)?, + max_stack: slots.max(width), + max_locals: slots, + code, + exception_table: Vec::new(), + attributes: Vec::new(), + }], + }); + } + Ok(()) +} diff --git a/java-linker/src/split_tests.rs b/java-linker/src/split_tests.rs index 86374236..7fc1bf82 100644 --- a/java-linker/src/split_tests.rs +++ b/java-linker/src/split_tests.rs @@ -1,3 +1,4 @@ +use crate::test_support::run_jar; use crate::*; use ristretto_classfile::{Method, MethodAccessFlags, ReferenceKind, Version}; @@ -104,16 +105,7 @@ fn oversized_holder_is_split_and_calls_and_handles_follow() { paths.push(path.to_string_lossy().into_owned()); let output = temp.path().join("output.jar"); pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); - let result = std::process::Command::new("java") - .args(["-Xverify:all", "-jar"]) - .arg(&output) - .output() - .expect("Java is required for classfile verification"); - assert!( - result.status.success(), - "{}", - String::from_utf8_lossy(&result.stderr) - ); + run_jar(&output); let mut jar = ZipArchive::new(fs::File::open(output).unwrap()).unwrap(); let mut names = HashSet::default(); let mut target = None; @@ -169,3 +161,80 @@ fn overflow_does_not_move_public_or_stateful_classes() { class.methods[0].name_index = class.constant_pool.add_utf8("").unwrap(); assert!(!split::eligible(&class)); } + +#[test] +fn split_codecs_keep_reflective_lookup_and_all_argument_widths() { + let owner = "test/Codecs_overflow"; + let mut first = class_file_from_data(&fragment(owner, 0).data).unwrap(); + first.methods.push(Method { + access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, + name_index: first.constant_pool.add_utf8("mixed").unwrap(), + descriptor_index: first + .constant_pool + .add_utf8("(IJFDLjava/lang/Object;[I)D") + .unwrap(), + attributes: vec![Attribute::Code { + name_index: first.constant_pool.add_utf8("Code").unwrap(), + max_stack: 2, + max_locals: 8, + code: vec![Instruction::Dload(4), Instruction::Dreturn], + exception_table: Vec::new(), + attributes: Vec::new(), + }], + }); + let temp = tempfile::tempdir().unwrap(); + let fragments = [ + serialize_class_file(&first).unwrap(), + fragment(owner, 100).data, + ]; + let paths = fragments + .iter() + .enumerate() + .map(|(i, bytes)| { + let path = temp.path().join(format!("{i}.class")); + fs::write(&path, bytes).unwrap(); + path.to_string_lossy().into_owned() + }) + .collect::>(); + let output = temp.path().join("codecs.jar"); + pipeline::link(&paths, &[], &[], &[], output.to_str().unwrap()).unwrap(); + let check = temp.path().join("Check.java"); + fs::write( + &check, + r#" +public class Check { + public static void main(String[] args) throws Exception { + Class codec = Class.forName("test.Codecs_overflow"); + codec.getMethod("f0").invoke(null); + codec.getMethod("f150").invoke(null); + Object result = codec.getMethod("mixed", int.class, long.class, float.class, + double.class, Object.class, int[].class) + .invoke(null, 1, 2L, 3.0f, 7.25, new Object(), new int[1]); + if (!result.equals(7.25)) throw new AssertionError(result); + } +} +"#, + ) + .unwrap(); + let compile = std::process::Command::new("javac") + .arg(&check) + .output() + .unwrap(); + assert!( + compile.status.success(), + "{}", + String::from_utf8_lossy(&compile.stderr) + ); + let classpath = std::env::join_paths([temp.path(), output.as_path()]).unwrap(); + let execution = std::process::Command::new("java") + .args(["-Xverify:all", "-cp"]) + .arg(classpath) + .arg("Check") + .output() + .unwrap(); + assert!( + execution.status.success(), + "{}", + String::from_utf8_lossy(&execution.stderr) + ); +} diff --git a/java-linker/src/test_support.rs b/java-linker/src/test_support.rs new file mode 100644 index 00000000..2d7f7d57 --- /dev/null +++ b/java-linker/src/test_support.rs @@ -0,0 +1,77 @@ +use crate::*; +use jvm_compiler_core::classfile::summary; + +pub(crate) fn holder(name: &str, private: bool) -> ClassFile<'static> { + let mut class = ClassFile { + version: ristretto_classfile::Version::Java8 { minor: 0 }, + ..Default::default() + }; + class.this_class = class.constant_pool.add_class(name).unwrap(); + class.super_class = class.constant_pool.add_class("java/lang/Object").unwrap(); + class.access_flags = ClassAccessFlags::PUBLIC | ClassAccessFlags::SUPER; + class.attributes.push(Attribute::SourceFile { + name_index: class.constant_pool.add_utf8("SourceFile").unwrap(), + source_file_index: class.constant_pool.add_utf8("main.rs").unwrap(), + }); + if private { + class.attributes.push(Attribute::Unknown { + name_index: class + .constant_pool + .add_utf8(summary::PRIVATE_ATTRIBUTE) + .unwrap(), + info: Vec::new(), + }); + } + class +} + +pub(crate) fn body( + class: &mut ClassFile<'static>, + name: &str, + descriptor: &str, + code: Vec, +) { + class.methods.push(ristretto_classfile::Method { + access_flags: ristretto_classfile::MethodAccessFlags::PUBLIC + | ristretto_classfile::MethodAccessFlags::STATIC, + name_index: class.constant_pool.add_utf8(name).unwrap(), + descriptor_index: class.constant_pool.add_utf8(descriptor).unwrap(), + attributes: vec![Attribute::Code { + name_index: class.constant_pool.add_utf8("Code").unwrap(), + max_stack: 2, + max_locals: 1, + code, + exception_table: Vec::new(), + attributes: Vec::new(), + }], + }); +} + +pub(crate) fn write_classes( + directory: &Path, + classes: impl IntoIterator>, +) -> Vec { + classes + .into_iter() + .enumerate() + .map(|(index, class)| { + let path = directory.join(format!("{index}.class")); + fs::write(&path, serialize_class_file(&class).unwrap()).unwrap(); + path.to_string_lossy().into_owned() + }) + .collect() +} + +pub(crate) fn run_jar(path: &Path) -> std::process::Output { + let output = std::process::Command::new("java") + .args(["-Xverify:all", "-jar"]) + .arg(path) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output +} diff --git a/java-linker/src/tests.rs b/java-linker/src/tests.rs index 6e3902d8..8e24a0bc 100644 --- a/java-linker/src/tests.rs +++ b/java-linker/src/tests.rs @@ -267,6 +267,15 @@ fn recognizes_msvc_output_argument_case_insensitively() { assert_eq!(msvc_output_path("/DEBUG"), None); } +#[test] +fn measures_short_branch_near_end_of_large_method_without_encoding_its_target() { + let mut code = vec![Instruction::Iinc_w(0, 1); 7_000]; + // The target is three bytes ahead. Treating its instruction index as a byte offset exceeds the branch limit. + code.extend([Instruction::Goto(7_001), Instruction::Return]); + let offsets = instruction_byte_offsets(&code).unwrap(); + assert_eq!(&offsets[7_000..], &[42_000, 42_003, 42_004]); +} + #[test] fn remaps_local_variable_ranges_when_ldc_widens() { let old_offsets = instruction_byte_offsets(&[Instruction::Ldc(1), Instruction::Return]) @@ -559,6 +568,60 @@ fn bundle_fragment(class: &[u8]) -> Vec { out } +#[test] +fn binary_resources_survive_archives_and_reject_conflicting_payloads() { + use jvm_compiler_core::classfile::resources::{BUNDLE_PREFIX, Resource}; + let directory = tempdir().unwrap(); + let resource = Resource::new(vec![0, 255, 128, 17]); + let name = format!("{BUNDLE_PREFIX}{}", resource.name); + let bundle = |payload: &[u8]| { + let class = bundle_fragment(&abstract_class_with_method("value")); + let mut out = CLASS_BUNDLE_MAGIC.to_vec(); + out.extend((name.len() as u32).to_le_bytes()); + out.extend((payload.len() as u64).to_le_bytes()); + out.extend(name.as_bytes()); + out.extend(payload); + out.extend_from_slice(&class[CLASS_BUNDLE_MAGIC.len()..]); + out + }; + let mut archive = b"!\n".to_vec(); + ar_member(&mut archive, "one.jvmbundle/", &bundle(&resource.bytes)); + ar_member(&mut archive, "two.jvmbundle/", &bundle(&resource.bytes)); + let path = directory.path().join("input.rlib"); + std::fs::write(&path, archive).unwrap(); + let output = directory.path().join("output.jar"); + crate::pipeline::link( + &[], + &[], + &[path.to_str().unwrap().into()], + &[], + output.to_str().unwrap(), + ) + .unwrap(); + let previous = std::fs::read(&output).unwrap(); + let mut jar = ZipArchive::new(File::open(&output).unwrap()).unwrap(); + let mut bytes = Vec::new(); + jar.by_name(&resource.name) + .unwrap() + .read_to_end(&mut bytes) + .unwrap(); + assert_eq!(bytes, resource.bytes); + assert_eq!(jar.len(), 3); + drop(jar); + let bad = directory.path().join("bad.jvmbundle"); + std::fs::write(&bad, bundle(&[1, 2, 3, 4])).unwrap(); + let error = crate::pipeline::link( + &[], + &[bad.to_str().unwrap().into()], + &[path.to_str().unwrap().into()], + &[], + output.to_str().unwrap(), + ) + .unwrap_err(); + assert!(error.to_string().contains("conflicting binary constant")); + assert_eq!(std::fs::read(output).unwrap(), previous); +} + fn ar_member(out: &mut Vec, name: &str, bytes: &[u8]) { out.extend( format!( @@ -640,8 +703,12 @@ fn borrowed_metadata_matches_full_reader_and_rejects_truncation() { assert_eq!( summary::read(&bytes).unwrap(), summary::Summary { + private: false, + opaque_reflection: false, + method_demands: false, name: name.into(), - has_main: true + has_main: true, + carrier: None, } ); for length in 0..bytes.len() { diff --git a/runtime/src/ArrayMemoryCodec.java b/runtime/src/ArrayMemoryCodec.java new file mode 100644 index 00000000..5d3a34c7 --- /dev/null +++ b/runtime/src/ArrayMemoryCodec.java @@ -0,0 +1,63 @@ +package org.rustlang.runtime; + +/** Shared little-endian codecs for primitive arrays, independent of Rust length. */ +final class ArrayMemoryCodec { + private ArrayMemoryCodec() { } + + static int elementSize(Class type) { + if (type == byte.class || type == boolean.class) return 1; + if (type == short.class || type == char.class) return 2; + if (type == int.class || type == float.class) return 4; + if (type == long.class || type == double.class) return 8; + throw new IllegalArgumentException("not a primitive array element: " + type); + } + + static void write(Object source, byte[] target, int offset) { + if (source instanceof byte[]) { + byte[] values = (byte[]) source; + System.arraycopy(values, 0, target, offset, values.length); + } else if (source instanceof boolean[]) { + for (boolean value : (boolean[]) source) target[offset++] = (byte) (value ? 1 : 0); + } else if (source instanceof short[]) { + for (short value : (short[]) source) { MemoryBytes.write(target, offset, 2, value); offset += 2; } + } else if (source instanceof char[]) { + for (char value : (char[]) source) { MemoryBytes.write(target, offset, 2, value); offset += 2; } + } else if (source instanceof int[]) { + for (int value : (int[]) source) { MemoryBytes.write(target, offset, 4, value); offset += 4; } + } else if (source instanceof float[]) { + for (float value : (float[]) source) { MemoryBytes.write(target, offset, 4, Float.floatToRawIntBits(value)); offset += 4; } + } else if (source instanceof long[]) { + for (long value : (long[]) source) { MemoryBytes.write(target, offset, 8, value); offset += 8; } + } else if (source instanceof double[]) { + for (double value : (double[]) source) { MemoryBytes.write(target, offset, 8, Double.doubleToRawLongBits(value)); offset += 8; } + } else throw new IllegalArgumentException("not a primitive array"); + } + + static void read(byte[] source, int offset, Object target) { + if (target instanceof byte[]) { + byte[] values = (byte[]) target; + System.arraycopy(source, offset, values, 0, values.length); + } else if (target instanceof boolean[]) { + boolean[] values = (boolean[]) target; + for (int i = 0; i < values.length; i++) values[i] = source[offset++] != 0; + } else if (target instanceof short[]) { + short[] values = (short[]) target; + for (int i = 0; i < values.length; i++, offset += 2) values[i] = (short) MemoryBytes.read(source, offset, 2); + } else if (target instanceof char[]) { + char[] values = (char[]) target; + for (int i = 0; i < values.length; i++, offset += 2) values[i] = (char) MemoryBytes.read(source, offset, 2); + } else if (target instanceof int[]) { + int[] values = (int[]) target; + for (int i = 0; i < values.length; i++, offset += 4) values[i] = (int) MemoryBytes.read(source, offset, 4); + } else if (target instanceof float[]) { + float[] values = (float[]) target; + for (int i = 0; i < values.length; i++, offset += 4) values[i] = Float.intBitsToFloat((int) MemoryBytes.read(source, offset, 4)); + } else if (target instanceof long[]) { + long[] values = (long[]) target; + for (int i = 0; i < values.length; i++, offset += 8) values[i] = MemoryBytes.read(source, offset, 8); + } else if (target instanceof double[]) { + double[] values = (double[]) target; + for (int i = 0; i < values.length; i++, offset += 8) values[i] = Double.longBitsToDouble(MemoryBytes.read(source, offset, 8)); + } else throw new IllegalArgumentException("not a primitive array"); + } +} diff --git a/runtime/src/BorrowedFieldPath.java b/runtime/src/BorrowedFieldPath.java new file mode 100644 index 00000000..d30867c3 --- /dev/null +++ b/runtime/src/BorrowedFieldPath.java @@ -0,0 +1,40 @@ +package org.rustlang.runtime; + +/** One allocation-owned projection recipe, shared by every reference to a field. */ +final class BorrowedFieldPath { + final int offset, size; + final boolean view; + final String codec; + final RustField[] fields; + + BorrowedFieldPath(Class type, int offset, int size, String path, boolean view, String codec) + throws ReflectiveOperationException { + this.offset = offset; + this.size = size; + this.view = view; + this.codec = codec; + String[] names = path.split("/"); + fields = new RustField[names.length]; + for (int i = 0; i < names.length; i++) { + fields[i] = RustField.find(type, names[i]); + type = fields[i].getType(); + } + } + + RustField field() { return fields[fields.length - 1]; } + + private Object owner(Object root) throws IllegalAccessException { + for (int i = 0; i < fields.length - 1; i++) root = fields[i].get(root); + return root; + } + + Object read(Object root, long[] metadata) { + try { return field().borrowedParts(owner(root), metadata); } + catch (IllegalAccessException error) { throw new IllegalStateException(error); } + } + + void write(Object root, Object backing, long offset, long length) { + try { field().setBorrowedParts(owner(root), backing, offset, length); } + catch (IllegalAccessException error) { throw new IllegalStateException(error); } + } +} diff --git a/runtime/src/BorrowedStorage.java b/runtime/src/BorrowedStorage.java new file mode 100644 index 00000000..23f32169 --- /dev/null +++ b/runtime/src/BorrowedStorage.java @@ -0,0 +1,47 @@ +package org.rustlang.runtime; + +/** Stores local reference components until a boundary is needed. + * After materialization, Cell.value holds the value shared by all aliases. + */ +final class BorrowedStorage extends Storage { + private Object root; + private long offset, length; + private int plan; + final boolean view; + boolean split; + + BorrowedStorage(Object value, int size, String codec, long metadata, String layout, boolean view) { + super(value, size, codec, metadata, layout); + this.view = view; + } + + void store(Object root, long offset, long length, int plan) { + this.root = root; + this.offset = offset; + this.length = length; + this.plan = plan; + value = null; + split = true; + } + + Object read(long[] metadata, boolean view) { + if (split) { + metadata[0] = offset; + if (view) metadata[1] = length; + return root; + } + metadata[0] = view ? RustField.viewStart((SliceView) value) : 0; + if (view) metadata[1] = RustField.viewLength((SliceView) value); + return view ? RustField.viewRoot((SliceView) value) : value; + } + + @Override void materialize() { + if (!split) return; + if (plan < 0) value = plan == -2 + ? new Utf8View(root, Math.toIntExact(offset), length) + : new SliceView(root, Math.toIntExact(offset), length); + else value = root == null && offset == 0 ? null : Pointer.addressFromParts(root, offset, plan); + split = false; + root = null; + } +} diff --git a/runtime/src/CodecCalls.java b/runtime/src/CodecCalls.java new file mode 100644 index 00000000..cb6e1d60 --- /dev/null +++ b/runtime/src/CodecCalls.java @@ -0,0 +1,74 @@ +package org.rustlang.runtime; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; + +/** One adapter class per operation, shared by every exact Rust layout. */ +final class CodecCalls { + private CodecCalls() {} + + interface Encoder { byte[] encode(Object value); } + interface RangeEncoder { void encode(Object value, byte[] bytes, int offset); } + interface Decoder { Object decode(byte[] bytes); } + interface RangeDecoder { Object decode(byte[] bytes, int offset); } + interface Binder { void bind(Pointer pointer, Object value); } + + static Encoder wholeEncoder(RangeEncoder operation, int byteSize) { + return value -> { + byte[] bytes = new byte[byteSize]; + operation.encode(value, bytes, 0); + return bytes; + }; + } + + static Decoder wholeDecoder(RangeDecoder operation) { + return bytes -> operation.decode(bytes, 0); + } + + static Encoder encoder(MethodHandle target) { + MethodHandle call = target.asType(MethodType.methodType(byte[].class, Object.class)); + return value -> { + try { return (byte[]) call.invokeExact(value); } + catch (Throwable error) { throw failure(error); } + }; + } + + static RangeEncoder rangeEncoder(MethodHandle target) { + MethodHandle call = target.asType(MethodType.methodType( + void.class, Object.class, byte[].class, int.class)); + return (value, bytes, offset) -> { + try { call.invokeExact(value, bytes, offset); } + catch (Throwable error) { throw failure(error); } + }; + } + + static Decoder decoder(MethodHandle target) { + MethodHandle call = target.asType(MethodType.methodType(Object.class, byte[].class)); + return bytes -> { + try { return (Object) call.invokeExact(bytes); } + catch (Throwable error) { throw failure(error); } + }; + } + + static RangeDecoder rangeDecoder(MethodHandle target) { + MethodHandle call = target.asType(MethodType.methodType(Object.class, byte[].class, int.class)); + return (bytes, offset) -> { + try { return (Object) call.invokeExact(bytes, offset); } + catch (Throwable error) { throw failure(error); } + }; + } + + static Binder binder(MethodHandle target) { + MethodHandle call = target.asType(MethodType.methodType(void.class, Pointer.class, Object.class)); + return (pointer, value) -> { + try { call.invokeExact(pointer, value); } + catch (Throwable error) { throw failure(error); } + }; + } + + private static RuntimeException failure(Throwable error) { + if (error instanceof RuntimeException) return (RuntimeException) error; + if (error instanceof Error) throw (Error) error; + return new IllegalStateException("Rust codec threw a checked exception", error); + } +} diff --git a/runtime/src/ConstantData.java b/runtime/src/ConstantData.java new file mode 100644 index 00000000..1788f615 --- /dev/null +++ b/runtime/src/ConstantData.java @@ -0,0 +1,133 @@ +package org.rustlang.runtime; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Lazy binary constants. Payload sharing never shares the destination allocation. */ +public final class ConstantData { + private ConstantData() {} + + private static final int MAX_BYTES = 4 * 1024 * 1024; + private static final int MAX_ENTRIES = 256; + // Entries retain only the per-owner map, never its Class or ClassLoader. + private static final ClassValue> OWNERS = new ClassValue>() { + protected Map computeValue(Class owner) { return new HashMap<>(); } + }; + private static final Map BLOCKS = new LinkedHashMap(16, 0.75f, true); + private static int retainedBytes; + + private static final class Block { + final Map owner; + final String name; + final ByteBuffer data; + Block(Map owner, String name, byte[] bytes) { + this.owner = owner; + this.name = name; + data = ByteBuffer.wrap(bytes).asReadOnlyBuffer(); + } + } + + private static ByteBuffer block(String name, Class owner, int size) { + if (size < 0 || size > 65536 || !name.startsWith("META-INF/rust-data/") + || name.indexOf("..") >= 0) throw new IllegalArgumentException("constant block"); + Map owned = OWNERS.get(owner); + synchronized (BLOCKS) { + Block cached = owned.get(name); + if (cached != null) { + if (cached.data.capacity() != size) throw new IllegalStateException("constant block size changed"); + BLOCKS.get(cached); + return cached.data; + } + } + // Resource I/O does not serialize unrelated constructor threads. + Block loaded = new Block(owned, name, readBlock(name, owner, size)); + synchronized (BLOCKS) { + Block existing = owned.get(name); + if (existing != null) return existing.data; + owned.put(name, loaded); + BLOCKS.put(loaded, Boolean.TRUE); + retainedBytes += size; + while (retainedBytes > MAX_BYTES || BLOCKS.size() > MAX_ENTRIES) { + java.util.Iterator oldest = BLOCKS.keySet().iterator(); + Block evicted = oldest.next(); + retainedBytes -= evicted.data.capacity(); + evicted.owner.remove(evicted.name); + oldest.remove(); + } + } + return loaded.data; + } + + /** Independent cursor over immutable, bounded, loader-local constant data. */ + public static ByteBuffer buffer(String name, Class owner, int size) { + return block(name, owner, size).duplicate().order(ByteOrder.LITTLE_ENDIAN); + } + + private static byte[] readBlock(String name, Class owner, int size) { + byte[] bytes = new byte[size]; + try (InputStream input = owner.getResourceAsStream("/" + name)) { + if (input == null) throw new IOException("missing Rust constant " + name); + for (int i = 0; i < size;) { + int read = input.read(bytes, i, size - i); + if (read < 0) throw new EOFException("truncated Rust constant"); + if (read == 0) { + int value = input.read(); + if (value < 0) throw new EOFException("truncated Rust constant"); + bytes[i++] = (byte) value; + } else i += read; + } + if (input.read() != -1) throw new IOException("oversized Rust constant"); + } catch (IOException error) { + throw new IllegalStateException("could not load Rust constant " + name, error); + } + return bytes; + } + + public static void readArray(Object target, ByteBuffer input) { + readArray(target, 0, java.lang.reflect.Array.getLength(target), input); + } + + public static void fill(Object target, int start, int count, String name, Class owner) { + int width; + if (target instanceof byte[] || target instanceof boolean[]) width = 1; + else if (target instanceof short[] || target instanceof char[]) width = 2; + else if (target instanceof int[] || target instanceof float[]) width = 4; + else if (target instanceof long[] || target instanceof double[]) width = 8; + else throw new IllegalArgumentException("not a primitive constant array"); + check(start, count, java.lang.reflect.Array.getLength(target)); + ByteBuffer input = buffer(name, owner, Math.multiplyExact(count, width)); + readArray(target, start, count, input); + } + + private static void readArray(Object target, int start, int count, ByteBuffer input) { + int end = start + count; + if (target instanceof byte[]) input.get((byte[]) target, start, count); + else if (target instanceof short[]) { + short[] a = (short[]) target; for (int i = start; i < end; i++) a[i] = input.getShort(); + } else if (target instanceof char[]) { + char[] a = (char[]) target; for (int i = start; i < end; i++) a[i] = input.getChar(); + } else if (target instanceof int[]) { + int[] a = (int[]) target; for (int i = start; i < end; i++) a[i] = input.getInt(); + } else if (target instanceof long[]) { + long[] a = (long[]) target; for (int i = start; i < end; i++) a[i] = input.getLong(); + } else if (target instanceof float[]) { + float[] a = (float[]) target; for (int i = start; i < end; i++) a[i] = input.getFloat(); + } else if (target instanceof double[]) { + double[] a = (double[]) target; for (int i = start; i < end; i++) a[i] = input.getDouble(); + } else if (target instanceof boolean[]) { + boolean[] a = (boolean[]) target; for (int i = start; i < end; i++) a[i] = input.get() != 0; + } else throw new IllegalArgumentException("not a primitive constant array"); + } + + private static void check(int start, int count, int length) { + if (start < 0 || count < 0 || start > length - count) { + throw new IndexOutOfBoundsException("constant array range"); + } + } +} diff --git a/runtime/src/FieldProjections.java b/runtime/src/FieldProjections.java new file mode 100644 index 00000000..ef27a9c5 --- /dev/null +++ b/runtime/src/FieldProjections.java @@ -0,0 +1,45 @@ +package org.rustlang.runtime; + +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.ref.WeakReference; + +/** Reuse repeated field borrows without keeping their owners alive. */ +public final class FieldProjections { + private static final MethodHandle FIELD; + private volatile WeakReference previous; + + static { + try { + FIELD = MethodHandles.lookup().findVirtual(FieldProjections.class, "field", + MethodType.methodType(Pointer.class, Object.class, String.class, long.class, String.class)); + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + + private FieldProjections() {} + + public static CallSite bootstrap(MethodHandles.Lookup lookup, String name, MethodType type) { + return new ConstantCallSite(FIELD.bindTo(new FieldProjections()).asType(type)); + } + + public static CallSite project(MethodHandles.Lookup lookup, String name, MethodType type, + Class owner, String field, long offset, long size, String codec) + throws ReflectiveOperationException { + return new ConstantCallSite(Pointer.fieldProjection(owner, field, offset, size, + codec.isEmpty() ? null : codec).asType(type)); + } + + private Pointer field(Object owner, String name, long size, String codec) { + WeakReference reference = previous; + Pointer pointer = reference == null ? null : reference.get(); + if (Pointer.cachedFieldMatches(pointer, owner, name, size, codec)) return pointer; + pointer = Pointer.field(owner, name, size, codec); + previous = new WeakReference<>(pointer); + return pointer; + } +} diff --git a/runtime/src/FunctionCallSite.java b/runtime/src/FunctionCallSite.java new file mode 100644 index 00000000..5eb946e2 --- /dev/null +++ b/runtime/src/FunctionCallSite.java @@ -0,0 +1,63 @@ +package org.rustlang.runtime; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; +import java.lang.invoke.WrongMethodTypeException; +import java.util.ArrayList; + +/** A few executed targets become direct calls without a class per cold target. */ +public final class FunctionCallSite extends MutableCallSite { + private static final int LIMIT = 4; + private static final MethodHandle SAME, LINK; + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + SAME = lookup.findStatic(FunctionCallSite.class, "same", + MethodType.methodType(boolean.class, MethodHandle.class, MethodHandle.class)); + LINK = lookup.findVirtual(FunctionCallSite.class, "link", + MethodType.methodType(Object.class, MethodHandle.class, Object[].class)); + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + private final ArrayList targets = new ArrayList<>(LIMIT); + private final MethodHandle linker, generic; + + private FunctionCallSite(MethodType type) { + super(type); + generic = MethodHandles.exactInvoker(type.dropParameterTypes(0, 1)); + linker = LINK.bindTo(this).asCollector(Object[].class, type.parameterCount() - 1).asType(type); + setTarget(linker); + } + + public static CallSite bootstrap(MethodHandles.Lookup lookup, String name, MethodType type) { + return new FunctionCallSite(type); + } + + private static boolean same(MethodHandle expected, MethodHandle actual) { + return expected == actual; + } + + private Object link(MethodHandle implementation, Object[] arguments) throws Throwable { + if (!implementation.type().equals(type().dropParameterTypes(0, 1))) { + throw new WrongMethodTypeException("Rust function ABI does not match its target"); + } + synchronized (this) { + if (targets.size() < LIMIT && !targets.contains(implementation)) { + targets.add(implementation); + MethodHandle chain = targets.size() == LIMIT ? generic : linker; + for (MethodHandle target : targets) { + chain = MethodHandles.guardWithTest(SAME.bindTo(target), + MethodHandles.dropArguments(target, 0, MethodHandle.class), chain); + } + setTarget(chain); + MutableCallSite.syncAll(new MutableCallSite[] {this}); + } + } + // Cached calls use the exact ABI and do not allocate an argument array. + return implementation.invokeWithArguments(arguments); + } +} diff --git a/runtime/src/FunctionPointers.java b/runtime/src/FunctionPointers.java index 66c7e1b2..89aa4291 100644 --- a/runtime/src/FunctionPointers.java +++ b/runtime/src/FunctionPointers.java @@ -23,6 +23,13 @@ protected Map computeValue(Class owner) { private FunctionPointers() {} + private static final ClassValue> CONSTANTS = + new ClassValue>() { + protected Map computeValue(Class signature) { + return new IdentityHashMap<>(); + } + }; + private static Object target(Class owner, String method) { Map methods = TARGETS.get(owner); synchronized (methods) { @@ -37,6 +44,10 @@ static Object identity(Object value) { return identity; } } + if (value instanceof MethodHandle) { + MethodHandleInfo info = MethodHandles.lookup().revealDirect((MethodHandle) value); + return target(info.getDeclaringClass(), info.getName() + ":" + info.getMethodType().toMethodDescriptorString()); + } if (value instanceof StaticFunctionPointer) { String name = ((StaticFunctionPointer) value).functionPointerIdentity(); int separator = name.indexOf("::"); @@ -51,18 +62,86 @@ static Object identity(Object value) { return value; } + /** Share the invocation class per exact ABI instead of per code target. */ + private static final ClassValue FACTORIES = new ClassValue() { + protected Factory computeValue(Class signature) { return new Factory(); } + }; + + private static final class Factory { + private MethodHandle factory; + private boolean initialized; + + synchronized MethodHandle get(MethodHandles.Lookup lookup, Class signature, + MethodType type) throws Throwable { + if (!initialized) { + if (signature.getName().startsWith("org.rustlang.runtime.FnPtr_")) { + try { + MethodHandle bridge = lookup.findStatic(signature, "$rust$invoke", + type.insertParameterTypes(0, MethodHandle.class)); + factory = LambdaMetafactory.metafactory(lookup, "call", + MethodType.methodType(signature, MethodHandle.class), type, + bridge, type).getTarget().asType( + MethodType.methodType(Object.class, MethodHandle.class)); + } catch (NoSuchMethodException absent) { + // Foreign/older interfaces retain their own Java SAM contract. + } + } + initialized = true; + } + return factory; + } + } + + public static Object bind(MethodHandles.Lookup lookup, Class signature, MethodHandle implementation) { + try { + return constant(lookup, signature, implementation); + } catch (RuntimeException | Error failure) { + throw failure; + } catch (Throwable failure) { + throw new IllegalStateException("could not link constant Rust function", failure); + } + } + + private static Object constant(MethodHandles.Lookup lookup, Class signature, + MethodHandle implementation) throws Throwable { + MethodType type = implementation.type(); + MethodHandleInfo info = lookup.revealDirect(implementation); + Object identity = target(info.getDeclaringClass(), + info.getName() + ":" + info.getMethodType().toMethodDescriptorString()); + Map constants = CONSTANTS.get(signature); + synchronized (constants) { + Object function = constants.get(identity); + if (function != null) return function; + MethodHandle factory = FACTORIES.get(signature).get(lookup, signature, type); + if (factory != null) { + function = (Object) factory.invokeExact(implementation); + } else { + CallSite site = LambdaMetafactory.metafactory(lookup, "call", + MethodType.methodType(signature), type, implementation, type); + function = site.getTarget().invoke(); + } + synchronized (IDENTITIES) { IDENTITIES.put(function, identity); } + constants.put(identity, function); + return function; + } + } + public static CallSite metafactory(MethodHandles.Lookup lookup, String name, MethodType factoryType, MethodType interfaceType, MethodHandle implementation, MethodType instantiatedType) throws Throwable { + if (!name.equals("call") || factoryType.parameterCount() != 0) { + return LambdaMetafactory.metafactory(lookup, name, factoryType, + interfaceType, implementation, instantiatedType); + } + // Executed reifications use direct JVM calls. Constant vtable entries use + // shared adapters in bind() to avoid hidden classes for unused targets. CallSite site = LambdaMetafactory.metafactory(lookup, name, factoryType, interfaceType, implementation, instantiatedType); Object function = site.getTarget().invoke(); MethodHandleInfo info = lookup.revealDirect(implementation); Object identity = target(info.getDeclaringClass(), info.getName() + ":" + info.getMethodType().toMethodDescriptorString()); - synchronized (IDENTITIES) { - IDENTITIES.put(function, identity); - } + synchronized (IDENTITIES) { IDENTITIES.put(function, identity); } return new ConstantCallSite(MethodHandles.constant(factoryType.returnType(), function)); } } diff --git a/runtime/src/Heap.java b/runtime/src/Heap.java new file mode 100644 index 00000000..839ec296 --- /dev/null +++ b/runtime/src/Heap.java @@ -0,0 +1,40 @@ +package org.rustlang.runtime; + +/** Keeps allocator storage separate from address offsets. */ +public final class Heap { + private Heap() {} + + public static Object allocate(long byteCount, long alignment) { + try { + int size = Pointer.checkedArrayLength(byteCount); + int checkedAlignment = Pointer.checkedAlignment(alignment); + byte[] bytes = new byte[size]; + Pointer.registerHeapAllocation(bytes, checkedAlignment); + return bytes; + } catch (IllegalArgumentException | ArithmeticException | OutOfMemoryError failure) { + return null; + } + } + + public static Object reallocate(Object root, long offset, long oldByteCount, + long alignment, long newByteCount) { + if (root == null) throw new NullPointerException("Rust realloc requires a non-null pointer"); + byte[] destination = null; + try { + int oldSize = Pointer.checkedArrayLength(oldByteCount); + int newSize = Pointer.checkedArrayLength(newByteCount); + destination = (byte[]) allocate(newSize, alignment); + if (destination == null) return null; + Pointer.copyHeapAllocation(root, offset, destination, Math.min(oldSize, newSize)); + deallocate(root, offset); + return destination; + } catch (IllegalArgumentException | ArithmeticException | OutOfMemoryError failure) { + if (destination != null) Pointer.releaseHeapAllocation(destination); + return null; + } + } + + public static void deallocate(Object root, long offset) { + Pointer.releaseHeapAllocation(root); + } +} diff --git a/runtime/src/KotlinFutureInterop.java b/runtime/src/KotlinFutureInterop.java index 37574e54..9446a0fb 100644 --- a/runtime/src/KotlinFutureInterop.java +++ b/runtime/src/KotlinFutureInterop.java @@ -1,7 +1,6 @@ package org.rustlang.runtime; import java.lang.reflect.Constructor; -import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -77,7 +76,6 @@ private static final class PollAdapter { private final Constructor rawWakerConstructor; private final Constructor wakerConstructor; private final Constructor contextConstructor; - private final Constructor pinConstructor; private final Method resume; private final Pointer vtablePointer; private final int futureSize; @@ -102,10 +100,7 @@ private PollAdapter( "org.rustlang.core.task.wake.Waker", true, loader); Class contextClass = Class.forName( "org.rustlang.core.task.wake.Context", true, loader); - Class pinClass = Class.forName( - "org.rustlang.core.pin.Pin_MutRef" + futureClass.getSimpleName(), - true, - loader); + resume = findResumeMethod(futureClass); rawWakerConstructor = rawWakerClass.getConstructor(Pointer.class, Pointer.class); Constructor vtableConstructor = constructorWithArity(rawWakerVTableClass, 4); @@ -163,8 +158,6 @@ public Object invoke(Object proxy, Method method, Object[] args) { wakerConstructor = wakerClass.getConstructor(rawWakerClass); contextConstructor = rustConstructor(contextClass); - pinConstructor = pinClass.getConstructor(Pointer.class); - resume = findResumeMethod(futureClass, pinClass); this.futureSize = futureSize; this.futureCodec = futureCodec; this.futureAlignment = futureAlignment; @@ -190,14 +183,13 @@ private Object poll(Object future, Runnable wake) { Object context = contextConstructor.newInstance(contextArguments); Pointer futurePointer = Pointer.receiverCellAligned( future, futureSize, futureCodec, futureAlignment); - Object pin = pinConstructor.newInstance(futurePointer); - Object result = resume.invoke(null, pin, Pointer.cell(context)); + Object result = resume.invoke(null, futurePointer, Pointer.cell(context)); - if (result.getClass().getName().endsWith("$Pending")) { + if (result.getClass().getSimpleName().equals("Pending")) { return PENDING; } try { - Field output = result.getClass().getField("value"); + RustField output = RustField.find(result.getClass(), "value"); return output.get(result); } catch (NoSuchFieldException noOutput) { return UNIT; @@ -235,8 +227,7 @@ private static Constructor rustConstructor(Class type) throws NoSuchMethod private static Object defaultRustValue(Class type) throws ReflectiveOperationException { if (type.isInterface()) { - Class none = Class.forName( - type.getName() + "$None", true, type.getClassLoader()); + Class none = RustClasses.nested(type, "None"); return none.getConstructor().newInstance(); } try { @@ -247,17 +238,18 @@ private static Object defaultRustValue(Class type) throws ReflectiveOperation } } - private static Method findResumeMethod(Class futureClass, Class pinClass) + private static Method findResumeMethod(Class futureClass) throws ReflectiveOperationException { + // Body is an executable namespace, never a shared storage carrier. Class bodyClass = Class.forName( futureClass.getName() + "$Body", true, futureClass.getClassLoader()); for (Method method : bodyClass.getMethods()) { Class[] parameters = method.getParameterTypes(); if (Modifier.isStatic(method.getModifiers()) && parameters.length == 2 - && parameters[0] == pinClass + && parameters[0] == Pointer.class && parameters[1] == Pointer.class - && method.getReturnType().getName().contains(".task.poll.Poll_")) { + && RustClasses.hasNested(method.getReturnType(), "Ready", "Pending")) { return method; } } diff --git a/runtime/src/MemoryBytes.java b/runtime/src/MemoryBytes.java index bbfd6a21..1b4147a3 100644 --- a/runtime/src/MemoryBytes.java +++ b/runtime/src/MemoryBytes.java @@ -1,9 +1,32 @@ package org.rustlang.runtime; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.nio.ByteOrder; + /** Little-endian scalar access shared by exact-layout codecs and pointer views. */ public final class MemoryBytes { + private static final MethodHandle READ_LONG = reader(long[].class); + private static final MethodHandle READ_INT = reader(int[].class); + private MemoryBytes() {} + // Java 9 adds unaligned scalar views. Link once and retain the Java 8 fallback. + private static MethodHandle reader(Class type) { + try { + Class handle = Class.forName("java.lang.invoke.VarHandle"); + Class mode = Class.forName("java.lang.invoke.VarHandle$AccessMode"); + Object view = MethodHandles.class.getMethod("byteArrayViewVarHandle", Class.class, ByteOrder.class) + .invoke(null, type, ByteOrder.LITTLE_ENDIAN); + return (MethodHandle) handle.getMethod("toMethodHandle", mode) + .invoke(view, mode.getField("GET").get(null)); + } catch (ClassNotFoundException | NoSuchMethodException unavailable) { + return null; + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + /** Copy an embedded Latin-1 byte chunk without allocating a temporary array. */ public static void fillConstant(byte[] target, int offset, String chunk) { for (int index = 0; index < chunk.length(); index++) { @@ -11,13 +34,43 @@ public static void fillConstant(byte[] target, int offset, String chunk) { } } + public static void clear(byte[] bytes, int offset, int size) { + java.util.Arrays.fill(bytes, offset, offset + size, (byte) 0); + } + public static void write(byte[] bytes, int offset, int size, long value) { + switch (size) { + case 8: + writeInt(bytes, offset, (int) value); + writeInt(bytes, offset + 4, (int) (value >>> 32)); + return; + case 4: + writeInt(bytes, offset, (int) value); + return; + case 2: + bytes[offset] = (byte) value; + bytes[offset + 1] = (byte) (value >>> 8); + return; + case 1: + bytes[offset] = (byte) value; + return; + } for (int index = 0; index < size; index++) { bytes[offset + index] = (byte) (value >>> (index * 8)); } } public static long read(byte[] bytes, int offset, int size) { + switch (size) { + case 8: + return readLong(bytes, offset); + case 4: + return readInt(bytes, offset) & 0xffffffffL; + case 2: + return (bytes[offset] & 255L) | ((bytes[offset + 1] & 255L) << 8); + case 1: + return bytes[offset] & 255L; + } long value = 0; for (int index = 0; index < size; index++) { value |= ((long) bytes[offset + index] & 0xffL) << (index * 8); @@ -25,6 +78,40 @@ public static long read(byte[] bytes, int offset, int size) { return value; } + private static int readInt(byte[] bytes, int offset) { + if (READ_INT != null) { + try { + return (int) READ_INT.invokeExact(bytes, offset); + } catch (RuntimeException | Error error) { + throw error; + } catch (Throwable error) { + throw new IllegalStateException(error); + } + } + return (bytes[offset] & 255) | ((bytes[offset + 1] & 255) << 8) + | ((bytes[offset + 2] & 255) << 16) | (bytes[offset + 3] << 24); + } + + private static long readLong(byte[] bytes, int offset) { + if (READ_LONG != null) { + try { + return (long) READ_LONG.invokeExact(bytes, offset); + } catch (RuntimeException | Error error) { + throw error; + } catch (Throwable error) { + throw new IllegalStateException(error); + } + } + return (readInt(bytes, offset) & 0xffffffffL) | ((long) readInt(bytes, offset + 4) << 32); + } + + private static void writeInt(byte[] bytes, int offset, int value) { + bytes[offset] = (byte) value; + bytes[offset + 1] = (byte) (value >>> 8); + bytes[offset + 2] = (byte) (value >>> 16); + bytes[offset + 3] = (byte) (value >>> 24); + } + public static void writeI128(byte[] bytes, int offset, I128 value) { write(bytes, offset, 8, value.low); write(bytes, offset + 8, 8, value.high); diff --git a/runtime/src/MemoryCodec.java b/runtime/src/MemoryCodec.java new file mode 100644 index 00000000..3c1e8d73 --- /dev/null +++ b/runtime/src/MemoryCodec.java @@ -0,0 +1,202 @@ +package org.rustlang.runtime; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Field; + +/** Resolves codec operations for an exact Rust layout only when needed. */ +final class MemoryCodec { + final Class encodeParameterType; + final int arrayElementSize; + final String arrayElementCodec; + private final Class owner; + private final String key; + private final int byteSize; + private final Object[] operations = new Object[5]; + private volatile int initialized; + + static MemoryCodec load(String recipe, ClassLoader loader) throws ReflectiveOperationException { + if (recipe.startsWith("@zero-sized:")) { + return new MemoryCodec(Class.forName(recipe.substring("@zero-sized:".length()).replace('/', '.'), false, loader)); + } + int first = recipe.indexOf('#'), second = recipe.indexOf('#', first + 1); + if (first < 1 || second <= first + 1) throw new IllegalArgumentException("invalid pointer codec recipe: " + recipe); + Class owner = Class.forName(recipe.substring(0, first).replace('/', '.'), false, loader); + int third = recipe.indexOf('#', second + 1); + String descriptor = third < 0 ? recipe.substring(second + 1) : recipe.substring(second + 1, third); + int size = third < 0 ? -1 : Integer.parseInt(recipe.substring(third + 1)); + if (third >= 0 && size < 0) throw new IllegalArgumentException("invalid codec byte size: " + recipe); + Class value = MethodType.fromMethodDescriptorString("()" + descriptor, loader).returnType(); + return new MemoryCodec(owner, recipe.substring(first + 1, second), value, size); + } + + private MemoryCodec(Class value) throws ReflectiveOperationException { + owner = null; + key = null; + byteSize = 0; + encodeParameterType = value; + arrayElementSize = -1; + arrayElementCodec = null; + MethodHandle constructor = MethodHandles.publicLookup().unreflectConstructor(value.getConstructor()) + .asType(MethodType.methodType(Object.class)); + operations[0] = (CodecCalls.Encoder) ignored -> new byte[0]; + operations[2] = (CodecCalls.Decoder) ignored -> { + try { return (Object) constructor.invokeExact(); } + catch (Throwable error) { throw failure(error); } + }; + initialized = 31; + } + + private MemoryCodec(Class owner, String key, Class value, int layoutSize) throws ReflectiveOperationException { + this.owner = owner; + this.key = key; + byteSize = layoutSize; + encodeParameterType = value; + if (owner == ArrayMemoryCodec.class && key.equals("array")) { + Class element = value.getComponentType(); + arrayElementSize = ArrayMemoryCodec.elementSize(element); + arrayElementCodec = null; + if (layoutSize < 0 || layoutSize % arrayElementSize != 0) + throw new IllegalArgumentException("invalid primitive array codec size"); + int length = layoutSize / arrayElementSize; + CodecCalls.RangeEncoder encoder = (array, bytes, offset) -> { + if (!value.isInstance(array) || java.lang.reflect.Array.getLength(array) != length) + throw new IllegalArgumentException("primitive array codec layout mismatch"); + Pointer.encodeArrayMemory(array, bytes, offset, arrayElementSize, null); + }; + CodecCalls.RangeDecoder decoder = (bytes, offset) -> { + Object array = java.lang.reflect.Array.newInstance(element, length); + Pointer.decodeArrayMemory(bytes, offset, array, arrayElementSize, null); + return array; + }; + operations[0] = CodecCalls.wholeEncoder(encoder, layoutSize); + operations[1] = encoder; + operations[2] = CodecCalls.wholeDecoder(decoder); + operations[3] = decoder; + initialized = 31; + return; + } + // Resolve array metadata only for array layouts. + if (value.isArray()) { + MethodHandle size = optional("s$", MethodType.methodType(int.class)); + MethodHandle codec = optional("c$", MethodType.methodType(String.class)); + try { + arrayElementSize = size == null ? -1 : (int) size.invokeExact(); + arrayElementCodec = codec == null ? null : (String) codec.invokeExact(); + } catch (Throwable error) { throw failure(error); } + } else { + arrayElementSize = -1; + arrayElementCodec = null; + } + } + + CodecCalls.Encoder encode() { return (CodecCalls.Encoder) operation(0); } + CodecCalls.RangeEncoder encodeAt() { return (CodecCalls.RangeEncoder) operation(1); } + CodecCalls.Decoder decode() { return (CodecCalls.Decoder) operation(2); } + CodecCalls.RangeDecoder decodeAt() { return (CodecCalls.RangeDecoder) operation(3); } + CodecCalls.Binder bind() { return (CodecCalls.Binder) operation(4); } + + MethodHandle directRangeDecoder() throws IllegalAccessException { + return owner == null || owner == ArrayMemoryCodec.class ? null + : optional("a$", MethodType.methodType(encodeParameterType, byte[].class, int.class)); + } + + private Object operation(int index) { + if ((initialized & (1 << index)) == 0) initialize(index); + return operations[index]; + } + + private synchronized void initialize(int index) { + int flag = 1 << index; + if ((initialized & flag) != 0) return; + try { + MethodHandle target; + switch (index) { + case 0: + target = optional("e$", MethodType.methodType(byte[].class, encodeParameterType)); + if (target != null) operations[index] = CodecCalls.encoder(target); + else if (byteSize >= 0 && encodeAt() != null) { + operations[index] = CodecCalls.wholeEncoder(encodeAt(), byteSize); + } else throw new NoSuchMethodException("codec has no encoder"); + break; + case 1: + target = optional("w$", MethodType.methodType(void.class, encodeParameterType, byte[].class, int.class)); + operations[index] = target == null ? null : CodecCalls.rangeEncoder(target); + break; + case 2: + target = optional("d$", MethodType.methodType(encodeParameterType, byte[].class)); + if (target != null) operations[index] = CodecCalls.decoder(target); + else if (decodeAt() != null) operations[index] = CodecCalls.wholeDecoder(decodeAt()); + else throw new NoSuchMethodException("codec has no decoder"); + break; + case 3: + target = optional("a$", MethodType.methodType(encodeParameterType, byte[].class, int.class)); + operations[index] = target == null ? null : CodecCalls.rangeDecoder(target); + break; + case 4: + target = optional("b$", MethodType.methodType(void.class, Pointer.class, encodeParameterType)); + operations[index] = target == null ? null : CodecCalls.binder(target); + break; + default: throw new AssertionError(index); + } + // Release publication also records a missing optional operation. + initialized |= flag; + } catch (ReflectiveOperationException error) { + throw new IllegalStateException("could not resolve Rust codec " + owner.getName() + "#" + key, error); + } + } + + private MethodHandle optional(String operation, MethodType type) throws IllegalAccessException { + try { return MethodHandles.lookup().findStatic(owner, operation + key, type); } + catch (NoSuchMethodException missing) { return null; } + } + + boolean isArrayCodecFor(Object value) { + return arrayElementSize > 0 && value != null && value.getClass().isArray(); + } + + // Resolve union fields only when a union byte operation needs them. + private static final ClassValue UNIONS = new ClassValue() { + protected UnionAccess computeValue(Class type) { return new UnionAccess(type); } + }; + + byte[] directUnionBytes(Object value) { + if (value == null || !encodeParameterType.isInstance(value)) return null; + MethodHandle read = UNIONS.get(encodeParameterType).bytes; + if (read == null) return null; + try { return (byte[]) read.invokeExact(value); } + catch (Throwable error) { throw failure(error); } + } + + Object[] directUnionObjects(Object value) { + if (value == null || !encodeParameterType.isInstance(value)) return null; + MethodHandle read = UNIONS.get(encodeParameterType).objects; + if (read == null) return null; + try { return (Object[]) read.invokeExact(value); } + catch (Throwable error) { throw failure(error); } + } + + private static final class UnionAccess { + final MethodHandle bytes, objects; + UnionAccess(Class type) { + MethodHandle b = null, o = null; + try { + Field bytes = type.getField("_bytes"), objects = type.getField("_objects"); + if (bytes.getType() == byte[].class && objects.getType() == Object[].class) { + b = MethodHandles.lookup().unreflectGetter(bytes).asType(MethodType.methodType(byte[].class, Object.class)); + o = MethodHandles.lookup().unreflectGetter(objects).asType(MethodType.methodType(Object[].class, Object.class)); + } + } catch (NoSuchFieldException ignored) { + } catch (IllegalAccessException error) { throw failure(error); } + this.bytes = b; + this.objects = o; + } + } + + private static RuntimeException failure(Throwable error) { + if (error instanceof RuntimeException) return (RuntimeException) error; + if (error instanceof Error) throw (Error) error; + return new IllegalStateException("Rust memory codec failed", error); + } +} diff --git a/runtime/src/OwnedReads.java b/runtime/src/OwnedReads.java new file mode 100644 index 00000000..5e5c847b --- /dev/null +++ b/runtime/src/OwnedReads.java @@ -0,0 +1,71 @@ +package org.rustlang.runtime; + +import java.lang.invoke.CallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MutableCallSite; + +/** Links owned reads to exact decoders while retaining general storage fallbacks. */ +public final class OwnedReads { + private static final MethodHandle FALLBACK, MATCHES, BYTES, OFFSET, GENERIC; + + static { + try { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + FALLBACK = lookup.findVirtual(Site.class, "read", + MethodType.methodType(Object.class, Object.class, int.class)); + MATCHES = lookup.findStatic(Pointer.class, "ownedSliceMatches", + MethodType.methodType(boolean.class, String.class, int.class, Object.class)); + BYTES = lookup.findStatic(Pointer.class, "ownedSliceBytes", + MethodType.methodType(byte[].class, Object.class)); + OFFSET = lookup.findStatic(Pointer.class, "ownedSliceOffset", + MethodType.methodType(int.class, Object.class, int.class)); + GENERIC = lookup.findStatic(Pointer.class, "sliceGetObjectCopy", + MethodType.methodType(Object.class, Object.class, int.class, String.class)); + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + + private OwnedReads() {} + + public static CallSite bootstrap(MethodHandles.Lookup lookup, String name, MethodType type) { + return new Site(type); + } + + private static final class Site extends MutableCallSite { + private final MethodHandle generic; + private int depth; + + Site(MethodType type) { + super(type); + String target = type.returnType().getName(); + generic = MethodHandles.insertArguments(GENERIC, 2, target).asType(type); + setTarget(FALLBACK.bindTo(this).asType(type)); + } + + Object read(Object backing, int index) throws Throwable { + link(backing); + return getTarget().invoke(backing, index); + } + + private synchronized void link(Object backing) throws Throwable { + String recipe = Pointer.ownedSliceRecipe(backing, type().returnType()); + MethodHandle decoder = recipe == null ? null + : Pointer.codecPlan(recipe).directRangeDecoder(); + if (decoder == null || depth >= 4) { + setTarget(generic); + } else { + int size = Pointer.ownedSliceSize(backing); + MethodHandle decode = MethodHandles.filterArguments(decoder, 0, BYTES); + decode = MethodHandles.collectArguments(decode, 1, OFFSET); + decode = MethodHandles.permuteArguments(decode, type(), 0, 0, 1); + MethodHandle test = MethodHandles.insertArguments(MATCHES, 0, recipe, size); + setTarget(MethodHandles.guardWithTest(test, decode, getTarget())); + depth++; + } + MutableCallSite.syncAll(new MutableCallSite[] {this}); + } + } +} diff --git a/runtime/src/Pointer.java b/runtime/src/Pointer.java index 6ca57649..a8dff93a 100644 --- a/runtime/src/Pointer.java +++ b/runtime/src/Pointer.java @@ -1,12 +1,9 @@ package org.rustlang.runtime; -import java.lang.invoke.CallSite; -import java.lang.invoke.LambdaMetafactory; import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; import java.lang.reflect.Array; import java.lang.reflect.Constructor; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -15,7 +12,6 @@ import java.lang.invoke.MethodType; import java.math.BigInteger; import java.nio.charset.StandardCharsets; -import java.util.AbstractMap; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -30,10 +26,7 @@ import java.util.concurrent.atomic.AtomicLongArray; public final class Pointer { - private static final String RELATIVE_POINTER_ELEMENT_OFFSET_SUFFIX = - "$rcj$elementOffset"; - private static final String RELATIVE_POINTER_BYTE_OFFSET_SUFFIX = - "$rcj$byteOffset"; + private static Object arrayGet(Object array, int index) { if (array instanceof byte[]) { @@ -131,44 +124,6 @@ public static void dropRustValue(Object value) { } } - private static void dropTraitPointer(Object pointer) { - if (pointer == null) { - return; - } - if (pointer instanceof TraitObjectCarrier) { - dropRustValue(((TraitObjectCarrier) pointer).rustTraitObjectPayload()); - return; - } - if (pointer instanceof Pointer) { - Object payload = ((Pointer) pointer).getObject(); - if (payload != pointer) { - dropRustValue(payload); - } - return; - } - if (isSliceViewCarrierType(pointer.getClass())) { - try { - SliceAccess access = SLICE_ACCESSES.get(pointer.getClass()); - Object backing = access.array(pointer); - int offset = access.offset(pointer); - if (backing instanceof Pointer) { - Object payload = ((Pointer) backing).add(offset).directCellValueOrSelf(); - if (payload != backing) { - dropRustValue(payload); - } - return; - } - if (backing != null && backing.getClass().isArray()) { - dropRustValue(arrayGet(backing, offset)); - return; - } - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("invalid Rust trait-object pointer", error); - } - } - dropRustValue(pointer); - } - public static boolean catchUnwind(Object tryFunction, Pointer data, Object catchFunction) { try { invokeRustFunction(tryFunction, data); @@ -193,11 +148,28 @@ static Object invokeRustFunction(Object function, Object... arguments) { } Method target = null; for (Method method : function.getClass().getMethods()) { - if (method.getName().equals("call") - && method.getParameterTypes().length == arguments.length) { - target = method; - break; - } + if (!method.getName().equals("call") || Modifier.isStatic(method.getModifiers())) continue; + Class[] params = method.getParameterTypes(); + if (params.length == arguments.length) { target = method; break; } + Object[] expanded = new Object[params.length]; + int next = 0; + for (Object argument : arguments) { + if (next >= params.length) { next = -1; break; } + if ((argument == null || argument instanceof Pointer) + && params[next] == Object.class && next + 1 < params.length + && params[next + 1] == long.class) { + expanded[next++] = argument; + expanded[next++] = 0L; + } else if ((argument == null || isSliceViewCarrierType(argument.getClass())) + && params[next] == Object.class && next + 2 < params.length + && params[next + 1] == int.class && params[next + 2] == long.class) { + SliceView view = (SliceView) argument; + expanded[next++] = view == null ? null : view.array; + expanded[next++] = view == null ? 0 : view.offset; + expanded[next++] = view == null ? 0L : view.rustLength; + } else { expanded[next++] = argument; } + } + if (next == params.length) { target = method; arguments = expanded; break; } } if (target == null) { throw new IllegalArgumentException( @@ -262,7 +234,7 @@ private static void rethrowUnchecked(Throwable failure) { new TreeMap<>(); private static final ReferenceQueue ALLOCATION_RANGE_QUEUE = new ReferenceQueue<>(); - private static final ConcurrentHashMap CODEC_METHODS = + private static final ConcurrentHashMap CODEC_METHODS = new ConcurrentHashMap<>(); private static final ThreadLocal RECENT_CODEC_PLANS = new ThreadLocal() { @@ -282,10 +254,6 @@ protected ResolvedClassCache initialValue() { new ConcurrentHashMap<>(); private static final Map SHARED_CONSTANT_ARRAYS = new IdentityHashMap<>(); - private static final ConcurrentHashMap DROP_METHOD_HANDLES = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap DROP_FIELDS_METHOD_HANDLES = - new ConcurrentHashMap<>(); private static final ConcurrentHashMap CODEC_DESCRIPTORS = new ConcurrentHashMap<>(); private static final ConcurrentHashMap BINARY_CLASS_NAMES = @@ -300,13 +268,6 @@ protected ResolvedClassCache initialValue() { new ConcurrentHashMap<>(); private static final Map>> RESOLVED_CLASSES = new IdentityHashMap<>(); - private static final ClassValue> INSTANCE_FIELDS = - new ClassValue>() { - @Override - protected ConcurrentHashMap computeValue(Class type) { - return new ConcurrentHashMap<>(); - } - }; private static final ClassValue> FIELD_ACCESSORS = new ClassValue>() { @Override @@ -314,37 +275,7 @@ protected ConcurrentHashMap computeValue(Class type) { return new ConcurrentHashMap<>(); } }; - private static final ClassValue SLICE_ACCESSES = - new ClassValue() { - @Override - protected SliceAccess computeValue(Class type) { - return new SliceAccess(type); - } - }; - private static final ClassValue PUBLIC_INSTANCE_FIELDS = - new ClassValue() { - @Override - protected Field[] computeValue(Class type) { - Field[] all = type.getFields(); - int count = 0; - for (Field field : all) { - if (!Modifier.isStatic(field.getModifiers()) - && !field.isSynthetic()) { - count++; - } - } - Field[] fields = new Field[count]; - int index = 0; - for (Field field : all) { - if (!Modifier.isStatic(field.getModifiers()) - && !field.isSynthetic()) { - field.setAccessible(true); - fields[index++] = field; - } - } - return fields; - } - }; + private static final ClassValue PUBLIC_INSTANCE_FIELDS = RustField.ALL; private static final ClassValue> PUBLIC_CONSTRUCTORS_BY_ARITY = new ClassValue>() { @Override @@ -364,36 +295,6 @@ protected Map computeValue(Class type) { return constructors; } }; - private static final ClassValue> SLICE_VIEW_CONSTRUCTORS = - new ClassValue>() { - @Override - protected Constructor computeValue(Class type) { - try { - Constructor constructor = - type.getConstructor(Object.class, int.class, int.class); - constructor.setAccessible(true); - return constructor; - } catch (NoSuchMethodException error) { - throw new IllegalStateException( - "Rust slice view has no array/offset/length constructor", error); - } - } - }; - private static final ClassValue> LONG_SLICE_VIEW_CONSTRUCTORS = - new ClassValue>() { - @Override - protected Constructor computeValue(Class type) { - try { - Constructor constructor = - type.getConstructor(Object.class, int.class, long.class); - constructor.setAccessible(true); - return constructor; - } catch (NoSuchMethodException error) { - throw new IllegalStateException( - "Rust slice view has no long-length constructor", error); - } - } - }; private static final ClassValue RUST_FUNCTION_POINTER_TYPES = new ClassValue() { @Override @@ -411,7 +312,7 @@ protected Boolean computeValue(Class type) { new ClassValue() { @Override protected ManagedCopyPlan computeValue(Class type) { - Field[] fields = PUBLIC_INSTANCE_FIELDS.get(type); + RustField[] fields = PUBLIC_INSTANCE_FIELDS.get(type); ManagedFieldPlan[] fieldPlans = new ManagedFieldPlan[fields.length]; for (int index = 0; index < fields.length; index++) { try { @@ -446,7 +347,8 @@ protected ManagedCopyPlan computeValue(Class type) { private static final Map TRAIT_METADATA_MARKERS = new HashMap<>(); private static final ConcurrentHashMap TRAIT_METADATA_INFO = new ConcurrentHashMap<>(); - private static final int STATE_STRIPE_COUNT = 256; + private static final int STATE_STRIPE_BITS = 8; + private static final int STATE_STRIPE_COUNT = 1 << STATE_STRIPE_BITS; private static final int LAZY_ARRAY_REPEAT_THRESHOLD = 2; private static final int REPEATED_ARRAY_FILTER_WORDS = 1 << 16; private static final long IDENTITY_FILTER_REBUILD_MARKS = 1L << 18; @@ -454,8 +356,7 @@ protected ManagedCopyPlan computeValue(Class type) { private static final class RebuildableIdentityFilter { private final int wordCount; private final long rebuildMarks; - private volatile AtomicLongArray primary; - private volatile AtomicLongArray secondary; + private volatile IdentityFilterState state; private final AtomicLong marks = new AtomicLong(); private final AtomicInteger rebuilding = new AtomicInteger(); @@ -466,7 +367,18 @@ private RebuildableIdentityFilter() { private RebuildableIdentityFilter(int wordCount, long rebuildMarks) { this.wordCount = wordCount; this.rebuildMarks = rebuildMarks; - primary = new AtomicLongArray(wordCount); + } + + private synchronized IdentityFilterState initialize() { + if (state == null) state = new IdentityFilterState(new AtomicLongArray(wordCount), null); + return state; + } + } + private static final class IdentityFilterState { + private final AtomicLongArray primary, secondary; + private IdentityFilterState(AtomicLongArray primary, AtomicLongArray secondary) { + this.primary = primary; + this.secondary = secondary; } } private static final AtomicLongArray REPEATED_ARRAY_FILTER = @@ -476,7 +388,7 @@ private RebuildableIdentityFilter(int wordCount, long rebuildMarks) { private static final RebuildableIdentityFilter MEMORY_VIEW_FILTER = new RebuildableIdentityFilter(); private static final RebuildableIdentityFilter MEMORY_VIEW_ORIGIN_FILTER = - new RebuildableIdentityFilter(); + new RebuildableIdentityFilter(REPEATED_ARRAY_FILTER_WORDS, MEMORY_VIEW_ORIGIN_FILTER_REBUILD_MARKS); private static final AtomicLongArray MEMORY_VIEW_EPOCHS = new AtomicLongArray(STATE_STRIPE_COUNT); private static final RebuildableIdentityFilter ENCODED_REFERENCE_FILTER = @@ -485,8 +397,8 @@ private RebuildableIdentityFilter(int wordCount, long rebuildMarks) { new RebuildableIdentityFilter(1 << 20, 1L << 22); private static final AtomicLongArray FIELD_CELL_FILTER = new AtomicLongArray(REPEATED_ARRAY_FILTER_WORDS); - private static final AtomicLongArray SHARED_CONSTANT_ARRAY_FILTER = - new AtomicLongArray(REPEATED_ARRAY_FILTER_WORDS); + private static final RebuildableIdentityFilter SHARED_CONSTANT_ARRAY_FILTER = + new RebuildableIdentityFilter(); private static final Map>[] STRUCTURAL_VIEWS = createWeakMapStripes(); private static final Map>[] MEMORY_VIEWS = @@ -582,7 +494,6 @@ private static void retainEncodedReference(Object owner, Object referencedAlloca || owner == referencedAllocation) { return; } - markIdentityFilter(ENCODED_REFERENCE_FILTER, owner); Map stripe = stateStripe(ENCODED_REFERENCES, owner); synchronized (stripe) { @@ -599,8 +510,9 @@ private static void retainEncodedReference(Object owner, Object referencedAlloca } references.allocations.put(referencedAllocation, Boolean.TRUE); } + markIdentityFilter(ENCODED_REFERENCE_FILTER, owner); } - maybeRebuildEncodedReferenceFilter(); + maybeRebuildIdentityFilter(ENCODED_REFERENCE_FILTER, ENCODED_REFERENCES); } private static void transferEncodedReferences(Object sourceOwner, Object targetOwner) { @@ -873,9 +785,11 @@ private static final class EncodedPointerState { private final String codec; private final ExposedTarget target; private final boolean targetUsesOwnerAllocation; + private final long address; private EncodedPointerState( - Object owner, int size, String codec, ExposedTarget target) { + Object owner, int size, String codec, ExposedTarget target, long address) { + this.address = address; this.size = size; this.codec = codec; targetUsesOwnerAllocation = target.allocation == owner; @@ -896,9 +810,11 @@ private static final class EncodedPointerCopy { private final int size; private final String codec; private final ExposedTarget target; + private final long address; private EncodedPointerCopy( - long offset, int size, String codec, ExposedTarget target) { + long offset, int size, String codec, ExposedTarget target, long address) { + this.address = address; this.offset = offset; this.size = size; this.codec = codec; @@ -911,25 +827,11 @@ private static void rememberEncodedPointer( long offset, int size, String codec, - Pointer pointer) { - rememberEncodedPointer( - owner, - offset, - size, - codec, - pointer == null ? null : pointer.exposedTarget()); - } - - private static void rememberEncodedPointer( - Object owner, - long offset, - int size, - String codec, - ExposedTarget target) { + ExposedTarget target, + long address) { if (owner == null || target == null || size <= 0 || codec == null) { return; } - markIdentityFilter(ENCODED_POINTER_FILTER, owner); Map> stripe = stateStripe(ENCODED_POINTERS, owner); synchronized (stripe) { @@ -941,9 +843,10 @@ private static void rememberEncodedPointer( removeOverlappingEncodedPointers(pointers, offset, size); pointers.put( offset, - new EncodedPointerState(owner, size, codec, target)); + new EncodedPointerState(owner, size, codec, target, address)); + markIdentityFilter(ENCODED_POINTER_FILTER, owner); } - maybeRebuildEncodedPointerFilter(); + maybeRebuildIdentityFilter(ENCODED_POINTER_FILTER, ENCODED_POINTERS); } private static Pointer encodedPointer( @@ -960,11 +863,10 @@ && mayBeInIdentityFilter(ENCODED_POINTER_FILTER, owner)) { } if (state != null && state.size == size + && state.address == address && codec.equals(state.codec)) { - Pointer pointer = pointerFromExposedTarget(state.target(owner)); - if (pointer.numericAddress() == address) { - return pointer; - } + return pointerFromExposedTarget(state.target, + state.targetUsesOwnerAllocation ? owner : state.target.allocation); } } return null; @@ -1005,7 +907,7 @@ private static void transferEncodedPointers( entryOffset, state.size, state.codec, - state.target(sourceOwner))); + state.target(sourceOwner), state.address)); } } if (move) { @@ -1018,7 +920,6 @@ private static void transferEncodedPointers( if (copied == null) { return; } - markIdentityFilter(ENCODED_POINTER_FILTER, targetOwner); Map> targetStripe = stateStripe(ENCODED_POINTERS, targetOwner); synchronized (targetStripe) { @@ -1038,8 +939,9 @@ private static void transferEncodedPointers( targetOwner, entry.size, entry.codec, - entry.target)); + entry.target, entry.address)); } + markIdentityFilter(ENCODED_POINTER_FILTER, targetOwner); } } @@ -1093,293 +995,6 @@ private static void discardEncodedPointers(Object owner, long offset, int size) } } - private static final class WeakIdentityMap extends AbstractMap { - private static final class OpenWeakReference extends WeakReference { - private final int identityHash; - - private OpenWeakReference(Object key, ReferenceQueue queue) { - super(key, queue); - identityHash = System.identityHashCode(key); - } - } - - private final ReferenceQueue collectedKeys = new ReferenceQueue<>(); - private OpenWeakReference[] keys; - private Object[] values; - private int size; - private int used; - private int readsUntilCleanup = 256; - - private WeakIdentityMap() { - this(16); - } - - private WeakIdentityMap(int initialCapacity) { - int capacity = 16; - while (capacity < initialCapacity) { - capacity <<= 1; - } - keys = newTable(capacity); - values = new Object[capacity]; - } - - private static OpenWeakReference[] newTable(int length) { - return new OpenWeakReference[length]; - } - - private static int tableIndex(int hash, int length) { - hash ^= hash >>> 16; - hash *= 0x7feb352d; - hash ^= hash >>> 15; - return hash & (length - 1); - } - - /** Removes an entry without leaving a tombstone in its probe chain. */ - private void deleteEntry(int deleted) { - if (values[deleted] != null) { - size--; - } - keys[deleted] = null; - values[deleted] = null; - used--; - - int mask = keys.length - 1; - for (int index = (deleted + 1) & mask; - keys[index] != null; - index = (index + 1) & mask) { - OpenWeakReference reference = keys[index]; - int home = tableIndex(reference.identityHash, keys.length); - if ((index < home && (home <= deleted || deleted <= index)) - || (home <= deleted && deleted <= index)) { - keys[deleted] = reference; - values[deleted] = values[index]; - keys[index] = null; - values[index] = null; - deleted = index; - } - } - } - - private void reset() { - keys = newTable(16); - values = new Object[16]; - size = 0; - used = 0; - readsUntilCleanup = 256; - while (collectedKeys.poll() != null) { - // Entries no longer exist after reset. - } - } - - private void discardCollectedKeys() { - OpenWeakReference collected; - while ((collected = (OpenWeakReference) collectedKeys.poll()) != null) { - int index = tableIndex(collected.identityHash, keys.length); - while (keys[index] != null) { - if (keys[index] == collected) { - deleteEntry(index); - break; - } - index = (index + 1) & (keys.length - 1); - } - } - } - - private void maybeDiscardCollectedKeys() { - if (--readsUntilCleanup == 0) { - discardCollectedKeys(); - readsUntilCleanup = 256; - } - } - - private void rehashForInsert() { - discardCollectedKeys(); - readsUntilCleanup = 256; - int newLength = - size * 4 >= keys.length * 3 - ? keys.length << 1 - : keys.length; - OpenWeakReference[] oldKeys = keys; - Object[] oldValues = values; - keys = newTable(newLength); - values = new Object[newLength]; - size = 0; - used = 0; - for (int oldIndex = 0; oldIndex < oldKeys.length; oldIndex++) { - OpenWeakReference reference = oldKeys[oldIndex]; - Object key = reference == null ? null : reference.get(); - if (key == null) { - continue; - } - int index = tableIndex(reference.identityHash, keys.length); - while (keys[index] != null) { - index = (index + 1) & (keys.length - 1); - } - keys[index] = reference; - values[index] = oldValues[oldIndex]; - size++; - used++; - } - } - - @SuppressWarnings("unchecked") - private V valueAt(int index) { - return (V) values[index]; - } - - @Override - public V get(Object key) { - maybeDiscardCollectedKeys(); - int index = tableIndex(System.identityHashCode(key), keys.length); - while (true) { - OpenWeakReference reference = keys[index]; - if (reference == null) { - return null; - } - Object live = reference.get(); - if (live == null) { - // The key can be collected after the batched queue drain; - // leave a tombstone until the next maintenance pass. - } else if (live == key) { - return valueAt(index); - } - index = (index + 1) & (keys.length - 1); - } - } - - @Override - public V put(Object key, V value) { - discardCollectedKeys(); - readsUntilCleanup = 256; - if (used * 4 >= keys.length * 3) { - rehashForInsert(); - } - int index = tableIndex(System.identityHashCode(key), keys.length); - while (true) { - OpenWeakReference reference = keys[index]; - if (reference == null) { - used++; - keys[index] = new OpenWeakReference(key, collectedKeys); - values[index] = value; - size++; - return null; - } - Object live = reference.get(); - if (live == null) { - deleteEntry(index); - continue; - } else if (live == key) { - V previous = valueAt(index); - values[index] = value; - return previous; - } - index = (index + 1) & (keys.length - 1); - } - } - - @Override - public V remove(Object key) { - discardCollectedKeys(); - readsUntilCleanup = 256; - int index = tableIndex(System.identityHashCode(key), keys.length); - while (true) { - OpenWeakReference reference = keys[index]; - if (reference == null) { - return null; - } - Object live = reference.get(); - if (live == null) { - deleteEntry(index); - continue; - } else if (live == key) { - V previous = valueAt(index); - deleteEntry(index); - if (size == 0) { - reset(); - } - return previous; - } - index = (index + 1) & (keys.length - 1); - } - } - - @Override - public int size() { - discardCollectedKeys(); - for (int index = 0; index < keys.length; ) { - OpenWeakReference reference = keys[index]; - if (reference != null && reference.get() == null) { - deleteEntry(index); - } else { - index++; - } - } - return size; - } - - @Override - public boolean isEmpty() { - discardCollectedKeys(); - if (size == 0) { - return true; - } - for (int index = 0; index < keys.length; ) { - OpenWeakReference reference = keys[index]; - if (reference == null) { - index++; - continue; - } - if (reference.get() != null) { - return false; - } - deleteEntry(index); - } - reset(); - return true; - } - - @Override - public Set> entrySet() { - discardCollectedKeys(); - Set> entries = new HashSet<>(); - for (int index = 0; index < keys.length; ) { - OpenWeakReference reference = keys[index]; - Object key = reference == null ? null : reference.get(); - if (key == null) { - if (reference != null) { - deleteEntry(index); - continue; - } - } else { - entries.add(new java.util.AbstractMap.SimpleImmutableEntry<>( - key, valueAt(index))); - } - index++; - } - return entries; - } - - private long markLiveKeys(AtomicLongArray filter) { - discardCollectedKeys(); - long count = 0; - for (int index = 0; index < keys.length; ) { - OpenWeakReference reference = keys[index]; - Object key = reference == null ? null : reference.get(); - if (key == null) { - if (reference != null) { - deleteEntry(index); - continue; - } - } else { - markIdentityFilter(filter, key); - count++; - } - index++; - } - return count; - } - } - /** Must be called while holding {@link #ALLOCATIONS}. */ private static AllocationInfo allocationInfo(Object allocation) { AllocationInfo info = ALLOCATIONS.get(allocation); @@ -1430,16 +1045,11 @@ private static void recordAlignment(Object allocation, int alignment) { } private static boolean isSliceViewType(Class type) { - return type != null && SLICE_VIEW_CLASS_NAME.equals(type.getName()); + return type == SliceView.class; } private static boolean isSliceViewCarrierType(Class type) { - for (Class current = type; current != null; current = current.getSuperclass()) { - if (SLICE_VIEW_CLASS_NAME.equals(current.getName())) { - return true; - } - } - return false; + return type != null && SliceView.class.isAssignableFrom(type); } /** @@ -1553,9 +1163,13 @@ private MemoryViewOrigin(Pointer pointer) { } private boolean matches(Pointer pointer) { + return matches(pointer, pointer.byteOffset); + } + + private boolean matches(Pointer pointer, long offset) { return allocation.get() == pointer.allocation && allocationElementSize == pointer.allocationElementSize - && byteOffset == pointer.byteOffset + && byteOffset == offset && viewSize == pointer.viewSize && java.util.Objects.equals( allocationCodecClassName, pointer.allocationCodecClassName) @@ -1580,11 +1194,11 @@ private static final class ManagedFieldPlan { private final MethodHandle getter; private final MethodHandle setter; - private ManagedFieldPlan(Field field) throws IllegalAccessException { + private ManagedFieldPlan(RustField field) throws IllegalAccessException { MethodHandles.Lookup lookup = MethodHandles.lookup(); - getter = lookup.unreflectGetter(field).asType(MethodType.methodType( + getter = field.getter().asType(MethodType.methodType( Object.class, Object.class)); - setter = lookup.unreflectSetter(field).asType(MethodType.methodType( + setter = field.setter().asType(MethodType.methodType( void.class, Object.class, Object.class)); } @@ -1622,176 +1236,20 @@ private Object newInstance(Object... arguments) throws ReflectiveOperationExcept } } - private static final class CodecPlan { - private final Class encodeParameterType; - private final CodecEncoder encode; - private final CodecDecoder decode; - private final CodecRangeDecoder decodeAt; - private final CodecBinder bind; - private final MethodHandle unionBytes; - private final MethodHandle unionObjects; - private final int arrayElementSize; - private final String arrayElementCodec; - - /** A fieldless Rust ZST needs only its concrete JVM constructor. */ - private CodecPlan(Class valueType) throws ReflectiveOperationException { - encodeParameterType = valueType; - MethodHandle constructor = MethodHandles.publicLookup() - .unreflectConstructor(valueType.getConstructor()) - .asType(MethodType.methodType(Object.class)); - encode = value -> new byte[0]; - decode = bytes -> { - try { - return (Object) constructor.invokeExact(); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException("could not construct zero-sized Rust value", error); - } - }; - bind = null; - decodeAt = null; - unionBytes = null; - unionObjects = null; - arrayElementSize = -1; - arrayElementCodec = null; - } - - private CodecPlan( - Method encode, - Method decode, - Method decodeAt, - Method bind, - Method arrayElementSize, - Method arrayElementCodec) - throws IllegalAccessException { - encodeParameterType = encode.getParameterTypes()[0]; - MethodHandles.Lookup lookup = MethodHandles.lookup(); - try { - this.encode = (CodecEncoder) - lambdaAdapter( - lookup, - "encode", - CodecEncoder.class, - MethodType.methodType(byte[].class, Object.class), - lookup.unreflect(encode), - MethodType.methodType( - byte[].class, encode.getParameterTypes()[0])); - this.decode = (CodecDecoder) - lambdaAdapter( - lookup, - "decode", - CodecDecoder.class, - MethodType.methodType(Object.class, byte[].class), - lookup.unreflect(decode), - MethodType.methodType( - decode.getReturnType(), byte[].class)); - this.decodeAt = decodeAt == null ? null : (CodecRangeDecoder) - lambdaAdapter( - lookup, - "decode", - CodecRangeDecoder.class, - MethodType.methodType(Object.class, byte[].class, int.class), - lookup.unreflect(decodeAt), - MethodType.methodType( - decodeAt.getReturnType(), byte[].class, int.class)); - this.bind = bind == null - ? null - : (CodecBinder) - lambdaAdapter( - lookup, - "bind", - CodecBinder.class, - MethodType.methodType( - void.class, Pointer.class, Object.class), - lookup.unreflect(bind), - MethodType.methodType( - void.class, - Pointer.class, - bind.getParameterTypes()[1])); - MethodHandle directUnionBytes = null; - MethodHandle directUnionObjects = null; - try { - Field bytes = encodeParameterType.getField("_bytes"); - Field objects = encodeParameterType.getField("_objects"); - if (bytes.getType() == byte[].class - && objects.getType() == Object[].class) { - directUnionBytes = lookup.unreflectGetter(bytes).asType( - MethodType.methodType(byte[].class, Object.class)); - directUnionObjects = lookup.unreflectGetter(objects).asType( - MethodType.methodType(Object[].class, Object.class)); - } - } catch (NoSuchFieldException ignored) { - // Ordinary generated aggregates use their encode/decode methods. - } - unionBytes = directUnionBytes; - unionObjects = directUnionObjects; - if (arrayElementSize == null || arrayElementCodec == null) { - this.arrayElementSize = -1; - this.arrayElementCodec = null; - } else { - this.arrayElementSize = - (int) lookup.unreflect(arrayElementSize).invokeExact(); - this.arrayElementCodec = - (String) lookup.unreflect(arrayElementCodec).invokeExact(); - } - } catch (Throwable error) { - throw new IllegalAccessException( - "could not create pointer codec call adapter: " + error); - } - } - - private byte[] directUnionBytes(Object value) { - if (unionBytes == null || value == null - || !encodeParameterType.isInstance(value)) { - return null; - } - try { - return (byte[]) unionBytes.invokeExact(value); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException( - "could not access generated Rust union storage", error); - } - } - - private Object[] directUnionObjects(Object value) { - if (unionObjects == null || value == null - || !encodeParameterType.isInstance(value)) { - return null; - } - try { - return (Object[]) unionObjects.invokeExact(value); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException( - "could not access generated Rust union references", error); - } - } - - private boolean isArrayCodecFor(Object value) { - return arrayElementSize > 0 - && value != null - && value.getClass().isArray(); - } - } - /** Two-entry per-thread cache for the codecs used by tight pointer loops. */ private static final class CodecPlanCache { private String firstName; - private CodecPlan firstPlan; + private MemoryCodec firstPlan; private String secondName; - private CodecPlan secondPlan; + private MemoryCodec secondPlan; - private CodecPlan get(String name) { + private MemoryCodec get(String name) { if (name == firstName || (firstName != null && firstName.equals(name))) { return firstPlan; } if (name == secondName || (secondName != null && secondName.equals(name))) { String previousName = firstName; - CodecPlan previousPlan = firstPlan; + MemoryCodec previousPlan = firstPlan; firstName = secondName; firstPlan = secondPlan; secondName = previousName; @@ -1801,7 +1259,7 @@ private CodecPlan get(String name) { return null; } - private void remember(String name, CodecPlan plan) { + private void remember(String name, MemoryCodec plan) { secondName = firstName; secondPlan = firstPlan; firstName = name; @@ -1849,40 +1307,6 @@ private void remember(String name, ClassLoader loader, Class resolved) { } } - private interface CodecEncoder { - byte[] encode(Object value); - } - - private interface CodecDecoder { - Object decode(byte[] bytes); - } - - private interface CodecRangeDecoder { - Object decode(byte[] bytes, int offset); - } - - private interface CodecBinder { - void bind(Pointer pointer, Object value); - } - - private static Object lambdaAdapter( - MethodHandles.Lookup lookup, - String methodName, - Class interfaceType, - MethodType erasedType, - MethodHandle implementation, - MethodType instantiatedType) - throws Throwable { - CallSite site = LambdaMetafactory.metafactory( - lookup, - methodName, - MethodType.methodType(interfaceType), - erasedType, - implementation, - instantiatedType); - return site.getTarget().invoke(); - } - private static final class RepeatedArrayState { private final Object template; @@ -1906,231 +1330,13 @@ private static ConstructorPlan constructorWithArity(Class type, int arity) { * the generated Rust value fields from the replacement object instead. */ public static void overwriteManagedObject(Object target, Object replacement) { - if (target == replacement) { - return; - } - if (target == null || replacement == null || target.getClass() != replacement.getClass()) { - throw new IllegalArgumentException("managed-object overwrite requires matching non-null classes"); - } - copyStructuralFields(replacement, target); - } - - /** Drops a Box pointee and always runs the Box deallocator during unwinding. */ - public static void dropBoxWithCleanup( - Object pointee, - Object box, - String pointeeOwner, - String pointeeMethod, - String pointeeDescriptor, - String boxOwner, - String boxMethod, - String boxDescriptor) { - Throwable pendingDropFailure = null; - try { - if (pointeeOwner.isEmpty()) { - dropTraitPointer(pointee); - } else { - invokeDropMethod(pointee, pointeeOwner, pointeeMethod, pointeeDescriptor); - } - } catch (Throwable failure) { - PanicSupport.abortIfStackOverflow(failure); - if (Boolean.getBoolean("org.rustlang.debugUnwind")) { - failure.printStackTrace(System.err); - } - pendingDropFailure = failure; - } - try { - invokeDropMethod(box, boxOwner, boxMethod, boxDescriptor); - } catch (Throwable failure) { - PanicSupport.abortIfStackOverflow(failure); - if (Boolean.getBoolean("org.rustlang.debugUnwind")) { - failure.printStackTrace(System.err); - } - if (pendingDropFailure != null) { - Runtime.getRuntime().halt(134); - } - pendingDropFailure = failure; - } - if (pendingDropFailure != null) { - rethrowUnchecked(pendingDropFailure); - } - } - - /** Runs a custom destructor and always drops the value's fields afterward. */ - public static void dropAdtWithCleanup( - Object value, - String owner, - String dropMethod, - String dropDescriptor, - String fieldsMethod) { - Throwable pendingDropFailure = null; - try { - invokeDropMethod(value, owner, dropMethod, dropDescriptor); - } catch (Throwable failure) { - PanicSupport.abortIfStackOverflow(failure); - if (Boolean.getBoolean("org.rustlang.debugUnwind")) { - failure.printStackTrace(System.err); - } - pendingDropFailure = failure; - } - try { - String key = value.getClass().getName() + '\0' + fieldsMethod; - MethodHandle handle = DROP_FIELDS_METHOD_HANDLES.get(key); - if (handle == null) { - MethodHandle resolved = MethodHandles.publicLookup().findVirtual( - value.getClass(), fieldsMethod, MethodType.methodType(void.class)); - MethodHandle previous = DROP_FIELDS_METHOD_HANDLES.putIfAbsent(key, resolved); - handle = previous == null ? resolved : previous; - } - handle.invokeWithArguments(value); - } catch (Throwable failure) { - PanicSupport.abortIfStackOverflow(failure); - if (Boolean.getBoolean("org.rustlang.debugUnwind")) { - failure.printStackTrace(System.err); - } - if (pendingDropFailure != null) { - Runtime.getRuntime().halt(134); - } - pendingDropFailure = failure; - } - if (pendingDropFailure != null) { - rethrowUnchecked(pendingDropFailure); - } - } - - private static void invokeDropMethod( - Object value, String ownerName, String methodName, String descriptor) - throws Throwable { - String key = ownerName + '\0' + methodName + '\0' + descriptor; - MethodHandle handle = DROP_METHOD_HANDLES.get(key); - if (handle == null) { - Class owner = resolvedRuntimeClass(ownerName); - MethodType methodType = - MethodType.fromMethodDescriptorString(descriptor, owner.getClassLoader()); - MethodHandle resolved = - MethodHandles.publicLookup().findStatic(owner, methodName, methodType); - MethodHandle previous = DROP_METHOD_HANDLES.putIfAbsent(key, resolved); - handle = previous == null ? resolved : previous; - } - Class parameterType = handle.type().parameterType(0); - Object argument = parameterType.isInstance(value) - ? value - : parameterType == Pointer.class && isSliceViewCarrierType(value.getClass()) - ? fromSlice(value) - : adaptStructuralField(value, parameterType); - handle.invokeWithArguments(argument); - } - - /** Runs generated Rust element drop glue over a dynamically sized slice. */ - public static void dropSlice(Object slice, String ownerClassName, String methodName) { - dropSlice( - slice, - ownerClassName, - methodName, - "(Lorg/rustlang/runtime/Pointer;)V", - -1, - null); - } - - /** Runs generated Rust element drop glue using its actual JVM carrier ABI. */ - public static void dropSlice( - Object slice, String ownerClassName, String methodName, String descriptor) { - dropSlice(slice, ownerClassName, methodName, descriptor, -1, null); - } - - /** Runs slice drop glue through the element type's logical memory view. */ - public static void dropSlice( - Object slice, - String ownerClassName, - String methodName, - String descriptor, - long elementViewSize, - String elementViewCodec) { - if (slice == null) { + if (target == replacement) { return; } - try { - Class sliceClass = slice.getClass(); - Object array = instanceField(sliceClass, "array").get(slice); - int offset = instanceField(sliceClass, "offset").getInt(slice); - int length = instanceField(sliceClass, "length").getInt(slice); - Pointer data = array instanceof Pointer - ? ((Pointer) array).sliceElementView().add(offset) - : null; - if (data == null && (array == null || !array.getClass().isArray())) { - throw new IllegalArgumentException("Rust slice drop requires array-backed storage"); - } - MethodHandle drop = null; - Throwable pendingDropFailure = null; - for (int index = 0; index < length; index++) { - Object stored = data == null ? arrayGet(array, offset + index) : null; - boolean alreadyTyped = stored != null - && elementViewSize >= 0 - && isGeneratedAggregateCodec(elementViewCodec) - && codecPlan(elementViewCodec).encodeParameterType.isInstance(stored); - Pointer element = alreadyTyped - ? Pointer.cell(stored, elementViewSize, elementViewCodec) - : data == null ? Pointer.cell(stored) : data.add(index); - if (!alreadyTyped && (elementViewSize >= 0 || elementViewCodec != null)) { - element = element.retype( - elementViewSize >= 0 ? elementViewSize : element.viewSize, - elementViewCodec); - if (elementViewSize == 0 && elementViewCodec != null) { - // `MaybeUninit` slice drop is an explicit initialized - // `T` view. Do not let the erased ZST source wrapper win - // over the target codec merely because both occupy zero bytes. - element.clearZeroSizedSourceView(); - } - } - Object managed = element.getObject(); - try { - if (managed instanceof RustDrop) { - ((RustDrop) managed).rustDrop(); - } else { - if (drop == null) { - Class owner = resolvedRuntimeClass(ownerClassName); - MethodType methodType = MethodType.fromMethodDescriptorString( - descriptor, owner.getClassLoader()); - drop = MethodHandles.publicLookup().findStatic( - owner, - methodName, - methodType); - } - Class parameterType = drop.type().parameterType(0); - Object argument = parameterType == Pointer.class - ? element - : adaptStructuralField(element.getObject(), parameterType); - drop.invokeWithArguments(argument); - } - } catch (Throwable failure) { - PanicSupport.abortIfStackOverflow(failure); - if (pendingDropFailure != null) { - Runtime.getRuntime().halt(134); - } - pendingDropFailure = failure; - } - } - if (pendingDropFailure instanceof RuntimeException) { - throw (RuntimeException) pendingDropFailure; - } - if (pendingDropFailure instanceof Error) { - throw (Error) pendingDropFailure; - } - if (pendingDropFailure != null) { - throw new IllegalStateException("Rust slice element drop failed", pendingDropFailure); - } - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not invoke Rust slice element drop glue", error); - } catch (Throwable error) { - PanicSupport.abortIfStackOverflow(error); - if (error instanceof RuntimeException) { - throw (RuntimeException) error; - } - if (error instanceof Error) { - throw (Error) error; - } - throw new IllegalStateException("Rust slice element drop failed", error); + if (target == null || replacement == null || target.getClass() != replacement.getClass()) { + throw new IllegalArgumentException("managed-object overwrite requires matching non-null classes"); } + copyStructuralFields(replacement, target); } /** Creates an independent JVM carrier for a copied Rust aggregate value. */ @@ -2312,6 +1518,71 @@ public static void fillArray(Object array, Object value, boolean copyValue) { } } + // Primitive repeat initialization keeps values unboxed and preserves live aliases. + public static void fillArray(boolean[] array, boolean value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetBoolean(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(byte[] array, byte value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetI8(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(short[] array, short value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetI16(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(char[] array, char value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetU16(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(int[] array, int value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetI32(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(long[] array, long value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetI64(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(float[] array, float value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetF32(array, index, value); + } else { + Arrays.fill(array, value); + } + } + + public static void fillArray(double[] array, double value) { + if (hasScalarWriteTracking(array)) { + for (int index = 0; index < array.length; index++) sliceSetF64(array, index, value); + } else { + Arrays.fill(array, value); + } + } + private static Object independentRepeatedArrayElement(Object array, int index) { int length = Array.getLength(array); if (index < 0 || index >= length) { @@ -2438,9 +1709,7 @@ private static void writeNestedPrimitiveArrayElement( } private static void markRepeatedArray(Object array) { - int hash = System.identityHashCode(array); - markFilterHash(REPEATED_ARRAY_FILTER, mixRepeatedArrayHash(hash)); - markFilterHash(REPEATED_ARRAY_FILTER, mixRepeatedArrayHash(hash ^ 0x9e37_79b9)); + markIdentityFilter(REPEATED_ARRAY_FILTER, array); } private static boolean mayBeRepeatedArray(Object array) { @@ -2454,26 +1723,6 @@ private static int mixRepeatedArrayHash(int hash) { return hash; } - private static boolean markFilterHash(AtomicLongArray filter, int hash) { - int bitIndex = hash & (filter.length() * Long.SIZE - 1); - int wordIndex = bitIndex >>> 6; - long bit = 1L << bitIndex; - while (true) { - long previous = filter.get(wordIndex); - if ((previous & bit) != 0) { - return false; - } - if (filter.compareAndSet(wordIndex, previous, previous | bit)) { - return true; - } - } - } - - private static boolean hasFilterHash(AtomicLongArray filter, int hash) { - int bitIndex = hash & (filter.length() * Long.SIZE - 1); - return (filter.get(bitIndex >>> 6) & (1L << bitIndex)) != 0; - } - /** Reads one reference-array element while preserving Rust array value semantics. */ public static Object arrayGetObject(Object array, int index) { Object value = independentRepeatedArrayElement(array, index); @@ -2504,11 +1753,13 @@ public static void encodeArrayMemory( elementCodec) .flushAllMemoryViews(); } - if (array instanceof byte[] && elementSize == 1 && elementCodec == null) { - System.arraycopy(array, 0, bytes, offset, length); + Class component = array.getClass().getComponentType(); + if (elementCodec == null && component.isPrimitive() + && elementSize == ArrayMemoryCodec.elementSize(component)) { + ArrayMemoryCodec.write(array, bytes, offset); return; } - CodecPlan aggregatePlan = isGeneratedAggregateCodec(elementCodec) + MemoryCodec aggregatePlan = isGeneratedAggregateCodec(elementCodec) ? codecPlan(elementCodec) : null; for (int index = 0; index < length; index++) { @@ -2545,11 +1796,12 @@ public static void encodeArrayMemory( public static void decodeArrayMemory( byte[] bytes, int offset, Object array, int elementSize, String elementCodec) { int length = Array.getLength(array); - if (array instanceof byte[] && elementSize == 1 && elementCodec == null) { - System.arraycopy(bytes, offset, array, 0, length); + Class componentType = array.getClass().getComponentType(); + if (elementCodec == null && componentType.isPrimitive() + && elementSize == ArrayMemoryCodec.elementSize(componentType)) { + ArrayMemoryCodec.read(bytes, offset, array); return; } - Class componentType = array.getClass().getComponentType(); for (int index = 0; index < length; index++) { Object element = decodeMemoryValue( bytes, offset + index * elementSize, elementSize, elementCodec, componentType); @@ -2801,7 +2053,7 @@ private static byte[] encodeFatPointer(Object value, int size, String codec) { String metadataClass = codec.substring(TRAIT_POINTER_VIEW_CODEC_PREFIX.length()); Pointer marker = traitMetadataMarker(pointer, metadataClass); dataAddress = erasedAddress(pointer); - rememberEncodedPointer(image, 0, wordSize, codec, pointer); + rememberEncodedPointer(image, 0, wordSize, codec, pointer.exposedTarget(), dataAddress); pointerMetadata = marker.address(); } @@ -2837,6 +2089,7 @@ private static Object decodeFatPointer( if (traitInterface == null) { data = data.retype(elementSize, elementCodec); } else { + data = independentFieldMetadata(data); Pointer marker = pointerObjectFromAddress(pointerMetadata); TraitMetadataInfo info = TRAIT_METADATA_INFO.get(marker.numericAddress()); data.traitMetadataMarker(marker); @@ -2885,14 +2138,7 @@ private static Object decodeFatPointer( data = typedPointerObjectFromAddress(dataAddress, codec); } data = data.retype(elementSize, elementCodec); - try { - Class viewClass = resolvedRuntimeClass(descriptor[0]); - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(viewClass) - .newInstance(data, 0, pointerMetadata); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not reconstruct Rust slice fat pointer", error); - } + return SliceView.create(descriptor[0], data, 0, pointerMetadata); } /** Writes a JVM fat-pointer carrier using Rust's native two-word layout. */ @@ -3027,28 +2273,13 @@ private static Class resolvedRuntimeClass(String className) return resolvedClass(className, RUNTIME_CLASS_LOADER); } - private static Field instanceField(Class owner, String name) + private static RustField instanceField(Class owner, String name) throws NoSuchFieldException { - ConcurrentHashMap fields = INSTANCE_FIELDS.get(owner); - Field cached = fields.get(name); - if (cached != null) { - return cached; - } - Field field = owner.getField(name); - if (Modifier.isStatic(field.getModifiers())) { - throw new NoSuchFieldException(owner.getName() + "." + name + " is static"); - } - field.setAccessible(true); - Field previous = fields.putIfAbsent(name, field); - return previous == null ? field : previous; + return RustField.find(owner, name); } - private static Field optionalInstanceField(Class owner, String name) { - try { - return instanceField(owner, name); - } catch (NoSuchFieldException ignored) { - return null; - } + private static RustField optionalInstanceField(Class owner, String name) { + return RustField.optional(owner, name); } private static FieldAccess fieldAccess(Class owner, String name) @@ -3065,15 +2296,15 @@ private static FieldAccess fieldAccess(Class owner, String name) private static long sliceLogicalLength(Object slice) throws ReflectiveOperationException { - return SLICE_ACCESSES.get(slice.getClass()).rustLength(slice); + return ((SliceView) slice).rustLength; } private static Object sliceBackingForArray(Object slice, Class targetArrayType) throws ReflectiveOperationException { - SliceAccess access = SLICE_ACCESSES.get(slice.getClass()); - Object backing = access.array(slice); - int offset = access.offset(slice); - int length = access.length(slice); + SliceView view = (SliceView) slice; + Object backing = view.array; + int offset = view.offset; + int length = view.length; if (backing instanceof Pointer) { Object result = ((Pointer) backing).backingArrayRange(targetArrayType, offset, length); if (result != null) { @@ -3216,8 +2447,7 @@ private static Object adaptStructuralField( return value; } if (isSliceViewCarrierType(targetType) && value.getClass().isArray()) { - Constructor constructor = SLICE_VIEW_CONSTRUCTORS.get(targetType); - return constructor.newInstance(value, 0, Array.getLength(value)); + return SliceView.create(targetType, value, 0, Array.getLength(value)); } if (targetType.isArray() && isSliceViewCarrierType(value.getClass())) { return sliceBackingForArray(value, targetType); @@ -3237,7 +2467,7 @@ private static Object constructStructuralView( Object source, Class targetClass, Object traitTailCarrier) { try { Object transparentInner = null; - for (Field field : PUBLIC_INSTANCE_FIELDS.get(source.getClass())) { + for (RustField field : PUBLIC_INSTANCE_FIELDS.get(source.getClass())) { if (transparentInner != null) { transparentInner = null; break; @@ -3247,7 +2477,7 @@ private static Object constructStructuralView( if (transparentInner != null && targetClass.isInstance(transparentInner)) { return transparentInner; } - Field[] targetFields = PUBLIC_INSTANCE_FIELDS.get(targetClass); + RustField[] targetFields = PUBLIC_INSTANCE_FIELDS.get(targetClass); ConstructorPlan constructor = structuralConstructor(targetClass, targetFields.length); Object[] args = new Object[constructor.parameterTypes.length]; java.lang.reflect.Parameter[] parameters = constructor.reflection.getParameters(); @@ -3264,7 +2494,7 @@ private static Object constructStructuralView( ? parameters[index].getName() : targetFields[index].getName(); try { - Field sourceField = instanceField(source.getClass(), fieldName); + RustField sourceField = instanceField(source.getClass(), fieldName); args[index] = adaptStructuralField( sourceField.get(source), parameters[index].getType(), index == parameters.length - 1 ? traitTailCarrier : null); @@ -3290,12 +2520,12 @@ private static Object constructStructuralView( private static void copyStructuralFields(Object source, Object target) { try { - Field[] targetFields = PUBLIC_INSTANCE_FIELDS.get(target.getClass()); + RustField[] targetFields = PUBLIC_INSTANCE_FIELDS.get(target.getClass()); if (targetFields.length == 1 && targetFields[0].getType().isInstance(source)) { targetFields[0].set(target, source); return; } - Field[] sourceFields = PUBLIC_INSTANCE_FIELDS.get(source.getClass()); + RustField[] sourceFields = PUBLIC_INSTANCE_FIELDS.get(source.getClass()); if (sourceFields.length == 1) { Object inner = sourceFields[0].get(source); if (inner != null && target.getClass().isInstance(inner)) { @@ -3303,8 +2533,8 @@ private static void copyStructuralFields(Object source, Object target) { return; } } - for (Field targetField : targetFields) { - Field sourceField = instanceField(source.getClass(), targetField.getName()); + for (RustField targetField : targetFields) { + RustField sourceField = instanceField(source.getClass(), targetField.getName()); Object sourceValue = sourceField.get(source); Object targetValue = targetField.get(target); Object adapted; @@ -3360,7 +2590,7 @@ private StructuralViewState structuralViewState(Object source, boolean create) { } } if (created) { - maybeRebuildStructuralViewFilter(); + maybeRebuildIdentityFilter(STRUCTURAL_VIEW_FILTER, STRUCTURAL_VIEWS); } return state; } @@ -3395,157 +2625,65 @@ private static boolean mayHaveStructuralView(Object allocation) { return mayBeInIdentityFilter(STRUCTURAL_VIEW_FILTER, allocation); } + // Both bits share one word so a lookup needs one atomic read. + private static long identityFilterBits(int hash) { + int first = hash >>> 20; + return (1L << first) | (1L << (first + 1 + ((hash >>> 26) & 31))); + } + private static boolean markIdentityFilter(AtomicLongArray filter, Object value) { - int hash = System.identityHashCode(value); - boolean first = markFilterHash(filter, mixRepeatedArrayHash(hash)); - boolean second = - markFilterHash(filter, mixRepeatedArrayHash(hash ^ 0x9e37_79b9)); - return first || second; + int hash = mixRepeatedArrayHash(System.identityHashCode(value)); + int index = hash & (filter.length() - 1); + long bits = identityFilterBits(hash); + while (true) { + long previous = filter.get(index); + if ((previous & bits) == bits) return false; + if (filter.compareAndSet(index, previous, previous | bits)) return true; + } } private static boolean mayBeInIdentityFilter(AtomicLongArray filter, Object value) { - int hash = System.identityHashCode(value); - return hasFilterHash(filter, mixRepeatedArrayHash(hash)) - && hasFilterHash(filter, mixRepeatedArrayHash(hash ^ 0x9e37_79b9)); + int hash = mixRepeatedArrayHash(System.identityHashCode(value)); + long bits = identityFilterBits(hash); + return (filter.get(hash & (filter.length() - 1)) & bits) == bits; } private static void markIdentityFilter(RebuildableIdentityFilter filter, Object value) { - AtomicLongArray primary = filter.primary; - boolean added = markIdentityFilter(primary, value); - AtomicLongArray secondary = filter.secondary; - if (secondary != null && secondary != primary) { - markIdentityFilter(secondary, value); - } - if (added) { - filter.marks.incrementAndGet(); - } + // Mark both generations and retry if a rebuild publishes a new state. + boolean added = false; + IdentityFilterState state; + do { + state = filter.state; + if (state == null) state = filter.initialize(); + added |= markIdentityFilter(state.primary, value); + if (state.secondary != null) added |= markIdentityFilter(state.secondary, value); + } while (state != filter.state); + if (added) filter.marks.incrementAndGet(); } private static boolean mayBeInIdentityFilter( RebuildableIdentityFilter filter, Object value) { - AtomicLongArray primary = filter.primary; - if (mayBeInIdentityFilter(primary, value)) { - return true; - } - AtomicLongArray secondary = filter.secondary; - return secondary != null - && secondary != primary - && mayBeInIdentityFilter(secondary, value); - } - - private static void maybeRebuildMemoryViewFilter() { - if (MEMORY_VIEW_FILTER.marks.get() < MEMORY_VIEW_FILTER.rebuildMarks - || !MEMORY_VIEW_FILTER.rebuilding.compareAndSet(0, 1)) { - return; - } - try { - AtomicLongArray rebuilt = - new AtomicLongArray(MEMORY_VIEW_FILTER.wordCount); - MEMORY_VIEW_FILTER.secondary = rebuilt; - for (Map> stripe : MEMORY_VIEWS) { - synchronized (stripe) { - ((WeakIdentityMap>) stripe) - .markLiveKeys(rebuilt); - } - } - MEMORY_VIEW_FILTER.primary = rebuilt; - MEMORY_VIEW_FILTER.secondary = null; - MEMORY_VIEW_FILTER.marks.set(0); - } finally { - MEMORY_VIEW_FILTER.rebuilding.set(0); - } - } - - private static void maybeRebuildMemoryViewOriginFilter() { - if (MEMORY_VIEW_ORIGIN_FILTER.marks.get() - < MEMORY_VIEW_ORIGIN_FILTER_REBUILD_MARKS - || !MEMORY_VIEW_ORIGIN_FILTER.rebuilding.compareAndSet(0, 1)) { - return; - } - try { - AtomicLongArray rebuilt = - new AtomicLongArray(MEMORY_VIEW_ORIGIN_FILTER.wordCount); - MEMORY_VIEW_ORIGIN_FILTER.secondary = rebuilt; - for (Map stripe : MEMORY_VIEW_ORIGINS) { - synchronized (stripe) { - ((WeakIdentityMap) stripe) - .markLiveKeys(rebuilt); - } - } - MEMORY_VIEW_ORIGIN_FILTER.primary = rebuilt; - MEMORY_VIEW_ORIGIN_FILTER.secondary = null; - MEMORY_VIEW_ORIGIN_FILTER.marks.set(0); - } finally { - MEMORY_VIEW_ORIGIN_FILTER.rebuilding.set(0); - } - } - - private static void maybeRebuildStructuralViewFilter() { - if (STRUCTURAL_VIEW_FILTER.marks.get() < STRUCTURAL_VIEW_FILTER.rebuildMarks - || !STRUCTURAL_VIEW_FILTER.rebuilding.compareAndSet(0, 1)) { - return; - } - try { - AtomicLongArray rebuilt = - new AtomicLongArray(STRUCTURAL_VIEW_FILTER.wordCount); - STRUCTURAL_VIEW_FILTER.secondary = rebuilt; - for (Map> stripe : STRUCTURAL_VIEWS) { - synchronized (stripe) { - ((WeakIdentityMap>) stripe) - .markLiveKeys(rebuilt); - } - } - STRUCTURAL_VIEW_FILTER.primary = rebuilt; - STRUCTURAL_VIEW_FILTER.secondary = null; - STRUCTURAL_VIEW_FILTER.marks.set(0); - } finally { - STRUCTURAL_VIEW_FILTER.rebuilding.set(0); - } - } - - private static void maybeRebuildEncodedReferenceFilter() { - if (ENCODED_REFERENCE_FILTER.marks.get() < ENCODED_REFERENCE_FILTER.rebuildMarks - || !ENCODED_REFERENCE_FILTER.rebuilding.compareAndSet(0, 1)) { - return; - } - try { - AtomicLongArray rebuilt = - new AtomicLongArray(ENCODED_REFERENCE_FILTER.wordCount); - ENCODED_REFERENCE_FILTER.secondary = rebuilt; - for (Map stripe : ENCODED_REFERENCES) { - synchronized (stripe) { - ((WeakIdentityMap) stripe).markLiveKeys(rebuilt); - } - } - ENCODED_REFERENCE_FILTER.primary = rebuilt; - ENCODED_REFERENCE_FILTER.secondary = null; - ENCODED_REFERENCE_FILTER.marks.set(0); - } finally { - ENCODED_REFERENCE_FILTER.rebuilding.set(0); - } + IdentityFilterState state = filter.state; + return state != null && (mayBeInIdentityFilter(state.primary, value) + || (state.secondary != null && mayBeInIdentityFilter(state.secondary, value))); } - private static void maybeRebuildEncodedPointerFilter() { - if (ENCODED_POINTER_FILTER.marks.get() - < ENCODED_POINTER_FILTER.rebuildMarks - || !ENCODED_POINTER_FILTER.rebuilding.compareAndSet(0, 1)) { - return; - } + private static void maybeRebuildIdentityFilter( + RebuildableIdentityFilter filter, Map[] stripes) { + if (filter.marks.get() < filter.rebuildMarks + || !filter.rebuilding.compareAndSet(0, 1)) return; try { - AtomicLongArray rebuilt = - new AtomicLongArray(ENCODED_POINTER_FILTER.wordCount); - ENCODED_POINTER_FILTER.secondary = rebuilt; - for (Map> stripe : ENCODED_POINTERS) { + AtomicLongArray rebuilt = new AtomicLongArray(filter.wordCount); + filter.state = new IdentityFilterState(filter.state.primary, rebuilt); + for (Map stripe : stripes) { synchronized (stripe) { - ((WeakIdentityMap>) stripe) - .markLiveKeys(rebuilt); + ((WeakIdentityMap) stripe).forEachLiveKey(key -> markIdentityFilter(rebuilt, key)); } } - ENCODED_POINTER_FILTER.primary = rebuilt; - ENCODED_POINTER_FILTER.secondary = null; - ENCODED_POINTER_FILTER.marks.set(0); + filter.state = new IdentityFilterState(rebuilt, null); + filter.marks.set(0); } finally { - ENCODED_POINTER_FILTER.rebuilding.set(0); + filter.rebuilding.set(0); } } @@ -3605,10 +2743,33 @@ private void flushMemoryViewsOverlapping(long offset, int size) { } private void flushMemoryViewsOverlapping(long offset, int size, boolean overwrite) { - if (directCellHasNoMemoryViews(allocation) - || !mayBeInIdentityFilter(MEMORY_VIEW_FILTER, allocation)) { + if (!directCellHasNoMemoryViews(allocation)) { + flushFilteredMemoryViews(offset, size, overwrite); + } + } + + private void flushFilteredMemoryViews(long offset, int size, boolean overwrite) { + int cached = absentViewState; + int stripe = cached == 0 ? stateStripeIndex(allocation) : cached & (STATE_STRIPE_COUNT - 1); + long epoch = MEMORY_VIEW_EPOCHS.get(stripe); + if (epoch + 1 == (cached >>> STATE_STRIPE_BITS)) return; + if (mayBeInIdentityFilter(MEMORY_VIEW_FILTER, allocation)) { + flushTrackedMemoryViews(offset, size, overwrite); + } else if (epoch >= 0 && epoch < (-1 >>> STATE_STRIPE_BITS)) { + absentViewState = ((int) (epoch + 1) << STATE_STRIPE_BITS) | stripe; + } + } + + private void flushTrackedMemoryViews(long offset, int size, boolean overwrite) { + if (MEMORY_VIEW_ABSENCE.get().matches(allocation, memoryViewEpoch(allocation))) { return; } + synchronized (atomicStripe(this)) { + flushMemoryViewsOverlappingLocked(offset, size, overwrite); + } + } + + private void flushMemoryViewsOverlappingLocked(long offset, int size, boolean overwrite) { long observedEpoch = memoryViewEpoch(allocation); MemoryViewAbsenceCache absence = MEMORY_VIEW_ABSENCE.get(); if (absence.matches(allocation, observedEpoch)) { @@ -3652,6 +2813,12 @@ private void flushAllMemoryViews() { || !mayBeInIdentityFilter(MEMORY_VIEW_FILTER, allocation)) { return; } + synchronized (atomicStripe(this)) { + flushAllMemoryViewsLocked(); + } + } + + private void flushAllMemoryViewsLocked() { LongRangeMap pending; Map> stripe = stateStripe(MEMORY_VIEWS, allocation); @@ -3761,6 +2928,30 @@ private Object decodedMemoryView() { } } + /** Reuse a decoded element only while its registered origin still matches. + * Keep the enclosing pointer bound to its own element. */ + private Object cachedSliceElement(long displacement) { + if (!(allocation instanceof byte[]) || viewSize <= 0 + || !isGeneratedAggregateCodec(viewCodecClassName) + || traitObjectCarrier() != null || traitMetadataCarrier() != null + || isDirectAllocationView()) return null; + long absolute = Math.addExact(byteOffset, displacement); + synchronized (atomicStripe(this)) { + Map> stripe = stateStripe(MEMORY_VIEWS, allocation); + synchronized (stripe) { + LongRangeMap views = stripe.get(allocation); + MemoryViewState cached = views == null ? null : views.get(absolute); + if (cached == null || !cached.active || cached.size != viewSize + || !cached.codecClassName.equals(viewCodecClassName) || cached.value == null) return null; + Map origins = stateStripe(MEMORY_VIEW_ORIGINS, cached.value); + synchronized (origins) { + MemoryViewOrigin origin = origins.get(cached.value); + return origin != null && origin.matches(this, absolute) ? cached.value : null; + } + } + } + } + private Object activeBoundMemoryViewValue(int materializedSize) { MemoryViewState bound = boundMemoryViewState(); if (bound != null @@ -3815,7 +3006,6 @@ private Object decodedMemoryViewLocked() { new MemoryViewState(materializedSize, viewCodecClassName, decoded, image); advanceMemoryViewEpoch(allocation); setDirectCellHasMemoryView(allocation, true); - markIdentityFilter(MEMORY_VIEW_FILTER, allocation); synchronized (stripe) { LongRangeMap views = stripe.get(allocation); if (views == null) { @@ -3823,12 +3013,10 @@ private Object decodedMemoryViewLocked() { stripe.put(allocation, views); } views.put(byteOffset, state); + markIdentityFilter(MEMORY_VIEW_FILTER, allocation); } advanceMemoryViewEpoch(allocation); - // Mark again after publishing the map entry so a concurrent filter - // rebuild cannot miss an insertion whose stripe it already scanned. - markIdentityFilter(MEMORY_VIEW_FILTER, allocation); - maybeRebuildMemoryViewFilter(); + maybeRebuildIdentityFilter(MEMORY_VIEW_FILTER, MEMORY_VIEWS); setBoundMemoryViewState(state); registerMemoryViewOrigin(decoded); bindDecodedMemoryView(decoded); @@ -3840,11 +3028,11 @@ private void attachDecodedTraitTail(Object decoded) { if (decoded == null || carrier == null) { return; } - Field[] fields = PUBLIC_INSTANCE_FIELDS.get(decoded.getClass()); + RustField[] fields = PUBLIC_INSTANCE_FIELDS.get(decoded.getClass()); if (fields.length == 0) { return; } - Field tail = fields[fields.length - 1]; + RustField tail = fields[fields.length - 1]; if (!tail.getType().isInstance(carrier)) { return; } @@ -3869,7 +3057,7 @@ private void registerMemoryViewOrigin(Object value) { markIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, value); previous = stripe.put(value, new MemoryViewOrigin(this)); } - maybeRebuildMemoryViewOriginFilter(); + maybeRebuildIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, MEMORY_VIEW_ORIGINS); Object previousAllocation = previous == null ? null : previous.allocation.get(); if (previousAllocation instanceof FieldCell && previousAllocation != allocation) { removeMemoryOriginView(previousAllocation, value); @@ -3937,7 +3125,7 @@ private void bindDecodedMemoryView(Object value) { || isBuiltInCodec(viewCodecClassName)) { return; } - CodecBinder bind = codecPlan(viewCodecClassName).bind; + CodecCalls.Binder bind = codecPlan(viewCodecClassName).bind(); if (bind == null) { return; } @@ -4056,7 +3244,7 @@ public void commitMemoryView() { // decoding this field separately. Nested direct writes still need // to reach the enclosing enum's original byte storage. if (allocation instanceof FieldCell) { - commitOriginMemoryView(((FieldCell) allocation).owner()); + ((FieldCell) allocation).commitOwners(); } return; } @@ -4100,6 +3288,7 @@ private void commitMemoryViewLocked() { } state.active = true; setDirectCellHasMemoryView(allocation, true); + markIdentityFilter(MEMORY_VIEW_FILTER, allocation); } advanceMemoryViewEpoch(allocation); registerMemoryViewOrigin(state.value); @@ -4127,21 +3316,54 @@ private Object managedViewObject() { // identity filter as short-lived field pointers accumulate in long runs. private static boolean mayHaveFieldCells(Object owner) { if (owner instanceof Cell) { - return ((Cell) owner).hasFieldCells; + return ((Cell) owner).fields != null; } if (owner instanceof FieldCell) { - return ((FieldCell) owner).hasFieldCells; + return ((FieldCell) owner).fields != null; } return owner != null && mayBeInIdentityFilter(FIELD_CELL_FILTER, owner); } - private static void markFieldCells(Object owner) { - if (owner instanceof Cell) { - ((Cell) owner).hasFieldCells = true; - } else if (owner instanceof FieldCell) { - ((FieldCell) owner).hasFieldCells = true; - } else { - markIdentityFilter(FIELD_CELL_FILTER, owner); + private static final class FieldCellCache extends WeakReference { + private volatile FieldCellCache next; + + private FieldCellCache(FieldCell field, FieldCellCache next) { + super(field); + this.next = next; + } + } + + private static FieldCellCache ownedFields(Object owner) { + return owner instanceof Cell ? ((Cell) owner).fields : ((FieldCell) owner).fields; + } + + /** Keep cached field cells weak. A decoded value can refer to its owner. + * A strong field cache would retain that cycle through the global view map. */ + private static FieldCell ownedField(Object owner, FieldAccess access) { + for (FieldCellCache entry = ownedFields(owner); entry != null; entry = entry.next) { + FieldCell field = entry.get(); + if (field != null && (field.access == access || field.access.cacheKey.equals(access.cacheKey))) + return field; + } + synchronized (owner) { + FieldCellCache head = ownedFields(owner), previous = null; + for (FieldCellCache entry = head; entry != null; entry = entry.next) { + FieldCell field = entry.get(); + if (field != null) { + if (field.access == access || field.access.cacheKey.equals(access.cacheKey)) return field; + previous = entry; + } else if (previous == null) { + head = entry.next; + } else { + // Keep the next link for readers that still hold this dead entry. + previous.next = entry.next; + } + } + FieldCell field = new FieldCell(owner, access, true); + FieldCellCache entry = new FieldCellCache(field, head); + if (owner instanceof Cell) ((Cell) owner).fields = entry; + else ((FieldCell) owner).fields = entry; + return field; } } @@ -4176,6 +3398,12 @@ private static java.util.List collectProjectedFieldViews( if (!mayHaveProjectedViews(owner)) { return projected; } + if (owner instanceof Cell || owner instanceof FieldCell) { + for (FieldCellCache entry = ownedFields(owner); entry != null; entry = entry.next) { + projected = collectProjectedFieldView(entry.get(), projected); + } + return projected; + } Map>> fieldStripe = stateStripe(FIELD_CELLS, owner); synchronized (fieldStripe) { @@ -4184,22 +3412,22 @@ private static java.util.List collectProjectedFieldViews( return projected; } for (WeakReference reference : fields.values()) { - FieldCell cell = reference.get(); - if (cell != null - && (cell.hasStructuralView - || cell.hasMemoryView - || cell.hasMemoryOrigins - || cell.hasProjectedViews)) { - if (projected == null) { - projected = new java.util.ArrayList<>(); - } - projected.add(cell); - } + projected = collectProjectedFieldView(reference.get(), projected); } } return projected; } + private static java.util.List collectProjectedFieldView( + FieldCell cell, java.util.List projected) { + if (cell != null && (cell.hasStructuralView || cell.hasMemoryView + || cell.hasMemoryOrigins || cell.hasProjectedViews)) { + if (projected == null) projected = new java.util.ArrayList<>(); + projected.add(cell); + } + return projected; + } + private static void discardProjectedFieldViews(Object owner) { java.util.List projected = collectProjectedFieldViews(owner, null); if (projected == null) { @@ -4233,6 +3461,12 @@ private static void discardProjectedFieldViews(Object owner) { } private static boolean hasProjectedFieldCells(Object owner) { + if (owner instanceof Cell || owner instanceof FieldCell) { + for (FieldCellCache entry = ownedFields(owner); entry != null; entry = entry.next) { + if (entry.get() != null) return true; + } + return false; + } if (!mayHaveFieldCells(owner)) { return false; } @@ -4272,7 +3506,7 @@ private Object transparentManagedView(Class targetClass) { } Object match = null; try { - for (Field field : PUBLIC_INSTANCE_FIELDS.get(owner.getClass())) { + for (RustField field : PUBLIC_INSTANCE_FIELDS.get(owner.getClass())) { Object candidate = field.get(owner); if (candidate != null && targetClass.isInstance(candidate)) { if (match != null) { @@ -4349,14 +3583,7 @@ private Object structTailSourceObject(String[] descriptor) { int elementSize = structTailPointerElementSize(descriptor); String elementCodec = descriptor[4].isEmpty() ? null : descriptor[4]; Pointer data = byteOffsetRetype(prefixSize, elementSize, elementCodec); - try { - Class viewClass = resolvedRuntimeClass(descriptor[2]); - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(viewClass) - .newInstance(data, 0, metadata()); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not construct Rust struct-tail source view", error); - } + return SliceView.create(descriptor[2], data, 0, metadata()); } private Object structuralSourceObject() { @@ -4394,7 +3621,7 @@ private static long inferredStructuralMetadata(Object value) { if (isSliceViewType(value.getClass())) { return sliceLogicalLength(value); } - for (Field field : PUBLIC_INSTANCE_FIELDS.get(value.getClass())) { + for (RustField field : PUBLIC_INSTANCE_FIELDS.get(value.getClass())) { if (!mayContainStructuralMetadata(field.getType())) { continue; } @@ -4445,7 +3672,7 @@ private static boolean mayContainStructuralMetadata( return false; } boolean result = false; - for (Field field : PUBLIC_INSTANCE_FIELDS.get(type)) { + for (RustField field : PUBLIC_INSTANCE_FIELDS.get(type)) { if (mayContainStructuralMetadata( field.getType(), visiting, encounteredCycle)) { result = true; @@ -4459,14 +3686,14 @@ private static boolean mayContainStructuralMetadata( return result; } - private static final class Cell { - private Object value; + static class Cell { + Object value; private volatile boolean hasStructuralView; private volatile boolean hasMemoryView; - private volatile boolean hasFieldCells; + private volatile FieldCellCache fields; private volatile boolean hasProjectedViews; - private Cell(Object value) { + Cell(Object value) { this.value = value; } } @@ -4502,13 +3729,17 @@ private static Object memoryViewAllocation(Object value) { } private static final class FieldCell { + // Share only the same immutable origin and offset. + private volatile AddressOriginState cachedAddressOrigin; + // Cache thin projections only. Exclude published addresses, decoded views, and dynamic metadata. + private volatile Pointer cachedProjection; private final Object fixedOwner; private final Object rootOwner; // Direct enum-payload borrows have no parent Pointer. Retain the // decoded owner's storage while this field cell remains reachable. private final Object memoryBacking; private final FieldAccess access; - private volatile boolean hasFieldCells; + private volatile FieldCellCache fields; private volatile boolean hasProjectedViews; private volatile boolean hasMemoryOrigins; private final int fieldNameHash; @@ -4516,25 +3747,13 @@ private static final class FieldCell { private volatile boolean hasMemoryView; private FieldCell(Object owner, FieldAccess access) { - fixedOwner = owner; - rootOwner = null; - memoryBacking = memoryViewAllocation(owner); - this.access = access; - fieldNameHash = access.fieldNameHash; - } - - private FieldCell(Cell owner, FieldAccess access) { - fixedOwner = null; - rootOwner = owner; - memoryBacking = null; - this.access = access; - fieldNameHash = access.fieldNameHash; + this(owner, access, false); } - private FieldCell(FieldCell owner, FieldAccess access) { - fixedOwner = null; - rootOwner = owner; - memoryBacking = null; + private FieldCell(Object owner, FieldAccess access, boolean rooted) { + fixedOwner = rooted ? null : owner; + rootOwner = rooted ? owner : null; + memoryBacking = rooted ? null : memoryViewAllocation(owner); this.access = access; fieldNameHash = access.fieldNameHash; } @@ -4556,14 +3775,6 @@ private Object get() { try { Object owner = owner(); Object value = (Object) access.getter.invokeExact(owner); - if (value instanceof Pointer && access.elementOffsetGetter != null) { - long elementOffset = - (long) access.elementOffsetGetter.invokeExact(owner); - long byteOffset = - (long) access.byteOffsetGetter.invokeExact(owner); - return materializeRelative( - (Pointer) value, elementOffset, byteOffset); - } return value; } catch (RuntimeException | Error error) { throw error; @@ -4576,10 +3787,6 @@ private void set(Object value) { Object owner = owner(); try { access.setter.invokeExact(owner, value); - if (access.elementOffsetSetter != null) { - access.elementOffsetSetter.invokeExact(owner, 0L); - access.byteOffsetSetter.invokeExact(owner, 0L); - } } catch (RuntimeException | Error error) { throw error; } catch (Throwable error) { @@ -4589,136 +3796,55 @@ private void set(Object value) { // of sibling fields (for example an inline vector's data when its // length changes before writing a newly inserted element). discardProjectedFieldViews(this); - commitOriginMemoryView(owner); - Object identity = ownerIdentity(); - if (identity != owner) { - commitOriginMemoryView(identity); + commitOwners(); + } + + private void commitOwners() { + // An enum payload can belong to a decoded enum that owns the byte storage. + // Intermediate field carriers need not have registered views. + FieldCell current = this; + while (true) { + Object owner = current.owner(); + commitOriginMemoryView(owner); + if (current.rootOwner instanceof FieldCell) { + current = (FieldCell) current.rootOwner; + } else { + Object identity = current.ownerIdentity(); + if (identity != owner) commitOriginMemoryView(identity); + return; + } } } } private static final class FieldAccess { + private final RustField field; private final MethodHandle getter; private final MethodHandle setter; - private final MethodHandle elementOffsetGetter; - private final MethodHandle byteOffsetGetter; - private final MethodHandle elementOffsetSetter; - private final MethodHandle byteOffsetSetter; private final int fieldNameHash; private final boolean primitive; - // Field metadata is shared; constructing this per projection hashes - // long generated class names on every Rust aggregate field access. + private final boolean borrowed; + // Reuse field metadata to avoid hashing generated class names on each projection. private final String cacheKey; - private FieldAccess(Field field) { + private FieldAccess(RustField field) { + this.field = field; cacheKey = field.getDeclaringClass().getName() + '\n' + field.getName(); fieldNameHash = field.getName().hashCode(); primitive = field.getType().isPrimitive(); + borrowed = field.isBorrowed(); try { - MethodHandles.Lookup lookup = MethodHandles.lookup(); - getter = lookup.unreflectGetter(field).asType( + getter = field.getter().asType( MethodType.methodType(Object.class, Object.class)); - setter = lookup.unreflectSetter(field).asType( + setter = field.setter().asType( MethodType.methodType(void.class, Object.class, Object.class)); - if (field.getType() == Pointer.class) { - Field elementOffset = optionalInstanceField( - field.getDeclaringClass(), - field.getName() + RELATIVE_POINTER_ELEMENT_OFFSET_SUFFIX); - Field byteOffset = optionalInstanceField( - field.getDeclaringClass(), - field.getName() + RELATIVE_POINTER_BYTE_OFFSET_SUFFIX); - if (elementOffset != null && byteOffset != null) { - elementOffsetGetter = lookup.unreflectGetter(elementOffset).asType( - MethodType.methodType(long.class, Object.class)); - byteOffsetGetter = lookup.unreflectGetter(byteOffset).asType( - MethodType.methodType(long.class, Object.class)); - elementOffsetSetter = lookup.unreflectSetter(elementOffset).asType( - MethodType.methodType(void.class, Object.class, long.class)); - byteOffsetSetter = lookup.unreflectSetter(byteOffset).asType( - MethodType.methodType(void.class, Object.class, long.class)); - } else { - elementOffsetGetter = null; - byteOffsetGetter = null; - elementOffsetSetter = null; - byteOffsetSetter = null; - } - } else { - elementOffsetGetter = null; - byteOffsetGetter = null; - elementOffsetSetter = null; - byteOffsetSetter = null; - } } catch (IllegalAccessException error) { throw new IllegalStateException("could not access Rust field pointer", error); } } } - private static final class SliceAccess { - private final MethodHandle array; - private final MethodHandle offset; - private final MethodHandle length; - private final MethodHandle rustLength; - - private SliceAccess(Class type) { - try { - MethodHandles.Lookup lookup = MethodHandles.lookup(); - array = lookup.unreflectGetter(instanceField(type, "array")).asType( - MethodType.methodType(Object.class, Object.class)); - offset = lookup.unreflectGetter(instanceField(type, "offset")).asType( - MethodType.methodType(int.class, Object.class)); - length = lookup.unreflectGetter(instanceField(type, "length")).asType( - MethodType.methodType(int.class, Object.class)); - rustLength = lookup.unreflectGetter(instanceField(type, "rustLength")).asType( - MethodType.methodType(long.class, Object.class)); - } catch (ReflectiveOperationException error) { - throw new IllegalArgumentException( - "invalid Rust slice view " + type.getName(), error); - } - } - - private Object array(Object slice) throws ReflectiveOperationException { - try { - return (Object) array.invokeExact(slice); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException("could not read Rust slice backing", error); - } - } - - private int offset(Object slice) throws ReflectiveOperationException { - try { - return (int) offset.invokeExact(slice); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException("could not read Rust slice offset", error); - } - } - - private int length(Object slice) throws ReflectiveOperationException { - try { - return (int) length.invokeExact(slice); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException("could not read Rust slice storage length", error); - } - } - - private long rustLength(Object slice) throws ReflectiveOperationException { - try { - return (long) rustLength.invokeExact(slice); - } catch (RuntimeException | Error error) { - throw error; - } catch (Throwable error) { - throw new IllegalStateException("could not read Rust slice length", error); - } - } - } - private static final class AllocationInfo { private Long base; private int alignment = 16; @@ -4983,6 +4109,8 @@ private synchronized boolean isEmpty() { } } + // Pack the stripe and its absent-view epoch plus one. Zero means no cache. + private int absentViewState; private final Object allocation; private final int allocationElementSize; private final long byteOffset; @@ -4994,9 +4122,15 @@ private synchronized boolean isEmpty() { private volatile RarePointerState rareState; private long metadata = -1; + /** Immutable provenance can be shared without coupling derived views. */ private static final class AddressOriginState { - private Pointer addressOrigin; - private long addressOriginOffset; + private final Pointer addressOrigin; + private final long addressOriginOffset; + + private AddressOriginState(Pointer origin, long offset) { + addressOrigin = origin; + addressOriginOffset = offset; + } } private static final class RarePointerState { @@ -5012,20 +4146,7 @@ private static final class RarePointerState { private long traitPointeeSize = -1; private long traitPointeeAlignment = -1; private String traitAdapterClassName; - private String traitPointeeCodecClassName; - } - - private AddressOriginState mutableAddressState() { - AddressOriginState current = addressState; - if (current != null) { - return current; - } - synchronized (this) { - if (addressState == null) { - addressState = new AddressOriginState(); - } - return addressState; - } + private String traitPointeeCodecClassName; } private RarePointerState mutableRareState() { @@ -5143,9 +4264,21 @@ private long addressOriginOffset() { } private void setAddressOrigin(Pointer origin, long offset) { - AddressOriginState current = mutableAddressState(); - current.addressOrigin = origin; - current.addressOriginOffset = offset; + AddressOriginState current = addressState; + if (current != null && current.addressOrigin == origin && current.addressOriginOffset == offset) { + return; + } + FieldCell field = byteOffset == 0 && allocation instanceof FieldCell ? (FieldCell) allocation : null; + if (field != null) { + current = field.cachedAddressOrigin; + if (current != null && current.addressOrigin == origin && current.addressOriginOffset == offset) { + addressState = current; + return; + } + } + current = new AddressOriginState(origin, offset); + addressState = current; + if (field != null) field.cachedAddressOrigin = current; } private long publishedAddress() { @@ -5270,6 +4403,11 @@ public static Pointer cell(Object value, long size, String codecClassName) { return cellAligned(value, checkedArrayLength(size), codecClassName, 16); } + public static Pointer cellAligned( + Object value, int size, String codec, int alignment, String scalarLayout) { + return cellAligned(value, size, codec, alignment); + } + public static Pointer cellAligned( Object value, int size, @@ -5291,6 +4429,71 @@ public static Pointer cellAligned( return metadata < 0 ? pointer : pointer.withMetadata(metadata); } + public static Object storage(Object value, int size, String codec) { + return storageAligned(value, size, codec, 16); + } + + public static Object storage(Object value, long size, String codec) { + return storageAligned(value, checkedArrayLength(size), codec, 16); + } + + /** Compiler-proven, nonzero typed storage, retaining the original identity. */ + public static Object storageAligned(Object value, int size, String codec, int alignment) { + return storageAligned(value, size, codec, alignment, null); + } + + public static Object storageAligned(Object value, int size, String codec, int alignment, String scalarLayout) { + return storageAligned(value, size, codec, alignment, scalarLayout, -1); + } + + public static Object borrowedStorage(Object value, int size, String codec, int kind) { + return borrowedStorageAligned(value, size, codec, 16, kind); + } + + public static Object borrowedStorage(Object value, long size, String codec, int kind) { + return borrowedStorage(value, checkedArrayLength(size), codec, kind); + } + + public static Object borrowedStorageAligned(Object value, int size, String codec, int alignment, int kind) { + return borrowedStorageAligned(value, size, codec, alignment, null, kind); + } + + public static Object borrowedStorageAligned(Object value, int size, String codec, int alignment, String layout, int kind) { + return storageAligned(value, size, codec, alignment, layout, kind); + } + + private static Object storageAligned(Object value, int size, String codec, int alignment, + String scalarLayout, int borrowed) { + if (size <= 0 || alignment <= 0 || (alignment & (alignment - 1)) != 0) { + throw new IllegalArgumentException("invalid typed Rust storage layout"); + } + long metadata = mayCarryStructuralMetadata(value, size) + ? inferredStructuralMetadata(value) : -1; + // Trait and DST addresses need the full codec carrier to preserve dynamic metadata. + Storage storage = borrowed >= 0 && (borrowed != 0 || size == 8) + ? new BorrowedStorage(value, size, codec, metadata, scalarLayout, borrowed != 0) + : new Storage(value, size, codec, metadata, scalarLayout); + recordAlignment(storage, alignment); + return storage; + } + + static Pointer storageBoundary(Storage storage) { + Pointer result = new Pointer(storage, storage.size, 0, storage.size, storage.codec); + return storage.metadata < 0 ? result : result.withMetadata(storage.metadata); + } + + private static boolean directStorage(Storage storage) { + return unescapedStorage(storage) + && (!(storage instanceof BorrowedStorage) || !((BorrowedStorage) storage).split); + } + + private static boolean unescapedStorage(Storage storage) { + Cell cell = storage; + return storage.boundary == null && !cell.hasStructuralView + && !cell.hasMemoryView && cell.fields == null && !cell.hasProjectedViews + && !isStructuralViewCodec(storage.codec); + } + /** Populate published static storage without changing the address captured * by self-references or by another static's initializer. Includes ZST carriers. */ public void initializeStatic(Object value) { @@ -5391,9 +4594,18 @@ public static Pointer cell(Object value) { /** Returns a stable, write-through pointer to a generated Rust value field. */ public static Pointer field( Object owner, String fieldName, int size, String codecClassName) { + return field(owner, fieldName, size, codecClassName, true); + } + + private static Pointer field( + Object owner, String fieldName, int size, String codecClassName, boolean cache) { if (owner == null) { throw new NullPointerException("Rust field pointer requires an owner"); } + // Erased zero-sized fields have no Java member. Avoid reflection errors and cache entries. + if (size == 0 && optionalInstanceField(owner.getClass(), fieldName) == null) { + return Pointer.cell(null, 0, codecClassName); + } FieldCell cell; Map>> stripe = stateStripe(FIELD_CELLS, owner); @@ -5418,9 +4630,15 @@ public static Pointer field( } fields.put(fieldName, new WeakReference<>(cell)); } - markFieldCells(owner); + markIdentityFilter(FIELD_CELL_FILTER, owner); } - return new Pointer(cell, size, 0, size, codecClassName); + Pointer cached = cell.cachedProjection; + if (cache && cached != null && cached.metadata == -1 && cached.rareState == null + && cached.addressState == null && cached.viewSize == size + && java.util.Objects.equals(cached.viewCodecClassName, codecClassName)) return cached; + Pointer result = new Pointer(cell, size, 0, size, codecClassName); + if (cache) cell.cachedProjection = result; + return result; } public static Pointer field( @@ -5459,37 +4677,51 @@ private static Pointer rootField( String fieldName, long size, String codecClassName) { + return rootField(owner, ownerClass, fieldName, size, codecClassName, null, 0); + } + + private static Pointer rootField(Object owner, Class ownerClass, String fieldName, + long size, String codecClassName, Pointer source, long displacement) { + // Zero-sized fields have no Java member. + if (size == 0 && optionalInstanceField(ownerClass, fieldName) == null) { + return finishFieldProjection(Pointer.cell(null, 0, codecClassName), source, displacement); + } FieldAccess access; try { access = fieldAccess(ownerClass, fieldName); } catch (NoSuchFieldException error) { if (size == 0) { - return Pointer.cell(null, 0, codecClassName); + return finishFieldProjection(Pointer.cell(null, 0, codecClassName), source, displacement); } throw new IllegalArgumentException( "unknown Rust field " + ownerClass.getName() + "." + fieldName, error); } - String cacheKey = access.cacheKey; - FieldCell cell; - Map>> stripe = - stateStripe(FIELD_CELLS, owner); - synchronized (stripe) { - Map> fields = stripe.get(owner); - if (fields == null) { - fields = new HashMap<>(); - stripe.put(owner, fields); - } - WeakReference reference = fields.get(cacheKey); - cell = reference == null ? null : reference.get(); - if (cell == null) { - cell = owner instanceof Cell - ? new FieldCell((Cell) owner, access) - : new FieldCell((FieldCell) owner, access); - fields.put(cacheKey, new WeakReference<>(cell)); - } - markFieldCells(owner); + return rootField(owner, access, size, codecClassName, source, displacement); + } + + private static Pointer rootField(Object owner, FieldAccess access, long size, + String codecClassName, Pointer source, long displacement) { + FieldCell cell = ownedField(owner, access); + Pointer cached = cell.cachedProjection; + boolean reusable = source != null && source.metadata == -1 && source.rareState == null; + if (reusable && cached != null && cached.metadata == -1 && cached.rareState == null + && cached.viewSize == size && java.util.Objects.equals(cached.viewCodecClassName, codecClassName)) { + AddressOriginState origin = source.addressState; + Pointer ownerOrigin = origin == null || origin.addressOrigin == null ? source : origin.addressOrigin; + long offset = origin == null || origin.addressOrigin == null ? displacement + : Math.addExact(origin.addressOriginOffset, displacement); + AddressOriginState previous = cached.addressState; + if (previous != null && previous.addressOrigin == ownerOrigin && previous.addressOriginOffset == offset) + return cached; } - return new Pointer(cell, checkedArrayLength(size), 0, size, codecClassName); + Pointer result = finishFieldProjection( + new Pointer(cell, checkedArrayLength(size), 0, size, codecClassName), source, displacement); + if (reusable) cell.cachedProjection = result; + return result; + } + + private static Pointer finishFieldProjection(Pointer pointer, Pointer source, long displacement) { + return source == null ? pointer : pointer.withMetadata(source.metadata).inheritAddressOrigin(source, displacement); } private Object compatibleStructView(String ownerClassName) { @@ -5517,10 +4749,10 @@ private boolean hasStableManagedCarrier() { protected Boolean computeValue(Class type) { Set> visited = new HashSet<>(); while (visited.add(type)) { - Field[] fields = PUBLIC_INSTANCE_FIELDS.get(type); + RustField[] fields = PUBLIC_INSTANCE_FIELDS.get(type); if (fields.length == 2) { - Field bytes = optionalInstanceField(type, "_bytes"); - Field objects = optionalInstanceField(type, "_objects"); + RustField bytes = optionalInstanceField(type, "_bytes"); + RustField objects = optionalInstanceField(type, "_objects"); return bytes != null && bytes.getType() == byte[].class && objects != null && objects.getType() == Object[].class; } @@ -5541,6 +4773,30 @@ private boolean hasByteStorageCarrier() { return value != null && value != this && BYTE_STORAGE_CARRIERS.get(value.getClass()); } + static MethodHandle fieldProjection(Class owner, String field, long offset, + long size, String codec) throws ReflectiveOperationException { + FieldAccess access = size == 0 && optionalInstanceField(owner, field) == null + ? null : fieldAccess(owner, field); + MethodHandle target = MethodHandles.lookup().findStatic(Pointer.class, "projectResolvedField", + MethodType.methodType(Pointer.class, Pointer.class, Class.class, FieldAccess.class, + String.class, long.class, long.class, String.class)); + return MethodHandles.insertArguments(target, 1, owner, access, field, offset, size, codec); + } + + private static Pointer projectResolvedField(Pointer pointer, Class owner, FieldAccess access, + String field, long offset, long size, String codec) { + if (pointer.allocation instanceof byte[] && pointer.traitMetadataCarrier() == null) { + return pointer.byteOffsetRetype(offset, size, codec); + } + if (access != null + && (pointer.allocation instanceof Cell || pointer.allocation instanceof FieldCell) + && pointer.byteOffset == 0 && pointer.viewSize == pointer.allocationElementSize + && owner.isInstance(directCellCarrier(pointer.allocation))) { + return rootField(pointer.allocation, access, size, codec, pointer, offset); + } + return pointer.projectStructField(owner.getName(), field, offset, size, codec); + } + public Pointer projectStructField( String ownerClassName, String fieldName, @@ -5561,20 +4817,11 @@ public Pointer projectStructField( long fieldOffset, long fieldSize, String fieldCodecClassName) { - Class ownerClass = null; - Class fieldType = null; - if (fieldSize != 0 || traitMetadataCarrier() != null) { - try { - ownerClass = resolvedRuntimeClass(ownerClassName); - fieldType = instanceField(ownerClass, fieldName).getType(); - } catch (ClassNotFoundException error) { - throw new IllegalArgumentException( - "unknown Rust aggregate class " + ownerClassName, error); - } catch (NoSuchFieldException error) { - throw new IllegalArgumentException( - "unknown Rust field " + ownerClassName + "." + fieldName, error); - } + // Rust field offsets locate byte storage without Java reflection or a decoded parent. + if (allocation instanceof byte[] && traitMetadataCarrier() == null) { + return byteOffsetRetype(fieldOffset, fieldSize, fieldCodecClassName); } + Class ownerClass = null; if ((allocation instanceof Cell || allocation instanceof FieldCell) && byteOffset == 0 && viewSize == allocationElementSize) { @@ -5588,15 +4835,26 @@ public Pointer projectStructField( ownerClass, fieldName, fieldSize, - fieldCodecClassName) - .withMetadata(metadata) - .inheritAddressOrigin(this, fieldOffset); + fieldCodecClassName, this, fieldOffset); } } catch (ClassNotFoundException error) { throw new IllegalArgumentException( "unknown Rust aggregate class " + ownerClassName, error); } } + Class fieldType = null; + if (fieldSize != 0 || traitMetadataCarrier() != null) { + try { + if (ownerClass == null) ownerClass = resolvedRuntimeClass(ownerClassName); + fieldType = instanceField(ownerClass, fieldName).getType(); + } catch (ClassNotFoundException error) { + throw new IllegalArgumentException( + "unknown Rust aggregate class " + ownerClassName, error); + } catch (NoSuchFieldException error) { + throw new IllegalArgumentException( + "unknown Rust field " + ownerClassName + "." + fieldName, error); + } + } boolean managedField = fieldType == null || !fieldType.isPrimitive(); // Replaceable roots and nested fields use rootField above. Only an // existing carrier may supply a live field here: decoding the whole @@ -5621,7 +4879,7 @@ public Pointer projectStructField( "unknown Rust aggregate class " + ownerClassName, error); } if (owner != null) { - return field(owner, fieldName, fieldSize, fieldCodecClassName) + return field(owner, fieldName, checkedArrayLength(fieldSize), fieldCodecClassName, false) .withMetadata(metadata) .inheritAddressOrigin(this, fieldOffset); } @@ -5668,14 +4926,7 @@ public Object projectStructSliceField( } Pointer data = byteOffsetRetype(fieldOffset, elementSize, elementCodecClassName); - try { - Class sliceView = resolvedRuntimeClass(SLICE_VIEW_CLASS_NAME); - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(sliceView) - .newInstance(data, 0, metadata()); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not construct Rust DST slice view", error); - } + return SliceView.create(SLICE_VIEW_CLASS_NAME, data, 0, metadata()); } public Object projectStructStrField( @@ -5692,14 +4943,7 @@ public Object projectStructStrField( } Pointer data = byteOffsetRetype(fieldOffset, 1, null); - try { - Class utf8View = resolvedRuntimeClass(UTF8_VIEW_CLASS_NAME); - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(utf8View) - .newInstance(data, 0, metadata()); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not construct Rust DST string view", error); - } + return SliceView.create(UTF8_VIEW_CLASS_NAME, data, 0, metadata()); } public static Pointer array( @@ -5732,8 +4976,7 @@ public static Pointer array( -1) .withMetadata(origin.metadata); if (activeMemoryViewMatches(array, origin)) { - return source.byte_offset(relativeOffset) - .retype(elementSize, codecClassName); + return source.byteOffsetRetype(relativeOffset, elementSize, codecClassName); } return new Pointer( array, @@ -5767,20 +5010,25 @@ public static Pointer array(Object array, int elementOffset) { } public static Pointer allocateBytes(long byteCount, long alignment) { - try { - int size = checkedArrayLength(byteCount); - int checkedAlignment = checkedAlignment(alignment); - byte[] bytes = new byte[size]; - recordAlignment(bytes, checkedAlignment); - Pointer pointer = new Pointer(bytes, 1, 0, 1, null); - synchronized (ALLOCATIONS) { - ALLOCATOR_OWNED_ALLOCATIONS.put(bytes, Boolean.TRUE); - } - return pointer; - } catch (IllegalArgumentException | ArithmeticException | OutOfMemoryError failure) { - // GlobalAlloc reports allocation failure with a null pointer. In - // particular, Rust's usize range is much larger than a JVM array. - return null; + return fromLocation(Heap.allocate(byteCount, alignment), 0, 1); + } + + static void registerHeapAllocation(Object allocation, int alignment) { + recordAlignment(allocation, alignment); + synchronized (ALLOCATIONS) { + ALLOCATOR_OWNED_ALLOCATIONS.put(allocation, Boolean.TRUE); + } + } + + static void copyHeapAllocation(Object root, long offset, byte[] destination, int size) { + if (root instanceof byte[] + && !mayBeInIdentityFilter(MEMORY_VIEW_FILTER, root) + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root) + && !mayBeInIdentityFilter(ENCODED_POINTER_FILTER, root) + && !mayBeInIdentityFilter(ENCODED_REFERENCE_FILTER, root)) { + System.arraycopy(root, Math.toIntExact(offset), destination, 0, size); + } else { + copy(fromLocation(root, offset, 1), array(destination, 0, 1), size); } } @@ -5891,7 +5139,6 @@ public static Pointer constantCell( }); } if (value != null - && !value.getClass().isArray() && pointer.allocation != null && pointer.allocation.getClass().isArray() && !pointer.allocation.getClass().getComponentType().isPrimitive() @@ -5998,38 +5245,32 @@ public static Pointer constantArray( recordAlignment(value, checkedAlignment); return array(value, 0, checkedSize, elementCodecClassName); }); + if (Array.getLength(value) == 1 + && pointer.allocation != null + && value.getClass().getComponentType().isArray() + && value.getClass().getComponentType() == pointer.allocation.getClass()) { + // CTFE can share an array with its sole element, which can also be an array. + // Preserve the element value and the original allocation identity. + return cellAligned(pointer.allocation, checkedSize, + elementCodecClassName, checkedAlignment) + .inheritAddressOrigin(pointer, 0); + } return pointer.retype(checkedSize, elementCodecClassName); } public static Pointer reallocateBytes( - Pointer source, - long oldByteCount, - long alignment, - long newByteCount) { - if (source == null) { - throw new NullPointerException("Rust realloc requires a non-null pointer"); - } - try { - int oldSize = checkedArrayLength(oldByteCount); - int newSize = checkedArrayLength(newByteCount); - Pointer destination = allocateBytes(newSize, alignment); - if (destination == null) { - return null; - } - copy(source, destination, Math.min(oldSize, newSize)); - deallocateBytes(source); - return destination; - } catch (IllegalArgumentException | ArithmeticException | OutOfMemoryError failure) { - return null; - } + Pointer source, long oldByteCount, long alignment, long newByteCount) { + return fromLocation(Heap.reallocate(source, 0, oldByteCount, alignment, newByteCount), 0, 1); } /** Releases a byte allocation after Rust's allocator has ended its lifetime. */ public static void deallocateBytes(Pointer pointer) { - if (pointer == null || pointer.allocation == null) { - return; - } - Object allocation = pointer.allocation; + Heap.deallocate(pointer, 0); + } + + static void releaseHeapAllocation(Object allocation) { + if (allocation instanceof Pointer) allocation = ((Pointer) allocation).allocation; + if (allocation == null) return; synchronized (ALLOCATIONS) { ALLOCATOR_OWNED_ALLOCATIONS.remove(allocation); AllocationInfo info = ALLOCATIONS.remove(allocation); @@ -6072,7 +5313,7 @@ public static void deallocateBytes(Pointer pointer) { discardEncodedReferences(allocation); } - private static int checkedAlignment(long alignment) { + static int checkedAlignment(long alignment) { if (alignment <= 0 || alignment > Integer.MAX_VALUE || (alignment & (alignment - 1L)) != 0) { @@ -6095,61 +5336,63 @@ public static Pointer fromSlice( if (sliceView instanceof Pointer) { return ((Pointer) sliceView).retype(elementSize, codecClassName); } - try { - SliceAccess access = SLICE_ACCESSES.get(sliceView.getClass()); - Object backing = access.array(sliceView); - int offset = access.offset(sliceView); - long length = access.rustLength(sliceView); - if (backing instanceof Pointer) { - return ((Pointer) backing) - .sliceStorageView(elementSize, codecClassName) - .add(offset) - .withMetadata(length); - } - MemoryViewOrigin origin = null; - if (mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, backing)) { - Map originStripe = - stateStripe(MEMORY_VIEW_ORIGINS, backing); - synchronized (originStripe) { - origin = originStripe.get(backing); - } + SliceView view = (SliceView) sliceView; + return fromSliceParts(view.array, view.offset, view.rustLength, elementSize, codecClassName); + } + + /** Extract an address from SSA view components without constructing a view object. */ + public static Pointer fromSliceParts( + Object backing, int offset, long length, int elementSize, String codecClassName) { + if (backing == null) { + return withoutProvenance(Math.multiplyExact((long) offset, elementSize), + elementSize, codecClassName).withMetadata(length); + } + if (backing instanceof Pointer) { + return ((Pointer) backing) + .sliceStorageView(elementSize, codecClassName) + .add(offset) + .withMetadata(length); + } + MemoryViewOrigin origin = null; + if (mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, backing)) { + Map originStripe = + stateStripe(MEMORY_VIEW_ORIGINS, backing); + synchronized (originStripe) { + origin = originStripe.get(backing); } - if (origin != null) { - Object allocation = origin.allocation.get(); - if (allocation != null) { - long relativeOffset = Math.multiplyExact((long) offset, elementSize); - Pointer source = new Pointer( - allocation, - origin.allocationElementSize, - origin.byteOffset, - origin.viewSize, - origin.allocationCodecClassName, - origin.allocationCodecClassName, - -1) - .withMetadata(origin.metadata); - if (activeMemoryViewMatches(backing, origin)) { - return source.byte_offset(relativeOffset) - .retype(elementSize, codecClassName) - .withMetadata(length); - } - return new Pointer( - backing, - elementSize, - relativeOffset, - elementSize, - codecClassName) - .inheritAddressOrigin(source, relativeOffset) + } + if (origin != null) { + Object allocation = origin.allocation.get(); + if (allocation != null) { + long relativeOffset = Math.multiplyExact((long) offset, elementSize); + Pointer source = new Pointer( + allocation, + origin.allocationElementSize, + origin.byteOffset, + origin.viewSize, + origin.allocationCodecClassName, + origin.allocationCodecClassName, + -1) + .withMetadata(origin.metadata); + if (activeMemoryViewMatches(backing, origin)) { + return source.byteOffsetRetype(relativeOffset, elementSize, codecClassName) .withMetadata(length); } + return new Pointer( + backing, + elementSize, + relativeOffset, + elementSize, + codecClassName) + .inheritAddressOrigin(source, relativeOffset) + .withMetadata(length); } - return array( - backing, - offset, - elementSize, - codecClassName).withMetadata(length); - } catch (ReflectiveOperationException error) { - throw new IllegalArgumentException("invalid Rust slice view", error); } + return array( + backing, + offset, + elementSize, + codecClassName).withMetadata(length); } public static Pointer fromSlice(Object sliceView, int elementSize) { @@ -6164,19 +5407,12 @@ public static Pointer fromSlice( if (sliceView instanceof Pointer) { return ((Pointer) sliceView).retype(elementSize, codecClassName); } - try { - SliceAccess access = SLICE_ACCESSES.get(sliceView.getClass()); - Object backing = access.array(sliceView); - if (backing instanceof Pointer) { - int offset = access.offset(sliceView); - long length = access.rustLength(sliceView); - return ((Pointer) backing) - .sliceStorageView(elementSize, codecClassName) - .add(offset) - .withMetadata(length); - } - } catch (ReflectiveOperationException error) { - throw new IllegalArgumentException("invalid Rust slice view", error); + SliceView view = (SliceView) sliceView; + if (view.array instanceof Pointer) { + return ((Pointer) view.array) + .sliceStorageView(elementSize, codecClassName) + .add(view.offset) + .withMetadata(view.rustLength); } return fromSlice(sliceView, checkedArrayLength(elementSize), codecClassName); } @@ -6189,17 +5425,12 @@ public static Pointer fromSlice(Object sliceView) { if (sliceView == null) { return nullPointer(); } - try { - Object array = SLICE_ACCESSES.get(sliceView.getClass()).array(sliceView); - if (array instanceof Pointer) { - Pointer pointer = (Pointer) array; - return fromSlice( - sliceView, pointer.viewSize, pointer.viewCodecClassName); - } - return fromSlice(sliceView, inferredArrayElementSize(array)); - } catch (ReflectiveOperationException error) { - throw new IllegalArgumentException("invalid Rust slice view", error); + Object array = ((SliceView) sliceView).array; + if (array instanceof Pointer) { + Pointer pointer = (Pointer) array; + return fromSlice(sliceView, pointer.viewSize, pointer.viewCodecClassName); } + return fromSlice(sliceView, inferredArrayElementSize(array)); } public static Pointer nullPointer(int viewSize) { @@ -6480,10 +5711,14 @@ private static Pointer canonicalFunctionPointer(Object value) { Map cells = FUNCTION_POINTER_CELLS; Object identity = FunctionPointers.identity(value); if (value instanceof FunctionPointerAdapter) { - identity = canonicalFunctionPointer( - ((FunctionPointerAdapter) value).functionPointerTarget()); - cells = FUNCTION_POINTER_ADAPTER_CELLS.computeIfAbsent( - value.getClass(), key -> new IdentityHashMap<>()); + Object target = ((FunctionPointerAdapter) value).functionPointerTarget(); + if (target instanceof MethodHandle) { + identity = FunctionPointers.identity(target); + } else { + identity = canonicalFunctionPointer(target); + cells = FUNCTION_POINTER_ADAPTER_CELLS.computeIfAbsent( + value.getClass(), key -> new IdentityHashMap<>()); + } } Pointer existing = cells.get(identity); if (existing != null) { @@ -6553,15 +5788,7 @@ public static Object stringView(String value, String viewClassName) { if (cached != null) { return cached; } - try { - Class viewClass = resolvedRuntimeClass(viewClassName); - cached = SLICE_VIEW_CONSTRUCTORS.get(viewClass) - .newInstance(views.bytes, 0, views.bytes.length); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException( - "could not construct cached Rust string view " + viewClassName, - error); - } + cached = SliceView.create(viewClassName, views.bytes, 0, views.bytes.length); if (utf8) { views.utf8 = cached; } else { @@ -6601,8 +5828,12 @@ private static Pointer typedPointerObjectFromAddress(long address, String pointe } private static Pointer pointerFromExposedTarget(ExposedTarget target) { + return pointerFromExposedTarget(target, target.allocation); + } + + private static Pointer pointerFromExposedTarget(ExposedTarget target, Object allocation) { Pointer pointer = new Pointer( - target.allocation, + allocation, target.allocationElementSize, target.byteOffset, target.viewSize, @@ -6640,8 +5871,8 @@ public static Pointer fromEncodedAddress( long viewSize, String viewCodecClassName, String pointerCodec) { - return typedPointerObjectFromAddress(address, pointerCodec) - .retype(viewSize, viewCodecClassName); + return retypeDecodedPointer(typedPointerObjectFromAddress(address, pointerCodec), + viewSize, viewCodecClassName); } /** Decodes an integer-to-pointer transmute without acquiring exposed provenance. */ @@ -6828,6 +6059,7 @@ public static Pointer attachTraitObjectCarrier( && (pointeeAlignment & (pointeeAlignment - 1)) != 0)) { throw new IllegalArgumentException("invalid trait-object pointee layout"); } + pointer = independentFieldMetadata(pointer); pointer.traitObjectCarrier(carrier); pointer.traitPointeeSize(pointeeSize); pointer.traitPointeeAlignment(pointeeAlignment); @@ -6879,6 +6111,7 @@ public static Pointer attachStructTailTraitMetadata(Pointer pointer, Pointer tai "struct-tail trait pointer does not carry dynamic metadata"); } TraitObjectCarrier traitCarrier = (TraitObjectCarrier) carrier; + pointer = independentFieldMetadata(pointer); pointer.traitMetadataCarrier(carrier); pointer.traitMetadataMarker(tailPointer.traitMetadataMarker()); pointer.traitPointeeSize(tailPointer.traitPointeeSize() >= 0 @@ -6947,6 +6180,7 @@ public static Pointer retypeStructTailWithMetadataOf( String newViewCodecClassName) { Pointer result = attachStructTailTraitMetadata( pointer.retype(newViewSize, newViewCodecClassName), metadataSource); + result.traitObjectCarrier(null); if (result.traitAdapterClassName() == null) { return result; } @@ -6968,10 +6202,8 @@ public static Pointer retypeStructTailFromTraitPointer( Pointer pointer, long newViewSize, String newViewCodecClassName) { - Pointer result = pointer.retype(newViewSize, newViewCodecClassName); - result.traitObjectCarrier(null); - result.traitMetadataCarrier(null); - return attachStructTailTraitMetadata(result, pointer); + return retypeStructTailWithMetadataOf( + pointer, pointer, newViewSize, newViewCodecClassName); } /** Creates a coherent JVM carrier view for a Rust struct-tail unsizing coercion. */ @@ -7006,7 +6238,7 @@ public static Pointer unsizeStruct( try { Object source = pointer.getObject(); Class targetClass = resolvedRuntimeClass(targetClassName); - for (Field targetField : PUBLIC_INSTANCE_FIELDS.get(targetClass)) { + for (RustField targetField : PUBLIC_INSTANCE_FIELDS.get(targetClass)) { if (!isSliceViewCarrierType(targetField.getType())) { continue; } @@ -7106,14 +6338,7 @@ public static Object restoreErasedSliceView(Pointer pointer, String viewClassNam throw new IllegalArgumentException("invalid erased Rust slice view"); } Pointer data = restoreErasedView(pointer); - try { - Class viewClass = resolvedRuntimeClass(viewClassName); - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(viewClass) - .newInstance(data, 0, pointer.metadata()); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not rebuild erased Rust slice view", error); - } + return SliceView.create(viewClassName, data, 0, pointer.metadata()); } private static Pointer traitMetadataPointer(Object metadata, int depth) @@ -7124,7 +6349,7 @@ private static Pointer traitMetadataPointer(Object metadata, int depth) if (metadata == null || depth == 0) { return null; } - for (Field field : PUBLIC_INSTANCE_FIELDS.get(metadata.getClass())) { + for (RustField field : PUBLIC_INSTANCE_FIELDS.get(metadata.getClass())) { Object nested = field.get(metadata); Pointer marker = traitMetadataPointer(nested, depth - 1); if (marker != null) { @@ -7176,6 +6401,15 @@ public static Pointer fromRawTraitParts(Pointer data, Object metadata) { } } + /** Rebuilds a trait-tailed struct using the supplied vtable, not stale data metadata. */ + public static Pointer fromRawStructTraitParts( + Pointer data, long prefixSize, String prefixCodec, Object metadata) { + Pointer tail = fromRawTraitParts(data.byte_offset(prefixSize), metadata); + Pointer result = data.retype(prefixSize, prefixCodec); + result.traitObjectCarrier(null); + return attachStructTailTraitMetadata(result, tail); + } + private Pointer withMetadata(long metadata) { this.metadata = metadata; return this; @@ -7205,36 +6439,45 @@ private Pointer copyDynamicMetadata(Pointer source) { } private Pointer inheritAddressOrigin(Pointer source, long additionalOffset) { - Pointer origin = source.addressOrigin(); - // A decoded carrier's origin is weakly indexed. Keep its backing cell - // alive while any projected pointer can still mutate that carrier; - // flattening past it lets GC detach live aliases from their storage. - if (origin == null || source.boundMemoryViewState() != null) { + AddressOriginState current = source.addressState; + // Keep decoded backing cells alive for as long as a projected alias. + if (current == null || current.addressOrigin == null || source.boundMemoryViewState() != null) { setAddressOrigin(source, additionalOffset); + } else if (additionalOffset == 0) { + addressState = current; } else { - setAddressOrigin( - origin, - Math.addExact(source.addressOriginOffset(), additionalOffset)); + setAddressOrigin(current.addressOrigin, + Math.addExact(current.addressOriginOffset, additionalOffset)); } return this; } private Pointer copyAddressOrigin(Pointer source, long additionalOffset) { - Pointer origin = source.addressOrigin(); - if (origin != null) { - setAddressOrigin( - origin, - Math.addExact(source.addressOriginOffset(), additionalOffset)); + AddressOriginState current = source.addressState; + if (current != null && current.addressOrigin != null) { + if (additionalOffset == 0) { + addressState = current; + } else { + // Origin offsets are address words and must support wrapping arithmetic. + setAddressOrigin(current.addressOrigin, + current.addressOriginOffset + additionalOffset); + } } return this; } + private static Pointer independentFieldMetadata(Pointer pointer) { + // Detach metadata even if this field pointer is no longer the cached projection. + return pointer.allocation instanceof FieldCell + ? pointer.retype(pointer.viewSize, pointer.viewCodecClassName) : pointer; + } + public static Pointer withMetadata(Pointer pointer, long metadata) { - return pointer.withMetadata(metadata); + return (pointer.metadata == metadata ? pointer : independentFieldMetadata(pointer)).withMetadata(metadata); } public static Pointer withMetadata(Pointer pointer, int metadata) { - return pointer.withMetadata(Integer.toUnsignedLong(metadata)); + return withMetadata(pointer, Integer.toUnsignedLong(metadata)); } public long metadata() { @@ -7545,17 +6788,24 @@ private Pointer wrappingByteOffset(long byteCount) { } public long align_offset(long alignment) { + return alignmentOffset(numericAddress(), viewSize, alignment); + } + + private static long alignmentOffset(long address, long stride, long alignment) { if (alignment <= 0 || (alignment & (alignment - 1)) != 0) { throw new IllegalArgumentException("Rust pointer alignment must be a power of two"); } - long current = numericAddress(); - long attempts = viewSize == 0 ? 1 : alignment; - for (long elements = 0; elements < attempts; elements++) { - if ((current + (long) elements * viewSize) % alignment == 0) { - return elements; - } - } - return -1; + if (stride < 0) throw new IllegalArgumentException("negative Rust pointee size"); + long divisor = stride == 0 ? alignment : Math.min(Long.lowestOneBit(stride), alignment); + if ((address & (divisor - 1)) != 0) return -1; + long mask = alignment / divisor - 1; + if (mask == 0) return 0; + // Solve address + n * stride = 0 modulo the power-of-two alignment. + // Divide by the gcd to get an odd stride with an inverse modulo 2^64. + long odd = stride / divisor; + long inverse = odd; + for (int i = 0; i < 6; i++) inverse *= 2 - odd * inverse; + return -(address >>> Long.numberOfTrailingZeros(divisor)) * inverse & mask; } public static long align_offset(Pointer pointer, long alignment) { @@ -7800,45 +7050,6 @@ private boolean samePointerMetadataWords(Pointer other) { return markerAdapter != null && markerAdapter.equals(unmaterializedAdapter); } - /** Compares deferred pointer data and metadata words without materialization. */ - public static boolean samePointerRelative( - Pointer left, - long leftElementOffset, - long leftByteOffset, - Pointer right, - long rightElementOffset, - long rightByteOffset) { - if (left == null || right == null) { - return false; - } - if (left.traitObjectCarrier() != null - || left.traitAdapterClassName() != null - || left.traitMetadataMarker() != null - || right.traitObjectCarrier() != null - || right.traitAdapterClassName() != null - || right.traitMetadataMarker() != null) { - return materializeRelative(left, leftElementOffset, leftByteOffset) - .samePointer(materializeRelative( - right, rightElementOffset, rightByteOffset)); - } - long leftDisplacement = Math.addExact( - Math.multiplyExact(leftElementOffset, left.viewSize), leftByteOffset); - long rightDisplacement = Math.addExact( - Math.multiplyExact(rightElementOffset, right.viewSize), rightByteOffset); - boolean sameAddress; - if (left.allocation != null && left.allocation == right.allocation) { - sameAddress = Math.addExact(left.byteOffset, leftDisplacement) - == Math.addExact(right.byteOffset, rightDisplacement); - } else if (left.allocation == null && right.allocation == null) { - sameAddress = Math.addExact(left.exposedAddress, leftDisplacement) - == Math.addExact(right.exposedAddress, rightDisplacement); - } else { - sameAddress = Math.addExact(left.numericAddress(), leftDisplacement) - == Math.addExact(right.numericAddress(), rightDisplacement); - } - return sameAddress && left.samePointerMetadataWords(right); - } - /** Compares both words of a Rust slice/str fat pointer. */ public static boolean fatPointerEquals(Object left, Object right) { if (left == right) { @@ -7901,6 +7112,7 @@ public int compareAddress(Pointer other) { if (other != null && allocation != null && allocation == other.allocation + && allocationElementSize > 0 && other.allocationElementSize > 0 && addressOrigin() == null && other.addressOrigin() == null && byteOffset >= 0 @@ -7962,6 +7174,26 @@ public boolean greaterOrEqual(Pointer other) { return compareAddress(other) >= 0; } + /** Test a reference or NonNull niche without exposing an address or creating a boundary. */ + public static long nullableLocationTag(Object root, long offset) { + if (root == null) return offset == 0 ? 0 : 1; + if (!(root instanceof Pointer)) return 1; + Pointer pointer = (Pointer) root; + if (pointer.allocation != null) return 1; + long address = pointer.addressOrigin() == null + ? pointer.exposedAddress : pointer.numericAddress(); + return address + offset == 0 ? 0 : 1; + } + + public static long nullableTag(Pointer pointer) { + return nullableLocationTag(pointer, 0); + } + + /** Test the data address for null. An empty array or string still has a non-null root. */ + public static long nullableViewLocationTag(Object root, int start) { + return start == 0 ? nullableLocationTag(root, 0) : 1; + } + public static boolean is_null(Pointer pointer) { return pointer == null || pointer.numericAddress() == 0; } @@ -8048,9 +7280,10 @@ private static void discardCollectedTypedExposedTargets(int limit) { } } - public static Object asRefOption(Pointer pointer, String optionClassName) { - String variantName = optionClassName + (is_null(pointer) ? "$None" : "$Some"); + public static Object asRefOption(Pointer pointer, String someClassName, String noneClassName) { + String variantName = is_null(pointer) ? noneClassName : someClassName; try { + // Shared enum interfaces need not own the payload class. Use the exact compiler-supplied variant. Class variant = resolvedRuntimeClass(variantName); if (is_null(pointer)) { return constructorWithArity(variant, 0).newInstance(); @@ -8069,14 +7302,14 @@ public static Object asRefOption(Pointer pointer, String optionClassName) { return constructor.newInstance(referent); } catch (ReflectiveOperationException error) { throw new IllegalStateException( - "could not construct Rust pointer option " + optionClassName, error); + "could not construct Rust pointer option " + variantName, error); } } private long numericAddress() { Pointer origin = addressOrigin(); if (origin != null) { - return Math.addExact(origin.numericAddress(), addressOriginOffset()); + return origin.numericAddress() + addressOriginOffset(); } if (allocation == null) { return exposedAddress; @@ -8088,11 +7321,12 @@ private long numericAddress() { Long cachedBase = cachedAllocationBase(ALLOCATION_BASE_CACHE, allocation); if (cachedBase != null) { - return Math.addExact(cachedBase.longValue(), byteOffset); + return cachedBase.longValue() + byteOffset; } synchronized (ALLOCATIONS) { AllocationInfo info = allocationInfo(allocation); - return Math.addExact(allocationBase(info), byteOffset); + // Address arithmetic can wrap. Memory accesses check their own ranges. + return allocationBase(info) + byteOffset; } } @@ -8102,6 +7336,10 @@ private static long numericAddress(Pointer pointer) { /** Must be called while holding {@link #ALLOCATIONS}. */ private long allocationBase(AllocationInfo info) { + return allocationBase(allocation, allocationElementSize, info); + } + + private static long allocationBase(Object allocation, int allocationElementSize, AllocationInfo info) { Long cached = cachedAllocationBase(ALLOCATION_BASE_CACHE, allocation); if (cached != null) { return cached.longValue(); @@ -8159,9 +7397,7 @@ public static long encodedAddress(Pointer pointer, Object owner) { } Pointer origin = pointer.addressOrigin(); if (origin != null) { - long address = Math.addExact( - encodedAddress(origin, owner), - pointer.addressOriginOffset()); + long address = encodedAddress(origin, owner) + pointer.addressOriginOffset(); pointer.setPublishedAddress(address); return address; } @@ -8172,8 +7408,7 @@ public static long encodedAddress(Pointer pointer, Object owner) { PUBLISHED_ALLOCATION_BASE_CACHE, pointer.allocation); long address; if (publishedBase != null) { - address = Math.addExact( - publishedBase.longValue(), pointer.byteOffset); + address = publishedBase.longValue() + pointer.byteOffset; } else { synchronized (ALLOCATIONS) { AllocationInfo info = allocationInfo(pointer.allocation); @@ -8182,7 +7417,7 @@ public static long encodedAddress(Pointer pointer, Object owner) { PUBLISHED_ALLOCATION_BASE_CACHE, pointer.allocation, base); - address = Math.addExact(base, pointer.byteOffset); + address = base + pointer.byteOffset; } } pointer.setPublishedAddress(address); @@ -8228,7 +7463,8 @@ public static long encodedAddress( ownerOffset, encodedSize, pointerCodec, - target); + target, + address); } return address; } @@ -8289,87 +7525,31 @@ private ExposedTarget exposedTargetAt(long displacement) { public long address() { Pointer origin = addressOrigin(); if (origin != null) { - long address = Math.addExact(origin.address(), addressOriginOffset()); + long address = origin.address() + addressOriginOffset(); setPublishedAddress(address); return address; } if (allocation == null) { - return exposedAddress; - } - ExposedTarget cachedTarget = publishedTarget(); - if (cachedTarget != null && cachedTarget.matches(this)) { - return publishedAddress(); - } - synchronized (ALLOCATIONS) { - AllocationInfo info = allocationInfo(allocation); - long base = publishAllocationRange(info); - long address = base + byteOffset; - ExposedTarget target = exposedTarget(); - registerExposedTarget(address, target); - setPublishedTarget(address, target); - return address; - } - } - - /** Returns a Rust pointer's address, including Java {@code null} as address zero. */ - public static long address(Pointer pointer) { - return pointer == null ? 0L : pointer.address(); - } - - public static long addressRelative( - Pointer base, long elementOffset, long byteOffset) { - if (base == null) { - return 0L; - } - long displacement = Math.addExact( - Math.multiplyExact(elementOffset, base.viewSize), byteOffset); - if (displacement == 0) { - return base.address(); - } - Pointer origin = base.addressOrigin(); - if (origin != null) { - return Math.addExact( - origin.address(), - Math.addExact(base.addressOriginOffset(), displacement)); + return exposedAddress; } - if (base.allocation == null) { - return Math.addExact(base.exposedAddress, displacement); + ExposedTarget cachedTarget = publishedTarget(); + if (cachedTarget != null && cachedTarget.matches(this)) { + return publishedAddress(); } synchronized (ALLOCATIONS) { - AllocationInfo info = allocationInfo(base.allocation); - long address = Math.addExact( - base.publishAllocationRange(info), - Math.addExact(base.byteOffset, displacement)); - registerExposedTarget(address, base.exposedTargetAt(displacement)); + AllocationInfo info = allocationInfo(allocation); + long base = publishAllocationRange(info); + long address = base + byteOffset; + ExposedTarget target = exposedTarget(); + registerExposedTarget(address, target); + setPublishedTarget(address, target); return address; } } - public static long addrRelative( - Pointer base, long elementOffset, long byteOffset) { - if (base == null) { - return 0L; - } - long displacement = Math.addExact( - Math.multiplyExact(elementOffset, base.viewSize), byteOffset); - return Math.addExact(base.numericAddress(), displacement); - } - - public static boolean isNullRelative( - Pointer base, long elementOffset, long byteOffset) { - return base == null || addrRelative(base, elementOffset, byteOffset) == 0; - } - - public static boolean isAlignedRelative( - Pointer base, - long elementOffset, - long byteOffset, - long alignment) { - if (alignment <= 0 || (alignment & (alignment - 1)) != 0) { - throw new IllegalArgumentException( - "is_aligned_to: align is not a power-of-two"); - } - return (addrRelative(base, elementOffset, byteOffset) & (alignment - 1)) == 0; + /** Returns a Rust pointer's address, including Java {@code null} as address zero. */ + public static long address(Pointer pointer) { + return pointer == null ? 0L : pointer.address(); } /** @@ -8585,7 +7765,7 @@ private static int primitiveArrayElementSize(Object value, int aggregateSize) { private static int loadPrimitiveArrayByte(Object array, int byteOffset, int elementSize) { int elementIndex = byteOffset / elementSize; int withinElement = byteOffset % elementSize; - long bits = valueBits(arrayGet(array, elementIndex), elementSize); + long bits = primitiveArrayBits(array, elementIndex); return (int) ((bits >>> (withinElement * 8)) & 0xffL); } @@ -8593,15 +7773,14 @@ private static void storePrimitiveArrayByte( Object array, int byteOffset, int elementSize, int incoming) { int elementIndex = byteOffset / elementSize; int withinElement = byteOffset % elementSize; - Object current = arrayGet(array, elementIndex); - long bits = valueBits(current, elementSize); + long bits = primitiveArrayBits(array, elementIndex); long mask = 0xffL << (withinElement * 8); long updated = (bits & ~mask) | (((long) incoming & 0xffL) << (withinElement * 8)); - arraySet(array, elementIndex, carrierFromBits(current, updated, elementSize)); + storePrimitiveArrayBits(array, elementIndex, updated); } private static long loadArrayCodecBits( - Object array, int byteOffset, int byteCount, CodecPlan arrayPlan) { + Object array, int byteOffset, int byteCount, MemoryCodec arrayPlan) { if (!arrayPlan.isArrayCodecFor(array)) { throw new IllegalArgumentException("pointer codec is not a fixed-array codec"); } @@ -8633,7 +7812,7 @@ private static long loadArrayCodecBits( byte[] image; boolean temporary = false; if (isGeneratedAggregateCodec(arrayPlan.arrayElementCodec)) { - CodecPlan elementPlan = codecPlan(arrayPlan.arrayElementCodec); + MemoryCodec elementPlan = codecPlan(arrayPlan.arrayElementCodec); if (elementPlan.isArrayCodecFor(element)) { long selected = loadArrayCodecBits(element, withinElement, chunk, elementPlan); @@ -8643,7 +7822,7 @@ private static long loadArrayCodecBits( } image = elementPlan.directUnionBytes(element); if (image == null) { - image = elementPlan.encode.encode(element); + image = elementPlan.encode().encode(element); temporary = true; } } else { @@ -8676,7 +7855,7 @@ private static void loadArrayCodecRange( byte[] target, int targetOffset, int byteCount, - CodecPlan arrayPlan) { + MemoryCodec arrayPlan) { if (!arrayPlan.isArrayCodecFor(array)) { throw new IllegalArgumentException("pointer codec is not a fixed-array codec"); } @@ -8711,7 +7890,7 @@ private static void loadArrayCodecRange( byte[] image; boolean temporary = false; if (isGeneratedAggregateCodec(arrayPlan.arrayElementCodec)) { - CodecPlan elementPlan = codecPlan(arrayPlan.arrayElementCodec); + MemoryCodec elementPlan = codecPlan(arrayPlan.arrayElementCodec); if (elementPlan.isArrayCodecFor(element)) { loadArrayCodecRange( element, @@ -8725,7 +7904,7 @@ private static void loadArrayCodecRange( } image = elementPlan.directUnionBytes(element); if (image == null) { - image = elementPlan.encode.encode(element); + image = elementPlan.encode().encode(element); temporary = true; } } else { @@ -8762,7 +7941,7 @@ private static void loadArrayCodecRange( } private static void storeArrayCodecBits( - Object array, int byteOffset, long incoming, int byteCount, CodecPlan arrayPlan) { + Object array, int byteOffset, long incoming, int byteCount, MemoryCodec arrayPlan) { if (!arrayPlan.isArrayCodecFor(array)) { throw new IllegalArgumentException("pointer codec is not a fixed-array codec"); } @@ -8799,7 +7978,7 @@ private static void storeArrayCodecBits( } if (isGeneratedAggregateCodec(arrayPlan.arrayElementCodec)) { - CodecPlan elementPlan = codecPlan(arrayPlan.arrayElementCodec); + MemoryCodec elementPlan = codecPlan(arrayPlan.arrayElementCodec); if (elementPlan.isArrayCodecFor(element)) { storeArrayCodecBits( element, withinElement, selected, chunk, elementPlan); @@ -8850,7 +8029,7 @@ private static void storeArrayCodecRange( byte[] source, int sourceOffset, int byteCount, - CodecPlan arrayPlan) { + MemoryCodec arrayPlan) { if (!arrayPlan.isArrayCodecFor(array)) { throw new IllegalArgumentException("pointer codec is not a fixed-array codec"); } @@ -8889,7 +8068,7 @@ private static void storeArrayCodecRange( } if (isGeneratedAggregateCodec(arrayPlan.arrayElementCodec)) { - CodecPlan elementPlan = codecPlan(arrayPlan.arrayElementCodec); + MemoryCodec elementPlan = codecPlan(arrayPlan.arrayElementCodec); if (elementPlan.isArrayCodecFor(element)) { storeArrayCodecRange( element, @@ -8993,6 +8172,8 @@ private boolean storeEmbeddedArrayBits( long currentBits = primitiveArrayBits(array, elementIndex); long updated = (currentBits & ~mask) | ((incoming & valueMask) << shift); storePrimitiveArrayBits(array, elementIndex, updated); + // In-place writes must reach the decoded field owner's storage. + if (allocation instanceof FieldCell) ((FieldCell) allocation).commitOwners(); return true; } @@ -9043,12 +8224,8 @@ private long loadUnsignedAt(long absoluteByteOffset, int byteCount) { throw new IndexOutOfBoundsException( "scalar load exceeds byte-addressable Rust storage"); } - flushMemoryViewsOverlapping(absoluteByteOffset, byteCount); - long result = 0; - for (int index = 0; index < byteCount; index++) { - result |= ((long) bytes[offset + index] & 0xffL) << (index * 8); - } - return result; + flushFilteredMemoryViews(absoluteByteOffset, byteCount, false); + return MemoryBytes.read(bytes, offset, byteCount); } return loadUnsignedAtSlow(absoluteByteOffset, byteCount); } @@ -9096,14 +8273,14 @@ private long loadUnsignedAtSlow(long absoluteByteOffset, int byteCount) { encoded = encodeFatPointer( value, allocationElementSize, allocationCodecClassName); } else if (isGeneratedAggregateCodec(allocationCodecClassName)) { - CodecPlan plan = codecPlan(allocationCodecClassName); + MemoryCodec plan = codecPlan(allocationCodecClassName); if (plan.isArrayCodecFor(value)) { return loadArrayCodecBits( value, withinElement, byteCount, plan); } directUnionBytes = plan.directUnionBytes(value); encoded = directUnionBytes == null - ? plan.encode.encode(value) + ? plan.encode().encode(value) : directUnionBytes; } if (encoded != null) { @@ -9194,12 +8371,12 @@ private int loadByte(long absoluteByteOffset, boolean flushMemoryViews) { return result; } if (isGeneratedAggregateCodec(allocationCodecClassName)) { - CodecPlan plan = codecPlan(allocationCodecClassName); + MemoryCodec plan = codecPlan(allocationCodecClassName); if (plan.isArrayCodecFor(value)) { return (int) loadArrayCodecBits(value, withinElement, 1, plan); } byte[] direct = plan.directUnionBytes(value); - byte[] bytes = direct == null ? plan.encode.encode(value) : direct; + byte[] bytes = direct == null ? plan.encode().encode(value) : direct; if (withinElement >= bytes.length) { if (direct == null) { discardEncodedReferences(bytes); @@ -9305,7 +8482,7 @@ private byte[] loadRange(int byteCount) { image = encodeFatPointer(value, allocationElementSize, allocationCodecClassName); } else if (!directPrimitiveArray && isGeneratedAggregateCodec(allocationCodecClassName)) { - CodecPlan plan = codecPlan(allocationCodecClassName); + MemoryCodec plan = codecPlan(allocationCodecClassName); if (plan.isArrayCodecFor(value)) { loadArrayCodecRange( value, @@ -9330,7 +8507,7 @@ && isGeneratedAggregateCodec(allocationCodecClassName)) { consumed += chunk; continue; } - image = plan.encode.encode(value); + image = plan.encode().encode(value); } if (image != null) { if (withinElement + chunk > image.length) { @@ -9378,9 +8555,7 @@ private void storeBytesAt(long absoluteByteOffset, long bits, int byteCount) { } prepareMemoryWrite(absoluteByteOffset, byteCount); discardEncodedPointers(allocation, absoluteByteOffset, byteCount); - for (int index = 0; index < byteCount; index++) { - bytes[offset + index] = (byte) (bits >>> (index * 8)); - } + MemoryBytes.write(bytes, offset, byteCount, bits); return; } if (storeEmbeddedArrayBits(absoluteByteOffset, bits, byteCount)) { @@ -9412,12 +8587,13 @@ private void storeBytesAt(long absoluteByteOffset, long bits, int byteCount) { encoded = encodeFatPointer( current, allocationElementSize, allocationCodecClassName); } else if (isGeneratedAggregateCodec(allocationCodecClassName)) { - CodecPlan plan = codecPlan(allocationCodecClassName); + MemoryCodec plan = codecPlan(allocationCodecClassName); if (plan.isArrayCodecFor(current)) { discardEncodedPointers( allocation, absoluteByteOffset, byteCount); storeArrayCodecBits( current, withinElement, bits, byteCount, plan); + if (allocation instanceof FieldCell) ((FieldCell) allocation).commitOwners(); return; } byte[] direct = plan.directUnionBytes(current); @@ -9428,7 +8604,7 @@ private void storeBytesAt(long absoluteByteOffset, long bits, int byteCount) { direct[0] = (byte) bits; return; } - encoded = plan.encode.encode(current); + encoded = plan.encode().encode(current); } if (encoded != null) { if (withinElement + byteCount > encoded.length) { @@ -9543,11 +8719,12 @@ private void storeByte(long absoluteByteOffset, int value) { return; } if (isGeneratedAggregateCodec(allocationCodecClassName)) { - CodecPlan plan = codecPlan(allocationCodecClassName); + MemoryCodec plan = codecPlan(allocationCodecClassName); if (plan.isArrayCodecFor(current)) { prepareMemoryWrite(absoluteByteOffset, 1); discardEncodedPointers(allocation, absoluteByteOffset, 1); storeArrayCodecBits(current, withinElement, value & 0xffL, 1, plan); + if (allocation instanceof FieldCell) ((FieldCell) allocation).commitOwners(); return; } byte[] direct = plan.directUnionBytes(current); @@ -9558,7 +8735,7 @@ private void storeByte(long absoluteByteOffset, int value) { direct[0] = (byte) value; return; } - byte[] bytes = plan.encode.encode(current); + byte[] bytes = plan.encode().encode(current); if (withinElement >= bytes.length) { throw new IndexOutOfBoundsException("aggregate codec returned a short memory image"); } @@ -9599,7 +8776,8 @@ private static Object[] createAtomicStripes() { return stripes; } - /** Maps aliases of a Rust atomic location to the same striped monitor. */ + /** Lock the allocation during atomic access and aggregate decoding or publication. + * A decoded view can overlap several atomic fields, so offset locks are insufficient. */ private static Object atomicStripe(Pointer pointer) { return ATOMIC_STRIPES[atomicStripeIndex(pointer)]; } @@ -9613,20 +8791,25 @@ static long atomicAddress(Pointer pointer) { } private static int atomicStripeIndex(Pointer pointer) { - Object identity = pointer.allocation; - int memberHash = 0; + return atomicStripeIndex(pointer.allocation, pointer.exposedAddress); + } + + private static Object atomicStripe(Object root, long offset) { + return root instanceof Pointer + ? ATOMIC_STRIPES[atomicStripeIndex(((Pointer) root).allocation, ((Pointer) root).exposedAddress + offset)] + : ATOMIC_STRIPES[atomicStripeIndex(root, offset)]; + } + + private static int atomicStripeIndex(Object identity, long exposedAddress) { if (identity instanceof ReceiverCell) { identity = ((ReceiverCell) identity).value; } else if (identity instanceof FieldCell) { FieldCell cell = (FieldCell) identity; identity = cell.owner(); - memberHash = cell.fieldNameHash; } long key = identity == null - ? pointer.exposedAddress - : ((long) System.identityHashCode(identity) << 32) - ^ ((long) memberHash << 1) - ^ pointer.byteOffset; + ? exposedAddress + : System.identityHashCode(identity); key ^= key >>> 33; key *= 0xff51afd7ed558ccdL; key ^= key >>> 33; @@ -9652,49 +8835,109 @@ private static long signExtendAtomic(long value, int byteCount) { return (value << shift) >> shift; } - private static void atomicStoreLocked(Pointer pointer, long value, int byteCount) { - pointer.prepareMemoryWrite(pointer.byteOffset, byteCount); - pointer.storeBytes(truncateAtomic(value, byteCount), byteCount); + public static long atomicLoad(Pointer pointer, int byteCount, int ordering) { + return atomicLoad(pointer, 0L, byteCount, ordering); } - private static long atomicLoadStriped(Pointer pointer, int byteCount) { - synchronized (atomicStripe(pointer)) { - long value = truncateAtomic(pointer.loadUnsigned(byteCount), byteCount); - return value; + public static void atomicStore(Pointer pointer, long value, int byteCount, int ordering) { + atomicStore(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicExchange(Pointer pointer, long value, int byteCount, int ordering) { + return atomicExchange(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicAdd(Pointer pointer, long value, int byteCount, int ordering) { + return atomicAdd(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicSubtract(Pointer pointer, long value, int byteCount, int ordering) { + return atomicSubtract(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicAnd(Pointer pointer, long value, int byteCount, int ordering) { + return atomicAnd(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicNand(Pointer pointer, long value, int byteCount, int ordering) { + return atomicNand(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicOr(Pointer pointer, long value, int byteCount, int ordering) { + return atomicOr(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicXor(Pointer pointer, long value, int byteCount, int ordering) { + return atomicXor(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicMax(Pointer pointer, long value, int byteCount, int ordering) { + return atomicMax(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicMin(Pointer pointer, long value, int byteCount, int ordering) { + return atomicMin(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicUnsignedMax(Pointer pointer, long value, int byteCount, int ordering) { + return atomicUnsignedMax(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicUnsignedMin(Pointer pointer, long value, int byteCount, int ordering) { + return atomicUnsignedMin(pointer, 0L, value, byteCount, ordering); + } + + public static long atomicCompareExchange(Pointer pointer, long expected, + long value, + int byteCount, + int successOrdering, + int failureOrdering) { + return atomicCompareExchange(pointer, 0L, expected, value, byteCount, successOrdering, failureOrdering); + } + + private static void atomicStoreLocked(Object root, long offset, long value, int byteCount) { + storeLocationBits(root, offset, truncateAtomic(value, byteCount), byteCount); + } + + private static long atomicLoadStriped(Object root, long offset, int byteCount) { + synchronized (atomicStripe(root, offset)) { + return truncateAtomic(loadLocationBits(root, offset, byteCount), byteCount); } } - public static long atomicLoad(Pointer pointer, int byteCount, int ordering) { + public static long atomicLoad(Object root, long offset, int byteCount, int ordering) { + root = normalizeLocationOrigin(root); checkedAtomicByteCount(byteCount); if (isSequentiallyConsistent(ordering)) { synchronized (ATOMIC_SEQUENCE_LOCK) { - return atomicLoadStriped(pointer, byteCount); + return atomicLoadStriped(root, offset, byteCount); } } - return atomicLoadStriped(pointer, byteCount); + return atomicLoadStriped(root, offset, byteCount); } - private static void atomicStoreStriped(Pointer pointer, long value, int byteCount) { - synchronized (atomicStripe(pointer)) { - atomicStoreLocked(pointer, value, byteCount); + private static void atomicStoreStriped(Object root, long offset, long value, int byteCount) { + synchronized (atomicStripe(root, offset)) { + atomicStoreLocked(root, offset, value, byteCount); } } - public static void atomicStore(Pointer pointer, long value, int byteCount, int ordering) { + public static void atomicStore(Object root, long offset, long value, int byteCount, int ordering) { + root = normalizeLocationOrigin(root); checkedAtomicByteCount(byteCount); if (isSequentiallyConsistent(ordering)) { synchronized (ATOMIC_SEQUENCE_LOCK) { - atomicStoreStriped(pointer, value, byteCount); + atomicStoreStriped(root, offset, value, byteCount); } return; } - atomicStoreStriped(pointer, value, byteCount); + atomicStoreStriped(root, offset, value, byteCount); } private static long atomicRmwStriped( - Pointer pointer, long operand, int byteCount, int operation) { - synchronized (atomicStripe(pointer)) { - long oldValue = truncateAtomic(pointer.loadUnsigned(byteCount), byteCount); + Object root, long offset, long operand, int byteCount, int operation) { + synchronized (atomicStripe(root, offset)) { + long oldValue = truncateAtomic(loadLocationBits(root, offset, byteCount), byteCount); long right = truncateAtomic(operand, byteCount); long newValue; switch (operation) { @@ -9740,97 +8983,99 @@ private static long atomicRmwStriped( default: throw new IllegalArgumentException("unknown Rust atomic operation " + operation); } - atomicStoreLocked(pointer, newValue, byteCount); + atomicStoreLocked(root, offset, newValue, byteCount); return oldValue; } } private static long atomicRmw( - Pointer pointer, long operand, int byteCount, int operation, int ordering) { + Object root, long offset, long operand, int byteCount, int operation, int ordering) { + root = normalizeLocationOrigin(root); checkedAtomicByteCount(byteCount); if (isSequentiallyConsistent(ordering)) { synchronized (ATOMIC_SEQUENCE_LOCK) { - return atomicRmwStriped(pointer, operand, byteCount, operation); + return atomicRmwStriped(root, offset, operand, byteCount, operation); } } - return atomicRmwStriped(pointer, operand, byteCount, operation); + return atomicRmwStriped(root, offset, operand, byteCount, operation); } public static long atomicExchange( - Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 0, ordering); + Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 0, ordering); } - public static long atomicAdd(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 1, ordering); + public static long atomicAdd(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 1, ordering); } public static long atomicSubtract( - Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 2, ordering); + Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 2, ordering); } - public static long atomicAnd(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 3, ordering); + public static long atomicAnd(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 3, ordering); } - public static long atomicNand(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 4, ordering); + public static long atomicNand(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 4, ordering); } - public static long atomicOr(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 5, ordering); + public static long atomicOr(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 5, ordering); } - public static long atomicXor(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 6, ordering); + public static long atomicXor(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 6, ordering); } - public static long atomicMax(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 7, ordering); + public static long atomicMax(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 7, ordering); } - public static long atomicMin(Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 8, ordering); + public static long atomicMin(Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 8, ordering); } public static long atomicUnsignedMax( - Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 9, ordering); + Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 9, ordering); } public static long atomicUnsignedMin( - Pointer pointer, long value, int byteCount, int ordering) { - return atomicRmw(pointer, value, byteCount, 10, ordering); + Object root, long offset, long value, int byteCount, int ordering) { + return atomicRmw(root, offset, value, byteCount, 10, ordering); } private static long atomicCompareExchangeStriped( - Pointer pointer, long expected, long value, int byteCount) { - synchronized (atomicStripe(pointer)) { - long oldValue = truncateAtomic(pointer.loadUnsigned(byteCount), byteCount); + Object root, long offset, long expected, long value, int byteCount) { + synchronized (atomicStripe(root, offset)) { + long oldValue = truncateAtomic(loadLocationBits(root, offset, byteCount), byteCount); if (oldValue == truncateAtomic(expected, byteCount)) { - atomicStoreLocked(pointer, value, byteCount); + atomicStoreLocked(root, offset, value, byteCount); } return oldValue; } } public static long atomicCompareExchange( - Pointer pointer, + Object root, long offset, long expected, long value, int byteCount, int successOrdering, int failureOrdering) { + root = normalizeLocationOrigin(root); checkedAtomicByteCount(byteCount); boolean sequentiallyConsistent = isSequentiallyConsistent(successOrdering) | isSequentiallyConsistent(failureOrdering); if (sequentiallyConsistent) { synchronized (ATOMIC_SEQUENCE_LOCK) { - return atomicCompareExchangeStriped(pointer, expected, value, byteCount); + return atomicCompareExchangeStriped(root, offset, expected, value, byteCount); } } - return atomicCompareExchangeStriped(pointer, expected, value, byteCount); + return atomicCompareExchangeStriped(root, offset, expected, value, byteCount); } public static void atomicFence(int ordering) { @@ -9846,125 +9091,945 @@ public static void atomicFence(int ordering) { } } - private void requireScalarViewSize(int expectedSize, String scalarType) { - if (viewSize == expectedSize) { + /** Thin-pointer comparison on storage components, preserving exposed addresses. */ + public static boolean sameLocation(Object left, long leftOffset, Object right, long rightOffset) { + if (left == right) return leftOffset == rightOffset; + if (left instanceof Pointer && right instanceof Pointer) { + Pointer a = (Pointer) left; + Pointer b = (Pointer) right; + if (a.allocation != null && a.allocation == b.allocation) { + return a.byteOffset + leftOffset == b.byteOffset + rightOffset; + } + } + return locationAddress(left) + leftOffset == locationAddress(right) + rightOffset; + } + + private static long locationAddress(Object root) { + return locationAddr(root, 0); + } + + /** Read an address word without materializing or exposing a Rust pointer. */ + public static long locationAddr(Object root, long offset) { + if (root == null) return offset; + if (root instanceof Pointer) return ((Pointer) root).numericAddress() + offset; + Object normalized = normalizeLocationOrigin(root); + if (normalized instanceof Pointer) return ((Pointer) normalized).numericAddress() + offset; + Long cached = cachedAllocationBase(ALLOCATION_BASE_CACHE, root); + if (cached != null) return cached.longValue() + offset; + int size = root instanceof Storage ? ((Storage) root).size : inferredArrayElementSize(root); + synchronized (ALLOCATIONS) { + return allocationBase(root, size, allocationInfo(root)) + offset; + } + } + + private static Object normalizeLocationOrigin(Object root) { + if (root == null || root instanceof Pointer || root instanceof Storage + || !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root)) return root; + Map stripe = stateStripe(MEMORY_VIEW_ORIGINS, root); + synchronized (stripe) { + MemoryViewOrigin origin = stripe.get(root); + // Filter matches can be stale or false. Only a live origin needs normalization. + if (origin == null || origin.allocation.get() == null) return root; + } + return array(root, 0, inferredArrayElementSize(root)); + } + + /** Pointer differences use allocation identity, never exposed address lookup. */ + public static long byteOffsetLocations(Object root, long offset, Object origin, long originOffset) { + root = normalizeLocationOrigin(root); + origin = normalizeLocationOrigin(origin); + Object allocation = root instanceof Pointer ? ((Pointer) root).provenanceAllocation() : root; + Object originAllocation = origin instanceof Pointer + ? ((Pointer) origin).provenanceAllocation() : origin; + if (allocation != originAllocation) { + throw new IllegalArgumentException("byte_offset_from requires pointers into one allocation"); + } + long start = root instanceof Pointer + ? Math.addExact(((Pointer) root).provenanceByteOffset(), offset) : offset; + long end = origin instanceof Pointer + ? Math.addExact(((Pointer) origin).provenanceByteOffset(), originOffset) : originOffset; + return Math.subtractExact(start, end); + } + + public static long offsetLocations(Object root, long offset, Object origin, long originOffset, long stride) { + if (stride == -1) stride = locationStride(root); + if (stride == 0) throw new ArithmeticException("offset_from is undefined for zero-sized pointees"); + long bytes = byteOffsetLocations(root, offset, origin, originOffset); + if (bytes % stride != 0) { + throw new ArithmeticException("pointer distance is not a whole number of elements"); + } + return bytes / stride; + } + + private static long unsignedLocationDistance(long distance) { + if (distance < 0) { + throw new ArithmeticException("offset_from_unsigned requires self at or after origin"); + } + return distance; + } + + public static long byteOffsetLocationsUnsigned(Object root, long offset, Object origin, long originOffset) { + return unsignedLocationDistance(byteOffsetLocations(root, offset, origin, originOffset)); + } + + public static long offsetLocationsUnsigned(Object root, long offset, Object origin, long originOffset, long stride) { + return unsignedLocationDistance(offsetLocations(root, offset, origin, originOffset, stride)); + } + + public static long alignLocation(Object root, long offset, long stride, long alignment) { + return alignmentOffset(locationAddr(root, offset), stride == -1 ? locationStride(root) : stride, alignment); + } + + /** Compare decomposed data addresses without creating temporary carriers. */ + public static int compareLocations(Object left, long leftOffset, Object right, long rightOffset) { + if (left == right && left != null && leftOffset >= 0 && rightOffset >= 0 + && (left instanceof Storage || (left.getClass().isArray() + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, left)))) { + return Long.compareUnsigned(leftOffset, rightOffset); + } + if (left instanceof Pointer && right instanceof Pointer) { + Pointer a = (Pointer) left, b = (Pointer) right; + long x = a.byteOffset + leftOffset, y = b.byteOffset + rightOffset; + if (a.allocation != null && a.allocation == b.allocation + && a.allocationElementSize > 0 && b.allocationElementSize > 0 + && a.addressOrigin() == null && b.addressOrigin() == null && x >= 0 && y >= 0) { + return Long.compareUnsigned(x, y); + } + } + // Other offsets and allocations require the full unsigned-address and provenance checks. + return Long.compareUnsigned(locationAddress(left) + leftOffset, locationAddress(right) + rightOffset); + } + + /** Element layout retained by a general storage root. */ + public static long locationStride(Object root) { + return root instanceof Storage ? ((Storage) root).size : ((Pointer) root).viewSize; + } + + /** The root carries the complete layout and provenance of a non-scalar view. */ + public static Pointer fromStorageLocation(Object root, long offset) { + if (root instanceof Storage) root = ((Storage) root).boundary(); + // Retain the pointer that binds a decoded view. A later commit must use the same owner. + if (offset == 0) return (Pointer) root; + return ((Pointer) root).byte_offset(offset); + } + + /** The physical ABI records a reconstruction plan, never an inferred JVM layout. */ + public static Pointer addressFromParts(Object root, long offset, int plan) { + if (plan == 64 || plan == 128) return fromBorrowedStorageLocation(root, offset, plan == 64); + return plan == 0 ? fromStorageLocation(root, offset) : fromLocation(root, offset, plan); + } + + private static BorrowedFieldPath borrowedPath(Storage storage, long offset, boolean view) { + StorageLayout layout = storage.layout(((Cell) storage).value); + return layout == null ? null : layout.borrowedAt(offset, view); + } + + private static Pointer fromBorrowedStorageLocation(Object root, long offset, boolean view) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + BorrowedFieldPath path = borrowedPath(storage, offset, view); + if (path != null) { + // Delay path materialization until a boundary needs it. Preserve replaceable parents after escape. + Pointer base = storage.boundary(); + Object owner = storage; + Pointer result = null; + for (int i = 0; i < path.fields.length; i++) { + RustField field = path.fields[i]; + boolean last = i == path.fields.length - 1; + result = rootField(owner, field.getDeclaringClass(), field.getName(), + last ? path.size : 0, last ? path.codec : null); + owner = result.allocation; + } + return result.inheritAddressOrigin(base, offset); + } + } + return fromStorageLocation(root, offset); + } + + /** The address of a stored borrow keeps its enclosing allocation root. */ + public static Object storageBorrowedFieldRoot(Object root, long offset, String owner, + String field, long fieldOffset, long size, String codec) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (directStorage(storage) && isGeneratedAggregateCodec(storage.codec)) { + long absolute = Math.addExact(offset, fieldOffset); + StorageLayout layout = storage.layout(((Cell) storage).value); + if (layout != null) { + BorrowedFieldPath path = layout.borrowedAt(absolute, true); + if (path == null) path = layout.borrowedAt(absolute, false); + if (path != null && path.size == size && path.field().getName().equals(field) + && matchesBinaryClassName(owner, path.field().getDeclaringClass().getName()) + && java.util.Objects.equals(codec, path.codec)) return storage; + } + } + } + return fromStorageLocation(root, offset).projectStructField(owner, field, fieldOffset, size, codec); + } + + /** Register the containing allocation when an escaping array borrow exposes its JVM array. */ + public static Object loadStorageArray(Object root, long offset, String target) { + return fromStorageLocation(root, offset).getObject(); + } + + public static Object loadStorageLocation(Object root, long offset, String target) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && directStorage(storage)) return ((Cell) storage).value; + root = storage.boundary(); + } + Pointer base = (Pointer) root; + if (base.rareState == null && base.addressState == null) { + return loadObjectLocation(base, offset, target); + } + Pointer view = fromStorageLocation(root, offset); + return target == null ? view.getObject() : view.getObjectAs(target); + } + + /** A stored borrow returns its components in the ordinary caller-owned ABI. */ + public static Object loadBorrowedView(Object root, long offset, long[] metadata) { + if (root instanceof BorrowedStorage && offset == 0 && unescapedStorage((Storage) root) + && ((BorrowedStorage) root).view) { + return ((BorrowedStorage) root).read(metadata, true); + } + if (root instanceof Storage) { + Storage storage = (Storage) root; + BorrowedFieldPath path = borrowedPath(storage, offset, true); + if (path != null && directStorage(storage)) return path.read(((Cell) storage).value, metadata); + root = fromBorrowedStorageLocation(root, offset, true); + offset = 0; + } + FieldCell field = directBorrowedField(root, offset, true); + if (field != null) return readBorrowedField(field, metadata); + SliceView view = (SliceView) loadStorageLocation(root, offset, SLICE_VIEW_CLASS_NAME); + metadata[0] = RustField.viewStart(view); + metadata[1] = RustField.viewLength(view); + return RustField.viewRoot(view); + } + + public static Object loadBorrowedAddress(Object root, long offset, long[] metadata) { + if (root instanceof BorrowedStorage && offset == 0 && unescapedStorage((Storage) root) + && !((BorrowedStorage) root).view) { + return ((BorrowedStorage) root).read(metadata, false); + } + if (root instanceof Storage) { + Storage storage = (Storage) root; + BorrowedFieldPath path = borrowedPath(storage, offset, false); + if (path != null && directStorage(storage)) return path.read(((Cell) storage).value, metadata); + root = fromBorrowedStorageLocation(root, offset, false); + offset = 0; + } + FieldCell field = directBorrowedField(root, offset, false); + if (field != null) return readBorrowedField(field, metadata); + Object value = loadStorageLocation(root, offset, null); + metadata[0] = 0; + return value; + } + + public static void storeBorrowedView(Object root, long offset, Object backing, int start, long length) { + storeBorrowedView(root, offset, backing, start, length, false); + } + + public static void storeBorrowedUtf8(Object root, long offset, Object backing, int start, long length) { + storeBorrowedView(root, offset, backing, start, length, true); + } + + private static void storeBorrowedView(Object root, long offset, Object backing, int start, long length, boolean utf8) { + if (root instanceof BorrowedStorage && offset == 0 && unescapedStorage((Storage) root) + && ((BorrowedStorage) root).view) { + ((BorrowedStorage) root).store(backing, start, length, utf8 ? -2 : -1); return; } - String sourceView = zeroSizedSourceViewSize() >= 0 - ? "; recorded erased source view is " + zeroSizedSourceViewSize() + " bytes" - : ""; - throw new IllegalStateException( - scalarType + " load requires a " + expectedSize + "-byte view, but pointer has a " - + viewSize + "-byte view" + sourceView); + if (root instanceof Storage) { + Storage storage = (Storage) root; + BorrowedFieldPath path = borrowedPath(storage, offset, true); + if (path != null && directStorage(storage)) { + path.write(((Cell) storage).value, backing, start, length); + return; + } + root = fromBorrowedStorageLocation(root, offset, true); + offset = 0; + } + FieldCell field = directBorrowedField(root, offset, true); + if (field != null) { + writeBorrowedField(field, backing, start, length); + return; + } + storeStorageLocation(root, offset, utf8 ? new Utf8View(backing, start, length) + : new SliceView(backing, start, length)); } - private long relativeByteOffset(long elementOffset, long additionalByteOffset) { - return Math.addExact( - byteOffset, - Math.addExact( - Math.multiplyExact(elementOffset, viewSize), - additionalByteOffset)); + public static void storeBorrowedAddress(Object root, long offset, Object backing, long displacement, int size) { + if (root instanceof BorrowedStorage && offset == 0 && unescapedStorage((Storage) root) + && !((BorrowedStorage) root).view && !(backing instanceof Storage)) { + ((BorrowedStorage) root).store(backing, displacement, 0, size); + return; + } + // Resolve referenced boundaries before assignment. Lazy materialization under owner locks could recurse through cycles. + if (root instanceof Storage) { + Storage storage = (Storage) root; + BorrowedFieldPath path = borrowedPath(storage, offset, false); + if (path != null && directStorage(storage)) { + path.write(((Cell) storage).value, backing, displacement, 0); + return; + } + root = fromBorrowedStorageLocation(root, offset, false); + offset = 0; + } + FieldCell field = directBorrowedField(root, offset, false); + if (field != null) { + writeBorrowedField(field, backing, displacement, 0); + return; + } + Pointer value = backing == null && displacement == 0 ? null + : addressFromParts(backing, displacement, size); + storeStorageLocation(root, offset, value); + } + + private static void writeBorrowedField(FieldCell cell, Object root, long offset, long length) { + try { cell.access.field.setBorrowedParts(cell.owner(), root, offset, length); } + catch (IllegalAccessException error) { + throw new IllegalStateException("could not write borrowed Rust field", error); + } + discardProjectedFieldViews(cell); + } + + private static Object readBorrowedField(FieldCell cell, long[] metadata) { + try { return cell.access.field.borrowedParts(cell.owner(), metadata); } + catch (IllegalAccessException error) { + throw new IllegalStateException("could not read borrowed Rust field", error); + } + } + + private static FieldCell directBorrowedField(Object root, long offset, boolean view) { + if (!(root instanceof Pointer)) return null; + Pointer pointer = (Pointer) root; + if (offset != 0 || pointer.byteOffset != 0 + || pointer.viewSize != pointer.allocationElementSize + || !pointer.isDirectAllocationView() + || pointer.rareState != null || !(pointer.allocation instanceof FieldCell)) return null; + FieldCell field = (FieldCell) pointer.allocation; + if (!field.access.field.borrowedShape(view)) return null; + Object owner = field; + for (int depth = 0; depth < 16; depth++) { + if (owner instanceof FieldCell) { + FieldCell current = (FieldCell) owner; + if (current.hasMemoryView || current.hasStructuralView || current.hasProjectedViews + || current.hasMemoryOrigins) return null; + owner = current.rootOwner == null ? current.fixedOwner : current.rootOwner; + } else if (owner instanceof Cell) { + Cell current = (Cell) owner; + if (current.hasMemoryView || current.hasStructuralView || current.hasProjectedViews) return null; + owner = current.value; + } else { + return mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, owner) ? null : field; + } + } + return null; } - /** - * Materializes a compiler-deferred derived pointer at an ABI or semantic boundary. - * A zero displacement preserves object identity and allocates nothing. - */ - public static Pointer materializeRelative( - Pointer base, long elementOffset, long byteOffset) { - if (elementOffset == 0 && byteOffset == 0) { - return base; + /** Copying a value never establishes a mutable decoded-view binding. */ + public static Object loadStorageCopy(Object root, long offset, String target) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && directStorage(storage)) { + return copyManagedValue(((Cell) storage).value); + } + root = storage.boundary(); + } + Pointer base = (Pointer) root; + if (base.allocation instanceof byte[] && base.rareState == null + && base.addressState == null && base.viewSize > 0 + && isGeneratedAggregateCodec(base.viewCodecClassName)) { + MemoryCodec plan = codecPlan(base.viewCodecClassName); + if (plan.decodeAt() != null && target != null + && matchesBinaryClassName(target, plan.encodeParameterType.getName())) { + byte[] bytes = (byte[]) base.allocation; + long absolute = Math.addExact(base.byteOffset, offset); + int start = Math.toIntExact(absolute); + int size = base.materializedViewSize(); + if (start < 0 || start > bytes.length - size) { + throw new IndexOutOfBoundsException("aggregate read exceeds byte-addressable Rust storage"); + } + base.flushMemoryViewsOverlapping(absolute, size); + return plan.decodeAt().decode(bytes, start); + } + } + return fromStorageLocation(base, offset).getObjectCopyAs(target); + } + + /** Owned field read without parent and projected address wrappers on byte storage. */ + public static Object loadStorageFieldCopy(Object root, long offset, String owner, + String field, long fieldOffset, long size, String codec, String target) { + if (size > 0 && size <= Integer.MAX_VALUE + && (root instanceof byte[] || (root instanceof Pointer + && ((Pointer) root).allocation instanceof byte[] + && ((Pointer) root).traitMetadataCarrier() == null))) { + return loadTypedStorageCopy(root, Math.addExact(offset, fieldOffset), + (int) size, codec, target); + } + return fromStorageLocation(root, offset) + .projectStructField(owner, field, fieldOffset, size, codec).getObjectCopyAs(target); + } + + /** Store an owned field through its exact layout without two projected wrappers. */ + public static void storeStorageField(Object root, long offset, String owner, String field, + long fieldOffset, long size, String codec, Object value) { + if (size > 0 && size <= Integer.MAX_VALUE + && (root instanceof byte[] + || (root instanceof Pointer && ((Pointer) root).allocation instanceof byte[] + && ((Pointer) root).traitMetadataCarrier() == null))) { + storeTypedStorage(root, Math.addExact(offset, fieldOffset), (int) size, codec, value); + return; + } + fromStorageLocation(root, offset).projectStructField(owner, field, fieldOffset, size, codec).set(value); + } + + /** Read a borrowed value without a temporary Pointer. Keep binding and commit behavior for decoded views. */ + public static Object loadTypedStorage(Object root, long offset, int size, String codec, String target) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && size == storage.size + && java.util.Objects.equals(codec, storage.codec) && directStorage(storage)) { + Object value = ((Cell) storage).value; + if (target != null && !target.isEmpty() || value == null || !value.getClass().isArray()) return value; + } + } + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if ((size == 4 || size == 8) && pointer.allocation instanceof byte[] + && pointer.allocationElementSize == 1 && pointer.allocationCodecClassName == null + && pointer.rareState == null && pointer.addressState == null + && target != null && matchesBinaryClassName(target, Pointer.class.getName()) + && isRawPointerCodec(codec) && !isArrayReferenceCodec(codec)) { + long absolute = Math.addExact(pointer.byteOffset, offset); + long address = pointer.loadUnsignedAt(absolute, size); + Pointer value = encodedPointer(pointer.allocation, absolute, size, codec, address); + return decodedRawPointer(value == null + ? typedPointerObjectFromAddress(address, codec) : value, codec); + } + if (size > 0 && pointer.viewSize == size + && java.util.Objects.equals(codec, pointer.viewCodecClassName) + && pointer.rareState == null && (pointer.addressState == null || offset == 0) + && pointer.isDirectAllocationView() && !mayHaveStructuralView(pointer.allocation)) { + Object value = loadObjectLocation(pointer, offset, target); + if (target != null && !target.isEmpty() || value == null || !value.getClass().isArray()) return value; + } + } + return fromTypedStorageLocation(root, offset, size, codec).getObjectAs(target); + } + + /** The copy's layout is compiler metadata, independent of the backing view. */ + public static Object loadTypedStorageCopy(Object root, long offset, int size, + String codec, String target) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && size == storage.size + && java.util.Objects.equals(codec, storage.codec) && directStorage(storage)) { + return copyManagedValue(((Cell) storage).value); + } + root = storage.boundary(); + } + if (root instanceof byte[] && size > 0 && isGeneratedAggregateCodec(codec) + && !mayBeInIdentityFilter(MEMORY_VIEW_FILTER, root) + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root)) { + MemoryCodec plan = codecPlan(codec); + if (plan.decodeAt() != null && target != null + && matchesBinaryClassName(target, plan.encodeParameterType.getName())) { + byte[] bytes = (byte[]) root; + int start = Math.toIntExact(offset); + if (start < 0 || start > bytes.length - size) { + throw new IndexOutOfBoundsException("aggregate read exceeds byte-addressable Rust storage"); + } + return plan.decodeAt().decode(bytes, start); + } + } + Pointer base = root instanceof Pointer ? (Pointer) root : fromLocation(root, 0, 1); + if (base.allocation instanceof byte[] && base.rareState == null + && base.addressState == null && size > 0 && isGeneratedAggregateCodec(codec)) { + MemoryCodec plan = codecPlan(codec); + if (plan.decodeAt() != null && target != null + && matchesBinaryClassName(target, plan.encodeParameterType.getName())) { + byte[] bytes = (byte[]) base.allocation; + long absolute = Math.addExact(base.byteOffset, offset); + int start = Math.toIntExact(absolute); + if (start < 0 || start > bytes.length - size) { + throw new IndexOutOfBoundsException("aggregate read exceeds byte-addressable Rust storage"); + } + base.flushMemoryViewsOverlapping(absolute, size); + return plan.decodeAt().decode(bytes, start); + } + } + Pointer escaped = base.escapedFieldStorage(offset); + return (escaped == null ? base.byteOffsetRetype(offset, size, codec) + : escaped.retype(size, codec)).getObjectCopyAs(target); + } + + /** Write with compiler-owned layout metadata, materializing only at a general boundary. */ + public static void storeTypedStorage(Object root, long offset, int size, String codec, Object value) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && size == storage.size + && java.util.Objects.equals(codec, storage.codec) && directStorage(storage)) { + Cell cell = storage; + cell.value = convertDirectValue(cell.value, value, size); + return; + } + } + if (root instanceof byte[] && size > 0 && isGeneratedAggregateCodec(codec) + && !mayBeInIdentityFilter(MEMORY_VIEW_FILTER, root) + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root) + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, value)) { + MemoryCodec plan = codecPlan(codec); + CodecCalls.RangeEncoder encode = plan.encodeAt(); + if (encode != null) { + byte[] bytes = (byte[]) root; + int start = Math.toIntExact(offset); + if (start < 0 || start > bytes.length - size) { + throw new IndexOutOfBoundsException("aggregate store exceeds byte-addressable Rust storage"); + } + discardEncodedPointers(bytes, start, size); + encode.encode(value, bytes, start); + return; + } + } + if (root instanceof Pointer + && ((Pointer) root).storeAggregateRange(offset, size, codec, value)) { + return; + } + fromTypedStorageLocation(root, offset, size, codec).set(value); + } + + /** Write an exact byte window without a temporary address or encoded image. */ + private boolean storeAggregateRange(long offset, long byteSize, String codec, Object value) { + if (!(allocation instanceof byte[]) || rareState != null || addressState != null + || byteSize <= 0 || !isGeneratedAggregateCodec(codec) + || mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, value)) { + return false; + } + CodecCalls.RangeEncoder encode = codecPlan(codec).encodeAt(); + if (encode == null) return false; + byte[] bytes = (byte[]) allocation; + int size = checkedArrayLength(byteSize); + int start = Math.toIntExact(Math.addExact(byteOffset, offset)); + if (start < 0 || start > bytes.length - size) { + throw new IndexOutOfBoundsException("aggregate store exceeds byte-addressable Rust storage"); + } + // Save pending writes outside this range before replacement. Live views use set() to preserve their origin. + prepareMemoryWrite(start, size); + discardEncodedPointers(bytes, start, size); + encode.encode(value, bytes, start); + return true; + } + + public static Object directStorageAggregate(Object root, long offset, Class type) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && directStorage(storage)) { + Object value = ((Cell) storage).value; + return type.isInstance(value) ? value : null; + } + root = storage.boundary(); + } + if (root instanceof byte[]) return null; + Pointer base = (Pointer) root; + if (base.allocation instanceof byte[] && base.rareState == null) return null; + if (offset == 0) { + Object direct = base.directAggregate(type); + if (direct != null) return direct; + } + Object embedded = base.directCellArrayElement(offset, type); + if (embedded != null || offset == 0) return embedded; + if (base.rareState != null || base.addressState != null + || !(base.allocation instanceof Object[]) + || base.allocationElementSize <= 0 + || !base.isDirectAllocationView() + || mayHaveStructuralView(base.allocation)) return null; + long absolute = Math.addExact(base.byteOffset, offset); + if (absolute % base.allocationElementSize != 0) return null; + base.flushMemoryViewsOverlapping(absolute, base.allocationElementSize); + Object value = base.readElement(Math.toIntExact(absolute / base.allocationElementSize)); + return type.isInstance(value) ? value : null; + } + + /** Borrow an aggregate element from a cell or field that owns a fixed array. */ + private Object directCellArrayElement(long offset, Class type) { + if (!(allocation instanceof Cell || allocation instanceof FieldCell) + || addressState != null && !(allocation instanceof FieldCell) + || traitObjectCarrier() != null || traitMetadataCarrier() != null + || zeroSizedSourceViewSize() >= 0 || mayHaveStructuralView(allocation) + || !isGeneratedAggregateCodec(allocationCodecClassName) + || !isGeneratedAggregateCodec(viewCodecClassName)) return null; + MemoryCodec plan = codecPlan(allocationCodecClassName); + int stride = plan.arrayElementSize; + if (stride <= 0 || viewSize != stride + || !java.util.Objects.equals(viewCodecClassName, plan.arrayElementCodec)) return null; + long absolute = Math.addExact(byteOffset, offset); + if (absolute < 0 || absolute % stride != 0) return null; + Object value = readElement(0); + if (!(value instanceof Object[]) || !plan.encodeParameterType.isInstance(value) + || (long) ((Object[]) value).length * stride != allocationElementSize + || absolute / stride >= ((Object[]) value).length) return null; + if (!directCellHasNoMemoryViews(allocation)) { + flushMemoryViewsOverlapping(absolute, stride); + // Flushing a byte alias can replace the array carrier. + value = readElement(0); + if (!(value instanceof Object[]) || !plan.encodeParameterType.isInstance(value) + || (long) ((Object[]) value).length * stride != allocationElementSize + || absolute / stride >= ((Object[]) value).length) return null; + } + Object element = independentRepeatedArrayElement(value, (int) (absolute / stride)); + // Nested array values require their separate origin-registration rules. + return element != null && !element.getClass().isArray() && type.isInstance(element) ? element : null; + } + + /** Keep a scalar projection on its typed owner while its layout is exact. */ + public static Object storageFieldRoot(Object root, long offset, String owner, + String field, long fieldOffset, long size, String codec) { + // Scalar components supply their own width. Byte storage needs no layout carrier. + if (codec == null && size > 0 && size <= 8 && hasPlainByteStorage(root)) { + return root; + } + if (root instanceof Storage && size > 0 && size <= 8) { + Storage storage = (Storage) root; + // Prepare byte storage before a later byte alias uses this root and offset. + if (directStorage(storage) && isGeneratedAggregateCodec(storage.codec)) { + StorageLayout layout = storage.layout(((Cell) storage).value); + if (layout != null && layout.at(Math.addExact(offset, fieldOffset), (int) size) != null) { + return storage; + } + } + } + return fromStorageLocation(root, offset).projectStructField(owner, field, fieldOffset, size, codec); + } + + public static long storageFieldOffset(Object root, Object base, long offset) { + return root == base ? offset : 0; + } + + public static void storeStorageLocation(Object root, long offset, Object value) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && directStorage(storage)) { + Cell cell = storage; + cell.value = convertDirectValue(cell.value, value, storage.size); + return; + } + root = storage.boundary(); + } + Pointer pointer = (Pointer) root; + if (pointer.storeAggregateRange(offset, pointer.viewSize, + pointer.viewCodecClassName, value)) { + return; + } + if (offset == 0) pointer.set(value); + else pointer.byte_offset(offset).set(value); + } + + public static void commitStorageLocation(Object root, long offset) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (offset == 0 && directStorage(storage)) { + discardProjectedFieldViews(((Cell) storage).value); + return; + } + root = storage.boundary(); + } + Pointer pointer = (Pointer) root; + if (offset == 0) pointer.commitMemoryView(); + else pointer.byte_offset(offset).commitMemoryView(); + } + + /** Materialize a scalar address only where a JVM object is required. */ + public static Pointer fromLocation(Object root, long offset, int size) { + return fromTypedStorageLocation(root, offset, size, null); + } + + /** Materialize an exact aggregate view only at a carrier boundary. */ + public static Pointer fromTypedStorageLocation(Object root, long offset, int size, String codec) { + if (root instanceof Storage) root = ((Storage) root).boundary(); + if (root == null && offset == 0) return null; + if (root == null) return fromUnprovenancedAddress(offset, size, codec); + if (!(root instanceof Pointer) && root.getClass().isArray() + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root)) { + return new Pointer(root, inferredArrayElementSize(root), offset, size, null, codec, -1); + } + Pointer pointer = root instanceof Pointer ? (Pointer) root + : array(root, 0, inferredArrayElementSize(root)); + Pointer escaped = pointer.escapedFieldStorage(offset); + if (escaped != null) return escaped.retype(size, codec); + // Combine offset and retype without an intermediate Pointer. + // Keep the result independent because later metadata changes must not affect the source. + Pointer result = new Pointer( + pointer.allocation, + pointer.allocationElementSize, + pointer.byteOffset + offset, + size, + pointer.allocationCodecClassName, + codec, + pointer.allocation == null ? pointer.exposedAddress + offset : -1) + .withMetadata(pointer.metadata) + .copyDynamicMetadata(pointer) + .copyAddressOrigin(pointer, offset); + if (offset == 0 && pointer.zeroSizedSourceViewSize() >= 0) { + result.setZeroSizedSourceView(pointer.zeroSizedSourceViewSize(), + pointer.zeroSizedSourceViewCodecClassName()); + } else if (pointer.viewSize == 0 && pointer.viewCodecClassName != null) { + result.setZeroSizedSourceView(pointer.viewSize, pointer.viewCodecClassName); + } + return result; + } + + private static Object directLocationSliceArray(Object root, int size) { + if (size <= 0) return null; + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if (pointer.rareState != null || pointer.allocationElementSize != size + || pointer.allocationCodecClassName != null) return null; + root = pointer.allocation; + } else if (mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root)) { + return null; + } + return root != null && root.getClass().isArray() + && root.getClass().getComponentType().isPrimitive() + && inferredArrayElementSize(root) == size ? root : null; + } + + /** Re-form a scalar slice directly from its decomposed storage location. */ + public static Object locationSliceBacking(Object root, long offset, int size) { + if (root == null && offset == 0) return null; + Object array = directLocationSliceArray(root, size); + if (array != null) return array; + if (retainedSliceRoot(root, offset, size, null)) return root; + return fromLocation(root, offset, size).sliceBackingArray(); + } + + public static int locationSliceOffset(Object root, long offset, int size) { + if (root == null && offset == 0) return 0; + if (directLocationSliceArray(root, size) == null) { + if (retainedSliceRoot(root, offset, size, null)) return (int) (offset / size); + return fromLocation(root, offset, size).sliceElementOffset(); + } + if (root instanceof Pointer) offset += ((Pointer) root).byteOffset; + if (offset % size != 0) { + throw new IllegalStateException("slice data pointer is not element-aligned"); + } + return Math.toIntExact(offset / size); + } + + private static boolean retainedSliceRoot(Object root, long offset, int size, String codec) { + if (!(root instanceof Pointer) || size <= 0 || offset % size != 0 + || offset / size != (int) (offset / size)) return false; + Pointer pointer = (Pointer) root; + return pointer.viewSize == size && java.util.Objects.equals(pointer.viewCodecClassName, codec); + } + + /** Retain the element owner. The slice start is relative to its current displacement. */ + public static Object typedLocationSliceBacking(Object root, long offset, int size, String codec) { + if (retainedSliceRoot(root, offset, size, codec)) return root; + return fromTypedStorageLocation(root, offset, size, codec).sliceBackingArray(); + } + + public static int typedLocationSliceOffset(Object root, long offset, int size, String codec) { + if (retainedSliceRoot(root, offset, size, codec)) return (int) (offset / size); + return fromTypedStorageLocation(root, offset, size, codec).sliceElementOffset(); + } + + /** Normalize the aggregate slice root once. Keep byte offsets and slice length separate. */ + public static Object sliceAddressRoot(Object backing, int size, String codec) { + if (backing == null) return withoutProvenance(0, size, codec); + if (backing instanceof Pointer) { + Pointer pointer = (Pointer) backing; + if (pointer.allocation instanceof byte[] && size > 0 && pointer.viewSize == size + && java.util.Objects.equals(pointer.viewCodecClassName, codec)) return pointer; + return pointer.sliceStorageView(size, codec); + } + // Preserve the backing owner of decoded arrays with a registered origin. + return fromSliceParts(backing, 0, 0L, size, codec); + } + + /** Normalize unusual decoded/transparent storage once before scalar iteration. */ + public static Object scalarSliceRoot(Object backing, int size) { + if (backing == null) return null; + if (backing.getClass().isArray() + && backing.getClass().getComponentType().isPrimitive() + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, backing)) { + return backing; + } + if (backing instanceof Pointer) { + Pointer pointer = (Pointer) backing; + if (pointer.allocation == null + || (pointer.allocation.getClass().isArray() + && pointer.allocation.getClass().getComponentType().isPrimitive())) { + return pointer; + } + // Root normalization must not change slice metadata. + return pointer.sliceStorageView(size, null); } - long displacement = Math.addExact( - Math.multiplyExact(elementOffset, base.viewSize), byteOffset); - return base.byte_offset(displacement); + return fromSliceParts(backing, 0, 0L, size, null); } - /** Retypes a deferred derived pointer with a single allocation. */ - public static Pointer retypeRelative( - Pointer base, - long elementOffset, - long byteOffset, - long newViewSize, - String newViewCodecClassName) { - long displacement = Math.addExact( - Math.multiplyExact(elementOffset, base.viewSize), byteOffset); - if (displacement == 0) { - return base.retype(newViewSize, newViewCodecClassName); - } - return base.byteOffsetRetype( - displacement, newViewSize, newViewCodecClassName); + /** Byte-backed places can access a known scalar offset without a field view. */ + public static boolean hasByteStorage(Object root) { + return root instanceof byte[] + || (root instanceof Pointer && ((Pointer) root).allocation instanceof byte[]); + } + + public static boolean hasPlainByteStorage(Object root) { + return hasByteStorage(root) + && (!(root instanceof Pointer) || ((Pointer) root).traitMetadataCarrier() == null); } - public static boolean getBooleanRelative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(1, "bool"); - return base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 1) - != 0; + /** Scalar field fallback after generated code has tried the managed carrier. */ + public static long loadScalarField(Object root, long offset, String owner, + String field, long fieldOffset, int size) { + if (hasByteStorage(root)) { + return loadLocationBits(root, Math.addExact(offset, fieldOffset), size); + } + Pointer projected = fromStorageLocation(root, offset) + .projectStructField(owner, field, fieldOffset, size, null); + return loadLocationBits(projected, 0, size); } - public static byte getI8Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(1, "i8/u8"); - return (byte) base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 1); + public static void storeScalarField(Object root, long offset, String owner, + String field, long fieldOffset, long bits, int size) { + if (hasByteStorage(root)) { + storeLocationBits(root, Math.addExact(offset, fieldOffset), bits, size); + } else { + Pointer projected = fromStorageLocation(root, offset) + .projectStructField(owner, field, fieldOffset, size, null); + storeLocationBits(projected, 0, bits, size); + } } - public static short getI16Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(2, "i16/u16/f16"); - return (short) base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 2); + public static long loadLocationBits(Object root, long offset, int size) { + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if (pointer.allocation instanceof byte[] && pointer.addressState == null) { + return pointer.loadUnsignedAt(Math.addExact(pointer.byteOffset, offset), size); + } + } + return loadLocationBitsSlow(root, offset, size); } - public static int getI32Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(4, "i32/u32/char"); - return (int) base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 4); + private static long loadLocationBitsSlow(Object root, long offset, int size) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + Object value = ((Cell) storage).value; + if (directStorage(storage)) { + StorageLayout layout = storage.layout(value); + StorageLayout.Leaf leaf = layout == null ? null : layout.at(offset, size); + if (leaf != null) return leaf.read(value, offset, size); + } + return loadLocationBits(storage.boundary(), offset, size); + } + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if (pointer.hasDirectPrimitiveArrayStorage()) { + return loadLocationBits(pointer.allocation, Math.addExact(pointer.byteOffset, offset), size); + } + Pointer escaped = pointer.escapedFieldStorage(offset); + return escaped != null ? escaped.loadUnsigned(size) + : pointer.loadUnsignedAt(Math.addExact(pointer.byteOffset, offset), size); + } + int elementSize = inferredArrayElementSize(root); + if (mayBeInIdentityFilter(MEMORY_VIEW_FILTER, root)) { + // Keep the array used by direct JVM access. + // array() can redirect a decoded view to its byte origin and detach later array reads. + return new Pointer(root, elementSize, 0, elementSize, null).loadUnsignedAt(offset, size); + } + if (root instanceof byte[]) return MemoryBytes.read((byte[]) root, Math.toIntExact(offset), size); + int index = Math.toIntExact(offset / elementSize); + int within = (int) (offset % elementSize); + if (within == 0 && size == elementSize) return primitiveArrayBits(root, index); + long bits = 0; + for (int i = 0; i < size; i++) { + bits |= (long) loadPrimitiveArrayByte(root, Math.toIntExact(offset + i), elementSize) << (8 * i); + } + return bits; } - public static long getI64Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(8, "i64/u64"); - return base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 8); + public static void storeLocationBits(Object root, long offset, long bits, int size) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + Object value = ((Cell) storage).value; + if (directStorage(storage)) { + StorageLayout layout = storage.layout(value); + StorageLayout.Leaf leaf = layout == null ? null : layout.at(offset, size); + if (leaf != null) { + leaf.write(value, offset, bits, size); + return; + } + } + root = storage.boundary(); + } + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if (pointer.hasDirectPrimitiveArrayStorage()) { + storeLocationBits(pointer.allocation, Math.addExact(pointer.byteOffset, offset), bits, size); + return; + } + Pointer escaped = pointer.escapedFieldStorage(offset); + if (escaped != null) escaped.storeBytes(bits, size); + else pointer.storeBytesAt(Math.addExact(pointer.byteOffset, offset), bits, size); + return; + } + int elementSize = inferredArrayElementSize(root); + if (hasScalarWriteTracking(root)) { + new Pointer(root, elementSize, 0, elementSize, null).storeBytesAt(offset, bits, size); + return; + } + if (root instanceof byte[]) { + MemoryBytes.write((byte[]) root, Math.toIntExact(offset), size, bits); + return; + } + int index = Math.toIntExact(offset / elementSize); + int within = (int) (offset % elementSize); + if (within == 0 && size == elementSize) { + storePrimitiveArrayBits(root, index, bits); + return; + } + for (int i = 0; i < size; i++) { + storePrimitiveArrayByte(root, Math.toIntExact(offset + i), elementSize, (int) (bits >>> (8 * i)) & 255); + } } - public static float getF32Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(4, "f32"); - return Float.intBitsToFloat((int) base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 4)); + private boolean hasDirectPrimitiveArrayStorage() { + return allocation != null && rareState == null && addressState == null + && allocation.getClass().isArray() + && allocation.getClass().getComponentType().isPrimitive() + && allocationElementSize == inferredArrayElementSize(allocation) + && !hasScalarWriteTracking(allocation); } - public static double getF64Relative( - Pointer base, long elementOffset, long byteOffset) { - base.requireScalarViewSize(8, "f64"); - return Double.longBitsToDouble(base.loadUnsignedAt( - base.relativeByteOffset(elementOffset, byteOffset), 8)); + private void requireScalarViewSize(int expectedSize, String scalarType) { + if (viewSize == expectedSize) { + return; + } + String sourceView = zeroSizedSourceViewSize() >= 0 + ? "; recorded erased source view is " + zeroSizedSourceViewSize() + " bytes" + : ""; + throw new IllegalStateException( + scalarType + " load requires a " + expectedSize + "-byte view, but pointer has a " + + viewSize + "-byte view" + sourceView); } /** * Loads a reference-valued pointee without allocating its derived pointer * when the JVM allocation already stores the requested Rust value directly. */ - public static Object getObjectRelative( + private static Object loadObjectLocation( Pointer base, - long elementOffset, long byteOffset, String targetClassName) { if (base == null) { throw new NullPointerException("attempted to dereference a null Rust pointer"); } long absoluteByteOffset = - base.relativeByteOffset(elementOffset, byteOffset); + Math.addExact(base.byteOffset, byteOffset); boolean sameCodec = base.allocationCodecClassName == null ? base.viewCodecClassName == null : base.allocationCodecClassName.equals(base.viewCodecClassName); - if (elementOffset == 0 - && byteOffset == 0 + if (byteOffset == 0 && base.allocation instanceof Cell && base.byteOffset == 0 && base.viewSize == base.allocationElementSize @@ -9982,18 +10047,21 @@ public static Object getObjectRelative( && !mayHaveStructuralView(base.allocation)) { int elementIndex = Math.toIntExact( Math.floorDiv(absoluteByteOffset, base.allocationElementSize)); - Object value = base.readElement(elementIndex); boolean trustedDirectCarrier = base.allocation instanceof Cell || base.allocation instanceof ReceiverCell || base.allocation instanceof FieldCell; if (targetClassName == null || targetClassName.isEmpty() - || value == null || trustedDirectCarrier) { base.flushMemoryViewsOverlapping( absoluteByteOffset, base.allocationElementSize); - value = base.readElement(elementIndex); - return value; + return base.readElement(elementIndex); + } + Object value = base.readElement(elementIndex); + if (value == null) { + base.flushMemoryViewsOverlapping( + absoluteByteOffset, base.allocationElementSize); + return base.readElement(elementIndex); } try { Class valueClass = value.getClass(); @@ -10023,7 +10091,7 @@ && matchesBinaryClassName( error); } } - Pointer pointer = materializeRelative(base, elementOffset, byteOffset); + Pointer pointer = fromStorageLocation(base, byteOffset); return targetClassName == null || targetClassName.isEmpty() ? pointer.getObject() : pointer.getObjectAs(targetClassName); @@ -10310,12 +10378,12 @@ public Object directAggregate(Class type) { /** Reads an owned aggregate value without creating a live alias of byte storage. */ public Object getObjectCopyAs(String targetClassName) { - if (allocation instanceof byte[] && rareState == null && viewSize > 0 + if (viewSize > 0 && !isDirectAllocationView() && isGeneratedAggregateCodec(viewCodecClassName)) { - CodecPlan plan = codecPlan(viewCodecClassName); + MemoryCodec plan = codecPlan(viewCodecClassName); if (targetClassName != null && matchesBinaryClassName(targetClassName, plan.encodeParameterType.getName())) { - if (plan.decodeAt != null) { + if (allocation instanceof byte[] && rareState == null && plan.decodeAt() != null) { byte[] bytes = (byte[]) allocation; int offset = Math.toIntExact(byteOffset); int size = materializedViewSize(); @@ -10324,10 +10392,10 @@ && matchesBinaryClassName(targetClassName, plan.encodeParameterType.getName())) "aggregate read exceeds byte-addressable Rust storage"); } flushMemoryViewsOverlapping(byteOffset, size); - return plan.decodeAt.decode(bytes, offset); + return plan.decodeAt().decode(bytes, offset); } - // Pointer-bearing and external codecs still use an independent - // image carrying the source's reference/provenance metadata. + // Owned reads need an independent image with provenance metadata. + // A live decoded view could overwrite source padding when flushed. return decodeAggregate(viewCodecClassName, loadRange(materializedViewSize())); } } @@ -10646,6 +10714,11 @@ public void set(Object value) { if (isDirectAllocationView()) { clearStructuralViewState(); int elementIndex = Math.toIntExact(byteOffset / allocationElementSize); + if (allocation instanceof FieldCell && ((FieldCell) allocation).access.borrowed) { + // Do not read the old borrowed value. Partially initialized storage may not contain a valid carrier. + writeElement(elementIndex, value); + return; + } Object current = readElement(elementIndex); writeElement( elementIndex, @@ -10701,6 +10774,100 @@ public void set(Object value) { storeBytes(incomingBits(value, materializedSize), materializedSize); } + /** Copy exact locations without constructing their boundary wrappers. */ + public static void copyStorage(Object source, long sourceOffset, int sourceSize, String sourceCodec, + Object destination, long destinationOffset, int destinationSize, String destinationCodec, + long byteCount, boolean nonoverlapping) { + int count = checkedArrayLength(byteCount); + if (count == 0) return; + Object from = plainCopyArray(source); + Object to = plainCopyArray(destination); + if (from != null && to != null && from.getClass() == to.getClass()) { + long fromOffset = sourceOffset + (source instanceof Pointer ? ((Pointer) source).byteOffset : 0); + long toOffset = destinationOffset + (destination instanceof Pointer ? ((Pointer) destination).byteOffset : 0); + int width = inferredArrayElementSize(from); + if (fromOffset >= 0 && toOffset >= 0 && fromOffset % width == 0 + && toOffset % width == 0 && count % width == 0) { + if (nonoverlapping && from == to + && fromOffset < Math.addExact(toOffset, count) + && toOffset < Math.addExact(fromOffset, count)) { + throw new IllegalArgumentException("copy_nonoverlapping regions overlap"); + } + System.arraycopy(from, Math.toIntExact(fromOffset / width), + to, Math.toIntExact(toOffset / width), count / width); + return; + } + } + if (count <= 8 && tryCopyScalarLocations(source, sourceOffset, + destination, destinationOffset, count, nonoverlapping)) return; + Pointer fromPointer = fromTypedStorageLocation(source, sourceOffset, sourceSize, sourceCodec); + Pointer toPointer = fromTypedStorageLocation(destination, destinationOffset, destinationSize, destinationCodec); + if (nonoverlapping) copyNonOverlapping(fromPointer, toPointer, count); + else copy(fromPointer, toPointer, count); + } + + /** Read all source bytes before writing so overlapping scalar copies preserve their input. */ + private static boolean tryCopyScalarLocations(Object source, long sourceOffset, + Object destination, long destinationOffset, int count, boolean nonoverlapping) { + Object from = scalarCopyAllocation(source, sourceOffset, count); + Object to = scalarCopyAllocation(destination, destinationOffset, count); + if (from == null || to == null || hasEncodedCopyState(from) || hasEncodedCopyState(to)) return false; + long fromOffset = source instanceof Pointer + ? Math.addExact(((Pointer) source).byteOffset, sourceOffset) : sourceOffset; + long toOffset = destination instanceof Pointer + ? Math.addExact(((Pointer) destination).byteOffset, destinationOffset) : destinationOffset; + if (from == to) { + if (nonoverlapping && fromOffset < Math.addExact(toOffset, count) + && toOffset < Math.addExact(fromOffset, count)) { + throw new IllegalArgumentException("copy_nonoverlapping regions overlap"); + } + if (fromOffset == toOffset) return true; + } + long bits = loadLocationBits(source, sourceOffset, count); + // Flushing can publish encoded references. The general copy must transfer their provenance and GC roots. + if (hasEncodedCopyState(from) || hasEncodedCopyState(to)) return false; + storeLocationBits(destination, destinationOffset, bits, count); + return true; + } + + private static boolean hasEncodedCopyState(Object allocation) { + return mayBeInIdentityFilter(ENCODED_POINTER_FILTER, allocation) + || mayBeInIdentityFilter(ENCODED_REFERENCE_FILTER, allocation); + } + + private static Object scalarCopyAllocation(Object root, long offset, int count) { + if (root instanceof Storage) { + Storage storage = (Storage) root; + if (!directStorage(storage)) return null; + StorageLayout layout = storage.layout(((Cell) storage).value); + return layout != null && layout.at(offset, count) != null ? storage : null; + } + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + // Projected fields can redirect arithmetic to a different owner. + if (pointer.addressState != null || pointer.allocationElementSize <= 0) return null; + root = pointer.allocation; + } else if (mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root)) { + return null; + } + return root != null && root.getClass().isArray() + && root.getClass().getComponentType().isPrimitive() ? root : null; + } + + private static Object plainCopyArray(Object root) { + if (root instanceof Pointer) { + Pointer pointer = (Pointer) root; + if (pointer.allocation == null || pointer.rareState != null || pointer.addressState != null + || pointer.allocationCodecClassName != null + || pointer.allocationElementSize != inferredArrayElementSize(pointer.allocation)) return null; + root = pointer.allocation; + } + return root != null && root.getClass().isArray() + && root.getClass().getComponentType().isPrimitive() + && !hasScalarWriteTracking(root) + && !mayBeInIdentityFilter(MEMORY_VIEW_ORIGIN_FILTER, root) ? root : null; + } + public static void copy(Pointer source, Pointer destination, int byteCount) { if (tryCopyScalarRange( source, @@ -10800,8 +10967,8 @@ private static boolean tryCopyDirectUnionRange( if (sourceCarrier == null || destinationCarrier == null) { return false; } - CodecPlan sourcePlan = codecPlan(source.allocationCodecClassName); - CodecPlan destinationPlan = codecPlan(destination.allocationCodecClassName); + MemoryCodec sourcePlan = codecPlan(source.allocationCodecClassName); + MemoryCodec destinationPlan = codecPlan(destination.allocationCodecClassName); byte[] sourceBytes = sourcePlan.directUnionBytes(sourceCarrier); byte[] destinationBytes = destinationPlan.directUnionBytes(destinationCarrier); if (sourceBytes == null || destinationBytes == null) { @@ -10895,100 +11062,6 @@ public static void copyElements( copy(source, destination, checkedElementByteCount(source, elementCount)); } - private static void copyRelativeBytes( - Pointer source, - long sourceElementOffset, - long sourceByteOffset, - Pointer destination, - long destinationElementOffset, - long destinationByteOffset, - int byteCount, - boolean nonOverlapping) { - long absoluteSourceByteOffset = - source.relativeByteOffset(sourceElementOffset, sourceByteOffset); - long absoluteDestinationByteOffset = - destination.relativeByteOffset(destinationElementOffset, destinationByteOffset); - if (nonOverlapping && source.allocation == destination.allocation) { - long sourceEnd = Math.addExact(absoluteSourceByteOffset, byteCount); - long destinationEnd = Math.addExact(absoluteDestinationByteOffset, byteCount); - if (absoluteSourceByteOffset < destinationEnd - && absoluteDestinationByteOffset < sourceEnd) { - throw new IllegalArgumentException("copy_nonoverlapping regions overlap"); - } - } - if (tryCopyScalarRange( - source, - absoluteSourceByteOffset, - destination, - absoluteDestinationByteOffset, - byteCount)) { - return; - } - if (tryCopyDirectUnionRange( - source, - absoluteSourceByteOffset, - destination, - absoluteDestinationByteOffset, - byteCount)) { - return; - } - if (tryCopyPrimitiveArrayRange( - source, - absoluteSourceByteOffset, - destination, - absoluteDestinationByteOffset, - byteCount)) { - return; - } - Pointer materializedSource = - materializeRelative(source, sourceElementOffset, sourceByteOffset); - Pointer materializedDestination = - materializeRelative(destination, destinationElementOffset, destinationByteOffset); - if (nonOverlapping) { - copyNonOverlapping(materializedSource, materializedDestination, byteCount); - } else { - copy(materializedSource, materializedDestination, byteCount); - } - } - - public static void copyRelative( - Pointer source, - long sourceElementOffset, - long sourceByteOffset, - Pointer destination, - long destinationElementOffset, - long destinationByteOffset, - long byteCount) { - copyRelativeBytes( - source, - sourceElementOffset, - sourceByteOffset, - destination, - destinationElementOffset, - destinationByteOffset, - checkedArrayLength(byteCount), - false); - } - - public static void copyElementsRelative( - Pointer source, - long sourceElementOffset, - long sourceByteOffset, - Pointer destination, - long destinationElementOffset, - long destinationByteOffset, - long elementCount) { - copyRelativeBytes( - source, - sourceElementOffset, - sourceByteOffset, - destination, - destinationElementOffset, - destinationByteOffset, - checkedElementByteCount(source, elementCount), - false); - } - public static void copyNonOverlapping(Pointer source, Pointer destination, int byteCount) { if (source.allocation == destination.allocation) { long sourceEnd = source.byteOffset + byteCount; @@ -11011,44 +11084,6 @@ public static void copyNonOverlappingElements( source, destination, checkedElementByteCount(source, elementCount)); } - public static void copyNonOverlappingRelative( - Pointer source, - long sourceElementOffset, - long sourceByteOffset, - Pointer destination, - long destinationElementOffset, - long destinationByteOffset, - long byteCount) { - copyRelativeBytes( - source, - sourceElementOffset, - sourceByteOffset, - destination, - destinationElementOffset, - destinationByteOffset, - checkedArrayLength(byteCount), - true); - } - - public static void copyNonOverlappingElementsRelative( - Pointer source, - long sourceElementOffset, - long sourceByteOffset, - Pointer destination, - long destinationElementOffset, - long destinationByteOffset, - long elementCount) { - copyRelativeBytes( - source, - sourceElementOffset, - sourceByteOffset, - destination, - destinationElementOffset, - destinationByteOffset, - checkedElementByteCount(source, elementCount), - true); - } - private static void swapBytes(Pointer left, Pointer right, int byteCount) { if (trySwapAlignedElements(left, right, byteCount)) { return; @@ -11124,7 +11159,7 @@ private static int rustIntegerCarrierValue(Object value) { if (current == null) { throw new IllegalArgumentException("Rust integer carrier was null"); } - Field[] fields = PUBLIC_INSTANCE_FIELDS.get(current.getClass()); + RustField[] fields = PUBLIC_INSTANCE_FIELDS.get(current.getClass()); if (fields.length != 1) { throw new IllegalArgumentException( "Rust integer carrier does not have one transparent field: " @@ -11140,15 +11175,102 @@ private static int rustIntegerCarrierValue(Object value) { } public static void writeBytes(Pointer destination, int value, int byteCount) { - byte[] bytes = new byte[byteCount]; - for (int index = 0; index < byteCount; index++) { - bytes[index] = (byte) value; - } - destination.storeRange(bytes); + writeBytes(destination, 0L, value, (long) byteCount); } public static void writeBytes(Pointer destination, int value, long byteCount) { - writeBytes(destination, value, checkedArrayLength(byteCount)); + writeBytes(destination, 0L, value, byteCount); + } + + /** Fill an exact byte range without allocating an address or a byte image. */ + public static void writeBytes(Object root, long offset, int value, long byteCount) { + int count = checkedArrayLength(byteCount); + if (count == 0) return; + long bits = (value & 255L) * 0x0101010101010101L; + if (root instanceof Storage && fillScalarStorage((Storage) root, offset, count, bits)) return; + root = normalizeLocationOrigin(root); + Pointer pointer = root instanceof Pointer ? (Pointer) root : null; + Object allocation = pointer == null ? root : pointer.allocation; + if (allocation != null && allocation.getClass().isArray() + && allocation.getClass().getComponentType().isPrimitive()) { + int width = inferredArrayElementSize(allocation); + if (pointer == null || pointer.allocationElementSize == width) { + long start = pointer == null ? offset : Math.addExact(pointer.byteOffset, offset); + long capacity = (long) Array.getLength(allocation) * width; + if (start < 0 || start > capacity - count) + throw new IndexOutOfBoundsException("byte fill exceeds primitive array storage"); + if (pointer == null && hasScalarWriteTracking(allocation)) + pointer = new Pointer(allocation, width, 0, width, null); + if (pointer != null) { + pointer.prepareMemoryWrite(start, count); + discardEncodedPointers(allocation, start, count); + } + fillPrimitiveArray(allocation, start, count, width, bits); + // Partial fills can leave other encoded references alive. + if (start == 0 && count == capacity) discardEncodedReferences(allocation); + return; + } + } + pointer = root instanceof Storage || root instanceof Pointer + ? fromStorageLocation(root, offset) : fromLocation(root, offset, 1); + byte[] bytes = new byte[count]; + Arrays.fill(bytes, (byte) value); + pointer.storeRange(bytes); + } + + private static boolean fillScalarStorage(Storage storage, long offset, int count, long bits) { + if (!directStorage(storage) || offset < 0 || offset > (long) storage.size - count) return false; + Object value = ((Cell) storage).value; + if (storage.size <= 8 && isPrimitiveScalarCarrier(value)) { + long mask = atomicMask(count) << (offset * 8); + long updated = (valueBits(value, storage.size) & ~mask) | ((bits << (offset * 8)) & mask); + ((Cell) storage).value = carrierFromBits(value, updated, storage.size); + return true; + } + StorageLayout layout = storage.layout(value); + if (layout == null) return false; + long end = offset + count; + // Validate all leaves first. Padding needs the general byte path without prior partial writes. + for (long cursor = offset; cursor < end; ) { + StorageLayout.Leaf leaf = layout.at(cursor, 1); + if (leaf == null) return false; + cursor += Math.min(end - cursor, leaf.size - (cursor - leaf.offset) % leaf.size); + } + for (long cursor = offset; cursor < end; ) { + StorageLayout.Leaf leaf = layout.at(cursor, 1); + int chunk = (int) Math.min(end - cursor, leaf.size - (cursor - leaf.offset) % leaf.size); + leaf.write(value, cursor, bits, chunk); + cursor += chunk; + } + return true; + } + + private static void fillPrimitiveArray(Object array, long offset, int count, int width, long bits) { + long end = offset + count; + int first = Math.toIntExact(offset / width), within = (int) (offset % width); + if (within != 0) { + int chunk = (int) Math.min(end - offset, width - within); + long mask = atomicMask(chunk) << (within * 8); + storePrimitiveArrayBits(array, first, + (primitiveArrayBits(array, first) & ~mask) | ((bits << (within * 8)) & mask)); + offset += chunk; + first++; + } + int last = Math.toIntExact(end / width); + if (first < last) { + if (array instanceof byte[]) Arrays.fill((byte[]) array, first, last, (byte) bits); + else if (array instanceof boolean[]) Arrays.fill((boolean[]) array, first, last, bits != 0); + else if (array instanceof short[]) Arrays.fill((short[]) array, first, last, (short) bits); + else if (array instanceof char[]) Arrays.fill((char[]) array, first, last, (char) bits); + else if (array instanceof int[]) Arrays.fill((int[]) array, first, last, (int) bits); + else if (array instanceof long[]) Arrays.fill((long[]) array, first, last, bits); + else if (array instanceof float[]) Arrays.fill((float[]) array, first, last, Float.intBitsToFloat((int) bits)); + else Arrays.fill((double[]) array, first, last, Double.longBitsToDouble(bits)); + } + if (offset < end && end % width != 0) { + long mask = atomicMask((int) (end % width)); + storePrimitiveArrayBits(array, last, (primitiveArrayBits(array, last) & ~mask) | (bits & mask)); + } } public static void writeElements(Pointer destination, int value, long elementCount) { @@ -11159,7 +11281,7 @@ private static int checkedElementByteCount(Pointer pointer, long elementCount) { return checkedArrayLength(Math.multiplyExact(elementCount, (long) pointer.viewSize)); } - private static int checkedArrayLength(long length) { + static int checkedArrayLength(long length) { if (length < 0 || length > Integer.MAX_VALUE) { throw new IllegalArgumentException("Rust memory operation exceeds JVM array limits"); } @@ -11180,6 +11302,21 @@ private static Pointer slicePointer(Object backing, int index) { : null; } + private long sliceScalarBits(int index, int size, String name) { + Pointer storage = sliceElementView(); + storage.requireScalarViewSize(size, name); + return loadLocationBits(storage, Math.multiplyExact((long) index, size), size); + } + + private static boolean storeSliceScalar(Object backing, int index, long bits, int size) { + if (!(backing instanceof Pointer)) return false; + Pointer storage = ((Pointer) backing).sliceElementView(); + // Aggregate/encoded layouts retain set()'s conversion and ownership rules. + if (storage.viewSize != size || storage.viewCodecClassName != null) return false; + storeLocationBits(storage, Math.multiplyExact((long) index, size), bits, size); + return true; + } + /** * Finds a slice carried through transparent single-field Rust wrappers, * such as {@code UnsafeCell<[T]>}. @@ -11195,7 +11332,7 @@ private static Object transparentSliceView(Pointer storage, Object value) { if (isSliceViewType(value.getClass())) { return value; } - Field[] fields = PUBLIC_INSTANCE_FIELDS.get(value.getClass()); + RustField[] fields = PUBLIC_INSTANCE_FIELDS.get(value.getClass()); if (fields.length != 1) { return null; } @@ -11260,6 +11397,12 @@ private Pointer sliceStorageView(long elementSize, String elementCodecClassName) String allocationCodec = storage.allocationCodecClassName != null ? storage.allocationCodecClassName : elementCodecClassName; + if (checkedElementSize > 0 && storage.allocationElementSize == checkedElementSize + && storage.viewSize == checkedElementSize + && java.util.Objects.equals(storage.viewCodecClassName, elementCodecClassName) + && java.util.Objects.equals(storage.allocationCodecClassName, allocationCodec)) { + return storage; + } if (checkedElementSize == 0) { return new Pointer( storage.allocation, @@ -11301,26 +11444,39 @@ private Pointer sliceStorageView(long elementSize, String elementCodecClassName) return storage.retype(checkedElementSize, elementCodecClassName); } + private static boolean hasScalarWriteTracking(Object root) { + return mayBeInIdentityFilter(MEMORY_VIEW_FILTER, root) + || mayBeInIdentityFilter(ENCODED_POINTER_FILTER, root) + || mayBeInIdentityFilter(ENCODED_REFERENCE_FILTER, root); + } + public static boolean sliceGetBoolean(Object backing, int index) { if (backing instanceof boolean[]) { - return ((boolean[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((boolean[]) backing)[index]; + } + return loadLocationBits(backing, index, 1) != 0; } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getBoolean() + return backing instanceof Pointer + ? ((Pointer) backing).sliceScalarBits(index, 1, "bool") != 0 : ((Boolean) arrayGet(backing, index)).booleanValue(); } public static byte sliceGetI8(Object backing, int index) { if (backing instanceof byte[]) { - return ((byte[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((byte[]) backing)[index]; + } + return (byte) loadLocationBits(backing, index, 1); } if (backing instanceof boolean[]) { - return (byte) (((boolean[]) backing)[index] ? 1 : 0); + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return (byte) (((boolean[]) backing)[index] ? 1 : 0); + } + return (byte) loadLocationBits(backing, index, 1); } - Pointer pointer = slicePointer(backing, index); - if (pointer != null) { - return pointer.getI8(); + if (backing instanceof Pointer) { + return (byte) ((Pointer) backing).sliceScalarBits(index, 1, "i8/u8"); } Object value = arrayGet(backing, index); return value instanceof Boolean @@ -11339,24 +11495,31 @@ public static byte[] sliceToByteArray(Object backing, int offset, int length) { public static short sliceGetI16(Object backing, int index) { if (backing instanceof short[]) { - return ((short[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((short[]) backing)[index]; + } + return (short) loadLocationBits(backing, (long) index * 2, 2); } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getI16() + return backing instanceof Pointer + ? (short) ((Pointer) backing).sliceScalarBits(index, 2, "i16/u16") : ((Number) arrayGet(backing, index)).shortValue(); } public static char sliceGetU16(Object backing, int index) { if (backing instanceof char[]) { - return ((char[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((char[]) backing)[index]; + } + return (char) loadLocationBits(backing, (long) index * 2, 2); } if (backing instanceof short[]) { - return (char) (((short[]) backing)[index] & 0xffff); + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return (char) (((short[]) backing)[index] & 0xffff); + } + return (char) loadLocationBits(backing, (long) index * 2, 2); } - Pointer pointer = slicePointer(backing, index); - if (pointer != null) { - return (char) (pointer.getI16() & 0xffff); + if (backing instanceof Pointer) { + return (char) ((Pointer) backing).sliceScalarBits(index, 2, "i16/u16"); } Object value = arrayGet(backing, index); return value instanceof Character @@ -11366,14 +11529,21 @@ public static char sliceGetU16(Object backing, int index) { public static int sliceGetI32(Object backing, int index) { if (backing instanceof int[]) { - return ((int[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((int[]) backing)[index]; + } + return (int) loadLocationBits(backing, (long) index * 4, 4); } if (backing instanceof char[]) { - return ((char[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((char[]) backing)[index]; + } + return (char) loadLocationBits(backing, (long) index * 2, 2); } - Pointer pointer = slicePointer(backing, index); - Object value = - pointer != null ? Integer.valueOf(pointer.getI32()) : arrayGet(backing, index); + if (backing instanceof Pointer) { + return (int) ((Pointer) backing).sliceScalarBits(index, 4, "i32/u32"); + } + Object value = arrayGet(backing, index); return value instanceof Character ? ((Character) value).charValue() : ((Number) value).intValue(); @@ -11381,93 +11551,172 @@ public static int sliceGetI32(Object backing, int index) { public static long sliceGetI64(Object backing, int index) { if (backing instanceof long[]) { - return ((long[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((long[]) backing)[index]; + } + return loadLocationBits(backing, (long) index * 8, 8); } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getI64() + return backing instanceof Pointer + ? ((Pointer) backing).sliceScalarBits(index, 8, "i64/u64") : ((Number) arrayGet(backing, index)).longValue(); } public static float sliceGetF32(Object backing, int index) { if (backing instanceof float[]) { - return ((float[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((float[]) backing)[index]; + } + return Float.intBitsToFloat((int) loadLocationBits(backing, (long) index * 4, 4)); } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getF32() + return backing instanceof Pointer + ? Float.intBitsToFloat((int) ((Pointer) backing).sliceScalarBits(index, 4, "f32")) : ((Number) arrayGet(backing, index)).floatValue(); } public static double sliceGetF64(Object backing, int index) { if (backing instanceof double[]) { - return ((double[]) backing)[index]; + if (!mayBeInIdentityFilter(MEMORY_VIEW_FILTER, backing)) { + return ((double[]) backing)[index]; + } + return Double.longBitsToDouble(loadLocationBits(backing, (long) index * 8, 8)); } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getF64() + return backing instanceof Pointer + ? Double.longBitsToDouble(((Pointer) backing).sliceScalarBits(index, 8, "f64")) : ((Number) arrayGet(backing, index)).doubleValue(); } + /** Read an owned element without registering a decoded memory view. */ + public static Object sliceGetObjectCopy(Object backing, int index, String target) { + if (backing instanceof Pointer) { + Pointer storage = ((Pointer) backing).sliceElementView(); + return loadStorageCopy(storage, Math.multiplyExact((long) index, storage.viewSize), target); + } + return copyManagedValue(independentRepeatedArrayElement(backing, index)); + } + + // Owned byte reads use the exact codec and do not consume pointer metadata. + static String ownedSliceRecipe(Object backing, Class target) { + if (!(backing instanceof Pointer)) return null; + Pointer pointer = (Pointer) backing; + if (!(pointer.allocation instanceof byte[]) || pointer.allocationElementSize != 1 + || pointer.allocationCodecClassName != null || pointer.viewSize <= 0 + || pointer.viewSize > Integer.MAX_VALUE + || !isGeneratedAggregateCodec(pointer.viewCodecClassName)) return null; + MemoryCodec plan = codecPlan(pointer.viewCodecClassName); + return plan.encodeParameterType == target ? pointer.viewCodecClassName : null; + } + + static boolean ownedSliceMatches(String codec, int size, Object backing) { + if (!(backing instanceof Pointer)) return false; + Pointer pointer = (Pointer) backing; + return pointer.allocation instanceof byte[] && pointer.allocationElementSize == 1 + && pointer.allocationCodecClassName == null && pointer.viewSize == size + && codec.equals(pointer.viewCodecClassName); + } + + static byte[] ownedSliceBytes(Object backing) { + return (byte[]) ((Pointer) backing).allocation; + } + + static int ownedSliceSize(Object backing) { + return (int) ((Pointer) backing).viewSize; + } + + static int ownedSliceOffset(Object backing, int index) { + Pointer pointer = (Pointer) backing; + long offset = Math.addExact(pointer.byteOffset, + Math.multiplyExact((long) index, pointer.viewSize)); + int start = Math.toIntExact(offset); + int size = (int) pointer.viewSize; + if (start < 0 || start > ((byte[]) pointer.allocation).length - size) { + throw new IndexOutOfBoundsException("aggregate read exceeds byte-addressable Rust storage"); + } + pointer.flushFilteredMemoryViews(offset, size, false); + return start; + } + public static Object sliceGetObject(Object backing, int index) { if (backing instanceof Object[]) { return independentRepeatedArrayElement(backing, index); } - Pointer pointer = slicePointer(backing, index); - return pointer != null - ? pointer.getObject() - : independentRepeatedArrayElement(backing, index); + if (backing instanceof Pointer) { + Pointer storage = ((Pointer) backing).sliceElementView(); + long displacement = Math.multiplyExact((long) index, storage.viewSize); + Object decoded = storage.cachedSliceElement(displacement); + if (decoded != null) return decoded; + if (storage.rareState == null && storage.addressState == null + && storage.isDirectAllocationView() && !mayHaveStructuralView(storage.allocation)) { + Object value = loadObjectLocation(storage, displacement, null); + // Array origins belong to the derived location, not this base. + if (value == null || !value.getClass().isArray()) return value; + } + return storage.add(index).getObject(); + } + return independentRepeatedArrayElement(backing, index); } public static void sliceSetBoolean(Object backing, int index, boolean value) { if (backing instanceof boolean[]) { - ((boolean[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, index, value ? 1 : 0, 1); + else ((boolean[]) backing)[index] = value; return; } + if (storeSliceScalar(backing, index, value ? 1 : 0, 1)) return; sliceSetObject(backing, index, Boolean.valueOf(value)); } public static void sliceSetI8(Object backing, int index, byte value) { if (backing instanceof byte[]) { - ((byte[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, index, value, 1); + else ((byte[]) backing)[index] = value; return; } if (backing instanceof boolean[]) { - ((boolean[]) backing)[index] = value != 0; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, index, value != 0 ? 1 : 0, 1); + else ((boolean[]) backing)[index] = value != 0; return; } + if (storeSliceScalar(backing, index, value, 1)) return; sliceSetObject(backing, index, Byte.valueOf(value)); } public static void sliceSetI16(Object backing, int index, short value) { if (backing instanceof short[]) { - ((short[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 2, value, 2); + else ((short[]) backing)[index] = value; return; } + if (storeSliceScalar(backing, index, value, 2)) return; sliceSetObject(backing, index, Short.valueOf(value)); } public static void sliceSetU16(Object backing, int index, char value) { if (backing instanceof char[]) { - ((char[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 2, value, 2); + else ((char[]) backing)[index] = value; return; } if (backing instanceof short[]) { - ((short[]) backing)[index] = (short) value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 2, value, 2); + else ((short[]) backing)[index] = (short) value; return; } + if (storeSliceScalar(backing, index, value, 2)) return; sliceSetObject(backing, index, Character.valueOf(value)); } public static void sliceSetI32(Object backing, int index, int value) { if (backing instanceof int[]) { - ((int[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 4, value, 4); + else ((int[]) backing)[index] = value; return; } if (backing instanceof char[]) { - ((char[]) backing)[index] = (char) value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 2, value, 2); + else ((char[]) backing)[index] = (char) value; return; } + if (storeSliceScalar(backing, index, value, 4)) return; Pointer pointer = slicePointer(backing, index); if (pointer != null) { pointer.set(Integer.valueOf(value)); @@ -11475,13 +11724,13 @@ public static void sliceSetI32(Object backing, int index, int value) { } Class component = backing.getClass().getComponentType(); if (component == byte.class) { - Array.setByte(backing, index, (byte) value); + storeLocationBits(backing, index, value, 1); } else if (component == short.class) { - Array.setShort(backing, index, (short) value); + storeLocationBits(backing, (long) index * 2, value, 2); } else if (component == char.class) { - Array.setChar(backing, index, (char) value); + storeLocationBits(backing, (long) index * 2, value, 2); } else if (component == int.class) { - Array.setInt(backing, index, value); + storeLocationBits(backing, (long) index * 4, value, 4); } else { arraySet(backing, index, Integer.valueOf(value)); } @@ -11489,25 +11738,31 @@ public static void sliceSetI32(Object backing, int index, int value) { public static void sliceSetI64(Object backing, int index, long value) { if (backing instanceof long[]) { - ((long[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 8, value, 8); + else ((long[]) backing)[index] = value; return; } + if (storeSliceScalar(backing, index, value, 8)) return; sliceSetObject(backing, index, Long.valueOf(value)); } public static void sliceSetF32(Object backing, int index, float value) { if (backing instanceof float[]) { - ((float[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 4, Float.floatToRawIntBits(value), 4); + else ((float[]) backing)[index] = value; return; } + if (storeSliceScalar(backing, index, Float.floatToRawIntBits(value), 4)) return; sliceSetObject(backing, index, Float.valueOf(value)); } public static void sliceSetF64(Object backing, int index, double value) { if (backing instanceof double[]) { - ((double[]) backing)[index] = value; + if (hasScalarWriteTracking(backing)) storeLocationBits(backing, (long) index * 8, Double.doubleToRawLongBits(value), 8); + else ((double[]) backing)[index] = value; return; } + if (storeSliceScalar(backing, index, Double.doubleToRawLongBits(value), 8)) return; sliceSetObject(backing, index, Double.valueOf(value)); } @@ -11534,7 +11789,9 @@ private void storeRange(byte[] source) { && allocationElementSize == inferredArrayElementSize(allocation) ? allocationElementSize : 0; - if (directPrimitiveElementSize > 0) { + if (allocation instanceof byte[] && directPrimitiveElementSize == 1) { + System.arraycopy(source, 0, allocation, Math.toIntExact(byteOffset), source.length); + } else if (directPrimitiveElementSize > 0) { for (int index = 0; index < source.length; index++) { storePrimitiveArrayByte( allocation, @@ -11554,7 +11811,7 @@ private void storeRange(byte[] source) { Math.toIntExact(Math.floorDiv(absoluteOffset, allocationElementSize)); int withinElement = (int) Math.floorMod(absoluteOffset, allocationElementSize); Object current = readElement(elementIndex); - CodecPlan plan = isBuiltInCodec(allocationCodecClassName) + MemoryCodec plan = isBuiltInCodec(allocationCodecClassName) ? null : codecPlan(allocationCodecClassName); int chunk = Math.min( source.length - consumed, @@ -11659,84 +11916,31 @@ && hasProjectedFieldCells(current)) { } } - private static CodecPlan codecPlan(String codecClassName) { + static MemoryCodec codecPlan(String codecClassName) { if (codecClassName == null) { throw new IllegalStateException("pointer view has no aggregate codec"); } CodecPlanCache recent = RECENT_CODEC_PLANS.get(); - CodecPlan local = recent.get(codecClassName); + MemoryCodec local = recent.get(codecClassName); if (local != null) { return local; } - CodecPlan cached = CODEC_METHODS.get(codecClassName); + MemoryCodec cached = CODEC_METHODS.get(codecClassName); if (cached != null) { recent.remember(codecClassName, cached); return cached; } try { - if (codecClassName.startsWith(ZERO_SIZED_CODEC_PREFIX)) { - Class valueType = resolvedRuntimeClass( - codecClassName.substring(ZERO_SIZED_CODEC_PREFIX.length())); - return rememberCodecPlan(codecClassName, new CodecPlan(valueType), recent); - } - Class codec = resolvedRuntimeClass(codecClassName); - Method encode = null; - Method decode = null; - Method decodeAt = null; - Method bind = null; - Method arrayElementSize = null; - Method arrayElementCodec = null; - for (Method method : codec.getMethods()) { - if (method.getName().equals("encode") && method.getParameterTypes().length == 1) { - encode = method; - } else if (method.getName().equals("decode") - && method.getParameterTypes().length == 1) { - decode = method; - } else if (method.getName().equals("decodeAt") - && java.util.Arrays.equals(method.getParameterTypes(), - new Class[] {byte[].class, int.class})) { - decodeAt = method; - } else if (method.getName().equals("bind") - && method.getParameterTypes().length == 2) { - bind = method; - } else if (method.getName().equals("_rustArrayElementSize") - && method.getParameterTypes().length == 0) { - arrayElementSize = method; - } else if (method.getName().equals("_rustArrayElementCodec") - && method.getParameterTypes().length == 0) { - arrayElementCodec = method; - } - } - if (encode == null || decode == null) { - throw new NoSuchMethodException( - "pointer codec must define public static encode/decode methods"); - } - encode.setAccessible(true); - decode.setAccessible(true); - if (decodeAt != null) { - decodeAt.setAccessible(true); - } - if (bind != null) { - bind.setAccessible(true); - } - if (arrayElementSize != null) { - arrayElementSize.setAccessible(true); - } - if (arrayElementCodec != null) { - arrayElementCodec.setAccessible(true); - } - CodecPlan plan = new CodecPlan( - encode, decode, decodeAt, bind, arrayElementSize, arrayElementCodec); - return rememberCodecPlan(codecClassName, plan, recent); + return rememberCodecPlan(codecClassName, MemoryCodec.load(codecClassName, RUNTIME_CLASS_LOADER), recent); } catch (ReflectiveOperationException error) { throw new IllegalStateException("could not load Rust pointer codec " + codecClassName, error); } } - private static CodecPlan rememberCodecPlan( - String name, CodecPlan plan, CodecPlanCache recent) { - CodecPlan previous = CODEC_METHODS.putIfAbsent(name, plan); - CodecPlan result = previous == null ? plan : previous; + private static MemoryCodec rememberCodecPlan( + String name, MemoryCodec plan, CodecPlanCache recent) { + MemoryCodec previous = CODEC_METHODS.putIfAbsent(name, plan); + MemoryCodec result = previous == null ? plan : previous; recent.remember(name, result); return result; } @@ -11810,7 +12014,7 @@ public static Object[] emptyUnionObjectStorage() { private static byte[] encodeAggregate(String codecClassName, Object value) { try { - return codecPlan(codecClassName).encode.encode(value); + return codecPlan(codecClassName).encode().encode(value); } catch (Throwable error) { throw new IllegalStateException( "could not encode Rust aggregate memory with " @@ -11823,7 +12027,7 @@ private static byte[] encodeAggregate(String codecClassName, Object value) { private static Object decodeAggregate(String codecClassName, byte[] bytes) { try { - return codecPlan(codecClassName).decode.decode(bytes); + return codecPlan(codecClassName).decode().decode(bytes); } catch (Throwable error) { throw new IllegalStateException("could not decode Rust aggregate memory", error); } @@ -11982,12 +12186,18 @@ private static Pointer decodedRawPointer(Pointer pointer, String codec) { return pointer; } if (isArrayReferenceCodec(codec)) { - return pointer.retype( + return retypeDecodedPointer(pointer, arrayReferenceElementSize(codec), arrayReferenceElementCodec(codec)); } - pointer = pointer.retype(rawPointerPointeeSize(codec), rawPointerPointeeCodec(codec)); - String pointeeClass = rawPointerPointeeClass(codec); - return pointeeClass.isEmpty() ? pointer : pointer.nominalManagedPointee(pointeeClass); + RawPointerLayout layout = rawPointerLayout(codec); + pointer = retypeDecodedPointer(pointer, layout.size, layout.codec); + return layout.owner.isEmpty() ? pointer : pointer.nominalManagedPointee(layout.owner); + } + + // Callers supply a new decoded carrier. No other value can observe its metadata. + private static Pointer retypeDecodedPointer(Pointer pointer, long size, String codec) { + return size > 0 && pointer.viewSize == size && java.util.Objects.equals(pointer.viewCodecClassName, codec) + ? pointer : pointer.retype(size, codec); } private static Object decodeArrayReference(long address, String codec, Class targetClass) { @@ -11998,63 +12208,51 @@ private static Object decodeArrayReference(long address, String codec, Class private static Object decodeArrayReference( Pointer pointer, String codec, Class targetClass) { Pointer data = decodedRawPointer(pointer, codec); - try { - return LONG_SLICE_VIEW_CONSTRUCTORS - .get(targetClass) - .newInstance(data, 0, arrayReferenceLength(codec)); - } catch (ReflectiveOperationException error) { - throw new IllegalStateException("could not reconstruct fixed-array reference", error); - } + return SliceView.create(targetClass, data, 0, arrayReferenceLength(codec)); } - private static long rawPointerPointeeSize(String codec) { - if (isArrayReferenceCodec(codec)) { - return Math.multiplyExact( - arrayReferenceLength(codec), (long) arrayReferenceElementSize(codec)); - } - int sizeStart = RAW_POINTER_VIEW_CODEC.length() + 1; - int codecStart = codec.indexOf('\n', sizeStart); - if (codecStart < 0) { - throw new IllegalArgumentException("invalid Rust raw-pointer codec descriptor"); - } - long pointeeSize; - try { - pointeeSize = Long.parseLong(codec.substring(sizeStart, codecStart)); - } catch (NumberFormatException error) { - throw new IllegalArgumentException("invalid Rust raw-pointer pointee size", error); - } - if (pointeeSize < 0) { - throw new IllegalArgumentException("negative Rust raw-pointer pointee size"); + private static final ConcurrentHashMap RAW_POINTER_LAYOUTS = + new ConcurrentHashMap<>(); + + private static final class RawPointerLayout { + final long size; + final String owner, codec; + + RawPointerLayout(String descriptor) { + int start = RAW_POINTER_VIEW_CODEC.length() + 1; + int ownerStart = descriptor.indexOf('\n', start); + int codecStart = ownerStart < 0 ? -1 : descriptor.indexOf('\n', ownerStart + 1); + if (codecStart < 0) { + throw new IllegalArgumentException("invalid Rust raw-pointer codec descriptor"); + } + try { + size = Long.parseLong(descriptor.substring(start, ownerStart)); + } catch (NumberFormatException error) { + throw new IllegalArgumentException("invalid Rust raw-pointer pointee size", error); + } + if (size < 0) throw new IllegalArgumentException("negative Rust raw-pointer pointee size"); + owner = descriptor.substring(ownerStart + 1, codecStart); + String element = descriptor.substring(codecStart + 1); + codec = element.isEmpty() ? null : element; } - return pointeeSize; } - private static String rawPointerPointeeClass(String codec) { - if (isArrayReferenceCodec(codec)) { - return ""; - } - int classStart = codec.indexOf('\n', RAW_POINTER_VIEW_CODEC.length() + 1); - if (classStart < 0) { - throw new IllegalArgumentException("invalid Rust raw-pointer codec descriptor"); - } - int classEnd = codec.indexOf('\n', classStart + 1); - if (classEnd < 0) { - throw new IllegalArgumentException("invalid Rust raw-pointer codec descriptor"); - } - return codec.substring(classStart + 1, classEnd); + private static RawPointerLayout rawPointerLayout(String codec) { + RawPointerLayout layout = RAW_POINTER_LAYOUTS.get(codec); + if (layout != null) return layout; + layout = new RawPointerLayout(codec); + RawPointerLayout previous = RAW_POINTER_LAYOUTS.putIfAbsent(codec, layout); + return previous == null ? layout : previous; + } + + private static long rawPointerPointeeSize(String codec) { + return isArrayReferenceCodec(codec) + ? Math.multiplyExact(arrayReferenceLength(codec), (long) arrayReferenceElementSize(codec)) + : rawPointerLayout(codec).size; } private static String rawPointerPointeeCodec(String codec) { - if (isArrayReferenceCodec(codec)) { - return arrayReferenceElementCodec(codec); - } - int classStart = codec.indexOf('\n', RAW_POINTER_VIEW_CODEC.length() + 1); - int codecStart = classStart < 0 ? -1 : codec.indexOf('\n', classStart + 1); - if (codecStart < 0) { - throw new IllegalArgumentException("invalid Rust raw-pointer codec descriptor"); - } - String pointeeCodec = codec.substring(codecStart + 1); - return pointeeCodec.isEmpty() ? null : pointeeCodec; + return isArrayReferenceCodec(codec) ? arrayReferenceElementCodec(codec) : rawPointerLayout(codec).codec; } /** @@ -12086,8 +12284,8 @@ private Pointer nominalManagedPointee(String targetClassName) { if (targetClass.isInstance(value)) { return this; } - Field match = null; - for (Field candidate : PUBLIC_INSTANCE_FIELDS.get(value.getClass())) { + RustField match = null; + for (RustField candidate : PUBLIC_INSTANCE_FIELDS.get(value.getClass())) { if (targetClass.isAssignableFrom(candidate.getType())) { if (match != null) { return this; @@ -12098,7 +12296,7 @@ private Pointer nominalManagedPointee(String targetClassName) { if (match == null) { return this; } - return field(value, match.getName(), viewSize, viewCodecClassName) + return field(value, match.getName(), checkedArrayLength(viewSize), viewCodecClassName, false) .withMetadata(metadata) .inheritAddressOrigin(this, 0); } catch (ClassNotFoundException error) { @@ -12380,4 +12578,13 @@ private static float halfToFloat(int half) { } return Float.intBitsToFloat(bits); } + + static boolean cachedFieldMatches(Pointer pointer, Object owner, String name, long size, String codec) { + if (pointer == null || pointer.metadata != -1 || pointer.rareState != null + || pointer.addressState != null || pointer.viewSize != size + || !(pointer.allocation instanceof FieldCell)) return false; + FieldCell cell = (FieldCell) pointer.allocation; + return cell.fixedOwner == owner && cell.access.field.getName().equals(name) + && java.util.Objects.equals(pointer.viewCodecClassName, codec); + } } diff --git a/runtime/src/RuntimeSupport.java b/runtime/src/RuntimeSupport.java index e184014d..f5fbdb53 100644 --- a/runtime/src/RuntimeSupport.java +++ b/runtime/src/RuntimeSupport.java @@ -231,9 +231,7 @@ public static long readStdin(Pointer destination, long length) { if (read < 0) { return 0; } - for (int index = 0; index < read; index++) { - destination.add(index).set(Byte.valueOf(copy[index])); - } + copyBytes(copy, read, destination); return read; } @@ -256,6 +254,10 @@ private static String utf8(Pointer bytes, long length) { static byte[] copyFromPointer(Pointer source, long length) { int checkedLength = Math.toIntExact(length); byte[] copy = new byte[checkedLength]; + if (checkedLength != 0 && Pointer.locationStride(source) == 1) { + Pointer.copy(source, Pointer.array(copy, 0, 1), checkedLength); + return copy; + } for (int index = 0; index < checkedLength; index++) { copy[index] = source.add(index).getI8(); } @@ -270,6 +272,10 @@ static void copyBytes(byte[] source, int length, Pointer destination) { if (length < 0 || length > source.length) { throw new IndexOutOfBoundsException("invalid byte copy length " + length); } + if (length != 0 && Pointer.locationStride(destination) == 1) { + Pointer.copy(Pointer.array(source, 0, 1), destination, length); + return; + } for (int index = 0; index < length; index++) { destination.add(index).set(Byte.valueOf(source[index])); } diff --git a/runtime/src/RustClasses.java b/runtime/src/RustClasses.java new file mode 100644 index 00000000..fa95bfcb --- /dev/null +++ b/runtime/src/RustClasses.java @@ -0,0 +1,31 @@ +package org.rustlang.runtime; + +import java.util.HashMap; +import java.util.Map; + +/** Nested storage identities come from class metadata, not binary-name suffixes. */ +final class RustClasses { + private static final ClassValue>> NESTED = new ClassValue>>() { + @Override + protected Map> computeValue(Class owner) { + Map> classes = new HashMap<>(); + for (Class nested : owner.getDeclaredClasses()) { + classes.put(nested.getSimpleName(), nested); + } + return classes; + } + }; + + private RustClasses() {} + + static boolean hasNested(Class owner, String first, String second) { + Map> classes = NESTED.get(owner); + return classes.containsKey(first) && classes.containsKey(second); + } + + static Class nested(Class owner, String name) throws ClassNotFoundException { + Class nested = NESTED.get(owner).get(name); + if (nested == null) throw new ClassNotFoundException(owner.getName() + " nested " + name); + return nested; + } +} diff --git a/runtime/src/RustField.java b/runtime/src/RustField.java new file mode 100644 index 00000000..37c8437a --- /dev/null +++ b/runtime/src/RustField.java @@ -0,0 +1,181 @@ +package org.rustlang.runtime; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Logical fields of Rust storage, independent of their physical JVM slots. */ +public final class RustField { + private static final ClassValue> FIELDS = + new ClassValue>() { + protected Map computeValue(Class type) { + Map result = new LinkedHashMap<>(); + Field[] fields = type.getFields(); + for (Field field : fields) { + if (Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) continue; + Field displacement = null; + String shape = null; + if (field.getType() == Object.class || field.getType() == long.class) { + for (Field candidate : fields) { + String name = candidate.getName(); + if (!candidate.isSynthetic() || Modifier.isStatic(candidate.getModifiers()) + || !name.startsWith("$rust$")) continue; + int separator = name.indexOf('$', 6); + if (separator < 0 || !name.substring(separator + 1).equals(field.getName())) continue; + String kind = name.substring(6, separator); + if (kind.equals("l")) continue; + if (field.getType() == long.class && !kind.equals("t")) continue; + if (field.getType() == Object.class && kind.equals("t")) continue; + displacement = candidate; + shape = kind; + break; + } + } + result.put(field.getName(), new RustField(field, displacement, shape)); + } + return result; + } + }; + + static final ClassValue ALL = new ClassValue() { + protected RustField[] computeValue(Class type) { + return FIELDS.get(type).values().toArray(new RustField[0]); + } + }; + + public static RustField find(Class owner, String name) throws NoSuchFieldException { + RustField field = optional(owner, name); + if (field == null) throw new NoSuchFieldException(owner.getName() + "." + name); + return field; + } + + static RustField optional(Class owner, String name) { + return FIELDS.get(owner).get(name); + } + + private final Field root; + private final Field displacement; + private final Field length; + private final int size; + private final String codec; + private final boolean typedAddress; + private final Class logicalType; + + private RustField(Field root, Field displacement, String shape) { + this.root = root; + this.displacement = displacement; + root.setAccessible(true); + if (displacement != null) displacement.setAccessible(true); + typedAddress = shape != null && shape.startsWith("a"); + String layoutCodec = null; + try { + if (typedAddress) { + size = Integer.parseInt(shape.substring(1)); + length = null; + logicalType = Pointer.class; + try { + Field metadata = root.getDeclaringClass().getField("$rust$c$" + root.getName()); + if (metadata.getType() != String.class || !Modifier.isStatic(metadata.getModifiers())) { + throw new IllegalStateException("invalid address layout " + metadata); + } + layoutCodec = (String) metadata.get(null); + } catch (NoSuchFieldException absent) { + // Scalar or untyped byte layouts have no codec constant. + } + } else if ("t".equals(shape)) { + size = 0; length = null; logicalType = TaggedLong.class; + } else if ("s".equals(shape) || "u".equals(shape)) { + size = 0; + length = root.getDeclaringClass().getField("$rust$l$" + root.getName()); + length.setAccessible(true); + logicalType = shape.equals("u") ? Utf8View.class : SliceView.class; + } else { + size = shape == null ? 0 : Integer.parseInt(shape); + length = null; + logicalType = displacement == null ? root.getType() : Pointer.class; + } + } catch (ReflectiveOperationException error) { + throw new IllegalStateException("invalid borrowed field " + root, error); + } + codec = layoutCodec; + } + + public String getName() { return root.getName(); } + public Class getType() { return logicalType; } + public Class getDeclaringClass() { return root.getDeclaringClass(); } + boolean isBorrowed() { return displacement != null && logicalType != TaggedLong.class; } + boolean borrowedShape(boolean view) { + return isBorrowed() && (length != null) == view; + } + + /** Read the physical fields without constructing a logical carrier. */ + Object borrowedParts(Object owner, long[] metadata) throws IllegalAccessException { + Object value = root.get(owner); + metadata[0] = length == null ? displacement.getLong(owner) : displacement.getInt(owner); + if (length != null) metadata[1] = length.getLong(owner); + return value; + } + void setBorrowedParts(Object owner, Object value, long offset, long count) throws IllegalAccessException { + if (length == null) displacement.setLong(owner, offset); + else { + displacement.setInt(owner, Math.toIntExact(offset)); + length.setLong(owner, count); + } + root.set(owner, value); + } + MethodHandle getter() throws IllegalAccessException { + if (displacement == null) return MethodHandles.lookup().unreflectGetter(root); + return access("get", MethodType.methodType(Object.class, Object.class)); + } + MethodHandle setter() throws IllegalAccessException { + if (displacement == null) return MethodHandles.lookup().unreflectSetter(root); + return access("set", MethodType.methodType(void.class, Object.class, Object.class)); + } + private MethodHandle access(String name, MethodType type) throws IllegalAccessException { + try { return MethodHandles.lookup().findVirtual(RustField.class, name, type).bindTo(this); } + catch (NoSuchMethodException error) { throw new AssertionError(error); } + } + public int getInt(Object owner) throws IllegalAccessException { return root.getInt(owner); } + + public Object get(Object owner) throws IllegalAccessException { + Object value = root.get(owner); + if (displacement == null) return value; + if (logicalType == TaggedLong.class) return new TaggedLong(((Long) value).longValue(), displacement.getLong(owner)); + if (length == null) return typedAddress + ? Pointer.fromTypedStorageLocation(value, displacement.getLong(owner), size, codec) + : Pointer.addressFromParts(value, displacement.getLong(owner), size); + return SliceView.create(logicalType, value, displacement.getInt(owner), length.getLong(owner)); + } + + public void set(Object owner, Object value) throws IllegalAccessException { + if (logicalType == TaggedLong.class && displacement != null) { + TaggedLong tagged = (TaggedLong) value; + root.setLong(owner, TaggedLong.value(tagged)); + displacement.setLong(owner, TaggedLong.tag(tagged)); + return; + } + if (length != null) { + displacement.setInt(owner, viewStart(value)); + length.setLong(owner, viewLength(value)); + value = viewRoot(value); + } else if (displacement != null) { + value = (Pointer) value; + displacement.setLong(owner, 0); + } + root.set(owner, value); + } + + public static Object viewRoot(Object value) { + return value == null ? null : ((SliceView) value).array; + } + public static int viewStart(Object value) { + return value == null ? 0 : ((SliceView) value).offset; + } + public static long viewLength(Object value) { + return value == null ? 0 : ((SliceView) value).rustLength; + } +} diff --git a/runtime/src/SliceView.java b/runtime/src/SliceView.java new file mode 100644 index 00000000..dd8ec2be --- /dev/null +++ b/runtime/src/SliceView.java @@ -0,0 +1,100 @@ +package org.rustlang.runtime; + +import java.lang.reflect.Array; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** Carries slices across Java and opaque boundaries. Rust calls use components. */ +public class SliceView { + public final Object array; + public final int offset; + public final int length; + public final long rustLength; + + public SliceView(Object array, int offset, int length) { + this(array, offset, (long) length); + } + + public SliceView(Object array, int offset, long length) { + this.array = array; + this.offset = offset; + this.length = (int) length; + this.rustLength = length; + } + + static SliceView create(String className, Object array, int offset, long length) { + if (className.equals("org/rustlang/runtime/SliceView") + || className.equals("org.rustlang.runtime.SliceView")) { + return new SliceView(array, offset, length); + } + if (className.equals("org/rustlang/runtime/Utf8View") + || className.equals("org.rustlang.runtime.Utf8View")) { + return new Utf8View(array, offset, length); + } + throw new IllegalArgumentException("unknown Rust view carrier " + className); + } + + static SliceView create(Class type, Object array, int offset, long length) { + if (type == SliceView.class) return new SliceView(array, offset, length); + if (type == Utf8View.class) return new Utf8View(array, offset, length); + // An explicit Java subclass is an interop boundary, not a Rust carrier. + try { + return (SliceView) type.getConstructor(Object.class, int.class, long.class) + .newInstance(array, offset, length); + } catch (ReflectiveOperationException failure) { + throw new IllegalArgumentException("invalid Rust view carrier " + type, failure); + } + } + + public final Object toArray() { + Object result = Array.newInstance(array.getClass().getComponentType(), length); + System.arraycopy(array, offset, result, 0, length); + return result; + } + + public static SliceView fromString(String value) { + return (SliceView) Pointer.stringView(value, "org/rustlang/runtime/SliceView"); + } + + public static String toUtf8String(SliceView value) { + return new String(Pointer.sliceToByteArray(value.array, value.offset, value.length), + StandardCharsets.UTF_8); + } + + public static Utf8View encodeUtf8(int value, SliceView target) { + SliceView bytes = fromString(String.valueOf(Character.toChars(value))); + System.arraycopy(bytes.array, bytes.offset, target.array, target.offset, bytes.length); + return new Utf8View(target.array, target.offset, bytes.length); + } + + public static boolean startsWith(SliceView value, SliceView prefix) { + if (prefix.length > value.length) return false; + for (int i = 0; i < prefix.length; i++) { + if (!Objects.equals(Pointer.sliceGetObject(value.array, value.offset + i), + Pointer.sliceGetObject(prefix.array, prefix.offset + i))) return false; + } + return true; + } + + public static boolean startsWithI8(SliceView value, SliceView prefix) { + if (prefix.length > value.length) return false; + for (int i = 0; i < prefix.length; i++) { + if (Pointer.sliceGetI8(value.array, value.offset + i) + != Pointer.sliceGetI8(prefix.array, prefix.offset + i)) return false; + } + return true; + } + + public static boolean startsWithI32(SliceView value, SliceView prefix) { + if (prefix.length > value.length) return false; + for (int i = 0; i < prefix.length; i++) { + if (Pointer.sliceGetI32(value.array, value.offset + i) + != Pointer.sliceGetI32(prefix.array, prefix.offset + i)) return false; + } + return true; + } + + public static Object $part$array(SliceView value) { return value == null ? null : value.array; } + public static int $part$offset(SliceView value) { return value == null ? 0 : value.offset; } + public static long $part$rustLength(SliceView value) { return value == null ? 0 : value.rustLength; } +} diff --git a/runtime/src/Storage.java b/runtime/src/Storage.java new file mode 100644 index 00000000..b1d3a8fe --- /dev/null +++ b/runtime/src/Storage.java @@ -0,0 +1,46 @@ +package org.rustlang.runtime; + +/** Owns a typed Rust location. Code carries its byte offset separately. + * Create a Pointer only when an operation needs the general memory API. + */ +class Storage extends Pointer.Cell { + final int size; + final String codec; + final long metadata; + final String scalarLayout; + private volatile StorageLayout layout; + volatile Pointer boundary; + + Storage(Object value, int size, String codec, long metadata, String scalarLayout) { + super(value); + this.size = size; + this.codec = codec; + this.metadata = metadata; + this.scalarLayout = scalarLayout; + } + + StorageLayout layout(Object value) { + StorageLayout current = layout; + if (current == null && value != null && scalarLayout != null) { + layout = current = StorageLayout.of(value.getClass(), scalarLayout); + } + return current; + } + + Pointer boundary() { + Pointer result = boundary; + if (result == null) { + synchronized (this) { + result = boundary; + if (result == null) { + materialize(); + result = Pointer.storageBoundary(this); + boundary = result; + } + } + } + return result; + } + + void materialize() {} +} diff --git a/runtime/src/StorageLayout.java b/runtime/src/StorageLayout.java new file mode 100644 index 00000000..4e0138e8 --- /dev/null +++ b/runtime/src/StorageLayout.java @@ -0,0 +1,170 @@ +package org.rustlang.runtime; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.concurrent.ConcurrentHashMap; + +/** Rust byte offsets mapped to authoritative JVM scalar fields. */ +final class StorageLayout { + private static final ClassValue> PLANS = + new ClassValue>() { + protected ConcurrentHashMap computeValue(Class type) { + return new ConcurrentHashMap<>(); + } + }; + + static StorageLayout of(Class type, String descriptor) { + return PLANS.get(type).computeIfAbsent(descriptor, key -> new StorageLayout(type, key)); + } + + private final Leaf[] leaves; + private final BorrowedFieldPath[] borrows; + + private StorageLayout(Class type, String descriptor) { + String[] entries = descriptor.split("\n", -1); + ArrayList scalars = new ArrayList<>(); + ArrayList borrowed = new ArrayList<>(); + try { + for (int i = 0; i < entries.length; i++) { + String[] parts = entries[i].split(",", -1); + if (parts.length == 4 && (parts[3].startsWith("p:") || parts[3].startsWith("v:"))) { + int lines = Integer.parseInt(parts[3].substring(2)); + if (lines < 1 || lines > entries.length - i - 1) + throw new IllegalArgumentException("invalid borrowed storage codec"); + String codec = String.join("\n", Arrays.copyOfRange(entries, i + 1, i + 1 + lines)); + borrowed.add(new BorrowedFieldPath(type, Integer.parseInt(parts[0]), + Integer.parseInt(parts[1]), parts[2], parts[3].charAt(0) == 'v', codec)); + i += lines; + continue; + } + scalars.add(new Leaf(type, Integer.parseInt(parts[0]), + Integer.parseInt(parts[1]), parts[2], + parts.length == 4 ? Integer.parseInt(parts[3]) : 0)); + } + } catch (ReflectiveOperationException error) { + throw new IllegalStateException("invalid typed Rust storage layout", error); + } + leaves = scalars.toArray(new Leaf[0]); + borrows = borrowed.toArray(new BorrowedFieldPath[0]); + Arrays.sort(leaves, Comparator.comparingInt(leaf -> leaf.offset)); + Arrays.sort(borrows, Comparator.comparingInt(leaf -> leaf.offset)); + } + + BorrowedFieldPath borrowedAt(long offset, boolean view) { + int low = 0, high = borrows.length; + while (low < high) { + int middle = (low + high) >>> 1; + BorrowedFieldPath path = borrows[middle]; + if (path.offset < offset) low = middle + 1; + else if (path.offset > offset) high = middle; + else return path.view == view && path.field().borrowedShape(view) ? path : null; + } + return null; + } + + Leaf at(long offset, int size) { + if (offset < 0 || size <= 0 || size > 8) return null; + int low = 0, high = leaves.length; + while (low < high) { + int middle = (low + high) >>> 1; + if (leaves[middle].offset <= offset) low = middle + 1; + else high = middle; + } + if (low == 0) return null; + Leaf leaf = leaves[low - 1]; + long relative = offset - leaf.offset; + long extent = (long) leaf.size * (leaf.count == 0 ? 1 : leaf.count); + return relative < extent && relative % leaf.size <= leaf.size - size ? leaf : null; + } + + static final class Leaf { + final int offset, size, count; + private final MethodHandle read, write; + + Leaf(Class type, int offset, int size, String path, int count) throws ReflectiveOperationException { + this.offset = offset; + this.size = size; + this.count = count; + String[] names = path.isEmpty() ? new String[0] : path.split("/"); + MethodHandle owner = MethodHandles.identity(Object.class); + int ownerPath = count == 0 ? names.length - 1 : names.length; + for (int i = 0; i < ownerPath; i++) { + RustField field = RustField.find(type, names[i]); + owner = MethodHandles.filterReturnValue(owner, + field.getter().asType(MethodType.methodType(Object.class, Object.class))); + type = field.getType(); + } + MethodHandle getter, setter; + Class scalar; + if (count == 0) { + RustField field = RustField.find(type, names[names.length - 1]); + getter = field.getter(); + setter = field.setter(); + scalar = field.getType(); + } else { + getter = MethodHandles.arrayElementGetter(type); + setter = MethodHandles.arrayElementSetter(type); + scalar = type.getComponentType(); + } + if (scalar == float.class || scalar == double.class) { + boolean single = scalar == float.class; + Class wrapper = single ? Float.class : Double.class; + Class bits = single ? int.class : long.class; + MethodHandle encode = MethodHandles.lookup().findStatic(wrapper, + single ? "floatToRawIntBits" : "doubleToRawLongBits", + MethodType.methodType(bits, scalar)); + MethodHandle decode = MethodHandles.lookup().findStatic(wrapper, + single ? "intBitsToFloat" : "longBitsToDouble", + MethodType.methodType(scalar, bits)); + getter = MethodHandles.filterReturnValue(getter, encode); + setter = MethodHandles.filterArguments(setter, count == 0 ? 1 : 2, decode); + } + read = MethodHandles.filterArguments(MethodHandles.explicitCastArguments(getter, + count == 0 ? MethodType.methodType(long.class, Object.class) + : MethodType.methodType(long.class, Object.class, int.class)), 0, owner); + write = MethodHandles.filterArguments(MethodHandles.explicitCastArguments(setter, + count == 0 ? MethodType.methodType(void.class, Object.class, long.class) + : MethodType.methodType(void.class, Object.class, int.class, long.class)), 0, owner); + } + + long read(Object root, long position, int count) { + try { + return (bits(root, position) >>> (((position - offset) % size) * 8)) & mask(count); + } catch (RuntimeException | Error error) { + throw error; + } catch (Throwable error) { + throw new IllegalStateException("could not read typed Rust field", error); + } + } + + void write(Object root, long position, long bits, int count) { + try { + int within = (int) ((position - offset) % size); + if (within != 0 || count != size) { + int shift = within * 8; + long mask = mask(count) << shift; + bits = (bits(root, position) & ~mask) | ((bits << shift) & mask); + } + if (this.count == 0) write.invokeExact(root, bits); + else write.invokeExact(root, Math.toIntExact((position - offset) / size), bits); + } catch (RuntimeException | Error error) { + throw error; + } catch (Throwable error) { + throw new IllegalStateException("could not write typed Rust field", error); + } + } + + private long bits(Object root, long position) throws Throwable { + return count == 0 ? (long) read.invokeExact(root) + : (long) read.invokeExact(root, Math.toIntExact((position - offset) / size)); + } + + private static long mask(int size) { + return size == 8 ? -1L : (1L << (size * 8)) - 1; + } + } +} diff --git a/runtime/src/TaggedLong.java b/runtime/src/TaggedLong.java new file mode 100644 index 00000000..36184b46 --- /dev/null +++ b/runtime/src/TaggedLong.java @@ -0,0 +1,22 @@ +package org.rustlang.runtime; + +/** Immutable boundary carrier for a scalar payload and independent Rust tag. */ +public final class TaggedLong { + public final long value; + public final long tag; + + public TaggedLong(long value, long tag) { + this.value = value; + this.tag = tag; + } + + public static TaggedLong of(long value, long tag) { return new TaggedLong(value, tag); } + + public static boolean optionEquals(TaggedLong left, TaggedLong right) { + long tag = tag(left); + return tag == tag(right) && (tag == 0 || value(left) == value(right)); + } + + public static long value(TaggedLong value) { return value == null ? 0 : value.value; } + public static long tag(TaggedLong value) { return value == null ? 0 : value.tag; } +} diff --git a/runtime/src/Utf8View.java b/runtime/src/Utf8View.java new file mode 100644 index 00000000..75716298 --- /dev/null +++ b/runtime/src/Utf8View.java @@ -0,0 +1,27 @@ +package org.rustlang.runtime; + +/** A UTF-8-valid boundary view. Length and offsets are measured in bytes. */ +public final class Utf8View extends SliceView { + public Utf8View(Object array, int offset, int length) { super(array, offset, length); } + public Utf8View(Object array, int offset, long length) { super(array, offset, length); } + + public static Utf8View fromJavaString(String value) { + return (Utf8View) Pointer.stringView(value, "org/rustlang/runtime/Utf8View"); + } + + public static String toJavaString(Utf8View value) { return toUtf8String(value); } + public static SliceView asSlice(Utf8View value) { return value; } + public static Utf8View fromSlice(SliceView value) { + return new Utf8View(value.array, value.offset, value.rustLength); + } + public static long len(Utf8View value) { return value.rustLength; } + public static boolean startsWith(Utf8View value, Utf8View prefix) { + return startsWithI8(value, prefix); + } + public static boolean equals(Utf8View value, Utf8View other) { + return value.rustLength == other.rustLength && startsWith(value, other); + } + public static boolean startsWithChar(Utf8View value, int character) { + return toJavaString(value).startsWith(String.valueOf(Character.toChars(character))); + } +} diff --git a/runtime/src/WeakIdentityMap.java b/runtime/src/WeakIdentityMap.java new file mode 100644 index 00000000..31cad297 --- /dev/null +++ b/runtime/src/WeakIdentityMap.java @@ -0,0 +1,287 @@ +package org.rustlang.runtime; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.AbstractMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** Identity-keyed weak table. Callers synchronize every operation externally. */ +final class WeakIdentityMap extends AbstractMap { + private static final class OpenWeakReference extends WeakReference { + private final int identityHash; + + private OpenWeakReference(Object key, ReferenceQueue queue) { + super(key, queue); + identityHash = System.identityHashCode(key); + } + } + + private final ReferenceQueue collectedKeys = new ReferenceQueue<>(); + private OpenWeakReference[] keys; + private Object[] values; + private int size; + private int used; + private int readsUntilCleanup = 256; + + WeakIdentityMap() { + this(16); + } + + WeakIdentityMap(int initialCapacity) { + int capacity = 16; + while (capacity < initialCapacity) { + capacity <<= 1; + } + keys = new OpenWeakReference[capacity]; + values = new Object[capacity]; + } + + private static int tableIndex(int hash, int length) { + hash ^= hash >>> 16; + hash *= 0x7feb352d; + hash ^= hash >>> 15; + return hash & (length - 1); + } + + /** Removes an entry without leaving a tombstone in its probe chain. */ + private void deleteEntry(int deleted) { + if (values[deleted] != null) { + size--; + } + keys[deleted] = null; + values[deleted] = null; + used--; + + int mask = keys.length - 1; + for (int index = (deleted + 1) & mask; + keys[index] != null; + index = (index + 1) & mask) { + OpenWeakReference reference = keys[index]; + int home = tableIndex(reference.identityHash, keys.length); + if ((index < home && (home <= deleted || deleted <= index)) + || (home <= deleted && deleted <= index)) { + keys[deleted] = reference; + values[deleted] = values[index]; + keys[index] = null; + values[index] = null; + deleted = index; + } + } + } + + private void reset() { + keys = new OpenWeakReference[16]; + values = new Object[16]; + size = 0; + used = 0; + readsUntilCleanup = 256; + while (collectedKeys.poll() != null) { + // Entries no longer exist after reset. + } + } + + private void discardCollectedKeys() { + OpenWeakReference collected; + while ((collected = (OpenWeakReference) collectedKeys.poll()) != null) { + int index = tableIndex(collected.identityHash, keys.length); + while (keys[index] != null) { + if (keys[index] == collected) { + deleteEntry(index); + break; + } + index = (index + 1) & (keys.length - 1); + } + } + } + + private void maybeDiscardCollectedKeys() { + if (--readsUntilCleanup == 0) { + discardCollectedKeys(); + readsUntilCleanup = 256; + } + } + + private void rehashForInsert() { + discardCollectedKeys(); + readsUntilCleanup = 256; + int newLength = + size * 4 >= keys.length * 3 + ? keys.length << 1 + : keys.length; + OpenWeakReference[] oldKeys = keys; + Object[] oldValues = values; + keys = new OpenWeakReference[newLength]; + values = new Object[newLength]; + size = 0; + used = 0; + for (int oldIndex = 0; oldIndex < oldKeys.length; oldIndex++) { + OpenWeakReference reference = oldKeys[oldIndex]; + Object key = reference == null ? null : reference.get(); + if (key == null) { + continue; + } + int index = tableIndex(reference.identityHash, keys.length); + while (keys[index] != null) { + index = (index + 1) & (keys.length - 1); + } + keys[index] = reference; + values[index] = oldValues[oldIndex]; + size++; + used++; + } + } + + @SuppressWarnings("unchecked") + private V valueAt(int index) { + return (V) values[index]; + } + + @Override + public V get(Object key) { + maybeDiscardCollectedKeys(); + int index = tableIndex(System.identityHashCode(key), keys.length); + while (true) { + OpenWeakReference reference = keys[index]; + if (reference == null) { + return null; + } + Object live = reference.get(); + // Keep dead entries in the probe chain until the next queue drain. + if (live != null && live == key) { + return valueAt(index); + } + index = (index + 1) & (keys.length - 1); + } + } + + @Override + public V put(Object key, V value) { + discardCollectedKeys(); + readsUntilCleanup = 256; + if (used * 4 >= keys.length * 3) { + rehashForInsert(); + } + int index = tableIndex(System.identityHashCode(key), keys.length); + while (true) { + OpenWeakReference reference = keys[index]; + if (reference == null) { + used++; + keys[index] = new OpenWeakReference(key, collectedKeys); + values[index] = value; + size++; + return null; + } + Object live = reference.get(); + if (live == null) { + deleteEntry(index); + continue; + } else if (live == key) { + V previous = valueAt(index); + values[index] = value; + return previous; + } + index = (index + 1) & (keys.length - 1); + } + } + + @Override + public V remove(Object key) { + discardCollectedKeys(); + readsUntilCleanup = 256; + int index = tableIndex(System.identityHashCode(key), keys.length); + while (true) { + OpenWeakReference reference = keys[index]; + if (reference == null) { + return null; + } + Object live = reference.get(); + if (live == null) { + deleteEntry(index); + continue; + } else if (live == key) { + V previous = valueAt(index); + deleteEntry(index); + if (size == 0) { + reset(); + } + return previous; + } + index = (index + 1) & (keys.length - 1); + } + } + + @Override + public int size() { + discardCollectedKeys(); + for (int index = 0; index < keys.length; ) { + OpenWeakReference reference = keys[index]; + if (reference != null && reference.get() == null) { + deleteEntry(index); + } else { + index++; + } + } + return size; + } + + @Override + public boolean isEmpty() { + discardCollectedKeys(); + if (size == 0) { + return true; + } + for (int index = 0; index < keys.length; ) { + OpenWeakReference reference = keys[index]; + if (reference == null) { + index++; + continue; + } + if (reference.get() != null) { + return false; + } + deleteEntry(index); + } + reset(); + return true; + } + + @Override + public Set> entrySet() { + discardCollectedKeys(); + Set> entries = new HashSet<>(); + for (int index = 0; index < keys.length; ) { + OpenWeakReference reference = keys[index]; + Object key = reference == null ? null : reference.get(); + if (key == null) { + if (reference != null) { + deleteEntry(index); + continue; + } + } else { + entries.add(new java.util.AbstractMap.SimpleImmutableEntry<>( + key, valueAt(index))); + } + index++; + } + return entries; + } + + void forEachLiveKey(java.util.function.Consumer visit) { + discardCollectedKeys(); + for (int index = 0; index < keys.length; ) { + OpenWeakReference reference = keys[index]; + Object key = reference == null ? null : reference.get(); + if (key == null) { + if (reference != null) { + deleteEntry(index); + continue; + } + } else { + visit.accept(key); + } + index++; + } + } +} diff --git a/src/allocator_shims.rs b/src/allocator_shims.rs index 722fd914..537aa07e 100644 --- a/src/allocator_shims.rs +++ b/src/allocator_shims.rs @@ -15,7 +15,7 @@ pub(super) fn allocator_shim_target_signature( } AllocatorTy::Ptr => params.push(( input.name.to_string(), - oomir::Type::Pointer(Box::new(oomir::Type::U8)), + oomir::Type::pointer(oomir::Type::U8), )), AllocatorTy::Usize => { params.push((input.name.to_string(), oomir::Type::U64)); @@ -27,7 +27,7 @@ pub(super) fn allocator_shim_target_signature( } let ret = match method.output { - AllocatorTy::ResultPtr => oomir::Type::Pointer(Box::new(oomir::Type::U8)), + AllocatorTy::ResultPtr => oomir::Type::pointer(oomir::Type::U8), AllocatorTy::Never | AllocatorTy::Unit => oomir::Type::Void, AllocatorTy::Layout | AllocatorTy::Ptr | AllocatorTy::Usize => { panic!("invalid allocator shim output type") @@ -215,10 +215,7 @@ pub(super) fn emit_allocator_shims<'tcx>( result.clone(), ); if matches!(method.output, AllocatorTy::Never) { - instructions.push(oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/AssertionError".to_string(), - message: "Diverging allocator call returned unexpectedly".to_string(), - }); + instructions.push(oomir::Instruction::Unreachable); } else { instructions.push(oomir::Instruction::Return { operand: result.map(|name| oomir::Operand::Variable { diff --git a/src/java_exports.rs b/src/java_exports.rs index 4d37e363..b54272bf 100644 --- a/src/java_exports.rs +++ b/src/java_exports.rs @@ -1,12 +1,66 @@ //! Discover and materialize the Rust library surface exposed to Java. use super::*; +/// Export attributes select the Java API. Rust visibility alone does not select it. +pub(crate) fn is_exported(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + fn scope(tcx: TyCtxt<'_>, mut current: Option) -> bool { + while let Some(def_id) = current { + #[allow(deprecated)] + if tcx.get_all_attrs(def_id).iter().any(|attribute| { + let path = attribute.path(); + path.len() == 2 && path[0].as_str() == "jvm_codegen" && path[1].as_str() == "export" + }) { + return true; + } + current = tcx.opt_parent(def_id); + } + false + } + if scope(tcx, Some(def_id)) { + return true; + } + // An exported type's inherent methods are part of its Java receiver API. + if let Some(implementation) = tcx + .opt_associated_item(def_id) + .and_then(|i| i.impl_container(tcx)) + { + let ty = tcx + .type_of(implementation) + .instantiate_identity() + .skip_norm_wip(); + if let TyKind::Adt(adt, _) = ty.kind() { + return scope(tcx, Some(adt.did())); + } + } + false +} + +/// KotlinFutureInterop requires these named schemas. Definition paths stay constant through +/// reexports. +pub(crate) fn is_runtime_type(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + tcx.crate_name(def_id.krate) == rustc_span::sym::core + && matches!( + lower1::jvm_names::class_for_def_id(tcx, def_id).as_str(), + "org/rustlang/core/task/wake/RawWaker" + | "org/rustlang/core/task/wake/RawWakerVTable" + | "org/rustlang/core/task/wake/Waker" + | "org/rustlang/core/task/wake/Context" + ) +} + pub(super) fn ensure_trait_interface<'tcx>( tcx: TyCtxt<'tcx>, trait_def_id: DefId, data_types: &mut Definitions<'tcx>, ) { let interface_name = lower1::jvm_names::class_for_def_id(tcx, trait_def_id); + if !trait_def_id.is_local() && data_types.has_upstream_type(&interface_name) { + data_types + .foreign_interfaces + .borrow_mut() + .insert(interface_name); + return; + } let methods = trait_interface_methods(tcx, trait_def_id, &interface_name, data_types); match data_types.get_mut(&interface_name) { @@ -63,9 +117,8 @@ pub(super) fn trait_interface_methods<'tcx>( continue; } - let mir_sig = tcx.instantiate_bound_regions_with_erased( - tcx.type_of(def_id).skip_binder().fn_sig(tcx), - ); + let mir_sig = tcx + .instantiate_bound_regions_with_erased(tcx.type_of(def_id).skip_binder().fn_sig(tcx)); let explicit_inputs = mir_sig.inputs(); let output = mir_sig.output(); let instance = Instance::new_raw( @@ -122,12 +175,6 @@ pub(super) fn trait_interface_methods<'tcx>( methods } -pub(super) fn crate_emits_library_artifact(tcx: TyCtxt<'_>) -> bool { - tcx.crate_types() - .iter() - .any(|crate_type| !matches!(crate_type, CrateType::Executable)) -} - pub(super) fn is_lowerable_java_public_function(tcx: TyCtxt<'_>, def_id: DefId) -> bool { if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) { return false; @@ -166,7 +213,8 @@ pub(super) fn java_public_surface_def_ids( JavaPublicSurface::Exported => effective_visibilities.is_exported(local_def_id), JavaPublicSurface::Reachable => effective_visibilities.is_reachable(local_def_id), }; - is_public_enough.then_some(local_def_id.to_def_id()) + (is_public_enough && is_exported(tcx, local_def_id.to_def_id())) + .then_some(local_def_id.to_def_id()) }) .collect(); @@ -199,15 +247,11 @@ pub(super) fn materialize_java_public_data_type<'tcx>( } } -pub(super) fn lower_public_library_exports<'tcx>( +pub(super) fn lower_java_exports<'tcx>( tcx: TyCtxt<'tcx>, oomir_module: &mut lower1::context::Module<'tcx>, lowered_instances: &Lock>>, ) { - if !crate_emits_library_artifact(tcx) { - return; - } - let function_defs = java_public_surface_def_ids(tcx, JavaPublicSurface::Exported) .into_iter() .filter(|def_id| is_lowerable_java_public_function(tcx, *def_id)) diff --git a/src/lib.rs b/src/lib.rs index 95f8869f..46c4524b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ extern crate rustc_abi; extern crate rustc_ast; extern crate rustc_attr_ir; extern crate rustc_codegen_ssa; +extern crate rustc_const_eval; extern crate rustc_data_structures; extern crate rustc_driver; extern crate rustc_hashes; @@ -78,7 +79,6 @@ use rustc_middle::{ }; use rustc_session::{IncrCompSession, Session, config::OutputFilenames}; use rustc_span::def_id::{DefId, LOCAL_CRATE}; -use rustc_structures::CrateType; use std::{ any::Any, io::{BufReader, BufWriter, Write}, @@ -281,13 +281,17 @@ fn empty_oomir_module<'tcx>( suppressed_data_types: HashSet::default(), shared_data_types: None, shared_context: None, - relative_static_methods: Arc::new(HashSet::default()), external_interfaces: HashSet::default(), statics: HashMap::default(), } } fn prepare_oomir_shard(mut module: lower1::context::Module<'_>) -> oomir::Module { + if !module.data_types.external_schemas.is_empty() { + module.shared_data_types = Some(Arc::new(std::mem::take( + &mut module.data_types.external_schemas, + ))); + } module.external_interfaces.extend(std::mem::take( &mut *module.data_types.foreign_interfaces.borrow_mut(), )); @@ -302,7 +306,6 @@ fn prepare_oomir_shard(mut module: lower1::context::Module<'_>) -> oomir::Module fn emit_oomir_shard( shard_name: &str, oomir_module: oomir::Module, - emit_runtime_views: bool, debug_info: lower2::DebugInfoOptions, emitted_class_registry: &lower2::EmittedClassRegistry, ) -> Vec<(String, PathBuf)> { @@ -318,16 +321,9 @@ fn emit_oomir_shard( ) ); - let generated_classes = lower2::oomir_to_jvm_bytecode( - oomir_module, - debug_info, - emit_runtime_views, - emitted_class_registry, + lower2::oomir_to_jvm_bytecode(oomir_module, debug_info, emitted_class_registry).unwrap_or_else( + |error| panic!("failed to lower OOMIR shard {shard_name} to JVM bytecode: {error}"), ) - .unwrap_or_else(|error| { - panic!("failed to lower OOMIR shard {shard_name} to JVM bytecode: {error}") - }); - generated_classes } impl CodegenBackend for MyBackend { @@ -371,18 +367,12 @@ impl CodegenBackend for MyBackend { scope, worker_count, OOMIR_SHARD_QUEUE_DEPTH, - |(ordinal, shard_name, module, emit_runtime_views): ( - usize, - String, - oomir::Module, - bool, - )| { + |(ordinal, shard_name, module): (usize, String, oomir::Module)| { ( ordinal, emit_oomir_shard( &shard_name, module, - emit_runtime_views, debug_info, &emitted_class_registry, ), @@ -399,11 +389,11 @@ impl CodegenBackend for MyBackend { // stream as their own job while ordinary owners are lowered. let mut export_module = empty_oomir_module(tcx, &crate_module_class, Arc::clone(&shared_lowering)); - lower_public_library_exports(tcx, &mut export_module, &lowered_instances); + lower_java_exports(tcx, &mut export_module, &lowered_instances); emit_allocator_shims(tcx, &mut export_module); let mut export_module = prepare_oomir_shard(export_module); canonical_data_types.collect(&mut export_module); - workers.submit((submitted, "java-exports".to_string(), export_module, true)); + workers.submit((submitted, "java-exports".to_string(), export_module)); submitted += 1; // Keep each JVM owner intact, and batch small owners so their @@ -470,7 +460,7 @@ impl CodegenBackend for MyBackend { ); let mut module = prepare_oomir_shard(module); canonical_data_types.borrow_mut().collect(&mut module); - producer.submit((ordinal, shard_name, module, false)); + producer.submit((ordinal, shard_name, module)); } }); }); @@ -510,7 +500,8 @@ impl CodegenBackend for MyBackend { drop(producer); let canonical_data_types = canonical_data_types.into_inner(); - let provided_symbols = shared_lowering.provided_symbols(); + let mut provided_symbols = shared_lowering.provided_symbols(); + provided_symbols.types = canonical_data_types.provided_names(); drop(shared_lowering); let canonical_timer = tcx.sess.timer("jvm_canonical_types"); @@ -524,7 +515,7 @@ impl CodegenBackend for MyBackend { .enumerate() { let shard_name = format!("canonical-types-{index}"); - workers.submit((submitted, shard_name, module, false)); + workers.submit((submitted, shard_name, module)); submitted += 1; } let mut results = tcx.sess.time("jvm_finish_emission", || workers.finish()); @@ -557,7 +548,7 @@ impl CodegenBackend for MyBackend { ) -> (CompiledModules, UnordMap) { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let (generated_classes, provided_symbols) = *ongoing_codegen - .downcast::<(Vec<(String, PathBuf)>, Vec)>() + .downcast::<(Vec<(String, PathBuf)>, symbols::Provided)>() .expect("in join_codegen: ongoing_codegen is not a generated-class list"); let temporary_directories: HashSet<_> = generated_classes diff --git a/src/lower1.rs b/src/lower1.rs index 3f03cb07..d37fb7b7 100644 --- a/src/lower1.rs +++ b/src/lower1.rs @@ -1,11 +1,4 @@ -//! This is the stage 1 lowering pass of the compiler. -//! It is responsible for coverting the MIR into a lower-level IR, called OOMIR (see src/oomir.rs). -//! It is a simple pass that converts the MIR into a more object-oriented representation. - -// lower1.rs -//! This module converts Rust MIR into an object-oriented MIR (OOMIR) -//! that sits between MIR and JVM bytecode. It supports a subset of Rust constructs -//! (arithmetic, branching, returns) and can be extended to support more of Rust. +//! Lower monomorphized Rust MIR into typed JVM operations for SSA construction. use crate::lower1::context::Definitions; use crate::oomir; @@ -13,16 +6,20 @@ use control_flow::convert_basic_block; use rustc_attr_ir::lang_items::LangItem; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use rustc_middle::{ - mir::{ - BasicBlock, Body, Local, OUTERMOST_SOURCE_SCOPE, Place, ProjectionElem, StatementKind, - TerminatorKind, VarDebugInfoContents, - }, + mir::{Body, Local, OUTERMOST_SOURCE_SCOPE, VarDebugInfoContents}, ty::{EarlyBinder, Instance, InstanceKind, ShimKind, TyCtxt}, }; use rustc_span::def_id::DefId; -use std::collections::VecDeque; use types::ty_to_oomir_type; +mod initialization; +mod inline; +mod inline_enums; +mod inline_niches; +mod inline_places; +mod inline_records; +use initialization::{MirControlFlow, class_locals_needing_initial_carriers}; + mod debug; pub(crate) use debug::source_location; use debug::{DebugScopeCache, local_variable_scope}; @@ -40,351 +37,6 @@ pub(crate) fn is_non_null_lang_item(tcx: TyCtxt<'_>, def_id: DefId) -> bool { tcx.is_lang_item(def_id, LangItem::NonNull) } -#[derive(Clone, Debug, PartialEq, Eq)] -struct LocalBitSet { - words: Vec, -} - -impl LocalBitSet { - fn empty(local_count: usize) -> Self { - Self { - words: vec![0; local_count.div_ceil(u64::BITS as usize)], - } - } - - fn all(local_count: usize) -> Self { - let mut result = Self { - words: vec![u64::MAX; local_count.div_ceil(u64::BITS as usize)], - }; - if let Some(last) = result.words.last_mut() - && !local_count.is_multiple_of(u64::BITS as usize) - { - *last = (1u64 << (local_count % u64::BITS as usize)) - 1; - } - result - } - - fn insert(&mut self, local: Local) { - let index = local.index(); - self.words[index / u64::BITS as usize] |= 1 << (index % u64::BITS as usize); - } - - fn remove(&mut self, local: Local) { - let index = local.index(); - self.words[index / u64::BITS as usize] &= !(1 << (index % u64::BITS as usize)); - } - - fn contains(&self, local: Local) -> bool { - let index = local.index(); - self.words[index / u64::BITS as usize] & (1 << (index % u64::BITS as usize)) != 0 - } - - fn assign_union(&mut self, left: &Self, right: &Self) { - for ((dest, left), right) in self.words.iter_mut().zip(&left.words).zip(&right.words) { - *dest = *left | *right; - } - } - - fn intersect_with_union(&mut self, left: &Self, right: &Self) { - for ((dest, left), right) in self.words.iter_mut().zip(&left.words).zip(&right.words) { - *dest &= *left | *right; - } - } - - fn assign_transfer(&mut self, input: &Self, killed: &Self, generated: &Self) { - for (((dest, input), killed), generated) in self - .words - .iter_mut() - .zip(&input.words) - .zip(&killed.words) - .zip(&generated.words) - { - *dest = (*input & !*killed) | *generated; - } - } - - fn intersect_with_transfer(&mut self, input: &Self, killed: &Self, generated: &Self) { - for (((dest, input), killed), generated) in self - .words - .iter_mut() - .zip(&input.words) - .zip(&killed.words) - .zip(&generated.words) - { - *dest &= (*input & !*killed) | *generated; - } - } - - fn into_locals(self) -> Vec { - let mut locals = Vec::new(); - for (word_index, mut word) in self.words.into_iter().enumerate() { - while word != 0 { - let bit = word.trailing_zeros() as usize; - locals.push(Local::from_usize(word_index * u64::BITS as usize + bit)); - word &= word - 1; - } - } - locals - } - - fn to_hash_set(&self) -> HashSet { - let mut locals = HashSet::default(); - for (word_index, word) in self.words.iter().copied().enumerate() { - let mut word = word; - while word != 0 { - let bit = word.trailing_zeros() as usize; - locals.insert(Local::from_usize(word_index * u64::BITS as usize + bit)); - word &= word - 1; - } - } - locals - } -} - -struct MirControlFlow { - predecessors: Vec>, - reachable: Vec, -} - -impl MirControlFlow { - fn new(mir: &Body<'_>) -> Self { - let mut predecessors = vec![Vec::new(); mir.basic_blocks.len()]; - for (block, data) in mir.basic_blocks.iter_enumerated() { - for successor in data.terminator().successors() { - predecessors[successor.index()].push(block); - } - } - - let mut reachable = vec![false; mir.basic_blocks.len()]; - let mut queue = VecDeque::from([BasicBlock::from_usize(0)]); - while let Some(block) = queue.pop_front() { - if std::mem::replace(&mut reachable[block.index()], true) { - continue; - } - queue.extend(mir.basic_blocks[block].terminator().successors()); - } - Self { - predecessors, - reachable, - } - } -} - -fn available_pointer_locals_at_block_entries<'tcx>( - mir: &Body<'tcx>, - pointer_origins: &control_flow::MutableBorrowMap<'tcx>, - control_flow: &MirControlFlow, -) -> Vec { - let local_count = mir.local_decls.len(); - let mut all_pointer_locals = LocalBitSet::empty(local_count); - for local in pointer_origins.keys().copied() { - all_pointer_locals.insert(local); - } - let mut generated = vec![LocalBitSet::empty(local_count); mir.basic_blocks.len()]; - for (block, data) in mir.basic_blocks.iter_enumerated() { - for statement in &data.statements { - let StatementKind::Assign(assignment) = &statement.kind else { - continue; - }; - let (place, _) = assignment.as_ref(); - if place.projection.is_empty() && pointer_origins.contains_key(&place.local) { - generated[block.index()].insert(place.local); - } - } - } - - let entry = BasicBlock::from_usize(0); - let mut available = mir - .basic_blocks - .indices() - .map(|block| { - if block == entry || !control_flow.reachable[block.index()] { - LocalBitSet::empty(local_count) - } else { - all_pointer_locals.clone() - } - }) - .collect::>(); - let mut incoming = LocalBitSet::empty(local_count); - - loop { - let mut changed = false; - for block in mir.basic_blocks.indices().filter(|block| *block != entry) { - if !control_flow.reachable[block.index()] { - continue; - } - let mut predecessors = control_flow.predecessors[block.index()] - .iter() - .copied() - .filter(|predecessor| control_flow.reachable[predecessor.index()]); - if let Some(first) = predecessors.next() { - incoming.assign_union(&available[first.index()], &generated[first.index()]); - for predecessor in predecessors { - incoming.intersect_with_union( - &available[predecessor.index()], - &generated[predecessor.index()], - ); - } - } else { - incoming = LocalBitSet::empty(local_count); - } - if available[block.index()] != incoming { - available[block.index()].clone_from(&incoming); - changed = true; - } - } - if !changed { - break; - } - } - - available -} - -fn update_class_carrier_state( - place: &Place<'_>, - initialized: &mut LocalBitSet, - needs_initial_carrier: Option<&mut LocalBitSet>, -) { - if place.projection.is_empty() { - initialized.insert(place.local); - return; - } - - if matches!(place.projection.first(), Some(ProjectionElem::Field(..))) { - if let Some(needs_initial_carrier) = needs_initial_carrier - && !initialized.contains(place.local) - { - needs_initial_carrier.insert(place.local); - } - initialized.insert(place.local); - } -} - -fn record_class_carrier_transfer( - place: &Place<'_>, - generated: &mut LocalBitSet, - killed: &mut LocalBitSet, -) { - if place.projection.is_empty() - || matches!(place.projection.first(), Some(ProjectionElem::Field(..))) - { - generated.insert(place.local); - killed.remove(place.local); - } -} - -fn class_locals_needing_initial_carriers( - mir: &Body<'_>, - control_flow: &MirControlFlow, -) -> Vec { - let entry = BasicBlock::from_usize(0); - let local_count = mir.local_decls.len(); - let all_locals = LocalBitSet::all(local_count); - let mut argument_locals = LocalBitSet::empty(local_count); - for index in 1..=mir.arg_count { - argument_locals.insert(Local::from_usize(index)); - } - let mut generated = vec![LocalBitSet::empty(local_count); mir.basic_blocks.len()]; - let mut killed = generated.clone(); - for (block, data) in mir.basic_blocks.iter_enumerated() { - let block_generated = &mut generated[block.index()]; - let block_killed = &mut killed[block.index()]; - for statement in &data.statements { - match &statement.kind { - StatementKind::Assign(assignment) => { - let (place, _) = assignment.as_ref(); - record_class_carrier_transfer(place, block_generated, block_killed); - } - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - block_generated.remove(*local); - block_killed.insert(*local); - } - _ => {} - } - } - if let TerminatorKind::Call { destination, .. } = &data.terminator().kind { - record_class_carrier_transfer(destination, block_generated, block_killed); - } - } - - let mut available = mir - .basic_blocks - .indices() - .map(|block| { - if block == entry { - argument_locals.clone() - } else if control_flow.reachable[block.index()] { - all_locals.clone() - } else { - LocalBitSet::empty(local_count) - } - }) - .collect::>(); - let mut incoming = LocalBitSet::empty(local_count); - - loop { - let mut changed = false; - for block in mir.basic_blocks.indices().filter(|block| *block != entry) { - if !control_flow.reachable[block.index()] { - continue; - } - let mut predecessors = control_flow.predecessors[block.index()] - .iter() - .copied() - .filter(|predecessor| control_flow.reachable[predecessor.index()]); - if let Some(first) = predecessors.next() { - incoming.assign_transfer( - &available[first.index()], - &killed[first.index()], - &generated[first.index()], - ); - for predecessor in predecessors { - incoming.intersect_with_transfer( - &available[predecessor.index()], - &killed[predecessor.index()], - &generated[predecessor.index()], - ); - } - } else { - incoming = LocalBitSet::empty(local_count); - } - if available[block.index()] != incoming { - available[block.index()].clone_from(&incoming); - changed = true; - } - } - if !changed { - break; - } - } - - let mut needs_initial_carrier = LocalBitSet::empty(local_count); - for block in mir.basic_blocks.indices() { - if !control_flow.reachable[block.index()] { - continue; - } - let mut state = available[block.index()].clone(); - let block_data = &mir.basic_blocks[block]; - for statement in &block_data.statements { - match &statement.kind { - StatementKind::Assign(assignment) => { - let (place, _) = assignment.as_ref(); - update_class_carrier_state(place, &mut state, Some(&mut needs_initial_carrier)) - } - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - state.remove(*local); - } - _ => {} - } - } - if let TerminatorKind::Call { destination, .. } = &block_data.terminator().kind { - update_class_carrier_state(destination, &mut state, Some(&mut needs_initial_carrier)); - } - } - needs_initial_carrier.into_locals() -} - fn jvm_default_operand(ty: &oomir::Type) -> oomir::Operand { use oomir::{Constant, Operand, Type}; @@ -404,10 +56,9 @@ fn jvm_default_operand(ty: &oomir::Type) -> oomir::Operand { Type::F32 => Constant::F32(0.0), Type::F64 => Constant::F64(0.0), Type::Pointer(_) - | Type::MutableReference(_) - | Type::Reference(_) | Type::Array(_) | Type::Slice(_) + | Type::TaggedI64 | Type::Str | Type::Class(_) | Type::Interface(_) => Constant::Null(ty.clone()), @@ -426,6 +77,8 @@ pub fn mir_to_oomir<'tcx>( data_types: &mut Definitions<'tcx>, external_interfaces: &mut HashSet, ) -> oomir::Function { + let expanded = inline::expand(tcx, instance, mir); + let mir = expanded.as_ref().unwrap_or(mir); data_types.with_body(mir, |data_types| { lower_body( tcx, @@ -538,6 +191,14 @@ fn lower_body<'tcx>( let return_oomir_ty: oomir::Type = ty_to_oomir_type(return_ty, tcx, data_types, instance); + if instance.def_id().is_local() + && crate::java_exports::is_lowerable_java_public_function(tcx, instance.def_id()) + && tcx.visibility(instance.def_id()).is_public() + && crate::java_exports::is_exported(tcx, instance.def_id()) + { + control_flow::rvalue::ensure_exported_closure_calls(return_ty, tcx, data_types, instance); + } + let mut signature = oomir::Signature { params: params_oomir, ret: Box::new(return_oomir_ty.clone()), // Clone here to pass to convert_basic_block @@ -600,7 +261,7 @@ fn lower_body<'tcx>( let (oomir_name, debug_type) = if data_types.local_uses_stable_cell(local) { ( place::local_cell_name(local), - oomir::Type::Pointer(Box::new(value_type)), + oomir::Type::pointer(value_type), ) } else { (format!("_{}", local.index()), value_type) @@ -655,10 +316,6 @@ fn lower_body<'tcx>( let entry_label = "bb0".to_string(); let mir_control_flow = MirControlFlow::new(mir); - let mut mutable_borrows = control_flow::collect_pointer_origins(mir, tcx, instance, data_types); - let available_pointer_locals = - available_pointer_locals_at_block_entries(mir, &mutable_borrows, &mir_control_flow); - for (bb, bb_data) in mir.basic_blocks.iter_enumerated() { let bb_ir = convert_basic_block( bb, @@ -670,10 +327,8 @@ fn lower_body<'tcx>( &mut basic_blocks, data_types, external_interfaces, - &mut mutable_borrows, &debug_variables, &debug_scope_cache, - available_pointer_locals[bb.index()].to_hash_set(), ); // Pass return type here basic_blocks.insert(bb_ir.label.clone(), bb_ir); } @@ -709,35 +364,39 @@ fn lower_body<'tcx>( let tuple_param_index = if closure_has_captures { 1 } else { 0 }; let tuple_param_local = format!("param_{tuple_param_index}"); if let Some((_tuple_param_name, tuple_param_ty)) = signature.params.get(tuple_param_index) { - // Check if it's a tuple/struct type that we need to unpack - if let oomir::Type::Class(class_name) = tuple_param_ty { - // Get the data type definition to see its fields - if let Some(oomir::DataType::Class { fields, .. }) = data_types.get(class_name) { - // Unpack each field from the tuple into the expected local variables - // Local 1 contains the tuple, we need to extract fields to locals 2, 3, 4... - for (field_idx, (field_name, field_ty)) in fields.iter().enumerate() { - let local_var_index = field_idx + 2; // Start from local 2 (local 1 is the tuple) - - // Get the field from the tuple object (local 1) - instrs.push(oomir::Instruction::GetField { - dest: format!("_{}", local_var_index), - object: oomir::Operand::Variable { - name: tuple_param_local.to_string(), - ty: tuple_param_ty.clone(), - }, - field_name: field_name.clone(), - field_ty: field_ty.clone(), - owner_class: class_name.clone(), - }); - } - } + let values = types::tuple_fields( + params_ty[0], + oomir::Operand::Variable { + name: tuple_param_local, + ty: tuple_param_ty.clone(), + }, + "closure_arg", + tcx, + data_types, + instance, + &mut instrs, + ); + for (index, src) in values.into_iter().enumerate() { + instrs.push(oomir::Instruction::Move { + dest: format!("_{}", index + 2), + src, + }); } } } let carrier_locals = class_locals_needing_initial_carriers(mir, &mir_control_flow); + let mut initialized_cells = HashSet::default(); for local in carrier_locals { - if data_types.local_uses_stable_cell(local) { + let rust_ty = data_types.normalize(tcx, mir.local_decls[local].ty, instance); + if let Some(word) = types::packed_word(rust_ty, tcx) { + instrs.push(oomir::Instruction::Move { + dest: format!("_{}", local.index()), + src: word.zero(), + }); + if data_types.local_uses_stable_cell(local) { + initialized_cells.insert(local); + } continue; } let oomir::Type::Class(class_name) = place::get_place_type( @@ -751,6 +410,7 @@ fn lower_body<'tcx>( }; let Some(oomir::DataType::Class { fields, + kind: crate::oomir::ClassKind::Value | crate::oomir::ClassKind::JavaValue, is_abstract: false, .. }) = data_types.get(&class_name) @@ -767,6 +427,9 @@ fn lower_body<'tcx>( class_name, args: constructor_args, }); + if data_types.local_uses_stable_cell(local) { + initialized_cells.insert(local); + } } for (local, _) in mir.local_decls.iter_enumerated() { @@ -788,19 +451,23 @@ fn lower_body<'tcx>( data_types, &mut instrs, ); - let initial_value = - if local.index() > 0 && local.index() <= mir.arg_count && value_type.has_jvm_value() { - oomir::Operand::Variable { - name: format!("_{}", local.index()), - ty: value_type.clone(), - } - } else if let Some(value) = implicit_zst { - value - } else { - oomir::Operand::Constant(oomir::Constant::Null(oomir::Type::Class( - "java/lang/Object".to_string(), - ))) - }; + let initial_value = if ((local.index() > 0 && local.index() <= mir.arg_count) + || initialized_cells.contains(&local)) + && value_type.has_jvm_value() + { + // An address does not initialize its value. Inlined clone shims can assign fields + // before they assign the complete aggregate. + oomir::Operand::Variable { + name: format!("_{}", local.index()), + ty: value_type.clone(), + } + } else if let Some(value) = implicit_zst { + value + } else { + oomir::Operand::Constant(oomir::Constant::Null(oomir::Type::Class( + "java/lang/Object".to_string(), + ))) + }; instrs.push(oomir::Instruction::InvokeStatic { dest: Some(place::local_cell_name(local)), class_name: oomir::POINTER_CLASS.to_string(), @@ -814,8 +481,9 @@ fn lower_body<'tcx>( ("size".to_string(), oomir::Type::I32), ("codec".to_string(), oomir::Type::java_string()), ("alignment".to_string(), oomir::Type::I32), + ("layout".to_string(), oomir::Type::java_string()), ], - ret: Box::new(oomir::Type::Pointer(Box::new(value_type))), + ret: Box::new(oomir::Type::pointer(value_type)), is_static: true, }, args: vec![ @@ -856,6 +524,7 @@ fn lower_body<'tcx>( ) .expect("stable local alignment exceeds the JVM runtime address space"), )), + types::scalar_storage_layout(mir.local_decls[local].ty, tcx, data_types, instance), ], }); } diff --git a/src/lower1/context.rs b/src/lower1/context.rs index bedb655d..3932c8d4 100644 --- a/src/lower1/context.rs +++ b/src/lower1/context.rs @@ -23,14 +23,15 @@ pub(crate) type CheckedIntrinsic = (String, String, String); /// No completed function bodies or serialized output are retained here. #[derive(Default)] pub(crate) struct CrateContext<'tcx> { - pub(crate) upstream_symbols: HashSet, + upstream: crate::symbols::Upstream, + provided_codecs: Lock>, provided_symbols: Lock>, normalized: Lock, GenericArgsRef<'tcx>), Ty<'tcx>>>, - tuple_abis: Lock>>, allocations: Lock>, checked_intrinsics: Lock>, union_bodies: Lock>>, storage_objects: Lock, bool>>, + storage_layouts: Lock, Option>>, byte_support: Lock, Result<(), String>>>, completed_codecs: Lock, super::types::PointerMemoryCodec>>, caller_locations: Lock>, @@ -41,12 +42,11 @@ pub(crate) struct CrateContext<'tcx> { impl<'tcx> CrateContext<'tcx> { pub(crate) fn with_upstream_symbols(tcx: TyCtxt<'tcx>) -> Self { Self { - upstream_symbols: crate::symbols::upstream(tcx) - .expect("could not read JVM symbol indexes"), + upstream: crate::symbols::upstream(tcx).expect("could not read JVM symbol indexes"), ..Self::default() } } - pub(crate) fn provided_symbols(&self) -> Vec { + pub(crate) fn provided_symbols(&self) -> crate::symbols::Provided { let mut symbols = self .provided_symbols .borrow() @@ -54,7 +54,18 @@ impl<'tcx> CrateContext<'tcx> { .copied() .collect::>(); symbols.sort_unstable(); - symbols + let mut codecs = self + .provided_codecs + .borrow() + .iter() + .cloned() + .collect::>(); + codecs.sort_unstable(); + crate::symbols::Provided { + bodies: symbols, + codecs, + types: Vec::new(), + } } pub(crate) fn take_references(&self) -> Vec> { @@ -65,11 +76,14 @@ impl<'tcx> CrateContext<'tcx> { #[derive(Default)] pub(crate) struct Definitions<'tcx> { values: HashMap, + /// Imported storage schemas inform the ABI without requesting class emission. + pub(crate) external_schemas: HashMap, // Owner metadata for imports and generated function-pointer adapters; // foreign interfaces must never acquire generated classfile definitions. pub(crate) foreign_interfaces: Lock>, pub(super) representations: HashMap, oomir::Type>, pub(super) defined_enums: HashSet, + pub(super) building_codecs: HashMap, super::types::PointerMemoryCodec>, shared: Shared<'tcx>, checked_intrinsics: Vec, next_temporary: usize, @@ -77,13 +91,29 @@ pub(crate) struct Definitions<'tcx> { } impl<'tcx> Definitions<'tcx> { + /// Read-only schema queries are independent of physical ownership. + pub(crate) fn get(&self, name: &str) -> Option<&oomir::DataType> { + self.values + .get(name) + .or_else(|| self.external_schemas.get(name)) + } + + pub(crate) fn storage_layout(&self, ty: Ty<'tcx>) -> Option> { + self.shared.storage_layouts.borrow().get(&ty).cloned() + } + + pub(crate) fn remember_storage_layout(&self, ty: Ty<'tcx>, layout: Option) { + self.shared.storage_layouts.borrow_mut().insert(ty, layout); + } pub(crate) fn new(shared: Shared<'tcx>) -> Self { Self { shared, values: HashMap::default(), + external_schemas: HashMap::default(), foreign_interfaces: Lock::default(), representations: HashMap::default(), defined_enums: HashSet::default(), + building_codecs: HashMap::default(), checked_intrinsics: Vec::new(), next_temporary: 0, body: BodyFacts::default(), @@ -126,17 +156,6 @@ impl<'tcx> Definitions<'tcx> { resolved } - pub(crate) fn tuple_name_conflicts(&self, name: &str, fields: &[oomir::Type]) -> bool { - let mut abis = self.shared.tuple_abis.borrow_mut(); - match abis.get(name) { - Some(previous) => previous != fields, - None => { - abis.insert(name.to_owned(), fields.to_vec()); - false - } - } - } - pub(crate) fn allocation_identity(&self, id: AllocId, candidate: String) -> String { self.shared .allocations @@ -147,7 +166,13 @@ impl<'tcx> Definitions<'tcx> { } pub(crate) fn has_upstream_body(&self, key: u64) -> bool { - self.shared.upstream_symbols.contains(&key) + self.shared.upstream.bodies.contains(&key) + } + pub(crate) fn has_upstream_type(&self, name: &str) -> bool { + self.shared.upstream.types.contains(name) + } + pub(crate) fn has_upstream_codec(&self, recipe: &str) -> bool { + self.shared.upstream.codecs.contains(recipe) } pub(crate) fn record_provided_body(&self, key: u64) { self.shared.provided_symbols.borrow_mut().insert(key); @@ -175,6 +200,15 @@ impl<'tcx> Definitions<'tcx> { } pub(super) fn complete_codec(&self, ty: Ty<'tcx>, codec: super::types::PointerMemoryCodec) { + if !self.has_upstream_codec(&codec.class_name) + && codec.class_name.contains('#') + && !codec.class_name.starts_with("org/rustlang/runtime/") + { + self.shared + .provided_codecs + .borrow_mut() + .insert(codec.class_name.clone()); + } self.shared.completed_codecs.borrow_mut().insert(ty, codec); } @@ -245,15 +279,8 @@ impl BodyFacts { } } } - for index in 1..=body.arg_count { - if matches!( - body.local_decls[mir::Local::from_usize(index)].ty.kind(), - rustc_middle::ty::TyKind::Ref(..) | rustc_middle::ty::TyKind::RawPtr(..) - ) { - // Incoming pointers already are stable addresses. - stable_cells[index] = false; - } - } + // A borrow of the parameter needs local storage. A borrow through the parameter uses the + // incoming allocation. Self { stable_cells } } } @@ -287,10 +314,6 @@ mod tests { assert_eq!(first.checked_intrinsics.len(), 1); assert!(second.checked_intrinsics.is_empty()); assert_eq!(separate.checked_intrinsics.len(), 1); - assert!(!first.tuple_name_conflicts("Tuple", &[oomir::Type::I32])); - assert!(!second.tuple_name_conflicts("Tuple", &[oomir::Type::I32])); - assert!(second.tuple_name_conflicts("Tuple", &[oomir::Type::I64])); - assert!(!separate.tuple_name_conflicts("Tuple", &[oomir::Type::I64])); assert_eq!( ( first.next_temporary(), diff --git a/src/lower1/control_flow.rs b/src/lower1/control_flow.rs index 15014d8e..83e7cec9 100644 --- a/src/lower1/control_flow.rs +++ b/src/lower1/control_flow.rs @@ -6,7 +6,7 @@ use super::{ emit_instructions_to_get_on_own, emit_instructions_to_set_value, emit_pointer_read, emit_pointer_slice_parts, emit_retyped_slice_data_pointer, place_to_string, }, - types::{enum_scoped_method_name, mir_int_to_oomir_const}, + types::mir_int_to_oomir_const, }; use crate::lower1::context::Definitions; use crate::oomir; @@ -21,21 +21,21 @@ use rustc_middle::{ Operand as MirOperand, Place, SourceInfo, StatementKind, TerminatorKind, UnwindAction, visit::Visitor, }, - ty::{EarlyBinder, Instance, InstanceKind, ShimKind, Ty, TyCtxt, TyKind, TypingEnv, VtblEntry}, + ty::{EarlyBinder, Instance, InstanceKind, ShimKind, Ty, TyCtxt, TyKind, TypingEnv}, }; use rustc_span::{Symbol, sym}; mod calls; +mod metadata; +use metadata::{emit_pointer_metadata, emit_raw_pointer_from_parts, emit_trait_object_metadata}; pub mod checked_intrinsics; mod checked_ops; mod comparisons; pub(super) use comparisons::*; mod diagnostics; pub(super) use diagnostics::*; -mod borrows; -pub(super) use borrows::*; -mod drop_lowering; -pub(super) use drop_lowering::*; +mod drop_glue; +pub(super) use drop_glue::*; mod atomic; pub(super) use atomic::*; @@ -68,108 +68,8 @@ fn is_core_ptr_free_function( .is_some_and(|parent| tcx.opt_item_name(parent) == Some(sym::ptr)) } -fn emit_trait_object_metadata( - pointer: oomir::Operand, - output_type: &oomir::Type, - dest: String, - temp_prefix: &str, - data_types: &HashMap, - instructions: &mut Vec, -) { - let metadata_class = output_type - .get_class_name() - .expect("trait-object metadata must be represented by a JVM class") - .to_string(); - let metadata_fields = match data_types.get(&metadata_class) { - Some(oomir::DataType::Class { fields, .. }) => fields.clone(), - other => panic!("trait-object metadata class is unavailable: {other:?}"), - }; - let [(_, vtable_type), (_, phantom_type)] = metadata_fields.as_slice() else { - panic!("trait-object metadata must contain vtable and phantom fields") - }; - let vtable_class = vtable_type - .get_class_name() - .expect("trait-object vtable pointer must be represented by a JVM class") - .to_string(); - let vtable_fields = match data_types.get(&vtable_class) { - Some(oomir::DataType::Class { fields, .. }) => fields.clone(), - other => panic!("trait-object vtable pointer class is unavailable: {other:?}"), - }; - let [(_, marker_type)] = vtable_fields.as_slice() else { - panic!("trait-object vtable pointer must contain exactly one pointer field") - }; - let phantom_class = phantom_type - .get_class_name() - .expect("trait-object metadata phantom field must be a JVM class") - .to_string(); - - let marker_name = format!("{temp_prefix}_vtable_marker"); - let pointer_type = pointer - .get_type() - .expect("trait-object metadata source must be typed"); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(marker_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "traitMetadataMarker".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), pointer_type), - ("metadata_class".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(marker_type.clone()), - is_static: true, - }, - args: vec![ - pointer, - oomir::Operand::Constant(oomir::Constant::String(metadata_class.clone())), - ], - }); - - let vtable_name = format!("{temp_prefix}_vtable"); - instructions.push(oomir::Instruction::ConstructObject { - dest: vtable_name.clone(), - class_name: vtable_class, - args: vec![( - oomir::Operand::Variable { - name: marker_name, - ty: marker_type.clone(), - }, - marker_type.clone(), - )], - }); - let phantom_name = format!("{temp_prefix}_phantom"); - instructions.push(oomir::Instruction::ConstructObject { - dest: phantom_name.clone(), - class_name: phantom_class, - args: Vec::new(), - }); - instructions.push(oomir::Instruction::ConstructObject { - dest, - class_name: metadata_class, - args: vec![ - ( - oomir::Operand::Variable { - name: vtable_name, - ty: vtable_type.clone(), - }, - vtable_type.clone(), - ), - ( - oomir::Operand::Variable { - name: phantom_name, - ty: phantom_type.clone(), - }, - phantom_type.clone(), - ), - ], - }); -} - fn requires_compiled_static_dispatch(ty: &oomir::Type) -> bool { - if let oomir::Type::MutableReference(inner) - | oomir::Type::Reference(inner) - | oomir::Type::Pointer(inner) = ty - { + if let oomir::Type::Pointer(oomir::Pointee { value: inner, .. }) = ty { return requires_compiled_static_dispatch(inner); } matches!(ty, oomir::Type::Unit | oomir::Type::Void) @@ -198,15 +98,12 @@ pub(super) fn convert_basic_block<'tcx>( basic_blocks: &mut HashMap, data_types: &mut Definitions<'tcx>, external_interfaces: &mut HashSet, - mutable_borrow_arrays: &mut MutableBorrowMap<'tcx>, debug_variables: &[oomir::DebugVariable], debug_scope_cache: &super::DebugScopeCache, - initially_available_pointer_locals: HashSet, ) -> oomir::BasicBlock { // Use the basic block index as its label. let label = format!("bb{}", bb.index()); let mut instructions = Vec::new(); - let mut initialized_borrows = initially_available_pointer_locals; // Convert each MIR statement in the block. for (statement_index, stmt) in bb_data.statements.iter().enumerate() { let statement_location = Location { @@ -236,138 +133,11 @@ pub(super) fn convert_basic_block<'tcx>( instance, data_types, external_interfaces, - mutable_borrow_arrays, - &initialized_borrows, ); // Add instructions needed to calculate the Rvalue instructions.extend(rvalue_instructions); - if let rustc_middle::mir::Rvalue::Ref(_, _, borrowed_place) = rvalue { - let dest_ty = place.ty(&mir.local_decls, tcx).ty; - let borrowed_is_trait_object = matches!(dest_ty.kind(), TyKind::Ref(..)) - && matches!(source_operand.get_type(), Some(oomir::Type::Interface(_))); - if borrowed_is_trait_object { - // Trait objects do not use the MutableReference array wrapper - } else { - // Check if the destination is a simple local (most common case for &mut assignment) - if place.projection.is_empty() { - if let oomir::Operand::Variable { - name: array_var_name, - ty: array_ty, - } = &source_operand - { - match array_ty { - oomir::Type::Pointer(element_ty) => { - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: Tracking mutable borrow array for place {:?} stored in local {:?}. Original: {:?}, ArrayVar: {}, ElementTy: {:?}", - place, - place.local, - borrowed_place, - array_var_name, - element_ty - ) - ); - mutable_borrow_arrays.insert( - place.local, - PointerOrigin { - original_place: borrowed_place.clone(), - carrier_name: place_to_string(place, tcx), - pointee_type: *element_ty.clone(), - writable: matches!( - rvalue, - rustc_middle::mir::Rvalue::Ref( - _, - rustc_middle::mir::BorrowKind::Mut { .. }, - _ - ) - ), - }, - ); - initialized_borrows.insert(place.local); - } - oomir::Type::Slice(_) | oomir::Type::Str => { - // Slice views already alias their backing array, so writes - // are visible directly and need no copy-out bookkeeping. - // Utf8View has the same canonical aliasing behaviour for - // `str` references. - } - _ => { - breadcrumbs::log!( - breadcrumbs::LogLevel::Warn, - "mir-lowering", - format!( - "Warning: Expected mutable-reference or slice representation, found {:?}", - array_ty - ) - ); - } - } - } else { - breadcrumbs::log!( - breadcrumbs::LogLevel::Warn, - "mir-lowering", - format!( - "Warning: Expected variable operand for mutable borrow ref assignment result, found {:?}", - source_operand - ) - ); - } - } else { - breadcrumbs::log!( - breadcrumbs::LogLevel::Warn, - "mir-lowering", - format!( - "Warning: Mutable borrow assigned to complex place {:?}, write-back might not work correctly.", - place - ) - ); - } - } - } - - if let rustc_middle::mir::Rvalue::RawPtr( - rustc_middle::mir::RawPtrKind::Const | rustc_middle::mir::RawPtrKind::Mut, - pointed_place, - ) = rvalue - && place.projection.is_empty() - && let oomir::Operand::Variable { - name: _, - ty: oomir::Type::Pointer(element_ty), - } = &source_operand - { - let inherited_origin = pointed_place - .projection - .last() - .filter(|projection| { - matches!(projection, rustc_middle::mir::ProjectionElem::Deref) - }) - .and_then(|_| mutable_borrow_arrays.get(&pointed_place.local)) - .map(|origin| origin.original_place.clone()); - mutable_borrow_arrays.insert( - place.local, - PointerOrigin { - original_place: inherited_origin - .unwrap_or_else(|| pointed_place.clone()), - // The assigned local is the stable carrier across - // block boundaries, not the rvalue temporary. - carrier_name: place_to_string(place, tcx), - pointee_type: element_ty.as_ref().clone(), - writable: matches!( - rvalue, - rustc_middle::mir::Rvalue::RawPtr( - rustc_middle::mir::RawPtrKind::Mut, - _ - ) - ), - }, - ); - initialized_borrows.insert(place.local); - } - // 2. Generate instructions to store the computed value into the destination place let assignment_instructions = emit_instructions_to_set_value( place, // The actual destination Place @@ -380,27 +150,6 @@ pub(super) fn convert_basic_block<'tcx>( // Add the final assignment instructions (Move, SetField, ArrayStore) instructions.extend(assignment_instructions); - instructions.extend(emit_pointer_origin_refreshes( - place, - &initialized_borrows, - mutable_borrow_arrays, - tcx, - instance, - mir, - data_types, - )); - if place.projection.iter().any(|projection| { - matches!(projection, rustc_middle::mir::ProjectionElem::Deref) - }) { - instructions.extend(emit_selected_mutable_borrow_writebacks( - [place.local], - mutable_borrow_arrays, - tcx, - instance, - mir, - data_types, - )); - } } StatementKind::StorageLive(_) | StatementKind::StorageDead(_) => { // no-op, currently @@ -441,7 +190,11 @@ pub(super) fn convert_basic_block<'tcx>( data_types, &mut instructions, ); - let source_rust_ty = copy.src.ty(&mir.local_decls, tcx); + // MIR can still contain generic parameters. The byte count requires the + // instantiated type. + let source_rust_ty = EarlyBinder::bind(tcx, copy.src.ty(&mir.local_decls, tcx)) + .instantiate(tcx, instance.args) + .skip_norm_wip(); let pointee = match source_rust_ty.kind() { TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) => *pointee, other => panic!( @@ -530,6 +283,12 @@ pub(super) fn convert_basic_block<'tcx>( instructions.extend(emit_instructions_to_set_value( place, value, tcx, instance, mir, data_types, )); + } else if super::types::tagged_scalar(enum_ty, tcx).is_some() { + assert_eq!(variant_index.as_u32(), 1); + } else if let Some(carrier) = super::types::enum_carrier(enum_ty, tcx) { + if *variant_index != carrier.variant { + instructions.push(oomir::Instruction::Unreachable); + } } else { breadcrumbs::log!( breadcrumbs::LogLevel::Warn, @@ -679,8 +438,6 @@ pub(super) fn convert_basic_block<'tcx>( func, destination, target, - mutable_borrow_arrays, - &mut initialized_borrows, ); } TerminatorKind::Assert { @@ -903,18 +660,27 @@ pub(super) fn convert_basic_block<'tcx>( .instantiate(tcx, instance.args) .skip_norm_wip(); if rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - let (value_name, value_instructions, value_ty) = - emit_instructions_to_get_on_own(place, tcx, instance, mir, data_types); - instructions.extend(value_instructions); - emit_rust_drop_value( - rust_ty, - oomir::Operand::Variable { - name: value_name, - ty: value_ty, - }, + let pointer_ty = crate::lower1::types::ty_to_oomir_type( + Ty::new_ptr(tcx, rust_ty, rustc_middle::ty::Mutability::Mut), + tcx, + data_types, + instance, + ); + let pointer = rvalue::emit_pointer_to_place( + place, + &pointer_ty, &format!("{label}_drop"), tcx, instance, + mir, + data_types, + &mut instructions, + ); + emit_drop_in_place( + rust_ty, + pointer, + tcx, + instance, data_types, &mut instructions, ); @@ -926,10 +692,7 @@ pub(super) fn convert_basic_block<'tcx>( }); } TerminatorKind::Unreachable => { - instructions.push(oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/RuntimeException".to_string(), - message: "Unreachable code reached".to_string(), - }); + instructions.push(oomir::Instruction::Unreachable); } TerminatorKind::UnwindResume => { instructions.push(oomir::Instruction::Rethrow); @@ -952,10 +715,7 @@ pub(super) fn convert_basic_block<'tcx>( ty: oomir::Type::Class("java/lang/Throwable".to_string()), }], }); - instructions.push(oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/AssertionError".to_string(), - message: "Rust abort unexpectedly returned".to_string(), - }); + instructions.push(oomir::Instruction::Unreachable); } // Other terminator kinds will be added as needed. _ => { diff --git a/src/lower1/control_flow/borrows.rs b/src/lower1/control_flow/borrows.rs deleted file mode 100644 index 19cb728c..00000000 --- a/src/lower1/control_flow/borrows.rs +++ /dev/null @@ -1,388 +0,0 @@ -//! Borrows. -use super::*; - -#[derive(Clone, Debug, PartialEq)] -pub(in crate::lower1) struct PointerOrigin<'tcx> { - pub original_place: Place<'tcx>, - pub carrier_name: String, - pub pointee_type: oomir::Type, - pub writable: bool, -} - -pub(in crate::lower1) type MutableBorrowMap<'tcx> = HashMap>; - -pub(in crate::lower1) fn collect_pointer_origins<'tcx>( - mir: &Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - data_types: &mut Definitions<'tcx>, -) -> MutableBorrowMap<'tcx> { - let mut origins = MutableBorrowMap::default(); - let assignment_count = mir - .basic_blocks - .iter() - .map(|block| block.statements.len()) - .sum::(); - - for _ in 0..=assignment_count { - let mut changed = false; - for block in mir.basic_blocks.iter() { - for statement in &block.statements { - let StatementKind::Assign(assignment) = &statement.kind else { - continue; - }; - let (destination, rvalue) = assignment.as_ref(); - if !destination.projection.is_empty() { - continue; - } - let destination_rust_ty = - EarlyBinder::bind(tcx, destination.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); - if !matches!( - destination_rust_ty.kind(), - TyKind::Ref(..) | TyKind::RawPtr(..) - ) { - continue; - } - let destination_ty = crate::lower1::place::get_place_type( - destination, - mir, - tcx, - instance, - data_types, - ); - let oomir::Type::Pointer(pointee_ty) = destination_ty else { - continue; - }; - - let inherit_deref_origin = |place: &Place<'tcx>| { - place - .projection - .last() - .filter(|projection| { - matches!(projection, rustc_middle::mir::ProjectionElem::Deref) - }) - .and_then(|_| origins.get(&place.local)) - .map(|origin| origin.original_place.clone()) - .unwrap_or_else(|| place.clone()) - }; - let origin = match rvalue { - rustc_middle::mir::Rvalue::Ref(_, borrow_kind, place) => Some(( - inherit_deref_origin(place), - pointee_ty.as_ref().clone(), - matches!(borrow_kind, rustc_middle::mir::BorrowKind::Mut { .. }), - )), - rustc_middle::mir::Rvalue::RawPtr(pointer_kind, place) => Some(( - inherit_deref_origin(place), - pointee_ty.as_ref().clone(), - matches!(pointer_kind, rustc_middle::mir::RawPtrKind::Mut), - )), - rustc_middle::mir::Rvalue::Use( - MirOperand::Copy(place) | MirOperand::Move(place), - _, - ) => origins.get(&place.local).map(|entry| { - ( - entry.original_place.clone(), - pointee_ty.as_ref().clone(), - entry.writable, - ) - }), - rustc_middle::mir::Rvalue::Cast( - _, - MirOperand::Copy(place) | MirOperand::Move(place), - _, - ) => origins.get(&place.local).map(|entry| { - ( - entry.original_place.clone(), - pointee_ty.as_ref().clone(), - entry.writable, - ) - }), - _ => None, - }; - let Some((origin, storage_ty, writable)) = origin else { - continue; - }; - let entry = PointerOrigin { - original_place: origin, - carrier_name: crate::lower1::place::place_to_string(destination, tcx), - pointee_type: storage_ty, - writable, - }; - if origins.get(&destination.local) != Some(&entry) { - origins.insert(destination.local, entry); - changed = true; - } - } - } - if !changed { - break; - } - } - origins -} - -pub(in crate::lower1) fn emit_mutable_borrow_writeback<'tcx>( - borrow_local: Local, - mutable_borrows: &MutableBorrowMap<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - mir: &Body<'tcx>, - data_types: &mut Definitions<'tcx>, - visited: &mut HashSet, -) -> Vec { - if !visited.insert(borrow_local) { - return Vec::new(); - } - let mut instructions = Vec::new(); - let Some(origin) = mutable_borrows.get(&borrow_local).cloned() else { - return instructions; - }; - if !origin.writable { - return instructions; - } - let original_place = origin.original_place; - if matches!( - original_place.projection.first(), - Some(rustc_middle::mir::ProjectionElem::Deref) - ) { - // A reborrow of existing pointer storage already writes through. The - // borrowed pointer local may itself be changed through another alias; - // copying it back would then overwrite the old pointee or read a stale - // local instead of its stable cell. - return instructions; - } - let carrier_name = origin.carrier_name; - let carrier_pointee_ty = origin.pointee_type; - if data_types.local_uses_stable_cell(original_place.local) { - // The pointer and all projected field views already share the root - // allocation. Copying a derived/address-only view back would both be - // redundant and could overwrite the field with the enclosing object. - return instructions; - } - if let Some((rustc_middle::mir::ProjectionElem::Field(_, _), base_projection)) = - original_place.projection.split_last() - { - let base_place = Place { - local: original_place.local, - projection: tcx.mk_place_elems(base_projection), - }; - let base_ty = EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); - if matches!(base_ty.kind(), TyKind::Tuple(_)) - || matches!( - base_ty.kind(), - TyKind::Adt(adt_def, _) - if adt_def.is_struct() || adt_def.is_enum() || adt_def.is_union() - ) - { - // Field pointers already write through to their original storage. - // Reassigning a union field here also detaches references returned - // by the call from the decoded view they still borrow. - return instructions; - } - } - let storage_rust_ty = EarlyBinder::bind(tcx, original_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); - let storage_ty = - crate::lower1::types::ty_to_oomir_type(storage_rust_ty, tcx, data_types, instance); - if !storage_ty.has_jvm_value() { - return instructions; - } - let value_name = format!( - "_writeback_{}_{}", - borrow_local.index(), - original_place.local.index() - ); - let carrier_ty = mutable_borrows - .get(&borrow_local) - .map(|origin| oomir::Type::Pointer(Box::new(origin.pointee_type.clone()))) - .unwrap_or_else(|| oomir::Type::Pointer(Box::new(carrier_pointee_ty.clone()))); - let mut carrier = oomir::Operand::Variable { - name: carrier_name, - ty: carrier_ty, - }; - if carrier_pointee_ty != storage_ty { - let storage_pointer_ty = oomir::Type::Pointer(Box::new(storage_ty.clone())); - let retyped_name = format!("{value_name}_storage_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(retyped_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), carrier.get_type().unwrap()), - ("size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(storage_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from( - crate::lower1::types::layout_size_bytes(tcx, storage_rust_ty) - .unwrap_or_else(|error| { - panic!("could not determine pointer write-back layout: {error}") - }), - ) - .expect("Rust pointer write-back layout exceeds u64"), - )), - crate::lower1::types::pointer_view_codec_operand( - storage_rust_ty, - tcx, - data_types, - instance, - ), - ], - operand: carrier, - }); - carrier = oomir::Operand::Variable { - name: retyped_name, - ty: storage_pointer_ty, - }; - } - emit_pointer_read(carrier, &storage_ty, &value_name, &mut instructions); - instructions.extend(emit_instructions_to_set_value( - &original_place, - oomir::Operand::Variable { - name: value_name, - ty: storage_ty, - }, - tcx, - instance, - mir, - data_types, - )); - if mutable_borrows.contains_key(&original_place.local) { - instructions.extend(emit_mutable_borrow_writeback( - original_place.local, - mutable_borrows, - tcx, - instance, - mir, - data_types, - visited, - )); - } - instructions -} - -pub(in crate::lower1) fn emit_pointer_origin_refreshes<'tcx>( - updated_place: &Place<'tcx>, - available_pointer_locals: &HashSet, - pointer_origins: &MutableBorrowMap<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - mir: &Body<'tcx>, - data_types: &mut Definitions<'tcx>, -) -> Vec { - if matches!( - updated_place.projection.first(), - Some(rustc_middle::mir::ProjectionElem::Deref) - ) { - // A reborrow retains the original pointer's storage. A store through - // that pointer is already visible to its aliases; reading the value - // back and storing it again only creates a redundant decoded view. - return Vec::new(); - } - let matching_origins = available_pointer_locals - .iter() - .filter_map(|local| pointer_origins.get(local)) - .filter(|origin| origin.original_place == *updated_place) - .filter(|origin| !data_types.local_uses_stable_cell(origin.original_place.local)) - .cloned() - .collect::>(); - if matching_origins.is_empty() { - return Vec::new(); - } - - let (value_name, mut instructions, value_type) = - emit_instructions_to_get_on_own(updated_place, tcx, instance, mir, data_types); - let updated_rust_ty = EarlyBinder::bind(tcx, updated_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); - for origin in matching_origins { - let mut carrier = oomir::Operand::Variable { - name: origin.carrier_name, - ty: oomir::Type::Pointer(Box::new(origin.pointee_type.clone())), - }; - if origin.pointee_type != value_type { - let storage_pointer_ty = oomir::Type::Pointer(Box::new(value_type.clone())); - let retyped_name = format!("{value_name}_refresh_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(retyped_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), carrier.get_type().unwrap()), - ("size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(storage_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from( - crate::lower1::types::layout_size_bytes(tcx, updated_rust_ty) - .unwrap_or_else(|error| { - panic!("could not determine pointer refresh layout: {error}") - }), - ) - .expect("Rust pointer refresh layout exceeds u64"), - )), - crate::lower1::types::pointer_view_codec_operand( - updated_rust_ty, - tcx, - data_types, - instance, - ), - ], - operand: carrier, - }); - carrier = oomir::Operand::Variable { - name: retyped_name, - ty: storage_pointer_ty, - }; - } - crate::lower1::place::emit_pointer_write( - carrier, - &value_type, - oomir::Operand::Variable { - name: value_name.clone(), - ty: value_type.clone(), - }, - &mut instructions, - ); - } - instructions -} - -pub(in crate::lower1) fn emit_selected_mutable_borrow_writebacks<'tcx>( - borrow_locals: impl IntoIterator, - mutable_borrows: &MutableBorrowMap<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - mir: &Body<'tcx>, - data_types: &mut Definitions<'tcx>, -) -> Vec { - let mut instructions = Vec::new(); - let mut visited = HashSet::default(); - for borrow_local in borrow_locals { - instructions.extend(emit_mutable_borrow_writeback( - borrow_local, - mutable_borrows, - tcx, - instance, - mir, - data_types, - &mut visited, - )); - } - instructions -} diff --git a/src/lower1/control_flow/calls.rs b/src/lower1/control_flow/calls.rs index 08994e8e..46f41be3 100644 --- a/src/lower1/control_flow/calls.rs +++ b/src/lower1/control_flow/calls.rs @@ -14,8 +14,6 @@ pub(super) fn emit<'tcx>( func: &MirOperand<'tcx>, destination: &Place<'tcx>, target: &Option, - mutable_borrow_arrays: &mut MutableBorrowMap<'tcx>, - initialized_borrows: &mut HashSet, ) { let mut pre_call_instructions = Vec::new(); let mut oomir_operands: Vec = args @@ -47,10 +45,6 @@ pub(super) fn emit<'tcx>( let destination_oomir_type = crate::lower1::place::get_place_type(destination, mir, tcx, instance, data_types); - let destination_pointer_pointee = match &destination_oomir_type { - oomir::Type::Pointer(pointee) => Some(pointee.as_ref().clone()), - _ => None, - }; let deferred_destination_store = !destination.projection.is_empty() || data_types.local_uses_stable_cell(destination.local); let dest_var_name = destination_oomir_type.has_jvm_value().then(|| { @@ -125,35 +119,22 @@ pub(super) fn emit<'tcx>( } if fn_inputs.len() > oomir_operands.len() && let Some(tuple_operand) = oomir_operands.last().cloned() - && let Some(tuple_class) = tuple_operand - .get_type() - .and_then(|ty| ty.get_class_name().map(str::to_string)) + && let Some(arg) = args.last() { - let prefix_len = oomir_operands.len() - 1; - let tuple_fields = match data_types.get(&tuple_class) { - Some(oomir::DataType::Class { fields, .. }) - if fields.len() == fn_inputs.len() - prefix_len => - { - Some(fields.clone()) - } - _ => None, - }; - if let Some(tuple_fields) = tuple_fields { + let tuple_ty = data_types.normalize(tcx, arg.node.ty(mir, tcx), instance); + let prefix = oomir_operands.len() - 1; + if matches!(tuple_ty.kind(), TyKind::Tuple(fields) if fields.len() == fn_inputs.len() - prefix) + { oomir_operands.pop(); - for (field_index, (field_name, field_ty)) in tuple_fields.into_iter().enumerate() { - let dest = format!("{label}_call_tuple_arg_{field_index}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: tuple_operand.clone(), - field_name, - field_ty: field_ty.clone(), - owner_class: tuple_class.clone(), - }); - oomir_operands.push(oomir::Operand::Variable { - name: dest, - ty: field_ty, - }); - } + oomir_operands.extend(crate::lower1::types::tuple_fields( + tuple_ty, + tuple_operand, + &format!("{label}_call_tuple_arg"), + tcx, + data_types, + instance, + instructions, + )); } } @@ -219,35 +200,10 @@ pub(super) fn emit<'tcx>( jvm_import, ); } else if let InstanceKind::Shim(ShimKind::DropGlue(_, drop_ty)) = func_instance.def { - if let Some(drop_ty) = drop_ty - && drop_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) - { - let drop_oomir_ty = - crate::lower1::types::ty_to_oomir_type(drop_ty, tcx, data_types, instance); - let pointer = oomir_operands[0].clone(); - let value = if matches!(drop_ty.kind(), TyKind::Slice(_)) { - match pointer { - oomir::Operand::Variable { name, .. } => oomir::Operand::Variable { - name, - ty: drop_oomir_ty.clone(), - }, - other => other, - } - } else if matches!(pointer.get_type(), Some(oomir::Type::Pointer(_))) { - let value_name = format!("{label}_drop_glue_value"); - crate::lower1::place::emit_pointer_read( - pointer, - &drop_oomir_ty, - &value_name, - &mut instructions, - ) - } else { - pointer - }; - emit_rust_drop_value( + if let Some(drop_ty) = drop_ty { + emit_drop_in_place( drop_ty, - value, - &format!("{label}_drop_glue"), + oomir_operands[0].clone(), tcx, instance, data_types, @@ -258,7 +214,52 @@ pub(super) fn emit<'tcx>( let TyKind::Adt(adt_def, _) = fn_output.kind() else { panic!("constructor returned non-ADT type {fn_output:?}") }; - if let Some(dest) = effective_dest { + if let Some(payload) = crate::lower1::types::transparent_payload(fn_output, tcx) { + if let Some(dest) = effective_dest { + instructions.push(oomir::Instruction::Move { + dest, + src: oomir_operands[payload.field].clone(), + }); + } + } else if let Some(word) = crate::lower1::types::packed_word(fn_output, tcx) { + if let Some(dest) = effective_dest { + let mut value = word.zero(); + for (index, field) in oomir_operands.into_iter().enumerate() { + value = word.insert( + value, + index, + field, + &format!("{dest}_field_{index}"), + &mut instructions, + ); + } + instructions.push(oomir::Instruction::Move { dest, src: value }); + } + } else if matches!(oomir_output_type, oomir::Type::TaggedI64) { + if let Some(dest) = effective_dest { + instructions.push(crate::lower1::types::tagged_value( + dest, + oomir_operands + .into_iter() + .next() + .expect("Some has one payload"), + 1, + )); + } + } else if let Some(carrier) = crate::lower1::types::enum_carrier(fn_output, tcx) { + let variant = tcx.parent(func_instance.def_id()); + if adt_def.variant(carrier.variant).def_id != variant { + instructions.push(oomir::Instruction::Unreachable); + } else if let Some(dest) = effective_dest { + instructions.push(oomir::Instruction::Move { + dest, + src: oomir_operands + .into_iter() + .next() + .expect("direct enum has one field"), + }); + } + } else if let Some(dest) = effective_dest { let base_class = oomir_output_type .get_class_name() .expect("constructor result has a JVM class"); @@ -267,7 +268,10 @@ pub(super) fn emit<'tcx>( format!( "{}${}", base_class, - jvm_names::member_name(tcx.item_name(variant_def_id).as_str()) + crate::lower1::types::enum_variant_name( + adt_def.variant(adt_def.variant_index_with_id(variant_def_id)), + tcx, + ) ) } else { base_class.to_string() @@ -375,75 +379,13 @@ pub(super) fn emit<'tcx>( )); } - let destination_ty = destination.ty(&mir.local_decls, tcx).ty; - if destination.projection.is_empty() - && matches!( - destination_ty.kind(), - TyKind::Ref(_, _, mutability) if mutability.is_mut() - ) - { - let aliases = args - .iter() - .filter_map(|arg| match &arg.node { - MirOperand::Move(place) | MirOperand::Copy(place) - if place.projection.is_empty() => - { - mutable_borrow_arrays.get(&place.local).cloned() - } - _ => None, - }) - .collect::>(); - // A returned `&mut U` is only a plain reborrow of an input - // `&mut T` when U and T are the same representation. Methods - // such as `ManuallyDrop::deref_mut` return a pointer derived - // from a field of `T`; that Pointer already carries the field - // address and must not be copied back as though it were T. - if let [alias] = aliases.as_slice() - && destination_pointer_pointee.as_ref() == Some(&alias.pointee_type) - { - mutable_borrow_arrays.insert( - destination.local, - PointerOrigin { - original_place: alias.original_place.clone(), - carrier_name: crate::lower1::place::place_to_string(destination, tcx), - pointee_type: alias.pointee_type.clone(), - writable: alias.writable, - }, - ); - initialized_borrows.insert(destination.local); - } - } - - let mut writeback_locals = HashSet::default(); - for arg in args { - if let MirOperand::Move(place) | MirOperand::Copy(place) = &arg.node - && place.projection.is_empty() - && mutable_borrow_arrays.contains_key(&place.local) - { - writeback_locals.insert(place.local); - } - } - let mut writeback_locals = writeback_locals.into_iter().collect::>(); - writeback_locals.sort_by_key(|local| local.index()); - instructions.extend(emit_selected_mutable_borrow_writebacks( - writeback_locals, - mutable_borrow_arrays, - tcx, - instance, - mir, - data_types, - )); - if let Some(target_bb) = target { let target_label = format!("bb{}", target_bb.index()); instructions.push(oomir::Instruction::Jump { target: target_label, }); } else { - instructions.push(oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/AssertionError".to_string(), - message: "Diverging Rust call returned unexpectedly".to_string(), - }); + instructions.push(oomir::Instruction::Unreachable); } } @@ -451,7 +393,6 @@ mod callables; mod imports; mod intrinsics; mod memory_intrinsics; -mod method_dispatch; mod methods; mod numeric_intrinsics; mod numeric_methods; diff --git a/src/lower1/control_flow/calls/callables.rs b/src/lower1/control_flow/calls/callables.rs index 8456688c..b2ac9c69 100644 --- a/src/lower1/control_flow/calls/callables.rs +++ b/src/lower1/control_flow/calls/callables.rs @@ -15,62 +15,32 @@ pub(super) fn function_item<'tcx>( let tuple_operand = explicit_method_args[0].clone(); let function_ty = function_item.ty(tcx, typing_env); let function_sig = tcx.instantiate_bound_regions_with_erased(function_ty.fn_sig(tcx)); - let value_input_count = function_sig - .inputs() - .iter() - .filter(|ty| { - crate::lower1::types::ty_to_oomir_type(**ty, tcx, data_types, function_item) - .has_jvm_value() + let tuple_ty = Ty::new_tup(tcx, function_sig.inputs()); + let fields = crate::lower1::types::tuple_fields( + tuple_ty, + tuple_operand, + &format!("{label}_fn_def_arg"), + tcx, + data_types, + function_item, + instructions, + ); + let function_args = fields + .into_iter() + .zip(function_sig.inputs()) + .enumerate() + .map(|(index, (field, input_ty))| { + crate::lower1::value_repr::adapt_operand_to_rust_type( + field, + *input_ty, + &format!("{label}_fn_def_arg_{index}_adapted"), + tcx, + instance, + data_types, + instructions, + ) }) - .count(); - let (tuple_class, tuple_fields) = if value_input_count == 0 { - (None, Vec::new()) - } else { - let tuple_class = tuple_operand - .get_type() - .and_then(|ty| ty.get_class_name().map(str::to_string)) - .expect("non-empty Fn argument tuple has a JVM class"); - let fields = match data_types.get(&tuple_class) { - Some(oomir::DataType::Class { fields, .. }) => fields.clone(), - _ => panic!("Fn argument tuple class {tuple_class} was not defined"), - }; - (Some(tuple_class), fields) - }; - let mut function_args = Vec::new(); - let mut tuple_fields = tuple_fields.into_iter(); - for (index, input_ty) in function_sig.inputs().iter().enumerate() { - let input_oomir_ty = - crate::lower1::types::ty_to_oomir_type(*input_ty, tcx, data_types, function_item); - if !input_oomir_ty.has_jvm_value() { - function_args.push(oomir::Operand::Constant(oomir::Constant::Unit)); - continue; - } - let (field_name, field_ty) = tuple_fields - .next() - .expect("Fn argument tuple has one field per JVM argument"); - let field_dest = format!("{label}_fn_def_arg_{index}"); - instructions.push(oomir::Instruction::GetField { - dest: field_dest.clone(), - object: tuple_operand.clone(), - field_name, - field_ty: field_ty.clone(), - owner_class: tuple_class - .clone() - .expect("value-bearing Fn tuple has a class"), - }); - function_args.push(crate::lower1::value_repr::adapt_operand_to_rust_type( - oomir::Operand::Variable { - name: field_dest, - ty: field_ty, - }, - *input_ty, - &format!("{label}_fn_def_arg_{index}_adapted"), - tcx, - instance, - data_types, - &mut instructions, - )); - } + .collect(); let target = data_types.function_name(tcx, function_item); instructions.push(oomir::Instruction::InvokeRustStatic { class_name: target @@ -164,35 +134,16 @@ pub(super) fn ordinary_call<'tcx>( } let flattened_closure_args = if is_closure_call && oomir_operands.len() == 2 && method_signature.params.len() > 1 { - oomir_operands[1] - .get_type() - .and_then(|ty| ty.get_class_name().map(str::to_string)) - .and_then(|tuple_class| { - let fields = match data_types.get(&tuple_class) { - Some(oomir::DataType::Class { fields, .. }) - if fields.len() == method_signature.params.len() => - { - fields.clone() - } - _ => return None, - }; - let mut flattened = Vec::new(); - for (field_index, (field_name, field_ty)) in fields.into_iter().enumerate() { - let dest = format!("{label}_closure_arg_{field_index}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: oomir_operands[1].clone(), - field_name, - field_ty: field_ty.clone(), - owner_class: tuple_class.clone(), - }); - flattened.push(oomir::Operand::Variable { - name: dest, - ty: field_ty, - }); - } - Some(flattened) - }) + let tuple_ty = Ty::new_tup(tcx, &fn_inputs); + Some(crate::lower1::types::tuple_fields( + tuple_ty, + oomir_operands[1].clone(), + &format!("{label}_closure_arg"), + tcx, + data_types, + func_instance, + instructions, + )) } else { None }; diff --git a/src/lower1/control_flow/calls/imports.rs b/src/lower1/control_flow/calls/imports.rs index 0aa489d9..ad8c49b0 100644 --- a/src/lower1/control_flow/calls/imports.rs +++ b/src/lower1/control_flow/calls/imports.rs @@ -33,6 +33,31 @@ pub(super) fn emit<'tcx>( ), ); } + let heap = if !jvm_import.interface && jvm_import.class_name == oomir::POINTER_CLASS { + match (jvm_import.method_name.as_str(), rust_descriptor.as_str()) { + ("allocateBytes", "(JJ)Lorg/rustlang/runtime/Pointer;") => { + Some(oomir::HeapOp::Allocate) + } + ( + "reallocateBytes", + "(Lorg/rustlang/runtime/Pointer;JJJ)Lorg/rustlang/runtime/Pointer;", + ) => Some(oomir::HeapOp::Reallocate), + ("deallocateBytes", "(Lorg/rustlang/runtime/Pointer;)V") => { + Some(oomir::HeapOp::Deallocate) + } + _ => None, + } + } else { + None + }; + if let Some(operation) = heap { + instructions.push(oomir::Instruction::Heap { + operation, + args: oomir_operands, + dest: effective_dest, + }); + return; + } instructions.push(oomir::Instruction::InvokeStatic { class_name: jvm_import.class_name, method_name: jvm_import.method_name, diff --git a/src/lower1/control_flow/calls/intrinsics.rs b/src/lower1/control_flow/calls/intrinsics.rs index 595275cf..6a64047d 100644 --- a/src/lower1/control_flow/calls/intrinsics.rs +++ b/src/lower1/control_flow/calls/intrinsics.rs @@ -484,9 +484,12 @@ pub(super) fn emit<'tcx>( && let Some(dest) = effective_dest.clone() { numeric_intrinsics::carrying_mul_add( + tcx, + data_types, + instance, + fn_output, &label, &mut instructions, - oomir_output_type, oomir_operands, dest, ); @@ -744,42 +747,19 @@ pub(super) fn emit<'tcx>( oomir_operands, effective_dest, ); - } else if is_ptr_metadata - && is_core_ptr - && (matches!( - oomir_operands[0].get_type(), - Some(oomir::Type::Slice(_) | oomir::Type::Str) - ) || fn_inputs.first().is_some_and(|input| { - matches!( - input.kind(), - TyKind::RawPtr(pointee, _) - if pointee.is_slice() || pointee.is_str() - ) - })) - { - if let Some(dest) = effective_dest.clone() { - instructions.push(oomir::Instruction::GetField { + } else if is_ptr_metadata { + if let Some(dest) = effective_dest { + emit_pointer_metadata( + tcx, + instance, + data_types, + fn_inputs[0], + oomir_operands[0].clone(), + oomir_output_type, dest, - object: oomir_operands[0].clone(), - field_name: "rustLength".to_string(), - field_ty: oomir::Type::U64, - owner_class: oomir::SLICE_VIEW_CLASS.to_string(), - }); + &mut instructions, + ); } - } else if is_ptr_metadata - && is_core_ptr - && matches!(oomir_operands[0].get_type(), Some(oomir::Type::Pointer(_))) - { - pointer_intrinsics::metadata( - tcx, - data_types, - &label, - &mut instructions, - fn_inputs, - oomir_output_type, - oomir_operands, - effective_dest, - ); } else if matches!( intrinsic_name.as_str(), "from_raw_parts" | "from_raw_parts_mut" diff --git a/src/lower1/control_flow/calls/memory_intrinsics.rs b/src/lower1/control_flow/calls/memory_intrinsics.rs index 4871275f..3118f15b 100644 --- a/src/lower1/control_flow/calls/memory_intrinsics.rs +++ b/src/lower1/control_flow/calls/memory_intrinsics.rs @@ -13,19 +13,14 @@ pub(super) fn arith_offset<'tcx>( if !matches!(pointer_ty, oomir::Type::Pointer(_)) { panic!("arith_offset requires a pointer operand, found {pointer_ty:?}"); } - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressOffset { dest: effective_dest.clone(), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "offset".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), pointer_ty.clone()), - ("count".to_string(), oomir::Type::I64), - ], - ret: Box::new(oomir_output_type.clone()), - is_static: true, - }, - args: vec![oomir_operands[0].clone(), oomir_operands[1].clone()], + source: oomir_operands[0].clone(), + count: oomir_operands[1].clone(), + ty: oomir_output_type.clone(), + bytes: false, + wrapping: false, + subtract: false, }); } pub(super) fn raw_eq<'tcx>( diff --git a/src/lower1/control_flow/calls/method_dispatch.rs b/src/lower1/control_flow/calls/method_dispatch.rs deleted file mode 100644 index 6d7aacc3..00000000 --- a/src/lower1/control_flow/calls/method_dispatch.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Method dispatch. -use super::*; - -pub(super) fn interface<'tcx>( - tcx: TyCtxt<'tcx>, - data_types: &Definitions<'tcx>, - instructions: &mut Vec, - func_instance: Instance<'tcx>, - oomir_operands: Vec, - effective_dest: Option, - method_signature: oomir::Signature, - receiver_operand: oomir::Operand, - method_args: Vec, - interface_method_name: String, - dispatch_receiver_ty: oomir::Type, - receiver_self_requires_static_dispatch: bool, - uses_concrete_trait_default: bool, - interface_name: String, -) { - let concrete_provided_trait_method = - uses_concrete_trait_default && !matches!(dispatch_receiver_ty, oomir::Type::Interface(_)); - if concrete_provided_trait_method - || receiver_self_requires_static_dispatch - || requires_compiled_static_dispatch(&dispatch_receiver_ty) - { - let target = data_types.function_name(tcx, func_instance); - let mut static_signature = method_signature; - static_signature.is_static = true; - instructions.push(oomir::Instruction::InvokeStatic { - class_name: target - .class_to_call_on - .expect("monomorphized functions have JVM owners"), - method_name: target.method_name, - method_ty: static_signature, - args: oomir_operands.clone(), - dest: effective_dest, - }); - } else { - // The method is from an interface - use InvokeInterface - instructions.push(oomir::Instruction::InvokeInterface { - class_name: interface_name, - method_name: interface_method_name, - method_ty: method_signature, - args: method_args, - dest: effective_dest, - operand: receiver_operand, - }); - } -} diff --git a/src/lower1/control_flow/calls/methods.rs b/src/lower1/control_flow/calls/methods.rs index 2cd0b023..565dc22a 100644 --- a/src/lower1/control_flow/calls/methods.rs +++ b/src/lower1/control_flow/calls/methods.rs @@ -48,11 +48,6 @@ pub(super) fn emit<'tcx>( declared_method_name.clone(), &method_signature.to_string(), ); - let is_provided_trait_method = item.trait_container(tcx).is_some_and(|trait_def_id| { - tcx.provided_trait_methods(trait_def_id).any(|provided| { - provided.def_id == item.def_id || item.trait_item_def_id() == Some(provided.def_id) - }) - }); let method_requires_sized_self = tcx.generics_require_sized_self(item.def_id); let transparent_virtual_receiver = if matches!(func_instance.def, InstanceKind::Virtual(..)) && !method_requires_sized_self @@ -111,34 +106,18 @@ pub(super) fn emit<'tcx>( .first() .cloned() .expect("Fn trait calls carry an argument tuple"); - let tuple_oomir_ty = tuple_operand - .get_type() - .expect("Fn trait argument tuple is typed"); - let tuple_class = tuple_oomir_ty - .get_class_name() - .expect("non-unit Fn argument tuple is a JVM class") - .to_string(); - let fields = match data_types.get(&tuple_class) { - Some(oomir::DataType::Class { fields, .. }) => fields.clone(), - _ => panic!("Fn argument tuple class {tuple_class} was not defined"), - }; - for (field_index, (field_name, field_ty)) in fields.into_iter().enumerate() { - if !field_ty.has_jvm_value() { - continue; - } - let field_dest = format!("{label}_callable_arg_{field_index}"); - instructions.push(oomir::Instruction::GetField { - dest: field_dest.clone(), - object: tuple_operand.clone(), - field_name, - field_ty: field_ty.clone(), - owner_class: tuple_class.clone(), - }); - flattened_args.push(oomir::Operand::Variable { - name: field_dest, - ty: field_ty, - }); - } + flattened_args = crate::lower1::types::tuple_fields( + callable_abi.tuple_ty, + tuple_operand, + &format!("{label}_callable_arg"), + tcx, + data_types, + instance, + instructions, + ) + .into_iter() + .filter(|value| value.get_type().is_some_and(|ty| ty.has_jvm_value())) + .collect(); } instructions.push(oomir::Instruction::InvokeInterface { class_name: callable_abi.interface_name, @@ -179,125 +158,21 @@ pub(super) fn emit<'tcx>( operand: receiver_operand, }); } else { - // Check if this method is declared in a trait (interface) - let trait_container = item.trait_container(tcx); - - // Check if the receiver operand is an interface type (after any casts) - let receiver_oomir_ty = receiver_operand.get_type(); - let has_concrete_receiver_method = match &receiver_oomir_ty { - Some(oomir::Type::Class(class_name)) => matches!( - data_types.get(class_name), - Some(oomir::DataType::Class { methods, .. }) - if methods.contains_key(&method_name) - ), - _ => false, + let use_interface = if matches!(func_instance.def, InstanceKind::Virtual(..)) { + match receiver_operand.get_type() { + Some(oomir::Type::Interface(name)) => Some(name), + _ => item + .trait_container(tcx) + .map(|trait_id| data_types.class_name(tcx, trait_id)), + } + } else { + None }; - let uses_concrete_trait_default = - is_provided_trait_method && !has_concrete_receiver_method; - - // Use InvokeInterface if: - // 1. The receiver type is explicitly an Interface type, OR - // 2. The method is declared in a trait (which maps to an interface) - let use_interface = - if let Some(oomir::Type::Interface(interface_name)) = &receiver_oomir_ty { - Some(interface_name.clone()) - } else if let Some(trait_def_id) = trait_container - && item.impl_container(tcx).is_none() - && !has_concrete_receiver_method - { - // Get the trait name and convert to interface name - let interface_name = data_types.class_name(tcx, trait_def_id); - matches!( - data_types.get(&interface_name), - Some(oomir::DataType::Interface { methods, .. }) - if methods.contains_key(&interface_method_name) - ) - .then_some(interface_name) - } else { - None - }; let dispatch_receiver_ty = crate::lower1::types::ty_to_oomir_type(receiver_mir_ty, tcx, data_types, instance); let resolved_receiver_mir_ty = EarlyBinder::bind(tcx, receiver_mir_ty) .instantiate(tcx, instance.args) .skip_norm_wip(); - let receiver_self_mir_ty = match resolved_receiver_mir_ty.kind() { - TyKind::Ref(_, pointee, _) => *pointee, - _ => resolved_receiver_mir_ty, - }; - let impl_container_mir_ty = item.impl_container(tcx).map(|impl_def_id| { - tcx.type_of(impl_def_id) - .instantiate(tcx, func_instance.args) - .skip_norm_wip() - }); - let inherent_container_mir_ty = item - .impl_container(tcx) - .filter(|impl_def_id| tcx.impl_opt_trait_ref(*impl_def_id).is_none()) - .and(impl_container_mir_ty); - let trait_impl_self_requires_static_dispatch = item - .impl_container(tcx) - .is_some_and(|impl_def_id| tcx.impl_opt_trait_ref(impl_def_id).is_some()) - && !has_concrete_receiver_method; - let has_arbitrary_self_receiver = inherent_container_mir_ty - .is_some_and(|container_ty| container_ty != receiver_self_mir_ty); - let mut receiver_value_mir_ty = receiver_self_mir_ty; - while let TyKind::Ref(_, pointee, _) = receiver_value_mir_ty.kind() { - receiver_value_mir_ty = *pointee; - } - let trait_object_method_requires_static_dispatch = if let TyKind::Dynamic(predicates, _) = - receiver_value_mir_ty.kind() - && let Some(principal) = predicates.principal() - && item.trait_container(tcx).is_some() - { - let trait_ref = tcx.instantiate_bound_regions_with_erased( - principal.with_self_ty(tcx, receiver_value_mir_ty), - ); - !tcx.vtable_entries(trait_ref).iter().any(|entry| { - let VtblEntry::Method(method) = entry else { - return false; - }; - method.def_id() == item.def_id - || tcx - .opt_associated_item(method.def_id()) - .and_then(|method_item| method_item.trait_item_def_id()) - == Some(item.def_id) - }) - } else { - false - }; - let has_enum_reference_receiver = matches!( - resolved_receiver_mir_ty.kind(), - TyKind::Ref(_, pointee, _) - if matches!(pointee.kind(), TyKind::Adt(adt_def, _) - if adt_def.is_enum()) - ); - let method_has_own_generic_params = !tcx - .generics_of(func_instance.def_id()) - .own_params - .is_empty(); - // The class behind a reference implements - // methods for its pointee, not blanket Rust - // impls whose `Self` is the reference itself - // when that pointee has no generated instance - // methods (notably closures and function items). - let receiver_self_requires_static_dispatch = inherent_container_mir_ty.is_some() - || has_enum_reference_receiver - || has_arbitrary_self_receiver - || trait_impl_self_requires_static_dispatch - || trait_object_method_requires_static_dispatch - || method_requires_sized_self - || matches!( - receiver_value_mir_ty.kind(), - TyKind::Closure(..) | TyKind::FnDef(..) | TyKind::FnPtr(..) - ) - || matches!( - receiver_self_mir_ty.kind(), - TyKind::Ref(..) if method_has_own_generic_params - ) - || matches!( - receiver_self_mir_ty.kind(), - TyKind::RawPtr(..) | TyKind::FnPtr(..) - ); let pointer_api_receiver = { let receiver_value_ty = match resolved_receiver_mir_ty.kind() { TyKind::Ref(_, pointee, _) => *pointee, @@ -369,19 +244,30 @@ pub(super) fn emit<'tcx>( ); let direct_non_null_fat_equality = non_null_carrier .is_some_and(|ty| matches!(ty, oomir::Type::Slice(_) | oomir::Type::Str)); - let direct_equality = matches!(declared_method_name.as_str(), "eq" | "ne") + // Only primitive Rust receivers use numeric shortcuts. Shared storage does not change + // Rust method selection. + let mut comparison_rust_ty = resolved_receiver_mir_ty; + while let TyKind::Ref(_, inner, _) = comparison_rust_ty.kind() { + comparison_rust_ty = *inner; + } + let nominal_value_receiver = + crate::lower1::types::direct_enum_payload(comparison_rust_ty, tcx).is_some() + || crate::lower1::types::transparent_payload(comparison_rust_ty, tcx).is_some() + || crate::lower1::types::value_scalar_ty(comparison_rust_ty, tcx).is_some(); + let direct_equality = !nominal_value_receiver + && matches!(declared_method_name.as_str(), "eq" | "ne") && (supports_direct_equality(&comparison_value_ty) || non_null_carrier.is_some_and(|ty| { supports_direct_equality(ty) || direct_non_null_fat_equality })) && comparison_rhs_ty.as_ref() == Some(&comparison_value_ty) && oomir_operands.len() == 2; - let direct_ordering = - matches!(declared_method_name.as_str(), "lt" | "le" | "gt" | "ge") - && (supports_direct_ordering(&comparison_value_ty) - || non_null_carrier.is_some_and(supports_direct_ordering)) - && comparison_rhs_ty.as_ref() == Some(&comparison_value_ty) - && oomir_operands.len() == 2; + let direct_ordering = !nominal_value_receiver + && matches!(declared_method_name.as_str(), "lt" | "le" | "gt" | "ge") + && (supports_direct_ordering(&comparison_value_ty) + || non_null_carrier.is_some_and(supports_direct_ordering)) + && comparison_rhs_ty.as_ref() == Some(&comparison_value_ty) + && oomir_operands.len() == 2; let direct_unit_comparison = comparison_value_ty == oomir::Type::Unit && comparison_rhs_ty.as_ref() == Some(&oomir::Type::Unit) && oomir_operands.len() == 2 @@ -434,7 +320,17 @@ pub(super) fn emit<'tcx>( && declared_method_name == "to_bits" && oomir_operands.len() == 1; - if direct_f16_to_bits { + if nominal_value_receiver && use_interface.is_none() { + let target = data_types.function_name(tcx, func_instance); + method_signature.is_static = true; + instructions.push(oomir::Instruction::InvokeRustStatic { + class_name: target.class_to_call_on.expect("nominal method owner"), + method_name: target.method_name, + method_ty: method_signature, + args: oomir_operands, + dest: effective_dest, + }); + } else if direct_f16_to_bits { if let Some(dest) = effective_dest { instructions.push(oomir::Instruction::InvokeStatic { dest: Some(dest), @@ -588,9 +484,18 @@ pub(super) fn emit<'tcx>( ) { if let Some(dest) = effective_dest { + let reference = crate::lower1::value_repr::adapt_operand_to_rust_type( + receiver_operand, + fn_output, + label, + tcx, + instance, + data_types, + &mut instructions, + ); instructions.push(oomir::Instruction::Move { dest, - src: receiver_operand, + src: reference, }); } } else if matches!(&dispatch_receiver_ty, oomir::Type::Pointer(_)) @@ -611,6 +516,7 @@ pub(super) fn emit<'tcx>( && declared_method_name == "to_raw_parts" { pointer_access::to_raw_parts( + tcx, data_types, &label, &mut instructions, @@ -749,22 +655,14 @@ pub(super) fn emit<'tcx>( dispatch_receiver_ty, ); } else if let Some(interface_name) = use_interface { - method_dispatch::interface( - tcx, - data_types, - &mut instructions, - func_instance, - oomir_operands, - effective_dest, - method_signature, - receiver_operand, - method_args, - interface_method_name, - dispatch_receiver_ty, - receiver_self_requires_static_dispatch, - uses_concrete_trait_default, - interface_name, - ); + instructions.push(oomir::Instruction::InvokeInterface { + class_name: interface_name, + method_name: interface_method_name, + method_ty: method_signature, + args: method_args, + dest: effective_dest, + operand: receiver_operand, + }); } else { runtime_methods::concrete( tcx, @@ -785,8 +683,6 @@ pub(super) fn emit<'tcx>( resolved_receiver_mir_ty, is_pointer_null_method, is_pointer_cast_method, - receiver_self_requires_static_dispatch, - uses_concrete_trait_default, comparison_rhs_ty, pointer_api_receiver, ); @@ -837,24 +733,6 @@ pub(super) fn emit<'tcx>( } generated = true; } - - if !generated - && let Some(class_name) = class_type.get_class_name() - && matches!( - data_types.get(class_name), - Some(oomir::DataType::Class { methods, .. }) - if methods.contains_key(&method_name) - ) - { - instructions.push(oomir::Instruction::InvokeRustStatic { - class_name: class_name.to_string(), - method_name: method_name.clone(), - method_ty: method_signature.clone(), - args: oomir_operands.clone(), - dest: effective_dest.clone(), // use effective_dest - }); - generated = true; - } } if !generated { diff --git a/src/lower1/control_flow/calls/numeric_intrinsics.rs b/src/lower1/control_flow/calls/numeric_intrinsics.rs index 12e28634..54def4d6 100644 --- a/src/lower1/control_flow/calls/numeric_intrinsics.rs +++ b/src/lower1/control_flow/calls/numeric_intrinsics.rs @@ -348,9 +348,12 @@ pub(super) fn integer_min<'tcx>( ); } pub(super) fn carrying_mul_add<'tcx>( + tcx: TyCtxt<'tcx>, + data_types: &mut Definitions<'tcx>, + instance: Instance<'tcx>, + output: Ty<'tcx>, label: &str, instructions: &mut Vec, - oomir_output_type: oomir::Type, oomir_operands: Vec, dest: String, ) { @@ -397,13 +400,9 @@ pub(super) fn carrying_mul_add<'tcx>( dest: Some(result_dest), }); } - instructions.push(oomir::Instruction::ConstructObject { - dest, - class_name: oomir_output_type - .get_class_name() - .expect("carrying_mul_add returns a tuple class") - .to_string(), - args: vec![ + crate::lower1::types::tuple_value( + output, + vec![ ( oomir::Operand::Variable { name: low_dest, @@ -419,7 +418,12 @@ pub(super) fn carrying_mul_add<'tcx>( input_ty, ), ], - }); + &dest, + tcx, + data_types, + instance, + instructions, + ); } pub(super) fn ctpop<'tcx>( instructions: &mut Vec, diff --git a/src/lower1/control_flow/calls/pointer_access.rs b/src/lower1/control_flow/calls/pointer_access.rs index 02aa7e24..aaa8b096 100644 --- a/src/lower1/control_flow/calls/pointer_access.rs +++ b/src/lower1/control_flow/calls/pointer_access.rs @@ -10,6 +10,14 @@ pub(super) fn as_ref<'tcx>( dispatch_receiver_ty: oomir::Type, ) { if let Some(dest) = effective_dest { + if matches!(oomir_output_type, oomir::Type::Pointer(_)) { + instructions.push(oomir::Instruction::Cast { + dest, + op: receiver_operand, + ty: oomir_output_type, + }); + return; + } let option_class = oomir_output_type .get_class_name() .expect("pointer as_ref/as_mut returns Option") @@ -21,14 +29,16 @@ pub(super) fn as_ref<'tcx>( method_ty: oomir::Signature { params: vec![ ("pointer".to_string(), dispatch_receiver_ty.clone()), - ("option_class".to_string(), oomir::Type::java_string()), + ("some_class".to_string(), oomir::Type::java_string()), + ("none_class".to_string(), oomir::Type::java_string()), ], ret: Box::new(oomir::Type::Class("java/lang/Object".to_string())), is_static: true, }, args: vec![ receiver_operand, - oomir::Operand::Constant(oomir::Constant::String(option_class)), + oomir::Operand::Constant(oomir::Constant::String(format!("{option_class}$Some"))), + oomir::Operand::Constant(oomir::Constant::String(format!("{option_class}$None"))), ], dest: Some(option_object.clone()), }); @@ -58,7 +68,10 @@ pub(super) fn non_null_ref<'tcx>( // raw-pointer methods with the same names which return Option. if let Some(dest) = effective_dest { if let oomir::Type::Pointer(non_null_ty) = &dispatch_receiver_ty - && matches!(non_null_ty.as_ref(), oomir::Type::Pointer(_)) + && matches!( + non_null_ty.as_ref(), + oomir::Type::Pointer(_) | oomir::Type::Slice(_) | oomir::Type::Str + ) { let pointer = crate::lower1::place::emit_pointer_read( receiver_operand, @@ -129,9 +142,18 @@ pub(super) fn non_null_ref<'tcx>( instructions.push(oomir::Instruction::Move { dest, src: adapted }); } } else { + let reference = crate::lower1::value_repr::adapt_operand_to_rust_type( + receiver_operand, + fn_output, + label, + tcx, + instance, + data_types, + &mut instructions, + ); instructions.push(oomir::Instruction::Move { dest, - src: receiver_operand, + src: reference, }); } } @@ -175,6 +197,7 @@ pub(super) fn is_aligned<'tcx>( }); } pub(super) fn to_raw_parts<'tcx>( + tcx: TyCtxt<'tcx>, data_types: &mut Definitions<'tcx>, label: &str, mut instructions: &mut Vec, @@ -203,28 +226,38 @@ pub(super) fn to_raw_parts<'tcx>( if matches!(pointee.kind(), TyKind::Dynamic(..)) ); let data_pointer_name = format!("{label}_raw_parts_data"); - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: if source_is_trait_object { - "traitObjectDataPointer".to_string() - } else { - "retype".to_string() - }, - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), dispatch_receiver_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), + instructions.push(if !source_is_trait_object { + oomir::Instruction::AddressRetype { + dest: Some(data_pointer_name.clone()), + source: receiver_operand.clone(), + layout: Box::new(oomir::AddressLayout { + pointer_type: data_pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64(0)), + codec: oomir::Operand::Constant(oomir::Constant::Null( + oomir::Type::java_string(), + )), + }), + } + } else { + oomir::Instruction::InvokeStatic { + class_name: oomir::POINTER_CLASS.to_string(), + method_name: "traitObjectDataPointer".to_string(), + method_ty: oomir::Signature { + params: vec![ + ("pointer".to_string(), dispatch_receiver_ty.clone()), + ("view_size".to_string(), oomir::Type::U64), + ("view_codec".to_string(), oomir::Type::java_string()), + ], + ret: Box::new(data_pointer_ty.clone()), + is_static: true, + }, + args: vec![ + receiver_operand.clone(), + oomir::Operand::Constant(oomir::Constant::U64(0)), + oomir::Operand::Constant(oomir::Constant::Null(oomir::Type::java_string())), ], - ret: Box::new(data_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - receiver_operand.clone(), - oomir::Operand::Constant(oomir::Constant::U64(0)), - oomir::Operand::Constant(oomir::Constant::Null(oomir::Type::java_string())), - ], - dest: Some(data_pointer_name.clone()), + dest: Some(data_pointer_name.clone()), + } }); let mut tuple_args = vec![( oomir::Operand::Variable { @@ -238,6 +271,13 @@ pub(super) fn to_raw_parts<'tcx>( let metadata_name = format!("{label}_raw_parts_metadata"); emit_trait_object_metadata( receiver_operand.clone(), + crate::lower1::types::stable_type_identity( + tcx, + match resolved_receiver_mir_ty.kind() { + TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) => *pointee, + _ => unreachable!("trait metadata source is a pointer"), + }, + ), &metadata_ty, metadata_name.clone(), &metadata_name, diff --git a/src/lower1/control_flow/calls/pointer_address.rs b/src/lower1/control_flow/calls/pointer_address.rs index c17c3fe2..2fc15d6b 100644 --- a/src/lower1/control_flow/calls/pointer_address.rs +++ b/src/lower1/control_flow/calls/pointer_address.rs @@ -144,15 +144,15 @@ pub(super) fn map_address<'tcx>( let closure_arg_oomir_ty = crate::lower1::types::ty_to_oomir_type(closure_arg_ty, tcx, data_types, instance); let closure_tuple_dest = format!("{label}_map_addr_closure_tuple"); - let closure_tuple_class = closure_arg_oomir_ty - .get_class_name() - .expect("FnOnce closure argument tuple is a JVM class") - .to_string(); - instructions.push(oomir::Instruction::ConstructObject { - dest: closure_tuple_dest.clone(), - class_name: closure_tuple_class, - args: vec![(address_operand.clone(), oomir::Type::U64)], - }); + crate::lower1::types::tuple_value( + closure_arg_ty, + vec![(address_operand.clone(), oomir::Type::U64)], + &closure_tuple_dest, + tcx, + data_types, + instance, + instructions, + ); let captures = closure_args .as_closure() diff --git a/src/lower1/control_flow/calls/pointer_intrinsics.rs b/src/lower1/control_flow/calls/pointer_intrinsics.rs index e99889fd..a4f9c619 100644 --- a/src/lower1/control_flow/calls/pointer_intrinsics.rs +++ b/src/lower1/control_flow/calls/pointer_intrinsics.rs @@ -35,8 +35,8 @@ pub(super) fn ptr_drop_in_place<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, data_types: &mut Definitions<'tcx>, - label: &str, - mut instructions: &mut Vec, + _label: &str, + instructions: &mut Vec, func_instance: Instance<'tcx>, oomir_operands: Vec, ) { @@ -45,41 +45,14 @@ pub(super) fn ptr_drop_in_place<'tcx>( .types() .next() .expect("drop_in_place has a pointee type argument"); - if pointee_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - let pointee_oomir_ty = - crate::lower1::types::ty_to_oomir_type(pointee_ty, tcx, data_types, instance); - let pointer = oomir_operands[0].clone(); - let value = if !matches!(pointee_ty.kind(), TyKind::Slice(_)) - && matches!(pointer.get_type(), Some(oomir::Type::Pointer(_))) - { - let value_name = format!("{label}_drop_in_place_value"); - crate::lower1::place::emit_pointer_read( - pointer, - &pointee_oomir_ty, - &value_name, - &mut instructions, - ) - } else { - // Fat slice references are already carried - // as SliceView values rather than Pointer. - match pointer { - oomir::Operand::Variable { name, .. } => oomir::Operand::Variable { - name, - ty: pointee_oomir_ty.clone(), - }, - other => other, - } - }; - emit_rust_drop_value( - pointee_ty, - value, - &format!("{label}_drop_in_place"), - tcx, - instance, - data_types, - &mut instructions, - ); - } + emit_drop_in_place( + pointee_ty, + oomir_operands[0].clone(), + tcx, + instance, + data_types, + instructions, + ); } pub(super) fn ptr_eq<'tcx>( instructions: &mut Vec, @@ -179,7 +152,7 @@ pub(super) fn ptr_slice_from_raw_parts<'tcx>( instance: Instance<'tcx>, data_types: &mut Definitions<'tcx>, label: &str, - mut instructions: &mut Vec, + instructions: &mut Vec, fn_output: Ty<'tcx>, oomir_output_type: oomir::Type, oomir_operands: Vec, @@ -203,12 +176,13 @@ pub(super) fn ptr_slice_from_raw_parts<'tcx>( }); let data = crate::lower1::place::emit_retyped_slice_data_pointer( data, + crate::lower1::types::ty_to_oomir_type(element_ty, tcx, data_types, instance), oomir::Operand::Constant(oomir::Constant::U64( u64::try_from(element_size).expect("Rust slice element layout exceeds u64"), )), crate::lower1::types::pointer_view_codec_operand(element_ty, tcx, data_types, instance), &format!("{label}_raw_slice"), - &mut instructions, + instructions, ); let view_class = if is_str { oomir::UTF8_VIEW_CLASS @@ -218,7 +192,7 @@ pub(super) fn ptr_slice_from_raw_parts<'tcx>( let (backing, offset) = crate::lower1::place::emit_pointer_slice_parts( data, &format!("{label}_raw_slice"), - &mut instructions, + instructions, ); let slice_object = format!("{label}_raw_slice_object"); instructions.push(oomir::Instruction::ConstructObject { @@ -227,14 +201,7 @@ pub(super) fn ptr_slice_from_raw_parts<'tcx>( args: vec![ (backing, oomir::Type::Class("java/lang/Object".to_string())), (offset, oomir::Type::I32), - ( - oomir_operands[1].clone(), - if is_str { - oomir::Type::I32 - } else { - oomir::Type::U64 - }, - ), + (oomir_operands[1].clone(), oomir::Type::U64), ], }); instructions.push(oomir::Instruction::Cast { @@ -247,66 +214,6 @@ pub(super) fn ptr_slice_from_raw_parts<'tcx>( }); } } -pub(super) fn metadata<'tcx>( - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - label: &str, - mut instructions: &mut Vec, - fn_inputs: Vec>, - oomir_output_type: oomir::Type, - oomir_operands: Vec, - effective_dest: Option, -) { - if let Some(dest) = effective_dest.clone() { - let pointee = fn_inputs.first().and_then(|input| { - let (TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _)) = input.kind() else { - return None; - }; - Some(*pointee) - }); - let slice_tailed_pointee = pointee.and_then(|pointee| { - let tail = tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); - (tail.is_slice() || tail.is_str()).then_some(pointee) - }); - let trait_tailed_pointee = pointee.is_some_and(|pointee| { - let tail = tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); - matches!(tail.kind(), TyKind::Dynamic(..)) - }); - if slice_tailed_pointee.is_some() { - let pointer_ty = oomir_operands[0] - .get_type() - .expect("DST metadata operand must be typed"); - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(dest), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "metadata".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), pointer_ty)], - ret: Box::new(oomir_output_type.clone()), - is_static: false, - }, - args: Vec::new(), - operand: oomir_operands[0].clone(), - }); - } else if pointee.is_some_and(|pointee| matches!(pointee.kind(), TyKind::Dynamic(..))) - || trait_tailed_pointee - { - emit_trait_object_metadata( - oomir_operands[0].clone(), - &oomir_output_type, - dest, - &format!("{label}_trait_metadata"), - data_types, - &mut instructions, - ); - } else { - instructions.push(oomir::Instruction::Move { - dest, - src: oomir::Operand::Constant(oomir::Constant::Null(oomir_output_type.clone())), - }); - } - } -} pub(super) fn ptr_without_provenance<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, @@ -352,7 +259,7 @@ pub(super) fn ptr_without_provenance<'tcx>( }); } pub(super) fn ptr_read<'tcx>( - mut instructions: &mut Vec, + instructions: &mut Vec, oomir_operands: Vec, effective_dest: Option, intrinsic_name: String, @@ -385,12 +292,12 @@ pub(super) fn ptr_read<'tcx>( oomir_operands[0].clone(), pointee.as_ref(), &dest, - &mut instructions, + instructions, ); } } pub(super) fn ptr_write<'tcx>( - mut instructions: &mut Vec, + instructions: &mut Vec, oomir_operands: Vec, intrinsic_name: String, is_compiler_intrinsic: bool, @@ -401,7 +308,7 @@ pub(super) fn ptr_write<'tcx>( oomir_operands[0].clone(), pointee.as_ref(), oomir_operands[1].clone(), - &mut instructions, + instructions, ); } if is_diagnostic_item(sym::ptr_write_volatile) diff --git a/src/lower1/control_flow/calls/pointer_metadata.rs b/src/lower1/control_flow/calls/pointer_metadata.rs index 6d9ff52f..67f14cf1 100644 --- a/src/lower1/control_flow/calls/pointer_metadata.rs +++ b/src/lower1/control_flow/calls/pointer_metadata.rs @@ -25,6 +25,7 @@ pub(super) fn with_metadata<'tcx>( }; let data = emit_retyped_slice_data_pointer( data, + crate::lower1::types::ty_to_oomir_type(element_ty, tcx, data_types, instance), oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( crate::lower1::types::layout_size_bytes(tcx, element_ty) @@ -96,67 +97,44 @@ pub(super) fn cast<'tcx>( let source_element_size = crate::lower1::types::layout_size_bytes(tcx, source_pointee) .expect("fat pointer element has a concrete layout"); let data_pointer = format!("{label}_fat_cast_data"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::ViewAddress { dest: Some(data_pointer.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir_output_type.clone()), - is_static: true, - }, - args: vec![ - receiver_operand, - oomir::Operand::Constant(oomir::Constant::U64( + source: receiver_operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: oomir_output_type.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from(source_element_size).expect("Rust slice element layout exceeds u64"), )), - crate::lower1::types::pointer_view_codec_operand( + codec: crate::lower1::types::pointer_view_codec_operand( source_pointee, tcx, data_types, instance, ), - ], + }), }); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: effective_dest, - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), oomir_output_type.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir_output_type.clone()), - is_static: false, + source: oomir::Operand::Variable { + name: data_pointer, + ty: oomir_output_type.clone(), }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( + layout: Box::new(oomir::AddressLayout { + pointer_type: oomir_output_type.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( crate::lower1::types::layout_size_bytes(tcx, target_pointee) .expect("fat pointer cast target has a concrete layout"), ) .expect("Rust pointer target layout exceeds u64"), )), - crate::lower1::types::pointer_view_codec_operand( + codec: crate::lower1::types::pointer_view_codec_operand( target_pointee, tcx, data_types, instance, ), - ], - operand: oomir::Operand::Variable { - name: data_pointer, - ty: oomir_output_type.clone(), - }, + }), }); } @@ -176,73 +154,15 @@ pub(super) fn from_raw_parts<'tcx>( panic!("from_raw_parts returned non-pointer type {other:?}") } }; - let source_ty = oomir_operands[0] - .get_type() - .expect("from_raw_parts data pointer is typed"); - if matches!(pointee.kind(), TyKind::Dynamic(_, _)) { - instructions.push(oomir::Instruction::InvokeStatic { - dest: effective_dest.clone(), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromRawTraitParts".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), source_ty), - ( - "metadata".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ], - ret: method_signature.ret.clone(), - is_static: true, - }, - args: vec![oomir_operands[0].clone(), oomir_operands[1].clone()], - }); - } else { - let tail = tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); - let carries_slice_metadata = tail.is_slice() || tail.is_str(); - let pointee_size = - crate::lower1::types::layout_size_bytes(tcx, pointee).unwrap_or_else(|error| { - panic!("could not determine from_raw_parts pointee size: {error}") - }); - instructions.push(oomir::Instruction::InvokeStatic { - dest: effective_dest.clone(), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: if carries_slice_metadata { - "retypeWithMetadata".to_string() - } else { - "retype".to_string() - }, - method_ty: oomir::Signature { - params: { - let mut params = vec![ - ("pointer".to_string(), source_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ]; - if carries_slice_metadata { - params.push(("metadata".to_string(), oomir::Type::U64)); - } - params - }, - ret: method_signature.ret.clone(), - is_static: true, - }, - args: { - let mut args = vec![ - oomir_operands[0].clone(), - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from(pointee_size) - .expect("Rust pointer target layout exceeds u64"), - )), - crate::lower1::types::pointer_view_codec_operand( - pointee, tcx, data_types, instance, - ), - ]; - if carries_slice_metadata { - args.push(oomir_operands[1].clone()); - } - args - }, - }); - } + emit_raw_pointer_from_parts( + tcx, + instance, + data_types, + pointee, + *method_signature.ret, + oomir_operands[0].clone(), + oomir_operands[1].clone(), + effective_dest, + instructions, + ); } diff --git a/src/lower1/control_flow/calls/runtime_methods.rs b/src/lower1/control_flow/calls/runtime_methods.rs index 7d2c8fca..e770c663 100644 --- a/src/lower1/control_flow/calls/runtime_methods.rs +++ b/src/lower1/control_flow/calls/runtime_methods.rs @@ -20,14 +20,44 @@ pub(super) fn concrete<'tcx>( resolved_receiver_mir_ty: Ty<'tcx>, is_pointer_null_method: bool, is_pointer_cast_method: bool, - receiver_self_requires_static_dispatch: bool, - uses_concrete_trait_default: bool, comparison_rhs_ty: Option, pointer_api_receiver: bool, ) { // The receiver is a concrete class - use InvokeVirtual let class_type = crate::lower1::types::ty_to_oomir_type(receiver_mir_ty, tcx, data_types, instance); + if pointer_api_receiver && matches!(class_type, oomir::Type::Pointer(_)) { + let wrapping = declared_method_name.starts_with("wrapping_"); + let operation = declared_method_name + .strip_prefix("wrapping_") + .unwrap_or(&declared_method_name); + if matches!( + operation, + "add" | "sub" | "offset" | "byte_add" | "byte_sub" | "byte_offset" + ) { + instructions.push(oomir::Instruction::AddressOffset { + dest: effective_dest, + source: receiver_operand, + count: method_args[0].clone(), + ty: *method_signature.ret, + bytes: operation.starts_with("byte_"), + wrapping, + subtract: operation.ends_with("sub"), + }); + return; + } + } + let receiver_value_ty = match resolved_receiver_mir_ty.kind() { + TyKind::Ref(_, pointee, _) => *pointee, + _ => resolved_receiver_mir_ty, + }; + let primitive_slice_equality = match receiver_value_ty.kind() { + TyKind::Slice(element) | TyKind::Array(element, _) => matches!( + data_types.normalize(tcx, *element, instance).kind(), + TyKind::Bool | TyKind::Char | TyKind::Int(_) | TyKind::Uint(_) + ), + _ => false, + }; let runtime_static_target = match &class_type { oomir::Type::Str if declared_method_name == "as_bytes" => { Some((oomir::UTF8_VIEW_CLASS.to_string(), "asSlice".to_string())) @@ -64,6 +94,7 @@ pub(super) fn concrete<'tcx>( } oomir::Type::Slice(element) if declared_method_name == "starts_with" + && primitive_slice_equality && matches!(element.as_ref(), oomir::Type::I8 | oomir::Type::U8) => { Some(( @@ -73,6 +104,7 @@ pub(super) fn concrete<'tcx>( } oomir::Type::Slice(element) if declared_method_name == "starts_with" + && primitive_slice_equality && matches!( element.as_ref(), oomir::Type::I32 | oomir::Type::U32 | oomir::Type::Char @@ -83,10 +115,14 @@ pub(super) fn concrete<'tcx>( "startsWithI32".to_string(), )) } - oomir::Type::Slice(_) if declared_method_name == "starts_with" => Some(( - oomir::SLICE_VIEW_CLASS.to_string(), - "startsWith".to_string(), - )), + oomir::Type::Slice(_) + if declared_method_name == "starts_with" && primitive_slice_equality => + { + Some(( + oomir::SLICE_VIEW_CLASS.to_string(), + "startsWith".to_string(), + )) + } oomir::Type::Slice(_) if matches!(declared_method_name.as_str(), "as_ptr" | "as_mut_ptr") => { @@ -182,8 +218,7 @@ pub(super) fn concrete<'tcx>( let static_target = runtime_static_target .map(|(class_name, method_name)| (class_name, method_name, false)) .or_else(|| { - (uses_concrete_trait_default - || receiver_self_requires_static_dispatch + (!matches!(func_instance.def, InstanceKind::Virtual(..)) || requires_compiled_static_dispatch(&class_type)) .then(|| { let target = data_types.function_name(tcx, func_instance); @@ -279,6 +314,33 @@ pub(super) fn concrete<'tcx>( static_signature.params.pop(); static_args.pop(); } + if !generated_rust_target + && class_name == oomir::POINTER_CLASS + && matches!(static_method_name.as_str(), "fromSlice" | "retype") + { + let mut args = static_args.into_iter(); + let source = args.next().expect("address source"); + let layout = Box::new(oomir::AddressLayout { + pointer_type: *static_signature.ret, + size: args.next().expect("address size"), + codec: args.next().expect("address codec"), + }); + assert!(args.next().is_none()); + instructions.push(if static_method_name == "retype" { + oomir::Instruction::AddressRetype { + dest: effective_dest, + source, + layout, + } + } else { + oomir::Instruction::ViewAddress { + dest: effective_dest, + source, + layout, + } + }); + return; + } instructions.push(if generated_rust_target { oomir::Instruction::InvokeRustStatic { class_name, diff --git a/src/lower1/control_flow/calls/type_intrinsics.rs b/src/lower1/control_flow/calls/type_intrinsics.rs index 7206cc3c..a5234b00 100644 --- a/src/lower1/control_flow/calls/type_intrinsics.rs +++ b/src/lower1/control_flow/calls/type_intrinsics.rs @@ -199,36 +199,46 @@ pub(super) fn type_id_eq<'tcx>( let type_id_ty = oomir_operands[0] .get_type() .expect("TypeId equality operands are typed"); - let oomir::Type::Class(type_id_class) = &type_id_ty else { - panic!("type_id_eq received non-class operand {type_id_ty:?}"); - }; - let data_ty = match data_types.get(type_id_class) { - Some(oomir::DataType::Class { fields, .. }) => fields - .iter() - .find(|(name, _)| name == "data") - .map(|(_, ty)| ty.clone()) - .expect("TypeId class has a data field"), - _ => panic!("TypeId class {type_id_class} is not defined"), + let (left_data, right_data, data_ty) = match &type_id_ty { + oomir::Type::Array(_) => ( + oomir_operands[0].clone(), + oomir_operands[1].clone(), + type_id_ty.clone(), + ), + oomir::Type::Class(type_id_class) => { + let (data_name, data_ty) = match data_types.get(type_id_class) { + Some(oomir::DataType::Class { fields, .. }) => fields + .first() + .cloned() + .expect("TypeId class has a data field"), + _ => panic!("TypeId class {type_id_class} is not defined"), + }; + let mut data = + oomir_operands + .into_iter() + .take(2) + .enumerate() + .map(|(index, operand)| { + let name = format!("{label}_type_id_data_{index}"); + instructions.push(oomir::Instruction::GetField { + dest: name.clone(), + object: operand, + field_name: data_name.clone(), + field_ty: data_ty.clone(), + owner_class: type_id_class.clone(), + }); + oomir::Operand::Variable { + name, + ty: data_ty.clone(), + } + }); + (data.next().unwrap(), data.next().unwrap(), data_ty) + } + _ => panic!("type_id_eq received unsupported operand {type_id_ty:?}"), }; let oomir::Type::Array(pointer_ty) = &data_ty else { - panic!("TypeId data field is not an array: {data_ty:?}"); + panic!("TypeId data is not an array: {data_ty:?}"); }; - let left_data = format!("{label}_type_id_left_data"); - let right_data = format!("{label}_type_id_right_data"); - instructions.push(oomir::Instruction::GetField { - dest: left_data.clone(), - object: oomir_operands[0].clone(), - field_name: "data".to_string(), - field_ty: data_ty.clone(), - owner_class: type_id_class.clone(), - }); - instructions.push(oomir::Instruction::GetField { - dest: right_data.clone(), - object: oomir_operands[1].clone(), - field_name: "data".to_string(), - field_ty: data_ty.clone(), - owner_class: type_id_class.clone(), - }); let limb_count = 16usize / usize::try_from(tcx.data_layout.pointer_size().bytes()).expect("pointer size fits usize"); @@ -240,18 +250,12 @@ pub(super) fn type_id_eq<'tcx>( let index = oomir::Operand::Constant(oomir::Constant::I32(limb as i32)); instructions.push(oomir::Instruction::ArrayGet { dest: left_limb.clone(), - array: oomir::Operand::Variable { - name: left_data.clone(), - ty: data_ty.clone(), - }, + array: left_data.clone(), index: index.clone(), }); instructions.push(oomir::Instruction::ArrayGet { dest: right_limb.clone(), - array: oomir::Operand::Variable { - name: right_data.clone(), - ty: data_ty.clone(), - }, + array: right_data.clone(), index, }); instructions.push(oomir::Instruction::InvokeVirtual { diff --git a/src/lower1/control_flow/checked_intrinsics.rs b/src/lower1/control_flow/checked_intrinsics.rs index 17531b89..22258d5f 100644 --- a/src/lower1/control_flow/checked_intrinsics.rs +++ b/src/lower1/control_flow/checked_intrinsics.rs @@ -299,6 +299,7 @@ pub fn emit_all_needed_intrinsics(needed_intrinsics: &[(String, String, String)] intrinsic_methods.insert(function.name.clone(), DataTypeMethod::Function(function)); } DataType::Class { + kind: crate::oomir::ClassKind::Static, is_abstract: false, super_class: Some("java/lang/Object".to_string()), fields: vec![], diff --git a/src/lower1/control_flow/checked_ops.rs b/src/lower1/control_flow/checked_ops.rs index f8db4bd9..21c83958 100644 --- a/src/lower1/control_flow/checked_ops.rs +++ b/src/lower1/control_flow/checked_ops.rs @@ -12,8 +12,6 @@ pub fn emit_checked_arithmetic_oomir_instructions( unique_id_offset: usize, // Used to ensure unique labels/temps result_tuple_class: &str, ) -> (Vec, String, String, String) { - // Instead of inlining, emit a call to a reusable checked arithmetic intrinsic function. - // The intrinsic function should be emitted once per type/operation elsewhere (e.g., at module init). let mut generated_instructions = Vec::new(); let unique_id = unique_id_offset; let tmp_pair = format!("{}_{}_chk_pair_{}", dest_base_name, operation, unique_id); @@ -39,6 +37,50 @@ pub fn emit_checked_arithmetic_oomir_instructions( let fn_name = checked_intrinsics::get_intrinsic_function_name(operation, ty_suffix, result_tuple_class); + if matches!(operation, "add" | "sub") && !matches!(op_ty, Type::Class(_)) { + let body = checked_intrinsics::emit_checked_arithmetic_intrinsic( + operation, + op_ty, + ty_suffix, + result_tuple_class, + ); + let rename = |name: &str| { + if name == "tmp_struct" { + tmp_pair.clone() + } else { + format!("{tmp_pair}${name}") + } + }; + let operand = |value: &mut Operand| { + if let Operand::Variable { name, .. } = value { + match name.as_str() { + "_1" => *value = op1.clone(), + "_2" => *value = op2.clone(), + _ => *name = rename(name), + } + } + }; + for mut instruction in body.body.basic_blocks["entry"].instructions.clone() { + match &mut instruction { + Instruction::Binary { dest, op1, op2, .. } => { + *dest = rename(dest); + operand(op1); + operand(op2); + } + Instruction::ConstructObject { dest, args, .. } => { + *dest = rename(dest); + for (arg, _) in args { + operand(arg); + } + } + Instruction::Return { .. } => continue, + _ => unreachable!(), + } + generated_instructions.push(instruction); + } + return (generated_instructions, tmp_pair, tmp_result, tmp_overflow); + } + // Register that this intrinsic is needed data_types.request_checked_intrinsic(operation, ty_suffix, result_tuple_class); diff --git a/src/lower1/control_flow/comparisons.rs b/src/lower1/control_flow/comparisons.rs index 85fa89b7..e0264e66 100644 --- a/src/lower1/control_flow/comparisons.rs +++ b/src/lower1/control_flow/comparisons.rs @@ -29,34 +29,23 @@ pub(in crate::lower1) fn emit_raw_eq_pointer<'tcx>( ); let element_oomir_ty = crate::lower1::types::ty_to_oomir_type(element_ty, tcx, data_types, instance); - let pointer_ty = oomir::Type::Pointer(Box::new(element_oomir_ty)); - instructions.push(oomir::Instruction::InvokeStatic { + let pointer_ty = oomir::Type::pointer(element_oomir_ty); + instructions.push(oomir::Instruction::ViewAddress { dest: Some(temp_name.to_string()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![ - operand, - oomir::Operand::Constant(oomir::Constant::U64( + source: operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( crate::lower1::types::layout_size_bytes(tcx, element_ty) .expect("raw_eq element has a concrete layout"), ) .expect("Rust raw_eq element layout exceeds u64"), )), - crate::lower1::types::pointer_view_codec_operand(element_ty, tcx, data_types, instance), - ], + codec: crate::lower1::types::pointer_view_codec_operand( + element_ty, tcx, data_types, instance, + ), + }), }); oomir::Operand::Variable { name: temp_name.to_string(), @@ -72,7 +61,7 @@ pub(in crate::lower1) fn comparison_value_type<'tcx>( let oomir::Type::Pointer(inner) = ty else { break; }; - ty = *inner; + ty = *inner.value; mir_ty = *pointee; } ty @@ -83,7 +72,7 @@ pub(in crate::lower1) fn emit_comparison_value<'tcx>( mut mir_ty: Ty<'tcx>, dest_prefix: &str, tcx: TyCtxt<'tcx>, - data_types: &HashMap, + data_types: &crate::lower1::context::Definitions<'_>, instructions: &mut Vec, ) -> oomir::Operand { let mut depth = 0; diff --git a/src/lower1/control_flow/diagnostics.rs b/src/lower1/control_flow/diagnostics.rs index 8ad62e13..d7a36175 100644 --- a/src/lower1/control_flow/diagnostics.rs +++ b/src/lower1/control_flow/diagnostics.rs @@ -123,8 +123,5 @@ pub(in crate::lower1) fn emit_panic_lang_item<'tcx>( }, args, }); - instructions.push(oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/AssertionError".to_string(), - message: "Diverging Rust panic call returned unexpectedly".to_string(), - }); + instructions.push(oomir::Instruction::Unreachable); } diff --git a/src/lower1/control_flow/drop_glue.rs b/src/lower1/control_flow/drop_glue.rs new file mode 100644 index 00000000..2f3ebf83 --- /dev/null +++ b/src/lower1/control_flow/drop_glue.rs @@ -0,0 +1,137 @@ +//! Rust types and locations control destruction. Erased enum wrappers still own their Drop +//! implementation. +use super::*; + +pub(in crate::lower1) fn emit_drop_in_place<'tcx>( + rust_ty: Ty<'tcx>, + pointer: oomir::Operand, + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + data_types: &mut Definitions<'tcx>, + instructions: &mut Vec, +) { + let rust_ty = data_types.normalize(tcx, rust_ty, instance); + if !rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { + return; + } + if matches!(rust_ty.kind(), TyKind::Dynamic(..)) { + // The dyn drop shim calls virtual Drop. Another call to the shim would cause recursion. + let value_ty = crate::lower1::types::ty_to_oomir_type(rust_ty, tcx, data_types, instance); + let prefix = format!("dyn_drop_{}", data_types.next_temporary()); + let value = if matches!(pointer.get_type(), Some(oomir::Type::Pointer(_))) { + emit_pointer_read(pointer, &value_ty, &prefix, instructions) + } else { + pointer + }; + instructions.push(oomir::Instruction::InvokeStatic { + class_name: oomir::POINTER_CLASS.to_string(), + method_name: "dropRustValue".to_string(), + method_ty: oomir::Signature { + params: vec![( + "value".to_string(), + oomir::Type::Class("java/lang/Object".to_string()), + )], + ret: Box::new(oomir::Type::Void), + is_static: true, + }, + args: vec![value], + dest: None, + }); + return; + } + let drop_instance = Instance::resolve_drop_glue(tcx, rust_ty); + let target = data_types.function_name(tcx, drop_instance); + // The shim can take &mut T. Its ABI differs from *mut T for arrays and trait objects. + let mir = tcx.instance_mir(drop_instance.def); + let parameter = + data_types.normalize(tcx, mir.local_decls[Local::from_usize(1)].ty, drop_instance); + let pointer_ty = + crate::lower1::types::ty_to_oomir_type(parameter, tcx, data_types, drop_instance); + let prefix = format!("drop_place_{}", data_types.next_temporary()); + let pointer = crate::lower1::value_repr::adapt_operand_to_rust_type( + pointer, + parameter, + &prefix, + tcx, + drop_instance, + data_types, + instructions, + ); + instructions.push(oomir::Instruction::InvokeRustStatic { + class_name: target.class_to_call_on.expect("drop glue has a JVM owner"), + method_name: target.method_name, + method_ty: oomir::Signature { + params: vec![("place".to_string(), pointer_ty)], + ret: Box::new(oomir::Type::Void), + is_static: true, + }, + args: vec![pointer], + dest: None, + }); +} + +/// Object callbacks supply owned values. MIR drops supply existing locations. +pub(in crate::lower1) fn emit_owned_drop<'tcx>( + rust_ty: Ty<'tcx>, + value: oomir::Operand, + prefix: &str, + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + data_types: &mut Definitions<'tcx>, + instructions: &mut Vec, +) { + if !rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { + return; + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(rust_ty)) + .expect("owned drop requires a layout"); + let value_ty = crate::lower1::types::ty_to_oomir_type(rust_ty, tcx, data_types, instance); + let pointer_ty = oomir::Type::pointer(value_ty.clone()); + let dest = format!("{prefix}_place"); + let codec = + crate::lower1::types::pointer_memory_codec_operand(rust_ty, tcx, data_types, instance); + instructions.push(oomir::Instruction::InvokeStatic { + class_name: oomir::POINTER_CLASS.to_string(), + method_name: "receiverCellAligned".to_string(), + method_ty: oomir::Signature { + params: vec![ + ( + "value".to_string(), + oomir::Type::Class("java/lang/Object".to_string()), + ), + ("size".to_string(), oomir::Type::I32), + ("codec".to_string(), oomir::Type::java_string()), + ("alignment".to_string(), oomir::Type::I32), + ], + ret: Box::new(pointer_ty.clone()), + is_static: true, + }, + args: vec![ + if value_ty.has_jvm_value() { + value + } else { + oomir::Operand::Constant(oomir::Constant::Unit) + }, + oomir::Operand::Constant(oomir::Constant::I32( + i32::try_from(layout.size.bytes()).expect("drop value exceeds JVM size"), + )), + codec, + oomir::Operand::Constant(oomir::Constant::I32( + i32::try_from(layout.align.abi.bytes()).expect("drop alignment exceeds JVM size"), + )), + ], + dest: Some(dest.clone()), + }); + emit_drop_in_place( + rust_ty, + oomir::Operand::Variable { + name: dest, + ty: pointer_ty, + }, + tcx, + instance, + data_types, + instructions, + ); +} diff --git a/src/lower1/control_flow/drop_lowering.rs b/src/lower1/control_flow/drop_lowering.rs deleted file mode 100644 index f000d04f..00000000 --- a/src/lower1/control_flow/drop_lowering.rs +++ /dev/null @@ -1,580 +0,0 @@ -//! Drop lowering. -use super::*; - -pub(in crate::lower1) fn emit_rust_drop_value<'tcx>( - rust_ty: Ty<'tcx>, - mut value: oomir::Operand, - temp_prefix: &str, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - data_types: &mut Definitions<'tcx>, - instructions: &mut Vec, -) { - let instantiated = EarlyBinder::bind(tcx, rust_ty).instantiate(tcx, instance.args); - let rust_ty = tcx - .try_normalize_erasing_regions(TypingEnv::fully_monomorphized(), instantiated) - .unwrap_or_else(|_| instantiated.skip_norm_wip()); - let oomir_ty = crate::lower1::types::ty_to_oomir_type(rust_ty, tcx, data_types, instance); - if !oomir_ty.has_jvm_value() { - let Some(materialized) = crate::lower1::value_repr::materialize_implicit_zst( - rust_ty, - &format!("{temp_prefix}_zst"), - tcx, - instance, - data_types, - instructions, - ) else { - return; - }; - value = materialized; - } - - match rust_ty.kind() { - TyKind::Adt(adt_def, substs) if adt_def.is_struct() => { - if adt_def.is_box() { - let pointee_ty = substs.type_at(0); - if pointee_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - let mut pointer = value.clone(); - let mut pointer_ty = oomir_ty.clone(); - let mut pointer_rust_ty = rust_ty; - for depth in 0..3 { - if matches!( - pointer_ty, - oomir::Type::Pointer(_) | oomir::Type::Slice(_) | oomir::Type::Str - ) { - break; - } - let class_name = pointer_ty - .get_class_name() - .expect("Box pointer carrier must be a JVM class") - .to_string(); - let TyKind::Adt(carrier_def, carrier_args) = pointer_rust_ty.kind() else { - panic!("Box pointer carrier {pointer_rust_ty:?} is not a struct"); - }; - let field = carrier_def - .variant(rustc_abi::VariantIdx::from_usize(0)) - .fields - .iter() - .next() - .expect("Box pointer carrier must have a field"); - let field_name = field.ident(tcx).to_string(); - let field_rust_ty = field.ty(tcx, carrier_args).skip_norm_wip(); - let field_ty = crate::lower1::types::ty_to_oomir_type( - field_rust_ty, - tcx, - data_types, - instance, - ); - let dest = format!("{temp_prefix}_box_pointer_{depth}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: pointer, - field_name, - field_ty: field_ty.clone(), - owner_class: class_name, - }); - pointer = oomir::Operand::Variable { - name: dest, - ty: field_ty.clone(), - }; - pointer_ty = field_ty; - pointer_rust_ty = field_rust_ty; - } - let drop_instance = Instance::resolve_drop_glue(tcx, pointee_ty); - let target = data_types.function_name(tcx, drop_instance); - let pointee_signature = oomir::Signature { - params: vec![("pointee".to_string(), pointer_ty)], - ret: Box::new(oomir::Type::Void), - is_static: true, - }; - let box_class = oomir_ty - .get_class_name() - .expect("Box has a JVM class") - .to_string(); - let box_signature = oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }; - let (pointee_owner, pointee_method, pointee_descriptor) = - if matches!(pointee_ty.kind(), TyKind::Dynamic(..)) { - (String::new(), String::new(), String::new()) - } else { - ( - target - .class_to_call_on - .expect("Box pointee drop glue has a JVM owner"), - target.method_name, - pointee_signature.to_jvm_descriptor_with_explicit_params(), - ) - }; - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropBoxWithCleanup".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "pointee".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - "box".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("pointee_owner".to_string(), oomir::Type::java_string()), - ("pointee_method".to_string(), oomir::Type::java_string()), - ("pointee_descriptor".to_string(), oomir::Type::java_string()), - ("box_owner".to_string(), oomir::Type::java_string()), - ("box_method".to_string(), oomir::Type::java_string()), - ("box_descriptor".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![ - pointer, - value, - oomir::Operand::Constant(oomir::Constant::String(pointee_owner)), - oomir::Operand::Constant(oomir::Constant::String(pointee_method)), - oomir::Operand::Constant(oomir::Constant::String(pointee_descriptor)), - oomir::Operand::Constant(oomir::Constant::String(box_class)), - oomir::Operand::Constant(oomir::Constant::String("drop".to_string())), - oomir::Operand::Constant(oomir::Constant::String( - box_signature.to_jvm_descriptor_with_explicit_params(), - )), - ], - dest: None, - }); - return; - } - } - - if adt_def.destructor(tcx).is_some() { - let class_name = oomir_ty - .get_class_name() - .expect("a Rust Drop ADT has a JVM class") - .to_string(); - if !adt_def.is_box() { - let drop_signature = oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }; - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropAdtWithCleanup".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "value".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("owner".to_string(), oomir::Type::java_string()), - ("drop_method".to_string(), oomir::Type::java_string()), - ("drop_descriptor".to_string(), oomir::Type::java_string()), - ("fields_method".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![ - value, - oomir::Operand::Constant(oomir::Constant::String(class_name)), - oomir::Operand::Constant(oomir::Constant::String("drop".to_string())), - oomir::Operand::Constant(oomir::Constant::String( - drop_signature.to_jvm_descriptor_with_explicit_params(), - )), - oomir::Operand::Constant(oomir::Constant::String( - "_rust_drop_fields".to_string(), - )), - ], - dest: None, - }); - return; - } - instructions.push(oomir::Instruction::InvokeStatic { - class_name, - method_name: "drop".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![value.clone()], - dest: None, - }); - } - - for (field_index, field) in adt_def - .variant(rustc_abi::VariantIdx::from_usize(0)) - .fields - .iter() - .enumerate() - { - let field_rust_ty = field.ty(tcx, substs).skip_norm_wip(); - if !field_rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let field_oomir_ty = crate::lower1::types::ty_to_oomir_type( - field_rust_ty, - tcx, - data_types, - instance, - ); - let field_value = if field_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_field_{field_index}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: field.ident(tcx).to_string(), - field_ty: field_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a struct field owner has a JVM class") - .to_string(), - }); - oomir::Operand::Variable { - name: dest, - ty: field_oomir_ty, - } - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_rust_drop_value( - field_rust_ty, - field_value, - &format!("{temp_prefix}_field_{field_index}_drop"), - tcx, - instance, - data_types, - instructions, - ); - } - } - TyKind::Tuple(fields) => { - for (field_index, field_rust_ty) in fields.iter().enumerate() { - if !field_rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let field_oomir_ty = crate::lower1::types::ty_to_oomir_type( - field_rust_ty, - tcx, - data_types, - instance, - ); - let field_value = if field_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_tuple_{field_index}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: format!("field{field_index}"), - field_ty: field_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a non-empty tuple has a JVM class") - .to_string(), - }); - oomir::Operand::Variable { - name: dest, - ty: field_oomir_ty, - } - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_rust_drop_value( - field_rust_ty, - field_value, - &format!("{temp_prefix}_tuple_{field_index}_drop"), - tcx, - instance, - data_types, - instructions, - ); - } - } - TyKind::Closure(_, closure_args) => { - for (capture_index, capture_rust_ty) in - closure_args.as_closure().upvar_tys().iter().enumerate() - { - if !capture_rust_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let capture_oomir_ty = crate::lower1::types::ty_to_oomir_type( - capture_rust_ty, - tcx, - data_types, - instance, - ); - let capture_value = if capture_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_capture_{capture_index}"); - instructions.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: format!("arg{capture_index}"), - field_ty: capture_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a closure with captures has a JVM class") - .to_string(), - }); - oomir::Operand::Variable { - name: dest, - ty: capture_oomir_ty, - } - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_rust_drop_value( - capture_rust_ty, - capture_value, - &format!("{temp_prefix}_capture_{capture_index}_drop"), - tcx, - instance, - data_types, - instructions, - ); - } - } - TyKind::Array(element_ty, length) => { - let Some(length) = length.try_to_target_usize(tcx) else { - return; - }; - if !element_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - return; - } - let element_oomir_ty = - crate::lower1::types::ty_to_oomir_type(*element_ty, tcx, data_types, instance); - for index in 0..length { - let element_value = if element_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_array_{index}"); - instructions.push(oomir::Instruction::ArrayGet { - dest: dest.clone(), - array: value.clone(), - index: oomir::Operand::Constant(oomir::Constant::I32(index as i32)), - }); - oomir::Operand::Variable { - name: dest, - ty: element_oomir_ty.clone(), - } - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_rust_drop_value( - *element_ty, - element_value, - &format!("{temp_prefix}_array_{index}_drop"), - tcx, - instance, - data_types, - instructions, - ); - } - } - TyKind::Slice(element_ty) => { - if !element_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - return; - } - let drop_instance = Instance::resolve_drop_glue(tcx, *element_ty); - let target = data_types.function_name(tcx, drop_instance); - let drop_mir = tcx.instance_mir(drop_instance.def); - let drop_param_ty = EarlyBinder::bind( - tcx, - drop_mir.local_decls[rustc_middle::mir::Local::from_usize(1)].ty, - ) - .instantiate(tcx, drop_instance.args) - .skip_norm_wip(); - let element_oomir_ty = crate::lower1::types::ty_to_oomir_type( - drop_param_ty, - tcx, - data_types, - drop_instance, - ); - let drop_signature = oomir::Signature { - params: vec![("element".to_string(), element_oomir_ty)], - ret: Box::new(oomir::Type::Void), - is_static: true, - }; - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("owner".to_string(), oomir::Type::java_string()), - ("method".to_string(), oomir::Type::java_string()), - ("descriptor".to_string(), oomir::Type::java_string()), - ("element_size".to_string(), oomir::Type::U64), - ("element_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![ - value, - oomir::Operand::Constant(oomir::Constant::String( - target.class_to_call_on.expect("drop glue has a JVM owner"), - )), - oomir::Operand::Constant(oomir::Constant::String(target.method_name)), - oomir::Operand::Constant(oomir::Constant::String( - drop_signature.to_jvm_descriptor_with_explicit_params(), - )), - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from( - crate::lower1::types::layout_size_bytes(tcx, *element_ty) - .unwrap_or_else(|error| { - panic!("could not determine slice drop element size: {error}") - }), - ) - .expect("slice drop element size exceeds the JVM address space"), - )), - crate::lower1::types::pointer_view_codec_operand( - *element_ty, - tcx, - data_types, - instance, - ), - ], - dest: None, - }); - } - TyKind::Coroutine(..) => { - let pointer_ty = oomir::Type::Pointer(Box::new(oomir_ty.clone())); - let pointer_name = format!("{temp_prefix}_coroutine_pointer"); - let size = crate::lower1::types::layout_size_bytes(tcx, rust_ty) - .unwrap_or_else(|error| panic!("could not size coroutine drop value: {error}")); - let alignment = crate::lower1::types::layout_align_bytes(tcx, rust_ty) - .unwrap_or_else(|error| panic!("could not align coroutine drop value: {error}")); - let codec = crate::lower1::types::pointer_memory_codec_operand( - rust_ty, tcx, data_types, instance, - ); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "receiverCellAligned".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "value".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("size".to_string(), oomir::Type::I32), - ("codec".to_string(), oomir::Type::java_string()), - ("alignment".to_string(), oomir::Type::I32), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![ - value, - oomir::Operand::Constant(oomir::Constant::I32( - i32::try_from(size) - .expect("coroutine drop value exceeds the JVM address space"), - )), - codec, - oomir::Operand::Constant(oomir::Constant::I32( - i32::try_from(alignment) - .expect("coroutine drop alignment exceeds the JVM address space"), - )), - ], - }); - let drop_instance = Instance::resolve_drop_glue(tcx, rust_ty); - let target = data_types.function_name(tcx, drop_instance); - instructions.push(oomir::Instruction::InvokeStatic { - class_name: target - .class_to_call_on - .expect("coroutine drop glue has a JVM owner"), - method_name: target.method_name, - method_ty: oomir::Signature { - params: vec![("coroutine".to_string(), pointer_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![oomir::Operand::Variable { - name: pointer_name, - ty: pointer_ty, - }], - dest: None, - }); - } - TyKind::Dynamic(..) => { - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropRustValue".to_string(), - method_ty: oomir::Signature { - params: vec![( - "value".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - )], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![value], - dest: None, - }); - } - // Rust unions require ManuallyDrop fields. Enums use a virtual helper - // generated on every variant so the active payload is destroyed without - // trying to reinterpret the JVM subclass as an inactive variant. - TyKind::Adt(adt_def, substs) if adt_def.is_enum() => { - crate::lower1::types::force_define_named_adt(rust_ty, tcx, data_types, instance); - let class_name = crate::lower1::types::generate_adt_jvm_class_name( - adt_def, substs, tcx, data_types, instance, - ); - if adt_def.destructor(tcx).is_some() { - let drop_signature = oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }; - instructions.push(oomir::Instruction::InvokeStatic { - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropAdtWithCleanup".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "value".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("owner".to_string(), oomir::Type::java_string()), - ("drop_method".to_string(), oomir::Type::java_string()), - ("drop_descriptor".to_string(), oomir::Type::java_string()), - ("fields_method".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![ - value, - oomir::Operand::Constant(oomir::Constant::String(class_name.clone())), - oomir::Operand::Constant(oomir::Constant::String("drop".to_string())), - oomir::Operand::Constant(oomir::Constant::String( - drop_signature.to_jvm_descriptor_with_explicit_params(), - )), - oomir::Operand::Constant(oomir::Constant::String(enum_scoped_method_name( - &class_name, - "_rust_drop_fields", - ))), - ], - dest: None, - }); - return; - } - instructions.push(oomir::Instruction::InvokeVirtual { - class_name: class_name.clone(), - method_name: enum_scoped_method_name(&class_name, "_rust_drop_fields"), - method_ty: oomir::Signature { - params: vec![("self".to_string(), oomir_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - args: Vec::new(), - dest: None, - operand: value, - }); - } - _ => {} - } -} diff --git a/src/lower1/control_flow/metadata.rs b/src/lower1/control_flow/metadata.rs new file mode 100644 index 00000000..34f8e0f4 --- /dev/null +++ b/src/lower1/control_flow/metadata.rs @@ -0,0 +1,242 @@ +//! Pointer metadata has the same semantics before and after MIR inlining. +use super::*; + +pub(super) fn emit_pointer_metadata<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + data_types: &mut Definitions<'tcx>, + pointer_ty: Ty<'tcx>, + pointer: oomir::Operand, + output_type: oomir::Type, + dest: String, + instructions: &mut Vec, +) -> oomir::Operand { + // Inlined intrinsics can contain generic parameters. Metadata classification requires the + // instantiated pointee. + let pointer_ty = data_types.normalize(tcx, pointer_ty, instance); + let (TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _)) = pointer_ty.kind() else { + panic!("metadata source must be a pointer: {pointer_ty:?}"); + }; + if !output_type.has_jvm_value() { + return oomir::Operand::Constant(oomir::Constant::Unit); + } + let tail = tcx.struct_tail_for_codegen(*pointee, TypingEnv::fully_monomorphized()); + if tail.is_slice() || tail.is_str() { + match pointer.get_type().expect("metadata source is typed") { + oomir::Type::Slice(_) | oomir::Type::Str => { + instructions.push(oomir::Instruction::GetField { + dest: dest.clone(), + object: pointer, + field_name: "rustLength".into(), + field_ty: oomir::Type::U64, + owner_class: oomir::SLICE_VIEW_CLASS.into(), + }); + } + ty @ oomir::Type::Pointer(_) => { + instructions.push(oomir::Instruction::InvokeVirtual { + dest: Some(dest.clone()), + class_name: oomir::POINTER_CLASS.into(), + method_name: "metadata".into(), + method_ty: oomir::Signature { + params: vec![("self".into(), ty)], + ret: Box::new(oomir::Type::U64), + is_static: false, + }, + args: vec![], + operand: pointer, + }); + } + other => panic!("unsupported slice metadata carrier: {other:?}"), + } + } else if matches!(tail.kind(), TyKind::Dynamic(..)) { + emit_trait_object_metadata( + pointer, + crate::lower1::types::stable_type_identity(tcx, tail), + &output_type, + dest.clone(), + &dest, + data_types, + instructions, + ); + } else { + panic!("non-unit metadata for {pointer_ty:?}: {output_type:?}"); + } + oomir::Operand::Variable { + name: dest, + ty: output_type, + } +} + +pub(super) fn emit_trait_object_metadata( + pointer: oomir::Operand, + identity: String, + output_type: &oomir::Type, + dest: String, + temp_prefix: &str, + data_types: &crate::lower1::context::Definitions<'_>, + instructions: &mut Vec, +) { + // FakeForPtrMetadata can expose an interface carrier. The runtime marker API requires a raw + // pointer. + let pointer = if matches!(pointer.get_type(), Some(oomir::Type::Pointer(_))) { + pointer + } else { + let pointer_ty = oomir::Type::pointer(pointer.get_type().expect("trait source is typed")); + crate::lower1::value_repr::emit_trait_object_reference_pointer( + pointer, + &pointer_ty, + temp_prefix, + instructions, + ) + }; + let mut wrappers = Vec::new(); + let mut marker_type = output_type.clone(); + while let oomir::Type::Class(class) = &marker_type { + assert!(wrappers.len() < 16, "recursive trait metadata carrier"); + let fields = match data_types.get(class) { + Some(oomir::DataType::Class { fields, .. }) => fields, + other => panic!("trait metadata class is unavailable: {other:?}"), + }; + let [(_, inner)] = fields.as_slice() else { + panic!("trait metadata wrapper must have one payload") + }; + wrappers.push((class.clone(), inner.clone())); + marker_type = inner.clone(); + } + assert!( + matches!(marker_type, oomir::Type::Pointer(_)), + "trait metadata needs an address" + ); + let marker_name = format!("{temp_prefix}_vtable_marker"); + instructions.push(oomir::Instruction::InvokeStatic { + dest: Some(marker_name.clone()), + class_name: oomir::POINTER_CLASS.to_string(), + method_name: "traitMetadataMarker".to_string(), + method_ty: oomir::Signature { + params: vec![ + ( + "pointer".to_string(), + pointer.get_type().expect("trait source is typed"), + ), + ("identity".to_string(), oomir::Type::java_string()), + ], + ret: Box::new(marker_type.clone()), + is_static: true, + }, + args: vec![ + pointer, + oomir::Operand::Constant(oomir::Constant::String(identity)), + ], + }); + let mut value = oomir::Operand::Variable { + name: marker_name, + ty: marker_type, + }; + for (index, (class_name, field_ty)) in wrappers.into_iter().rev().enumerate() { + let name = format!("{temp_prefix}_metadata_{index}"); + instructions.push(oomir::Instruction::ConstructObject { + dest: name.clone(), + class_name: class_name.clone(), + args: vec![(value, field_ty)], + }); + value = oomir::Operand::Variable { + name, + ty: oomir::Type::Class(class_name), + }; + } + instructions.push(oomir::Instruction::Move { dest, src: value }); +} + +/// Reconstruct both pointer words; the data address alone cannot recover a vtable. +pub(super) fn emit_trait_pointer_from_parts( + data: oomir::Operand, + metadata: oomir::Operand, + pointer_type: oomir::Type, + dest: Option, + instructions: &mut Vec, +) { + instructions.push(oomir::Instruction::InvokeStatic { + dest, + class_name: oomir::POINTER_CLASS.into(), + method_name: "fromRawTraitParts".into(), + method_ty: oomir::Signature { + params: vec![ + ( + "pointer".into(), + data.get_type().expect("raw pointer data is typed"), + ), + ( + "metadata".into(), + oomir::Type::Class("java/lang/Object".into()), + ), + ], + ret: Box::new(pointer_type), + is_static: true, + }, + args: vec![data, metadata], + }); +} + +/// RawPtr MIR and uninlined from_raw_parts calls share reconstruction rules. +pub(super) fn emit_raw_pointer_from_parts<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + data_types: &mut Definitions<'tcx>, + pointee: Ty<'tcx>, + pointer_type: oomir::Type, + data: oomir::Operand, + metadata: oomir::Operand, + dest: Option, + instructions: &mut Vec, +) { + let pointee = data_types.normalize(tcx, pointee, instance); + if matches!(pointee.kind(), TyKind::Dynamic(..)) { + emit_trait_pointer_from_parts(data, metadata, pointer_type, dest, instructions); + return; + } + let tail = tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); + let size = oomir::Operand::Constant(oomir::Constant::U64( + crate::lower1::types::layout_size_bytes(tcx, pointee) + .expect("raw pointer pointee has a static prefix layout") as u64, + )); + let codec = + crate::lower1::types::pointer_view_codec_operand(pointee, tcx, data_types, instance); + let (method, metadata_type) = if tail.is_slice() || tail.is_str() { + ("retypeWithMetadata", oomir::Type::U64) + } else if matches!(tail.kind(), TyKind::Dynamic(..)) { + ( + "fromRawStructTraitParts", + oomir::Type::Class("java/lang/Object".into()), + ) + } else { + instructions.push(oomir::Instruction::AddressRetype { + dest, + source: data, + layout: Box::new(oomir::AddressLayout { + pointer_type, + size, + codec, + }), + }); + return; + }; + instructions.push(oomir::Instruction::InvokeStatic { + dest, + class_name: oomir::POINTER_CLASS.into(), + method_name: method.into(), + method_ty: oomir::Signature { + params: vec![ + ( + "pointer".into(), + data.get_type().expect("raw pointer data is typed"), + ), + ("prefix_size".into(), oomir::Type::U64), + ("codec".into(), oomir::Type::java_string()), + ("metadata".into(), metadata_type), + ], + ret: Box::new(pointer_type), + is_static: true, + }, + args: vec![data, size, codec, metadata], + }); +} diff --git a/src/lower1/control_flow/rvalue.rs b/src/lower1/control_flow/rvalue.rs index 0e7e4d03..1a8e234d 100644 --- a/src/lower1/control_flow/rvalue.rs +++ b/src/lower1/control_flow/rvalue.rs @@ -38,17 +38,25 @@ use rustc_middle::{ mod pointer_layout; use pointer_layout::*; mod struct_tails; +pub(crate) use struct_tails::emit_struct_tail_reborrow_view; use struct_tails::*; mod unsize; use unsize::*; +mod array_places; +use array_places::*; mod array_views; use array_views::*; mod addressing; +pub(crate) use addressing::emit_pointer_to_place; use addressing::*; mod aggregates; use aggregates::*; +mod callable_handles; mod function_pointers; +pub(crate) use callable_handles::callable_handle; pub(crate) use function_pointers::*; +mod exported_closures; +pub(crate) use exported_closures::ensure_exported_closure_calls; mod closures; pub(crate) use closures::*; mod arithmetic; @@ -63,8 +71,6 @@ struct RvalueContext<'b, 'tcx> { instance: Instance<'tcx>, data_types: &'b mut Definitions<'tcx>, external_interfaces: &'b mut HashSet, - pointer_origins: &'b crate::lower1::control_flow::MutableBorrowMap<'tcx>, - available_pointer_locals: &'b HashSet, } fn generate_temp_var_name(data_types: &mut Definitions<'_>, base_name: &str) -> String { @@ -116,8 +122,6 @@ pub(super) fn convert_rvalue_to_operand<'a>( instance: Instance<'a>, data_types: &mut Definitions<'a>, external_interfaces: &mut HashSet, - pointer_origins: &super::MutableBorrowMap<'a>, - available_pointer_locals: &HashSet, ) -> (Vec, oomir::Operand) { breadcrumbs::log!( breadcrumbs::LogLevel::Info, @@ -131,8 +135,6 @@ pub(super) fn convert_rvalue_to_operand<'a>( instance, data_types, external_interfaces, - pointer_origins, - available_pointer_locals, }; match rvalue { Rvalue::Use(..) => context.lower_values(rvalue), diff --git a/src/lower1/control_flow/rvalue/addressing.rs b/src/lower1/control_flow/rvalue/addressing.rs index 8585d8dd..31aa1779 100644 --- a/src/lower1/control_flow/rvalue/addressing.rs +++ b/src/lower1/control_flow/rvalue/addressing.rs @@ -7,6 +7,7 @@ pub(super) fn emit_pointer_factory( dest: &str, instructions: &mut Vec, ) -> oomir::Operand { + assert!(matches!(pointer_ty, oomir::Type::Pointer(_))); let params = match method_name { "cell" => vec![ ( @@ -45,91 +46,11 @@ pub(super) fn emit_pointer_factory( } } -pub(super) fn emit_array_pointer( - array_or_slice: oomir::Operand, - index: oomir::Operand, - element_size: oomir::Operand, - codec: oomir::Operand, - pointer_ty: &oomir::Type, - dest: &str, - instructions: &mut Vec, -) -> oomir::Operand { - let source_ty = array_or_slice - .get_type() - .expect("array pointer source must be typed"); - if matches!(source_ty, oomir::Type::Slice(_)) { - let index = if index.get_type() == Some(oomir::Type::U64) { - index - } else { - let index_u64 = format!("{dest}_slice_index_u64"); - instructions.push(oomir::Instruction::Cast { - dest: index_u64.clone(), - op: index, - ty: oomir::Type::U64, - }); - oomir::Operand::Variable { - name: index_u64, - ty: oomir::Type::U64, - } - }; - let base_dest = format!("{dest}_slice_base"); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(base_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![array_or_slice, element_size, codec], - }); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(dest.to_string()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "add".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), pointer_ty.clone()), - ("count".to_string(), oomir::Type::U64), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![ - oomir::Operand::Variable { - name: base_dest, - ty: pointer_ty.clone(), - }, - index, - ], - }); - return oomir::Operand::Variable { - name: dest.to_string(), - ty: pointer_ty.clone(), - }; - } - emit_pointer_factory( - "array", - vec![array_or_slice, index, element_size, codec], - pointer_ty, - dest, - instructions, - ) -} - /// Takes the address of a MIR place. Array elements retain their real backing /// allocation; scalar/aggregate locals and fields receive a stable heap cell. /// A dereference followed by another borrow is a reborrow and therefore keeps /// the exact pointer object/allocation identity. -pub(super) fn emit_pointer_to_place<'tcx>( +pub(crate) fn emit_pointer_to_place<'tcx>( place: &Place<'tcx>, pointer_ty: &oomir::Type, temp_prefix: &str, @@ -139,6 +60,54 @@ pub(super) fn emit_pointer_to_place<'tcx>( data_types: &mut Definitions<'tcx>, instructions: &mut Vec, ) -> oomir::Operand { + if matches!(pointer_ty, oomir::Type::Slice(_) | oomir::Type::Str) { + let rust_ty = normalize_unsize_ty(place.ty(&mir.local_decls, tcx).ty, tcx, instance); + let projects_struct_tail = place.projection.split_last().is_some_and(|(last, prefix)| { + matches!(last, ProjectionElem::Field(..)) + && crate::lower1::place::has_slice_or_str_struct_tail( + tcx, + normalize_unsize_ty( + Place { + local: place.local, + projection: tcx.mk_place_elems(prefix), + } + .ty(&mir.local_decls, tcx) + .ty, + tcx, + instance, + ), + ) + }); + if projects_struct_tail { + let storage = emit_pointer_to_place( + place, + &oomir::Type::pointer(pointer_ty.clone()), + &format!("{temp_prefix}_unsized_storage"), + tcx, + instance, + mir, + data_types, + instructions, + ); + return emit_slice_pointer_carrier( + Ty::new_ptr(tcx, rust_ty, rustc_middle::ty::Mutability::Mut), + storage, + temp_prefix, + tcx, + instance, + data_types, + instructions, + ) + .expect("slice-like place must retain its metadata"); + } + // Slice and string places already contain an address and length. A new cell would store the + // view itself. + let (name, get_instructions, ty) = + emit_instructions_to_get_on_own(place, tcx, instance, mir, data_types); + instructions.extend(get_instructions); + assert_eq!(&ty, pointer_ty, "unsized borrow must preserve its carrier"); + return oomir::Operand::Variable { name, ty }; + } if place.projection.is_empty() && data_types.local_uses_stable_cell(place.local) { let dest = format!("{temp_prefix}_local_ptr"); instructions.push(oomir::Instruction::Move { @@ -168,7 +137,7 @@ pub(super) fn emit_pointer_to_place<'tcx>( let base_rust_ty = normalize_unsize_ty(base_place.ty(&mir.local_decls, tcx).ty, tcx, instance); let base_oomir_ty = get_place_type(&base_place, mir, tcx, instance, data_types); - let base_pointer_ty = oomir::Type::Pointer(Box::new(base_oomir_ty)); + let base_pointer_ty = oomir::Type::pointer(base_oomir_ty); let base_pointer = emit_pointer_to_place( &base_place, &base_pointer_ty, @@ -180,29 +149,19 @@ pub(super) fn emit_pointer_to_place<'tcx>( instructions, ); let dest = format!("{temp_prefix}_downcast_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), base_pointer_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![ - base_pointer, - rust_layout_size_operand(base_rust_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: base_pointer, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: rust_layout_size_operand(base_rust_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( base_rust_ty, tcx, data_types, instance, ), - ], + }), }); return oomir::Operand::Variable { name: dest, @@ -284,26 +243,16 @@ pub(super) fn emit_pointer_to_place<'tcx>( instructions, ); let dest = format!("{temp_prefix}_slice_array"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), pointer_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: false, - }, - args: vec![ - rust_layout_size_operand(pointee_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: element_pointer, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: rust_layout_size_operand(pointee_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( pointee_ty, tcx, data_types, instance, ), - ], - operand: element_pointer, + }), }); return oomir::Operand::Variable { name: dest, @@ -312,37 +261,16 @@ pub(super) fn emit_pointer_to_place<'tcx>( } } ProjectionElem::Index(index_local) => { - let direct_slice_base = - base_place - .projection - .split_last() - .and_then(|(projection, prefix)| { - if !matches!(projection, ProjectionElem::Deref) { - return None; - } - let reference_place = Place { - local: base_place.local, - projection: tcx.mk_place_elems(prefix), - }; - matches!( - get_place_type(&reference_place, mir, tcx, instance, data_types), - oomir::Type::Slice(_) - ) - .then(|| { - emit_instructions_to_get_on_own( - &reference_place, - tcx, - instance, - mir, - data_types, - ) - }) - }); - let (base_name, base_instructions, base_ty) = - direct_slice_base.unwrap_or_else(|| { - emit_instructions_to_get_on_own(&base_place, tcx, instance, mir, data_types) - }); - instructions.extend(base_instructions); + let base = array_place_storage( + &base_place, + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + let base_ty = base.get_type().expect("array place type"); if matches!(base_ty, oomir::Type::Array(_) | oomir::Type::Slice(_)) { let index = convert_operand( &MirOperand::Copy(Place::from(*index_local)), @@ -353,10 +281,7 @@ pub(super) fn emit_pointer_to_place<'tcx>( instructions, ); return emit_array_pointer( - oomir::Operand::Variable { - name: base_name, - ty: base_ty, - }, + base, index, rust_layout_size_operand(place.ty(&mir.local_decls, tcx).ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( @@ -374,42 +299,18 @@ pub(super) fn emit_pointer_to_place<'tcx>( ProjectionElem::ConstantIndex { offset, from_end, .. } => { - let direct_slice_base = - base_place - .projection - .split_last() - .and_then(|(projection, prefix)| { - if !matches!(projection, ProjectionElem::Deref) { - return None; - } - let reference_place = Place { - local: base_place.local, - projection: tcx.mk_place_elems(prefix), - }; - matches!( - get_place_type(&reference_place, mir, tcx, instance, data_types), - oomir::Type::Slice(_) - ) - .then(|| { - emit_instructions_to_get_on_own( - &reference_place, - tcx, - instance, - mir, - data_types, - ) - }) - }); - let (base_name, base_instructions, base_ty) = - direct_slice_base.unwrap_or_else(|| { - emit_instructions_to_get_on_own(&base_place, tcx, instance, mir, data_types) - }); - instructions.extend(base_instructions); + let base = array_place_storage( + &base_place, + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + let base_ty = base.get_type().expect("array place type"); if matches!(base_ty, oomir::Type::Array(_) | oomir::Type::Slice(_)) { - let base_operand = oomir::Operand::Variable { - name: base_name, - ty: base_ty, - }; + let base_operand = base; let index = if *from_end { let length_name = format!("{temp_prefix}_pointer_length"); instructions.push(oomir::Instruction::Length { @@ -454,6 +355,189 @@ pub(super) fn emit_pointer_to_place<'tcx>( let base_rust_ty = EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) .instantiate(tcx, instance.args) .skip_norm_wip(); + if field_index.index() == 0 + && matches!(base_rust_ty.kind(), TyKind::Adt(def, args) + if crate::lower1::is_non_null_lang_item(tcx, def.did()) + && crate::lower1::types::is_codegen_sized(args.type_at(0), tcx)) + { + return emit_pointer_to_place( + &base_place, + pointer_ty, + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + } + if let Some(payload) = crate::lower1::types::transparent_payload(base_rust_ty, tcx) + { + if field_index.index() == payload.field { + return emit_pointer_to_place( + &base_place, + pointer_ty, + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + } + let parent_ty = get_place_type(&base_place, mir, tcx, instance, data_types); + let mut parent = emit_pointer_to_place( + &base_place, + &oomir::Type::pointer(parent_ty), + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(base_rust_ty)) + .expect("owner layout was checked by transparent_payload"); + let offset = layout.fields.offset(field_index.index()).bytes(); + if offset != 0 { + let dest = format!("{temp_prefix}_marker_offset"); + let ty = parent.get_type().expect("owner has an address"); + instructions.push(oomir::Instruction::AddressOffset { + dest: Some(dest.clone()), + source: parent, + count: oomir::Operand::Constant(oomir::Constant::U64(offset)), + ty: ty.clone(), + bytes: true, + wrapping: false, + subtract: false, + }); + parent = oomir::Operand::Variable { name: dest, ty }; + } + let dest = format!("{temp_prefix}_marker_pointer"); + instructions.push(oomir::Instruction::AddressRetype { + dest: Some(dest.clone()), + source: parent, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: rust_layout_size_operand(field_rust_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( + field_rust_ty, + tcx, + data_types, + instance, + ), + }), + }); + return oomir::Operand::Variable { + name: dest, + ty: pointer_ty.clone(), + }; + } + if let Some(word) = crate::lower1::types::packed_word(base_rust_ty, tcx) { + let owner_type = oomir::Type::pointer(word.jvm_type()); + let owner = emit_pointer_to_place( + &base_place, + &owner_type, + &format!("{temp_prefix}_owner"), + tcx, + instance, + mir, + data_types, + instructions, + ); + let field_type = ty_to_oomir_type(field_rust_ty, tcx, data_types, instance); + return word.field_address( + owner, + field_index.index(), + &field_type, + temp_prefix, + instructions, + ); + } + if field_index.index() == 0 + && let Some(layout) = crate::lower1::types::tagged_scalar(base_rust_ty, tcx) + { + let Some((ProjectionElem::Downcast(_, variant), parent)) = prefix.split_last() + else { + panic!("tagged payload needs a variant"); + }; + assert_eq!(variant.as_u32(), 1); + let owner_type = oomir::Type::pointer(oomir::Type::TaggedI64); + let owner = emit_pointer_to_place( + &Place { + local: place.local, + projection: tcx.mk_place_elems(parent), + }, + &owner_type, + &format!("{temp_prefix}_owner"), + tcx, + instance, + mir, + data_types, + instructions, + ); + let offset = format!("{temp_prefix}_payload_offset"); + instructions.push(oomir::Instruction::AddressOffset { + dest: Some(offset.clone()), + source: owner, + count: oomir::Operand::Constant(oomir::Constant::U64( + layout.payload_offset as u64, + )), + ty: owner_type.clone(), + bytes: true, + wrapping: false, + subtract: false, + }); + let dest = format!("{temp_prefix}_payload_address"); + instructions.push(oomir::Instruction::AddressRetype { + dest: Some(dest.clone()), + source: oomir::Operand::Variable { + name: offset, + ty: owner_type, + }, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::I32(8)), + codec: oomir::Operand::Constant(oomir::Constant::Null( + oomir::Type::java_string(), + )), + }), + }); + return oomir::Operand::Variable { + name: dest, + ty: pointer_ty.clone(), + }; + } + if field_index.index() == 0 + && crate::lower1::types::direct_enum_payload(base_rust_ty, tcx).is_some() + { + let Some((ProjectionElem::Downcast(_, variant), parent)) = prefix.split_last() + else { + panic!("nullable enum field needs a variant projection"); + }; + if *variant + != crate::lower1::types::enum_carrier(base_rust_ty, tcx) + .unwrap() + .variant + { + instructions.push(oomir::Instruction::Unreachable); + return oomir::Operand::Constant(oomir::Constant::Null(pointer_ty.clone())); + } + return emit_pointer_to_place( + &Place { + local: place.local, + projection: tcx.mk_place_elems(parent), + }, + pointer_ty, + temp_prefix, + tcx, + instance, + mir, + data_types, + instructions, + ); + } let managed_base_rust_ty = match base_rust_ty.kind() { TyKind::Ref(_, inner, _) => *inner, _ => base_rust_ty, @@ -519,7 +603,7 @@ pub(super) fn emit_pointer_to_place<'tcx>( if let Some((base_oomir_ty, owner_class, field_name)) = managed_field { let base_pointer_ty = match base_oomir_ty { oomir::Type::Pointer(_) => base_oomir_ty.clone(), - _ => oomir::Type::Pointer(Box::new(base_oomir_ty.clone())), + _ => oomir::Type::pointer(base_oomir_ty.clone()), }; let base_pointer = emit_pointer_to_place( &base_place, @@ -548,39 +632,17 @@ pub(super) fn emit_pointer_to_place<'tcx>( .offset((*field_index).into()) .bytes_usize(); let result_name = format!("{temp_prefix}_struct_field_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(result_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "projectStructField".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), base_pointer_ty.clone()), - ("owner_class".to_string(), oomir::Type::java_string()), - ("field_name".to_string(), oomir::Type::java_string()), - ("field_offset".to_string(), oomir::Type::U64), - ("field_size".to_string(), oomir::Type::U64), - ("field_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::String(owner_class)), - oomir::Operand::Constant(oomir::Constant::String(field_name)), - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from(field_offset) - .expect("Rust struct field offset exceeds u64"), - )), - rust_layout_size_operand(field_rust_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( - field_rust_ty, - tcx, - data_types, - instance, - ), - ], - operand: base_pointer, - }); + instructions.push(crate::lower1::place::project_memory( + result_name.clone(), + base_pointer, + owner_class, + field_name, + field_rust_ty, + field_offset as u64, + tcx, + instance, + data_types, + )); return oomir::Operand::Variable { name: result_name, ty: pointer_ty.clone(), @@ -607,7 +669,7 @@ pub(super) fn emit_pointer_to_place<'tcx>( ]; let mut field_args = vec![ base_value, - oomir::Operand::Constant(oomir::Constant::String(field_name)), + oomir::Operand::Constant(oomir::Constant::LiteralString(field_name)), rust_layout_size_operand(field_rust_ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( field_rust_ty, @@ -659,7 +721,7 @@ pub(super) fn emit_pointer_to_place<'tcx>( .offset((*field_index).into()) .bytes_usize(); let base_oomir_ty = ty_to_oomir_type(base_rust_ty, tcx, data_types, instance); - let base_pointer_ty = oomir::Type::Pointer(Box::new(base_oomir_ty)); + let base_pointer_ty = oomir::Type::pointer(base_oomir_ty); let base_pointer = emit_pointer_to_place( &base_place, &base_pointer_ty, @@ -671,57 +733,39 @@ pub(super) fn emit_pointer_to_place<'tcx>( instructions, ); let offset_pointer_name = format!("{temp_prefix}_field_offset_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressOffset { dest: Some(offset_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "byte_offset".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), base_pointer_ty.clone()), - ("byte_count".to_string(), oomir::Type::U64), - ], - ret: Box::new(base_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - base_pointer, - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from(field_offset) - .expect("Rust aggregate field offset exceeds u64"), - )), - ], + source: base_pointer, + count: oomir::Operand::Constant(oomir::Constant::U64( + u64::try_from(field_offset) + .expect("Rust aggregate field offset exceeds u64"), + )), + ty: base_pointer_ty.clone(), + bytes: true, + wrapping: false, + subtract: false, }); let result_name = format!("{temp_prefix}_field_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(result_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), base_pointer_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, + source: oomir::Operand::Variable { + name: offset_pointer_name, + ty: pointer_ty.clone(), }, - args: vec![ - oomir::Operand::Variable { - name: offset_pointer_name, - ty: pointer_ty.clone(), - }, - rust_layout_size_operand( + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: rust_layout_size_operand( place.ty(&mir.local_decls, tcx).ty, tcx, instance, ), - crate::lower1::types::pointer_view_codec_operand( + codec: crate::lower1::types::pointer_view_codec_operand( place.ty(&mir.local_decls, tcx).ty, tcx, data_types, instance, ), - ], + }), }); return oomir::Operand::Variable { name: result_name, @@ -756,50 +800,3 @@ pub(super) fn emit_pointer_to_place<'tcx>( instructions, ) } - -pub(super) fn reuse_pointer_to_place<'tcx>( - place: &Place<'tcx>, - pointer_ty: &oomir::Type, - temp_prefix: &str, - pointer_origins: &crate::lower1::control_flow::MutableBorrowMap<'tcx>, - available_pointer_locals: &HashSet, - data_types: &Definitions<'tcx>, - instructions: &mut Vec, -) -> Option { - // A dereference is tied to the current value of its base pointer, not only - // to the syntactic MIR place. Loop-carried raw pointers can be reassigned - // while an older borrow of `*pointer` is still available; reusing that - // borrow would retain the previous address. Reborrowing through - // emit_pointer_to_place is allocation-free and preserves the current - // pointer's provenance, so always materialize dereferenced places afresh. - if place - .projection - .iter() - .any(|projection| matches!(projection, ProjectionElem::Deref)) - { - return None; - } - - let oomir::Type::Pointer(target_inner) = pointer_ty else { - return None; - }; - let (_, origin) = pointer_origins.iter().find(|(local, origin)| { - available_pointer_locals.contains(local) - && !data_types.local_uses_stable_cell(**local) - && origin.original_place == *place - && origin.carrier_name != temp_prefix - && origin.pointee_type == *target_inner.as_ref() - })?; - let dest = format!("{temp_prefix}_existing_ptr"); - instructions.push(oomir::Instruction::Move { - dest: dest.clone(), - src: oomir::Operand::Variable { - name: origin.carrier_name.clone(), - ty: pointer_ty.clone(), - }, - }); - Some(oomir::Operand::Variable { - name: dest, - ty: pointer_ty.clone(), - }) -} diff --git a/src/lower1/control_flow/rvalue/aggregates.rs b/src/lower1/control_flow/rvalue/aggregates.rs index ecd6922a..0fdb2d7c 100644 --- a/src/lower1/control_flow/rvalue/aggregates.rs +++ b/src/lower1/control_flow/rvalue/aggregates.rs @@ -107,6 +107,7 @@ pub(super) fn adapt_value_for_field<'tcx>( data_types.get(class_name), Some(oomir::DataType::Class { fields, + kind: crate::oomir::ClassKind::Value, is_abstract: false, .. }) if fields.is_empty() @@ -155,24 +156,24 @@ impl<'tcx> RvalueContext<'_, 'tcx> { let aggregate_has_jvm_value = aggregate_oomir_type.has_jvm_value(); match kind { - rustc_middle::mir::AggregateKind::Tuple if !aggregate_has_jvm_value => { + rustc_middle::mir::AggregateKind::Tuple + | rustc_middle::mir::AggregateKind::Adt(..) + | rustc_middle::mir::AggregateKind::Closure(..) + if !aggregate_has_jvm_value => + { // Coroutine MIR represents unit yields and returns as empty // tuple aggregates, which have no JVM value or constructor. - debug_assert!(operands.is_empty()); + debug_assert!(operands.iter().all(|operand| { + !ty_to_oomir_type( + operand.ty(&mir.local_decls, tcx), + tcx, + data_types, + instance, + ) + .has_jvm_value() + })); } rustc_middle::mir::AggregateKind::Tuple => { - let tuple_class_name = match &aggregate_oomir_type { - oomir::Type::Class(name) => name.clone(), - _ => panic!("Tuple aggregate type error"), - }; - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: Handling Tuple Aggregate -> Temp Var '{}'", - temp_aggregate_var - ) - ); let place_ty = original_dest_place.ty(&mir.local_decls, tcx).ty; let mut constructor_args = Vec::new(); for (i, mir_op) in operands.iter().enumerate() { @@ -203,11 +204,15 @@ impl<'tcx> RvalueContext<'_, 'tcx> { ); constructor_args.push((value_operand, element_oomir_type)); } - instructions.push(oomir::Instruction::ConstructObject { - dest: temp_aggregate_var.clone(), - class_name: tuple_class_name.clone(), - args: constructor_args, - }); + crate::lower1::types::tuple_value( + place_ty, + constructor_args, + &temp_aggregate_var, + tcx, + data_types, + instance, + &mut instructions, + ); } rustc_middle::mir::AggregateKind::Array(mir_element_ty) => { breadcrumbs::log!( @@ -262,18 +267,8 @@ impl<'tcx> RvalueContext<'_, 'tcx> { } } rustc_middle::mir::AggregateKind::Closure(_, _) => { - let closure_ty = original_dest_place.ty(&mir.local_decls, tcx).ty; - if let Some(callable_abi) = - closure_callable_abi(closure_ty, tcx, data_types, instance) - { - ensure_closure_callable_bridge( - closure_ty, - &callable_abi, - data_types, - tcx, - instance, - ); - } + // Only dyn Fn coercions require callable interfaces. Static calls already + // identify the closure body. let closure_class_name = match &aggregate_oomir_type { oomir::Type::Class(name) => name.clone(), _ => panic!("Closure aggregate type error"), @@ -395,7 +390,106 @@ impl<'tcx> RvalueContext<'_, 'tcx> { instance, ); } - if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) + if let Some(payload) = crate::lower1::types::transparent_payload( + data_types.normalize(tcx, Ty::new_adt(tcx, adt_def, substs), instance), + tcx, + ) { + let value = convert_operand( + &operands[FieldIdx::from_usize(payload.field)], + tcx, + instance, + mir, + data_types, + &mut instructions, + ); + instructions.push(oomir::Instruction::Move { + dest: temp_aggregate_var.clone(), + src: value, + }); + } else if let Some(word) = crate::lower1::types::packed_word( + data_types.normalize(tcx, Ty::new_adt(tcx, adt_def, substs), instance), + tcx, + ) { + let mut value = word.zero(); + for (index, operand) in operands.iter().enumerate() { + let field = convert_operand( + operand, + tcx, + instance, + mir, + data_types, + &mut instructions, + ); + value = word.insert( + value, + index, + field, + &format!("{temp_aggregate_var}_packed_{index}"), + &mut instructions, + ); + } + instructions.push(oomir::Instruction::Move { + dest: temp_aggregate_var.clone(), + src: value, + }); + } else if let Some(value) = crate::lower1::types::enum_scalar_variant( + data_types.normalize(tcx, Ty::new_adt(tcx, adt_def, substs), instance), + *variant_idx, + tcx, + ) { + instructions.push(oomir::Instruction::Move { + dest: temp_aggregate_var.clone(), + src: oomir::Operand::Constant(value), + }); + } else if matches!(aggregate_oomir_type, oomir::Type::TaggedI64) { + let value = operands + .iter() + .next() + .map(|operand| { + convert_operand( + operand, + tcx, + instance, + mir, + data_types, + &mut instructions, + ) + }) + .unwrap_or(oomir::Operand::Constant(oomir::Constant::I64(0))); + instructions.push(crate::lower1::types::tagged_value( + temp_aggregate_var.clone(), + value, + variant_idx.as_u32() as u64, + )); + } else if let Some(carrier) = crate::lower1::types::enum_carrier( + data_types.normalize(tcx, Ty::new_adt(tcx, adt_def, substs), instance), + tcx, + ) { + if !carrier.nullable && *variant_idx != carrier.variant { + // Specialized MIR can construct an uninhabited variant. That + // variant has no payload for the remaining live representation. + instructions.push(oomir::Instruction::Unreachable); + return (instructions, jvm_default_value(&aggregate_oomir_type)); + } + let value = if carrier.nullable && variant_idx.as_u32() == 0 { + oomir::Operand::Constant(oomir::Constant::Null( + aggregate_oomir_type.clone(), + )) + } else { + convert_operand( + operands.iter().next().expect("direct enum has one field"), + tcx, + instance, + mir, + data_types, + &mut instructions, + ) + }; + instructions.push(oomir::Instruction::Move { + dest: temp_aggregate_var.clone(), + src: value, + }); + } else if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) && matches!(aggregate_oomir_type, oomir::Type::Pointer(_)) { let operand = operands @@ -430,16 +524,22 @@ impl<'tcx> RvalueContext<'_, 'tcx> { let oomir_fields: Vec<(String, oomir::Type)> = variant .fields .iter() - .filter_map(|f| { + .enumerate() + .filter_map(|(index, f)| { let field_ty = ty_to_oomir_type( f.ty(tcx, substs).skip_norm_wip(), tcx, data_types, instance, ); - field_ty - .has_jvm_value() - .then(|| (f.ident(tcx).to_string(), field_ty)) + field_ty.has_jvm_value().then(|| { + ( + crate::lower1::types::struct_field_name( + tcx, &adt_def, index, + ), + field_ty, + ) + }) }) .collect(); if should_define_data_type && !data_types.contains_key(&jvm_class_name) @@ -457,6 +557,9 @@ impl<'tcx> RvalueContext<'_, 'tcx> { jvm_class_name.clone(), oomir::DataType::Class { fields: oomir_fields.clone(), + kind: crate::lower1::types::adt_class_kind( + tcx, &adt_def, substs, + ), is_abstract: false, methods, super_class: None, @@ -526,7 +629,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { let variant_class_name = format!( "{}${}", base_enum_name, - jvm_names::member_name(&variant_def.name.to_string()) + crate::lower1::types::enum_variant_name(variant_def, tcx) ); let transparent_payload = jvm_subtype_payload_ty(&adt_def, variant_def, substs, tcx); @@ -653,9 +756,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { } } rustc_middle::mir::AggregateKind::RawPtr(pointee_ty, _) => { - let pointee_ty = EarlyBinder::bind(tcx, *pointee_ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let pointee_ty = data_types.normalize(tcx, *pointee_ty, instance); let destination_ty = normalize_unsize_ty( original_dest_place.ty(&mir.local_decls, tcx).ty, tcx, @@ -703,6 +804,9 @@ impl<'tcx> RvalueContext<'_, 'tcx> { }; let data = emit_retyped_slice_data_pointer( data, + crate::lower1::types::ty_to_oomir_type( + element_ty, tcx, data_types, instance, + ), rust_layout_size_operand(element_ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( element_ty, tcx, data_types, instance, @@ -726,26 +830,10 @@ impl<'tcx> RvalueContext<'_, 'tcx> { args: vec![ (backing, oomir::Type::Class("java/lang/Object".to_string())), (offset, oomir::Type::I32), - ( - metadata, - if is_str { - oomir::Type::I32 - } else { - oomir::Type::U64 - }, - ), + (metadata, oomir::Type::U64), ], }); - } else if { - let tail = tcx.struct_tail_for_codegen( - pointee_ty, - TypingEnv::fully_monomorphized(), - ); - tail.is_slice() || tail.is_str() - } { - let data_ty = data - .get_type() - .expect("slice-tailed raw pointer data pointer is typed"); + } else { let metadata = convert_operand( &operands[FieldIdx::from_usize(1)], tcx, @@ -754,97 +842,17 @@ impl<'tcx> RvalueContext<'_, 'tcx> { data_types, &mut instructions, ); - let pointee_size = - crate::lower1::types::layout_size_bytes(tcx, pointee_ty) - .expect("slice-tailed raw pointer has a static prefix layout"); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(temp_aggregate_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retypeWithMetadata".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), data_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ("metadata".to_string(), oomir::Type::U64), - ], - ret: Box::new(aggregate_oomir_type.clone()), - is_static: true, - }, - args: vec![ - data, - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from(pointee_size) - .expect("Rust raw DST prefix layout exceeds u64"), - )), - crate::lower1::types::pointer_view_codec_operand( - pointee_ty, tcx, data_types, instance, - ), - metadata, - ], - }); - } else if matches!(pointee_ty.kind(), TyKind::Dynamic(_, _)) { - let data_ty = data - .get_type() - .expect("raw trait-object aggregate data pointer is typed"); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(temp_aggregate_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "restoreErasedView".to_string(), - method_ty: oomir::Signature { - params: vec![("pointer".to_string(), data_ty)], - ret: Box::new(aggregate_oomir_type.clone()), - is_static: true, - }, - args: vec![data], - }); - } else { - let data_ty = data - .get_type() - .expect("thin raw pointer aggregate data pointer is typed"); - if let Ok(pointee_size) = - crate::lower1::types::layout_size_bytes(tcx, pointee_ty) - { - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(temp_aggregate_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), data_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(aggregate_oomir_type.clone()), - is_static: true, - }, - args: vec![ - data, - oomir::Operand::Constant(oomir::Constant::U64( - u64::try_from(pointee_size) - .expect("Rust raw pointer layout exceeds u64"), - )), - crate::lower1::types::pointer_view_codec_operand( - pointee_ty, tcx, data_types, instance, - ), - ], - }); - } else { - // A generic thin pointer has no metadata operand carrying T's - // layout. Its data pointer came through an erased (`*const ()`) - // view, which records the prior concrete view in Pointer. - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(temp_aggregate_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "restoreErasedView".to_string(), - method_ty: oomir::Signature { - params: vec![("pointer".to_string(), data_ty)], - ret: Box::new(aggregate_oomir_type.clone()), - is_static: true, - }, - args: vec![data], - }); - } + crate::lower1::control_flow::emit_raw_pointer_from_parts( + tcx, + instance, + data_types, + pointee_ty, + aggregate_oomir_type.clone(), + data, + metadata, + Some(temp_aggregate_var.clone()), + &mut instructions, + ); } } _ => { diff --git a/src/lower1/control_flow/rvalue/arithmetic.rs b/src/lower1/control_flow/rvalue/arithmetic.rs index 4e87c89a..39ea6877 100644 --- a/src/lower1/control_flow/rvalue/arithmetic.rs +++ b/src/lower1/control_flow/rvalue/arithmetic.rs @@ -32,20 +32,14 @@ impl<'tcx> RvalueContext<'_, 'tcx> { BinOp::Offset if matches!(oomir_op1.get_type(), Some(oomir::Type::Pointer(_))) => { - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressOffset { dest: Some(temp_binop_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "offset".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), oomir_op1.get_type().unwrap()), - ("elements".to_string(), oomir::Type::I64), - ], - ret: Box::new(oomir_result_type.clone()), - is_static: false, - }, - args: vec![oomir_op2], - operand: oomir_op1, + source: oomir_op1, + count: oomir_op2, + ty: oomir_result_type.clone(), + bytes: false, + wrapping: false, + subtract: false, }); } BinOp::Eq | BinOp::Ne @@ -380,7 +374,6 @@ impl<'tcx> RvalueContext<'_, 'tcx> { convert_operand(operand, tcx, instance, mir, data_types, &mut instructions); let oomir_result_type = get_place_type(original_dest_place, mir, tcx, instance, data_types); - let mut produced_value = false; match operation { UnOp::Not => { @@ -388,123 +381,32 @@ impl<'tcx> RvalueContext<'_, 'tcx> { dest: temp_unop_var.clone(), src: oomir_src_operand, }); - produced_value = true; } UnOp::Neg => { instructions.push(oomir::Instruction::Neg { dest: temp_unop_var.clone(), src: oomir_src_operand, }); - produced_value = true; } UnOp::PtrMetadata => { - let operand_ty = operand.ty(&mir.local_decls, tcx); - let pointee = match operand_ty.kind() { - TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) => *pointee, - _ => operand_ty, - }; - if pointee.is_slice() || pointee.is_str() { - match oomir_src_operand.get_type().unwrap() { - oomir::Type::Slice(_) | oomir::Type::Str => { - instructions.push(oomir::Instruction::GetField { - dest: temp_unop_var.clone(), - object: oomir_src_operand, - field_name: "rustLength".into(), - field_ty: oomir::Type::U64, - owner_class: oomir::SLICE_VIEW_CLASS.into(), - }); - } - ty @ oomir::Type::Pointer(_) => { - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(temp_unop_var.clone()), - class_name: oomir::POINTER_CLASS.into(), - method_name: "metadata".into(), - method_ty: oomir::Signature { - params: vec![("self".into(), ty)], - ret: Box::new(oomir::Type::U64), - is_static: false, - }, - args: vec![], - operand: oomir_src_operand, - }); - } - _ => { - let length_i32 = format!("{temp_unop_var}_i32"); - instructions.push(oomir::Instruction::Length { - dest: length_i32.clone(), - array: oomir_src_operand, - }); - instructions.push(oomir::Instruction::Cast { - dest: temp_unop_var.clone(), - op: oomir::Operand::Variable { - name: length_i32, - ty: oomir::Type::I32, - }, - ty: oomir::Type::U64, - }); - } - } - produced_value = true; - } else if matches!(pointee.kind(), TyKind::Dynamic(..)) - && matches!(oomir_src_operand.get_type(), Some(oomir::Type::Pointer(_))) - { - crate::lower1::control_flow::emit_trait_object_metadata( - oomir_src_operand, - &oomir_result_type, - temp_unop_var.clone(), - &format!("{base_temp_name}_trait_metadata"), - data_types, - &mut instructions, - ); - produced_value = true; - } else { - let tail = tcx - .struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); - if (tail.is_slice() || tail.is_str()) - && matches!( - oomir_src_operand.get_type(), - Some(oomir::Type::Pointer(_)) - ) - { - instructions.push(oomir::Instruction::InvokeVirtual { - dest: Some(temp_unop_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "metadata".to_string(), - method_ty: oomir::Signature { - params: vec![( - "self".to_string(), - oomir_src_operand - .get_type() - .expect("DST metadata pointer must be typed"), - )], - ret: Box::new(oomir::Type::U64), - is_static: false, - }, - args: Vec::new(), - operand: oomir_src_operand, - }); - produced_value = true; - } - } + let value = crate::lower1::control_flow::emit_pointer_metadata( + tcx, + instance, + data_types, + operand.ty(&mir.local_decls, tcx), + oomir_src_operand, + oomir_result_type, + temp_unop_var, + &mut instructions, + ); + return (instructions, value); } } - if produced_value { - result_operand = oomir::Operand::Variable { - name: temp_unop_var, - ty: oomir_result_type, - }; - } else if !oomir_result_type.has_jvm_value() { - result_operand = oomir::Operand::Constant(oomir::Constant::Unit); - } else { - result_operand = get_placeholder_operand( - original_dest_place, - mir, - tcx, - instance, - data_types, - ); - } + result_operand = oomir::Operand::Variable { + name: temp_unop_var, + ty: oomir_result_type, + }; } _ => unreachable!("rvalue routed to arithmetic"), diff --git a/src/lower1/control_flow/rvalue/array_places.rs b/src/lower1/control_flow/rvalue/array_places.rs new file mode 100644 index 00000000..0f2ef56f --- /dev/null +++ b/src/lower1/control_flow/rvalue/array_places.rs @@ -0,0 +1,113 @@ +//! Array locations retain their enclosing owner through field replacement. +use super::*; + +pub(super) fn array_place_storage<'tcx>( + place: &Place<'tcx>, + prefix: &str, + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + definitions: &mut Definitions<'tcx>, + instructions: &mut Vec, +) -> oomir::Operand { + let rust_ty = normalize_unsize_ty(place.ty(&mir.local_decls, tcx).ty, tcx, instance); + if matches!(rust_ty.kind(), TyKind::Array(..)) + && (!place.projection.is_empty() || definitions.local_uses_stable_cell(place.local)) + && let Some(view) = emit_borrowed_projected_array_view( + place, + rust_ty, + &format!("{prefix}_array_storage"), + tcx, + instance, + mir, + definitions, + instructions, + ) + { + return view; + } + let direct = place + .projection + .split_last() + .and_then(|(projection, prefix)| { + if !matches!(projection, ProjectionElem::Deref) { + return None; + } + let reference = Place { + local: place.local, + projection: tcx.mk_place_elems(prefix), + }; + matches!( + get_place_type(&reference, mir, tcx, instance, definitions), + oomir::Type::Slice(_) + ) + .then(|| emit_instructions_to_get_on_own(&reference, tcx, instance, mir, definitions)) + }); + let (name, code, ty) = direct + .unwrap_or_else(|| emit_instructions_to_get_on_own(place, tcx, instance, mir, definitions)); + instructions.extend(code); + oomir::Operand::Variable { name, ty } +} + +pub(super) fn emit_array_pointer( + array_or_slice: oomir::Operand, + index: oomir::Operand, + element_size: oomir::Operand, + codec: oomir::Operand, + pointer_ty: &oomir::Type, + dest: &str, + instructions: &mut Vec, +) -> oomir::Operand { + let source_ty = array_or_slice + .get_type() + .expect("array pointer source must be typed"); + if matches!(source_ty, oomir::Type::Slice(_)) { + let index = if index.get_type() == Some(oomir::Type::U64) { + index + } else { + let index_u64 = format!("{dest}_slice_index_u64"); + instructions.push(oomir::Instruction::Cast { + dest: index_u64.clone(), + op: index, + ty: oomir::Type::U64, + }); + oomir::Operand::Variable { + name: index_u64, + ty: oomir::Type::U64, + } + }; + let base_dest = format!("{dest}_slice_base"); + instructions.push(oomir::Instruction::ViewAddress { + dest: Some(base_dest.clone()), + source: array_or_slice, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: element_size, + codec: codec, + }), + }); + instructions.push(oomir::Instruction::AddressOffset { + dest: Some(dest.to_string()), + source: oomir::Operand::Variable { + name: base_dest, + ty: pointer_ty.clone(), + }, + count: index, + ty: pointer_ty.clone(), + bytes: false, + wrapping: false, + subtract: false, + }); + return oomir::Operand::Variable { + name: dest.to_string(), + ty: pointer_ty.clone(), + }; + } + emit_pointer_factory( + "array", + vec![array_or_slice, index, element_size, codec], + pointer_ty, + dest, + instructions, + ) +} diff --git a/src/lower1/control_flow/rvalue/array_views.rs b/src/lower1/control_flow/rvalue/array_views.rs index 101e01d7..18d3177c 100644 --- a/src/lower1/control_flow/rvalue/array_views.rs +++ b/src/lower1/control_flow/rvalue/array_views.rs @@ -35,7 +35,7 @@ pub(super) fn emit_borrowed_array_view<'tcx>( // typed zero-byte cell so every logical index can materialize the value // while retaining Rust's rule that all ZST element addresses are equal. let backing = if element_type.has_jvm_value() { - let pointer_type = oomir::Type::Pointer(Box::new(element_type.clone())); + let pointer_type = oomir::Type::pointer(element_type.clone()); let template = const_eval::read_zero_sized_constant(tcx, *element_ty, data_types, instance) .unwrap_or_else(|error| { panic!("could not materialize borrowed ZST array element {element_ty:?}: {error}") @@ -114,7 +114,7 @@ pub(super) fn emit_borrowed_projected_array_view<'tcx>( .try_to_target_usize(tcx)?; let array_type = ty_to_oomir_type(array_ty, tcx, data_types, instance); let element_type = ty_to_oomir_type(*element_ty, tcx, data_types, instance); - let array_pointer_type = oomir::Type::Pointer(Box::new(array_type)); + let array_pointer_type = oomir::Type::pointer(array_type); let array_pointer = emit_pointer_to_place( source_place, &array_pointer_type, @@ -125,67 +125,31 @@ pub(super) fn emit_borrowed_projected_array_view<'tcx>( data_types, instructions, ); - let element_pointer_type = oomir::Type::Pointer(Box::new(element_type.clone())); + let element_pointer_type = oomir::Type::pointer(element_type.clone()); let element_pointer_name = format!("{dest}_element_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(element_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), array_pointer_type), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_type.clone()), - is_static: true, - }, - args: vec![ - array_pointer, - rust_layout_size_operand(*element_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: array_pointer, + layout: Box::new(oomir::AddressLayout { + pointer_type: element_pointer_type.clone(), + size: rust_layout_size_operand(*element_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( *element_ty, tcx, data_types, instance, ), - ], - }); - let object = format!("{dest}_object"); - instructions.push(oomir::Instruction::ConstructObject { - dest: object.clone(), - class_name: oomir::SLICE_VIEW_CLASS.to_string(), - args: vec![ - ( - oomir::Operand::Variable { - name: element_pointer_name, - ty: element_pointer_type, - }, - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - oomir::Operand::Constant(oomir::Constant::I32(0)), - oomir::Type::I32, - ), - ( - oomir::Operand::Constant(oomir::Constant::U64(length)), - oomir::Type::U64, - ), - ], + }), }); - let slice_type = oomir::Type::Slice(Box::new(element_type)); - instructions.push(oomir::Instruction::Cast { - dest: dest.to_string(), - op: oomir::Operand::Variable { - name: object, - ty: oomir::Type::Class(oomir::SLICE_VIEW_CLASS.to_string()), + Some(crate::lower1::place::emit_pointer_slice_view( + oomir::Operand::Variable { + name: element_pointer_name, + ty: element_pointer_type, }, - ty: slice_type.clone(), - }); - Some(oomir::Operand::Variable { - name: dest.to_string(), - ty: slice_type, - }) + oomir::Operand::Constant(oomir::Constant::U64(length)), + dest, + instructions, + )) } pub(super) fn emit_raw_array_pointer_unsize<'tcx>( @@ -303,44 +267,22 @@ pub(super) fn emit_raw_array_pointer_unsize<'tcx>( && matches!(target_oomir_ty, oomir::Type::Pointer(_)) { let erased_pointer_dest = format!("{dest}_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(erased_pointer_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_oomir_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ], - ret: Box::new(target_oomir_ty.clone()), - is_static: false, - }, - args: vec![oomir::Operand::Constant(oomir::Constant::U64(0))], - operand: source.clone(), + source: source.clone(), + layout: Box::new(oomir::AddressLayout { + pointer_type: target_oomir_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64(0)), + codec: oomir::Operand::Constant(oomir::Constant::Null(oomir::Type::java_string())), + }), }); let erased_pointer = oomir::Operand::Variable { name: erased_pointer_dest, ty: target_oomir_ty.clone(), }; - if callable_closure_bridge { - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(dest.to_string()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "attachPointeeTraitObjectCarrier".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), target_oomir_ty.clone()), - ("pointee_size".to_string(), oomir::Type::U64), - ("pointee_alignment".to_string(), oomir::Type::U64), - ], - ret: Box::new(target_oomir_ty.clone()), - is_static: true, - }, - args: std::iter::once(erased_pointer) - .chain(layout_args()) - .collect(), - }); - } else if let Some(adapter_class) = callable_fn_def_adapter { + // Raw callable pointers require payload layout and vtable metadata. The call interface + // alone does not supply them. + if let Some(adapter_class) = callable_fn_def_adapter { let carrier_dest = format!("{dest}_carrier"); instructions.push(oomir::Instruction::ConstructObject { dest: carrier_dest.clone(), @@ -450,69 +392,34 @@ pub(super) fn emit_raw_array_pointer_unsize<'tcx>( } let element_oomir_ty = ty_to_oomir_type(target_element, tcx, data_types, instance); - let element_pointer_ty = oomir::Type::Pointer(Box::new(element_oomir_ty)); + let element_pointer_ty = oomir::Type::pointer(element_oomir_ty); let element_pointer = format!("{dest}_element_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(element_pointer.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), source_oomir_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - source, - rust_layout_size_operand(target_element, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: element_pointer_ty.clone(), + size: rust_layout_size_operand(target_element, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( target_element, tcx, data_types, instance, ), - ], + }), }); let length = EarlyBinder::bind(tcx, *length) .instantiate(tcx, instance.args) .skip_norm_wip() .try_to_target_usize(tcx)?; - let slice_object = format!("{dest}_slice_object"); - instructions.push(oomir::Instruction::ConstructObject { - dest: slice_object.clone(), - class_name: oomir::SLICE_VIEW_CLASS.to_string(), - args: vec![ - ( - oomir::Operand::Variable { - name: element_pointer, - ty: element_pointer_ty, - }, - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - oomir::Operand::Constant(oomir::Constant::I32(0)), - oomir::Type::I32, - ), - ( - oomir::Operand::Constant(oomir::Constant::U64(length)), - oomir::Type::U64, - ), - ], - }); - instructions.push(oomir::Instruction::Cast { - dest: dest.to_string(), - op: oomir::Operand::Variable { - name: slice_object, - ty: oomir::Type::Class(oomir::SLICE_VIEW_CLASS.to_string()), + Some(crate::lower1::place::emit_pointer_slice_view( + oomir::Operand::Variable { + name: element_pointer, + ty: element_pointer_ty, }, - ty: target_oomir_ty.clone(), - }); - Some(oomir::Operand::Variable { - name: dest.to_string(), - ty: target_oomir_ty, - }) + oomir::Operand::Constant(oomir::Constant::U64(length)), + dest, + instructions, + )) } diff --git a/src/lower1/control_flow/rvalue/borrows.rs b/src/lower1/control_flow/rvalue/borrows.rs index dbf558d3..392d2d42 100644 --- a/src/lower1/control_flow/rvalue/borrows.rs +++ b/src/lower1/control_flow/rvalue/borrows.rs @@ -11,65 +11,36 @@ impl<'tcx> RvalueContext<'_, 'tcx> { tcx, instance, data_types, - pointer_origins, - available_pointer_locals, .. } = self; let mut instructions = Vec::new(); let result_operand; let base_temp_name = place_to_string(original_dest_place, tcx); match rvalue { - Rvalue::Ref(_region, borrow_kind, source_place) => { - // Check if the result type (destination place type) is a trait object reference + Rvalue::Ref(_, _, source_place) => { let dest_ty = original_dest_place.ty(&mir.local_decls, tcx).ty; - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Rvalue::Ref start: original_dest_place={:?}, dest_ty={:?}, borrow_kind={:?}, source_place={:?}, source_ty={:?}", - original_dest_place, - dest_ty, - borrow_kind, + let reference_ty = ty_to_oomir_type(dest_ty, tcx, data_types, instance); + if matches!(reference_ty, oomir::Type::Pointer(_)) { + result_operand = emit_pointer_to_place( source_place, - source_place.ty(&mir.local_decls, tcx).ty - ) - ); - let reference_oomir_ty = ty_to_oomir_type(dest_ty, tcx, data_types, instance); - let is_trait_object = matches!(dest_ty.kind(), TyKind::Ref(..)) - && matches!(reference_oomir_ty, oomir::Type::Interface(_)); - if matches!(reference_oomir_ty, oomir::Type::Pointer(_)) { - result_operand = reuse_pointer_to_place( - source_place, - &reference_oomir_ty, + &reference_ty, &base_temp_name, - pointer_origins, - available_pointer_locals, + tcx, + instance, + mir, data_types, &mut instructions, - ) - .unwrap_or_else(|| { - emit_pointer_to_place( - source_place, - &reference_oomir_ty, - &base_temp_name, - tcx, - instance, - mir, - data_types, - &mut instructions, - ) - }); + ); return (instructions, result_operand); } - - let source_mir_ty = source_place.ty(&mir.local_decls, tcx).ty; - let normalized_source_mir_ty = normalize_unsize_ty(source_mir_ty, tcx, instance); + let source_ty = source_place.ty(&mir.local_decls, tcx).ty; + let normalized = normalize_unsize_ty(source_ty, tcx, instance); if !source_place.projection.is_empty() - && matches!(normalized_source_mir_ty.kind(), TyKind::Array(..)) - && matches!(reference_oomir_ty, oomir::Type::Slice(_)) - && let Some(array_reference) = emit_borrowed_projected_array_view( + && matches!(normalized.kind(), TyKind::Array(..)) + && matches!(reference_ty, oomir::Type::Slice(_)) + && let Some(view) = emit_borrowed_projected_array_view( source_place, - normalized_source_mir_ty, + normalized, &generate_temp_var_name(data_types, &base_temp_name), tcx, instance, @@ -78,218 +49,31 @@ impl<'tcx> RvalueContext<'_, 'tcx> { &mut instructions, ) { - return (instructions, array_reference); + return (instructions, view); } - - match borrow_kind { - MirBorrowKind::Mut { .. } if !is_trait_object => { - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: Handling Rvalue::Ref(Mut) for place '{}' -> Temp Array Var", - place_to_string(source_place, tcx) - ) - ); - - // 1. Get the value of the place being referenced (the 'pointee'). - let (pointee_value_var_name, pointee_get_instructions, pointee_oomir_type) = - emit_instructions_to_get_on_own( - source_place, - tcx, - instance, - mir, - data_types, - ); - instructions.extend(pointee_get_instructions); // Add instructions to get the value - - if matches!(pointee_oomir_type, oomir::Type::Array(_)) { - let slice_name = generate_temp_var_name(data_types, &base_temp_name); - if let Some(slice) = emit_borrowed_array_view( - oomir::Operand::Variable { - name: pointee_value_var_name.clone(), - ty: pointee_oomir_type.clone(), - }, - source_place.ty(&mir.local_decls, tcx).ty, - &slice_name, - tcx, - instance, - data_types, - &mut instructions, - ) { - return (instructions, slice); - } - } - - if matches!(pointee_oomir_type, oomir::Type::Slice(_) | oomir::Type::Str) { - result_operand = oomir::Operand::Variable { - name: pointee_value_var_name, - ty: pointee_oomir_type, - }; - return (instructions, result_operand); - } - - // 2. Determine the OOMIR type for the array reference itself. - let array_ref_oomir_type = - oomir::Type::MutableReference(Box::new(pointee_oomir_type.clone())); - - // 3. Create a temporary variable name for the new array. - let array_ref_var_name = - generate_temp_var_name(data_types, &base_temp_name); - - // 4. Emit instruction to allocate the single-element array (new T[1]). - instructions.push(oomir::Instruction::NewArray { - dest: array_ref_var_name.clone(), - element_type: pointee_oomir_type.clone(), - size: oomir::Operand::Constant(oomir::Constant::I32(1)), - }); - - // 5. Emit instruction to store the pointee's value into the array's first element. - let pointee_value_operand = oomir::Operand::Variable { - name: pointee_value_var_name, - ty: pointee_oomir_type, - }; - instructions.push(oomir::Instruction::ArrayStore { - array: oomir::Operand::Variable { - name: array_ref_var_name.clone(), - ty: array_ref_oomir_type.clone(), - }, - index: oomir::Operand::Constant(oomir::Constant::I32(0)), - value: pointee_value_operand, - copy_value: false, - }); - - // 6. The result is the reference to the newly created array. - result_operand = oomir::Operand::Variable { - name: array_ref_var_name, - ty: array_ref_oomir_type, - }; - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: -> Temp Array Var '{}' ({:?})", - result_operand.get_name().unwrap_or(""), - result_operand.get_type() - ) - ); - } - MirBorrowKind::Mut { .. } - | MirBorrowKind::Shared - | MirBorrowKind::Fake { .. } => { - // Treat Fake like Shared (used for closures etc.) - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: Handling Rvalue::Ref({:?}) for place '{}' -> Direct Value", - borrow_kind, - place_to_string(source_place, tcx) - ) - ); - - let source_mir_ty = source_place.ty(&mir.local_decls, tcx).ty; - if let TyKind::Closure(_, closure_args) = source_mir_ty.kind() { - let closure_oomir_type = - ty_to_oomir_type(source_mir_ty, tcx, data_types, instance); - if let oomir::Type::Class(class_name) = closure_oomir_type.clone() { - let has_captures = closure_args - .as_closure() - .upvar_tys() - .iter() - .next() - .is_some(); - if has_captures { - let (temp_var_name, get_instructions, temp_var_type) = - emit_instructions_to_get_on_own( - source_place, - tcx, - instance, - mir, - data_types, - ); - instructions.extend(get_instructions); - result_operand = oomir::Operand::Variable { - name: temp_var_name, - ty: temp_var_type, - }; - } else { - let closure_var_name = - generate_temp_var_name(data_types, &base_temp_name); - instructions.push(oomir::Instruction::ConstructObject { - dest: closure_var_name.clone(), - class_name, - args: Vec::new(), - }); - result_operand = oomir::Operand::Variable { - name: closure_var_name, - ty: closure_oomir_type, - }; - } - } else { - result_operand = get_placeholder_operand( - original_dest_place, - mir, - tcx, - instance, - data_types, - ); - } - } else { - // 1. Get the value/reference of the place being borrowed directly. - // `emit_instructions_to_get_on_own` handles loading/accessing the value. - let ( - pointee_value_var_name, - pointee_get_instructions, - pointee_oomir_type, - ) = emit_instructions_to_get_on_own( - source_place, - tcx, - instance, - mir, - data_types, - ); - - // 2. Add the instructions needed to get this value. - instructions.extend(pointee_get_instructions); - - if matches!(pointee_oomir_type, oomir::Type::Array(_)) { - let slice_name = - generate_temp_var_name(data_types, &base_temp_name); - if let Some(slice) = emit_borrowed_array_view( - oomir::Operand::Variable { - name: pointee_value_var_name.clone(), - ty: pointee_oomir_type.clone(), - }, - source_mir_ty, - &slice_name, - tcx, - instance, - data_types, - &mut instructions, - ) { - return (instructions, slice); - } - } - - // 3. The result *is* the operand representing the borrowed value itself. - // No array wrapping is done. - result_operand = oomir::Operand::Variable { - name: pointee_value_var_name, - ty: pointee_oomir_type, - }; - } - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "mir-lowering", - format!( - "Info: -> Direct Value Operand '{}' ({:?})", - result_operand.get_name().unwrap_or(""), - result_operand.get_type() - ) - ); - } + // These carriers already identify their storage. Mutable borrows use the same + // representation. + let (name, read, ty) = + emit_instructions_to_get_on_own(source_place, tcx, instance, mir, data_types); + instructions.extend(read); + let source = oomir::Operand::Variable { + name, + ty: ty.clone(), + }; + if matches!(ty, oomir::Type::Array(_)) + && let Some(view) = emit_borrowed_array_view( + source.clone(), + source_ty, + &generate_temp_var_name(data_types, &base_temp_name), + tcx, + instance, + data_types, + &mut instructions, + ) + { + return (instructions, view); } + result_operand = source; } Rvalue::RawPtr(kind, place) => { @@ -322,59 +106,31 @@ impl<'tcx> RvalueContext<'_, 'tcx> { let pointer_oomir_type = get_place_type(original_dest_place, mir, tcx, instance, data_types); if !matches!(pointer_oomir_type, oomir::Type::Pointer(_)) { - let pointer_mir_ty = normalize_unsize_ty( - original_dest_place.ty(&mir.local_decls, tcx).ty, - tcx, - instance, - ); - let pointee_mir_ty = pointer_pointee_ty(pointer_mir_ty); - let projects_struct_tail = - place.projection.split_last().is_some_and(|(last, prefix)| { - if !matches!(last, ProjectionElem::Field(..)) { - return false; - } - let base_place = Place { - local: place.local, - projection: tcx.mk_place_elems(prefix), - }; - let base_ty = normalize_unsize_ty( - base_place.ty(&mir.local_decls, tcx).ty, - tcx, - instance, - ); - crate::lower1::place::has_slice_or_str_struct_tail(tcx, base_ty) - }); - if projects_struct_tail - && matches!(pointee_mir_ty.kind(), TyKind::Slice(_) | TyKind::Str) - { - let storage_pointer_ty = oomir::Type::Pointer(Box::new( - ty_to_oomir_type(pointee_mir_ty, tcx, data_types, instance), - )); - let storage = emit_pointer_to_place( + if matches!( + pointer_oomir_type, + oomir::Type::Slice(_) | oomir::Type::Str + ) { + result_operand = emit_pointer_to_place( place, - &storage_pointer_ty, - &format!("{base_temp_name}_unsized_storage"), + &pointer_oomir_type, + &base_temp_name, tcx, instance, mir, data_types, &mut instructions, ); - result_operand = emit_slice_pointer_carrier( - pointer_mir_ty, - storage, - &format!("{base_temp_name}_unsized_pointer"), - tcx, - instance, - data_types, - &mut instructions, - ) - .expect("slice-like raw pointer must have a slice carrier"); return (instructions, result_operand); } + let pointer_mir_ty = normalize_unsize_ty( + original_dest_place.ty(&mir.local_decls, tcx).ty, + tcx, + instance, + ); + let pointee_mir_ty = pointer_pointee_ty(pointer_mir_ty, tcx); if matches!(pointee_mir_ty.kind(), TyKind::Dynamic(..)) { let storage_pointer_ty = - oomir::Type::Pointer(Box::new(pointer_oomir_type.clone())); + oomir::Type::pointer(pointer_oomir_type.clone()); result_operand = emit_pointer_to_place( place, &storage_pointer_ty, @@ -400,27 +156,16 @@ impl<'tcx> RvalueContext<'_, 'tcx> { result_operand = oomir::Operand::Variable { name, ty }; return (instructions, result_operand); } - result_operand = reuse_pointer_to_place( + result_operand = emit_pointer_to_place( place, &pointer_oomir_type, &base_temp_name, - pointer_origins, - available_pointer_locals, + tcx, + instance, + mir, data_types, &mut instructions, - ) - .unwrap_or_else(|| { - emit_pointer_to_place( - place, - &pointer_oomir_type, - &base_temp_name, - tcx, - instance, - mir, - data_types, - &mut instructions, - ) - }); + ); } } } diff --git a/src/lower1/control_flow/rvalue/callable_handles.rs b/src/lower1/control_flow/rvalue/callable_handles.rs new file mode 100644 index 00000000..1cc34a7b --- /dev/null +++ b/src/lower1/control_flow/rvalue/callable_handles.rs @@ -0,0 +1,20 @@ +//! Constant callable identities remain symbolic until the JVM links their site. +use super::*; + +pub(crate) fn callable_handle( + signature: &oomir::Signature, + interface: &str, + owner: String, + name: String, + target_is_interface: bool, +) -> oomir::Constant { + oomir::Constant::FunctionHandle { + interface_name: interface.into(), + owner, + name, + descriptor: signature + .component_signature() + .to_jvm_descriptor_with_explicit_params(), + interface: target_is_interface, + } +} diff --git a/src/lower1/control_flow/rvalue/casts.rs b/src/lower1/control_flow/rvalue/casts.rs index c2b244cd..7f54fa53 100644 --- a/src/lower1/control_flow/rvalue/casts.rs +++ b/src/lower1/control_flow/rvalue/casts.rs @@ -94,6 +94,8 @@ impl<'tcx> RvalueContext<'_, 'tcx> { } if let Some(FnPointerTarget::Static(target)) = &callable_target && let Some(target_class_name) = &target.class_to_call_on + && (!signature.needs_component_abi() + || oomir::component_method(target_class_name, &target.method_name)) { instructions.push(oomir::Instruction::CreateFunctionPointer { dest: temp_cast_var.clone(), @@ -164,6 +166,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { if let Ok(bridge_class) = ensure_erased_receiver_fn_pointer_bridge( data_types, + source_mir_ty, &source_signature, &source_interface, &target_signature, @@ -228,7 +231,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { ); if matches!(source_mir_ty.kind(), TyKind::RawPtr(..) | TyKind::Ref(..)) && matches!( - pointer_pointee_ty(source_mir_ty).kind(), + pointer_pointee_ty(source_mir_ty, tcx).kind(), TyKind::Dynamic(..) ) && matches!( @@ -236,11 +239,11 @@ impl<'tcx> RvalueContext<'_, 'tcx> { TyKind::RawPtr(..) | TyKind::Ref(..) ) && !matches!( - pointer_pointee_ty(resolved_target_mir_ty).kind(), + pointer_pointee_ty(resolved_target_mir_ty, tcx).kind(), TyKind::Dynamic(..) ) && crate::lower1::types::is_codegen_sized( - pointer_pointee_ty(resolved_target_mir_ty), + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, ) && matches!(oomir_target_type, oomir::Type::Pointer(_)) @@ -262,7 +265,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { raw_oomir_operand, pointer_view_size_operand(*target_mir_ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( - pointer_pointee_ty(*target_mir_ty), + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, data_types, instance, @@ -422,7 +425,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { oomir_operand, pointer_view_size_operand(*target_mir_ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( - pointer_pointee_ty(*target_mir_ty), + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, data_types, instance, @@ -588,7 +591,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { address, pointer_view_size_operand(*target_mir_ty, tcx, instance), crate::lower1::types::pointer_view_codec_operand( - pointer_pointee_ty(*target_mir_ty), + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, data_types, instance, diff --git a/src/lower1/control_flow/rvalue/casts/representations.rs b/src/lower1/control_flow/rvalue/casts/representations.rs index 4b81751a..652ce176 100644 --- a/src/lower1/control_flow/rvalue/casts/representations.rs +++ b/src/lower1/control_flow/rvalue/casts/representations.rs @@ -38,6 +38,40 @@ impl<'tcx> PreparedCast<'_, 'tcx> { callable_fn_def_adapter, trait_object_adapter, } = self; + if matches!( + cast_kind, + CastKind::PointerCoercion(PointerCoercion::Unsize, _) + ) && matches!(source_mir_ty.kind(), TyKind::Adt(..)) + && let Some(result) = emit_unsize_value( + source_mir_ty, + *target_mir_ty, + oomir_operand.clone(), + &temp_cast_var, + tcx, + instance, + data_types, + &mut instructions, + ) + { + return (instructions, result); + } + if matches!(cast_kind, CastKind::Transmute) + && oomir_source_type == oomir_target_type + && matches!(oomir_target_type, oomir::Type::Slice(_) | oomir::Type::Str) + && same_view_pointee(source_mir_ty, resolved_target_mir_ty, tcx) + { + return (instructions, oomir_operand); + } + if matches!(cast_kind, CastKind::Transmute) + && let Some(result) = crate::lower1::types::emit_direct_transmute( + oomir_operand.clone(), + &oomir_target_type, + &temp_cast_var, + &mut instructions, + ) + { + return (instructions, result); + } let result_operand; let exact_transmute_helper = matches!(cast_kind, CastKind::Transmute) .then(|| { @@ -126,82 +160,59 @@ impl<'tcx> PreparedCast<'_, 'tcx> { }); } else if matches!(&oomir_source_type, oomir::Type::Slice(_)) && matches!(&oomir_target_type, oomir::Type::Slice(_)) - && pointer_pointee_ty(source_mir_ty).is_slice() - && pointer_pointee_ty(*target_mir_ty).is_slice() - && pointer_pointee_ty(source_mir_ty).sequence_element_type(tcx) - != pointer_pointee_ty(*target_mir_ty).sequence_element_type(tcx) + && pointer_pointee_ty(source_mir_ty, tcx).is_slice() + && pointer_pointee_ty(resolved_target_mir_ty, tcx).is_slice() + && pointer_pointee_ty(source_mir_ty, tcx).sequence_element_type(tcx) + != pointer_pointee_ty(resolved_target_mir_ty, tcx).sequence_element_type(tcx) { let source_element = EarlyBinder::bind( tcx, - pointer_pointee_ty(source_mir_ty).sequence_element_type(tcx), + pointer_pointee_ty(source_mir_ty, tcx).sequence_element_type(tcx), ) .instantiate(tcx, instance.args) .skip_norm_wip(); let target_element = EarlyBinder::bind( tcx, - pointer_pointee_ty(*target_mir_ty).sequence_element_type(tcx), + pointer_pointee_ty(resolved_target_mir_ty, tcx).sequence_element_type(tcx), ) .instantiate(tcx, instance.args) .skip_norm_wip(); let source_element_oomir = ty_to_oomir_type(source_element, tcx, data_types, instance); let target_element_oomir = ty_to_oomir_type(target_element, tcx, data_types, instance); - let source_pointer_ty = oomir::Type::Pointer(Box::new(source_element_oomir)); - let target_pointer_ty = oomir::Type::Pointer(Box::new(target_element_oomir)); + let source_pointer_ty = oomir::Type::pointer(source_element_oomir); + let target_pointer_ty = oomir::Type::pointer(target_element_oomir); let data_name = format!("{temp_cast_var}_slice_data"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::ViewAddress { dest: Some(data_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(source_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - oomir_operand.clone(), - rust_layout_size_operand(source_element, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: oomir_operand.clone(), + layout: Box::new(oomir::AddressLayout { + pointer_type: source_pointer_ty.clone(), + size: rust_layout_size_operand(source_element, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( source_element, tcx, data_types, instance, ), - ], + }), }); let retyped_name = format!("{temp_cast_var}_slice_retyped"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(retyped_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_pointer_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(target_pointer_ty.clone()), - is_static: false, + source: oomir::Operand::Variable { + name: data_name, + ty: source_pointer_ty, }, - args: vec![ - rust_layout_size_operand(target_element, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + layout: Box::new(oomir::AddressLayout { + pointer_type: target_pointer_ty.clone(), + size: rust_layout_size_operand(target_element, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( target_element, tcx, data_types, instance, ), - ], - operand: oomir::Operand::Variable { - name: data_name, - ty: source_pointer_ty, - }, + }), }); let length_name = format!("{temp_cast_var}_slice_length"); instructions.push(oomir::Instruction::GetField { @@ -211,39 +222,18 @@ impl<'tcx> PreparedCast<'_, 'tcx> { field_ty: oomir::Type::U64, owner_class: oomir::SLICE_VIEW_CLASS.to_string(), }); - let slice_object_name = format!("{temp_cast_var}_slice_object"); - instructions.push(oomir::Instruction::ConstructObject { - dest: slice_object_name.clone(), - class_name: oomir::SLICE_VIEW_CLASS.to_string(), - args: vec![ - ( - oomir::Operand::Variable { - name: retyped_name, - ty: target_pointer_ty, - }, - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - oomir::Operand::Constant(oomir::Constant::I32(0)), - oomir::Type::I32, - ), - ( - oomir::Operand::Variable { - name: length_name, - ty: oomir::Type::U64, - }, - oomir::Type::U64, - ), - ], - }); - instructions.push(oomir::Instruction::Cast { - op: oomir::Operand::Variable { - name: slice_object_name, - ty: oomir::Type::Class(oomir::SLICE_VIEW_CLASS.to_string()), + crate::lower1::place::emit_pointer_slice_view( + oomir::Operand::Variable { + name: retyped_name, + ty: target_pointer_ty, }, - ty: oomir_target_type.clone(), - dest: temp_cast_var.clone(), - }); + oomir::Operand::Variable { + name: length_name, + ty: oomir::Type::U64, + }, + &temp_cast_var, + &mut instructions, + ); } else if matches!(oomir_source_type, oomir::Type::Slice(_) | oomir::Type::Str) && matches!(oomir_target_type, oomir::Type::Pointer(_)) { @@ -254,91 +244,59 @@ impl<'tcx> PreparedCast<'_, 'tcx> { } else if pointee.is_str() { tcx.types.u8 } else { - pointer_pointee_ty(*target_mir_ty) + pointer_pointee_ty(resolved_target_mir_ty, tcx) } } - _ => pointer_pointee_ty(*target_mir_ty), + _ => pointer_pointee_ty(resolved_target_mir_ty, tcx), }; let source_element_size = crate::lower1::types::layout_size_bytes(tcx, source_pointee) .expect("fat pointer element has a concrete layout"); + // DST targets require the slice length. Only sized targets can discard it. + let source_pointer_ty = if pointer_pointee_ty(resolved_target_mir_ty, tcx) + .is_sized(tcx, TypingEnv::fully_monomorphized()) + { + oomir::Type::pointer(ty_to_oomir_type(source_pointee, tcx, data_types, instance)) + } else { + oomir_target_type.clone() + }; let data_pointer = format!("{temp_cast_var}_fat_data"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::ViewAddress { dest: Some(data_pointer.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir_target_type.clone()), - is_static: true, - }, - args: vec![ - oomir_operand, - oomir::Operand::Constant(oomir::Constant::U64( + source: oomir_operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: source_pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from(source_element_size) .expect("Rust slice element layout exceeds u64"), )), - crate::lower1::types::pointer_view_codec_operand( + codec: crate::lower1::types::pointer_view_codec_operand( source_pointee, tcx, data_types, instance, ), - ], + }), }); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(temp_cast_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), oomir_target_type.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir_target_type.clone()), - is_static: false, + source: oomir::Operand::Variable { + name: data_pointer, + ty: source_pointer_ty, }, - args: vec![ - pointer_view_size_operand(*target_mir_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( - pointer_pointee_ty(*target_mir_ty), + layout: Box::new(oomir::AddressLayout { + pointer_type: oomir_target_type.clone(), + size: pointer_view_size_operand(*target_mir_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, data_types, instance, ), - ], - operand: oomir::Operand::Variable { - name: data_pointer, - ty: oomir_target_type.clone(), - }, + }), }); } else if matches!( cast_kind, CastKind::PointerCoercion(PointerCoercion::Unsize, _) - ) && matches!(source_mir_ty.kind(), TyKind::Adt(..)) - && emit_unsize_value( - source_mir_ty, - *target_mir_ty, - oomir_operand.clone(), - &temp_cast_var, - tcx, - instance, - data_types, - &mut instructions, - ) - .is_some() - { - // The recursive helper emitted the complete target wrapper. - } else if matches!( - cast_kind, - CastKind::PointerCoercion(PointerCoercion::Unsize, _) ) && emit_struct_trait_tail_pointer_unsize( source_mir_ty, resolved_target_mir_ty, @@ -364,7 +322,7 @@ impl<'tcx> PreparedCast<'_, 'tcx> { instance, ) { let source_pointee = - normalize_unsize_ty(pointer_pointee_ty(source_mir_ty), tcx, instance); + normalize_unsize_ty(pointer_pointee_ty(source_mir_ty, tcx), tcx, instance); let source_tail = tcx.struct_tail_for_codegen(source_pointee, TypingEnv::fully_monomorphized()); let TyKind::Array(element_ty, length) = source_tail.kind() else { @@ -373,8 +331,11 @@ impl<'tcx> PreparedCast<'_, 'tcx> { let length = length .try_to_target_usize(tcx) .expect("struct-tail array length is concrete"); - let target_pointee = - normalize_unsize_ty(pointer_pointee_ty(resolved_target_mir_ty), tcx, instance); + let target_pointee = normalize_unsize_ty( + pointer_pointee_ty(resolved_target_mir_ty, tcx), + tcx, + instance, + ); let target_tail = tcx.struct_tail_for_codegen(target_pointee, TypingEnv::fully_monomorphized()); let tail_view_class = if target_tail.is_str() { @@ -418,7 +379,7 @@ impl<'tcx> PreparedCast<'_, 'tcx> { && matches!(oomir_target_type, oomir::Type::Pointer(_)) { if let Some(pointer) = emit_struct_tail_reborrow_view( - pointer_pointee_ty(resolved_target_mir_ty), + pointer_pointee_ty(resolved_target_mir_ty, tcx), oomir_operand.clone(), &oomir_target_type, &temp_cast_var, @@ -460,23 +421,19 @@ impl<'tcx> PreparedCast<'_, 'tcx> { ); } if matches!( - pointer_pointee_ty(*target_mir_ty).kind(), + pointer_pointee_ty(resolved_target_mir_ty, tcx).kind(), TyKind::Dynamic(_, _) ) { - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(temp_cast_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), oomir_source_type), - ("view_size".to_string(), oomir::Type::U64), - ], - ret: Box::new(oomir_target_type.clone()), - is_static: false, - }, - args: vec![oomir::Operand::Constant(oomir::Constant::U64(0))], - operand: oomir_operand, + source: oomir_operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: oomir_target_type.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64(0)), + codec: oomir::Operand::Constant(oomir::Constant::Null( + oomir::Type::java_string(), + )), + }), }); return ( instructions, @@ -486,29 +443,19 @@ impl<'tcx> PreparedCast<'_, 'tcx> { }, ); } - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(temp_cast_var.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), oomir_source_type.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(oomir_target_type.clone()), - is_static: false, - }, - args: vec![ - pointer_view_size_operand(*target_mir_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( - pointer_pointee_ty(*target_mir_ty), + source: oomir_operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: oomir_target_type.clone(), + size: pointer_view_size_operand(*target_mir_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( + pointer_pointee_ty(resolved_target_mir_ty, tcx), tcx, data_types, instance, ), - ], - operand: oomir_operand, + }), }); } else if matches!( (&oomir_source_type, &oomir_target_type), @@ -546,10 +493,6 @@ impl<'tcx> PreparedCast<'_, 'tcx> { && (matches!( oomir_source_type, oomir::Type::Array(_) | oomir::Type::Slice(_) - ) || matches!( - &oomir_source_type, - oomir::Type::MutableReference(inner) - if matches!(inner.as_ref(), oomir::Type::Array(_)) ) || matches!( &oomir_source_type, oomir::Type::Pointer(inner) @@ -558,7 +501,7 @@ impl<'tcx> PreparedCast<'_, 'tcx> { { let slice_source = match &oomir_source_type { oomir::Type::Pointer(inner) if matches!(inner.as_ref(), oomir::Type::Array(_)) => { - let source_array_ty = pointer_pointee_ty(source_mir_ty); + let source_array_ty = pointer_pointee_ty(source_mir_ty, tcx); let TyKind::Array(element_rust_ty, length) = source_array_ty.kind() else { unreachable!("array pointer OOMIR carrier has non-array Rust pointee") }; @@ -566,85 +509,37 @@ impl<'tcx> PreparedCast<'_, 'tcx> { oomir::Type::Slice(element) => element.as_ref().clone(), _ => unreachable!(), }; - let element_pointer_ty = oomir::Type::Pointer(Box::new(element_oomir_ty)); + let element_pointer_ty = oomir::Type::pointer(element_oomir_ty); let element_pointer_name = format!("{temp_cast_var}_element_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(element_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), oomir_source_type.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - oomir_operand, - rust_layout_size_operand(*element_rust_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + source: oomir_operand, + layout: Box::new(oomir::AddressLayout { + pointer_type: element_pointer_ty.clone(), + size: rust_layout_size_operand(*element_rust_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( *element_rust_ty, tcx, data_types, instance, ), - ], + }), }); let length = EarlyBinder::bind(tcx, *length) .instantiate(tcx, instance.args) .skip_norm_wip() .try_to_target_usize(tcx) .expect("array-to-slice coercion length must be concrete"); - let slice_object_name = format!("{temp_cast_var}_slice_object"); - instructions.push(oomir::Instruction::ConstructObject { - dest: slice_object_name.clone(), - class_name: oomir::SLICE_VIEW_CLASS.to_string(), - args: vec![ - ( - oomir::Operand::Variable { - name: element_pointer_name, - ty: element_pointer_ty, - }, - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - oomir::Operand::Constant(oomir::Constant::I32(0)), - oomir::Type::I32, - ), - ( - oomir::Operand::Constant(oomir::Constant::U64(length)), - oomir::Type::U64, - ), - ], - }); - instructions.push(oomir::Instruction::Cast { - op: oomir::Operand::Variable { - name: slice_object_name, - ty: oomir::Type::Class(oomir::SLICE_VIEW_CLASS.to_string()), - }, - ty: oomir_target_type.clone(), - dest: temp_cast_var.clone(), - }); - None - } - oomir::Type::MutableReference(inner) - if matches!(inner.as_ref(), oomir::Type::Array(_)) => - { - let unwrapped_name = format!("{}_array", temp_cast_var); - instructions.push(oomir::Instruction::ArrayGet { - dest: unwrapped_name.clone(), - array: oomir_operand, - index: oomir::Operand::Constant(oomir::Constant::I32(0)), - }); - Some(( + crate::lower1::place::emit_pointer_slice_view( oomir::Operand::Variable { - name: unwrapped_name, - ty: inner.as_ref().clone(), + name: element_pointer_name, + ty: element_pointer_ty, }, - inner.as_ref().clone(), - )) + oomir::Operand::Constant(oomir::Constant::U64(length)), + &temp_cast_var, + &mut instructions, + ); + None } _ => Some((oomir_operand, oomir_source_type.clone())), }; @@ -751,3 +646,14 @@ impl<'tcx> PreparedCast<'_, 'tcx> { (instructions, result_operand) } } + +fn same_view_pointee<'tcx>(source: Ty<'tcx>, target: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool { + let pointee = |ty: Ty<'tcx>| match ty.kind() { + TyKind::Ref(_, inner, _) | TyKind::RawPtr(inner, _) => Some(*inner), + TyKind::Adt(def, args) if crate::lower1::is_non_null_lang_item(tcx, def.did()) => { + Some(args.type_at(0)) + } + _ => None, + }; + pointee(source).is_some_and(|source| pointee(target) == Some(source)) +} diff --git a/src/lower1/control_flow/rvalue/closures.rs b/src/lower1/control_flow/rvalue/closures.rs index 47f2ef83..4a73e946 100644 --- a/src/lower1/control_flow/rvalue/closures.rs +++ b/src/lower1/control_flow/rvalue/closures.rs @@ -1,13 +1,12 @@ use super::*; -pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( +pub(crate) fn ensure_closure_fn_pointer_bridge<'tcx>( data_types: &mut Definitions<'tcx>, closure_instance: Instance<'tcx>, signature: &oomir::Signature, - interface_name: &str, tcx: TyCtxt<'tcx>, instance_context: Instance<'tcx>, -) -> String { +) -> (String, String) { let typing_env = TypingEnv::fully_monomorphized(); let closure_ty = closure_instance.ty(tcx, typing_env); let TyKind::Closure(_, closure_args) = closure_ty.kind() else { @@ -31,21 +30,15 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( let target_owner = target.class_to_call_on.expect("closure has a JVM owner"); let target_method = target.method_name; let descriptor = signature.to_jvm_descriptor_with_explicit_params(); - let target_name = format!("{target_owner}::{target_method}"); - let identity = format!("{target_name}:{descriptor}"); - let base_name = jvm_names::path_segment(&target_method); - let interface_token = interface_name.rsplit('/').next().unwrap_or(interface_name); - let local_name = crate::stable_hash::readable_or_hashed_name( - "ClosureFnPtrImpl", - &format!("{base_name}_{interface_token}"), - &identity, - 180, + let method_name = format!( + "_fp${}", + crate::stable_hash::short_hash(&format!("{target_method}:{descriptor}"), 16) ); - let class_name = jvm_names::synthetic_class_for_instance(tcx, instance_context, local_name); - - let mut method_params = Vec::with_capacity(signature.params.len() + 1); - method_params.push(("self".to_string(), oomir::Type::Class(class_name.clone()))); - method_params.extend(signature.params.iter().cloned()); + if matches!(data_types.get(&target_owner), Some(oomir::DataType::Class { methods, .. } | oomir::DataType::Interface { methods, .. }) if methods.contains_key(&method_name)) + { + return (target_owner, method_name); + } + let method_params = signature.params.clone(); let tuple_dest = "_closure_args".to_string(); let tuple_args = signature @@ -56,7 +49,7 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( .map(|(index, (_, ty))| { ( oomir::Operand::Variable { - name: format!("_{}", index + 2), + name: format!("_{}", index + 1), ty: ty.clone(), }, ty.clone(), @@ -65,15 +58,15 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( .collect::>(); let mut instructions = Vec::new(); let closure_call_args = if tuple_oomir_ty.has_jvm_value() { - let tuple_class = tuple_oomir_ty - .get_class_name() - .expect("non-unit closure argument tuple is represented by a JVM class") - .to_string(); - instructions.push(oomir::Instruction::ConstructObject { - dest: tuple_dest.clone(), - class_name: tuple_class, - args: tuple_args, - }); + crate::lower1::types::tuple_value( + tuple_ty, + tuple_args, + &tuple_dest, + tcx, + data_types, + instance_context, + &mut instructions, + ); vec![oomir::Operand::Variable { name: tuple_dest, ty: tuple_oomir_ty.clone(), @@ -90,7 +83,7 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( let call_dest = signature.ret.has_jvm_value().then(|| "_ret".to_string()); instructions.push(oomir::Instruction::InvokeRustStatic { dest: call_dest.clone(), - class_name: target_owner, + class_name: target_owner.clone(), method_name: target_method, method_ty: oomir::Signature { params: closure_method_params, @@ -107,13 +100,13 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( }); let call_method = oomir::DataTypeMethod::Function(oomir::Function { - name: "call".to_string(), + name: method_name.clone(), owner_class: None, debug_variables: Vec::new(), signature: oomir::Signature { params: method_params, ret: signature.ret.clone(), - is_static: false, + is_static: true, }, body: oomir::CodeBlock { entry: "bb0".to_string(), @@ -127,53 +120,21 @@ pub(crate) fn ensure_closure_fn_pointer_adapter_class<'tcx>( } .into(), }); - let relative_method_name = format!("call{}", oomir::RELATIVE_POINTER_METHOD_SUFFIX); - let relative_call_method = signature.supports_relative_pointer_abi().then(|| { - let oomir::DataTypeMethod::Function(mut function) = call_method.clone() else { - unreachable!("closure function-pointer adapters are OOMIR functions"); - }; - function.name = relative_method_name.clone(); - oomir::DataTypeMethod::Function(function) - }); - - match data_types.get_mut(&class_name) { - Some(oomir::DataType::Class { - methods, - interfaces, - .. - }) => { - methods.entry("call".to_string()).or_insert(call_method); - if let Some(relative_call_method) = relative_call_method { - methods - .entry(relative_method_name) - .or_insert(relative_call_method); - } - if !interfaces.iter().any(|name| name == interface_name) { - interfaces.push(interface_name.to_string()); - } - } - Some(oomir::DataType::Interface { .. }) => { - panic!("closure function-pointer adapter name collided with an interface") - } - None => { - let mut methods = HashMap::from_iter([("call".to_string(), call_method)]); - if let Some(relative_call_method) = relative_call_method { - methods.insert(relative_method_name, relative_call_method); - } - data_types.insert( - class_name.clone(), - oomir::DataType::Class { - fields: Vec::new(), - is_abstract: false, - methods, - super_class: Some("java/lang/Object".to_string()), - interfaces: vec![interface_name.to_string()], - }, - ); - } - } - class_name + let entry = data_types + .entry(target_owner.clone()) + .or_insert_with(|| oomir::DataType::Class { + fields: Vec::new(), + kind: oomir::ClassKind::Static, + is_abstract: false, + methods: HashMap::default(), + super_class: None, + interfaces: Vec::new(), + }); + let (oomir::DataType::Class { methods, .. } | oomir::DataType::Interface { methods, .. }) = + entry; + methods.entry(method_name.clone()).or_insert(call_method); + (target_owner, method_name) } pub(super) fn ensure_non_capturing_closure_fn_pointer_bridge<'tcx>( @@ -192,142 +153,56 @@ pub(super) fn ensure_non_capturing_closure_fn_pointer_bridge<'tcx>( closure_args.as_closure().upvar_tys().is_empty(), "only non-capturing closures can become function pointers" ); - let oomir::Type::Class(closure_class) = - ty_to_oomir_type(closure_ty, tcx, data_types, instance_context) - else { - panic!("closure type did not lower to a JVM class"); + let target = data_types.function_name(tcx, closure_instance); + let owner = target + .class_to_call_on + .clone() + .expect("closure has a JVM owner"); + let method_name = target.method_name.clone(); + let entry = data_types + .entry(owner.clone()) + .or_insert_with(|| oomir::DataType::Class { + fields: Vec::new(), + kind: oomir::ClassKind::Static, + is_abstract: false, + methods: HashMap::default(), + super_class: Some("java/lang/Object".to_string()), + interfaces: Vec::new(), + }); + let oomir::DataType::Class { methods, .. } = entry else { + panic!("closure implementation holder must be a class"); }; - let method_name = "_fn_ptr_call".to_string(); - let already_defined = matches!( - data_types.get(&closure_class), - Some(oomir::DataType::Class { methods, .. }) if methods.contains_key(&method_name) - ); - if already_defined { - return (closure_class, method_name); - } - - // Optimised MIR can remove the standalone closure mono item when its only - // use is this coercion. Keep the implementation alongside the bridge in - // the closure's existing class so the LambdaMetafactory target never - // depends on that separate reachability decision. - let implementation_method_name = "_fn_ptr_impl".to_string(); - let implementation_already_defined = matches!( - data_types.get(&closure_class), - Some(oomir::DataType::Class { methods, .. }) - if methods.contains_key(&implementation_method_name) - ); - if !implementation_already_defined { - let closure_mir = tcx.instance_mir(closure_instance.def); + // MIR can omit a closure mono item used only by coercion. The adapter still requires its + // canonical static body. + if !methods.contains_key(&method_name) { let implementation = crate::lower1::mir_to_oomir( tcx, closure_instance, - closure_mir, - Some(crate::lower1::naming::FnNameData { - class_to_call_on: Some(closure_class.clone()), - method_name: implementation_method_name.clone(), - }), + tcx.instance_mir(closure_instance.def), + Some(target), true, data_types, external_interfaces, ); - let Some(oomir::DataType::Class { methods, .. }) = data_types.get_mut(&closure_class) - else { - panic!("closure class disappeared while adding its function-pointer implementation"); + let Some(oomir::DataType::Class { methods, .. }) = data_types.get_mut(&owner) else { + unreachable!("closure implementation holder disappeared"); }; - methods.insert( - implementation_method_name.clone(), - DataTypeMethod::Function(implementation), - ); + methods.insert(method_name, DataTypeMethod::Function(implementation)); } - - let closure_sig = tcx.instantiate_bound_regions_with_erased(closure_args.as_closure().sig()); - let tuple_ty = *closure_sig - .inputs() - .first() - .expect("closure call ABI always has a tuple argument"); - let tuple_oomir_ty = ty_to_oomir_type(tuple_ty, tcx, data_types, instance_context); - let mut instructions = Vec::new(); - let closure_args = if tuple_oomir_ty.has_jvm_value() { - let tuple_dest = "_closure_args".to_string(); - instructions.push(oomir::Instruction::ConstructObject { - dest: tuple_dest.clone(), - class_name: tuple_oomir_ty - .get_class_name() - .expect("non-unit closure tuple is represented by a JVM class") - .to_string(), - args: signature - .params - .iter() - .enumerate() - .map(|(index, (_, ty))| { - ( - oomir::Operand::Variable { - name: format!("_{}", index + 1), - ty: ty.clone(), - }, - ty.clone(), - ) - }) - .collect(), - }); - vec![oomir::Operand::Variable { - name: tuple_dest, - ty: tuple_oomir_ty.clone(), - }] - } else { - Vec::new() - }; - let call_dest = signature.ret.has_jvm_value().then(|| "_ret".to_string()); - instructions.push(oomir::Instruction::InvokeStatic { - dest: call_dest.clone(), - class_name: closure_class.clone(), - method_name: implementation_method_name, - method_ty: oomir::Signature { - params: tuple_oomir_ty - .has_jvm_value() - .then(|| ("args".to_string(), tuple_oomir_ty)) - .into_iter() - .collect(), - ret: signature.ret.clone(), - is_static: true, - }, - args: closure_args, - }); - instructions.push(oomir::Instruction::Return { - operand: call_dest.map(|name| oomir::Operand::Variable { - name, - ty: signature.ret.as_ref().clone(), - }), - }); - let bridge = DataTypeMethod::Function(oomir::Function { - name: method_name.clone(), - owner_class: None, - signature: signature.clone(), - debug_variables: Vec::new(), - body: oomir::CodeBlock { - entry: "bb0".to_string(), - basic_blocks: HashMap::from_iter([( - "bb0".to_string(), - oomir::BasicBlock { - label: "bb0".to_string(), - instructions, - }, - )]), - } - .into(), - }); - let Some(oomir::DataType::Class { methods, .. }) = data_types.get_mut(&closure_class) else { - panic!("closure class disappeared while adding its function-pointer bridge"); - }; - methods.insert(method_name.clone(), bridge); - (closure_class, method_name) + ensure_closure_fn_pointer_bridge( + data_types, + closure_instance, + signature, + tcx, + instance_context, + ) } /// Makes a concrete Rust closure directly implement the primitive-specialized /// JVM SAM used by `dyn Fn*`. The bridge rebuilds Rust's tuple call argument /// and, for a capturing closure, supplies the address-like environment carrier /// expected by the lowered closure body. -pub(super) fn ensure_closure_callable_bridge<'tcx>( +pub(crate) fn ensure_closure_callable_bridge<'tcx>( closure_ty: rustc_middle::ty::Ty<'tcx>, callable_abi: &crate::lower1::types::CallableTraitObjectAbi<'tcx>, data_types: &mut Definitions<'tcx>, @@ -361,10 +236,6 @@ pub(super) fn ensure_closure_callable_bridge<'tcx>( let mut instructions = Vec::new(); let tuple_dest = "_closure_args".to_string(); let tuple_call_arg = if tuple_oomir_ty.has_jvm_value() { - let tuple_class = tuple_oomir_ty - .get_class_name() - .expect("non-unit closure argument tuple is represented by a JVM class") - .to_string(); let tuple_args = callable_abi .signature .params @@ -380,11 +251,15 @@ pub(super) fn ensure_closure_callable_bridge<'tcx>( ) }) .collect(); - instructions.push(oomir::Instruction::ConstructObject { - dest: tuple_dest.clone(), - class_name: tuple_class, - args: tuple_args, - }); + crate::lower1::types::tuple_value( + callable_abi.tuple_ty, + tuple_args, + &tuple_dest, + tcx, + data_types, + instance_context, + &mut instructions, + ); Some(oomir::Operand::Variable { name: tuple_dest, ty: tuple_oomir_ty.clone(), @@ -530,48 +405,19 @@ pub(super) fn ensure_closure_callable_bridge<'tcx>( { interfaces.push(callable_abi.interface_name.clone()); } + crate::lower1::types::ensure_drop_callback( + closure_ty, + &closure_class, + tcx, + data_types, + instance_context, + ); true } -pub(super) fn closure_callable_abi<'tcx>( - closure_ty: Ty<'tcx>, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance: Instance<'tcx>, -) -> Option> { - let closure_ty = normalize_unsize_ty(closure_ty, tcx, instance); - let TyKind::Closure(_, closure_args) = closure_ty.kind() else { - return None; - }; - let closure_signature = tcx.instantiate_bound_regions_with_erased(closure_args.as_closure().sig()); - let tuple_ty = *closure_signature.inputs().first()?; - let TyKind::Tuple(tuple_elements) = tuple_ty.kind() else { - return None; - }; - let params = tuple_elements - .iter() - .enumerate() - .filter_map(|(index, element_ty)| { - let ty = ty_to_oomir_type(element_ty, tcx, data_types, instance); - ty.has_jvm_value().then(|| (format!("arg{index}"), ty)) - }) - .collect(); - let output_ty = closure_signature.output(); - let signature = oomir::Signature { - params, - ret: Box::new(ty_to_oomir_type(output_ty, tcx, data_types, instance)), - is_static: true, - }; - let interface_name = ensure_fn_ptr_interface(&signature, data_types, tcx, instance); - Some(crate::lower1::types::CallableTraitObjectAbi { - tuple_ty, - signature, - interface_name, - }) -} - pub(super) fn ensure_erased_receiver_fn_pointer_bridge<'tcx>( data_types: &mut Definitions<'tcx>, + source_rust_ty: Ty<'tcx>, source_signature: &oomir::Signature, source_interface: &str, target_signature: &oomir::Signature, @@ -650,14 +496,15 @@ pub(super) fn ensure_erased_receiver_fn_pointer_bridge<'tcx>( let source_descriptor = source_signature.to_jvm_descriptor_with_explicit_params(); let target_descriptor = target_signature.to_jvm_descriptor_with_explicit_params(); let identity = format!( - "{source_signature:?}:{source_descriptor}->{target_signature:?}:{target_descriptor}" + "{source_rust_ty:?}:{source_signature:?}:{source_descriptor}->{target_signature:?}:{target_descriptor}" ); let local_name = crate::stable_hash::readable_or_hashed_name( "FnPtrErasedReceiverBridge", &format!( - "{}_to_{}", + "{}_to_{}_{}", source_signature.fn_ptr_interface_name(), - target_signature.fn_ptr_interface_name() + target_signature.fn_ptr_interface_name(), + crate::stable_hash::short_hash(&identity, 16) ), &identity, 180, @@ -703,7 +550,41 @@ pub(super) fn ensure_erased_receiver_fn_pointer_bridge<'tcx>( } else { erased_receiver }; - if erased_pointer_to_slice { + let rust_signature = tcx.instantiate_bound_regions_with_erased(source_rust_ty.fn_sig(tcx)); + let array_receiver = rust_signature.inputs().first().and_then(|receiver| { + let (TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _)) = receiver.kind() else { + return None; + }; + let TyKind::Array(element, length) = pointee.kind() else { + return None; + }; + Some((*element, length.try_to_target_usize(tcx)?)) + }); + if erased_pointer_to_slice && let Some((element, length)) = array_receiver { + let element_type = ty_to_oomir_type(element, tcx, data_types, instance); + let pointer_type = oomir::Type::pointer(element_type); + let pointer_name = "_array_data".to_string(); + instructions.push(oomir::Instruction::AddressRetype { + dest: Some(pointer_name.clone()), + source: erased_payload, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_type.clone(), + size: rust_layout_size_operand(element, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( + element, tcx, data_types, instance, + ), + }), + }); + crate::lower1::place::emit_pointer_slice_view( + oomir::Operand::Variable { + name: pointer_name, + ty: pointer_type, + }, + oomir::Operand::Constant(oomir::Constant::U64(length)), + &typed_receiver_name, + &mut instructions, + ); + } else if erased_pointer_to_slice { let restored_object_name = "_restored_slice_object".to_string(); let view_class_name = match source_receiver_ty { oomir::Type::Slice(_) => oomir::SLICE_VIEW_CLASS, @@ -862,6 +743,7 @@ pub(super) fn ensure_erased_receiver_fn_pointer_bridge<'tcx>( "function".into(), oomir::Type::Interface(source_interface.into()), )], + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods: HashMap::from_iter([ ("call".into(), call_method), diff --git a/src/lower1/control_flow/rvalue/exported_closures.rs b/src/lower1/control_flow/rvalue/exported_closures.rs new file mode 100644 index 00000000..24f4fd26 --- /dev/null +++ b/src/lower1/control_flow/rvalue/exported_closures.rs @@ -0,0 +1,59 @@ +//! Returned Java closures require a callable interface even without a Rust trait coercion. +use super::*; + +pub(crate) fn ensure_exported_closure_calls<'tcx>( + result: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + data_types: &mut Definitions<'tcx>, + instance: Instance<'tcx>, +) { + let result = normalize_unsize_ty(result, tcx, instance); + for argument in result.walk() { + let Some(ty) = argument.as_type() else { + continue; + }; + if matches!(ty.kind(), TyKind::Closure(..)) { + if let Some(abi) = closure_callable_abi(ty, tcx, data_types, instance) { + ensure_closure_callable_bridge(ty, &abi, data_types, tcx, instance); + } + } + } +} + +fn closure_callable_abi<'tcx>( + closure_ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + data_types: &mut Definitions<'tcx>, + instance: Instance<'tcx>, +) -> Option> { + let closure_ty = normalize_unsize_ty(closure_ty, tcx, instance); + let TyKind::Closure(_, closure_args) = closure_ty.kind() else { + return None; + }; + let closure_signature = + tcx.instantiate_bound_regions_with_erased(closure_args.as_closure().sig()); + let tuple_ty = *closure_signature.inputs().first()?; + let TyKind::Tuple(tuple_elements) = tuple_ty.kind() else { + return None; + }; + let params = tuple_elements + .iter() + .enumerate() + .filter_map(|(index, element_ty)| { + let ty = ty_to_oomir_type(element_ty, tcx, data_types, instance); + ty.has_jvm_value().then(|| (format!("arg{index}"), ty)) + }) + .collect(); + let output_ty = closure_signature.output(); + let signature = oomir::Signature { + params, + ret: Box::new(ty_to_oomir_type(output_ty, tcx, data_types, instance)), + is_static: true, + }; + let interface_name = ensure_fn_ptr_interface(&signature, data_types, tcx, instance); + Some(crate::lower1::types::CallableTraitObjectAbi { + tuple_ty, + signature, + interface_name, + }) +} diff --git a/src/lower1/control_flow/rvalue/function_pointers.rs b/src/lower1/control_flow/rvalue/function_pointers.rs index 6079f790..0e0f8e5a 100644 --- a/src/lower1/control_flow/rvalue/function_pointers.rs +++ b/src/lower1/control_flow/rvalue/function_pointers.rs @@ -186,7 +186,7 @@ pub(crate) fn fn_pointer_target<'tcx>( } let (_, receiver_ty) = signature.params.first()?; let (receiver_ty, indirect_receiver) = match receiver_ty { - oomir::Type::Pointer(inner) | oomir::Type::Reference(inner) => (inner.as_ref(), true), + oomir::Type::Pointer(oomir::Pointee { value: inner, .. }) => (inner.as_ref(), true), other => (other, false), }; let mut method_signature = signature.clone(); @@ -464,14 +464,7 @@ pub(crate) fn ensure_fn_pointer_adapter_class<'tcx>( } .into(), }); - let relative_method_name = format!("call{}", oomir::RELATIVE_POINTER_METHOD_SUFFIX); - let relative_call_method = signature.supports_relative_pointer_abi().then(|| { - let oomir::DataTypeMethod::Function(mut function) = call_method.clone() else { - unreachable!("function-pointer adapters are OOMIR functions"); - }; - function.name = relative_method_name.clone(); - oomir::DataTypeMethod::Function(function) - }); + let code_identity = match target_function { Some(FnPointerTarget::Static(_) | FnPointerTarget::ImportedStatic(_)) => identity, _ => format!("{class_name}::call:{descriptor}"), @@ -513,11 +506,7 @@ pub(crate) fn ensure_fn_pointer_adapter_class<'tcx>( interfaces.push(identity_interface); } methods.entry("call".to_string()).or_insert(call_method); - if let Some(relative_call_method) = relative_call_method { - methods - .entry(relative_method_name) - .or_insert(relative_call_method); - } + if !interfaces .iter() .any(|interface| interface == interface_name) @@ -536,17 +525,16 @@ pub(crate) fn ensure_fn_pointer_adapter_class<'tcx>( ); } None => { - let mut methods = HashMap::from_iter([ + let methods = HashMap::from_iter([ ("call".to_string(), call_method), ("functionPointerIdentity".into(), identity_method), ]); - if let Some(relative_call_method) = relative_call_method { - methods.insert(relative_method_name, relative_call_method); - } + data_types.insert( class_name.clone(), oomir::DataType::Class { fields: vec![], + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods, super_class: Some("java/lang/Object".to_string()), diff --git a/src/lower1/control_flow/rvalue/pointer_layout.rs b/src/lower1/control_flow/rvalue/pointer_layout.rs index 60e6e2cb..520883e3 100644 --- a/src/lower1/control_flow/rvalue/pointer_layout.rs +++ b/src/lower1/control_flow/rvalue/pointer_layout.rs @@ -34,13 +34,10 @@ pub(super) fn pointer_view_size_operand<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ) -> oomir::Operand { - let pointee = match pointer_ty.kind() { - TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _) => *pointee, - other => panic!("expected pointer/reference type, found {other:?}"), - }; - let pointee = EarlyBinder::bind(tcx, pointee) + let pointer_ty = EarlyBinder::bind(tcx, pointer_ty) .instantiate(tcx, instance.args) .skip_norm_wip(); + let pointee = pointer_pointee_ty(pointer_ty, tcx); let size = crate::lower1::types::layout_size_bytes(tcx, pointee).unwrap_or_else(|error| { panic!("could not determine pointer view size for {pointee:?}: {error}") }); @@ -50,10 +47,22 @@ pub(super) fn pointer_view_size_operand<'tcx>( } pub(super) fn pointer_pointee_ty<'tcx>( - pointer_ty: rustc_middle::ty::Ty<'tcx>, + mut pointer_ty: rustc_middle::ty::Ty<'tcx>, + tcx: TyCtxt<'tcx>, ) -> rustc_middle::ty::Ty<'tcx> { - match pointer_ty.kind() { - TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _) => *pointee, - other => panic!("expected pointer/reference type, found {other:?}"), + loop { + match pointer_ty.kind() { + TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _) => return *pointee, + TyKind::Pat(inner, _) => pointer_ty = *inner, + _ => { + // The proven transparent field supplies the pointee. The owner can have other + // generic arguments. + pointer_ty = crate::lower1::types::transparent_payload(pointer_ty, tcx) + .unwrap_or_else(|| { + panic!("expected pointer/reference type, found {pointer_ty:?}") + }) + .ty; + } + } } } diff --git a/src/lower1/control_flow/rvalue/struct_tails.rs b/src/lower1/control_flow/rvalue/struct_tails.rs index 4d2192c4..512cdb39 100644 --- a/src/lower1/control_flow/rvalue/struct_tails.rs +++ b/src/lower1/control_flow/rvalue/struct_tails.rs @@ -90,7 +90,7 @@ pub(super) fn struct_tail_pointer_target_class<'tcx>( /// Restores the nominal view of a slice-tailed DST when reborrowing through a /// pointer wrapper. Allocator-backed pointers can retain the correct data word /// and metadata while carrying only the allocation's element view. -pub(super) fn emit_struct_tail_reborrow_view<'tcx>( +pub(crate) fn emit_struct_tail_reborrow_view<'tcx>( pointee_ty: Ty<'tcx>, source: oomir::Operand, pointer_ty: &oomir::Type, @@ -101,6 +101,9 @@ pub(super) fn emit_struct_tail_reborrow_view<'tcx>( instructions: &mut Vec, ) -> Option { let pointee_ty = normalize_unsize_ty(pointee_ty, tcx, instance); + if !matches!(pointee_ty.kind(), TyKind::Adt(def, _) if def.is_struct()) { + return None; + } let tail = tcx.struct_tail_for_codegen(pointee_ty, TypingEnv::fully_monomorphized()); if tail == pointee_ty { return None; @@ -176,8 +179,10 @@ pub(super) fn emit_struct_tail_pointer_cast<'tcx>( data_types: &mut Definitions<'tcx>, instructions: &mut Vec, ) -> Option { - let source_pointee = normalize_unsize_ty(pointer_pointee_ty(source_pointer_ty), tcx, instance); - let target_pointee = normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty), tcx, instance); + let source_pointee = + normalize_unsize_ty(pointer_pointee_ty(source_pointer_ty, tcx), tcx, instance); + let target_pointee = + normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty, tcx), tcx, instance); let TyKind::Adt(source_def, _) = source_pointee.kind() else { return None; }; @@ -217,7 +222,8 @@ pub(super) fn emit_slice_pointer_carrier<'tcx>( data_types: &mut Definitions<'tcx>, instructions: &mut Vec, ) -> Option { - let target_pointee = normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty), tcx, instance); + let target_pointee = + normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty, tcx), tcx, instance); let source_ty = source.get_type()?; let oomir::Type::Pointer(_) = &source_ty else { return None; @@ -233,7 +239,7 @@ pub(super) fn emit_slice_pointer_carrier<'tcx>( target_pointee.sequence_element_type(tcx) }; let element_type = ty_to_oomir_type(element_ty, tcx, data_types, instance); - let element_pointer_type = oomir::Type::Pointer(Box::new(element_type)); + let element_pointer_type = oomir::Type::pointer(element_type); let metadata = format!("{dest}_metadata"); instructions.push(oomir::Instruction::InvokeVirtual { dest: Some(metadata.clone()), @@ -248,24 +254,16 @@ pub(super) fn emit_slice_pointer_carrier<'tcx>( operand: source.clone(), }); let retyped = format!("{dest}_data"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(retyped.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_type.clone()), - is_static: false, - }, - args: vec![ - rust_layout_size_operand(element_ty, tcx, instance), - crate::lower1::types::pointer_view_codec_operand(element_ty, tcx, data_types, instance), - ], - operand: source, + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: element_pointer_type.clone(), + size: rust_layout_size_operand(element_ty, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( + element_ty, tcx, data_types, instance, + ), + }), }); let view_class = if target_pointee.is_str() { oomir::UTF8_VIEW_CLASS @@ -321,11 +319,13 @@ pub(super) fn emit_trait_object_to_struct_tail_cast<'tcx>( data_types: &mut Definitions<'tcx>, instructions: &mut Vec, ) -> Option { - let source_pointee = normalize_unsize_ty(pointer_pointee_ty(source_pointer_ty), tcx, instance); + let source_pointee = + normalize_unsize_ty(pointer_pointee_ty(source_pointer_ty, tcx), tcx, instance); if !matches!(source_pointee.kind(), TyKind::Dynamic(..)) { return None; } - let target_pointee = normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty), tcx, instance); + let target_pointee = + normalize_unsize_ty(pointer_pointee_ty(target_pointer_ty, tcx), tcx, instance); let TyKind::Adt(target_def, _) = target_pointee.kind() else { return None; }; @@ -467,50 +467,32 @@ pub(super) fn emit_struct_trait_tail_pointer_unsize<'tcx>( let source_tail_oomir_ty = ty_to_oomir_type(source_tail_pointer_ty, tcx, data_types, instance); let offset_pointer = format!("{dest}_tail_offset"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressOffset { dest: Some(offset_pointer.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "byte_offset".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_oomir_ty.clone()), - ("byte_count".to_string(), oomir::Type::U64), - ], - ret: Box::new(source_oomir_ty.clone()), - is_static: false, - }, - args: vec![oomir::Operand::Constant(oomir::Constant::U64( - info.source_tail_offset, - ))], - operand: source.clone(), + source: source.clone(), + count: oomir::Operand::Constant(oomir::Constant::U64(info.source_tail_offset)), + ty: source_oomir_ty.clone(), + bytes: true, + wrapping: false, + subtract: false, }); let source_tail_pointer = format!("{dest}_tail_source"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(source_tail_pointer.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_oomir_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(source_tail_oomir_ty.clone()), - is_static: false, + source: oomir::Operand::Variable { + name: offset_pointer, + ty: source_oomir_ty.clone(), }, - args: vec![ - rust_layout_size_operand(info.source_tail, tcx, instance), - crate::lower1::types::pointer_view_codec_operand( + layout: Box::new(oomir::AddressLayout { + pointer_type: source_tail_oomir_ty.clone(), + size: rust_layout_size_operand(info.source_tail, tcx, instance), + codec: crate::lower1::types::pointer_view_codec_operand( info.source_tail, tcx, data_types, instance, ), - ], - operand: oomir::Operand::Variable { - name: offset_pointer, - ty: source_oomir_ty.clone(), - }, + }), }); let tail_trait_dest = format!("{dest}_tail_trait"); let tail_trait = emit_unsize_value( diff --git a/src/lower1/control_flow/rvalue/unsize.rs b/src/lower1/control_flow/rvalue/unsize.rs index 9d30bb60..519c176e 100644 --- a/src/lower1/control_flow/rvalue/unsize.rs +++ b/src/lower1/control_flow/rvalue/unsize.rs @@ -37,7 +37,23 @@ pub(super) fn emit_unsize_value<'tcx>( let source_oomir_ty = ty_to_oomir_type(source_ty, tcx, data_types, instance); let target_oomir_ty = ty_to_oomir_type(target_ty, tcx, data_types, instance); - let result = if source_oomir_ty == target_oomir_ty { + let result = if let (TyKind::Adt(source_def, _), TyKind::Adt(target_def, _)) = + (source_ty.kind(), target_ty.kind()) + && source_def.did() == target_def.did() + && let Some(source_payload) = crate::lower1::types::transparent_payload(source_ty, tcx) + && let Some(target_payload) = crate::lower1::types::transparent_payload(target_ty, tcx) + { + emit_unsize_value( + source_payload.ty, + target_payload.ty, + source, + dest, + tcx, + instance, + data_types, + instructions, + ) + } else if source_oomir_ty == target_oomir_ty { instructions.push(oomir::Instruction::Move { dest: dest.to_string(), src: source, @@ -51,7 +67,10 @@ pub(super) fn emit_unsize_value<'tcx>( && source_def.did() == target_def.did() && crate::lower1::is_non_null_lang_item(tcx, source_def.did()) && matches!(source_oomir_ty, oomir::Type::Pointer(_)) - && let oomir::Type::Class(target_class) = &target_oomir_ty + && matches!( + target_oomir_ty, + oomir::Type::Class(_) | oomir::Type::Slice(_) | oomir::Type::Str + ) { let field = source_def .variant(0usize.into()) @@ -74,6 +93,9 @@ pub(super) fn emit_unsize_value<'tcx>( instructions, ) .map(|pointer| { + let oomir::Type::Class(target_class) = &target_oomir_ty else { + return pointer; + }; let pointer_ty = pointer .get_type() .expect("unsized NonNull pointer has a JVM value"); @@ -130,7 +152,11 @@ pub(super) fn emit_unsize_value<'tcx>( instructions.push(oomir::Instruction::GetField { dest: source_field_name.clone(), object: source.clone(), - field_name: field.ident(tcx).to_string(), + field_name: crate::lower1::types::struct_field_name( + tcx, + source_def, + field_index, + ), field_ty: source_field_oomir_ty.clone(), owner_class: source_class.clone(), }); @@ -190,7 +216,7 @@ pub(super) fn emit_unsize_value<'tcx>( tcx, instance, ) { - let source_pointee = normalize_unsize_ty(pointer_pointee_ty(source_ty), tcx, instance); + let source_pointee = normalize_unsize_ty(pointer_pointee_ty(source_ty, tcx), tcx, instance); let TyKind::Array(element_ty, length) = tcx .struct_tail_for_codegen(source_pointee, TypingEnv::fully_monomorphized()) .kind() @@ -198,7 +224,7 @@ pub(super) fn emit_unsize_value<'tcx>( unreachable!("struct-tail unsizing source was validated") }; let length = length.try_to_target_usize(tcx)?; - let target_pointee = normalize_unsize_ty(pointer_pointee_ty(target_ty), tcx, instance); + let target_pointee = normalize_unsize_ty(pointer_pointee_ty(target_ty, tcx), tcx, instance); let target_tail = tcx.struct_tail_for_codegen(target_pointee, TypingEnv::fully_monomorphized()); let tail_view_class = if target_tail.is_str() { diff --git a/src/lower1/control_flow/rvalue/values.rs b/src/lower1/control_flow/rvalue/values.rs index d0251eec..32690cb0 100644 --- a/src/lower1/control_flow/rvalue/values.rs +++ b/src/lower1/control_flow/rvalue/values.rs @@ -103,41 +103,149 @@ impl<'tcx> RvalueContext<'_, 'tcx> { } } Rvalue::Discriminant(place) => { - // 1. Generate instructions to get the actual value from the place + let temp_discriminant_var = generate_temp_var_name(data_types, &base_temp_name); + let place_mir_ty = + normalize_unsize_ty(place.ty(&mir.local_decls, tcx).ty, tcx, instance); + let carrier = crate::lower1::types::enum_carrier(place_mir_ty, tcx); + let single_variant = carrier + .filter(|carrier| !carrier.nullable) + .map(|carrier| carrier.variant) + .or_else(|| { + let layout = tcx + .layout_of( + rustc_middle::ty::TypingEnv::fully_monomorphized() + .as_query_input(place_mir_ty), + ) + .ok()?; + if let rustc_abi::Variants::Single { index } = layout.variants { + Some(index) + } else { + None + } + }); + if let Some(variant) = single_variant { + // A single inhabited variant needs no payload read. + let TyKind::Adt(def, _) = place_mir_ty.kind() else { + unreachable!() + }; + let discriminant = def.discriminant_for_variant(tcx, variant).val; + let result_ty = + get_place_type(original_dest_place, mir, tcx, instance, data_types); + instructions.push(oomir::Instruction::Cast { + dest: temp_discriminant_var.clone(), + ty: result_ty.clone(), + op: oomir::Operand::Constant(oomir::Constant::I64(discriminant as i64)), + }); + return ( + instructions, + oomir::Operand::Variable { + name: temp_discriminant_var, + ty: result_ty, + }, + ); + } let (actual_value_var_name, get_instructions, actual_value_oomir_type) = emit_instructions_to_get_on_own(place, tcx, instance, mir, data_types); - - // Add the instructions needed to get the value (e.g., ArrayGet) instructions.extend(get_instructions); - - // 2. Now operate on the variable holding the actual value - let temp_discriminant_var = generate_temp_var_name(data_types, &base_temp_name); + if crate::lower1::types::enum_scalar_ty(place_mir_ty, tcx).is_some() { + let result_ty = + get_place_type(original_dest_place, mir, tcx, instance, data_types); + instructions.push(oomir::Instruction::Cast { + dest: temp_discriminant_var.clone(), + ty: result_ty.clone(), + op: oomir::Operand::Variable { + name: actual_value_var_name, + ty: actual_value_oomir_type, + }, + }); + return ( + instructions, + oomir::Operand::Variable { + name: temp_discriminant_var, + ty: result_ty, + }, + ); + } + if matches!(actual_value_oomir_type, oomir::Type::TaggedI64) { + let result_ty = + get_place_type(original_dest_place, mir, tcx, instance, data_types); + let part = format!("{temp_discriminant_var}_tag"); + instructions.push(oomir::Instruction::TaggedPart { + dest: part.clone(), + value: oomir::Operand::Variable { + name: actual_value_var_name, + ty: actual_value_oomir_type, + }, + index: 1, + }); + instructions.push(oomir::Instruction::Cast { + dest: temp_discriminant_var.clone(), + ty: result_ty.clone(), + op: oomir::Operand::Variable { + name: part, + ty: oomir::Type::I64, + }, + }); + return ( + instructions, + oomir::Operand::Variable { + name: temp_discriminant_var, + ty: result_ty, + }, + ); + } + if carrier.is_some() { + let result_ty = + get_place_type(original_dest_place, mir, tcx, instance, data_types); + instructions.push(oomir::Instruction::InvokeStatic { + dest: Some(temp_discriminant_var.clone()), + class_name: oomir::POINTER_CLASS.to_string(), + method_name: if matches!( + actual_value_oomir_type, + oomir::Type::Slice(_) | oomir::Type::Str + ) { + "nullableViewTag" + } else { + "nullableTag" + } + .to_string(), + method_ty: oomir::Signature { + params: vec![("value".to_string(), actual_value_oomir_type.clone())], + ret: Box::new(oomir::Type::I64), + is_static: true, + }, + args: vec![oomir::Operand::Variable { + name: actual_value_var_name, + ty: actual_value_oomir_type, + }], + }); + let cast = format!("{temp_discriminant_var}_cast"); + instructions.push(oomir::Instruction::Cast { + dest: cast.clone(), + ty: result_ty.clone(), + op: oomir::Operand::Variable { + name: temp_discriminant_var, + ty: oomir::Type::I64, + }, + }); + return ( + instructions, + oomir::Operand::Variable { + name: cast, + ty: result_ty, + }, + ); + } let place_class_name = match actual_value_oomir_type.clone() { oomir::Type::Class(name) => name.clone(), // Handle potential references if get_on_own returns Ref(Class) - oomir::Type::Reference(inner) => { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - panic!("Discriminant on Ref to non-class type: {:?}", inner) - } - } - oomir::Type::MutableReference(inner) => { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - panic!("Discriminant on MutableRef to non-class type: {:?}", inner) - } - } _ => panic!( "Discriminant on non-class type: {:?}", actual_value_oomir_type ), }; - let place_mir_ty = - normalize_unsize_ty(place.ty(&mir.local_decls, tcx).ty, tcx, instance); if matches!(place_mir_ty.kind(), TyKind::Coroutine(..)) { instructions.push(oomir::Instruction::GetField { dest: temp_discriminant_var.clone(), @@ -175,7 +283,7 @@ impl<'tcx> RvalueContext<'_, 'tcx> { } let (enum_class_name, use_numeric_discriminant) = match place_mir_ty.kind() { TyKind::Adt(adt_def, substs) if adt_def.is_enum() => { - force_define_named_adt(place_mir_ty, tcx, data_types, instance); + ty_to_oomir_type(place_mir_ty, tcx, data_types, instance); ( generate_adt_jvm_class_name(adt_def, substs, tcx, data_types, instance), enum_union_discriminant_supported(adt_def, tcx), diff --git a/src/lower1/control_flow/trait_objects.rs b/src/lower1/control_flow/trait_objects.rs index 5fbb967a..1e024f3f 100644 --- a/src/lower1/control_flow/trait_objects.rs +++ b/src/lower1/control_flow/trait_objects.rs @@ -8,9 +8,7 @@ use rustc_middle::ty::{ use super::super::{ jvm_names, - types::{ - pointer_view_codec_operand, readable_rust_type_name, sanitize_name_token, ty_to_oomir_type, - }, + types::{pointer_view_codec_operand, stable_type_identity, ty_to_oomir_type}, }; use crate::oomir; @@ -23,7 +21,7 @@ const RUNTIME_TRAIT_OBJECT_ALIGNMENT_METHOD: &str = "rustTraitObjectAlignment"; pub(super) fn carrier_needs_trait_object_adapter( carrier_ty: &oomir::Type, interface_name: &str, - data_types: &HashMap, + data_types: &crate::lower1::context::Definitions<'_>, ) -> bool { match carrier_ty { oomir::Type::Class(class_name) => !matches!( @@ -107,21 +105,16 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( ); let identity = format!( - "{dynamic_ty:?}:{trait_ref:?}:{}", + "{}:{}:{}", + stable_type_identity(tcx, concrete_ty), + stable_type_identity(tcx, dynamic_ty), carrier_ty.to_jvm_descriptor() ); - let concrete_token = sanitize_name_token(&readable_rust_type_name( - concrete_ty, - tcx, - data_types, - instance_context, - )); - let interface_token = interface_name.rsplit('/').next().unwrap_or(interface_name); - let local_name = crate::stable_hash::readable_or_hashed_name( - "TraitObjectCarrier", - &format!("{concrete_token}_as_{interface_token}"), - &identity, - 180, + // Different Rust callables can share one physical carrier. Dispatch requires their Rust + // identity. + let local_name = format!( + "TraitObjectCarrier_{}", + crate::stable_hash::short_hash(&identity, 16) ); let class_name = jvm_names::synthetic_class_for_instance(tcx, instance_context, local_name); if data_types.contains_key(&class_name) { @@ -130,13 +123,35 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( let mut adapter_methods = HashMap::default(); let mut interface_methods = HashMap::default(); - let callable_fn_pointer = callable_abi + let callable_value = callable_abi .as_ref() - .filter(|_| matches!(concrete_ty.kind(), TyKind::FnPtr(..))); - if let Some(callable) = callable_fn_pointer { - // Fn/FnMut/FnOnce share one flattened JVM call method. Forward through - // the stored function pointer instead of adding Rust's tuple-taking - // vtable shims as extra abstract methods on the functional interface. + .filter(|_| matches!(concrete_ty.kind(), TyKind::FnPtr(..) | TyKind::Closure(..))); + if let Some(callable) = callable_value { + let erased_closure = match concrete_ty.kind() { + TyKind::Closure(def, args) if args.as_closure().upvar_tys().is_empty() => { + Some(Instance::new_raw(*def, args)) + } + _ => None, + }; + let value_ty = + if erased_closure.is_none() && matches!(concrete_ty.kind(), TyKind::Closure(..)) { + if !super::rvalue::ensure_closure_callable_bridge( + concrete_ty, + callable, + data_types, + tcx, + instance_context, + ) { + return Err(format!( + "could not generate callable closure bridge for {concrete_ty:?}" + )); + } + ty_to_oomir_type(concrete_ty, tcx, data_types, instance_context) + } else { + oomir::Type::Interface(callable.interface_name.clone()) + }; + // Fn, FnMut, and FnOnce share one JVM call method. Extra tuple shims would break the + // functional interface. let payload = oomir::Operand::Variable { name: "_callable".to_string(), ty: carrier_ty.clone(), @@ -151,36 +166,86 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( field_ty: carrier_ty.clone(), owner_class: class_name.clone(), }]; - let function = if matches!(carrier_ty, oomir::Type::Pointer(_)) { - crate::lower1::place::emit_pointer_read( - payload, - &oomir::Type::Interface(callable.interface_name.clone()), - "_function", - &mut instructions, - ) - } else { - payload - }; let result = callable .signature .ret .has_jvm_value() .then(|| "_ret".to_string()); - instructions.push(oomir::Instruction::CallIndirect { - dest: result.clone(), - function_ptr: Box::new(function), - args: callable - .signature - .params - .iter() - .enumerate() - .map(|(index, (_, ty))| oomir::Operand::Variable { - name: format!("_{}", index + 2), - ty: ty.clone(), - }) - .collect(), - signature: callable.signature.clone(), - }); + let arguments = callable + .signature + .params + .iter() + .enumerate() + .map(|(index, (_, ty))| oomir::Operand::Variable { + name: format!("_{}", index + 2), + ty: ty.clone(), + }) + .collect::>(); + if let Some(closure) = erased_closure { + instructions.clear(); + let tuple_ty = ty_to_oomir_type(callable.tuple_ty, tcx, data_types, instance_context); + let mut params = Vec::new(); + let mut args = Vec::new(); + if tuple_ty.has_jvm_value() { + let argument = crate::lower1::types::tuple_value( + callable.tuple_ty, + arguments + .into_iter() + .zip(callable.signature.params.iter().map(|(_, ty)| ty.clone())) + .collect(), + "_arguments", + tcx, + data_types, + instance_context, + &mut instructions, + ); + params.push(("args".into(), tuple_ty.clone())); + args.push(argument); + } + let target = data_types.function_name(tcx, closure); + instructions.push(oomir::Instruction::InvokeStatic { + dest: result.clone(), + class_name: target.class_to_call_on.expect("closure owner"), + method_name: target.method_name, + method_ty: oomir::Signature { + params, + ret: callable.signature.ret.clone(), + is_static: true, + }, + args, + }); + } else { + let function = if matches!(carrier_ty, oomir::Type::Pointer(_)) { + crate::lower1::place::emit_pointer_read( + payload, + &value_ty, + "_function", + &mut instructions, + ) + } else { + payload + }; + let interface_ty = oomir::Type::Interface(callable.interface_name.clone()); + let function = if function.get_type().as_ref() != Some(&interface_ty) { + instructions.push(oomir::Instruction::Cast { + op: function, + ty: interface_ty.clone(), + dest: "_function_interface".into(), + }); + oomir::Operand::Variable { + name: "_function_interface".into(), + ty: interface_ty, + } + } else { + function + }; + instructions.push(oomir::Instruction::CallIndirect { + dest: result.clone(), + function_ptr: Box::new(function), + args: arguments, + signature: callable.signature.clone(), + }); + } instructions.push(oomir::Instruction::Return { operand: result.map(|name| oomir::Operand::Variable { name, @@ -216,7 +281,7 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( } for entry in trait_ref .into_iter() - .filter(|_| callable_fn_pointer.is_none()) + .filter(|_| callable_value.is_none()) .flat_map(|trait_ref| tcx.vtable_entries(trait_ref).iter()) { let VtblEntry::Method(target_instance) = entry else { @@ -227,144 +292,6 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( .type_of(target_instance.def_id()) .instantiate(tcx, target_instance.args) .skip_norm_wip(); - if let TyKind::Closure(_, closure_args) = target_instance_ty.kind() { - let Some(callable_abi) = &callable_abi else { - return Err(format!( - "closure vtable entry does not target a callable trait: {target_instance:?}" - )); - }; - let closure_signature = - tcx.instantiate_bound_regions_with_erased(closure_args.as_closure().sig()); - let tuple_ty = *closure_signature - .inputs() - .first() - .ok_or_else(|| { - format!("closure call signature has no argument tuple: {target_instance:?}") - })?; - let tuple_oomir_ty = ty_to_oomir_type(tuple_ty, tcx, data_types, *target_instance); - let return_ty = callable_abi.signature.ret.as_ref().clone(); - let payload_name = "_trait_object_payload".to_string(); - let mut instructions = vec![oomir::Instruction::GetField { - dest: payload_name.clone(), - object: oomir::Operand::Variable { - name: "_1".to_string(), - ty: oomir::Type::Class(class_name.clone()), - }, - field_name: "value".to_string(), - field_ty: carrier_ty.clone(), - owner_class: class_name.clone(), - }]; - let mut target_params = Vec::new(); - let mut target_args = Vec::new(); - if !closure_args.as_closure().upvar_tys().is_empty() { - target_params.push(("closure_env".to_string(), carrier_ty.clone())); - target_args.push(oomir::Operand::Variable { - name: payload_name, - ty: carrier_ty.clone(), - }); - } - if tuple_oomir_ty.has_jvm_value() { - let tuple_name = "_trait_object_call_args".to_string(); - instructions.push(oomir::Instruction::ConstructObject { - dest: tuple_name.clone(), - class_name: tuple_oomir_ty - .get_class_name() - .ok_or_else(|| { - format!("closure argument tuple has no JVM class: {tuple_oomir_ty:?}") - })? - .to_string(), - args: callable_abi - .signature - .params - .iter() - .enumerate() - .map(|(index, (_, ty))| { - ( - oomir::Operand::Variable { - name: format!("_{}", index + 2), - ty: ty.clone(), - }, - ty.clone(), - ) - }) - .collect(), - }); - target_params.push(("args".to_string(), tuple_oomir_ty.clone())); - target_args.push(oomir::Operand::Variable { - name: tuple_name, - ty: tuple_oomir_ty, - }); - } - let call_dest = return_ty.has_jvm_value().then(|| "_ret".to_string()); - let target = data_types.function_name(tcx, *target_instance); - instructions.extend([ - oomir::Instruction::InvokeStatic { - dest: call_dest.clone(), - class_name: target.class_to_call_on.expect("closure has a JVM owner"), - method_name: target.method_name, - method_ty: oomir::Signature { - params: target_params, - ret: Box::new(return_ty.clone()), - is_static: true, - }, - args: target_args, - }, - oomir::Instruction::Return { - operand: call_dest.map(|name| oomir::Operand::Variable { - name, - ty: return_ty.clone(), - }), - }, - ]); - - let interface_params = callable_abi.signature.params.clone(); - let method_name = "call".to_string(); - interface_methods.insert( - method_name.clone(), - oomir::Signature { - params: interface_params.clone(), - ret: Box::new(return_ty.clone()), - is_static: false, - }, - ); - adapter_methods.insert( - method_name.clone(), - oomir::DataTypeMethod::Function(oomir::Function { - name: method_name, - owner_class: None, - debug_variables: Vec::new(), - signature: oomir::Signature { - params: std::iter::once(( - "self".to_string(), - oomir::Type::Class(class_name.clone()), - )) - .chain(interface_params) - .collect(), - ret: Box::new(return_ty), - is_static: false, - }, - body: oomir::CodeBlock { - entry: "bb0".to_string(), - basic_blocks: HashMap::from_iter([( - "bb0".to_string(), - oomir::BasicBlock { - label: "bb0".to_string(), - instructions, - }, - )]), - } - .into(), - }), - ); - continue; - } - if callable_abi.is_some() && matches!(concrete_ty.kind(), TyKind::Closure(..)) { - // The JVM callable interface deliberately exposes one flattened - // `call` method. The remaining Rust vtable entries are Fn/FnMut/ - // FnOnce shims for the same closure body, not additional methods - // that belong on that functional interface. - continue; - } let is_coroutine = matches!(target_instance_ty.kind(), TyKind::Coroutine(..)); let (target_inputs, target_output) = if is_coroutine || matches!(target_instance.def, InstanceKind::Shim(ShimKind::VTable(_))) @@ -426,28 +353,13 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( })?; let element_ty = *element_ty; let element_oomir_ty = ty_to_oomir_type(element_ty, tcx, data_types, instance_context); - let element_pointer_ty = oomir::Type::Pointer(Box::new(element_oomir_ty)); + let element_pointer_ty = oomir::Type::pointer(element_oomir_ty); let element_pointer_name = "_trait_object_array_data".to_string(); let receiver_object_name = "_trait_object_receiver_object".to_string(); let receiver_name = "_trait_object_receiver".to_string(); ( vec![ - oomir::Instruction::InvokeStatic { - dest: Some(element_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), carrier_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_ty.clone()), - is_static: true, - }, - args: vec![ - payload_operand, - oomir::Operand::Constant(oomir::Constant::U64( + oomir::Instruction::AddressRetype { dest: Some(element_pointer_name.clone()), source: payload_operand, layout: Box::new(oomir::AddressLayout { pointer_type: element_pointer_ty.clone(), size: oomir::Operand::Constant(oomir::Constant::U64( super::super::types::layout_size_bytes(tcx, element_ty)? .try_into() .map_err(|_| { @@ -455,15 +367,12 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( "trait-object array element is too large: {element_ty:?}" ) })?, - )), - pointer_view_codec_operand( + )), codec: pointer_view_codec_operand( element_ty, tcx, data_types, instance_context, - ), - ], - }, + ) }) }, oomir::Instruction::ConstructObject { dest: receiver_object_name.clone(), class_name: oomir::SLICE_VIEW_CLASS.to_string(), @@ -580,19 +489,21 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( field_ty: carrier_ty.clone(), owner_class: class_name.clone(), }; - let receiver_has_interface = match &target_receiver_ty { - oomir::Type::Pointer(inner) | oomir::Type::Reference(inner) => match inner.as_ref() { - oomir::Type::Class(receiver_class) => matches!( - data_types.get(receiver_class), - Some(oomir::DataType::Class { interfaces, .. }) - if interfaces.iter().any(|interface| interface == interface_name) - ), + let receiver_has_interface = super::super::types::tagged_scalar(concrete_ty, tcx).is_none() + && super::super::types::enum_carrier(concrete_ty, tcx).is_none() + && match &target_receiver_ty { + oomir::Type::Pointer(oomir::Pointee { value: inner, .. }) => match inner.as_ref() { + oomir::Type::Class(receiver_class) => matches!( + data_types.get(receiver_class), + Some(oomir::DataType::Class { interfaces, .. }) + if interfaces.iter().any(|interface| interface == interface_name) + ), + _ => false, + }, _ => false, - }, - _ => false, - }; + }; let call = match &target_receiver_ty { - oomir::Type::Pointer(inner) | oomir::Type::Reference(inner) + oomir::Type::Pointer(oomir::Pointee { value: inner, .. }) if matches!(inner.as_ref(), oomir::Type::Class(_)) && receiver_has_interface => { let oomir::Type::Class(receiver_class) = inner.as_ref() else { @@ -743,12 +654,44 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( .into(), }), ); - let adapter_drops_payload = concrete_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) - && matches!(carrier_ty, oomir::Type::Pointer(_)); + let adapter_drops_payload = concrete_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()); if adapter_drops_payload { - let drop_instance = Instance::resolve_drop_glue(tcx, concrete_ty); - let target = data_types.function_name(tcx, drop_instance); let payload_name = "_trait_object_drop_payload".to_string(); + let payload = oomir::Operand::Variable { + name: payload_name.clone(), + ty: carrier_ty.clone(), + }; + let mut instructions = vec![oomir::Instruction::GetField { + dest: payload_name, + object: oomir::Operand::Variable { + name: "_1".to_string(), + ty: oomir::Type::Class(class_name.clone()), + }, + field_name: "value".to_string(), + field_ty: carrier_ty.clone(), + owner_class: class_name.clone(), + }]; + if matches!(carrier_ty, oomir::Type::Pointer(_)) { + super::emit_drop_in_place( + concrete_ty, + payload, + tcx, + instance_context, + data_types, + &mut instructions, + ); + } else { + super::emit_owned_drop( + concrete_ty, + payload, + "_trait_drop", + tcx, + instance_context, + data_types, + &mut instructions, + ); + } + instructions.push(oomir::Instruction::Return { operand: None }); adapter_methods.insert( "rustDrop".to_string(), oomir::DataTypeMethod::Function(oomir::Function { @@ -766,35 +709,7 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( "bb0".to_string(), oomir::BasicBlock { label: "bb0".to_string(), - instructions: vec![ - oomir::Instruction::GetField { - dest: payload_name.clone(), - object: oomir::Operand::Variable { - name: "_1".to_string(), - ty: oomir::Type::Class(class_name.clone()), - }, - field_name: "value".to_string(), - field_ty: carrier_ty.clone(), - owner_class: class_name.clone(), - }, - oomir::Instruction::InvokeStatic { - dest: None, - class_name: target - .class_to_call_on - .expect("trait-object payload drop glue has a JVM owner"), - method_name: target.method_name, - method_ty: oomir::Signature { - params: vec![("pointee".to_string(), carrier_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![oomir::Operand::Variable { - name: payload_name, - ty: carrier_ty.clone(), - }], - }, - oomir::Instruction::Return { operand: None }, - ], + instructions, }, )]), } @@ -875,6 +790,7 @@ pub(crate) fn ensure_trait_object_adapter_class_for_pointees<'tcx>( class_name.clone(), oomir::DataType::Class { fields: vec![("value".to_string(), carrier_ty.clone())], + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods: adapter_methods, super_class: Some("java/lang/Object".to_string()), diff --git a/src/lower1/initialization.rs b/src/lower1/initialization.rs new file mode 100644 index 00000000..cb744756 --- /dev/null +++ b/src/lower1/initialization.rs @@ -0,0 +1,274 @@ +//! Definite initialization tracks only the locals used by lowering decisions. +use rustc_hash::FxHashSet as HashSet; +use rustc_middle::mir::{ + BasicBlock, Body, Local, Place, ProjectionElem, StatementKind, TerminatorKind, +}; +use std::collections::VecDeque; + +struct Domain { + slots: Vec, + locals: Vec, +} + +impl Domain { + fn new(local_count: usize, locals: impl IntoIterator) -> Self { + let mut result = Self { + slots: Vec::new(), + locals: Vec::new(), + }; + for local in locals { + if result.slots.is_empty() { + result.slots.resize(local_count, u32::MAX); + } + if result.slots[local.index()] == u32::MAX { + result.slots[local.index()] = result.locals.len() as u32; + result.locals.push(local); + } + } + result + } + + fn slot(&self, local: Local) -> Option { + self.slots + .get(local.index()) + .copied() + .filter(|&slot| slot != u32::MAX) + .map(|slot| slot as usize) + } + + fn empty(&self) -> Bits { + Bits(vec![0; self.locals.len().div_ceil(64)]) + } + fn all(&self) -> Bits { + let mut bits = self.empty(); + bits.0.fill(u64::MAX); + let tail = self.locals.len() % 64; + if tail != 0 + && let Some(last) = bits.0.last_mut() + { + *last = (1 << tail) - 1; + } + bits + } +} + +#[derive(Clone, PartialEq, Eq)] +struct Bits(Vec); + +impl Bits { + fn insert(&mut self, slot: usize) { + self.0[slot / 64] |= 1 << (slot % 64); + } + fn remove(&mut self, slot: usize) { + self.0[slot / 64] &= !(1 << (slot % 64)); + } + fn contains(&self, slot: usize) -> bool { + self.0[slot / 64] & (1 << (slot % 64)) != 0 + } + + fn transfer(&mut self, input: &Self, killed: Option<&Self>, generated: &Self, intersect: bool) { + for (index, dest) in self.0.iter_mut().enumerate() { + let value = (input.0[index] & !killed.map_or(0, |k| k.0[index])) | generated.0[index]; + *dest = if intersect { *dest & value } else { value }; + } + } + + fn locals<'a>(&'a self, domain: &'a Domain) -> impl Iterator + 'a { + self.0.iter().enumerate().flat_map(move |(index, &word)| { + let mut word = word; + std::iter::from_fn(move || { + if word == 0 { + return None; + } + let bit = word.trailing_zeros() as usize; + word &= word - 1; + Some(domain.locals[index * 64 + bit]) + }) + }) + } +} + +pub(super) struct MirControlFlow { + predecessors: Vec>, + reachable: Vec, +} + +impl MirControlFlow { + pub(super) fn new(mir: &Body<'_>) -> Self { + let mut predecessors = vec![Vec::new(); mir.basic_blocks.len()]; + for (block, data) in mir.basic_blocks.iter_enumerated() { + for successor in data.terminator().successors() { + predecessors[successor.index()].push(block); + } + } + let mut reachable = vec![false; mir.basic_blocks.len()]; + let mut queue = VecDeque::from([BasicBlock::from_usize(0)]); + while let Some(block) = queue.pop_front() { + if std::mem::replace(&mut reachable[block.index()], true) { + continue; + } + queue.extend(mir.basic_blocks[block].terminator().successors()); + } + Self { + predecessors, + reachable, + } + } + + fn initialized( + &self, + domain: &Domain, + generated: &[Bits], + killed: Option<&[Bits]>, + entry: Bits, + ) -> Vec { + let all = domain.all(); + let mut available = self + .reachable + .iter() + .enumerate() + .map(|(block, &reachable)| { + if block == 0 { + entry.clone() + } else if reachable { + all.clone() + } else { + domain.empty() + } + }) + .collect::>(); + let mut incoming = domain.empty(); + loop { + let mut changed = false; + for block in 1..self.reachable.len() { + if !self.reachable[block] { + continue; + } + let mut first = true; + for predecessor in &self.predecessors[block] { + let index = predecessor.index(); + if !self.reachable[index] { + continue; + } + incoming.transfer( + &available[index], + killed.map(|k| &k[index]), + &generated[index], + !first, + ); + first = false; + } + if first { + incoming.0.fill(0); + } + if available[block] != incoming { + available[block].clone_from(&incoming); + changed = true; + } + } + if !changed { + return available; + } + } + } +} + +fn field_assignment(place: Place<'_>) -> bool { + matches!(place.projection.first(), Some(ProjectionElem::Field(..))) +} + +#[derive(Clone, Copy)] +enum Write { + Whole, + Field, + Reset, +} + +/// Visit writes and storage-lifetime resets in the order they take effect. +fn writes(mir: &Body<'_>, block: BasicBlock, mut visit: impl FnMut(Local, Write)) { + fn assignment(place: Place<'_>, visit: &mut impl FnMut(Local, Write)) { + if field_assignment(place) { + visit(place.local, Write::Field); + } else if place.projection.is_empty() { + visit(place.local, Write::Whole); + } + } + let data = &mir.basic_blocks[block]; + for statement in &data.statements { + match &statement.kind { + StatementKind::Assign(value) => assignment(value.0, &mut visit), + StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + visit(*local, Write::Reset) + } + _ => {} + } + } + if let TerminatorKind::Call { destination, .. } = &data.terminator().kind { + assignment(*destination, &mut visit); + } +} + +pub(super) fn class_locals_needing_initial_carriers( + mir: &Body<'_>, + flow: &MirControlFlow, +) -> Vec { + let mut candidates = Vec::new(); + for (block, _) in mir.basic_blocks.iter_enumerated() { + writes(mir, block, |local, write| { + if matches!(write, Write::Field) { + candidates.push(local); + } + }); + } + let domain = Domain::new(mir.local_decls.len(), candidates); + if domain.locals.is_empty() { + return Vec::new(); + } + let mut generated = vec![domain.empty(); mir.basic_blocks.len()]; + let mut killed = generated.clone(); + for (block, _) in mir.basic_blocks.iter_enumerated() { + writes(mir, block, |local, write| { + if let Some(slot) = domain.slot(local) { + if matches!(write, Write::Reset) { + generated[block.index()].remove(slot); + killed[block.index()].insert(slot); + } else { + generated[block.index()].insert(slot); + killed[block.index()].remove(slot); + } + } + }); + } + let mut entry = domain.empty(); + for (slot, local) in domain.locals.iter().enumerate() { + if local.index() > 0 && local.index() <= mir.arg_count { + entry.insert(slot); + } + } + let available = flow.initialized(&domain, &generated, Some(&killed), entry); + let mut needed = domain.empty(); + for (block, _) in mir.basic_blocks.iter_enumerated() { + if !flow.reachable[block.index()] { + continue; + } + let mut state = available[block.index()].clone(); + writes(mir, block, |local, write| { + if let Some(slot) = domain.slot(local) { + if matches!(write, Write::Reset) { + state.remove(slot); + } else { + if matches!(write, Write::Field) && !state.contains(slot) { + needed.insert(slot); + } + state.insert(slot); + } + } + }); + } + let mut result = needed.locals(&domain).collect::>(); + result.sort_unstable(); + result +} + +#[cfg(test)] +mod tests; diff --git a/src/lower1/initialization/tests.rs b/src/lower1/initialization/tests.rs new file mode 100644 index 00000000..dcb79a0a --- /dev/null +++ b/src/lower1/initialization/tests.rs @@ -0,0 +1,59 @@ +use super::*; + +#[test] +fn definite_initialization_intersects_branches_and_loop_backedges() { + let local = Local::from_usize; + let domain = Domain::new(10_000, [local(1), local(999), local(64)]); + let flow = MirControlFlow { + predecessors: [ + vec![], + vec![0], + vec![0], + vec![1, 2, 4, 5], + vec![3], + vec![], + vec![3], + ] + .into_iter() + .map(|p| p.into_iter().map(BasicBlock::from_usize).collect()) + .collect(), + reachable: vec![true, true, true, true, true, false, true], + }; + let mut generated = vec![domain.empty(); flow.reachable.len()]; + let mut killed = generated.clone(); + let mut entry = domain.empty(); + entry.insert(domain.slot(local(1)).unwrap()); + for block in [1, 2] { + generated[block].insert(domain.slot(local(999)).unwrap()); + } + generated[4].insert(domain.slot(local(64)).unwrap()); + // A storage lifetime ends on one branch and on the loop's backedge. + killed[2].insert(domain.slot(local(1)).unwrap()); + killed[4].insert(domain.slot(local(999)).unwrap()); + let available = flow.initialized(&domain, &generated, Some(&killed), entry.clone()); + assert_eq!(available[1].locals(&domain).collect::>(), [local(1)]); + for block in [3, 4, 5, 6] { + assert_eq!(available[block].locals(&domain).count(), 0); + } + let available = flow.initialized(&domain, &generated, None, entry); + for block in [3, 4, 6] { + assert_eq!( + available[block].locals(&domain).collect::>(), + [local(1), local(999)] + ); + } +} + +#[test] +fn sparse_local_numbers_do_not_expand_per_block_state() { + let locals = (0..65) + .map(|i| Local::from_usize(9_000 - i * 17)) + .collect::>(); + let domain = Domain::new(10_000, locals.iter().copied().chain(locals.iter().copied())); + assert_eq!(domain.all().locals(&domain).collect::>(), locals); + assert_eq!(domain.empty().0.len(), 2); + assert!(domain.slot(Local::from_usize(8)).is_none()); + let domain = Domain::new(10_000, []); + assert!(domain.slots.is_empty()); + assert!(domain.empty().0.is_empty()); +} diff --git a/src/lower1/inline.rs b/src/lower1/inline.rs new file mode 100644 index 00000000..f4cc690f --- /dev/null +++ b/src/lower1/inline.rs @@ -0,0 +1,382 @@ +//! Expose small monomorphized operations before storage lowering. +use rustc_abi::ExternAbi; +use rustc_attr_ir::InlineAttr; +use rustc_middle::{ + mir::{self, visit::MutVisitor, *}, + ty::{self, EarlyBinder, Instance, InstanceKind, ShimKind, TyCtxt, TypingEnv}, +}; + +pub(super) fn expand<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + original: &Body<'tcx>, +) -> Option> { + if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No + || matches!( + tcx.codegen_fn_attrs(instance.def_id()).optimize, + rustc_attr_ir::OptimizeAttr::DoNotOptimize + ) + || !original.basic_blocks.iter().any(|block| { + matches!( + block.terminator().kind, + TerminatorKind::Call { + target: Some(_), + .. + } + ) + }) + || original.coroutine.is_some() + || crate::lower2::debug_info_options(tcx).local_variables + { + return None; + } + let mut body = instance + .try_instantiate_mir_and_normalize_erasing_regions( + tcx, + TypingEnv::fully_monomorphized(), + EarlyBinder::bind(tcx, original.clone()), + ) + .ok()?; + let mut history = vec![instance]; + let mut budget = 2400usize; + let end = body.basic_blocks.len(); + let changed = expand_blocks(tcx, &mut body, 0..end, &mut history, &mut budget); + if changed { + super::inline_niches::expand(tcx, &mut body); + for block in body.basic_blocks.as_mut() { + if let TerminatorKind::Drop { place, target, .. } = block.terminator().kind + && !place + .ty(&body.local_decls, tcx) + .ty + .needs_drop(tcx, TypingEnv::fully_monomorphized()) + { + block.terminator_mut().kind = TerminatorKind::Goto { target }; + } + } + super::inline_places::simplify(tcx, &mut body); + for _ in 0..8 { + let records = super::inline_records::simplify(tcx, &mut body); + let enums = super::inline_enums::simplify(tcx, &mut body); + if !records && !enums { + break; + } + super::inline_places::simplify(tcx, &mut body); + } + super::inline_places::simplify(tcx, &mut body); + } + changed.then_some(body) +} + +fn expand_blocks<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + blocks: std::ops::Range, + history: &mut Vec>, + budget: &mut usize, +) -> bool { + let mut changed = false; + for index in blocks { + if history.len() > 20 || *budget == 0 { + break; + } + let block = BasicBlock::from_usize(index); + let TerminatorKind::Call { + func, + args, + destination, + target: Some(target), + unwind, + .. + } = &body[block].terminator().kind + else { + continue; + }; + let Some(destination) = destination.as_local() else { + continue; + }; + let ty::FnDef(def_id, generic_args) = *func.ty(&body.local_decls, tcx).kind() else { + continue; + }; + let Some(generic_args) = generic_args.no_bound_vars() else { + continue; + }; + let Some(instance) = + Instance::try_resolve(tcx, TypingEnv::fully_monomorphized(), def_id, generic_args) + .ok() + .flatten() + else { + continue; + }; + if matches!( + tcx.def_kind(instance.def_id()), + rustc_hir::def::DefKind::Ctor(..) + ) && let ty::Adt(def, args_ty) = *body.local_decls[destination].ty.kind() + && let Some((variant, _)) = def + .variants() + .iter_enumerated() + .find(|(_, v)| v.ctor_def_id() == Some(instance.def_id())) + { + let source = body[block].terminator().source_info; + let assignment = Rvalue::Aggregate( + Box::new(AggregateKind::Adt(def.did(), variant, args_ty, None, None)), + args.iter().map(|arg| arg.node.clone()).collect(), + ); + let target = *target; + body[block].statements.push(Statement::new( + source, + StatementKind::Assign(Box::new((destination.into(), assignment))), + )); + body[block].terminator_mut().kind = TerminatorKind::Goto { target }; + changed = true; + continue; + } + if !matches!( + instance.def, + InstanceKind::Item(_) + | InstanceKind::Shim(ShimKind::ClosureOnce { .. } | ShimKind::FnPtr(..)) + ) || history.contains(&instance) + || (matches!(instance.def, InstanceKind::Item(_)) + && !tcx.is_mir_available(instance.def_id())) + { + continue; + } + if matches!( + tcx.constness(instance.def_id()), + rustc_hir::Constness::Const { always: true } + ) { + continue; + } + let attrs = tcx.codegen_fn_attrs(instance.def_id()); + if matches!(attrs.inline, InlineAttr::Never) + || matches!(attrs.optimize, rustc_attr_ir::OptimizeAttr::DoNotOptimize) + || !attrs.target_features.is_empty() + { + continue; + } + let sig = tcx + .fn_sig(def_id) + .instantiate(tcx, generic_args) + .skip_norm_wip(); + if !matches!(sig.abi(), ExternAbi::Rust | ExternAbi::RustCall) { + continue; + } + let original = tcx.instance_mir(instance.def); + if original.coroutine.is_some() { + continue; + } + let cost: usize = original + .basic_blocks + .iter() + .map(|b| b.statements.len() + 5) + .sum(); + let scalar_state = tcx.is_closure_like(instance.def_id()) + || sig.inputs_and_output().iter().any(|ty| { + let ty = tcx.instantiate_bound_regions_with_erased(ty); + let candidate = match ty.kind() { + ty::Tuple(fields) => !fields.is_empty() && fields.len() <= 8, + ty::Closure(..) => true, + ty::Adt(def, _) => { + !def.is_union() + && def.variants().len() <= 4 + && def.variants().iter().all(|v| v.fields.len() <= 8) + } + _ => false, + }; + candidate && !ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) + }); + if cost > if scalar_state { 300 } else { 80 } + || (matches!(attrs.inline, InlineAttr::None) + && !tcx.is_closure_like(instance.def_id()) + && cost > 150) + || cost > *budget + || original.basic_blocks.iter().any(|b| { + matches!( + b.terminator().kind, + TerminatorKind::TailCall { .. } + | TerminatorKind::InlineAsm { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::CoroutineDrop + ) + }) + { + continue; + } + let Ok(mut callee) = instance.try_instantiate_mir_and_normalize_erasing_regions( + tcx, + TypingEnv::fully_monomorphized(), + EarlyBinder::bind(tcx, original.clone()), + ) else { + continue; + }; + if callee.return_ty() != body.local_decls[destination].ty { + continue; + } + let mut arguments: Vec<_> = args.iter().map(|a| a.node.clone()).collect(); + if sig.abi() == ExternAbi::RustCall && callee.spread_arg.is_none() { + let Some(packed) = arguments.pop() else { + continue; + }; + let ty::Tuple(fields) = packed.ty(&body.local_decls, tcx).kind() else { + continue; + }; + let unpacked = if let Some(tuple) = packed.place() { + fields + .iter() + .enumerate() + .map(|(i, ty)| { + Operand::Move(tcx.mk_place_field( + tuple, + rustc_abi::FieldIdx::from_usize(i), + ty, + )) + }) + .collect::>() + } else if fields.is_empty() { + Vec::new() + } else if let Operand::Constant(constant) = packed + && let Some((_, operands)) = + super::inline_records::constant_fields(tcx, &constant, false) + { + operands + } else { + continue; + }; + arguments.extend(unpacked); + } + if arguments.len() != callee.arg_count + || arguments + .iter() + .zip(callee.args_iter()) + .any(|(a, l)| a.ty(&body.local_decls, tcx) != callee.local_decls[l].ty) + { + continue; + } + let source_info = body[block].terminator().source_info; + let target = *target; + let unwind = *unwind; + let local_start = body.local_decls.len(); + let block_start = body.basic_blocks.len(); + let mut locals = vec![destination]; + for decl in callee.local_decls.iter().skip(1) { + locals.push(body.local_decls.push(decl.clone())); + } + for (argument, local) in arguments.into_iter().zip(locals.iter().copied().skip(1)) { + body[block].statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + local.into(), + Rvalue::Use(argument, WithRetag::Yes), + ))), + )); + } + let cleanup = body[block].is_cleanup; + let scope_start = body.source_scopes.len(); + let scope_id = |scope: SourceScope| SourceScope::from_usize(scope_start + scope.index()); + let parent = &body.source_scopes[source_info.scope]; + let inlined_parent = if parent.inlined.is_some() { + Some(source_info.scope) + } else { + parent.inlined_parent_scope + }; + for scope in &mut callee.source_scopes { + if scope.parent_scope.is_none() { + scope.parent_scope = Some(source_info.scope); + scope.inlined_parent_scope = inlined_parent; + scope.inlined = Some((instance, source_info.span)); + } else { + scope.parent_scope = scope.parent_scope.map(scope_id); + scope.inlined_parent_scope = Some(scope_id( + scope.inlined_parent_scope.unwrap_or(OUTERMOST_SOURCE_SCOPE), + )); + } + } + body.source_scopes.append(&mut callee.source_scopes); + let mut remap = Remap { + tcx, + locals: &locals, + block_start, + target, + unwind, + scope_start, + }; + for (bb, data) in callee.basic_blocks_mut().iter_enumerated_mut() { + remap.visit_basic_block_data(bb, data); + data.is_cleanup |= cleanup; + } + for decl in body.local_decls.iter_mut().skip(local_start) { + decl.source_info.scope = scope_id(decl.source_info.scope); + } + body.basic_blocks_mut().append(callee.basic_blocks_mut()); + body[block].terminator_mut().kind = TerminatorKind::Goto { + target: BasicBlock::from_usize(block_start), + }; + *budget -= cost; + changed = true; + history.push(instance); + let end = body.basic_blocks.len(); + expand_blocks(tcx, body, block_start..end, history, budget); + history.pop(); + } + changed +} + +struct Remap<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + locals: &'a [Local], + block_start: usize, + target: BasicBlock, + unwind: UnwindAction, + scope_start: usize, +} +impl Remap<'_, '_> { + fn block(&self, block: BasicBlock) -> BasicBlock { + BasicBlock::from_usize(self.block_start + block.index()) + } + fn unwind(&self, unwind: UnwindAction) -> UnwindAction { + match unwind { + UnwindAction::Cleanup(bb) => UnwindAction::Cleanup(self.block(bb)), + UnwindAction::Continue => self.unwind, + other => other, + } + } +} +impl<'tcx> MutVisitor<'tcx> for Remap<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + fn visit_local(&mut self, local: &mut Local, _: mir::visit::PlaceContext, _: Location) { + *local = self.locals[local.index()]; + } + fn visit_source_scope(&mut self, scope: &mut SourceScope) { + *scope = SourceScope::from_usize(self.scope_start + scope.index()); + } + fn visit_terminator(&mut self, term: &mut Terminator<'tcx>, location: Location) { + if !matches!(term.kind, TerminatorKind::Return) { + self.super_terminator(term, location); + } else { + self.visit_source_info(&mut term.source_info); + } + match &mut term.kind { + TerminatorKind::Return => { + term.kind = TerminatorKind::Goto { + target: self.target, + } + } + TerminatorKind::UnwindResume => { + term.kind = match self.unwind { + UnwindAction::Cleanup(bb) => TerminatorKind::Goto { target: bb }, + UnwindAction::Continue => TerminatorKind::UnwindResume, + UnwindAction::Unreachable => TerminatorKind::Unreachable, + UnwindAction::Terminate(reason) => TerminatorKind::UnwindTerminate(reason), + } + } + other => { + let unwind = other.unwind_mut().map(|u| self.unwind(*u)); + other.successors_mut(|successor| *successor = self.block(*successor)); + if let Some(mapped) = unwind { + *other.unwind_mut().unwrap() = mapped; + } + } + } + } +} diff --git a/src/lower1/inline_enums.rs b/src/lower1/inline_enums.rs new file mode 100644 index 00000000..b27c998b --- /dev/null +++ b/src/lower1/inline_enums.rs @@ -0,0 +1,382 @@ +//! Keep private enum discriminants and payloads in separate locals. +use rustc_middle::ty::util::IntTypeExt; +use rustc_middle::{ + mir::{ + visit::{MutVisitor, PlaceContext, Visitor}, + *, + }, + ty::{self, TyCtxt, TypingEnv}, +}; + +struct Layout<'tcx> { + tag: Local, + fields: Vec)>>, +} + +pub(super) fn simplify<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let mut candidates = body + .local_decls + .iter_enumerated() + .map(|(local, decl)| { + if local.index() <= body.arg_count { + return None; + } + let ty::Adt(def, args) = decl.ty.kind() else { + return None; + }; + if !def.is_enum() + || def.variants().len() > 4 + || decl.ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) + { + return None; + } + let count: usize = def.variants().iter().map(|v| v.fields.len()).sum(); + if count > 8 { + return None; + } + Some((*def, *args)) + }) + .collect::>(); + for data in body.basic_blocks.as_mut() { + for statement in &mut data.statements { + if let StatementKind::Assign(a) = &statement.kind + && a.0.projection.is_empty() + && let Some((def, _)) = candidates[a.0.local.index()] + && let Rvalue::Use(Operand::Constant(constant), _) = &a.1 + && let Some((variant, operands)) = + super::inline_records::constant_fields(tcx, constant, true) + { + let (_, args) = candidates[a.0.local.index()].unwrap(); + statement.kind = StatementKind::Assign(Box::new(( + a.0, + Rvalue::Aggregate( + Box::new(AggregateKind::Adt(def.did(), variant, args, None, None)), + operands.into_iter().collect(), + ), + ))); + } + } + } + struct Uses<'a, 'tcx> { + candidates: &'a mut [Option<(ty::AdtDef<'tcx>, ty::GenericArgsRef<'tcx>)>], + copies: Vec<(Local, Local)>, + defined: Vec, + } + impl<'tcx> Visitor<'tcx> for Uses<'_, 'tcx> { + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + if let StatementKind::Assign(a) = &statement.kind + && let Some(dest) = a.0.as_local() + { + self.defined[dest.index()] = true; + if let Rvalue::Use(Operand::Copy(source) | Operand::Move(source), _) = a.1 + && let Some(source) = source.as_local() + { + self.copies.push((source, dest)); + return; + } + } + match &statement.kind { + StatementKind::Assign(a) + if a.0.projection.is_empty() + && matches!(&a.1, Rvalue::Aggregate(_, _) | Rvalue::Discriminant(_)) => + { + if let Rvalue::Discriminant(p) = &a.1 { + if p.projection.is_empty() { + return; + } + } + self.visit_rvalue(&a.1, location); + } + StatementKind::SetDiscriminant { place, .. } if place.projection.is_empty() => { + self.defined[place.local.index()] = true; + } + _ => self.super_statement(statement, location), + } + } + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + if !matches!(context, PlaceContext::NonUse(_)) + && (context.is_borrow() + || context.is_address_of() + || !matches!( + place.projection.as_ref(), + [ProjectionElem::Downcast(..), ProjectionElem::Field(..), ..] + )) + { + self.candidates[place.local.index()] = None; + } + self.super_place(place, context, location); + } + } + let mut uses = Uses { + candidates: &mut candidates, + copies: Vec::new(), + defined: vec![false; body.local_decls.len()], + }; + for (bb, data) in body.basic_blocks.iter_enumerated() { + uses.visit_basic_block_data(bb, data); + } + let Uses { + copies, defined, .. + } = uses; + for (candidate, defined) in candidates.iter_mut().zip(defined) { + if !defined { + *candidate = None; + } + } + loop { + let mut changed = false; + for &(source, dest) in &copies { + if candidates[dest.index()].is_none() && candidates[source.index()].take().is_some() { + changed = true; + } + } + if !changed { + break; + } + } + if candidates.iter().all(Option::is_none) { + return false; + } + split_copies(tcx, body, &candidates); + let mut layouts = Vec::new(); + for candidate in &candidates { + let Some((def, args)) = candidate else { + layouts.push(None); + continue; + }; + let tag = body.local_decls.push(LocalDecl::new( + def.repr().discr_type().to_ty(tcx), + body.span, + )); + let fields = def + .variants() + .iter() + .map(|v| { + v.fields + .iter() + .map(|f| { + let ty = f.ty(tcx, args).skip_norm_wip(); + (body.local_decls.push(LocalDecl::new(ty, body.span)), ty) + }) + .collect() + }) + .collect(); + layouts.push(Some(Layout { tag, fields })); + } + layouts.resize_with(body.local_decls.len(), || None); + for data in body.basic_blocks_mut() { + let mut result = Vec::new(); + for mut statement in std::mem::take(&mut data.statements) { + let source = statement.source_info; + match &mut statement.kind { + StatementKind::Assign(a) => { + if a.0.projection.is_empty() + && let Some(layout) = &layouts[a.0.local.index()] + { + let Rvalue::Aggregate(kind, operands) = &a.1 else { + unreachable!() + }; + let AggregateKind::Adt(_, variant, ..) = **kind else { + unreachable!() + }; + let (def, _) = candidates[a.0.local.index()].unwrap(); + let ty = def.repr().discr_type().to_ty(tcx); + let tag = def.discriminant_for_variant(tcx, variant).val; + let value = Operand::const_from_scalar( + tcx, + ty, + rustc_middle::mir::interpret::Scalar::from_uint( + tag, + tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .unwrap() + .size, + ), + source.span, + ); + result.push(Statement::new( + source, + StatementKind::Assign(Box::new(( + layout.tag.into(), + Rvalue::Use(value, WithRetag::Yes), + ))), + )); + for ((local, _), operand) in + layout.fields[variant.index()].iter().zip(operands.iter()) + { + result.push(Statement::new( + source, + StatementKind::Assign(Box::new(( + (*local).into(), + Rvalue::Use(operand.clone(), WithRetag::Yes), + ))), + )); + } + continue; + } + if let Rvalue::Discriminant(place) = a.1 + && place.projection.is_empty() + && let Some(layout) = &layouts[place.local.index()] + { + a.1 = Rvalue::Use(Operand::Copy(layout.tag.into()), WithRetag::Yes); + } + } + StatementKind::SetDiscriminant { + place, + variant_index, + } if place.projection.is_empty() && layouts[place.local.index()].is_some() => { + let (def, _) = candidates[place.local.index()].unwrap(); + let ty = def.repr().discr_type().to_ty(tcx); + let value = Operand::const_from_scalar( + tcx, + ty, + rustc_middle::mir::interpret::Scalar::from_uint( + def.discriminant_for_variant(tcx, *variant_index).val, + tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .unwrap() + .size, + ), + source.span, + ); + statement.kind = StatementKind::Assign(Box::new(( + layouts[place.local.index()].as_ref().unwrap().tag.into(), + Rvalue::Use(value, WithRetag::Yes), + ))); + } + _ => {} + } + result.push(statement); + } + data.statements = result; + } + struct Fields<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + layouts: &'a [Option>], + } + impl<'tcx> MutVisitor<'tcx> for Fields<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, loc: Location) { + if let Some(Some(layout)) = self.layouts.get(place.local.index()) + && let [ + ProjectionElem::Downcast(_, variant), + ProjectionElem::Field(field, _), + rest @ .., + ] = place.projection.as_ref() + { + *place = Place::from(layout.fields[variant.index()][field.index()].0) + .project_deeper(rest, self.tcx); + } + self.super_place(place, context, loc); + } + } + let mut visitor = Fields { + tcx, + layouts: &layouts, + }; + for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { + visitor.visit_basic_block_data(bb, data); + } + true +} + +fn split_copies<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + candidates: &[Option<(ty::AdtDef<'tcx>, ty::GenericArgsRef<'tcx>)>], +) { + let mut index = 0; + while index < body.basic_blocks.len() { + let block = BasicBlock::from_usize(index); + index += 1; + let found = body[block] + .statements + .iter() + .enumerate() + .find_map(|(position, statement)| { + let StatementKind::Assign(a) = &statement.kind else { + return None; + }; + let Some((def, args)) = candidates.get(a.0.local.index()).copied().flatten() else { + return None; + }; + let Rvalue::Use(Operand::Copy(source) | Operand::Move(source), _) = a.1 else { + return None; + }; + a.0.as_local() + .map(|dest| (position, dest, source, def, args, statement.source_info)) + }); + let Some((position, dest, source, def, args, source_info)) = found else { + continue; + }; + let is_cleanup = body[block].is_cleanup; + let statements = body[block].statements.split_off(position + 1); + body[block].statements.pop(); + let terminator = body[block].terminator.take(); + let continuation = body.basic_blocks_mut().push(BasicBlockData::new_stmts( + statements, terminator, is_cleanup, + )); + let tag = body.local_decls.push(LocalDecl::new( + def.repr().discr_type().to_ty(tcx), + source_info.span, + )); + body[block].statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new((tag.into(), Rvalue::Discriminant(source)))), + )); + let mut targets = Vec::new(); + for (variant, data) in def.variants().iter_enumerated() { + let mut statements = vec![Statement::new( + source_info, + StatementKind::SetDiscriminant { + place: Box::new(dest.into()), + variant_index: variant, + }, + )]; + for (field, decl) in data.fields.iter_enumerated() { + let ty = decl.ty(tcx, args).skip_norm_wip(); + let projections = [ + ProjectionElem::Downcast(None, variant), + ProjectionElem::Field(field, ty), + ]; + let dest = Place::from(dest).project_deeper(&projections, tcx); + let source = source.project_deeper(&projections, tcx); + statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + dest, + Rvalue::Use(Operand::Copy(source), WithRetag::Yes), + ))), + )); + } + let target = body.basic_blocks_mut().push(BasicBlockData::new_stmts( + statements, + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { + target: continuation, + }, + loop_hint_attrs: Default::default(), + }), + is_cleanup, + )); + targets.push((def.discriminant_for_variant(tcx, variant).val, target)); + } + let invalid = body.basic_blocks_mut().push(BasicBlockData::new( + Some(Terminator { + source_info, + kind: TerminatorKind::Unreachable, + loop_hint_attrs: Default::default(), + }), + is_cleanup, + )); + body[block].terminator = Some(Terminator { + source_info, + loop_hint_attrs: Default::default(), + kind: TerminatorKind::SwitchInt { + discr: Operand::Copy(tag.into()), + targets: SwitchTargets::new(targets.into_iter(), invalid), + }, + }); + } +} diff --git a/src/lower1/inline_niches.rs b/src/lower1/inline_niches.rs new file mode 100644 index 00000000..f303d2dc --- /dev/null +++ b/src/lower1/inline_niches.rs @@ -0,0 +1,138 @@ +//! Expose scalar niche casts before local enum decomposition. +use rustc_abi::{TagEncoding, Variants}; +use rustc_middle::{ + mir::*, + ty::{self, TyCtxt, TypingEnv}, +}; + +pub(super) fn expand<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let env = TypingEnv::fully_monomorphized(); + let mut index = 0; + while index < body.basic_blocks.len() { + let block = BasicBlock::from_usize(index); + index += 1; + let found = body[block] + .statements + .iter() + .enumerate() + .find_map(|(position, statement)| { + let StatementKind::Assign(a) = &statement.kind else { + return None; + }; + let Rvalue::Cast(CastKind::Transmute, operand, ty) = &a.1 else { + return None; + }; + if !operand.ty(&body.local_decls, tcx).is_integral() { + return None; + } + let ty::Adt(def, args) = ty.kind() else { + return None; + }; + if !def.is_enum() || def.variants().len() != 2 || ty.needs_drop(tcx, env) { + return None; + } + let layout = tcx.layout_of(env.as_query_input(*ty)).ok()?; + let Variants::Multiple { + tag, + tag_encoding: + TagEncoding::Niche { + untagged_variant, + niche_variants, + niche_start, + }, + tag_field, + .. + } = &layout.variants + else { + return None; + }; + if tag.size(&tcx.data_layout) != layout.size + || niche_variants.start != niche_variants.last + || layout.fields.offset((*tag_field).into()).bytes() != 0 + || !def.variant(niche_variants.start).fields.is_empty() + { + return None; + } + let fields = &def.variant(*untagged_variant).fields; + if fields.len() != 1 { + return None; + } + let field = fields[rustc_abi::FieldIdx::ZERO] + .ty(tcx, args) + .skip_norm_wip(); + if tcx.layout_of(env.as_query_input(field)).ok()?.size != layout.size { + return None; + } + Some(( + position, + a.0, + operand.clone(), + *def, + *args, + *untagged_variant, + niche_variants.start, + *niche_start, + field, + statement.source_info, + )) + }); + let Some((position, dest, operand, def, args, full, empty, niche, field, source_info)) = + found + else { + continue; + }; + let cleanup = body[block].is_cleanup; + let rest = body[block].statements.split_off(position + 1); + body[block].statements.pop(); + let terminator = body[block].terminator.take(); + let done = body + .basic_blocks_mut() + .push(BasicBlockData::new_stmts(rest, terminator, cleanup)); + let payload = body + .local_decls + .push(LocalDecl::new(field, source_info.span)); + let mut targets = Vec::new(); + for variant in [empty, full] { + let mut statements = Vec::new(); + let values = if variant == full { + statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + payload.into(), + Rvalue::Cast(CastKind::Transmute, operand.clone(), field), + ))), + )); + vec![Operand::Move(payload.into())] + } else { + Vec::new() + }; + statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + dest, + Rvalue::Aggregate( + Box::new(AggregateKind::Adt(def.did(), variant, args, None, None)), + values.into_iter().collect(), + ), + ))), + )); + targets.push(body.basic_blocks_mut().push(BasicBlockData::new_stmts( + statements, + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { target: done }, + loop_hint_attrs: Default::default(), + }), + cleanup, + ))); + } + body[block].terminator = Some(Terminator { + source_info, + kind: TerminatorKind::SwitchInt { + discr: operand, + targets: SwitchTargets::new(std::iter::once((niche, targets[0])), targets[1]), + }, + loop_hint_attrs: Default::default(), + }); + } +} diff --git a/src/lower1/inline_places.rs b/src/lower1/inline_places.rs new file mode 100644 index 00000000..b11c5f34 --- /dev/null +++ b/src/lower1/inline_places.rs @@ -0,0 +1,173 @@ +//! Substitute local places for private reference temporaries. +use rustc_middle::{ + mir::{ + visit::{MutVisitor, PlaceContext, Visitor}, + *, + }, + ty::{self, TyCtxt}, +}; + +pub(super) fn simplify<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut writes = Writes(vec![0; body.local_decls.len()]); + for local in body.args_iter() { + writes.0[local.index()] = 2; + } + writes.visit_body(body); + let mut places = vec![None; body.local_decls.len()]; + loop { + let mut assigned = vec![0; places.len()]; + let mut candidates = vec![None; places.len()]; + let mut conflicts = vec![false; places.len()]; + for block in body.basic_blocks.iter() { + for statement in &block.statements { + let StatementKind::Assign(assignment) = &statement.kind else { + continue; + }; + let Some(local) = assignment.0.as_local() else { + continue; + }; + let index = local.index(); + if writes.0[index] == 0 || places[index].is_some() { + continue; + } + assigned[index] += 1; + let place = match &assignment.1 { + Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => { + Some(expand(tcx, *place, &places)) + } + Rvalue::Use(Operand::Copy(place) | Operand::Move(place), _) + if place.projection.is_empty() => + { + places[place.local.index()] + } + _ => None, + }; + if let Some(place) = place { + if candidates[index].is_some_and(|previous| previous != place) { + conflicts[index] = true; + } + candidates[index] = Some(place); + } else { + conflicts[index] = true; + } + } + } + let mut changed = false; + for (index, candidate) in candidates.into_iter().enumerate() { + if conflicts[index] || assigned[index] != writes.0[index] { + continue; + } + let Some(place) = candidate else { + continue; + }; + let borrowed = matches!( + body.local_decls[Local::from_usize(index)].ty.kind(), + ty::Ref(..) + ); + let stable = place.iter_projections().all(|(base, projection)| { + let ty = base.ty(&body.local_decls, tcx).ty; + match projection { + // Union fields share storage and must retain their address. + ProjectionElem::Field(..) => match ty.kind() { + ty::Adt(def, _) => !def.is_union(), + ty::Tuple(_) | ty::Closure(..) => true, + _ => false, + }, + // All payload accesses must refer to the same variant storage. + ProjectionElem::Downcast(..) if borrowed => matches!(ty.kind(), + ty::Adt(def, _) if def.variants().iter().filter(|v| !v.fields.is_empty()).count() == 1), + _ => false, + } + }); + if place.local.index() == index || !stable { + continue; + } + places[index] = Some(place); + changed = true; + } + if !changed { + break; + } + } + let mut visitor = Replace { + tcx, + places: &places, + }; + for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { + visitor.visit_basic_block_data(bb, data); + } + loop { + let mut reads = Reads(vec![false; body.local_decls.len()]); + for (bb, data) in body.basic_blocks.iter_enumerated() { + reads.visit_basic_block_data(bb, data); + } + let mut changed = false; + for data in body.basic_blocks_mut() { + for statement in &mut data.statements { + if let StatementKind::Assign(assignment) = &statement.kind + && let Some(local) = assignment.0.as_local() + && !reads.0[local.index()] + && assignment.1.is_safe_to_remove() + { + statement.kind = StatementKind::Nop; + changed = true; + } + } + } + if !changed { + break; + } + } +} + +fn expand<'tcx>( + tcx: TyCtxt<'tcx>, + place: Place<'tcx>, + places: &[Option>], +) -> Place<'tcx> { + if place.projection.first() == Some(&ProjectionElem::Deref) + && let Some(base) = places[place.local.index()] + { + return base.project_deeper(&place.projection[1..], tcx); + } + place +} + +struct Writes(Vec); +impl<'tcx> Visitor<'tcx> for Writes { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + if place.projection.first() != Some(&ProjectionElem::Deref) + && (context.is_place_assignment() || context.is_borrow() || context.is_address_of()) + { + self.0[place.local.index()] += 1; + } + self.super_place(place, context, location); + } +} +struct Replace<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + places: &'a [Option>], +} +impl<'tcx> MutVisitor<'tcx> for Replace<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + *place = expand(self.tcx, *place, self.places); + self.super_place(place, context, location); + } +} +struct Reads(Vec); +impl<'tcx> Visitor<'tcx> for Reads { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + if place.projection.is_empty() && context.is_place_assignment() { + return; + } + self.super_place(place, context, location); + } + fn visit_local(&mut self, local: Local, context: PlaceContext, _: Location) { + if context.is_use() { + self.0[local.index()] = true; + } + } +} diff --git a/src/lower1/inline_records.rs b/src/lower1/inline_records.rs new file mode 100644 index 00000000..dc7964de --- /dev/null +++ b/src/lower1/inline_records.rs @@ -0,0 +1,308 @@ +//! Split private records before reference lowering. +use rustc_middle::{ + mir::{ + visit::{MutVisitor, PlaceContext, Visitor}, + *, + }, + ty::{self, TyCtxt, TypingEnv}, +}; + +pub(super) fn simplify<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let mut fields = body + .local_decls + .iter_enumerated() + .map(|(local, decl)| { + if local.index() <= body.arg_count + || decl.ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) + { + return None; + } + let fields: Vec<_> = match decl.ty.kind() { + ty::Tuple(fields) => fields.to_vec(), + ty::Adt(def, args) + if def.is_struct() + && !tcx.is_lang_item( + def.did(), + rustc_attr_ir::lang_items::LangItem::UnsafeCell, + ) => + { + def.non_enum_variant() + .fields + .iter() + .map(|f| f.ty(tcx, args).skip_norm_wip()) + .collect() + } + ty::Closure(_, args) => args.as_closure().upvar_tys().iter().collect(), + _ => return None, + }; + (fields.len() <= 8).then_some(fields) + }) + .collect::>(); + for data in body.basic_blocks.as_mut() { + for statement in &mut data.statements { + let StatementKind::Assign(a) = &mut statement.kind else { + continue; + }; + let Some(local) = a.0.as_local() else { + continue; + }; + if fields[local.index()].is_none() { + continue; + } + let Rvalue::Use(Operand::Constant(constant), _) = &a.1 else { + continue; + }; + let ty = body.local_decls[local].ty; + let Some((_, operands)) = constant_fields(tcx, constant, false) else { + continue; + }; + let kind = match ty.kind() { + ty::Tuple(_) => AggregateKind::Tuple, + ty::Adt(def, args) => { + AggregateKind::Adt(def.did(), rustc_abi::VariantIdx::ZERO, args, None, None) + } + ty::Closure(def, args) => AggregateKind::Closure(*def, args), + _ => continue, + }; + a.1 = Rvalue::Aggregate(Box::new(kind), operands.into_iter().collect()); + } + } + for index in 0..body.basic_blocks.len() { + let bb = BasicBlock::from_usize(index); + let mut output = Vec::new(); + for mut statement in std::mem::take(&mut body[bb].statements) { + if let StatementKind::Assign(a) = &mut statement.kind + && let Some(local) = a.0.as_local() + && let Some(record) = &fields[local.index()] + && record.len() == 1 + && let Rvalue::Cast(CastKind::Transmute, value, ty) = &a.1 + && value.ty(&body.local_decls, tcx).is_integral() + && let ty::Adt(def, args) = ty.kind() + && let Ok(layout) = + tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(*ty)) + && let Ok(field_layout) = + tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(record[0])) + && layout.size == field_layout.size + && layout.fields.offset(0).bytes() == 0 + { + let field = body + .local_decls + .push(LocalDecl::new(record[0], statement.source_info.span)); + output.push(Statement::new( + statement.source_info, + StatementKind::Assign(Box::new(( + field.into(), + Rvalue::Cast(CastKind::Transmute, value.clone(), record[0]), + ))), + )); + a.1 = Rvalue::Aggregate( + Box::new(AggregateKind::Adt( + def.did(), + rustc_abi::VariantIdx::ZERO, + args, + None, + None, + )), + std::iter::once(Operand::Move(field.into())).collect(), + ); + } + output.push(statement); + } + body[bb].statements = output; + } + fields.resize(body.local_decls.len(), None); + struct Uses<'a, 'tcx> { + fields: &'a mut [Option>>], + copies: Vec<(Local, Local)>, + defined: Vec, + } + impl<'tcx> Visitor<'tcx> for Uses<'_, 'tcx> { + fn visit_statement(&mut self, s: &Statement<'tcx>, loc: Location) { + if let StatementKind::Assign(a) = &s.kind + && let Some(dest) = a.0.as_local() + { + if matches!(a.1, Rvalue::Aggregate(_, _)) { + self.defined[dest.index()] = true; + self.visit_rvalue(&a.1, loc); + return; + } + if let Rvalue::Use(Operand::Copy(source) | Operand::Move(source), _) = a.1 + && let Some(source) = source.as_local() + { + self.defined[dest.index()] = true; + self.copies.push((source, dest)); + return; + } + } + self.super_statement(s, loc); + } + fn visit_place(&mut self, p: &Place<'tcx>, context: PlaceContext, loc: Location) { + if context.is_use() + && (context.is_borrow() + || context.is_address_of() + || !matches!(p.projection.first(), Some(ProjectionElem::Field(..)))) + { + self.fields[p.local.index()] = None; + } + self.super_place(p, context, loc); + } + } + let mut uses = Uses { + fields: &mut fields, + copies: Vec::new(), + defined: vec![false; body.local_decls.len()], + }; + for (bb, data) in body.basic_blocks.iter_enumerated() { + uses.visit_basic_block_data(bb, data); + } + let Uses { + copies, defined, .. + } = uses; + for (field, defined) in fields.iter_mut().zip(defined) { + if !defined { + *field = None; + } + } + loop { + let mut changed = false; + for &(source, dest) in &copies { + if fields[dest.index()].is_none() && fields[source.index()].take().is_some() { + changed = true; + } + } + if !changed { + break; + } + } + if fields.iter().all(Option::is_none) { + return false; + } + let layouts: Vec<_> = fields + .iter() + .map(|f| { + f.as_ref().map(|f| { + f.iter() + .map(|&ty| body.local_decls.push(LocalDecl::new(ty, body.span))) + .collect::>() + }) + }) + .collect(); + for data in body.basic_blocks_mut() { + let mut output = Vec::new(); + for statement in std::mem::take(&mut data.statements) { + if let StatementKind::Assign(a) = &statement.kind + && let Some(dest) = a.0.as_local() + && let Some(locals) = &layouts[dest.index()] + { + let operands: Vec<_> = match &a.1 { + Rvalue::Aggregate(_, operands) => operands.iter().cloned().collect(), + Rvalue::Use(Operand::Copy(source) | Operand::Move(source), _) => fields + [dest.index()] + .as_ref() + .unwrap() + .iter() + .enumerate() + .map(|(i, &ty)| { + Operand::Copy(tcx.mk_place_field( + *source, + rustc_abi::FieldIdx::from_usize(i), + ty, + )) + }) + .collect(), + _ => unreachable!(), + }; + assert_eq!(locals.len(), operands.len()); + for (&local, operand) in locals.iter().zip(operands) { + output.push(Statement::new( + statement.source_info, + StatementKind::Assign(Box::new(( + local.into(), + Rvalue::Use(operand, WithRetag::Yes), + ))), + )); + } + } else { + output.push(statement); + } + } + data.statements = output; + } + struct Fields<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + layouts: &'a [Option>], + } + impl<'tcx> MutVisitor<'tcx> for Fields<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, loc: Location) { + if let Some(Some(fields)) = self.layouts.get(place.local.index()) + && let [ProjectionElem::Field(field, _), rest @ ..] = place.projection.as_ref() + { + *place = Place::from(fields[field.index()]).project_deeper(rest, self.tcx); + } + self.super_place(place, context, loc); + } + } + let mut visitor = Fields { + tcx, + layouts: &layouts, + }; + for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { + visitor.visit_basic_block_data(bb, data); + } + true +} + +pub(super) fn constant_fields<'tcx>( + tcx: TyCtxt<'tcx>, + constant: &ConstOperand<'tcx>, + is_enum: bool, +) -> Option<(rustc_abi::VariantIdx, Vec>)> { + let env = TypingEnv::fully_monomorphized(); + let value = constant.const_.eval(tcx, env, constant.span).ok()?; + let (ecx, operand) = rustc_const_eval::const_eval::mk_eval_cx_for_const_val( + tcx.at(constant.span), + env, + value, + constant.const_.ty(), + )?; + let variant = if is_enum { + ecx.read_discriminant(&operand).discard_err()? + } else { + rustc_abi::VariantIdx::ZERO + }; + let operand = if is_enum { + ecx.project_downcast(&operand, variant).discard_err()? + } else { + operand + }; + let fields = (0..operand.layout.fields.count()) + .map(|index| { + let field = ecx + .project_field(&operand, rustc_abi::FieldIdx::from_usize(index)) + .discard_err()?; + if field.layout.is_zst() { + return Some(Operand::zero_sized_constant(field.layout.ty, constant.span)); + } + if !matches!( + field.layout.ty.kind(), + ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) + ) { + return None; + } + let immediate = ecx.read_immediate(&field).discard_err()?; + let rustc_const_eval::interpret::Immediate::Scalar(value) = *immediate else { + return None; + }; + Some(Operand::const_from_scalar( + tcx, + field.layout.ty, + value, + constant.span, + )) + }) + .collect::>>()?; + Some((variant, fields)) +} diff --git a/src/lower1/naming.rs b/src/lower1/naming.rs index 3cd8fea8..752a18b3 100644 --- a/src/lower1/naming.rs +++ b/src/lower1/naming.rs @@ -4,9 +4,7 @@ use super::jvm_names; use rustc_attr_ir::lang_items::LangItem; use rustc_hir::def::DefKind; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::{ - GenericArg, Instance, InstanceKind, ShimKind, TyCtxt, TyKind, TypeVisitableExt, -}; +use rustc_middle::ty::{GenericArg, Instance, InstanceKind, ShimKind, TyCtxt, TyKind}; use rustc_span::{def_id::DefId, sym}; const MAX_MONO_FN_NAME_LEN: usize = 128; @@ -151,6 +149,20 @@ pub fn parse_jvm_class_link_name(link_name: &str) -> Result { Ok(class_name.to_string()) } +fn internal_instance(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + tcx.generics_of(def_id).requires_monomorphization(tcx) + || tcx.opt_associated_item(def_id).is_some() + || (matches!(tcx.def_kind(def_id), DefKind::Fn) + && !(tcx.visibility(def_id).is_public() + && crate::java_exports::is_exported(tcx, def_id)) + // The launcher discovers the Rust entry point by its Java name. + && tcx.entry_fn(()).is_none_or(|(entry, _)| entry != def_id)) + || matches!( + tcx.def_kind(tcx.parent(def_id)), + DefKind::Fn | DefKind::AssocFn | DefKind::Closure + ) +} + pub fn mono_owner_class<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> String { let def_id = instance.def_id(); // The runtime polls an async body through a carrier companion, independently of @@ -161,19 +173,11 @@ pub fn mono_owner_class<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> St jvm_names::anonymous_class_for_args(tcx, def_id, instance.args, true) ); } - if tcx.generics_of(def_id).requires_monomorphization(tcx) { - // Definition-crate ownership agrees across all downstream emitters. + if internal_instance(tcx, def_id) { + // Java receiver methods forward to private Rust implementations. The private ABI does not + // require boxed borrows. let identity = super::types::stable_instance_identity(tcx, def_id, instance.args); generic_mono_owner(tcx, def_id, &identity) - } else if let Some(trait_def_id) = tcx - .opt_associated_item(def_id) - .and_then(|item| item.trait_container(tcx)) - { - // A Java interface carries its dynamically dispatched method, but - // monomorphized Rust default bodies are static functions. Put those - // bodies beside the trait in its module rather than generating a - // second class file for the interface name. - jvm_names::owner_class_for_function(tcx, trait_def_id) } else { jvm_names::owner_class_for_function(tcx, def_id) } @@ -645,14 +649,7 @@ pub fn mono_fn_name_from_instance<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'t let base = jvm_names::method_for_function(tcx, instance.def_id()); // Internal names identify the Rust definition and its substitutions. // Deriving a readable name must never construct carrier/codec bodies. - let internal = tcx - .generics_of(instance.def_id()) - .requires_monomorphization(tcx) - || tcx.opt_associated_item(instance.def_id()).is_some() - || matches!( - tcx.def_kind(tcx.parent(instance.def_id())), - DefKind::Fn | DefKind::AssocFn | DefKind::Closure - ); + let internal = internal_instance(tcx, instance.def_id()); let mut method_name = if internal { let identity = super::types::stable_instance_identity(tcx, instance.def_id(), instance.args); diff --git a/src/lower1/operand.rs b/src/lower1/operand.rs index 93ea118e..7046deb6 100644 --- a/src/lower1/operand.rs +++ b/src/lower1/operand.rs @@ -220,27 +220,10 @@ pub fn convert_operand<'tcx>( } let copy_id = data_types.next_temporary(); - let object_ty = oomir::Type::Class("java/lang/Object".to_string()); - let object_dest = format!("{final_var_name}_copy_object_{copy_id}"); let copy_dest = format!("{final_var_name}_copy_{copy_id}"); - instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(object_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "copyManagedValue".to_string(), - method_ty: oomir::Signature { - params: vec![("value".to_string(), object_ty.clone())], - ret: Box::new(object_ty.clone()), - is_static: true, - }, - args: vec![loaded], - }); - instructions.push(oomir::Instruction::Cast { - op: oomir::Operand::Variable { - name: object_dest, - ty: object_ty, - }, - ty: final_type.clone(), + instructions.push(oomir::Instruction::ValueCopy { dest: copy_dest.clone(), + source: loaded, }); oomir::Operand::Variable { name: copy_dest, diff --git a/src/lower1/operand/const_eval.rs b/src/lower1/operand/const_eval.rs index 9b5860ea..9423ba1c 100644 --- a/src/lower1/operand/const_eval.rs +++ b/src/lower1/operand/const_eval.rs @@ -15,7 +15,8 @@ use rustc_span::def_id::LOCAL_CRATE; use super::super::{ control_flow::rvalue::{ - ensure_closure_fn_pointer_adapter_class, ensure_fn_pointer_adapter_class, fn_pointer_target, + FnPointerTarget, callable_handle, ensure_closure_fn_pointer_bridge, + ensure_fn_pointer_adapter_class, fn_pointer_target, }, control_flow::trait_objects::ensure_trait_object_adapter_class_for_pointees, jvm_names, ty_to_oomir_type, @@ -138,40 +139,49 @@ pub fn read_pointer_constant<'tcx>( ) } } - _ => { - let layout = tcx - .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) - .map_err(|error| format!("Could not determine scalar constant layout: {error:?}"))?; - if !matches!(layout.backend_repr, BackendRepr::Scalar(_)) - || layout.size != tcx.data_layout.pointer_size() - { - return Err(format!("Unexpected pointer scalar for constant type {ty:?}")); - } - // This pointer is the aggregate's contents, not its address. Put - // the scalar in temporary storage so the normal decoder preserves - // enum niches, field types, and function-pointer provenance. - let mut allocation = - ConstAllocation::new(layout.size, layout.align.abi, AllocInit::Uninit, ()); - allocation - .write_scalar( - &tcx.data_layout, - AllocRange { - start: Size::ZERO, - size: layout.size, - }, - Scalar::from_pointer(pointer, &tcx.data_layout), - ) - .map_err(|error| format!("Could not store pointer scalar: {error:?}"))?; - read_constant_value_from_memory( - tcx, - &allocation, - Size::ZERO, - ty, - oomir_data_types, - instance, - ) - } + _ => read_scalar_aggregate_constant( + tcx, + Scalar::from_pointer(pointer, &tcx.data_layout), + ty, + oomir_data_types, + instance, + ), + } +} + +fn read_scalar_aggregate_constant<'tcx>( + tcx: TyCtxt<'tcx>, + scalar: Scalar, + ty: Ty<'tcx>, + data_types: &mut Definitions<'tcx>, + instance: Instance<'tcx>, +) -> Result { + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .map_err(|error| format!("Could not determine scalar constant layout: {error:?}"))?; + let size = match scalar { + Scalar::Int(value) => value.size(), + Scalar::Ptr(..) => tcx.data_layout.pointer_size(), + }; + if !matches!(layout.backend_repr, BackendRepr::Scalar(_)) || layout.size != size { + return Err(format!( + "Unexpected {size:?} scalar for constant type {ty:?}" + )); } + // The scalar contains aggregate data. The decoder requires storage to recover fields, enum + // niches, and pointer provenance. + let mut allocation = ConstAllocation::new(layout.size, layout.align.abi, AllocInit::Uninit, ()); + allocation + .write_scalar( + &tcx.data_layout, + AllocRange { + start: Size::ZERO, + size, + }, + scalar, + ) + .map_err(|error| format!("Could not store aggregate scalar: {error:?}"))?; + read_constant_value_from_memory(tcx, &allocation, Size::ZERO, ty, data_types, instance) } fn pointer_references_static(tcx: TyCtxt<'_>, pointer: Pointer) -> bool { @@ -525,27 +535,47 @@ fn read_function_pointer_constant<'tcx>( _ => None, }; - let adapter_class = if let Some(closure_instance) = closure_instance { - ensure_closure_fn_pointer_adapter_class( + if let Some(closure_instance) = closure_instance { + let (owner, name) = ensure_closure_fn_pointer_bridge( oomir_data_types, closure_instance, &signature, - &interface_name, tcx, instance, - ) - } else { - let callable_target = - fn_pointer_target(tcx, oomir_data_types, function_instance, &signature); - ensure_fn_pointer_adapter_class( - oomir_data_types, - callable_target.as_ref(), + ); + return Ok(callable_handle( &signature, &interface_name, - tcx, - instance, - ) - }; + owner, + name, + false, + )); + } + let target = fn_pointer_target(tcx, oomir_data_types, function_instance, &signature); + if let Some(FnPointerTarget::Static(target) | FnPointerTarget::ImportedStatic(target)) = &target + { + if let Some(owner) = &target.class_to_call_on + && (!signature.needs_component_abi() + || oomir::component_method(owner, &target.method_name)) + { + let is_interface = oomir_data_types.foreign_interfaces.borrow().contains(owner); + return Ok(callable_handle( + &signature, + &interface_name, + owner.clone(), + target.method_name.clone(), + is_interface, + )); + } + } + let adapter_class = ensure_fn_pointer_adapter_class( + oomir_data_types, + target.as_ref(), + &signature, + &interface_name, + tcx, + instance, + ); Ok(oomir::Constant::FunctionPointer { adapter_class, @@ -713,7 +743,7 @@ fn read_pointee_constant<'tcx>( .layout_of(TypingEnv::fully_monomorphized().as_query_input(concrete_ty)) .map_err(|error| format!("Could not determine vtable layout: {error:?}"))?; let concrete_oomir_ty = ty_to_oomir_type(concrete_ty, tcx, oomir_data_types, instance); - let pointer_oomir_ty = oomir::Type::Pointer(Box::new(concrete_oomir_ty)); + let pointer_oomir_ty = oomir::Type::pointer(concrete_oomir_ty); let oomir::Type::Interface(interface_name) = ty_to_oomir_type(dyn_ty, tcx, oomir_data_types, instance) else { @@ -750,12 +780,12 @@ fn read_pointee_constant<'tcx>( pointee_codec, ], param_types: Vec::new(), - ty: oomir::Type::Pointer(Box::new(ty_to_oomir_type( + ty: oomir::Type::pointer(ty_to_oomir_type( pointee_ty, tcx, oomir_data_types, instance, - ))), + )), }) } GlobalAlloc::TypeId { ty } => Err(format!("Unsupported constant pointer to TypeId {ty:?}")), diff --git a/src/lower1/operand/const_eval/memory.rs b/src/lower1/operand/const_eval/memory.rs index 8ecc1a13..b9622f29 100644 --- a/src/lower1/operand/const_eval/memory.rs +++ b/src/lower1/operand/const_eval/memory.rs @@ -23,6 +23,99 @@ pub(crate) fn read_constant_value_from_memory<'tcx>( .layout_of(pci) .map_err(|_| "Couldn't get layout.".to_string())?; + if let Some(layout) = crate::lower1::types::tagged_scalar(ty, tcx) { + let tag = read_constant_value_from_memory( + tcx, + allocation, + offset + Size::from_bytes(layout.tag_offset), + tcx.types.u64, + oomir_data_types, + instance, + )?; + let present = matches!(tag, oomir::Constant::U64(1)); + let value = if present { + read_constant_value_from_memory( + tcx, + allocation, + offset + Size::from_bytes(layout.payload_offset), + layout.payload, + oomir_data_types, + instance, + )? + } else { + oomir::Constant::I64(0) + }; + return Ok(oomir::Constant::Instance { + class_name: oomir::TAGGED_LONG_CLASS.into(), + params: vec![value, tag], + param_types: vec![oomir::Type::I64, oomir::Type::I64], + }); + } + if let Some(scalar) = crate::lower1::types::value_scalar_ty(ty, tcx) { + return read_constant_value_from_memory( + tcx, + allocation, + offset, + scalar, + oomir_data_types, + instance, + ); + } + if let Some(payload) = crate::lower1::types::transparent_payload(ty, tcx) { + return read_constant_value_from_memory( + tcx, + allocation, + offset, + payload.ty, + oomir_data_types, + instance, + ); + } + + if let Some(carrier) = crate::lower1::types::enum_carrier(ty, tcx) { + let payload = carrier.payload; + if !carrier.nullable { + return read_constant_value_from_memory( + tcx, + allocation, + offset, + payload, + oomir_data_types, + instance, + ); + } + let scalar = allocation + .read_scalar( + &tcx.data_layout, + AllocRange { + start: offset, + // None initializes only the pointer word. Slice metadata can remain + // uninitialized. + size: tcx.data_layout.pointer_size(), + }, + true, + ) + .map_err(|error| format!("Failed to read nullable pointer: {error:?}"))?; + if let Scalar::Int(value) = scalar + && value.to_target_usize(tcx) == 0 + { + return Ok(oomir::Constant::Null(ty_to_oomir_type( + payload, + tcx, + oomir_data_types, + instance, + ))); + } + return read_constant_value_from_memory( + tcx, + allocation, + offset, + payload, + oomir_data_types, + instance, + ); + } + // A reference to a ZST can point at a zero-byte allocation, so there are // no bytes to decode. Reconstruct its nominal JVM value from the type. if layout.is_zst() { @@ -415,7 +508,11 @@ pub(crate) fn read_constant_value_from_memory<'tcx>( .then_some(value) .into_iter() .collect(), - param_types: Vec::new(), + param_types: field_oomir_ty + .has_jvm_value() + .then_some(field_oomir_ty) + .into_iter() + .collect(), ty: oomir::Type::Class(class_name), }); } diff --git a/src/lower1/operand/const_eval/objects.rs b/src/lower1/operand/const_eval/objects.rs index aefb2bcc..221cc54f 100644 --- a/src/lower1/operand/const_eval/objects.rs +++ b/src/lower1/operand/const_eval/objects.rs @@ -390,7 +390,7 @@ pub(crate) fn handle_constant_enum<'tcx>( let variant_class_name = format!( "{}${}", // Using '$' as inner class separator is common in JVM base_enum_name, - jvm_names::member_name(&variant_def.ident(tcx).to_string()) + crate::lower1::types::enum_variant_name(variant_def, tcx) ); Ok(oomir::Constant::Instance { diff --git a/src/lower1/operand/const_eval/scalars.rs b/src/lower1/operand/const_eval/scalars.rs index 181f27a2..8bedbedb 100644 --- a/src/lower1/operand/const_eval/scalars.rs +++ b/src/lower1/operand/const_eval/scalars.rs @@ -11,6 +11,25 @@ pub(crate) fn read_scalar_int_constant<'tcx>( let ty = EarlyBinder::bind(tcx, ty) .instantiate(tcx, instance.args) .skip_norm_wip(); + if let Some(payload) = crate::lower1::types::transparent_payload(ty, tcx) { + return read_scalar_int_constant(tcx, scalar_int, payload.ty, oomir_data_types, instance); + } + if let Some(scalar) = crate::lower1::types::value_scalar_ty(ty, tcx) { + return scalar_int_to_oomir_constant(tcx, scalar_int, scalar); + } + if let Some(carrier) = crate::lower1::types::enum_carrier(ty, tcx) { + let payload = carrier.payload; + return if carrier.nullable && scalar_int.to_target_usize(tcx) == 0 { + Ok(oomir::Constant::Null(ty_to_oomir_type( + payload, + tcx, + oomir_data_types, + instance, + ))) + } else { + read_scalar_int_constant(tcx, scalar_int, payload, oomir_data_types, instance) + }; + } if let TyKind::Pat(base_ty, _) = ty.kind() { return read_scalar_int_constant(tcx, scalar_int, *base_ty, oomir_data_types, instance); } @@ -38,52 +57,16 @@ pub(crate) fn read_scalar_int_constant<'tcx>( }); } - if let TyKind::Closure(_, closure_args) = ty.kind() { - let class_name = match ty_to_oomir_type(ty, tcx, oomir_data_types, instance) { - oomir::Type::Class(class_name) => class_name, - other => { - return Err(format!( - "Scalar closure {ty:?} did not map to a JVM class: {other:?}" - )); - } - }; - let mut params = Vec::new(); - let mut param_types = Vec::new(); - let mut non_zst_captures = 0usize; - for capture_ty in closure_args.as_closure().upvar_tys().iter() { - let capture_ty = EarlyBinder::bind(tcx, capture_ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); - let capture_layout = tcx - .layout_of(TypingEnv::fully_monomorphized().as_query_input(capture_ty)) - .map_err(|error| { - format!( - "Could not determine closure capture layout for {capture_ty:?}: {error:?}" - ) - })?; - let capture_jvm_ty = ty_to_oomir_type(capture_ty, tcx, oomir_data_types, instance); - if !capture_jvm_ty.has_jvm_value() { - continue; - } - let capture = if capture_layout.is_zst() { - read_zero_sized_constant(tcx, capture_ty, oomir_data_types, instance)? - } else { - non_zst_captures += 1; - read_scalar_int_constant(tcx, scalar_int, capture_ty, oomir_data_types, instance)? - }; - params.push(capture); - param_types.push(capture_jvm_ty); - } - if non_zst_captures != 1 { - return Err(format!( - "Scalar closure {ty:?} has {non_zst_captures} non-ZST captures, expected exactly one" - )); - } - return Ok(oomir::Constant::Instance { - class_name, - params, - param_types, - }); + if matches!(ty.kind(), TyKind::Closure(..)) { + // A scalar closure environment can contain nested fields. The decoder must preserve their + // layout and enum niches. + return read_scalar_aggregate_constant( + tcx, + Scalar::Int(scalar_int), + ty, + oomir_data_types, + instance, + ); } let scalar_carrier_ty = || match scalar_int.size().bytes() { @@ -186,7 +169,7 @@ pub(crate) fn scalar_struct_field_ty<'tcx>( pub(crate) fn instance_constant_with_declared_fields( class_name: String, named_values: Vec<(String, oomir::Constant)>, - oomir_data_types: &HashMap, + oomir_data_types: &crate::lower1::context::Definitions<'_>, ) -> oomir::Constant { let declared_fields: &[(String, oomir::Type)] = match oomir_data_types.get(&class_name) { Some(oomir::DataType::Class { fields, .. }) => fields.as_slice(), @@ -227,6 +210,9 @@ pub(crate) fn read_zero_sized_constant<'tcx>( if !layout.is_zst() { return Err(format!("Type {ty:?} is not zero-sized")); } + if let Some(payload) = crate::lower1::types::direct_enum_payload(ty, tcx) { + return read_zero_sized_constant(tcx, payload, oomir_data_types, instance); + } let oomir_ty = ty_to_oomir_type(ty, tcx, oomir_data_types, instance); if !oomir_ty.has_jvm_value() { @@ -288,14 +274,19 @@ pub(crate) fn read_zero_sized_constant<'tcx>( return Err(format!("ZST struct {ty:?} did not map to a class")); }; let mut values = Vec::new(); - for field in &adt_def.variant(VariantIdx::from_usize(0)).fields { + for (index, field) in adt_def + .variant(VariantIdx::from_usize(0)) + .fields + .iter() + .enumerate() + { let field_ty = EarlyBinder::bind(tcx, field.ty(tcx, substs).skip_norm_wip()) .instantiate(tcx, instance.args) .skip_norm_wip(); let field_jvm_ty = ty_to_oomir_type(field_ty, tcx, oomir_data_types, instance); if field_jvm_ty.has_jvm_value() { values.push(( - field.ident(tcx).to_string(), + crate::lower1::types::struct_field_name(tcx, adt_def, index), read_zero_sized_constant(tcx, field_ty, oomir_data_types, instance)?, )); } @@ -335,7 +326,7 @@ pub(crate) fn read_zero_sized_constant<'tcx>( let class_name = format!( "{}${}", base_class, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); let mut values = Vec::new(); for (field_index, field) in variant.fields.iter().enumerate() { diff --git a/src/lower1/operand/const_eval/views.rs b/src/lower1/operand/const_eval/views.rs index e9fd362c..73a342bb 100644 --- a/src/lower1/operand/const_eval/views.rs +++ b/src/lower1/operand/const_eval/views.rs @@ -136,7 +136,7 @@ pub(crate) fn slice_tailed_pointer_constant<'tcx>( }; let java_object = oomir::Type::Class("java/lang/Object".to_string()); let java_string = oomir::Type::java_string(); - let data_pointer_type = oomir::Type::Pointer(Box::new(element_type)); + let data_pointer_type = oomir::Type::pointer(element_type); let data_pointer = oomir::Constant::StaticCall { owner_class: oomir::POINTER_CLASS.to_string(), method_name: "fromSlice".to_string(), @@ -169,7 +169,7 @@ pub(crate) fn slice_tailed_pointer_constant<'tcx>( java_string, oomir::Type::U64, ], - ty: oomir::Type::Pointer(Box::new(pointee_type)), + ty: oomir::Type::pointer(pointee_type), }) } @@ -442,7 +442,7 @@ pub(crate) fn slice_ref_constant<'tcx>( element_codec, ], param_types: Vec::new(), - ty: oomir::Type::Pointer(Box::new(element_type.clone())), + ty: oomir::Type::pointer(element_type.clone()), } } else { backing diff --git a/src/lower1/place.rs b/src/lower1/place.rs index 1c2a7c8d..d24197a7 100644 --- a/src/lower1/place.rs +++ b/src/lower1/place.rs @@ -1,5 +1,4 @@ use super::{ - jvm_names, operand::convert_operand, types::{ adapt_simple_enum_operand, enum_variant_field_name, force_define_named_adt, @@ -17,17 +16,19 @@ use rustc_middle::{ }; pub(crate) fn has_slice_or_str_struct_tail<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + struct_tail_for_place(tcx, ty).is_some_and(|tail| tail.is_slice() || tail.is_str()) +} + +fn struct_tail_for_place<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option> { if !matches!(ty.kind(), TyKind::Adt(adt_def, _) if adt_def.is_struct()) { - return false; + return None; } tcx.try_normalize_erasing_regions( TypingEnv::fully_monomorphized(), rustc_middle::ty::Unnormalized::new_wip(ty), ) - .is_ok_and(|normalized| { - let tail = tcx.struct_tail_for_codegen(normalized, TypingEnv::fully_monomorphized()); - tail.is_slice() || tail.is_str() - }) + .ok() + .map(|normalized| tcx.struct_tail_for_codegen(normalized, TypingEnv::fully_monomorphized())) } pub fn place_to_string<'tcx>(place: &Place<'tcx>, _tcx: TyCtxt<'tcx>) -> String { @@ -39,6 +40,38 @@ pub(crate) fn local_cell_name(local: Local) -> String { format!("_cell_{}", local.index()) } +pub(in crate::lower1) fn project_memory<'tcx>( + dest: String, + base: Operand, + owner: String, + field: String, + field_ty: Ty<'tcx>, + offset: u64, + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + definitions: &mut Definitions<'tcx>, +) -> Instruction { + let field_ty = definitions.normalize(tcx, field_ty, instance); + let codec = match pointer_view_codec_operand(field_ty, tcx, definitions, instance) { + Operand::Constant(oomir::Constant::String(codec)) => Some(codec), + Operand::Constant(oomir::Constant::Null(_)) => None, + _ => unreachable!("field memory layout must be constant"), + }; + Instruction::MemoryProject { + dest, + base, + projection: Box::new(oomir::MemoryProjection { + owner, + field, + pointee: ty_to_oomir_type(field_ty, tcx, definitions, instance), + offset, + size: super::types::layout_size_bytes(tcx, field_ty) + .expect("sized struct field must have a layout") as u64, + codec, + }), + } +} + fn union_parts_from_ty<'tcx>(ty: Ty<'tcx>) -> Option<(AdtDef<'tcx>, GenericArgsRef<'tcx>)> { match ty.kind() { TyKind::Adt(adt_def, substs) if adt_def.is_union() => Some((*adt_def, substs)), @@ -108,10 +141,6 @@ fn collect_union_writebacks<'tcx>( }; let class_name = match ty_to_oomir_type(union_base_ty, tcx, data_types, instance) { oomir::Type::Class(name) => name, - oomir::Type::Reference(inner) => match inner.as_ref() { - oomir::Type::Class(name) => name.clone(), - other => panic!("Union field access through non-class reference: {other:?}"), - }, other => panic!("Union field access on non-class type: {other:?}"), }; let field_name = union_field_name(union_def, field_index.index(), tcx); @@ -189,17 +218,12 @@ fn collect_memory_view_writebacks(get_instructions: &[Instruction]) -> Vec - { - Some(operand.clone()) - } + } if pointee.is_jvm_reference_type() => Some(pointer.clone()), _ => None, }) .collect() @@ -207,20 +231,8 @@ fn collect_memory_view_writebacks(get_instructions: &[Instruction]) -> Vec) { for pointer in writebacks.iter().rev() { - let pointer_ty = pointer - .get_type() - .expect("a memory-view writeback requires a typed pointer"); - instructions.push(Instruction::InvokeVirtual { - dest: None, - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "commitMemoryView".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), pointer_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - args: vec![], - operand: pointer.clone(), + instructions.push(Instruction::MemoryCommit { + pointer: pointer.clone(), }); } } @@ -233,11 +245,9 @@ fn field_name_from_rust_ty<'tcx>( match ty.kind() { TyKind::Ref(_, inner_ty, _) => field_name_from_rust_ty(*inner_ty, field_index, tcx), TyKind::Tuple(_) => Some(format!("field{}", field_index)), - TyKind::Adt(adt_def, _) if adt_def.is_struct() => adt_def - .variant(0usize.into()) - .fields - .get(rustc_abi::FieldIdx::from_usize(field_index)) - .map(|field| field.ident(tcx).to_string()), + TyKind::Adt(adt_def, _) if adt_def.is_struct() => { + Some(super::types::struct_field_name(tcx, adt_def, field_index)) + } TyKind::Adt(adt_def, _) if adt_def.is_enum() => None, TyKind::Coroutine(_, _) => Some(format!("arg{}", field_index)), _ => None, @@ -270,7 +280,7 @@ pub(super) fn field_name_for_projection<'tcx>( field_index: usize, base_rust_ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, - data_types: &HashMap, + data_types: &crate::lower1::context::Definitions<'_>, ) -> Result { let direct_rust_ty = match base_rust_ty.kind() { TyKind::Ref(_, inner_ty, _) => *inner_ty, @@ -281,7 +291,7 @@ pub(super) fn field_name_for_projection<'tcx>( && let Some(variant) = adt_def.variants().iter().find(|variant| { owner_class_name.ends_with(&format!( "${}", - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) )) }) { @@ -328,7 +338,10 @@ pub fn get_place_type<'tcx>( let base_ty = EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) .instantiate(tcx, instance.args) .skip_norm_wip(); - if matches!(base_ty.kind(), TyKind::Coroutine(..)) { + if matches!(base_ty.kind(), TyKind::Coroutine(..)) + || super::types::tagged_scalar(base_ty, tcx).is_some() + || super::types::direct_enum_payload(base_ty, tcx).is_some() + { return ty_to_oomir_type(base_ty, tcx, data_types, instance); } let (adt_def, substs) = match base_ty.kind() { @@ -349,7 +362,7 @@ pub fn get_place_type<'tcx>( return oomir::Type::Class(format!( "{}${}", base_class, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) )); } } @@ -362,10 +375,12 @@ pub fn get_place_type<'tcx>( ty_to_oomir_type(instantiated_ty, tcx, data_types, instance) } +mod memory_place; /// Generates OOMIR instructions to "get" the value from a Place. /// This function now supports nested projections by calling /// `emit_instructions_to_get_recursive`. mod views; +use memory_place::indirect_field_address; pub(crate) use views::*; mod read; diff --git a/src/lower1/place/memory_place.rs b/src/lower1/place/memory_place.rs new file mode 100644 index 00000000..8334c464 --- /dev/null +++ b/src/lower1/place/memory_place.rs @@ -0,0 +1,60 @@ +//! Field projections keep the original storage owner. They do not decode the enclosing allocation. +use super::*; + +pub(super) fn indirect_field_address<'tcx>( + place: &Place<'tcx>, + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + definitions: &mut Definitions<'tcx>, + instructions: &mut Vec, +) -> Option<(Operand, oomir::Type)> { + let deref = place + .projection + .iter() + .rposition(|p| matches!(p, ProjectionElem::Deref))?; + let fields = &place.projection[deref + 1..]; + if fields.is_empty() + || fields.len() > 32 + || !fields + .iter() + .all(|p| matches!(p, ProjectionElem::Field(..))) + { + return None; + } + let reference = projection_prefix_place(place, deref, tcx); + if !matches!( + get_place_type(&reference, mir, tcx, instance, definitions), + oomir::Type::Pointer(_) + ) { + return None; + } + for index in deref + 1..place.projection.len() { + let parent = projection_prefix_place(place, index, tcx); + let ty = definitions.normalize(tcx, parent.ty(&mir.local_decls, tcx).ty, instance); + if !matches!(ty.kind(), TyKind::Tuple(_)) + && !matches!(ty.kind(), TyKind::Adt(def, _) if def.is_struct()) + { + return None; + } + } + let rust_ty = definitions.normalize(tcx, place.ty(&mir.local_decls, tcx).ty, instance); + if !super::super::types::is_codegen_sized(rust_ty, tcx) { + return None; + } + let ty = ty_to_oomir_type(rust_ty, tcx, definitions, instance); + if !ty.has_jvm_value() { + return None; + } + let pointer = super::super::control_flow::rvalue::emit_pointer_to_place( + place, + &oomir::Type::pointer(ty.clone()), + &format!("{}_storage_field", place_to_string(place, tcx)), + tcx, + instance, + mir, + definitions, + instructions, + ); + Some((pointer, ty)) +} diff --git a/src/lower1/place/read.rs b/src/lower1/place/read.rs index 4f85b74a..2cf7be94 100644 --- a/src/lower1/place/read.rs +++ b/src/lower1/place/read.rs @@ -1,6 +1,12 @@ //! Read operations on Rust places. use super::*; +fn is_struct_tail_place<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + struct_tail_for_place(tcx, ty).is_some_and(|tail| { + tail.is_slice() || tail.is_str() || matches!(tail.kind(), TyKind::Dynamic(..)) + }) +} + pub(crate) fn emit_instructions_to_get_recursive<'tcx>( place: &Place<'tcx>, tcx: TyCtxt<'tcx>, @@ -8,6 +14,14 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( mir: &Body<'tcx>, data_types: &mut Definitions<'tcx>, ) -> (String, Vec, oomir::Type) { + let mut instructions = Vec::new(); + if let Some((pointer, ty)) = + indirect_field_address(place, tcx, instance, mir, data_types, &mut instructions) + { + let dest = format!("{}_field_value", place_to_string(place, tcx)); + emit_pointer_read(pointer, &ty, &dest, &mut instructions); + return (dest, instructions, ty); + } // Start with the base local. let current_place = Place { local: place.local, @@ -15,7 +29,6 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( }; let mut current_var = place_to_string(¤t_place, tcx); let mut current_type = get_place_type(¤t_place, mir, tcx, instance, data_types); - let mut instructions = vec![]; let mut coroutine_variant = None; if data_types.local_uses_stable_cell(place.local) { let pointee_type = current_type.clone(); @@ -23,7 +36,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( let value = emit_pointer_read( Operand::Variable { name: local_cell_name(place.local), - ty: oomir::Type::Pointer(Box::new(pointee_type.clone())), + ty: oomir::Type::pointer(pointee_type.clone()), }, &pointee_type, &value_name, @@ -58,11 +71,70 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( match proj { ProjectionElem::Field(field_index, field_ty) => { let base_place_for_field = projection_prefix_place(place, proj_index, tcx); - let base_rust_ty = - EarlyBinder::bind(tcx, base_place_for_field.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let base_rust_ty = data_types.normalize( + tcx, + base_place_for_field.ty(&mir.local_decls, tcx).ty, + instance, + ); + if let Some(payload) = super::super::types::transparent_payload(base_rust_ty, tcx) { + if field_index.index() == payload.field { + current_type = ty_to_oomir_type(field_ty, tcx, data_types, instance); + } else { + let ty = data_types.normalize(tcx, field_ty, instance); + let dest = format!("{current_var}_marker_{}", field_index.index()); + current_type = ty_to_oomir_type(ty, tcx, data_types, instance); + let field = if current_type.has_jvm_value() { + super::super::value_repr::materialize_implicit_zst( + ty, + &dest, + tcx, + instance, + data_types, + &mut instructions, + ) + .expect("owner marker is a zero-sized value") + } else { + oomir::Operand::Constant(oomir::Constant::Unit) + }; + instructions.push(Instruction::Move { + dest: dest.clone(), + src: field, + }); + current_var = dest; + } + continue; + } + + if let Some(word) = super::super::types::packed_word(base_rust_ty, tcx) { + let field_type = ty_to_oomir_type(field_ty, tcx, data_types, instance); + let dest = format!("{current_var}_packed_{}", field_index.index()); + let source = Operand::Variable { + name: current_var, + ty: current_type.clone(), + }; + let value = if matches!(current_type, oomir::Type::Pointer(_)) { + let address = word.field_address( + source, + field_index.index(), + &field_type, + &dest, + &mut instructions, + ); + emit_pointer_read(address, &field_type, &dest, &mut instructions) + } else { + word.extract( + source, + field_index.index(), + &field_type, + &dest, + &mut instructions, + ) + }; + current_var = value.get_name().unwrap().to_string(); + current_type = field_type; + continue; + } if field_index.index() == 0 && proj_index > 0 && let ProjectionElem::Downcast(_, variant_idx) = @@ -78,6 +150,27 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( continue; } + if field_index.index() == 0 && matches!(current_type, oomir::Type::TaggedI64) { + let dest = format!("{current_var}_payload"); + instructions.push(Instruction::TaggedPart { + dest: dest.clone(), + value: Operand::Variable { + name: current_var, + ty: current_type, + }, + index: 0, + }); + current_var = dest; + current_type = ty_to_oomir_type(field_ty, tcx, data_types, instance); + continue; + } + if field_index.index() == 0 + && super::super::types::direct_enum_payload(base_rust_ty, tcx).is_some() + { + current_type = ty_to_oomir_type(field_ty, tcx, data_types, instance); + continue; + } + if field_index.index() == 0 && matches!( base_rust_ty.kind(), @@ -107,9 +200,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( ) }); let field_offset = layout.fields.offset(field_index.index()).bytes_usize(); - let field_rust_ty = EarlyBinder::bind(tcx, field_ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let field_rust_ty = data_types.normalize(tcx, field_ty, instance); let base_pointer_name = current_var.clone(); let base_pointer_ty = current_type.clone(); let oomir::Type::Pointer(base_pointee_ty) = &base_pointer_ty else { @@ -127,6 +218,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( data_types, ) .unwrap_or_else(|error| panic!("Error getting struct field name: {error}")); + let memory_offset = field_offset as u64; let field_offset = Operand::Constant(oomir::Constant::U64( u64::try_from(field_offset).expect("Rust struct field offset exceeds u64"), )); @@ -150,7 +242,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( }, args: vec![ Operand::Constant(oomir::Constant::String(owner_class)), - Operand::Constant(oomir::Constant::String(field_name)), + Operand::Constant(oomir::Constant::LiteralString(field_name)), field_offset, ], operand: Operand::Variable { @@ -191,7 +283,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( }, args: vec![ Operand::Constant(oomir::Constant::String(owner_class)), - Operand::Constant(oomir::Constant::String(field_name)), + Operand::Constant(oomir::Constant::LiteralString(field_name)), field_offset, Operand::Constant(oomir::Constant::U64( u64::try_from( @@ -227,47 +319,29 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( current_var = next_var; } else { current_type = ty_to_oomir_type(field_rust_ty, tcx, data_types, instance); - let field_pointer_ty = oomir::Type::Pointer(Box::new(current_type.clone())); + let field_pointer_ty = oomir::Type::pointer(current_type.clone()); let typed_pointer_name = format!("{current_var}_typed_field_pointer"); - instructions.push(Instruction::InvokeVirtual { - dest: Some(typed_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "projectStructField".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), base_pointer_ty.clone()), - ("owner_class".to_string(), oomir::Type::java_string()), - ("field_name".to_string(), oomir::Type::java_string()), - ("field_offset".to_string(), oomir::Type::U64), - ("field_size".to_string(), oomir::Type::U64), - ("field_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(field_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - Operand::Constant(oomir::Constant::String(owner_class)), - Operand::Constant(oomir::Constant::String(field_name)), - field_offset, - Operand::Constant(oomir::Constant::U64( - u64::try_from( - super::super::types::layout_size_bytes(tcx, field_rust_ty) - .expect("sized struct field must have a layout"), - ) - .expect("Rust struct field layout exceeds u64"), - )), - pointer_view_codec_operand( - field_rust_ty, - tcx, - data_types, - instance, - ), - ], - operand: Operand::Variable { + instructions.push(super::project_memory( + typed_pointer_name.clone(), + Operand::Variable { name: base_pointer_name, ty: base_pointer_ty, }, - }); + owner_class, + field_name, + field_rust_ty, + memory_offset, + tcx, + instance, + data_types, + )); + if is_struct_tail_place(tcx, field_rust_ty) { + // Nested DST fields require the address and length. Their static prefix + // does not describe the complete value. + current_var = typed_pointer_name; + current_type = field_pointer_ty; + continue; + } let next_var = format!("{current_var}_{}", field_index.index()); let value = emit_pointer_read( Operand::Variable { @@ -286,15 +360,6 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( let owner_class_name = match ty_to_oomir_type(base_rust_ty, tcx, data_types, instance) { oomir::Type::Class(name) => name, - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Class(_)) => - { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - unreachable!() - } - } other => { panic!("Union field access on non-class OOMIR type: {:?}", other) } @@ -329,18 +394,8 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( // Get the owner class name and field name. let owner_class_name = match ¤t_type { oomir::Type::Class(name) => name.clone(), - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Class(_)) => - { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - unreachable!() - } - } _ => panic!( - "Field access on non-class type: current var '{}' has type: {:?}", - current_var, current_type + "Field access on non-class type: current var '{current_var}' has type {current_type:?}, Rust type {base_rust_ty:?}, in {instance:?}" ), }; @@ -359,7 +414,9 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( ) }) { Ok(name) => name, - Err(e) => panic!("Error getting field name: {}", e), + Err(e) => panic!( + "Error getting field name: {e}, Rust type {base_rust_ty:?}, in {instance:?}" + ), }; coroutine_variant = None; @@ -407,15 +464,6 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( // Determine element type from the current type (which should be an array or reference-to-array). current_type = match ¤t_type { oomir::Type::Array(inner) | oomir::Type::Slice(inner) => inner.as_ref().clone(), - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Array(_)) => - { - if let oomir::Type::Array(element_type) = inner.as_ref() { - element_type.as_ref().clone() - } else { - unreachable!() - } - } _ => panic!( "Index access on non-array type: current var '{}' has type: {:?}", current_var, current_type @@ -441,15 +489,6 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( // Determine element type based on current_type being an array or reference-to-array. current_type = match ¤t_type { oomir::Type::Array(inner) | oomir::Type::Slice(inner) => inner.as_ref().clone(), - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Array(_)) => - { - if let oomir::Type::Array(element_type) = inner.as_ref() { - element_type.as_ref().clone() - } else { - unreachable!() - } - } _ => panic!( "Constant index access on non-array type: current var '{}' has type: {:?}", current_var, current_type @@ -526,8 +565,8 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) => Some(*pointee), _ => None, }; - let preserve_slice_tailed_pointer = pointer_pointee - .is_some_and(|pointee| has_slice_or_str_struct_tail(tcx, pointee)); + let preserve_unsized_pointer = + pointer_pointee.is_some_and(|pointee| is_struct_tail_place(tcx, pointee)); let projects_field = matches!(base_rust_ty.kind(), TyKind::RawPtr(..)) && matches!( place.projection.get(proj_index + 1), @@ -548,7 +587,12 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( // A raw field projection must not read neighboring, possibly // uninitialized fields. References already guarantee a valid // enclosing value, so retain their direct object access. - if preserve_slice_tailed_pointer || projects_field { + let projects_packed_field = matches!( + place.projection.get(proj_index + 1), + Some(ProjectionElem::Field(..)) + ) && pointer_pointee + .is_some_and(|ty| super::super::types::packed_word(ty, tcx).is_some()); + if preserve_unsized_pointer || projects_field || projects_packed_field { continue; } if matches!(type_before_proj, oomir::Type::Slice(_)) @@ -588,51 +632,6 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( current_var = result.get_name().unwrap_or(&next_var).to_string(); current_type = element_type.as_ref().clone(); } - oomir::Type::MutableReference(_) => { - let type_before_deref = current_type.clone(); - - match type_before_deref.clone() { - oomir::Type::MutableReference(element_type) => { - // Create a temporary variable name for the dereferenced value - let next_var = format!("{}_deref", current_var); - - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "place-lowering", - format!( - "Info: Handling Deref: Var '{}' ({:?}) -> Temp Var '{}' (Type: {:?})", - current_var, - type_before_deref, - next_var, - element_type.as_ref() - ) - ); - - instructions.push(oomir::Instruction::ArrayGet { - dest: next_var.clone(), - array: Operand::Variable { - name: current_var.clone(), - ty: type_before_deref, // The type is Array(T) - }, - // Index is always 0 for our reference representation - index: Operand::Constant(oomir::Constant::I32(0)), - }); - - // Update current_var and current_type for subsequent projections - current_var = next_var; - current_type = element_type.as_ref().clone(); // Type becomes T - } - _ => { - panic!( - "Attempted to Deref a non-reference (non-array) type: \ - Variable '{}' has type {:?}. Place: {:?}", - current_var, - current_type, // Use the original current_type for the error - place - ); - } - } - } _ => { // no op } @@ -653,6 +652,12 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( .instantiate(tcx, instance.args) .skip_norm_wip(); + if super::super::types::tagged_scalar(base_rust_ty, tcx).is_some() + || super::super::types::direct_enum_payload(base_rust_ty, tcx).is_some() + { + continue; + } + let coroutine_ty = match base_rust_ty.kind() { TyKind::Coroutine(..) => Some(base_rust_ty), TyKind::Ref(_, inner, _) if matches!(inner.kind(), TyKind::Coroutine(..)) => { @@ -711,7 +716,7 @@ pub(crate) fn emit_instructions_to_get_recursive<'tcx>( let variant_class_name = format!( "{}${}", base_enum_oomir_name, // Use OOMIR name already derived - jvm_names::member_name(&variant_def.name.to_string()) + crate::lower1::types::enum_variant_name(variant_def, tcx) ); let transparent_payload = jvm_subtype_payload_ty(&adt_def, variant_def, substs, tcx); diff --git a/src/lower1/place/views.rs b/src/lower1/place/views.rs index 84df1d43..6f56caf3 100644 --- a/src/lower1/place/views.rs +++ b/src/lower1/place/views.rs @@ -1,171 +1,56 @@ //! Views operations on Rust places. use super::*; -pub(crate) fn pointer_getter_for_type(ty: &oomir::Type) -> (&'static str, oomir::Type) { - match ty { - oomir::Type::Boolean => ("getBoolean", oomir::Type::Boolean), - oomir::Type::I8 | oomir::Type::U8 => ("getI8", oomir::Type::I8), - oomir::Type::I16 | oomir::Type::U16 => ("getI16", oomir::Type::I16), - oomir::Type::F16 => ("getI16", oomir::Type::F16), - oomir::Type::I32 | oomir::Type::U32 | oomir::Type::Char => ("getI32", oomir::Type::I32), - oomir::Type::I64 | oomir::Type::U64 => ("getI64", oomir::Type::I64), - oomir::Type::F32 => ("getF32", oomir::Type::F32), - oomir::Type::F64 => ("getF64", oomir::Type::F64), - _ => ( - "getObject", - oomir::Type::Class("java/lang/Object".to_string()), - ), - } -} - -/// Loads the pointee while retaining the pointer itself as a first-class JVM -/// value. Reference-valued pointees use Object at the runtime boundary and are -/// cast back to their precise OOMIR type immediately afterwards. pub(crate) fn emit_pointer_read( pointer: Operand, pointee_ty: &oomir::Type, dest: &str, instructions: &mut Vec, ) -> Operand { - if !pointee_ty.has_jvm_value() { - return Operand::Constant(oomir::Constant::Unit); - } - let pointer_ty = pointer - .get_type() - .expect("a pointer read requires a typed operand"); - let (mut method_name, runtime_ret_ty) = pointer_getter_for_type(pointee_ty); - let requested_class = match pointee_ty { - oomir::Type::Class(class_name) => Some(class_name.clone()), - oomir::Type::Slice(_) => Some(oomir::SLICE_VIEW_CLASS.to_string()), - _ => None, - }; - if requested_class.is_some() { - method_name = "getObjectAs"; - } - let runtime_dest = if runtime_ret_ty == *pointee_ty { - dest.to_string() - } else { - format!("{dest}_object") - }; - instructions.push(Instruction::InvokeVirtual { - dest: Some(runtime_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: method_name.to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), pointer_ty)] - .into_iter() - .chain( - requested_class - .is_some() - .then_some(("target_class".to_string(), oomir::Type::java_string())), - ) - .collect(), - ret: Box::new(runtime_ret_ty.clone()), - is_static: false, - }, - args: requested_class - .map(|class_name| vec![Operand::Constant(oomir::Constant::String(class_name))]) - .unwrap_or_default(), - operand: pointer, - }); - if runtime_ret_ty != *pointee_ty { - instructions.push(Instruction::Cast { - op: Operand::Variable { - name: runtime_dest, - ty: runtime_ret_ty, - }, - ty: pointee_ty.clone(), - dest: dest.to_string(), - }); - } - Operand::Variable { - name: dest.to_string(), - ty: pointee_ty.clone(), - } + memory_read(pointer, pointee_ty, dest, false, instructions) } -/// Implements Rust's `ptr::read` family. Unlike an ordinary dereference, these -/// operations produce an independent bitwise copy of the pointee. JVM-backed -/// aggregate carriers therefore need to be detached from the memory view. +/// An owned read must not establish a mutable decoded-view binding. pub(crate) fn emit_pointer_read_copy( pointer: Operand, pointee_ty: &oomir::Type, dest: &str, instructions: &mut Vec, ) -> Operand { - if !pointee_ty.is_jvm_reference_type() { - return emit_pointer_read(pointer, pointee_ty, dest, instructions); - } + memory_read(pointer, pointee_ty, dest, true, instructions) +} - let read_start = instructions.len(); - let loaded_dest = format!("{dest}_loaded"); - let loaded = emit_pointer_read(pointer, pointee_ty, &loaded_dest, instructions); - if detach_pointer_read(&mut instructions[read_start..], &loaded_dest) { - instructions.push(Instruction::Move { - dest: dest.to_string(), - src: loaded, - }); - return Operand::Variable { - name: dest.to_string(), - ty: pointee_ty.clone(), - }; +fn memory_read( + pointer: Operand, + pointee_ty: &oomir::Type, + dest: &str, + owned: bool, + instructions: &mut Vec, +) -> Operand { + if !pointee_ty.has_jvm_value() { + return Operand::Constant(oomir::Constant::Unit); } - let object_dest = format!("{dest}_copy_object"); - let object_ty = oomir::Type::Class("java/lang/Object".to_string()); - instructions.push(Instruction::InvokeStatic { - dest: Some(object_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "copyManagedValue".to_string(), - method_ty: oomir::Signature { - params: vec![("value".to_string(), object_ty.clone())], - ret: Box::new(object_ty.clone()), - is_static: true, - }, - args: vec![loaded], - }); - instructions.push(Instruction::Cast { - op: Operand::Variable { - name: object_dest, - ty: object_ty, - }, - ty: pointee_ty.clone(), - dest: dest.to_string(), + instructions.push(Instruction::MemoryLoad { + dest: dest.into(), + pointer, + pointee: pointee_ty.clone(), + owned, }); Operand::Variable { - name: dest.to_string(), + name: dest.into(), ty: pointee_ty.clone(), } } -/// A value copy can decode directly into its owned carrier. Only fuse the final -/// load: earlier pointer reads may be needed as live views for field projections. +/// Only the final read makes an owned copy. Earlier projections need the live owner for writeback. pub(crate) fn detach_pointer_read(instructions: &mut [Instruction], value_name: &str) -> bool { - let Some( - [ - Instruction::InvokeVirtual { - dest: Some(result), - class_name, - method_name, - .. - }, - Instruction::Cast { - dest, - op: Operand::Variable { name, .. }, - .. - }, - ], - ) = instructions.last_chunk_mut::<2>() - else { + let Some(Instruction::MemoryLoad { dest, owned, .. }) = instructions.last_mut() else { return false; }; - if class_name != oomir::POINTER_CLASS - || method_name != "getObjectAs" - || result != name - || dest != value_name - { + if dest != value_name { return false; } - *method_name = "getObjectCopyAs".to_string(); + *owned = true; true } @@ -175,32 +60,13 @@ pub(crate) fn emit_pointer_write( value: Operand, instructions: &mut Vec, ) { - if !pointee_ty.has_jvm_value() { - return; + if pointee_ty.has_jvm_value() { + instructions.push(Instruction::MemoryStore { + pointer, + pointee: pointee_ty.clone(), + value, + }); } - let pointer_ty = pointer - .get_type() - .expect("a pointer write requires a typed operand"); - let value_ty = if pointee_ty.is_jvm_primitive() { - pointee_ty.clone() - } else { - oomir::Type::Class("java/lang/Object".to_string()) - }; - instructions.push(Instruction::InvokeVirtual { - dest: None, - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "set".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), pointer_ty), - ("value".to_string(), value_ty), - ], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - args: vec![value], - operand: pointer, - }); } pub(crate) fn emit_slice_view( @@ -365,11 +231,50 @@ pub(crate) fn emit_pointer_slice_parts( ) } +pub(crate) fn emit_pointer_slice_view( + data: Operand, + length: Operand, + dest: &str, + instructions: &mut Vec, +) -> Operand { + let Some(oomir::Type::Pointer(element)) = data.get_type() else { + panic!("slice data must be an element pointer") + }; + let (backing, start) = if data.get_type().unwrap().scalar_address_size().is_some() { + emit_pointer_slice_parts(data, dest, instructions) + } else { + (data, Operand::Constant(oomir::Constant::I32(0))) + }; + let object = oomir::Type::Class("java/lang/Object".into()); + let carrier = oomir::Type::Class(oomir::SLICE_VIEW_CLASS.into()); + let name = format!("{dest}_view"); + instructions.push(Instruction::ConstructObject { + dest: name.clone(), + class_name: oomir::SLICE_VIEW_CLASS.into(), + args: vec![ + (backing, object), + (start, oomir::Type::I32), + (length, oomir::Type::U64), + ], + }); + let ty = oomir::Type::Slice(element.value); + instructions.push(Instruction::Cast { + dest: dest.into(), + op: Operand::Variable { name, ty: carrier }, + ty: ty.clone(), + }); + Operand::Variable { + name: dest.into(), + ty, + } +} + /// Gives a raw slice data pointer the element view used by `SliceView` accessors. /// This matters when MIR constructs a fat pointer from an erased or whole-array /// view: the backing `Pointer` must advance and load one Rust element at a time. pub(crate) fn emit_retyped_slice_data_pointer( data: Operand, + element_type: oomir::Type, element_size: Operand, element_codec: Operand, dest_prefix: &str, @@ -378,24 +283,19 @@ pub(crate) fn emit_retyped_slice_data_pointer( let data_ty = data .get_type() .expect("slice data pointer must have an OOMIR type"); + let result_ty = oomir::Type::pointer(element_type); let dest = format!("{dest_prefix}_element_pointer"); - instructions.push(Instruction::InvokeStatic { + instructions.push(Instruction::AddressRetype { dest: Some(dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("pointer".to_string(), data_ty.clone()), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(data_ty.clone()), - is_static: true, - }, - args: vec![data, element_size, element_codec], + source: data, + layout: Box::new(oomir::AddressLayout { + pointer_type: result_ty.clone(), + size: element_size, + codec: element_codec, + }), }); Operand::Variable { name: dest, - ty: data_ty, + ty: result_ty, } } diff --git a/src/lower1/place/write.rs b/src/lower1/place/write.rs index bf51b6aa..956d768b 100644 --- a/src/lower1/place/write.rs +++ b/src/lower1/place/write.rs @@ -25,6 +25,17 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( data_types, &mut instructions, ); + if let Some((pointer, ty)) = indirect_field_address( + dest_place, + tcx, + instance, + mir, + data_types, + &mut instructions, + ) { + emit_pointer_write(pointer, &ty, source_operand, &mut instructions); + return instructions; + } if dest_place.projection.is_empty() { if data_types.local_uses_stable_cell(dest_place.local) { @@ -32,7 +43,7 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( emit_pointer_write( Operand::Variable { name: local_cell_name(dest_place.local), - ty: oomir::Type::Pointer(Box::new(pointee_type.clone())), + ty: oomir::Type::pointer(pointee_type.clone()), }, &pointee_type, source_operand, @@ -53,10 +64,135 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( local: dest_place.local, projection: tcx.mk_place_elems(base_projection_elems), }; + if let ProjectionElem::Field(field, field_ty) = last_projection { + let base_rust_ty = + data_types.normalize(tcx, base_place.ty(&mir.local_decls, tcx).ty, instance); + if let Some(payload) = super::super::types::transparent_payload(base_rust_ty, tcx) { + if field.index() == payload.field { + instructions.extend(emit_instructions_to_set_value( + &base_place, + source_operand, + tcx, + instance, + mir, + data_types, + )); + } + return instructions; + } + if let Some(word) = super::super::types::packed_word(base_rust_ty, tcx) { + let dest = format!("{}_packed_store", place_to_string(dest_place, tcx)); + if let Some((ProjectionElem::Deref, prefix)) = base_place.projection.split_last() { + // Partial field writes must preserve adjacent bytes. + let pointer_place = Place { + local: base_place.local, + projection: tcx.mk_place_elems(prefix), + }; + let (name, code, ty) = emit_instructions_to_get_on_own( + &pointer_place, + tcx, + instance, + mir, + data_types, + ); + instructions.extend(code); + let field_type = ty_to_oomir_type(*field_ty, tcx, data_types, instance); + let address = word.field_address( + Operand::Variable { name, ty }, + field.index(), + &field_type, + &dest, + &mut instructions, + ); + emit_pointer_write(address, &field_type, source_operand, &mut instructions); + } else { + let (name, code, ty) = emit_instructions_to_get_on_own( + &base_place, + tcx, + instance, + mir, + data_types, + ); + instructions.extend(code); + let value = word.insert( + Operand::Variable { name, ty }, + field.index(), + source_operand, + &dest, + &mut instructions, + ); + instructions.extend(emit_instructions_to_set_value( + &base_place, + value, + tcx, + instance, + mir, + data_types, + )); + } + return instructions; + } + } if matches!(last_projection, ProjectionElem::Field(field, _) if field.index() == 0) { - let base_rust_ty = EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let base_rust_ty = + data_types.normalize(tcx, base_place.ty(&mir.local_decls, tcx).ty, instance); + if super::super::types::tagged_scalar(base_rust_ty, tcx).is_some() { + let Some((ProjectionElem::Downcast(_, variant), prefix)) = + base_projection_elems.split_last() + else { + panic!("tagged payload needs a variant"); + }; + assert_eq!(variant.as_u32(), 1); + let dest = format!("{}_tagged", place_to_string(dest_place, tcx)); + instructions.push(super::super::types::tagged_value( + dest.clone(), + source_operand, + 1, + )); + instructions.extend(emit_instructions_to_set_value( + &Place { + local: dest_place.local, + projection: tcx.mk_place_elems(prefix), + }, + Operand::Variable { + name: dest, + ty: oomir::Type::TaggedI64, + }, + tcx, + instance, + mir, + data_types, + )); + return instructions; + } + if super::super::types::direct_enum_payload(base_rust_ty, tcx).is_some() { + let Some((ProjectionElem::Downcast(_, variant), prefix)) = + base_projection_elems.split_last() + else { + panic!("nullable enum field needs a variant projection"); + }; + if *variant + != super::super::types::enum_carrier(base_rust_ty, tcx) + .unwrap() + .variant + { + instructions.push(oomir::Instruction::Unreachable); + return instructions; + } + let storage = Place { + local: dest_place.local, + projection: tcx.mk_place_elems(prefix), + }; + instructions.extend(emit_instructions_to_set_value( + &storage, + source_operand, + tcx, + instance, + mir, + data_types, + )); + return instructions; + } if matches!( base_rust_ty.kind(), TyKind::Adt(adt_def, _) @@ -111,10 +247,11 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( && matches!(&reference_type, oomir::Type::Pointer(inner) if matches!(inner.as_ref(), oomir::Type::Class(_))) && { - let ty = - EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let ty = data_types.normalize( + tcx, + base_place.ty(&mir.local_decls, tcx).ty, + instance, + ); matches!(ty.kind(), TyKind::Tuple(_)) || matches!(ty.kind(), TyKind::Adt(def, _) if def.is_struct()) }; @@ -146,9 +283,8 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( // 3. Generate the final store instruction based on the *last* projection. match last_projection { ProjectionElem::Field(field_index, field_mir_ty) => { - let base_rust_ty = EarlyBinder::bind(tcx, base_place.ty(&mir.local_decls, tcx).ty) - .instantiate(tcx, instance.args) - .skip_norm_wip(); + let base_rust_ty = + data_types.normalize(tcx, base_place.ty(&mir.local_decls, tcx).ty, instance); if matches!(&base_oomir_type, oomir::Type::Pointer(inner) if matches!(inner.as_ref(), oomir::Type::Class(_))) && (matches!(base_rust_ty.kind(), TyKind::Tuple(_)) @@ -166,7 +302,7 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( .instantiate(tcx, instance.args) .skip_norm_wip(); let field_oomir_ty = ty_to_oomir_type(field_rust_ty, tcx, data_types, instance); - let field_pointer_ty = oomir::Type::Pointer(Box::new(field_oomir_ty.clone())); + let field_pointer_ty = oomir::Type::pointer(field_oomir_ty.clone()); let oomir::Type::Pointer(base_pointee_ty) = &base_oomir_type else { unreachable!(); }; @@ -183,43 +319,20 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( ) .unwrap_or_else(|error| panic!("Error getting struct field name: {error}")); let typed_pointer_name = format!("{base_var_name}_typed_field_pointer"); - instructions.push(Instruction::InvokeVirtual { - dest: Some(typed_pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "projectStructField".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), base_oomir_type.clone()), - ("owner_class".to_string(), oomir::Type::java_string()), - ("field_name".to_string(), oomir::Type::java_string()), - ("field_offset".to_string(), oomir::Type::U64), - ("field_size".to_string(), oomir::Type::U64), - ("field_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(field_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - Operand::Constant(oomir::Constant::String(owner_class)), - Operand::Constant(oomir::Constant::String(field_name)), - Operand::Constant(oomir::Constant::U64( - u64::try_from(field_offset) - .expect("Rust struct field offset exceeds u64"), - )), - Operand::Constant(oomir::Constant::U64( - u64::try_from( - super::super::types::layout_size_bytes(tcx, field_rust_ty) - .expect("sized struct field must have a layout"), - ) - .expect("Rust struct field layout exceeds u64"), - )), - pointer_view_codec_operand(field_rust_ty, tcx, data_types, instance), - ], - operand: Operand::Variable { + instructions.push(super::project_memory( + typed_pointer_name.clone(), + Operand::Variable { name: base_var_name, ty: base_oomir_type, }, - }); + owner_class, + field_name, + field_rust_ty, + field_offset as u64, + tcx, + instance, + data_types, + )); emit_pointer_write( Operand::Variable { name: typed_pointer_name, @@ -237,15 +350,6 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( let owner_class_name = match ty_to_oomir_type(base_rust_ty, tcx, data_types, instance) { oomir::Type::Class(name) => name, - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Class(_)) => - { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - unreachable!() - } - } other => panic!( "Union field assignment on non-class OOMIR type: {:?}", other @@ -294,15 +398,6 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( // Target is a field: base_var_name.field = source_operand let owner_class_name = match &base_oomir_type { oomir::Type::Class(name) => name.clone(), - oomir::Type::Reference(inner) - if matches!(inner.as_ref(), oomir::Type::Class(_)) => - { - if let oomir::Type::Class(name) = inner.as_ref() { - name.clone() - } else { - unreachable!() - } - } _ => panic!( "SetField target base '{}' (Place: {:?}) is not a class or reference-to-class type: {:?}", base_var_name, base_place, base_oomir_type @@ -351,7 +446,7 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( // Ensure the base is actually an array or ref-to-array match &base_oomir_type { oomir::Type::Array(_) | oomir::Type::Slice(_) => {} - oomir::Type::Reference(t) if matches!(t.as_ref(), oomir::Type::Array(_)) => {} + _ => panic!( "ArrayStore target base '{}' (Place: {:?}) is not an array or reference-to-array type: {:?}", base_var_name, base_place, base_oomir_type @@ -389,7 +484,7 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( // Ensure the base is actually an array or ref-to-array match &base_oomir_type { oomir::Type::Array(_) | oomir::Type::Slice(_) => {} - oomir::Type::Reference(t) if matches!(t.as_ref(), oomir::Type::Array(_)) => {} + _ => panic!( "ArrayStore target base '{}' (Place: {:?}) is not an array or reference-to-array type: {:?}", base_var_name, base_place, base_oomir_type @@ -469,18 +564,6 @@ pub(crate) fn emit_instructions_to_set_value<'tcx>( &mut instructions, ); } - oomir::Type::MutableReference(_element_type) => { - instructions.push(Instruction::ArrayStore { - array: oomir::Operand::Variable { - name: base_var_name.clone(), - ty: base_oomir_type.clone(), - }, // The variable holding the array reference - // Index is always 0 for our reference representation - index: Operand::Constant(oomir::Constant::I32(0)), - value: source_operand, // The value being assigned - copy_value: false, - }); - } _ => { // no-op - non-mutable reference } diff --git a/src/lower1/statics.rs b/src/lower1/statics.rs index 206db5f8..2a2c8494 100644 --- a/src/lower1/statics.rs +++ b/src/lower1/statics.rs @@ -32,12 +32,12 @@ pub fn static_ref_constant<'tcx>( data_types: &mut Definitions<'tcx>, instance: Instance<'tcx>, ) -> oomir::Constant { - let rust_ty = tcx.type_of(def_id).skip_binder(); + let rust_ty = data_types.normalize(tcx, tcx.type_of(def_id).skip_binder(), instance); let value_type = ty_to_oomir_type(rust_ty, tcx, data_types, instance); - let ty = if matches!(value_type, oomir::Type::Array(_)) { + let ty = if rust_ty.is_array() { value_type } else { - oomir::Type::Pointer(Box::new(value_type)) + oomir::Type::pointer(value_type) }; let (owner_class, field_name) = identity(tcx, def_id); oomir::Constant::StaticRef { @@ -57,13 +57,16 @@ pub fn lower_static<'tcx>( if is_nested(tcx, def_id) { return Ok(()); } - let rust_ty = tcx.type_of(def_id).skip_binder(); let instance = Instance::mono(tcx, def_id); + let rust_ty = module + .data_types + .normalize(tcx, tcx.type_of(def_id).skip_binder(), instance); let value_type = ty_to_oomir_type(rust_ty, tcx, &mut module.data_types, instance); - let storage_type = if matches!(value_type, oomir::Type::Array(_)) { + // Only Rust arrays use direct array static storage. Struct wrappers keep the struct borrow ABI. + let storage_type = if rust_ty.is_array() { value_type.clone() } else { - oomir::Type::Pointer(Box::new(value_type.clone())) + oomir::Type::pointer(value_type.clone()) }; let allocation = tcx .eval_static_initializer(def_id) @@ -85,6 +88,8 @@ pub fn lower_static<'tcx>( _ => None, }); let (owner_class, field_name) = identity(tcx, def_id); + let attrs = tcx.codegen_fn_attrs(def_id); + use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; let static_value = oomir::Static { owner_class, field_name, @@ -94,6 +99,11 @@ pub fn lower_static<'tcx>( allocation_alignment, allocation_codec_class_name, is_thread_local: tcx.is_thread_local_static(def_id), + is_private: !crate::java_exports::is_exported(tcx, def_id) + && !attrs.contains_extern_indicator() + && !attrs + .flags + .intersects(CodegenFnAttrFlags::USED_COMPILER | CodegenFnAttrFlags::USED_LINKER), }; module.statics.insert(static_value.key(), static_value); Ok(()) diff --git a/src/lower1/types.rs b/src/lower1/types.rs index 187fceec..874070d5 100644 --- a/src/lower1/types.rs +++ b/src/lower1/types.rs @@ -17,12 +17,23 @@ use rustc_span::{Symbol, def_id::DefId, sym}; mod mapping; pub(crate) use mapping::{is_codegen_sized, mir_int_to_oomir_const, ty_to_oomir_type}; +mod tuples; +pub(crate) use tuples::{tuple_fields, tuple_value}; +mod packed; +pub(crate) use packed::{packed_word, value_scalar_ty}; +mod transparent; +pub(crate) use transparent::transparent_payload; +mod tagged; +pub(crate) use tagged::{tagged_scalar, tagged_value}; +mod nullable; +pub(crate) use nullable::{direct_enum_payload, enum_carrier}; +mod enum_scalars; +pub(crate) use enum_scalars::{enum_scalar_ty, enum_scalar_variant}; mod enums; use enums::*; pub(crate) use enums::{ - adapt_simple_enum_operand, enum_scoped_method_name, enum_variant_field_name, - jvm_subtype_payload_ty, union_from_method_name, union_getter_method_name, - union_setter_method_name, + adapt_simple_enum_operand, enum_variant_field_name, enum_variant_name, jvm_subtype_payload_ty, + union_from_method_name, union_getter_method_name, union_setter_method_name, }; mod abi; pub(crate) use abi::{ @@ -39,6 +50,7 @@ pub(crate) use layout::{ }; mod adt; use adt::*; +pub(crate) use adt::{adt_class_kind, ensure_drop_callback, struct_field_name}; pub(crate) use adt::{force_define_named_adt, should_define_named_data_type}; mod drop; use drop::*; @@ -48,12 +60,19 @@ mod bytes; use bytes::*; mod enum_codecs; use enum_codecs::*; +mod enum_codec_dispatch; +use enum_codec_dispatch::*; mod write; use write::*; mod read; use read::*; mod transmute; pub(crate) use transmute::ensure_exact_transmute_helper; +mod bitcasts; +pub(crate) use bitcasts::emit_direct_transmute; +mod range_encoder; +mod storage_layout; +pub(crate) use storage_layout::scalar_storage_layout; mod pointer_codecs; use pointer_codecs::*; pub(crate) use pointer_codecs::{ @@ -68,8 +87,8 @@ mod names; use names::*; pub(crate) use names::{ generate_adt_jvm_class_name, generate_tuple_jvm_class_name, get_field_name_from_index, - readable_rust_generic_arg_name, readable_rust_type_name, sanitize_name_token, short_hash, - stable_def_identity, stable_def_path, stable_instance_identity, stable_type_identity, + readable_rust_generic_arg_name, sanitize_name_token, stable_def_identity, stable_def_path, + stable_instance_identity, stable_type_identity, }; pub const UNION_BYTES_FIELD: &str = "_bytes"; @@ -82,9 +101,6 @@ const STRUCT_TAIL_POINTER_VIEW_CODEC_PREFIX: &str = "@struct-tail-pointer\n"; const TRAIT_POINTER_VIEW_CODEC_PREFIX: &str = "@trait-pointer\n"; pub(super) const ENUM_UNION_DISCRIMINANT_METHOD: &str = "_unionDiscriminant"; const ENUM_FROM_UNION_DISCRIMINANT_METHOD: &str = "_fromUnionDiscriminant"; -const ENUM_WRITE_UNION_STORAGE_METHOD: &str = "_writeUnionStorage"; -const ENUM_READ_UNION_STORAGE_METHOD: &str = "_readUnionStorage"; -const ENUM_DROP_FIELDS_METHOD: &str = "_rust_drop_fields"; const MANAGED_DROP_METHOD: &str = "rustDrop"; const MANAGED_DROP_INTERFACE: &str = "org/rustlang/runtime/RustDrop"; @@ -205,6 +221,31 @@ pub(crate) struct PointerMemoryCodec { pub class_name: String, } +impl PointerMemoryCodec { + fn owner(&self) -> &str { + self.class_name.split('#').next().unwrap() + } + + fn method(&self, name: &str) -> String { + let key = self + .class_name + .split('#') + .nth(1) + .expect("generated codec recipe"); + let prefix = match name { + "encode" => "e", + "encodeAt" => "w", + "decode" => "d", + "decodeAt" => "a", + "bind" => "b", + "_rustArrayElementSize" => "s", + "_rustArrayElementCodec" => "c", + _ => unreachable!("unknown codec operation {name}"), + }; + format!("{prefix}${key}") + } +} + // Keep ordinary nested generic/tuple names readable for Java callers. Hashing // is only a last-resort guard against unwieldy class-file names, consistent // with the other generated-name families. diff --git a/src/lower1/types/adt.rs b/src/lower1/types/adt.rs index 9d57b6c8..8815f152 100644 --- a/src/lower1/types/adt.rs +++ b/src/lower1/types/adt.rs @@ -1,6 +1,151 @@ use super::*; use crate::lower1::context::Definitions; +pub(crate) fn adt_class_kind<'tcx>( + tcx: TyCtxt<'tcx>, + adt: &AdtDef<'tcx>, + args: GenericArgsRef<'tcx>, +) -> oomir::ClassKind { + if !is_codegen_sized(Ty::new_adt(tcx, *adt, args), tcx) { + oomir::ClassKind::MemoryView + } else if has_java_adt_identity(tcx, adt) { + oomir::ClassKind::JavaValue + } else { + oomir::ClassKind::Value + } +} + +pub(super) fn has_java_adt_identity(tcx: TyCtxt<'_>, adt: &AdtDef<'_>) -> bool { + tcx.generics_of(adt.did()).own_params.is_empty() + && tcx.visibility(adt.did()).is_public() + && (crate::java_exports::is_exported(tcx, adt.did()) + || crate::java_exports::is_runtime_type(tcx, adt.did())) +} + +/// Positional private fields permit shared storage classes. Java exports keep their declared names. +pub(crate) fn struct_field_name<'tcx>( + tcx: TyCtxt<'tcx>, + adt: &AdtDef<'tcx>, + index: usize, +) -> String { + // NonNull's fallback carrier participates in the compiler's pointer ABI. + if !has_java_adt_identity(tcx, adt) && !crate::lower1::is_non_null_lang_item(tcx, adt.did()) { + format!("field{index}") + } else { + adt.non_enum_variant().fields[FieldIdx::from_usize(index)] + .ident(tcx) + .to_string() + } +} + +/// Downstream codecs require the complete field schema, including fields whose helpers this crate +/// does not emit. +pub(super) fn remember_external_fields<'tcx>( + adt: &AdtDef<'tcx>, + args: GenericArgsRef<'tcx>, + name: &str, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, +) { + if adt.is_union() || definitions.external_schemas.contains_key(name) { + return; + } + let schema = |fields, interfaces| oomir::DataType::Class { + kind: adt_class_kind(tcx, adt, args), + fields, + is_abstract: false, + methods: HashMap::default(), + super_class: None, + interfaces, + }; + definitions.external_schemas.insert( + name.into(), + if adt.is_enum() { + oomir::DataType::Interface { + methods: imported_enum_methods(adt, args, tcx), + interfaces: Vec::new(), + is_enum: true, + } + } else { + schema(Vec::new(), Vec::new()) + }, + ); + for variant in adt.variants() { + if adt.is_enum() && is_jvm_subtype_variant(tcx, variant) { + continue; + } + let fields = variant + .fields + .iter() + .enumerate() + .filter_map(|(index, field)| { + let ty = ty_to_oomir_type( + field.ty(tcx, args).skip_norm_wip(), + tcx, + definitions, + instance, + ); + let name = if adt.is_enum() { + enum_variant_field_name(variant, index, tcx) + } else { + struct_field_name(tcx, adt, index) + }; + ty.has_jvm_value().then_some((name, ty)) + }) + .collect(); + let (owner, interfaces) = if adt.is_enum() { + ( + format!( + "{name}${}", + crate::lower1::types::enum_variant_name(variant, tcx) + ), + vec![name.to_string()], + ) + } else { + (name.to_string(), Vec::new()) + }; + definitions + .external_schemas + .insert(owner, schema(fields, interfaces)); + } +} + +// Imported enums need tag factory declarations for scalar conversion. They do not need duplicate +// factory bodies. +fn imported_enum_methods<'tcx>( + adt: &AdtDef<'tcx>, + args: GenericArgsRef<'tcx>, + tcx: TyCtxt<'tcx>, +) -> HashMap { + let mut names = vec!["eq", "variantIndex"]; + if tcx.is_lang_item(adt.did(), rustc_attr_ir::lang_items::LangItem::Option) { + names.extend(["is_none", "is_some"]); + } + let transparent = adt + .variants() + .iter() + .any(|v| is_jvm_subtype_variant(tcx, v)); + if enum_union_discriminant_supported(adt, tcx) { + names.push(ENUM_UNION_DISCRIMINANT_METHOD); + if !transparent && adt_class_kind(tcx, adt, args) == oomir::ClassKind::Value { + names.push(oomir::ENUM_TAG_METHOD); + } + } + if !transparent && simple_enum_union_size(adt, tcx).is_ok() { + names.push(ENUM_FROM_UNION_DISCRIMINANT_METHOD); + } + names + .into_iter() + .map(|name| { + ( + name.to_string(), + DataTypeMethod::SimpleConstantReturn(oomir::Type::Void, None), + ) + }) + .collect() +} + pub(crate) fn should_define_named_data_type<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { def_id.is_local() || matches!(tcx.crate_name(def_id.krate), sym::core | sym::alloc) } @@ -21,6 +166,7 @@ pub(super) fn ensure_adt_data_type<'tcx>( jvm_name.to_string(), oomir::DataType::Class { fields: vec![], + kind: adt_class_kind(tcx, adt_def, substs), is_abstract: false, methods: HashMap::default(), super_class: None, @@ -31,8 +177,9 @@ pub(super) fn ensure_adt_data_type<'tcx>( let oomir_fields = variant .fields .iter() - .filter_map(|field_def| { - let field_name = field_def.ident(tcx).to_string(); + .enumerate() + .filter_map(|(index, field_def)| { + let field_name = struct_field_name(tcx, adt_def, index); let field_ty = field_def.ty(tcx, substs); let field_mir_ty = if field_ty.has_param() || field_ty.has_escaping_bound_vars() { @@ -88,37 +235,6 @@ pub(super) fn ensure_adt_data_type<'tcx>( } let rust_ty = Ty::new_adt(tcx, *adt_def, substs); - let needs_custom_drop_fields = adt_def.is_struct() - && !adt_def.is_box() - && adt_def.destructor(tcx).is_some() - && !rust_ty.has_param() - && !rust_ty.has_escaping_bound_vars() - && matches!( - data_types.get(jvm_name), - Some(oomir::DataType::Class { methods, .. }) - if !methods.contains_key(ENUM_DROP_FIELDS_METHOD) - ); - if needs_custom_drop_fields { - if let Some(oomir::DataType::Class { methods, .. }) = data_types.get_mut(jvm_name) { - methods.insert( - ENUM_DROP_FIELDS_METHOD.to_string(), - DataTypeMethod::SimpleConstantReturn(oomir::Type::Void, None), - ); - } - let fields_method = managed_struct_drop_fields_function( - rust_ty, - jvm_name, - tcx, - data_types, - instance_context, - ); - if let Some(oomir::DataType::Class { methods, .. }) = data_types.get_mut(jvm_name) { - methods.insert( - ENUM_DROP_FIELDS_METHOD.to_string(), - DataTypeMethod::Function(fields_method), - ); - } - } ensure_managed_drop(rust_ty, jvm_name, tcx, data_types, instance_context); } @@ -128,6 +244,26 @@ pub(super) fn ensure_managed_drop<'tcx>( tcx: TyCtxt<'tcx>, data_types: &mut Definitions<'tcx>, instance_context: rustc_middle::ty::Instance<'tcx>, +) { + // Rust drops call their glue directly. Only Java exports and trait adapters require object + // callbacks. + let java_surface = matches!( + rust_ty.kind(), + TyKind::Adt(def, _) if has_java_adt_identity(tcx, def) + ); + if !java_surface { + return; + } + ensure_drop_callback(rust_ty, jvm_name, tcx, data_types, instance_context); +} + +/// Adds destruction only when an erased or Java-facing receiver needs it. +pub(crate) fn ensure_drop_callback<'tcx>( + rust_ty: Ty<'tcx>, + jvm_name: &str, + tcx: TyCtxt<'tcx>, + data_types: &mut Definitions<'tcx>, + instance_context: rustc_middle::ty::Instance<'tcx>, ) { let needs_managed_drop = !rust_ty.has_param() && !rust_ty.has_escaping_bound_vars() @@ -184,9 +320,25 @@ pub(crate) fn force_define_named_adt<'tcx>( data_types: &mut Definitions<'tcx>, instance_context: rustc_middle::ty::Instance<'tcx>, ) -> oomir::Type { + let ty = data_types.normalize(tcx, ty, instance_context); let TyKind::Adt(adt_def, substs) = ty.kind() else { return ty_to_oomir_type(ty, tcx, data_types, instance_context); }; + if tcx.lang_items().phantom_data() == Some(adt_def.did()) { + return oomir::Type::Unit; + } + if tagged_scalar(ty, tcx).is_some() { + return oomir::Type::TaggedI64; + } + if let Some(payload) = direct_enum_payload(ty, tcx) { + return ty_to_oomir_type(payload, tcx, data_types, instance_context); + } + if let Some(scalar) = value_scalar_ty(ty, tcx) { + return ty_to_oomir_type(scalar, tcx, data_types, instance_context); + } + if let Some(payload) = transparent_payload(ty, tcx) { + return ty_to_oomir_type(payload.ty, tcx, data_types, instance_context); + } if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) { let lowered = ty_to_oomir_type(ty, tcx, data_types, instance_context); if matches!(lowered, oomir::Type::Pointer(_)) { diff --git a/src/lower1/types/bitcasts.rs b/src/lower1/types/bitcasts.rs new file mode 100644 index 00000000..69b06bee --- /dev/null +++ b/src/lower1/types/bitcasts.rs @@ -0,0 +1,221 @@ +//! Exact scalar and thin-address conversions need no generated codec or method. +use crate::oomir::{self, Instruction, Operand, Type}; + +pub(super) fn scalar_wrapper_transmute<'tcx>( + source: super::Ty<'tcx>, + target: super::Ty<'tcx>, + tcx: super::TyCtxt<'tcx>, + definitions: &mut crate::lower1::context::Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, +) -> Option> { + use super::*; + + fn path<'tcx>( + mut ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + definitions: &mut crate::lower1::context::Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, + ) -> Option<(Type, Vec<(String, String, Type)>)> { + let mut fields = Vec::new(); + for _ in 0..16 { + let inner = match ty.kind() { + TyKind::Bool + | TyKind::Char + | TyKind::Int(_) + | TyKind::Uint(_) + | TyKind::Float(_) => { + return Some((ty_to_oomir_type(ty, tcx, definitions, instance), fields)); + } + TyKind::Pat(inner, _) => { + ty = *inner; + continue; + } + TyKind::Adt(def, args) + if def.is_struct() && def.non_enum_variant().fields.len() == 1 => + { + normalize_union_ty( + tcx, + def.non_enum_variant().fields[FieldIdx::from_usize(0)] + .ty(tcx, args) + .skip_norm_wip(), + ) + .ok()? + } + TyKind::Tuple(elements) if elements.len() == 1 => elements[0], + _ => return None, + }; + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + if layout.fields.offset(0).bytes() != 0 + || layout.size.bytes_usize() != layout_size_bytes(tcx, inner).ok()? + { + return None; + } + let outer = ty_to_oomir_type(ty, tcx, definitions, instance); + let nested = ty_to_oomir_type(inner, tcx, definitions, instance); + if outer != nested { + let Type::Class(owner) = outer else { + return None; + }; + let Some(DataType::Class { + fields: members, + kind: oomir::ClassKind::Value, + is_abstract: false, + .. + }) = definitions.get(&owner) + else { + return None; + }; + let [(name, field)] = members.as_slice() else { + return None; + }; + if *field != nested { + return None; + } + fields.push((owner, name.clone(), nested)); + } + ty = inner; + } + None + } + + let (source_scalar, source_fields) = path(source, tcx, definitions, instance)?; + let (target_scalar, target_fields) = path(target, tcx, definitions, instance)?; + let mut instructions = Vec::new(); + let mut value = Operand::Variable { + name: "_1".into(), + ty: source_fields + .first() + .map_or(source_scalar, |(owner, _, _)| Type::Class(owner.clone())), + }; + for (index, (owner, name, ty)) in source_fields.into_iter().enumerate() { + let dest = format!("_scalar_source_{index}"); + instructions.push(Instruction::GetField { + dest: dest.clone(), + object: value, + field_name: name, + field_ty: ty.clone(), + owner_class: owner, + }); + value = Operand::Variable { name: dest, ty }; + } + value = emit_direct_transmute(value, &target_scalar, "_scalar_bits", &mut instructions)?; + for (index, (owner, _, ty)) in target_fields.into_iter().rev().enumerate() { + let dest = format!("_scalar_target_{index}"); + instructions.push(Instruction::ConstructObject { + dest: dest.clone(), + class_name: owner.clone(), + args: vec![(value, ty)], + }); + value = Operand::Variable { + name: dest, + ty: Type::Class(owner), + }; + } + instructions.push(Instruction::Return { + operand: Some(value), + }); + Some(instructions) +} + +pub(crate) fn emit_direct_transmute( + source: Operand, + target: &Type, + dest: &str, + instructions: &mut Vec, +) -> Option { + let from = source.get_type()?; + let integer_size = |ty: &Type| match ty { + Type::Boolean | Type::I8 | Type::U8 => Some(1), + Type::I16 | Type::U16 => Some(2), + Type::I32 | Type::U32 => Some(4), + Type::I64 | Type::U64 => Some(8), + _ => None, + }; + let direct = (from == *target + && (integer_size(&from).is_some() + || matches!(from, Type::Unit | Type::F32 | Type::F64 | Type::Pointer(_)))) + || (integer_size(&from).is_some() && integer_size(&from) == integer_size(target)) + || (from.scalar_address_size().is_some() && target.scalar_address_size().is_some()); + if direct { + if !target.has_jvm_value() { + return Some(Operand::Constant(oomir::Constant::Unit)); + } + instructions.push(Instruction::Cast { + op: source, + ty: target.clone(), + dest: dest.into(), + }); + } else { + let (owner, method) = match (&from, target) { + (Type::F32, Type::I32 | Type::U32) => ("java/lang/Float", "floatToRawIntBits"), + (Type::I32 | Type::U32, Type::F32) => ("java/lang/Float", "intBitsToFloat"), + (Type::F64, Type::I64 | Type::U64) => ("java/lang/Double", "doubleToRawLongBits"), + (Type::I64 | Type::U64, Type::F64) => ("java/lang/Double", "longBitsToDouble"), + _ => return None, + }; + instructions.push(Instruction::InvokeStatic { + dest: Some(dest.into()), + class_name: owner.into(), + method_name: method.into(), + method_ty: oomir::Signature { + params: vec![("value".into(), from)], + ret: Box::new(target.clone()), + is_static: true, + }, + args: vec![source], + }); + } + Some(Operand::Variable { + name: dest.into(), + ty: target.clone(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pointer_identity_and_float_bits_need_no_codec() { + let mut code = Vec::new(); + let address = Type::pointer(Type::U64); + let result = emit_direct_transmute( + Operand::Variable { + name: "p".into(), + ty: address, + }, + &Type::pointer(Type::F64), + "q", + &mut code, + ); + assert!(result.is_some()); + assert!(matches!(&code[..], [Instruction::Cast { .. }])); + code.clear(); + assert!( + emit_direct_transmute( + Operand::Constant(oomir::Constant::U64(0x7ff8000000001234)), + &Type::F64, + "f", + &mut code + ) + .is_some() + ); + assert!( + matches!(&code[..], [Instruction::InvokeStatic { class_name, method_name, .. }] + if class_name == "java/lang/Double" && method_name == "longBitsToDouble") + ); + code.clear(); + assert!( + emit_direct_transmute( + Operand::Constant(oomir::Constant::U64(0)), + &Type::pointer(Type::U64), + "p", + &mut code + ) + .is_none() + ); + assert!(code.is_empty()); + } +} diff --git a/src/lower1/types/coroutines.rs b/src/lower1/types/coroutines.rs index 4d867c79..462d863a 100644 --- a/src/lower1/types/coroutines.rs +++ b/src/lower1/types/coroutines.rs @@ -353,7 +353,7 @@ pub(super) fn coroutine_pointer_codec_methods<'tcx>( decode_variants, ); - let pointer_ty = oomir::Type::Pointer(Box::new(value_ty.clone())); + let pointer_ty = oomir::Type::pointer(value_ty.clone()); let mut bind_entry = Vec::new(); emit_aggregate_memory_bindings( &UnionAggregateLayout { diff --git a/src/lower1/types/drop.rs b/src/lower1/types/drop.rs index bf1ce100..887e810a 100644 --- a/src/lower1/types/drop.rs +++ b/src/lower1/types/drop.rs @@ -1,574 +1,7 @@ +//! Java callbacks use Rust destruction. MIR controls drop order and unwind cleanup. use super::*; use crate::lower1::context::Definitions; -pub(super) fn enum_variant_drop_glue_function<'tcx>( - variant: &rustc_middle::ty::VariantDef, - substs: GenericArgsRef<'tcx>, - enum_class_name: &str, - variant_class_name: &str, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> oomir::Function { - let self_ty = oomir::Type::Class(variant_class_name.to_string()); - let self_operand = operand_var("_1", self_ty.clone()); - let mut actions = Vec::new(); - for (field_index, field) in variant.fields.iter().enumerate() { - let raw_field_ty = field.ty(tcx, substs).skip_norm_wip(); - let field_ty = - resolve_union_ty(tcx, raw_field_ty, instance_context).unwrap_or(raw_field_ty); - let field_oomir_ty = ty_to_oomir_type(field_ty, tcx, data_types, instance_context); - let mut action = Vec::new(); - let field_value = if field_oomir_ty.has_jvm_value() { - let field_name = enum_variant_field_name(variant, field_index, tcx); - let dest = format!("_drop_field_{field_index}"); - action.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: self_operand.clone(), - field_name, - field_ty: field_oomir_ty.clone(), - owner_class: variant_class_name.to_string(), - }); - operand_var(dest, field_oomir_ty) - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - - // Enum data types can first be discovered below a higher-ranked - // function, leaving otherwise irrelevant bound lifetimes in a payload - // type. `needs_drop` cannot query a type with escaping bound vars, but - // lifetimes do not affect either JVM representation or drop glue. - let drop_field_ty = erase_escaping_regions(tcx, field_ty); - if drop_field_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - emit_managed_value_drop( - drop_field_ty, - field_value, - &format!("_drop_field_{field_index}"), - tcx, - instance_context, - data_types, - &mut action, - ); - actions.push(action); - } - } - - oomir::Function { - name: enum_scoped_method_name(enum_class_name, ENUM_DROP_FIELDS_METHOD), - owner_class: None, - debug_variables: Vec::new(), - signature: oomir::Signature { - params: vec![("self".to_string(), self_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - body: cleanup_safe_drop_body(actions).into(), - } -} - -pub(super) fn enum_transparent_variant_drop_glue_function<'tcx>( - payload_ty: Ty<'tcx>, - enum_class_name: &str, - receiver_class_name: &str, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> oomir::Function { - let self_ty = oomir::Type::Class(receiver_class_name.to_string()); - let mut instructions = Vec::new(); - crate::lower1::control_flow::emit_rust_drop_value( - payload_ty, - operand_var("_1", self_ty.clone()), - "_transparent_enum_drop", - tcx, - instance_context, - data_types, - &mut instructions, - ); - instructions.push(oomir::Instruction::Return { operand: None }); - oomir::Function { - name: enum_scoped_method_name(enum_class_name, ENUM_DROP_FIELDS_METHOD), - owner_class: None, - debug_variables: Vec::new(), - signature: oomir::Signature { - params: vec![("self".to_string(), self_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - body: simple_body(instructions).into(), - } -} - -fn erase_escaping_regions<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> { - // Preserve regions owned by nested function/closure binders: erasing those - // changes their trait implementations and can prevent projection resolution. - tcx.replace_escaping_bound_vars_uncached( - ty, - rustc_middle::ty::FnMutDelegate { - regions: &mut |_| tcx.lifetimes.re_erased, - types: &mut |ty| panic!("unexpected bound type in drop glue: {ty:?}"), - consts: &mut |ct| panic!("unexpected bound constant in drop glue: {ct:?}"), - }, - ) -} - -pub(super) fn emit_managed_value_drop<'tcx>( - rust_ty: Ty<'tcx>, - mut value: oomir::Operand, - temp_prefix: &str, - tcx: TyCtxt<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, - data_types: &mut Definitions<'tcx>, - instructions: &mut Vec, -) { - let value_ty = ty_to_oomir_type(rust_ty, tcx, data_types, instance_context); - if !value_ty.has_jvm_value() { - let Some(materialized) = crate::lower1::value_repr::materialize_implicit_zst( - rust_ty, - &format!("{temp_prefix}_zst"), - tcx, - instance_context, - data_types, - instructions, - ) else { - return; - }; - value = materialized; - } - instructions.push(oomir::Instruction::InvokeStatic { - dest: None, - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropRustValue".to_string(), - method_ty: oomir::Signature { - params: vec![( - "value".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - )], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![value], - }); -} - -pub(super) fn cleanup_safe_drop_body(actions: Vec>) -> oomir::CodeBlock { - if actions.is_empty() { - return simple_body(vec![oomir::Instruction::Return { operand: None }]); - } - - let mut basic_blocks = HashMap::default(); - for (index, action) in actions.iter().enumerate() { - let mut instructions = vec![oomir::Instruction::UnwindStart { - target: if index + 1 < actions.len() { - format!("cleanup_{}", index + 1) - } else { - "rethrow".to_string() - }, - }]; - instructions.extend(action.clone()); - instructions.push(oomir::Instruction::UnwindEnd); - instructions.push(if index + 1 < actions.len() { - oomir::Instruction::Jump { - target: format!("drop_{}", index + 1), - } - } else { - oomir::Instruction::Return { operand: None } - }); - let label = format!("drop_{index}"); - basic_blocks.insert( - label.clone(), - oomir::BasicBlock { - label, - instructions, - }, - ); - } - - for (index, action) in actions.iter().enumerate().skip(1) { - let mut instructions = vec![oomir::Instruction::UnwindStart { - target: "abort".to_string(), - }]; - instructions.extend(action.clone()); - instructions.push(oomir::Instruction::UnwindEnd); - instructions.push(if index + 1 < actions.len() { - oomir::Instruction::Jump { - target: format!("cleanup_{}", index + 1), - } - } else { - oomir::Instruction::Rethrow - }); - let label = format!("cleanup_{index}"); - basic_blocks.insert( - label.clone(), - oomir::BasicBlock { - label, - instructions, - }, - ); - } - - basic_blocks.insert( - "rethrow".to_string(), - oomir::BasicBlock { - label: "rethrow".to_string(), - instructions: vec![oomir::Instruction::Rethrow], - }, - ); - basic_blocks.insert( - "abort".to_string(), - oomir::BasicBlock { - label: "abort".to_string(), - instructions: vec![ - oomir::Instruction::InvokeStatic { - dest: None, - class_name: "org/rustlang/runtime/PanicSupport".to_string(), - method_name: "abort".to_string(), - method_ty: oomir::Signature { - params: vec![( - "failure".to_string(), - oomir::Type::Class("java/lang/Throwable".to_string()), - )], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![operand_var( - "__rust_unwind_exception", - oomir::Type::Class("java/lang/Throwable".to_string()), - )], - }, - oomir::Instruction::ThrowNewWithMessage { - exception_class: "java/lang/AssertionError".to_string(), - message: "Rust abort unexpectedly returned".to_string(), - }, - ], - }, - ); - - oomir::CodeBlock { - entry: "drop_0".to_string(), - basic_blocks, - } -} - -pub(super) fn managed_drop_actions<'tcx>( - rust_ty: Ty<'tcx>, - value: oomir::Operand, - temp_prefix: &str, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> Vec> { - let oomir_ty = ty_to_oomir_type(rust_ty, tcx, data_types, instance_context); - let mut actions = Vec::new(); - - match rust_ty.kind() { - TyKind::Adt(adt_def, substs) if adt_def.is_struct() => { - if adt_def.is_box() { - let pointee_ty = substs.type_at(0); - if pointee_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - let mut action = Vec::new(); - let mut pointer = value.clone(); - let mut pointer_ty = oomir_ty.clone(); - let mut pointer_rust_ty = rust_ty; - for depth in 0..3 { - let class_name = pointer_ty - .get_class_name() - .expect("Box pointer carrier must be a JVM class") - .to_string(); - let TyKind::Adt(carrier_def, carrier_args) = pointer_rust_ty.kind() else { - panic!("Box pointer carrier {pointer_rust_ty:?} is not a struct"); - }; - let field = carrier_def - .variant(VariantIdx::from_usize(0)) - .fields - .iter() - .next() - .expect("Box pointer carrier must have a field"); - let field_rust_ty = field.ty(tcx, carrier_args).skip_norm_wip(); - let field_ty = - ty_to_oomir_type(field_rust_ty, tcx, data_types, instance_context); - let dest = format!("{temp_prefix}_box_pointer_{depth}"); - action.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: pointer, - field_name: field.ident(tcx).to_string(), - field_ty: field_ty.clone(), - owner_class: class_name, - }); - pointer = operand_var(dest, field_ty.clone()); - pointer_ty = field_ty; - pointer_rust_ty = field_rust_ty; - } - if matches!(pointee_ty.kind(), TyKind::Dynamic(..)) { - action.push(oomir::Instruction::InvokeStatic { - dest: None, - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "dropTraitPointer".to_string(), - method_ty: oomir::Signature { - params: vec![( - "pointer".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - )], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![pointer], - }); - } else if matches!(pointee_ty.kind(), TyKind::Slice(_)) { - crate::lower1::control_flow::emit_rust_drop_value( - pointee_ty, - pointer, - &format!("{temp_prefix}_box_slice"), - tcx, - instance_context, - data_types, - &mut action, - ); - } else { - let pointee = format!("{temp_prefix}_box_pointee"); - action.push(oomir::Instruction::InvokeVirtual { - dest: Some(pointee.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "getObject".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), pointer_ty)], - ret: Box::new(oomir::Type::Class("java/lang/Object".to_string())), - is_static: false, - }, - args: Vec::new(), - operand: pointer, - }); - emit_managed_value_drop( - pointee_ty, - operand_var( - pointee, - oomir::Type::Class("java/lang/Object".to_string()), - ), - &format!("{temp_prefix}_box_pointee"), - tcx, - instance_context, - data_types, - &mut action, - ); - } - actions.push(action); - } - } - - if adt_def.destructor(tcx).is_some() { - actions.push(vec![oomir::Instruction::InvokeStatic { - class_name: oomir_ty - .get_class_name() - .expect("a Rust Drop ADT has a JVM class") - .to_string(), - method_name: "drop".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![value.clone()], - dest: None, - }]); - } - - for (field_index, field) in adt_def - .variant(VariantIdx::from_usize(0)) - .fields - .iter() - .enumerate() - { - let field_ty = field.ty(tcx, substs).skip_norm_wip(); - if !field_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let field_oomir_ty = ty_to_oomir_type(field_ty, tcx, data_types, instance_context); - let mut action = Vec::new(); - let field_value = if field_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_field_{field_index}"); - action.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: field.ident(tcx).to_string(), - field_ty: field_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a struct field owner has a JVM class") - .to_string(), - }); - operand_var(dest, field_oomir_ty) - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_managed_value_drop( - field_ty, - field_value, - &format!("{temp_prefix}_field_{field_index}"), - tcx, - instance_context, - data_types, - &mut action, - ); - actions.push(action); - } - } - TyKind::Adt(adt_def, _) if adt_def.is_enum() => { - let enum_class = oomir_ty - .get_class_name() - .expect("a Rust enum has a JVM class") - .to_string(); - if adt_def.destructor(tcx).is_some() { - actions.push(vec![oomir::Instruction::InvokeStatic { - class_name: enum_class.clone(), - method_name: "drop".to_string(), - method_ty: oomir::Signature { - params: vec![("self".to_string(), oomir_ty.clone())], - ret: Box::new(oomir::Type::Void), - is_static: true, - }, - args: vec![value.clone()], - dest: None, - }]); - } - actions.push(vec![oomir::Instruction::InvokeVirtual { - class_name: enum_class.clone(), - method_name: enum_scoped_method_name(&enum_class, ENUM_DROP_FIELDS_METHOD), - method_ty: oomir::Signature { - params: vec![("self".to_string(), oomir_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - args: Vec::new(), - dest: None, - operand: value, - }]); - } - TyKind::Tuple(fields) => { - for (field_index, field_ty) in fields.iter().enumerate() { - if !field_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let field_oomir_ty = ty_to_oomir_type(field_ty, tcx, data_types, instance_context); - let mut action = Vec::new(); - let field_value = if field_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_tuple_{field_index}"); - action.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: format!("field{field_index}"), - field_ty: field_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a non-empty tuple has a JVM class") - .to_string(), - }); - operand_var(dest, field_oomir_ty) - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_managed_value_drop( - field_ty, - field_value, - &format!("{temp_prefix}_tuple_{field_index}"), - tcx, - instance_context, - data_types, - &mut action, - ); - actions.push(action); - } - } - TyKind::Closure(_, closure_args) => { - for (capture_index, capture_ty) in - closure_args.as_closure().upvar_tys().iter().enumerate() - { - if !capture_ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { - continue; - } - let capture_oomir_ty = - ty_to_oomir_type(capture_ty, tcx, data_types, instance_context); - let mut action = Vec::new(); - let capture_value = if capture_oomir_ty.has_jvm_value() { - let dest = format!("{temp_prefix}_capture_{capture_index}"); - action.push(oomir::Instruction::GetField { - dest: dest.clone(), - object: value.clone(), - field_name: format!("arg{capture_index}"), - field_ty: capture_oomir_ty.clone(), - owner_class: oomir_ty - .get_class_name() - .expect("a closure with captures has a JVM class") - .to_string(), - }); - operand_var(dest, capture_oomir_ty) - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }; - emit_managed_value_drop( - capture_ty, - capture_value, - &format!("{temp_prefix}_capture_{capture_index}"), - tcx, - instance_context, - data_types, - &mut action, - ); - actions.push(action); - } - } - TyKind::Array(..) | TyKind::Dynamic(..) => { - let mut action = Vec::new(); - emit_managed_value_drop( - rust_ty, - value, - temp_prefix, - tcx, - instance_context, - data_types, - &mut action, - ); - actions.push(action); - } - _ => {} - } - actions -} - -pub(super) fn managed_struct_drop_fields_function<'tcx>( - rust_ty: Ty<'tcx>, - class_name: &str, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> oomir::Function { - let self_ty = oomir::Type::Class(class_name.to_string()); - let mut actions = managed_drop_actions( - rust_ty, - operand_var("_1", self_ty.clone()), - "_managed_drop_fields", - tcx, - data_types, - instance_context, - ); - let TyKind::Adt(adt_def, _) = rust_ty.kind() else { - unreachable!("managed struct drop fields require an ADT"); - }; - if adt_def.destructor(tcx).is_some() { - actions.remove(0); - } - oomir::Function { - name: ENUM_DROP_FIELDS_METHOD.to_string(), - owner_class: None, - debug_variables: Vec::new(), - signature: oomir::Signature { - params: vec![("self".to_string(), self_ty)], - ret: Box::new(oomir::Type::Void), - is_static: false, - }, - body: cleanup_safe_drop_body(actions).into(), - } -} - pub(super) fn managed_drop_glue_function<'tcx>( rust_ty: Ty<'tcx>, class_name: &str, @@ -578,7 +11,7 @@ pub(super) fn managed_drop_glue_function<'tcx>( ) -> oomir::Function { let self_ty = oomir::Type::Class(class_name.to_string()); let mut instructions = Vec::new(); - crate::lower1::control_flow::emit_rust_drop_value( + crate::lower1::control_flow::emit_owned_drop( rust_ty, operand_var("_1", self_ty.clone()), "_managed_drop", diff --git a/src/lower1/types/enum_codec_dispatch.rs b/src/lower1/types/enum_codec_dispatch.rs new file mode 100644 index 00000000..5d5c0882 --- /dev/null +++ b/src/lower1/types/enum_codec_dispatch.rs @@ -0,0 +1,238 @@ +//! Exact Rust enum layouts belong to codecs, not to shareable variant storage. +use super::*; +use crate::lower1::context::Definitions; + +pub(super) struct EnumCodec { + pub owner: String, + pub writer: String, + pub reader: String, +} + +pub(super) fn ensure_enum_union_codec<'tcx>( + adt_def: &AdtDef<'tcx>, + substs: GenericArgsRef<'tcx>, + enum_ty: Ty<'tcx>, + enum_class: &str, + tcx: TyCtxt<'tcx>, + data_types: &mut Definitions<'tcx>, + instance_context: rustc_middle::ty::Instance<'tcx>, +) -> Result { + let key = crate::stable_hash::short_hash(enum_class, 16); + let codec = EnumCodec { + owner: pointer_codec_class_name(enum_ty, tcx, &format!("Codecs_{}", &key[..2])), + writer: format!("ew${key}"), + reader: format!("er${key}"), + }; + if !data_types.contains_key(enum_class) { + ensure_enum_data_types( + adt_def, + substs, + enum_class, + tcx, + data_types, + instance_context, + ); + } + if !data_types.contains_key(&codec.owner) { + data_types.insert( + codec.owner.clone(), + DataType::Class { + fields: vec![], + kind: oomir::ClassKind::Static, + is_abstract: false, + methods: HashMap::default(), + super_class: None, + interfaces: vec![], + }, + ); + } + let Some(DataType::Class { methods, .. }) = data_types.get_mut(&codec.owner) else { + return Err(format!("enum codec holder {} is not a class", codec.owner)); + }; + // The placeholder prevents recursion through payload codecs. Emission requires the complete + // body. + if methods.contains_key(&codec.writer) { + return Ok(codec); + } + methods.insert( + codec.writer.clone(), + DataTypeMethod::Abstract(enum_union_write_signature(enum_class)), + ); + let generated = (|| { + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(enum_ty)) + .map_err(|err| format!("could not get layout for {enum_ty:?}: {err:?}"))?; + let tag = union_enum_tag(&layout, tcx)?; + let mut writers = Vec::new(); + for index in 0..adt_def.variants().len() { + let (receiver, function) = enum_variant_union_writer( + format!("ev${key}${index}"), + adt_def, + substs, + enum_class, + &layout, + &tag, + VariantIdx::from_usize(index), + tcx, + data_types, + instance_context, + )?; + writers.push((receiver, function)); + } + let reader = enum_union_reader( + codec.reader.clone(), + adt_def, + substs, + enum_class, + &layout, + &tag, + tcx, + data_types, + instance_context, + )?; + let tagged = matches!(data_types.get(enum_class), Some(DataType::Interface { methods, .. }) if methods.contains_key(oomir::ENUM_TAG_METHOD)); + let dispatcher = dispatcher( + &codec, + enum_class, + &writers, + tagged.then(|| { + adt_def + .discriminants(tcx) + .map(|(_, d)| d.val as i64) + .collect::>() + }), + ); + Ok::<_, String>((writers, reader, dispatcher)) + })(); + let Some(DataType::Class { methods, .. }) = data_types.get_mut(&codec.owner) else { + unreachable!() + }; + match generated { + Ok((writers, reader, dispatcher)) => { + for function in writers + .into_iter() + .map(|(_, f)| f) + .chain([reader, dispatcher]) + { + methods.insert(function.name.clone(), DataTypeMethod::Function(function)); + } + Ok(codec) + } + Err(error) => { + methods.remove(&codec.writer); + Err(error) + } + } +} + +fn dispatcher( + codec: &EnumCodec, + enum_class: &str, + writers: &[(String, oomir::Function)], + tags: Option>, +) -> oomir::Function { + use oomir::{Constant as C, Instruction as I, Type as T}; + let receiver = operand_var("_1", T::Class(enum_class.into())); + let mut blocks = HashMap::default(); + let (query, discriminator) = if tags.is_some() { + ( + I::InvokeVirtual { + dest: Some("_tag".into()), + class_name: enum_class.into(), + method_name: oomir::ENUM_TAG_METHOD.into(), + method_ty: oomir::Signature { + params: vec![("self".into(), T::Class(enum_class.into()))], + ret: Box::new(T::I64), + is_static: false, + }, + args: vec![], + operand: receiver.clone(), + }, + T::I64, + ) + } else { + ( + I::InvokeStatic { + dest: Some("_tag".into()), + class_name: enum_class.into(), + method_name: "variantIndex".into(), + method_ty: oomir::Signature { + params: vec![("value".into(), T::Class(enum_class.into()))], + ret: Box::new(T::I32), + is_static: true, + }, + args: vec![receiver.clone()], + }, + T::I32, + ) + }; + let mut targets = Vec::new(); + for (i, (class, writer)) in writers.iter().enumerate() { + let label = format!("variant_{i}"); + targets.push(( + tags.as_ref().map_or(C::I32(i as i32), |t| C::I64(t[i])), + label.clone(), + )); + blocks.insert( + label.clone(), + oomir::BasicBlock { + label, + instructions: vec![ + I::Cast { + dest: "_variant".into(), + op: receiver.clone(), + ty: T::Class(class.clone()), + }, + I::InvokeStatic { + dest: None, + class_name: codec.owner.clone(), + method_name: writer.name.clone(), + method_ty: writer.signature.clone(), + args: vec![ + operand_var("_variant", T::Class(class.clone())), + operand_var("_2", byte_array_type()), + operand_var("_3", object_array_type()), + operand_var("_4", T::I32), + ], + }, + I::Return { operand: None }, + ], + }, + ); + } + blocks.insert( + "entry".into(), + oomir::BasicBlock { + label: "entry".into(), + instructions: vec![ + query, + I::Switch { + discr: operand_var("_tag", discriminator), + targets, + otherwise: "invalid".into(), + }, + ], + }, + ); + blocks.insert( + "invalid".into(), + oomir::BasicBlock { + label: "invalid".into(), + instructions: vec![I::ThrowNewWithMessage { + exception_class: "java/lang/IllegalArgumentException".into(), + message: "invalid enum variant while writing union storage".into(), + }], + }, + ); + oomir::Function { + name: codec.writer.clone(), + owner_class: None, + debug_variables: vec![], + signature: enum_union_write_signature(enum_class), + body: oomir::CodeBlock { + entry: "entry".into(), + basic_blocks: blocks, + } + .into(), + } +} diff --git a/src/lower1/types/enum_codecs.rs b/src/lower1/types/enum_codecs.rs index 17b1cbac..5c821fed 100644 --- a/src/lower1/types/enum_codecs.rs +++ b/src/lower1/types/enum_codecs.rs @@ -2,6 +2,7 @@ use super::*; use crate::lower1::context::Definitions; pub(super) fn enum_variant_union_writer<'tcx>( + name: String, adt_def: &AdtDef<'tcx>, substs: GenericArgsRef<'tcx>, enum_class: &str, @@ -142,7 +143,7 @@ pub(super) fn enum_variant_union_writer<'tcx>( Ok(( receiver_class.clone(), oomir::Function { - name: enum_scoped_method_name(enum_class, ENUM_WRITE_UNION_STORAGE_METHOD), + name, owner_class: None, debug_variables: Vec::new(), signature: enum_union_write_signature(&receiver_class), @@ -152,6 +153,7 @@ pub(super) fn enum_variant_union_writer<'tcx>( } pub(super) fn enum_union_reader<'tcx>( + name: String, adt_def: &AdtDef<'tcx>, substs: GenericArgsRef<'tcx>, enum_class: &str, @@ -383,7 +385,7 @@ pub(super) fn enum_union_reader<'tcx>( }; Ok(oomir::Function { - name: ENUM_READ_UNION_STORAGE_METHOD.to_string(), + name, owner_class: None, debug_variables: Vec::new(), signature: enum_union_read_signature(enum_class), @@ -394,110 +396,3 @@ pub(super) fn enum_union_reader<'tcx>( .into(), }) } - -pub(super) fn ensure_enum_union_codec<'tcx>( - adt_def: &AdtDef<'tcx>, - substs: GenericArgsRef<'tcx>, - enum_ty: Ty<'tcx>, - enum_class: &str, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> Result<(), String> { - let writer_method = enum_scoped_method_name(enum_class, ENUM_WRITE_UNION_STORAGE_METHOD); - if !data_types.contains_key(enum_class) { - ensure_enum_data_types( - adt_def, - substs, - enum_class, - tcx, - data_types, - instance_context, - ); - } - if let Some(DataType::Interface { methods, .. }) = data_types.get(enum_class) { - if methods.contains_key(ENUM_READ_UNION_STORAGE_METHOD) { - return Ok(()); - } - if methods.contains_key(&writer_method) { - // The writer is installed before recursively constructing variant - // codecs. Encountering it without the reader means this enum is - // already being generated through a recursive field. - return Ok(()); - } - } - - if let Some(DataType::Interface { methods, .. }) = data_types.get_mut(enum_class) { - let mut abstract_writer_signature = enum_union_write_signature(enum_class); - abstract_writer_signature.params.remove(0); - methods.insert( - writer_method.clone(), - DataTypeMethod::Abstract(abstract_writer_signature), - ); - } else { - return Err(format!("enum JVM interface {enum_class} was not defined")); - } - - let generated = (|| { - let layout = tcx - .layout_of(TypingEnv::fully_monomorphized().as_query_input(enum_ty)) - .map_err(|err| format!("could not get layout for {enum_ty:?}: {err:?}"))?; - let tag = union_enum_tag(&layout, tcx)?; - let mut writers = Vec::new(); - for variant_index in 0..adt_def.variants().len() { - let variant_idx = VariantIdx::from_usize(variant_index); - writers.push(enum_variant_union_writer( - adt_def, - substs, - enum_class, - &layout, - &tag, - variant_idx, - tcx, - data_types, - instance_context, - )?); - } - let reader = enum_union_reader( - adt_def, - substs, - enum_class, - &layout, - &tag, - tcx, - data_types, - instance_context, - )?; - Ok::<_, String>((writers, reader)) - })(); - let (writers, reader) = match generated { - Ok(generated) => generated, - Err(error) => { - if let Some(DataType::Interface { methods, .. }) = data_types.get_mut(enum_class) { - methods.remove(&writer_method); - } - return Err(error); - } - }; - - for (variant_class, writer) in writers { - match data_types.get_mut(&variant_class) { - Some(DataType::Class { methods, .. }) | Some(DataType::Interface { methods, .. }) => { - methods.insert(writer_method.clone(), DataTypeMethod::Function(writer)); - } - None => { - return Err(format!( - "enum variant JVM type {variant_class} was not defined" - )); - } - } - } - let Some(DataType::Interface { methods, .. }) = data_types.get_mut(enum_class) else { - return Err(format!("enum JVM interface {enum_class} was not defined")); - }; - methods.insert( - ENUM_READ_UNION_STORAGE_METHOD.to_string(), - DataTypeMethod::Function(reader), - ); - Ok(()) -} diff --git a/src/lower1/types/enum_scalars.rs b/src/lower1/types/enum_scalars.rs new file mode 100644 index 00000000..c49e7aee --- /dev/null +++ b/src/lower1/types/enum_scalars.rs @@ -0,0 +1,65 @@ +//! Private fieldless enums use Rust integer tags. Rust identity still controls methods, validity, +//! and destruction. +use super::*; + +pub(crate) fn enum_scalar_ty<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if !def.is_enum() + || def + .variants() + .iter() + .any(|variant| !variant.fields.is_empty()) + || adt_class_kind(tcx, def, args) != oomir::ClassKind::Value + { + return None; + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let Variants::Multiple { + tag, + tag_encoding: TagEncoding::Direct, + tag_field, + .. + } = &layout.variants + else { + return None; + }; + if layout.fields.offset((*tag_field).into()).bytes() != 0 + || layout.size != tag.size(&tcx.data_layout) + { + return None; + } + let rustc_abi::Primitive::Int(_, signed) = tag.primitive() else { + return None; + }; + Some(match (layout.size.bytes(), signed) { + (1, true) => tcx.types.i8, + (1, false) => tcx.types.u8, + (2, true) => tcx.types.i16, + (2, false) => tcx.types.u16, + (4, true) => tcx.types.i32, + (4, false) => tcx.types.u32, + (8, true) => tcx.types.i64, + (8, false) => tcx.types.u64, + _ => return None, + }) +} + +pub(crate) fn enum_scalar_variant<'tcx>( + ty: Ty<'tcx>, + variant: VariantIdx, + tcx: TyCtxt<'tcx>, +) -> Option { + let scalar = enum_scalar_ty(ty, tcx)?; + let TyKind::Adt(def, _) = ty.kind() else { + unreachable!() + }; + Some(mir_int_to_oomir_const( + def.discriminant_for_variant(tcx, variant).val, + scalar, + tcx, + )) +} diff --git a/src/lower1/types/enums.rs b/src/lower1/types/enums.rs index cdad56be..26ab63d5 100644 --- a/src/lower1/types/enums.rs +++ b/src/lower1/types/enums.rs @@ -1,10 +1,21 @@ use super::*; use crate::lower1::context::Definitions; -/// Returns the public JVM field name for a Rust enum variant payload. -/// -/// Struct-like variants retain their source field names. Tuple-like variants -/// use `value` when there is exactly one payload and `_0`, `_1`, ... otherwise. +/// Private variants use positional names. Java exports and Option/Poll adapters keep their required +/// names. +pub(crate) fn enum_variant_name(variant: &rustc_middle::ty::VariantDef, tcx: TyCtxt<'_>) -> String { + let adt = tcx.adt_def(tcx.parent(variant.def_id)); + if super::adt::has_java_adt_identity(tcx, &adt) + || matches!(variant.name.as_str(), "None" | "Some" | "Ready" | "Pending") + { + jvm_names::member_name(&variant.name.to_string()) + } else { + format!("V{}", adt.variant_index_with_id(variant.def_id).as_usize()) + } +} + +/// Private payload fields use positional names. Exported struct variants keep their declared field +/// names. pub(crate) fn enum_variant_field_name( variant: &rustc_middle::ty::VariantDef, field_index: usize, @@ -20,7 +31,8 @@ pub(crate) fn enum_variant_field_name( ) }); let source_name = field.ident(tcx).to_string(); - if source_name.parse::().is_ok() { + let adt = tcx.adt_def(tcx.parent(variant.def_id)); + if !super::adt::has_java_adt_identity(tcx, &adt) || source_name.parse::().is_ok() { if variant.fields.len() == 1 { "value".to_string() } else { @@ -31,15 +43,6 @@ pub(crate) fn enum_variant_field_name( } } -pub(crate) fn enum_scoped_method_name(enum_class: &str, method: &str) -> String { - let mut hash = 0xcbf2_9ce4_8422_2325u64; - for byte in enum_class.bytes() { - hash ^= u64::from(byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - format!("{method}${hash:016x}") -} - pub(crate) fn union_from_method_name(field_name: &str) -> String { format!("$union$from${}", jvm_names::member_name(field_name)) } @@ -170,7 +173,7 @@ pub(super) fn enum_from_union_discriminant_function<'tcx>( let variant_class_name = format!( "{}${}", base_enum_name, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); let block_name = format!("variant_{}", variant_idx.as_u32()); let result_name = format!("_variant_{}", variant_idx.as_u32()); @@ -286,13 +289,14 @@ pub(super) fn ensure_enum_data_types<'tcx>( let variant_class_name = format!( "{}${}", base_enum_name, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); if !data_types.contains_key(&variant_class_name) { data_types.insert( variant_class_name.clone(), oomir::DataType::Class { fields: vec![], + kind: adt_class_kind(tcx, adt_def, substs), is_abstract: false, methods: HashMap::default(), super_class: None, @@ -311,6 +315,11 @@ pub(super) fn ensure_enum_data_types<'tcx>( .variants() .iter() .any(|variant| is_jvm_subtype_variant(tcx, variant)); + let private = adt_class_kind(tcx, adt_def, substs) == oomir::ClassKind::Value; + // Concrete variants have one enum parent. Transparent parents use instanceof to select the + // correct tag. + let discriminant_dispatch = (private && has_numeric_discriminant && !has_transparent_variant) + .then(|| oomir::ENUM_TAG_METHOD.to_owned()); let union_factory = (!has_transparent_variant) .then_some(union_size) .flatten() @@ -319,7 +328,7 @@ pub(super) fn ensure_enum_data_types<'tcx>( .variants() .iter() .map(|variant| { - let variant_name = jvm_names::member_name(&variant.name.to_string()); + let variant_name = crate::lower1::types::enum_variant_name(variant, tcx); let fields = variant .fields .iter() @@ -382,17 +391,13 @@ pub(super) fn ensure_enum_data_types<'tcx>( variants_info.clone(), tcx.is_lang_item(adt_def.did(), rustc_attr_ir::lang_items::LangItem::Option), ); - methods - .entry(enum_scoped_method_name( - base_enum_name, - ENUM_DROP_FIELDS_METHOD, - )) - .or_insert(DataTypeMethod::Abstract(oomir::Signature { - params: vec![], - ret: Box::new(oomir::Type::Void), - is_static: false, - })); if has_numeric_discriminant { + if let Some(name) = &discriminant_dispatch { + methods.insert( + name.clone(), + DataTypeMethod::SimpleConstantReturn(oomir::Type::I64, None), + ); + } methods .entry(ENUM_UNION_DISCRIMINANT_METHOD.to_string()) .or_insert(DataTypeMethod::AdtHelperMethod { @@ -403,6 +408,7 @@ pub(super) fn ensure_enum_data_types<'tcx>( .discriminants(tcx) .map(|(_, discriminant)| discriminant.val as i64) .collect(), + dispatch: discriminant_dispatch.clone(), }, }); } @@ -413,37 +419,15 @@ pub(super) fn ensure_enum_data_types<'tcx>( } } - for (_variant_idx, variant) in adt_def.variants().iter().enumerate() { - if let Some(payload_ty) = jvm_subtype_payload_ty(adt_def, variant, substs, tcx) { - let receiver_class = ty_to_oomir_type(payload_ty, tcx, data_types, instance_context) - .get_class_name() - .expect("transparent enum payload is an enum interface") - .to_string(); - let drop_function = enum_transparent_variant_drop_glue_function( - payload_ty, - base_enum_name, - &receiver_class, - tcx, - data_types, - instance_context, - ); - let Some(oomir::DataType::Interface { methods, .. }) = - data_types.get_mut(&receiver_class) - else { - tcx.dcx().fatal(format!( - "transparent enum payload {receiver_class} is not an interface" - )); - }; - methods.insert( - enum_scoped_method_name(base_enum_name, ENUM_DROP_FIELDS_METHOD), - DataTypeMethod::Function(drop_function), - ); + for (variant_idx, discriminant) in adt_def.discriminants(tcx) { + let variant = adt_def.variant(variant_idx); + if jvm_subtype_payload_ty(adt_def, variant, substs, tcx).is_some() { continue; } let variant_class_name = format!( "{}${}", base_enum_name, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); if !created_placeholders.contains(&variant_class_name) { if let Some(oomir::DataType::Class { .. }) = data_types.get_mut(&variant_class_name) { @@ -468,19 +452,18 @@ pub(super) fn ensure_enum_data_types<'tcx>( }) .collect(); let mut methods = HashMap::default(); - methods.insert( - enum_scoped_method_name(base_enum_name, ENUM_DROP_FIELDS_METHOD), - DataTypeMethod::Function(enum_variant_drop_glue_function( - variant, - substs, - base_enum_name, - &variant_class_name, - tcx, - data_types, - instance_context, - )), - ); - for (component_index, (field_name, field_ty)) in fields.iter().enumerate() { + if let Some(name) = &discriminant_dispatch { + methods.insert( + name.clone(), + DataTypeMethod::SimpleConstantReturn( + oomir::Type::I64, + Some(oomir::Constant::I64(discriminant.val as i64)), + ), + ); + } + for (component_index, (field_name, field_ty)) in + fields.iter().enumerate().filter(|_| !private) + { methods.insert( format!("component{}", component_index + 1), DataTypeMethod::AdtHelperMethod { @@ -525,7 +508,7 @@ pub(crate) fn adapt_simple_enum_operand( source: oomir::Operand, target_ty: &oomir::Type, temp_prefix: &str, - data_types: &HashMap, + data_types: &crate::lower1::context::Definitions<'_>, instructions: &mut Vec, ) -> oomir::Operand { let oomir::Type::Class(enum_class) = target_ty else { diff --git a/src/lower1/types/layout.rs b/src/lower1/types/layout.rs index 147c3e34..5170ef0f 100644 --- a/src/lower1/types/layout.rs +++ b/src/lower1/types/layout.rs @@ -242,6 +242,9 @@ pub(super) fn union_aggregate_layout<'tcx>( instance_context: rustc_middle::ty::Instance<'tcx>, ) -> Result>, String> { let ty = resolve_union_ty(tcx, ty, instance_context)?; + if value_scalar_ty(ty, tcx).is_some() || transparent_payload(ty, tcx).is_some() { + return Ok(None); + } let layout = tcx .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) .map_err(|err| format!("could not get layout for {:?}: {:?}", ty, err))?; @@ -282,7 +285,7 @@ pub(super) fn union_aggregate_layout<'tcx>( field.ty(tcx, substs).skip_norm_wip(), instance_context, )?, - field.ident(tcx).to_string(), + struct_field_name(tcx, adt_def, index), layout .fields .offset(FieldIdx::from_usize(index).into()) @@ -538,7 +541,7 @@ pub(super) fn union_enum_variant_layout<'tcx>( let variant_class = format!( "{}${}", enum_class, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); let mut fields = Vec::new(); for (field_index, field) in variant.fields.iter().enumerate() { @@ -580,7 +583,7 @@ pub(super) fn enum_union_write_signature(receiver_class: &str) -> oomir::Signatu ("offset".to_string(), oomir::Type::I32), ], ret: Box::new(oomir::Type::Void), - is_static: false, + is_static: true, } } diff --git a/src/lower1/types/mapping.rs b/src/lower1/types/mapping.rs index 68457c2f..f869ca4a 100644 --- a/src/lower1/types/mapping.rs +++ b/src/lower1/types/mapping.rs @@ -41,7 +41,31 @@ pub(crate) fn ty_to_oomir_type<'tcx>( return cached; } crate::metrics::record_type_cache_miss(); - let lowered = ty_to_oomir_type_resolved(resolved_ty, tcx, data_types, instance_context); + let mut lowered = ty_to_oomir_type_resolved(resolved_ty, tcx, data_types, instance_context); + // JVM array descriptors omit length. Raw addresses keep the exact Rust stride and codec. + let static_pointee = match resolved_ty.kind() { + TyKind::RawPtr(pointee, _) => Some(*pointee), + TyKind::Adt(def, args) if crate::lower1::is_non_null_lang_item(tcx, def.did()) => { + args.iter().find_map(|arg| arg.as_type()) + } + _ => None, + }; + if matches!(lowered, oomir::Type::Pointer(_)) + && let Some(pointee) = static_pointee + && let TyKind::Array(element, _) = pointee.kind() + && matches!( + element.kind(), + TyKind::Bool | TyKind::Char | TyKind::Int(_) | TyKind::Uint(_) | TyKind::Float(_) + ) + && let Ok(size) = layout_size_bytes(tcx, pointee) + && let Ok(size) = u32::try_from(size) + && size > 0 + { + let codec = pointer_view_codec_operand(pointee, tcx, data_types, instance_context); + if let oomir::Operand::Constant(oomir::Constant::String(codec)) = codec { + lowered = lowered.with_address_layout(size, Some(codec)); + } + } data_types .representations .insert(resolved_ty, lowered.clone()); @@ -88,21 +112,47 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( FloatTy::F128 => oomir::Type::Class(crate::lower2::F128_CLASS.to_string()), }, rustc_middle::ty::TyKind::Adt(adt_def, substs) => { + if tcx.lang_items().phantom_data() == Some(adt_def.did()) { + return oomir::Type::Unit; + } + if tagged_scalar(resolved_ty, tcx).is_some() { + return oomir::Type::TaggedI64; + } + if let Some(payload) = direct_enum_payload(resolved_ty, tcx) { + return ty_to_oomir_type(payload, tcx, data_types, instance_context); + } + if let Some(scalar) = value_scalar_ty(resolved_ty, tcx) { + return ty_to_oomir_type(scalar, tcx, data_types, instance_context); + } + if let Some(payload) = transparent_payload(resolved_ty, tcx) { + return ty_to_oomir_type(payload.ty, tcx, data_types, instance_context); + } if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) && let Some(pointee) = substs.iter().find_map(|arg| arg.as_type()) && is_codegen_sized(pointee, tcx) { - return oomir::Type::Pointer(Box::new(ty_to_oomir_type( + return oomir::Type::pointer(ty_to_oomir_type( pointee, tcx, data_types, instance_context, - ))); + )); } let jvm_name_full = generate_adt_jvm_class_name(&adt_def, substs, tcx, data_types, instance_context); - if !should_define_named_data_type(tcx, adt_def.did()) && substs.is_empty() { + if !adt_def.did().is_local() + && !adt_def.is_union() + && data_types.has_upstream_type(&jvm_name_full) + { + remember_external_fields( + adt_def, + substs, + &jvm_name_full, + tcx, + data_types, + instance_context, + ); return oomir::Type::Class(jvm_name_full); } @@ -177,7 +227,7 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( // Sized references carry a stable address. Mutability remains // a Rust type-system property; both reference kinds use the // same JVM pointer representation so reborrows preserve identity. - oomir::Type::Pointer(Box::new(pointee_oomir_type)) + oomir::Type::pointer(pointee_oomir_type) } } rustc_middle::ty::TyKind::RawPtr(ty, _mutability) => { @@ -200,7 +250,7 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( if matches!(pointee_oomir_type, oomir::Type::Class(_)) { pointee_oomir_type } else { - oomir::Type::Pointer(Box::new(pointee_oomir_type)) + oomir::Type::pointer(pointee_oomir_type) } } else { // Sized raw pointers and sized references intentionally share @@ -208,9 +258,9 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( // const/mut casts and makes arithmetic independent of the JVM ABI. let oomir_pointee_type = ty_to_oomir_type(*ty, tcx, data_types, instance_context); if matches!(oomir_pointee_type, oomir::Type::Void) { - oomir::Type::Pointer(Box::new(oomir::Type::Unit)) + oomir::Type::pointer(oomir::Type::Unit) } else { - oomir::Type::Pointer(Box::new(oomir_pointee_type)) + oomir::Type::pointer(oomir_pointee_type) } } } @@ -229,99 +279,58 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( instance_context, ))) } - rustc_middle::ty::TyKind::Tuple(tuple_elements) => { - // Unit is an inhabited Rust value, but occupies no JVM stack or local slot. - if tuple_elements.is_empty() { + TyKind::Tuple(elements) => { + if let Some(word) = packed_word(resolved_ty, tcx) { + return word.jvm_type(); + } + if elements.is_empty() { return oomir::Type::Unit; } - - // Handle non-empty tuples -> generate a class - let element_mir_tys: Vec> = tuple_elements.iter().collect(); // Collect MIR types - - // Generate the JVM class name for this specific tuple type - let tuple_class_name = - generate_tuple_jvm_class_name(&element_mir_tys, tcx, data_types, instance_context); - - // Check if we've already created the DataType for this tuple signature - if !data_types.contains_key(&tuple_class_name) { - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "type-mapping", - format!( - "Info: Defining new tuple type class: {} for MIR type {:?}", - tuple_class_name, resolved_ty - ) - ); - // Create the fields ("field0", "field1", ...) and their OOMIR types - let oomir_fields = element_mir_tys - .iter() + let fields = elements.iter().collect::>(); + let name = generate_tuple_jvm_class_name(&fields, tcx, data_types, instance_context); + if data_types.get(&name).is_none() { + let fields = fields + .into_iter() .enumerate() - .map(|(i, &elem_ty)| { - let field_name = format!("field{}", i); - // Recursively convert element type to OOMIR type - let field_oomir_type = - ty_to_oomir_type(elem_ty, tcx, data_types, instance_context); - (field_name, field_oomir_type) + .map(|(index, ty)| { + ( + format!("field{index}"), + ty_to_oomir_type(ty, tcx, data_types, instance_context), + ) }) .collect::>(); - - let mut methods = HashMap::default(); - methods.insert( - "eq".to_string(), - DataTypeMethod::AdtHelperMethod { - kind: oomir::AdtHelperKind::PartialEqClass { - fields: oomir_fields.clone(), - }, - }, - ); - - // Create and insert the DataType definition - let tuple_data_type = oomir::DataType::Class { - fields: oomir_fields, - is_abstract: false, - methods, - super_class: None, - interfaces: vec![], - }; - data_types.insert(tuple_class_name.clone(), tuple_data_type); - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "type-mapping", - format!(" -> Added DataType: {:?}", data_types[&tuple_class_name]) - ); - } else { - if let Some(oomir::DataType::Class { - fields, methods, .. - }) = data_types.get_mut(&tuple_class_name) - { - methods.entry("eq".to_string()).or_insert_with(|| { + let imported = data_types.has_upstream_type(&name); + let methods = if imported { + HashMap::default() + } else { + HashMap::from_iter([( + "eq".into(), DataTypeMethod::AdtHelperMethod { kind: oomir::AdtHelperKind::PartialEqClass { fields: fields.clone(), }, - } - }); + }, + )]) + }; + let schema = oomir::DataType::Class { + fields, + methods, + is_abstract: false, + super_class: None, + interfaces: Vec::new(), + kind: if is_codegen_sized(resolved_ty, tcx) { + oomir::ClassKind::Value + } else { + oomir::ClassKind::MemoryView + }, + }; + if imported { + data_types.external_schemas.insert(name.clone(), schema); + } else { + data_types.insert(name.clone(), schema); } - breadcrumbs::log!( - breadcrumbs::LogLevel::Info, - "type-mapping", - format!( - "Info: Reusing existing tuple type class: {}", - tuple_class_name - ) - ); } - - ensure_managed_drop( - resolved_ty, - &tuple_class_name, - tcx, - data_types, - instance_context, - ); - - // Return the OOMIR type as a Class reference - oomir::Type::Class(tuple_class_name) + oomir::Type::Class(name) } rustc_middle::ty::TyKind::Slice(component_ty) => { // Special case for slices of string references @@ -486,6 +495,11 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( ty = resolved_ty, ), rustc_middle::ty::TyKind::Closure(def_id, args) => { + // Static closure calls already identify their body. Dynamic calls obtain that identity + // from the adapter. + if args.as_closure().upvar_tys().is_empty() { + return oomir::Type::Unit; + } let safe_name = data_types.closure_class_name(tcx, *def_id, args, false); // Define the closure class struct if not already present @@ -506,6 +520,7 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( safe_name.clone(), oomir::DataType::Class { fields, + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods: HashMap::default(), // 'call' is handled via MIR lowering logic super_class: Some("java/lang/Object".to_string()), @@ -529,6 +544,7 @@ pub(super) fn ty_to_oomir_type_resolved<'tcx>( safe_name.clone(), oomir::DataType::Class { fields: vec![("__state".to_string(), oomir::Type::I32)], + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods: HashMap::default(), super_class: Some("java/lang/Object".to_string()), diff --git a/src/lower1/types/names.rs b/src/lower1/types/names.rs index c1ba82cc..16e99cab 100644 --- a/src/lower1/types/names.rs +++ b/src/lower1/types/names.rs @@ -106,6 +106,7 @@ pub(crate) fn readable_oomir_type_name(t: &oomir::Type) -> String { Type::F32 => "f32".to_string(), Type::F64 => "f64".to_string(), Type::Str => "Str".to_string(), + Type::TaggedI64 => "TaggedI64".to_string(), Type::Void => "Void".to_string(), Type::Unit => "Unit".to_string(), Type::Class(name) => { @@ -115,8 +116,6 @@ pub(crate) fn readable_oomir_type_name(t: &oomir::Type) -> String { Type::Array(inner) => format!("{}Array", readable_oomir_type_name(inner)), Type::Slice(inner) => format!("{}Slice", readable_oomir_type_name(inner)), Type::Pointer(inner) => format!("Ptr{}", readable_oomir_type_name(inner)), - Type::Reference(inner) => format!("Ref{}", readable_oomir_type_name(inner)), - Type::MutableReference(inner) => format!("Ref{}", readable_oomir_type_name(inner)), Type::Interface(name) => { // prefix interfaces with I to avoid conflicts with classes let seg = name.rsplit('/').next().unwrap_or(name); @@ -133,9 +132,6 @@ pub(super) fn readable_tuple_abi_type_name(t: &oomir::Type) -> String { Type::Array(inner) => format!("{}Array", readable_tuple_abi_type_name(inner)), Type::Slice(inner) => format!("{}Slice", readable_tuple_abi_type_name(inner)), Type::Pointer(inner) => format!("Ptr{}", readable_tuple_abi_type_name(inner)), - Type::Reference(inner) | Type::MutableReference(inner) => { - format!("Ref{}", readable_tuple_abi_type_name(inner)) - } _ => readable_oomir_type_name(t), } } @@ -305,121 +301,6 @@ pub(crate) fn readable_rust_type_name<'tcx>( } } -pub(super) fn readable_pointer_codec_type_name<'tcx>( - ty: Ty<'tcx>, - tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, - instance_context: rustc_middle::ty::Instance<'tcx>, -) -> String { - if let Some(name) = primitive_rust_type_name(ty) { - return name.to_owned(); - } - let instantiated = EarlyBinder::bind(tcx, ty).instantiate(tcx, instance_context.args); - let ty = tcx - .try_normalize_erasing_regions(TypingEnv::fully_monomorphized(), instantiated) - .unwrap_or_else(|_| instantiated.skip_norm_wip()); - match ty.kind() { - TyKind::Ref(_, inner, mutability) => format!( - "{}_to_{}", - if mutability.is_mut() { "MutRef" } else { "Ref" }, - readable_pointer_codec_type_name(*inner, tcx, data_types, instance_context) - ), - TyKind::RawPtr(inner, mutability) => format!( - "{}_to_{}", - if mutability.is_mut() { - "MutPtr" - } else { - "ConstPtr" - }, - readable_pointer_codec_type_name(*inner, tcx, data_types, instance_context) - ), - TyKind::Array(inner, length) => { - let length = length - .try_to_target_usize(tcx) - .map(|length| length.to_string()) - .unwrap_or_else(|| "Unknown".to_string()); - format!( - "Array{}_of_{}", - length, - readable_pointer_codec_type_name(*inner, tcx, data_types, instance_context) - ) - } - TyKind::Slice(inner) => format!( - "Slice_of_{}", - readable_pointer_codec_type_name(*inner, tcx, data_types, instance_context) - ), - TyKind::Tuple(elements) if elements.is_empty() => "Unit".to_string(), - TyKind::Tuple(elements) => format!( - "Tuple_of_{}", - elements - .iter() - .map(|element| readable_pointer_codec_type_name( - element, - tcx, - data_types, - instance_context, - )) - .collect::>() - .join("_and_") - ), - TyKind::Dynamic(predicates, _) => { - let base = readable_tuple_abi_type_name(&ty_to_oomir_type( - ty, - tcx, - data_types, - instance_context, - )); - let auto_traits = predicates - .iter() - .filter_map(|predicate| match predicate.skip_binder() { - ExistentialPredicate::AutoTrait(def_id) => { - Some(data_types.readable_class_name(tcx, def_id)) - } - _ => None, - }) - .collect::>(); - if auto_traits.is_empty() { - base - } else { - format!("{}_and_{}", base, auto_traits.join("_and_")) - } - } - TyKind::Adt(adt_def, substs) => { - let base = data_types.readable_class_name(tcx, adt_def.did()); - let args = substs - .iter() - .filter_map(|arg| { - if let Some(arg_ty) = arg.as_type() { - Some(readable_pointer_codec_type_name( - arg_ty, - tcx, - data_types, - instance_context, - )) - } else { - arg.as_const().map(|constant| { - format!( - "Const_{}", - readable_rust_const_name(constant, tcx, instance_context) - ) - }) - } - }) - .collect::>(); - if args.is_empty() { - base - } else { - format!("{}_of_{}", base, args.join("_and_")) - } - } - TyKind::FnDef(def_id, _) => readable_qualified_function_item_path(tcx, *def_id), - TyKind::Char => "char".to_string(), - TyKind::Int(IntTy::Isize) => "isize".to_string(), - TyKind::Uint(UintTy::Usize) => "usize".to_string(), - _ => readable_oomir_type_name(&ty_to_oomir_type(ty, tcx, data_types, instance_context)), - } -} - pub(super) fn readable_rust_const_name<'tcx>( constant: rustc_middle::ty::Const<'tcx>, tcx: TyCtxt<'tcx>, @@ -478,8 +359,7 @@ pub(crate) fn sanitize_name_token(s: &str) -> String { } } -/// Generate a JVM-safe ADT name, retaining raw tokens for the long-name hash. -/// A token may recursively name a large generic type; calculate it only once. +/// ADT names use stable Rust type identity. pub(crate) fn generate_adt_jvm_class_name<'tcx>( adt_def: &AdtDef<'tcx>, substs: GenericArgsRef<'tcx>, @@ -488,87 +368,110 @@ pub(crate) fn generate_adt_jvm_class_name<'tcx>( instance_context: rustc_middle::ty::Instance<'tcx>, ) -> String { let base = data_types.class_name(tcx, adt_def.did()); - let tokens = substs + if !substs .iter() - .filter_map(|arg| readable_rust_generic_arg_name(arg, tcx, data_types, instance_context)) - .collect::>(); - let mut name = base.clone(); - for token in &tokens { - name.push('_'); - name.push_str(&sanitize_name_token(token)); + .any(|arg| arg.as_type().is_some() || arg.as_const().is_some()) + { + return base; } - if name.len() <= MAX_TUPLE_NAME_LEN { - return name; + let concrete = data_types.normalize(tcx, Ty::new_adt(tcx, *adt_def, substs), instance_context); + let identity = stable_type_identity(tcx, concrete); + format!("{base}_{identity}") +} + +/// Tuple names use the complete field representation. Pointer> and Array> +/// have the same readable token. +fn tuple_carrier_name(fields: &[oomir::Type], unsized_tail: bool) -> String { + let prefix = if unsized_tail { "TupleDst" } else { "Tuple" }; + let hash = crate::stable_hash::short_hash_value(&fields, 16); + let tokens = fields + .iter() + .map(|ty| sanitize_name_token(&readable_tuple_abi_type_name(ty))) + .collect::>() + .join("_"); + let readable = format!("org/rustlang/core/{prefix}_{tokens}_{hash}"); + if readable.len() <= MAX_TUPLE_NAME_LEN { + readable + } else { + format!("org/rustlang/core/{prefix}_{hash}") } - let mut identity = base.clone(); - identity.push('_'); - for token in tokens { - identity.push_str(&token); - identity.push('_'); +} + +#[cfg(test)] +mod tuple_names_tests { + use super::{MAX_TUPLE_NAME_LEN, readable_tuple_abi_type_name, tuple_carrier_name}; + use crate::oomir::Type; + + #[test] + fn nested_carriers_cannot_claim_each_others_upstream_tuple() { + let address = Type::pointer(Type::Array(Box::new(Type::U8))); + let array = Type::Array(Box::new(Type::pointer(Type::U8))); + assert_eq!( + readable_tuple_abi_type_name(&address), + readable_tuple_abi_type_name(&array) + ); + let address_name = tuple_carrier_name(std::slice::from_ref(&address), false); + let array_name = tuple_carrier_name(&[array], false); + assert_ne!(address_name, array_name); + } + + #[test] + fn tuple_identity_preserves_field_paths_and_address_layouts() { + let left = Type::Class("example/A_B".into()); + let right = Type::Class("example_A/B".into()); + assert_eq!( + readable_tuple_abi_type_name(&left), + readable_tuple_abi_type_name(&right) + ); + assert_ne!( + tuple_carrier_name(&[left], false), + tuple_carrier_name(&[right], false) + ); + let pointer = Type::pointer(Type::Array(Box::new(Type::U8))); + let four = pointer + .clone() + .with_address_layout(4, Some("array4".into())); + let eight = pointer.with_address_layout(8, Some("array8".into())); + assert_ne!( + tuple_carrier_name(&[four], false), + tuple_carrier_name(&[eight], false) + ); + } + + #[test] + fn identical_representations_share_bounded_names() { + let fields = [Type::U64, Type::Class("example/LongName".repeat(30))]; + let name = tuple_carrier_name(&fields, false); + assert!(name.len() <= MAX_TUPLE_NAME_LEN); + assert_ne!(name, tuple_carrier_name(&fields, true)); + assert_ne!(name, tuple_carrier_name(&fields[..1], false)); } - format!("{base}_{}", short_hash(&identity, 10)) } -/// Generates a readable JVM class name for a tuple type. Rust types that share -/// the same JVM field carriers (such as `usize` and `u64`) deliberately reuse a -/// tuple class. A qualified stable hash is added only when the readable name is -/// already occupied by an ABI-incompatible tuple, such as two unrelated enums -/// both named `Ordering`. +/// Equal OOMIR fields share a tuple class. Address layout remains part of the identity for field +/// accessors. pub(crate) fn generate_tuple_jvm_class_name<'tcx>( element_tys: &[Ty<'tcx>], tcx: TyCtxt<'tcx>, - data_types: &mut Definitions<'tcx>, // Needed for recursive calls + data_types: &mut Definitions<'tcx>, instance_context: rustc_middle::ty::Instance<'tcx>, ) -> String { - // First attempt: build a human-readable name like `Tuple_i32_String`. - let mut tokens: Vec = Vec::new(); - let mut oomir_element_types = Vec::new(); - for ty in element_tys { - let oomir_ty = ty_to_oomir_type(*ty, tcx, data_types, instance_context); - // Downstream monomorphizations have a fresh collision registry, so keep - // carrier paths in tuple names to prevent incompatible linker fragments. - let token = readable_tuple_abi_type_name(&oomir_ty); - tokens.push(sanitize_name_token(&token)); - oomir_element_types.push(oomir_ty); - } - - let readable_name = format!("org/rustlang/core/Tuple_{}", tokens.join("_")); - let local_incompatible_collision = match data_types.get(&readable_name) { - Some(oomir::DataType::Class { fields, .. }) => fields - .iter() - .map(|(_, field_ty)| field_ty) - .ne(oomir_element_types.iter()), - Some(_) => true, - None => false, - }; - - if readable_name.len() <= MAX_TUPLE_NAME_LEN { - let crate_incompatible_collision = - data_types.tuple_name_conflicts(&readable_name, &oomir_element_types); - if !local_incompatible_collision && !crate_incompatible_collision { - return readable_name; - } - } - - let identity = element_tys + let fields = element_tys .iter() - .map(|ty| readable_rust_type_name(*ty, tcx, data_types, instance_context)) - .collect::>() - .join("_"); - let hash = short_hash(&identity, 10); - let disambiguated = format!("{readable_name}_{hash}"); - if disambiguated.len() <= MAX_TUPLE_NAME_LEN { - disambiguated - } else { - format!("org/rustlang/core/Tuple_{hash}") - } + .map(|ty| ty_to_oomir_type(*ty, tcx, data_types, instance_context)) + .collect::>(); + let unsized_tail = element_tys.last().is_some_and(|ty| { + let ty = data_types.normalize(tcx, *ty, instance_context); + !is_codegen_sized(ty, tcx) + }); + tuple_carrier_name(&fields, unsized_tail) } // Helper to get field name from index using DataType info pub(crate) fn get_field_name_from_index( owner_class_name: &str, index: usize, - data_types: &HashMap, + data_types: &crate::lower1::context::Definitions<'_>, ) -> Result { // Return Result for error handling data_types diff --git a/src/lower1/types/nullable.rs b/src/lower1/types/nullable.rs new file mode 100644 index 00000000..535351d0 --- /dev/null +++ b/src/lower1/types/nullable.rs @@ -0,0 +1,143 @@ +//! Enum carriers use proven Rust layouts. Rust identity and discriminants remain separate from Java +//! storage. +use super::*; +use rustc_middle::ty::TypeVisitableExt; + +fn nullable_pointer_payload<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if !tcx.is_lang_item(def.did(), rustc_attr_ir::lang_items::LangItem::Option) { + return None; + } + let payload = args.type_at(0); + let mut physical = payload; + while let Some(owner) = transparent_payload(physical, tcx) { + physical = owner.ty; + } + let pointee = match physical.kind() { + TyKind::Ref(_, pointee, _) if !matches!(pointee.kind(), TyKind::Foreign(_)) => *pointee, + TyKind::RawPtr(pointee, _) + if physical != payload && !matches!(pointee.kind(), TyKind::Foreign(_)) => + { + *pointee + } + // Sized NonNull values use direct addresses. Unsized NonNull values with nominal wrappers + // cannot use null niches. + TyKind::Adt(def, args) + if crate::lower1::is_non_null_lang_item(tcx, def.did()) + && is_codegen_sized(args.type_at(0), tcx) => + { + args.type_at(0) + } + _ => return None, + }; + // Trait objects and struct tails require separate metadata. Only slices and strings use these + // view components. + if !is_codegen_sized(pointee, tcx) && !matches!(pointee.kind(), TyKind::Slice(_) | TyKind::Str) + { + return None; + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let Variants::Multiple { + tag_encoding: + TagEncoding::Niche { + untagged_variant, + niche_variants, + niche_start, + }, + tag_field, + tag, + .. + } = &layout.variants + else { + return None; + }; + (untagged_variant.as_u32() == 1 + && niche_variants.start.as_u32() == 0 + && niche_variants.last.as_u32() == 0 + && *niche_start == 0 + && layout.fields.offset((*tag_field).into()).bytes() == 0 + && tag.size(&tcx.data_layout) == tcx.data_layout.pointer_size() + && tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(payload)) + .ok()? + .size + == layout.size) + .then_some(payload) +} + +#[derive(Clone, Copy)] +pub(crate) struct EnumCarrier<'tcx> { + pub payload: Ty<'tcx>, + pub variant: VariantIdx, + pub nullable: bool, +} + +pub(crate) fn enum_carrier<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + if ty.has_param() || ty.has_escaping_bound_vars() { + return None; + } + let ty = normalize_union_ty(tcx, ty).ok()?; + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if !def.is_enum() { + return None; + } + if let Some(payload) = nullable_pointer_payload(ty, tcx) { + if !super::transparent::finite_representation(ty, tcx) { + return None; + } + return Some(EnumCarrier { + payload, + variant: VariantIdx::from_usize(1), + nullable: true, + }); + } + // Java subtype enums have an explicit nominal interoperability contract. + if def + .variants() + .iter() + .any(|variant| is_jvm_subtype_variant(tcx, variant)) + { + return None; + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let Variants::Single { index } = layout.variants else { + return None; + }; + let variant = def.variant(index); + if variant.fields.len() != 1 || layout.fields.offset(0).bytes() != 0 { + return None; + } + let payload = normalize_union_ty( + tcx, + variant.fields[FieldIdx::from_usize(0)] + .ty(tcx, args) + .skip_norm_wip(), + ) + .ok()?; + let payload_layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(payload)) + .ok()?; + if payload_layout.size != layout.size || payload_layout.align != layout.align { + return None; + } + if !super::transparent::finite_representation(ty, tcx) { + return None; + } + Some(EnumCarrier { + payload, + variant: index, + nullable: false, + }) +} + +pub(crate) fn direct_enum_payload<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + enum_carrier(ty, tcx).map(|carrier| carrier.payload) +} diff --git a/src/lower1/types/packed.rs b/src/lower1/types/packed.rs new file mode 100644 index 00000000..264faa4a --- /dev/null +++ b/src/lower1/types/packed.rs @@ -0,0 +1,274 @@ +//! Small integer aggregates use their storage word. Rust identity still controls methods, traits, +//! layout, and destruction. +use super::*; +use crate::oomir::{BinaryOp, Constant, Instruction, Operand, Type}; + +#[derive(Clone, Copy)] +pub(crate) struct PackedWord<'tcx> { + pub scalar: Ty<'tcx>, + size: u8, + fields: [(u8, u8); 4], +} + +pub(crate) fn value_scalar_ty<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + enum_scalar_ty(ty, tcx).or_else(|| packed_word(ty, tcx).map(|word| word.scalar)) +} + +pub(crate) fn packed_word<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + if !matches!( + ty.kind(), + TyKind::Adt(..) | TyKind::Tuple(..) | TyKind::Alias(..) + ) || ty.has_param() + || ty.has_escaping_bound_vars() + { + return None; + } + let ty = normalize_union_ty(tcx, ty).ok()?; + let mut fields = [tcx.types.unit; 4]; + let len = match ty.kind() { + TyKind::Adt(def, args) + if def.is_struct() + && !tcx + .is_lang_item(def.did(), rustc_attr_ir::lang_items::LangItem::UnsafeCell) + && !def.has_dtor(tcx) + && adt_class_kind(tcx, def, args) == oomir::ClassKind::Value => + { + let members = &def.non_enum_variant().fields; + if members.len() > fields.len() { + return None; + } + for (i, field) in members.iter().enumerate() { + fields[i] = normalize_union_ty(tcx, field.ty(tcx, args).skip_norm_wip()).ok()?; + } + members.len() + } + TyKind::Tuple(members) if members.len() <= fields.len() => { + for (i, field) in members.iter().enumerate() { + fields[i] = field; + } + members.len() + } + _ => return None, + }; + let fields = &fields[..len]; + if fields.is_empty() { + return None; + } + // Pointer identity, floats, padding and interior mutation are not words. + for &field in fields { + let mut ty = field; + while let TyKind::Pat(inner, _) = ty.kind() { + ty = *inner; + } + if !matches!( + ty.kind(), + TyKind::Int(IntTy::I8 | IntTy::I16 | IntTy::I32 | IntTy::I64 | IntTy::Isize) + | TyKind::Uint( + UintTy::U8 | UintTy::U16 | UintTy::U32 | UintTy::U64 | UintTy::Usize + ) + ) { + return None; + } + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let size = layout.size.bytes(); + let scalar = match size { + 1 => tcx.types.u8, + 2 => tcx.types.u16, + 4 => tcx.types.u32, + 8 => tcx.types.u64, + _ => return None, + }; + let mut result = PackedWord { + scalar, + size: size as u8, + fields: [(0, 0); 4], + }; + let mut covered = 0; + for (index, field) in fields.iter().enumerate() { + let field_size = layout_size_bytes(tcx, *field).ok()?; + let offset = layout.fields.offset(index).bytes_usize(); + if offset + field_size > size as usize { + return None; + } + result.fields[index] = (offset as u8, field_size as u8); + covered += field_size; + } + // Padding may be uninitialized: never turn it into a scalar load. + (covered == size as usize).then_some(result) +} + +impl PackedWord<'_> { + pub(crate) fn jvm_type(self) -> Type { + match self.size { + 1 => Type::U8, + 2 => Type::U16, + 4 => Type::U32, + 8 => Type::U64, + _ => unreachable!(), + } + } + + pub(crate) fn zero(self) -> Operand { + Operand::Constant(match self.size { + 1 => Constant::U8(0), + 2 => Constant::U16(0), + 4 => Constant::U32(0), + 8 => Constant::U64(0), + _ => unreachable!(), + }) + } + + fn bits_type(self) -> Type { + if self.size == 8 { Type::U64 } else { Type::U32 } + } + + fn constant(self, bits: u64) -> Operand { + Operand::Constant(if self.size == 8 { + Constant::U64(bits) + } else { + Constant::U32(bits as u32) + }) + } + + fn mask(self, index: usize) -> u64 { + let size = self.fields[index].1; + if size == 8 { + u64::MAX + } else { + (1u64 << (size * 8)) - 1 + } + } + + pub(crate) fn extract( + self, + value: Operand, + index: usize, + ty: &Type, + dest: &str, + code: &mut Vec, + ) -> Operand { + let value = cast(value, self.bits_type(), &format!("{dest}_bits"), code); + let shift = self.fields[index].0 * 8; + let value = if shift == 0 { + value + } else { + binary( + BinaryOp::Shr, + value, + self.constant(shift as u64), + &format!("{dest}_shift"), + code, + ) + }; + cast(value, ty.clone(), dest, code) + } + + pub(crate) fn insert( + self, + value: Operand, + index: usize, + field: Operand, + dest: &str, + code: &mut Vec, + ) -> Operand { + let shift = self.fields[index].0 * 8; + let mask = self.mask(index); + let value = cast(value, self.bits_type(), &format!("{dest}_bits"), code); + let field = cast(field, self.bits_type(), &format!("{dest}_field_bits"), code); + let field = binary( + BinaryOp::BitAnd, + field, + self.constant(mask), + &format!("{dest}_masked"), + code, + ); + let field = if shift == 0 { + field + } else { + binary( + BinaryOp::Shl, + field, + self.constant(shift as u64), + &format!("{dest}_shift"), + code, + ) + }; + let value = binary( + BinaryOp::BitAnd, + value, + self.constant(!(mask << shift)), + &format!("{dest}_clear"), + code, + ); + let value = binary( + BinaryOp::BitOr, + value, + field, + &format!("{dest}_merge"), + code, + ); + cast(value, self.jvm_type(), dest, code) + } + + pub(crate) fn field_address( + self, + owner: Operand, + index: usize, + field: &Type, + dest: &str, + code: &mut Vec, + ) -> Operand { + let owner_type = owner.get_type().expect("packed owner has a type"); + let offset = format!("{dest}_offset"); + code.push(Instruction::AddressOffset { + dest: Some(offset.clone()), + source: owner, + count: Operand::Constant(Constant::U64(self.fields[index].0 as u64)), + ty: owner_type.clone(), + bytes: true, + wrapping: false, + subtract: false, + }); + let pointer_type = Type::pointer(field.clone()); + let address = format!("{dest}_address"); + code.push(Instruction::AddressRetype { + dest: Some(address.clone()), + source: operand_var(offset, owner_type), + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_type.clone(), + size: Operand::Constant(Constant::I32(self.fields[index].1 as i32)), + codec: Operand::Constant(Constant::Null(Type::java_string())), + }), + }); + operand_var(address, pointer_type) + } +} + +fn cast(value: Operand, ty: Type, dest: &str, code: &mut Vec) -> Operand { + code.push(Instruction::Cast { + dest: dest.into(), + op: value, + ty: ty.clone(), + }); + operand_var(dest, ty) +} + +fn binary( + op: BinaryOp, + left: Operand, + right: Operand, + dest: &str, + code: &mut Vec, +) -> Operand { + let ty = left.get_type().unwrap(); + code.push(Instruction::Binary { + dest: dest.into(), + op, + op1: left, + op2: right, + }); + operand_var(dest, ty) +} diff --git a/src/lower1/types/pointer_codecs.rs b/src/lower1/types/pointer_codecs.rs index 3c0ee75e..70e204ec 100644 --- a/src/lower1/types/pointer_codecs.rs +++ b/src/lower1/types/pointer_codecs.rs @@ -7,19 +7,14 @@ pub(super) fn pointer_codec_class_name<'tcx>( ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, local_name: &str, - value_ty: &oomir::Type, ) -> String { let root = match ty.kind() { - // Structural tuple/array types are language-level types. Give them the - // same stable owner regardless of the crate which instantiates them. TyKind::Tuple(_) | TyKind::Array(_, _) => "org/rustlang/core".to_string(), - _ => match value_ty { - oomir::Type::Class(class_name) => class_name - .rsplit_once('/') - .map(|(package, _)| package.to_string()) - .unwrap_or_else(|| jvm_names::crate_root(tcx, rustc_span::def_id::LOCAL_CRATE)), - _ => jvm_names::crate_root(tcx, rustc_span::def_id::LOCAL_CRATE), - }, + TyKind::Adt(def, _) => jvm_names::crate_root(tcx, def.did().krate), + TyKind::Closure(def, _) | TyKind::Coroutine(def, _) => { + jvm_names::crate_root(tcx, def.krate) + } + _ => unreachable!("codec owner requires an aggregate"), }; format!("{root}/{}", jvm_names::path_segment(local_name)) } @@ -55,30 +50,54 @@ pub(crate) fn ensure_pointer_memory_codec<'tcx>( instance_context: rustc_middle::ty::Instance<'tcx>, ) -> Result, String> { let ty = resolve_union_ty(tcx, ty, instance_context)?; - let cached = data_types.completed_codec(ty); - if let Some(cached) = cached - && (!data_types.contains_key(&cached.class_name) - || codec_is_complete(data_types, &cached.class_name)) - { - // The first requesting shard owns the codec body. Other shards still - // register the value carrier's declarations, but do not rebuild the - // encoder/decoder merely to discard them during canonical merging. + if value_scalar_ty(ty, tcx).is_some() || transparent_payload(ty, tcx).is_some() { + return Ok(None); + } + if let Some(cached) = data_types.completed_codec(ty) { + // The first requesting shard owns the body. Other shards require only its declarations. ty_to_oomir_type(ty, tcx, data_types, instance_context); return Ok(Some(cached)); } - let result = build_pointer_memory_codec(ty, tcx, data_types, instance_context)?; - if let Some(codec) = &result - && (codec.class_name.starts_with(ZERO_SIZED_CODEC_PREFIX) - || codec_is_complete(data_types, &codec.class_name)) - { + if let Some(codec) = data_types.building_codecs.get(&ty) { + return Ok(Some(codec.clone())); + } + let result = build_pointer_memory_codec(ty, tcx, data_types, instance_context); + data_types.building_codecs.remove(&ty); + if let Ok(Some(codec)) = &result { data_types.complete_codec(ty, codec.clone()); } - Ok(result) + result } -fn codec_is_complete(data_types: &HashMap, class: &str) -> bool { - matches!(data_types.get(class), Some(oomir::DataType::Class { methods, .. }) - if methods.contains_key("encode") && methods.contains_key("decode")) +/// Codec recipes share holders. Method enumeration would also resolve unrelated codec types. +fn register_codec( + data_types: &mut Definitions<'_>, + codec: &PointerMemoryCodec, + methods: HashMap, +) { + let entry = data_types + .entry(codec.owner().to_owned()) + .or_insert_with(|| oomir::DataType::Class { + fields: Vec::new(), + kind: oomir::ClassKind::Static, + is_abstract: false, + methods: HashMap::default(), + super_class: Some("java/lang/Object".to_owned()), + interfaces: Vec::new(), + }); + let oomir::DataType::Class { + methods: existing, .. + } = entry + else { + unreachable!("codec holder cannot be an interface"); + }; + for (name, method) in methods { + let DataTypeMethod::Function(mut function) = method else { + unreachable!("codec operation must have a body"); + }; + function.name = codec.method(&name); + existing.insert(function.name.clone(), DataTypeMethod::Function(function)); + } } fn build_pointer_memory_codec<'tcx>( @@ -102,6 +121,20 @@ fn build_pointer_memory_codec<'tcx>( if !value_ty.has_jvm_value() { return Ok(None); } + // Primitive array codecs share runtime methods. Each recipe supplies its length and element + // type. + if let TyKind::Array(element, _) = ty.kind() + && let oomir::Type::Array(value) = &value_ty + && let Some(width) = oomir::Type::pointer((**value).clone()).scalar_address_size() + && layout_size_bytes(tcx, *element)? == width as usize + { + return Ok(Some(PointerMemoryCodec { + class_name: format!( + "org/rustlang/runtime/ArrayMemoryCodec#array#{}#{size}", + value_ty.to_jvm_descriptor() + ), + })); + } // These carriers have a public no-argument constructor and no state to // encode or bind. Retain the concrete class identity in the codec recipe; // no generated codec class or computational helper bodies are needed. @@ -118,17 +151,6 @@ fn build_pointer_memory_codec<'tcx>( class_name: format!("{ZERO_SIZED_CODEC_PREFIX}{class_name}"), })); } - exact_bytes_supported(ty, tcx, instance_context, data_types)?; - let readable = format!( - "{}_{}bytes", - sanitize_name_token(&readable_pointer_codec_type_name( - ty, - tcx, - data_types, - instance_context, - )), - size - ); // A pretty-printed `Ty` can use a different crate alias in an upstream // crate and a downstream monomorphization (for example `alloc::borrow` // versus `std::borrow`). Pointer views of the same Rust allocation must @@ -141,40 +163,16 @@ fn build_pointer_memory_codec<'tcx>( stable_type_identity(tcx, ty), value_ty.to_jvm_descriptor() ); - let local_name = - crate::stable_hash::readable_disambiguated_name("PointerCodec", &readable, &identity, 180); - let class_name = pointer_codec_class_name(ty, tcx, &local_name, &value_ty); - if codec_is_complete(data_types, &class_name) { - return Ok(Some(PointerMemoryCodec { class_name })); - } - - if matches!( - data_types.get(&class_name), - Some(oomir::DataType::Class { methods, .. }) if methods.is_empty() - ) { - // Recursive pointees may refer back to this codec while its encode and - // decode bodies are still being constructed. The final class will own - // both methods once the outer generation completes. - return Ok(Some(PointerMemoryCodec { class_name })); - } - if matches!( - data_types.get(&class_name), - Some(oomir::DataType::Interface { .. }) - ) { - return Err(format!( - "pointer codec helper name {class_name} is already an interface" - )); + let key = crate::stable_hash::short_hash(&identity, 16); + let owner = pointer_codec_class_name(ty, tcx, &format!("Codecs_{}", &key[..2])); + let codec = PointerMemoryCodec { + class_name: format!("{owner}#{key}#{}#{size}", value_ty.to_jvm_descriptor()), + }; + if data_types.has_upstream_codec(&codec.class_name) { + return Ok(Some(codec)); } - data_types.insert( - class_name.clone(), - oomir::DataType::Class { - fields: Vec::new(), - is_abstract: false, - methods: HashMap::default(), - super_class: Some("java/lang/Object".to_string()), - interfaces: Vec::new(), - }, - ); + exact_bytes_supported(ty, tcx, instance_context, data_types)?; + data_types.building_codecs.insert(ty, codec.clone()); if matches!(ty.kind(), TyKind::Coroutine(_, _)) { let generated = coroutine_pointer_codec_methods( @@ -184,67 +182,91 @@ fn build_pointer_memory_codec<'tcx>( data_types, instance_context, ); - let methods = match generated { - Ok(methods) => methods, - Err(error) => { - data_types.remove(&class_name); - return Err(error); - } - }; - match data_types.get_mut(&class_name) { - Some(oomir::DataType::Class { - methods: existing, .. - }) => existing.extend(methods), - _ => unreachable!("coroutine pointer codec placeholder disappeared"), - } - return Ok(Some(PointerMemoryCodec { class_name })); + register_codec(data_types, &codec, generated?); + return Ok(Some(codec)); } let bytes_ty = byte_array_type(); let object_storage_size = union_object_storage_size(ty, size, tcx, instance_context, data_types); - let mut encode_instructions = vec![ - oomir::Instruction::NewArray { - dest: "_bytes".to_string(), - element_type: oomir::Type::I8, - size: oomir::Operand::Constant(oomir::Constant::I32(size as i32)), - }, - allocate_union_object_storage("_objects", object_storage_size), - ]; - let encode_storage = JvmUnionStorage::at_start("_bytes", "_objects"); + let encode_at_offset = size > 0 && object_storage_size == 0 && range_encoder::supports(ty, tcx); + let mut encode_instructions = vec![allocate_union_object_storage( + "_objects", + object_storage_size, + )]; + if !encode_at_offset { + encode_instructions.insert( + 0, + oomir::Instruction::NewArray { + dest: "_bytes".to_string(), + element_type: oomir::Type::I8, + size: oomir::Operand::Constant(oomir::Constant::I32(size as i32)), + }, + ); + } + let encode_storage = if encode_at_offset { + JvmUnionStorage::at_offset("_2", "_objects", operand_var("_3", oomir::Type::I32)) + } else { + JvmUnionStorage::at_start("_bytes", "_objects") + }; let mut encode_counter = 0; - if let Err(error) = emit_ty_to_union_bytes( - ty, - if value_ty.has_jvm_value() { - operand_var("_1", value_ty.clone()) - } else { - oomir::Operand::Constant(oomir::Constant::Unit) - }, - &encode_storage, - 0, - tcx, - data_types, - instance_context, - &mut encode_instructions, - &mut encode_counter, - ) { - data_types.remove(&class_name); - return Err(error); + let source = if value_ty.has_jvm_value() { + operand_var("_1", value_ty.clone()) + } else { + oomir::Operand::Constant(oomir::Constant::Unit) + }; + if encode_at_offset { + range_encoder::emit( + ty, + source, + &encode_storage, + tcx, + data_types, + instance_context, + &mut encode_instructions, + &mut encode_counter, + )?; + } else { + emit_ty_to_union_bytes( + ty, + source, + &encode_storage, + 0, + tcx, + data_types, + instance_context, + &mut encode_instructions, + &mut encode_counter, + )?; } encode_instructions.push(oomir::Instruction::Return { - operand: Some(operand_var("_bytes", bytes_ty.clone())), + operand: (!encode_at_offset).then(|| operand_var("_bytes", bytes_ty.clone())), }); let encode = oomir::Function { - name: "encode".to_string(), + name: if encode_at_offset { + "encodeAt" + } else { + "encode" + } + .to_string(), owner_class: None, debug_variables: Vec::new(), signature: oomir::Signature { - params: value_ty - .has_jvm_value() - .then(|| vec![("value".to_string(), value_ty.clone())]) - .unwrap_or_default(), - ret: Box::new(bytes_ty.clone()), + params: if encode_at_offset { + vec![ + ("value".to_string(), value_ty.clone()), + ("bytes".to_string(), bytes_ty.clone()), + ("offset".to_string(), oomir::Type::I32), + ] + } else { + vec![("value".to_string(), value_ty.clone())] + }, + ret: Box::new(if encode_at_offset { + oomir::Type::Void + } else { + bytes_ty.clone() + }), is_static: true, }, body: simple_body(encode_instructions).into(), @@ -276,7 +298,6 @@ fn build_pointer_memory_codec<'tcx>( ) { Ok(decoded) => decoded, Err(error) => { - data_types.remove(&class_name); return Err(error); } }; @@ -307,7 +328,7 @@ fn build_pointer_memory_codec<'tcx>( body: simple_body(decode_instructions).into(), }; - let pointer_ty = oomir::Type::Pointer(Box::new(value_ty.clone())); + let pointer_ty = oomir::Type::pointer(value_ty.clone()); let mut bind_instructions = Vec::new(); let mut bind_counter = 0; if let Err(error) = emit_memory_view_bindings( @@ -320,52 +341,13 @@ fn build_pointer_memory_codec<'tcx>( &mut bind_instructions, &mut bind_counter, ) { - data_types.remove(&class_name); return Err(error); } let mut methods = HashMap::from_iter([ - ("encode".to_string(), DataTypeMethod::Function(encode)), + (encode.name.clone(), DataTypeMethod::Function(encode)), (decode.name.clone(), DataTypeMethod::Function(decode)), ]); - if decode_at_offset { - let decoded = operand_var("_decoded", value_ty.clone()); - let decode = oomir::Function { - name: "decode".to_string(), - owner_class: None, - debug_variables: Vec::new(), - signature: oomir::Signature { - params: vec![("bytes".to_string(), bytes_ty.clone())], - ret: Box::new(value_ty.clone()), - is_static: true, - }, - body: simple_body(vec![ - oomir::Instruction::InvokeStatic { - dest: Some("_decoded".to_string()), - class_name: class_name.clone(), - method_name: "decodeAt".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("bytes".to_string(), bytes_ty.clone()), - ("offset".to_string(), oomir::Type::I32), - ], - ret: Box::new(value_ty.clone()), - is_static: true, - }, - args: vec![ - operand_var("_1", bytes_ty), - oomir::Operand::Constant(oomir::Constant::I32(0)), - ], - }, - oomir::Instruction::Return { - operand: Some(decoded), - }, - ]) - .into(), - }; - methods.insert("decode".to_string(), DataTypeMethod::Function(decode)); - } - // The runtime treats a missing binder as a no-op. Do not compile a method - // whose only instruction would be `return`. + // The runtime supplies whole-buffer adapters and treats absent binders as no-ops. if !bind_instructions.is_empty() { bind_instructions.push(oomir::Instruction::Return { operand: None }); let bind = oomir::Function { @@ -426,18 +408,8 @@ fn build_pointer_memory_codec<'tcx>( }), ); } - match data_types.get_mut(&class_name) { - Some(oomir::DataType::Class { - methods: existing, .. - }) => existing.extend(methods), - Some(oomir::DataType::Interface { .. }) => { - return Err(format!( - "pointer codec helper name {class_name} is already an interface" - )); - } - None => unreachable!("pointer codec placeholder disappeared during generation"), - } - Ok(Some(PointerMemoryCodec { class_name })) + register_codec(data_types, &codec, methods); + Ok(Some(codec)) } pub(crate) fn pointer_memory_codec_operand<'tcx>( @@ -446,6 +418,13 @@ pub(crate) fn pointer_memory_codec_operand<'tcx>( data_types: &mut Definitions<'tcx>, instance_context: rustc_middle::ty::Instance<'tcx>, ) -> oomir::Operand { + let ty = data_types.normalize(tcx, ty, instance_context); + if let TyKind::Pat(inner, _) = ty.kind() { + return pointer_memory_codec_operand(*inner, tcx, data_types, instance_context); + } + if let Some(payload) = transparent_payload(ty, tcx) { + return pointer_memory_codec_operand(payload.ty, tcx, data_types, instance_context); + } if let Some(codec) = fat_pointer_codec_operand(ty, tcx, data_types, instance_context) { return codec; } @@ -476,6 +455,13 @@ pub(crate) fn pointer_view_codec_operand<'tcx>( data_types: &mut Definitions<'tcx>, instance_context: rustc_middle::ty::Instance<'tcx>, ) -> oomir::Operand { + let ty = data_types.normalize(tcx, ty, instance_context); + if let TyKind::Pat(inner, _) = ty.kind() { + return pointer_view_codec_operand(*inner, tcx, data_types, instance_context); + } + if let Some(payload) = transparent_payload(ty, tcx) { + return pointer_view_codec_operand(payload.ty, tcx, data_types, instance_context); + } if let Some(codec) = fat_pointer_codec_operand(ty, tcx, data_types, instance_context) { return codec; } diff --git a/src/lower1/types/range_encoder.rs b/src/lower1/types/range_encoder.rs new file mode 100644 index 00000000..50782b6c --- /dev/null +++ b/src/lower1/types/range_encoder.rs @@ -0,0 +1,177 @@ +//! Bounded scalar snapshots for allocation-free, overlap-safe aggregate stores. +use super::*; +use crate::lower1::context::Definitions; + +pub(super) fn supports<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool { + fn visit<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, budget: &mut usize) -> bool { + if *budget == 0 { + return false; + } + *budget -= 1; + match ty.kind() { + TyKind::Bool | TyKind::Char | TyKind::Int(_) | TyKind::Uint(_) => true, + TyKind::Float(kind) => *kind != FloatTy::F128, + TyKind::Tuple(fields) => fields.iter().all(|ty| visit(ty, tcx, budget)), + TyKind::Adt(def, args) if def.is_struct() => def + .non_enum_variant() + .fields + .iter() + .all(|field| visit(field.ty(tcx, args).skip_norm_wip(), tcx, budget)), + TyKind::Array(element, length) => { + length.try_to_target_usize(tcx).is_some_and(|length| { + length <= *budget as u64 && (0..length).all(|_| visit(*element, tcx, budget)) + }) + } + _ => false, + } + } + visit(ty, tcx, &mut 64) +} + +pub(super) fn emit<'tcx>( + ty: Ty<'tcx>, + source: oomir::Operand, + storage: &JvmUnionStorage, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, + instructions: &mut Vec, + counter: &mut usize, +) -> Result<(), String> { + let mut leaves = Vec::new(); + snapshot( + ty, + source, + 0, + tcx, + definitions, + instance, + instructions, + counter, + &mut leaves, + )?; + // The source array can also be the destination. All source reads must precede padding and + // destination writes. + let offset = storage.byte_index(0, instructions, counter); + instructions.push(oomir::Instruction::InvokeStatic { + dest: None, + class_name: MEMORY_BYTES_CLASS.into(), + method_name: "clear".into(), + method_ty: oomir::Signature { + params: vec![ + ("bytes".into(), byte_array_type()), + ("offset".into(), oomir::Type::I32), + ("size".into(), oomir::Type::I32), + ], + ret: Box::new(oomir::Type::Void), + is_static: true, + }, + args: vec![ + operand_var(storage.bytes_var.clone(), byte_array_type()), + offset, + oomir::Operand::Constant(oomir::Constant::I32(layout_size_bytes(tcx, ty)? as i32)), + ], + }); + for (ty, value, offset) in leaves { + emit_scalar_to_union_bytes( + ty, + value, + storage, + offset, + tcx, + definitions, + instance, + instructions, + counter, + )?; + } + Ok(()) +} + +fn snapshot<'tcx>( + ty: Ty<'tcx>, + source: oomir::Operand, + offset: usize, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, + instructions: &mut Vec, + counter: &mut usize, + leaves: &mut Vec<(Ty<'tcx>, oomir::Operand, usize)>, +) -> Result<(), String> { + let ty = value_scalar_ty(ty, tcx).unwrap_or(ty); + if let Some(payload) = transparent_payload(ty, tcx) { + return snapshot( + payload.ty, + source, + offset, + tcx, + definitions, + instance, + instructions, + counter, + leaves, + ); + } + if layout_size_bytes(tcx, ty)? == 0 { + return Ok(()); + } + match ty.kind() { + TyKind::Array(element, length) => { + let length = length + .try_to_target_usize(tcx) + .ok_or("unknown range encoder array length")?; + let stride = layout_size_bytes(tcx, *element)?; + let element_ty = ty_to_oomir_type(*element, tcx, definitions, instance); + for index in 0..length { + let dest = next_union_temp("range_element", counter); + instructions.push(oomir::Instruction::ArrayGet { + dest: dest.clone(), + array: source.clone(), + index: oomir::Operand::Constant(oomir::Constant::I32(index as i32)), + }); + snapshot( + *element, + operand_var(dest, element_ty.clone()), + offset + index as usize * stride, + tcx, + definitions, + instance, + instructions, + counter, + leaves, + )?; + } + } + TyKind::Tuple(_) | TyKind::Adt(_, _) => { + let layout = union_aggregate_layout(ty, tcx, definitions, instance)? + .ok_or("range encoder aggregate has no field layout")?; + for field in layout.fields { + if layout_size_bytes(tcx, field.rust_ty)? == 0 { + continue; + } + let dest = next_union_temp("range_field", counter); + instructions.push(oomir::Instruction::GetField { + dest: dest.clone(), + object: source.clone(), + field_name: field.jvm_name, + field_ty: field.jvm_ty.clone(), + owner_class: layout.class_name.clone(), + }); + snapshot( + field.rust_ty, + operand_var(dest, field.jvm_ty), + offset + field.offset, + tcx, + definitions, + instance, + instructions, + counter, + leaves, + )?; + } + } + _ => leaves.push((ty, source, offset)), + } + Ok(()) +} diff --git a/src/lower1/types/read.rs b/src/lower1/types/read.rs index 042af002..71c648b3 100644 --- a/src/lower1/types/read.rs +++ b/src/lower1/types/read.rs @@ -12,6 +12,61 @@ pub(super) fn emit_ty_from_union_bytes<'tcx>( temp_counter: &mut usize, ) -> Result { let ty = resolve_union_ty(tcx, ty, instance_context)?; + if let Some(layout) = tagged_scalar(ty, tcx) { + let tag = emit_ty_from_union_bytes( + tcx.types.u64, + storage, + base_offset + layout.tag_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + )?; + let value = emit_ty_from_union_bytes( + layout.payload, + storage, + base_offset + layout.payload_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + )?; + let dest = next_union_temp("tagged", temp_counter); + instructions.push(oomir::Instruction::TaggedPack { + dest: dest.clone(), + value, + tag, + }); + return Ok(operand_var(dest, oomir::Type::TaggedI64)); + } + if let Some(scalar) = value_scalar_ty(ty, tcx) { + return emit_ty_from_union_bytes( + scalar, + storage, + base_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + ); + } + if let Some(payload) = + direct_enum_payload(ty, tcx).or_else(|| transparent_payload(ty, tcx).map(|p| p.ty)) + { + return emit_ty_from_union_bytes( + payload, + storage, + base_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + ); + } if layout_size_bytes(tcx, ty)? == 0 { let jvm_ty = ty_to_oomir_type(ty, tcx, data_types, instance_context); if !jvm_ty.has_jvm_value() { @@ -139,8 +194,8 @@ pub(super) fn emit_ty_from_union_bytes<'tcx>( let decoded = next_union_temp("nested_coroutine_value", temp_counter); instructions.push(oomir::Instruction::InvokeStatic { dest: Some(decoded.clone()), - class_name: codec.class_name, - method_name: "decode".to_string(), + class_name: codec.owner().to_owned(), + method_name: codec.method("decode"), method_ty: oomir::Signature { params: vec![("bytes".to_string(), byte_array_type())], ret: Box::new(value_ty.clone()), @@ -174,7 +229,7 @@ pub(super) fn emit_ty_from_union_bytes<'tcx>( .ok_or_else(|| format!("array length is not concrete for {inner:?}"))?; let element_oomir_ty = ty_to_oomir_type(*element, tcx, data_types, instance_context); let slice_ty = oomir::Type::Slice(Box::new(element_oomir_ty.clone())); - let pointer_ty = oomir::Type::Pointer(Box::new(element_oomir_ty)); + let pointer_ty = oomir::Type::pointer(element_oomir_ty); let address = emit_bits_from_union_bytes( oomir::Type::U64, layout_size_bytes(tcx, ty)?, @@ -393,7 +448,7 @@ pub(super) fn emit_ty_from_union_bytes<'tcx>( let oomir::Type::Class(enum_class) = &enum_oomir_ty else { return Err(format!("enum {ty:?} did not map to a JVM class")); }; - ensure_enum_union_codec( + let codec = ensure_enum_union_codec( adt_def, substs, ty, @@ -406,8 +461,8 @@ pub(super) fn emit_ty_from_union_bytes<'tcx>( let offset = storage.byte_index(base_offset, instructions, temp_counter); instructions.push(oomir::Instruction::InvokeStatic { dest: Some(enum_dest.clone()), - class_name: enum_class.clone(), - method_name: ENUM_READ_UNION_STORAGE_METHOD.to_string(), + class_name: codec.owner, + method_name: codec.reader, method_ty: enum_union_read_signature(enum_class), args: vec![ operand_var(storage.bytes_var.clone(), byte_array_type()), diff --git a/src/lower1/types/storage_layout.rs b/src/lower1/types/storage_layout.rs new file mode 100644 index 00000000..08432c51 --- /dev/null +++ b/src/lower1/types/storage_layout.rs @@ -0,0 +1,149 @@ +//! Compact typed field paths for borrowed scalar projections. +use super::*; +use crate::lower1::context::Definitions; + +pub(crate) fn scalar_storage_layout<'tcx>( + ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, +) -> oomir::Operand { + let ty = definitions.normalize(tcx, ty, instance); + let descriptor = if let Some(cached) = definitions.storage_layout(ty) { + cached + } else { + let mut fields = Vec::new(); + collect( + ty, + 0, + &mut Vec::new(), + &mut 64, + &mut fields, + tcx, + definitions, + instance, + ); + let result = (!fields.is_empty()).then(|| fields.join("\n")); + definitions.remember_storage_layout(ty, result.clone()); + result + }; + oomir::Operand::Constant(match descriptor { + Some(descriptor) => oomir::Constant::String(descriptor), + None => oomir::Constant::Null(oomir::Type::java_string()), + }) +} + +fn collect<'tcx>( + ty: Ty<'tcx>, + offset: usize, + path: &mut Vec, + budget: &mut usize, + fields: &mut Vec, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: rustc_middle::ty::Instance<'tcx>, +) { + if *budget == 0 { + return; + } + *budget -= 1; + let Ok(ty) = resolve_union_ty(tcx, ty, instance) else { + return; + }; + if let Some(payload) = transparent_payload(ty, tcx) { + collect( + payload.ty, + offset, + path, + budget, + fields, + tcx, + definitions, + instance, + ); + return; + } + if !is_codegen_sized(ty, tcx) { + return; + } + if !path.is_empty() + && matches!(ty.kind(), TyKind::Ref(..) | TyKind::RawPtr(..)) + && let Some(shape) = ty_to_oomir_type(ty, tcx, definitions, instance).component_shape() + && let Ok(size) = layout_size_bytes(tcx, ty) + && matches!(size, 8 | 16) + { + let codec = match pointer_view_codec_operand(ty, tcx, definitions, instance) { + oomir::Operand::Constant(oomir::Constant::String(codec)) => codec, + _ => return, + }; + let kind = if shape == jvm_compiler_core::ir::ComponentShape::View { + 'v' + } else { + 'p' + }; + fields.push(format!( + "{offset},{size},{},{kind}:{}\n{codec}", + path.join("/"), + codec.split('\n').count() + )); + return; + } + if let TyKind::Array(element, length) = ty.kind() + && matches!( + element.kind(), + TyKind::Bool | TyKind::Int(_) | TyKind::Uint(_) | TyKind::Float(_) + ) + && let Some(length) = length.try_to_target_usize(tcx) + && length > 0 + && length <= i32::MAX as u64 + && let Ok(size) = layout_size_bytes(tcx, *element) + && matches!(size, 1 | 2 | 4 | 8) + && length + .checked_mul(size as u64) + .is_some_and(|n| n <= i32::MAX as u64) + { + // Each access resolves the array field from its current owner. Whole-array replacement can + // change that field. + fields.push(format!("{offset},{size},{},{length}", path.join("/"))); + return; + } + let scalar = matches!( + ty.kind(), + TyKind::Bool | TyKind::Int(_) | TyKind::Uint(_) | TyKind::Float(_) + ) || value_scalar_ty(ty, tcx).is_some(); + if scalar { + if path.is_empty() { + return; + } + let Ok(size) = layout_size_bytes(tcx, ty) else { + return; + }; + if matches!(size, 1 | 2 | 4 | 8) { + fields.push(format!("{offset},{size},{}", path.join("/"))); + } + return; + } + // JVM field order does not describe enum payloads, pointer contents, or aggregate arrays. + if !matches!(ty.kind(), TyKind::Tuple(_)) + && !matches!(ty.kind(), TyKind::Adt(def, _) if def.is_struct()) + { + return; + } + let Ok(Some(layout)) = union_aggregate_layout(ty, tcx, definitions, instance) else { + return; + }; + for field in layout.fields { + path.push(field.jvm_name); + collect( + field.rust_ty, + offset + field.offset, + path, + budget, + fields, + tcx, + definitions, + instance, + ); + path.pop(); + } +} diff --git a/src/lower1/types/tagged.rs b/src/lower1/types/tagged.rs new file mode 100644 index 00000000..1c2068fa --- /dev/null +++ b/src/lower1/types/tagged.rs @@ -0,0 +1,59 @@ +//! Small tagged values retain their Rust layout separately from JVM components. +use super::*; + +#[derive(Clone, Copy)] +pub(crate) struct TaggedScalar<'tcx> { + pub payload: Ty<'tcx>, + pub tag_offset: usize, + pub payload_offset: usize, +} + +pub(crate) fn tagged_scalar<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option> { + if ty.has_param() || ty.has_escaping_bound_vars() { + return None; + } + let ty = normalize_union_ty(tcx, ty).ok()?; + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if !tcx.is_lang_item(def.did(), rustc_attr_ir::lang_items::LangItem::Option) { + return None; + } + let payload = args.type_at(0); + if !matches!( + payload.kind(), + TyKind::Int(IntTy::I64 | IntTy::Isize) | TyKind::Uint(UintTy::U64 | UintTy::Usize) + ) { + return None; + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let Variants::Multiple { + tag, + tag_encoding: TagEncoding::Direct, + tag_field, + ref variants, + .. + } = layout.variants + else { + return None; + }; + if tag.size(&tcx.data_layout).bytes() != 8 || layout.size.bytes() != 16 { + return None; + } + Some(TaggedScalar { + payload, + tag_offset: layout.fields.offset(tag_field.into()).bytes_usize(), + payload_offset: variants[VariantIdx::from_usize(1)].field_offsets[FieldIdx::from_usize(0)] + .bytes_usize(), + }) +} + +pub(crate) fn tagged_value(dest: String, value: oomir::Operand, tag: u64) -> oomir::Instruction { + oomir::Instruction::TaggedPack { + dest, + value, + tag: oomir::Operand::Constant(oomir::Constant::U64(tag)), + } +} diff --git a/src/lower1/types/transmute.rs b/src/lower1/types/transmute.rs index a6ce1fce..3c661f03 100644 --- a/src/lower1/types/transmute.rs +++ b/src/lower1/types/transmute.rs @@ -109,6 +109,7 @@ pub(super) fn force_define_transmute_adts<'tcx>( } match ty.kind() { + TyKind::Adt(adt_def, _) if tcx.lang_items().phantom_data() == Some(adt_def.did()) => {} TyKind::Adt(adt_def, substs) => { if !should_define_named_data_type(tcx, adt_def.did()) && substs.is_empty() { force_define_named_adt(ty, tcx, data_types, instance_context); @@ -172,7 +173,6 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( let source_oomir_ty = ty_to_oomir_type(source_ty, tcx, data_types, instance_context); let target_oomir_ty = ty_to_oomir_type(target_ty, tcx, data_types, instance_context); - let method_name = "transmute".to_string(); let signature = oomir::Signature { params: source_oomir_ty .has_jvm_value() @@ -181,21 +181,6 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( ret: Box::new(target_oomir_ty.clone()), is_static: true, }; - let readable = format!( - "{}_to_{}", - sanitize_name_token(&readable_rust_type_name( - source_ty, - tcx, - data_types, - instance_context, - )), - sanitize_name_token(&readable_rust_type_name( - target_ty, - tcx, - data_types, - instance_context, - )) - ); let identity = format!( "{}:{}->{}:{}", stable_type_identity(tcx, source_ty), @@ -203,16 +188,13 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( stable_type_identity(tcx, target_ty), target_oomir_ty.to_jvm_descriptor() ); - // The same readable Rust types can use different JVM carriers across - // upstream and downstream monomorphizations. Include the descriptors in - // the class identity so their helper methods cannot collide at link time. - let local_name = crate::stable_hash::readable_disambiguated_name( - "ExactTransmute", - &readable, - &identity, - 180, + let identity = crate::stable_hash::short_hash(&identity, 16); + let method_name = format!("transmute${identity}"); + let class_name = format!( + "{}/mono/Mono_{}", + jvm_names::crate_root(tcx, instance_context.def_id().krate), + &identity[..2], ); - let class_name = jvm_names::synthetic_class_for_instance(tcx, instance_context, local_name); let helper = ExactTransmuteHelper { class_name: class_name.clone(), method_name: method_name.clone(), @@ -229,7 +211,10 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) => Some(*pointee), TyKind::Adt(adt_def, args) if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) - && matches!(oomir_ty, oomir::Type::Pointer(_)) => + && matches!( + oomir_ty, + oomir::Type::Pointer(_) | oomir::Type::Slice(_) | oomir::Type::Str + ) => { args.iter().find_map(|arg| arg.as_type()) } @@ -283,7 +268,10 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( } TyKind::Adt(adt_def, args) if crate::lower1::is_non_null_lang_item(tcx, adt_def.did()) - && matches!(target_oomir_ty, oomir::Type::Pointer(_)) => + && matches!( + target_oomir_ty, + oomir::Type::Pointer(_) | oomir::Type::Slice(_) | oomir::Type::Str + ) => { args.iter() .find_map(|arg| arg.as_type()) @@ -299,23 +287,6 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( .map(|(_, field_ty)| (field_ty.clone(), *pointee, Some(class_name.clone()))) }), }; - let target_struct_tail_view = target_pointee.and_then(|pointee| { - let tail = tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()); - let tail_view = if tail.is_str() { - Some(oomir::UTF8_VIEW_CLASS) - } else if matches!(tail.kind(), TyKind::Slice(_)) { - Some(oomir::SLICE_VIEW_CLASS) - } else { - None - }?; - let oomir::Type::Pointer(target) = &target_oomir_ty else { - return None; - }; - let oomir::Type::Class(target_class) = target.as_ref() else { - return None; - }; - Some((target_class.clone(), tail_view.to_string())) - }); let mut pointer_instructions = Vec::new(); let source_pointer = if source_pointee.is_some() && source_oomir_ty.has_jvm_value() { Some(( @@ -350,35 +321,54 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( |((source_pointer, source_pointer_ty), (target_pointer_ty, pointee, wrapper))| { let retyped = if source_pointer_pointee == Some(pointee) { source_pointer - } else if let Some((target_class, tail_view_class)) = &target_struct_tail_view { - if !matches!(source_pointer_ty, oomir::Type::Pointer(_)) - || !matches!(target_pointer_ty, oomir::Type::Pointer(_)) - { - return Ok(None); - } - let retyped_name = "_retargeted_struct_tail_pointer".to_string(); + } else if matches!(pointee.kind(), TyKind::Adt(def, _) if def.is_struct()) + && matches!( + tcx.struct_tail_for_codegen(pointee, TypingEnv::fully_monomorphized()) + .kind(), + TyKind::Dynamic(..) + ) + && matches!(source_pointer_ty, oomir::Type::Pointer(_)) + { + // MIR can express a pointer cast as a transmute. The DST adapter belongs at the + // new tail offset. + let name = "_trait_tail_pointer".to_string(); pointer_instructions.push(oomir::Instruction::InvokeStatic { - dest: Some(retyped_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retargetStructTail".to_string(), + dest: Some(name.clone()), + class_name: oomir::POINTER_CLASS.into(), + method_name: "retypeStructTailFromTraitPointer".into(), method_ty: oomir::Signature { params: vec![ - ("pointer".to_string(), source_pointer_ty), - ("target_class".to_string(), oomir::Type::java_string()), - ("tail_view_class".to_string(), oomir::Type::java_string()), + ("pointer".into(), source_pointer_ty), + ("prefix_size".into(), oomir::Type::U64), + ("codec".into(), oomir::Type::java_string()), ], ret: Box::new(target_pointer_ty.clone()), is_static: true, }, args: vec![ source_pointer, - oomir::Operand::Constant(oomir::Constant::String(target_class.clone())), - oomir::Operand::Constant(oomir::Constant::String( - tail_view_class.clone(), - )), + oomir::Operand::Constant(oomir::Constant::U64(layout_size_bytes( + tcx, pointee, + )? + as u64)), + pointer_view_codec_operand(pointee, tcx, data_types, instance_context), ], }); - operand_var(retyped_name, target_pointer_ty.clone()) + operand_var(name, target_pointer_ty.clone()) + } else if matches!(source_pointer_ty, oomir::Type::Pointer(_)) + && let Some(retyped) = + crate::lower1::control_flow::rvalue::emit_struct_tail_reborrow_view( + pointee, + source_pointer.clone(), + &target_pointer_ty, + "_retargeted_struct_tail_pointer", + tcx, + instance_context, + data_types, + &mut pointer_instructions, + ) + { + retyped } else if source_pointer_ty == target_pointer_ty && matches!(source_pointer_ty, oomir::Type::Pointer(_)) { @@ -387,27 +377,22 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( && matches!(target_pointer_ty, oomir::Type::Pointer(_)) { let retyped_name = "_retyped_pointer".to_string(); - pointer_instructions.push(oomir::Instruction::InvokeVirtual { + pointer_instructions.push(oomir::Instruction::AddressRetype { dest: Some(retyped_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_pointer_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(target_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( + source: source_pointer, + layout: Box::new(oomir::AddressLayout { + pointer_type: target_pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from(layout_size_bytes(tcx, pointee)?) .map_err(|_| "pointer transmute view exceeds u64")?, )), - pointer_view_codec_operand(pointee, tcx, data_types, instance_context), - ], - operand: source_pointer, + codec: pointer_view_codec_operand( + pointee, + tcx, + data_types, + instance_context, + ), + }), }); operand_var(retyped_name, target_pointer_ty.clone()) } else { @@ -450,6 +435,14 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( operand: Some(operand_var(result, target_oomir_ty.clone())), }, ] + } else if let Some(instructions) = super::bitcasts::scalar_wrapper_transmute( + source_ty, + target_ty, + tcx, + data_types, + instance_context, + ) { + instructions } else { let bytes_name = "_exact_bytes".to_string(); let objects_name = "_exact_objects".to_string(); @@ -537,6 +530,7 @@ pub(crate) fn ensure_exact_transmute_helper<'tcx>( class_name, oomir::DataType::Class { fields: Vec::new(), + kind: crate::oomir::ClassKind::Static, is_abstract: false, methods: HashMap::from_iter([( method_name, diff --git a/src/lower1/types/transparent.rs b/src/lower1/types/transparent.rs new file mode 100644 index 00000000..1acc6ceb --- /dev/null +++ b/src/lower1/types/transparent.rs @@ -0,0 +1,149 @@ +//! Private wrappers share payload storage when Rust layout and destruction permit it. +use super::*; + +#[derive(Clone, Copy)] +pub(crate) struct TransparentPayload<'tcx> { + pub ty: Ty<'tcx>, + pub field: usize, +} + +fn candidate<'tcx>( + ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + budget: usize, +) -> Option> { + if budget == 0 || ty.has_param() || ty.has_escaping_bound_vars() { + return None; + } + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if !def.is_struct() + || def.non_enum_variant().fields.len() > 4 + || tcx.is_lang_item(def.did(), rustc_attr_ir::lang_items::LangItem::UnsafeCell) + || adt_class_kind(tcx, def, args) != oomir::ClassKind::Value + { + return None; + } + let mut payload = None; + for (index, field) in def.non_enum_variant().fields.iter().enumerate() { + let mut ty = normalize_union_ty(tcx, field.ty(tcx, args).skip_norm_wip()).ok()?; + // NonNull slice and string metadata already uses view components. Other DSTs keep their + // existing carriers. + if crate::lower1::is_non_null_lang_item(tcx, def.did()) + && let TyKind::Pat(inner, _) = ty.kind() + && let TyKind::RawPtr(pointee, _) = inner.kind() + && matches!(pointee.kind(), TyKind::Slice(_) | TyKind::Str) + { + ty = *inner; + } + if layout_size_bytes(tcx, ty).ok()? == 0 { + // Omitted fields must not require destruction. Stateful allocators keep their complete + // schema. + if ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) { + return None; + } + } else if payload + .replace(TransparentPayload { ty, field: index }) + .is_some() + { + return None; + } + } + let payload = payload?; + match payload.ty.kind() { + TyKind::RawPtr(pointee, _) | TyKind::Ref(_, pointee, _) + if !matches!(pointee.kind(), TyKind::Foreign(_)) => {} + TyKind::Adt(..) if candidate(payload.ty, tcx, budget - 1).is_some() => {} + // Types that require destruction keep their owner until drop adapters can distinguish types + // with the same Java carrier. + TyKind::Adt(inner, args) + if inner.is_struct() + && adt_class_kind(tcx, inner, args) == oomir::ClassKind::Value + && !ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) => {} + TyKind::Array(..) | TyKind::Tuple(..) + if !ty.needs_drop(tcx, TypingEnv::fully_monomorphized()) => {} + TyKind::Bool | TyKind::Char | TyKind::Float(FloatTy::F32 | FloatTy::F64) => {} + _ => return None, + } + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .ok()?; + let inner = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(payload.ty)) + .ok()?; + (layout.size == inner.size + && layout.align == inner.align + && layout.fields.offset(payload.field).bytes() == 0) + .then_some(payload) +} + +pub(crate) fn transparent_payload<'tcx>( + ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, +) -> Option> { + if !matches!(ty.kind(), TyKind::Adt(..) | TyKind::Alias(..)) { + return None; + } + let ty = normalize_union_ty(tcx, ty).ok()?; + let payload = candidate(ty, tcx, 16)?; + finite_representation(ty, tcx).then_some(payload) +} + +pub(super) fn finite_representation<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool { + // Every recursive type path requires a nominal boundary. The depth and work limits keep large + // graphs nominal. + fn finite<'tcx>( + ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + path: &mut [Ty<'tcx>; 32], + depth: usize, + remaining: &mut usize, + ) -> bool { + if depth == path.len() || *remaining == 0 || path[..depth].contains(&ty) { + return false; + } + *remaining -= 1; + path[depth] = ty; + let mut follow = |next| finite(next, tcx, path, depth + 1, remaining); + match ty.kind() { + TyKind::Ref(_, inner, _) + | TyKind::RawPtr(inner, _) + | TyKind::Array(inner, _) + | TyKind::Slice(inner) + | TyKind::Pat(inner, _) => follow(*inner), + TyKind::Tuple(elements) => elements.iter().all(follow), + TyKind::Alias(..) => normalize_union_ty(tcx, ty).is_ok_and(follow), + TyKind::FnPtr(..) => { + let signature = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx)); + signature.inputs_and_output.iter().all(follow) + } + TyKind::Adt(def, args) + if crate::lower1::is_non_null_lang_item(tcx, def.did()) + && is_codegen_sized(args.type_at(0), tcx) => + { + follow(args.type_at(0)) + } + TyKind::Adt(def, args) if def.is_enum() => { + // enum_carrier queries transparent_payload recursively. This check must inspect the + // layout directly. + if tcx.is_lang_item(def.did(), rustc_attr_ir::lang_items::LangItem::Option) { + return follow(args.type_at(0)); + } + let Ok(layout) = tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + else { + return false; + }; + if let Variants::Single { index } = layout.variants + && let [field] = def.variant(index).fields.raw.as_slice() + { + follow(field.ty(tcx, args).skip_norm_wip()) + } else { + true + } + } + _ => candidate(ty, tcx, 16).is_none_or(|payload| follow(payload.ty)), + } + } + finite(ty, tcx, &mut [ty; 32], 0, &mut 128) +} diff --git a/src/lower1/types/tuples.rs b/src/lower1/types/tuples.rs new file mode 100644 index 00000000..cc32a596 --- /dev/null +++ b/src/lower1/types/tuples.rs @@ -0,0 +1,81 @@ +use super::*; +use crate::lower1::context::Definitions; +use crate::oomir::{Instruction, Operand, Type}; +use rustc_middle::ty::Instance; + +pub(crate) fn tuple_value<'tcx>( + ty: Ty<'tcx>, + args: Vec<(Operand, Type)>, + dest: &str, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: Instance<'tcx>, + code: &mut Vec, +) -> Operand { + let ty = definitions.normalize(tcx, ty, instance); + let jvm = ty_to_oomir_type(ty, tcx, definitions, instance); + if !jvm.has_jvm_value() { + return Operand::Constant(oomir::Constant::Unit); + } + if let Some(word) = packed_word(ty, tcx) { + let mut value = word.zero(); + for (index, (field, _)) in args.into_iter().enumerate() { + value = word.insert(value, index, field, &format!("{dest}_field_{index}"), code); + } + code.push(Instruction::Move { + dest: dest.into(), + src: value, + }); + } else { + code.push(Instruction::ConstructObject { + dest: dest.into(), + class_name: jvm.get_class_name().expect("tuple carrier").into(), + args, + }); + } + operand_var(dest, jvm) +} + +pub(crate) fn tuple_fields<'tcx>( + ty: Ty<'tcx>, + value: Operand, + dest: &str, + tcx: TyCtxt<'tcx>, + definitions: &mut Definitions<'tcx>, + instance: Instance<'tcx>, + code: &mut Vec, +) -> Vec { + let ty = definitions.normalize(tcx, ty, instance); + let TyKind::Tuple(fields) = ty.kind() else { + panic!("expected tuple, found {ty:?}") + }; + let word = packed_word(ty, tcx); + fields + .iter() + .enumerate() + .map(|(index, ty)| { + let jvm = ty_to_oomir_type(ty, tcx, definitions, instance); + if !jvm.has_jvm_value() { + return Operand::Constant(oomir::Constant::Unit); + } + let name = format!("{dest}_{index}"); + if let Some(word) = word { + word.extract(value.clone(), index, &jvm, &name, code) + } else { + code.push(Instruction::GetField { + dest: name.clone(), + object: value.clone(), + field_name: format!("field{index}"), + field_ty: jvm.clone(), + owner_class: value + .get_type() + .unwrap() + .get_class_name() + .expect("tuple carrier") + .into(), + }); + operand_var(name, jvm) + } + }) + .collect() +} diff --git a/src/lower1/types/unions.rs b/src/lower1/types/unions.rs index 62befe32..42fc1e93 100644 --- a/src/lower1/types/unions.rs +++ b/src/lower1/types/unions.rs @@ -300,6 +300,7 @@ pub(crate) fn ensure_union_data_type<'tcx>( (UNION_BYTES_FIELD.to_string(), byte_array_type()), (UNION_OBJECTS_FIELD.to_string(), object_array_type()), ], + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods: HashMap::default(), super_class: Some("java/lang/Object".to_string()), @@ -405,6 +406,7 @@ pub(crate) fn ensure_union_data_type<'tcx>( union_class.clone(), oomir::DataType::Class { fields: union_fields, + kind: crate::oomir::ClassKind::Value, is_abstract: false, methods, super_class: Some("java/lang/Object".to_string()), diff --git a/src/lower1/types/write.rs b/src/lower1/types/write.rs index 9bb35e67..6fb9eed3 100644 --- a/src/lower1/types/write.rs +++ b/src/lower1/types/write.rs @@ -13,6 +13,56 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( temp_counter: &mut usize, ) -> Result<(), String> { let ty = resolve_union_ty(tcx, ty, instance_context)?; + if let Some(layout) = tagged_scalar(ty, tcx) { + for (index, offset) in [(0, layout.payload_offset), (1, layout.tag_offset)] { + let dest = next_union_temp("tagged_part", temp_counter); + instructions.push(oomir::Instruction::TaggedPart { + dest: dest.clone(), + value: source.clone(), + index, + }); + emit_ty_to_union_bytes( + tcx.types.i64, + operand_var(dest, oomir::Type::I64), + storage, + base_offset + offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + )?; + } + return Ok(()); + } + if let Some(scalar) = value_scalar_ty(ty, tcx) { + return emit_ty_to_union_bytes( + scalar, + source, + storage, + base_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + ); + } + if let Some(payload) = + direct_enum_payload(ty, tcx).or_else(|| transparent_payload(ty, tcx).map(|p| p.ty)) + { + return emit_ty_to_union_bytes( + payload, + source, + storage, + base_offset, + tcx, + data_types, + instance_context, + instructions, + temp_counter, + ); + } if layout_size_bytes(tcx, ty)? == 0 { return Ok(()); } @@ -95,8 +145,8 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( let value_ty = ty_to_oomir_type(ty, tcx, data_types, instance_context); instructions.push(oomir::Instruction::InvokeStatic { dest: Some(encoded.clone()), - class_name: codec.class_name, - method_name: "encode".to_string(), + class_name: codec.owner().to_owned(), + method_name: codec.method("encode"), method_ty: oomir::Signature { params: vec![("value".to_string(), value_ty)], ret: Box::new(byte_array_type()), @@ -141,37 +191,24 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( // SliceView. Transmutes (notably fmt::Arguments::new) must encode the // data address, not the managed identity of the SliceView wrapper. let element_size = layout_size_bytes(tcx, *element)?; - let pointer_ty = oomir::Type::Pointer(Box::new(ty_to_oomir_type( + let pointer_ty = oomir::Type::pointer(ty_to_oomir_type( *element, tcx, data_types, instance_context, - ))); + )); let pointer_dest = next_union_temp("union_array_reference_pointer", temp_counter); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::ViewAddress { dest: Some(pointer_dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(pointer_ty.clone()), - is_static: true, - }, - args: vec![ - source, - oomir::Operand::Constant(oomir::Constant::U64( + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from(element_size) .map_err(|_| "Rust array element layout exceeds u64")?, )), - pointer_view_codec_operand(*element, tcx, data_types, instance_context), - ], + codec: pointer_view_codec_operand(*element, tcx, data_types, instance_context), + }), }); let address_dest = next_union_temp("union_array_reference_address", temp_counter); let pointer_codec = @@ -200,12 +237,12 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( args: vec![ operand_var( pointer_dest, - oomir::Type::Pointer(Box::new(ty_to_oomir_type( + oomir::Type::pointer(ty_to_oomir_type( *element, tcx, data_types, instance_context, - ))), + )), ), operand_var(storage.bytes_var.clone(), byte_array_type()), storage_offset, @@ -366,7 +403,7 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( let oomir::Type::Class(enum_class) = &enum_oomir_ty else { return Err(format!("enum {ty:?} did not map to a JVM class")); }; - ensure_enum_union_codec( + let codec = ensure_enum_union_codec( adt_def, substs, ty, @@ -376,17 +413,17 @@ pub(super) fn emit_ty_to_union_bytes<'tcx>( instance_context, )?; let offset = storage.byte_index(base_offset, instructions, temp_counter); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::InvokeStatic { dest: None, - class_name: enum_class.clone(), - method_name: enum_scoped_method_name(enum_class, ENUM_WRITE_UNION_STORAGE_METHOD), + class_name: codec.owner, + method_name: codec.writer, method_ty: enum_union_write_signature(enum_class), args: vec![ + source, operand_var(storage.bytes_var.clone(), byte_array_type()), operand_var(storage.objects_var.clone(), object_array_type()), offset, ], - operand: source, }); Ok(()) } diff --git a/src/lower1/value_repr.rs b/src/lower1/value_repr.rs index 503c7112..8c9911ba 100644 --- a/src/lower1/value_repr.rs +++ b/src/lower1/value_repr.rs @@ -5,12 +5,9 @@ use rustc_middle::ty::{EarlyBinder, Instance, Ty, TyCtxt, TyKind, TypingEnv}; use crate::oomir; -use super::{ - jvm_names, - types::{ - adapt_simple_enum_operand, ensure_exact_transmute_helper, ensure_union_data_type, - generate_adt_jvm_class_name, generate_tuple_jvm_class_name, ty_to_oomir_type, - }, +use super::types::{ + adapt_simple_enum_operand, ensure_exact_transmute_helper, ensure_union_data_type, + generate_adt_jvm_class_name, generate_tuple_jvm_class_name, ty_to_oomir_type, }; enum RustValueRepresentation { @@ -54,7 +51,11 @@ fn representation<'tcx>( let TyKind::Adt(adt_def, substs) = rust_ty.kind() else { return RustValueRepresentation::Direct; }; - if !adt_def.is_enum() { + if matches!(jvm_ty, oomir::Type::TaggedI64) + || !adt_def.is_enum() + || super::types::direct_enum_payload(rust_ty, tcx).is_some() + || super::types::enum_scalar_ty(rust_ty, tcx).is_some() + { return RustValueRepresentation::Direct; } @@ -185,6 +186,27 @@ pub(super) fn construct_fieldless_enum_variant<'tcx>( return None; } let variant_def = adt_def.variant(variant); + if let Some(value) = super::types::enum_scalar_variant(enum_ty, variant, tcx) { + return Some(oomir::Operand::Constant(value)); + } + if super::types::tagged_scalar(enum_ty, tcx).is_some() { + return (variant.as_u32() == 0).then(|| { + let dest = format!("{temp_prefix}_none"); + instructions.push(super::types::tagged_value( + dest.clone(), + oomir::Operand::Constant(oomir::Constant::I64(0)), + 0, + )); + operand_var(dest, oomir::Type::TaggedI64) + }); + } + if let Some(carrier) = super::types::enum_carrier(enum_ty, tcx) { + return (carrier.nullable && variant.as_u32() == 0).then(|| { + oomir::Operand::Constant(oomir::Constant::Null(ty_to_oomir_type( + enum_ty, tcx, data_types, instance, + ))) + }); + } if variant_def.fields.iter().any(|field| { let field_ty = resolved_ty(field.ty(tcx, substs).skip_norm_wip(), tcx, instance); ty_to_oomir_type(field_ty, tcx, data_types, instance).has_jvm_value() @@ -196,7 +218,7 @@ pub(super) fn construct_fieldless_enum_variant<'tcx>( let variant_class = format!( "{}${}", base_class, - jvm_names::member_name(&variant_def.name.to_string()) + crate::lower1::types::enum_variant_name(variant_def, tcx) ); let dest = format!("{temp_prefix}_enum_value"); instructions.push(oomir::Instruction::ConstructObject { @@ -222,6 +244,19 @@ pub(super) fn materialize_implicit_zst<'tcx>( if layout.size.bytes() != 0 { return None; } + if !ty_to_oomir_type(rust_ty, tcx, data_types, instance).has_jvm_value() { + return None; + } + if let Some(payload) = super::types::direct_enum_payload(rust_ty, tcx) { + return materialize_implicit_zst( + payload, + temp_prefix, + tcx, + instance, + data_types, + instructions, + ); + } match rust_ty.kind() { TyKind::Adt(adt_def, substs) if adt_def.is_enum() => { @@ -234,7 +269,7 @@ pub(super) fn materialize_implicit_zst<'tcx>( let variant_class = format!( "{}${}", base_class, - jvm_names::member_name(&variant.name.to_string()) + crate::lower1::types::enum_variant_name(variant, tcx) ); let mut args = Vec::new(); for (field_index, field) in variant.fields.iter().enumerate() { @@ -494,6 +529,15 @@ fn adapt_physical_scalar<'tcx>( if !target_jvm_ty.is_jvm_reference_type() { return None; } + let dest = format!("{temp_prefix}_layout_value"); + if let Some(value) = crate::lower1::types::emit_direct_transmute( + source.clone(), + &target_jvm_ty, + &dest, + instructions, + ) { + return Some(value); + } let target_layout = tcx .layout_of(TypingEnv::fully_monomorphized().as_query_input(target_rust_ty)) .ok()?; @@ -502,7 +546,6 @@ fn adapt_physical_scalar<'tcx>( let helper = ensure_exact_transmute_helper(source_rust_ty, target_rust_ty, tcx, data_types, instance) .ok()?; - let dest = format!("{temp_prefix}_layout_value"); instructions.push(oomir::Instruction::InvokeStatic { dest: Some(dest.clone()), class_name: helper.class_name, @@ -606,7 +649,7 @@ pub(super) fn adapt_operand_to_rust_type<'tcx>( ) -> oomir::Operand { let target_rust_ty = resolved_ty(target_rust_ty, tcx, instance); let target_jvm_ty = ty_to_oomir_type(target_rust_ty, tcx, data_types, instance); - let source = adapt_mutable_reference_carrier( + let source = adapt_reference_carrier( source, target_rust_ty, &target_jvm_ty, @@ -744,7 +787,7 @@ pub(super) fn adapt_operand_to_rust_type<'tcx>( let variant_class = format!( "{}${}", base_class, - jvm_names::member_name(&variant_def.name.to_string()) + crate::lower1::types::enum_variant_name(variant_def, tcx) ); let dest = format!("{temp_prefix}_enum_value"); instructions.push(oomir::Instruction::ConstructObject { diff --git a/src/lower1/value_repr/pointers.rs b/src/lower1/value_repr/pointers.rs index abd849eb..bfa1fd04 100644 --- a/src/lower1/value_repr/pointers.rs +++ b/src/lower1/value_repr/pointers.rs @@ -1,23 +1,9 @@ //! Pointer and mutable-carrier representation boundaries. use super::*; -pub(super) fn mutable_reference_chain_contains(source: &oomir::Type, target: &oomir::Type) -> bool { - let mut current = source; - while let oomir::Type::MutableReference(inner) = current { - if inner.as_ref() == target { - return true; - } - current = inner; - } - false -} - pub(super) fn reference_chain_contains(outer: &oomir::Type, candidate: &oomir::Type) -> bool { let mut current = outer; - while let oomir::Type::MutableReference(inner) - | oomir::Type::Reference(inner) - | oomir::Type::Pointer(inner) = current - { + while let oomir::Type::Pointer(oomir::Pointee { value: inner, .. }) = current { if inner.as_ref() == candidate { return true; } @@ -26,32 +12,7 @@ pub(super) fn reference_chain_contains(outer: &oomir::Type, candidate: &oomir::T false } -pub(super) fn unwrap_mutable_references( - mut source: oomir::Operand, - target_jvm_ty: &oomir::Type, - temp_prefix: &str, - instructions: &mut Vec, -) -> oomir::Operand { - let mut depth = 0; - loop { - let Some(oomir::Type::MutableReference(inner)) = source.get_type() else { - return source; - }; - let dest = format!("{temp_prefix}_deref_{depth}"); - instructions.push(oomir::Instruction::ArrayGet { - dest: dest.clone(), - array: source, - index: oomir::Operand::Constant(oomir::Constant::I32(0)), - }); - source = operand_var(dest, inner.as_ref().clone()); - if inner.as_ref() == target_jvm_ty { - return source; - } - depth += 1; - } -} - -pub(super) fn adapt_mutable_reference_carrier<'tcx>( +pub(super) fn adapt_reference_carrier<'tcx>( source: oomir::Operand, target_rust_ty: Ty<'tcx>, target_jvm_ty: &oomir::Type, @@ -65,10 +26,8 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( return source; } if source.get_type().as_ref().is_some_and(|source_ty| { - matches!( - source_ty, - oomir::Type::MutableReference(_) | oomir::Type::Reference(_) | oomir::Type::Pointer(_) - ) && reference_chain_contains(target_jvm_ty, source_ty) + matches!(source_ty, oomir::Type::Pointer(_)) + && reference_chain_contains(target_jvm_ty, source_ty) }) { return source; } @@ -82,23 +41,14 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( let source_ty = source .get_type() .expect("fixed-array reference pointer is typed"); - let element_pointer_ty = oomir::Type::Pointer(element_ty.clone()); + let element_pointer_ty = oomir::Type::pointer(*element_ty.clone()); let pointer_name = format!("{temp_prefix}_array_ref_element_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(pointer_name.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(element_pointer_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: element_pointer_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( super::super::types::layout_size_bytes( tcx, @@ -108,70 +58,32 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( ) .expect("Rust array element layout exceeds u64"), )), - super::super::types::pointer_view_codec_operand( + codec: super::super::types::pointer_view_codec_operand( resolved_ty(*array_element_ty, tcx, instance), tcx, data_types, instance, ), - ], - operand: source, + }), }); - let object_name = format!("{temp_prefix}_array_ref_view_object"); - instructions.push(oomir::Instruction::ConstructObject { - dest: object_name.clone(), - class_name: oomir::SLICE_VIEW_CLASS.to_string(), - args: vec![ - ( - operand_var(pointer_name, element_pointer_ty), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ( - oomir::Operand::Constant(oomir::Constant::I32(0)), - oomir::Type::I32, - ), - ( - oomir::Operand::Constant(oomir::Constant::U64(length)), - oomir::Type::U64, - ), - ], - }); - let dest = format!("{temp_prefix}_array_ref_view"); - let slice_ty = oomir::Type::Slice(element_ty.clone()); - instructions.push(oomir::Instruction::Cast { - dest: dest.clone(), - op: operand_var( - object_name, - oomir::Type::Class(oomir::SLICE_VIEW_CLASS.to_string()), - ), - ty: slice_ty.clone(), - }); - return operand_var(dest, slice_ty); + return crate::lower1::place::emit_pointer_slice_view( + operand_var(pointer_name, element_pointer_ty), + oomir::Operand::Constant(oomir::Constant::U64(length)), + &format!("{temp_prefix}_array_ref_view"), + instructions, + ); } if matches!(source.get_type(), Some(oomir::Type::Slice(_))) && matches!(target_jvm_ty, oomir::Type::Pointer(_)) { let dest = format!("{temp_prefix}_slice_pointer"); - instructions.push(oomir::Instruction::InvokeStatic { + instructions.push(oomir::Instruction::ViewAddress { dest: Some(dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "fromSlice".to_string(), - method_ty: oomir::Signature { - params: vec![ - ( - "slice".to_string(), - oomir::Type::Class("java/lang/Object".to_string()), - ), - ("element_size".to_string(), oomir::Type::U64), - ("codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(target_jvm_ty.clone()), - is_static: true, - }, - args: vec![ - source, - oomir::Operand::Constant(oomir::Constant::U64( + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: target_jvm_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( match target_rust_ty.kind() { TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _) => { @@ -188,7 +100,7 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( ) .expect("Rust slice pointer layout exceeds u64"), )), - match target_rust_ty.kind() { + codec: match target_rust_ty.kind() { TyKind::Ref(_, pointee, _) | TyKind::RawPtr(pointee, _) => { super::super::types::pointer_view_codec_operand( resolved_ty(*pointee, tcx, instance), @@ -199,11 +111,24 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( } _ => unreachable!("pointer carrier target must be a pointer type"), }, - ], + }), }); return operand_var(dest, target_jvm_ty.clone()); } + if matches!(source.get_type(), Some(oomir::Type::Pointer(_))) + && let TyKind::Ref(_, pointee, _) = target_rust_ty.kind() + && matches!(pointee.kind(), TyKind::Dynamic(..)) + { + // Raw dyn pointers contain an address and vtable. References use a JVM trait adapter. + return crate::lower1::place::emit_pointer_read( + source, + target_jvm_ty, + &format!("{temp_prefix}_trait_reference"), + instructions, + ); + } + let target_pointer_ty = match target_rust_ty.kind() { TyKind::Pat(inner, _) => resolved_ty(*inner, tcx, instance), _ => target_rust_ty, @@ -232,21 +157,12 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( let pointee_ty = resolved_ty(*pointee_ty, tcx, instance); let source_ty = oomir::Type::Pointer(source_inner); let dest = format!("{temp_prefix}_retyped_pointer"); - instructions.push(oomir::Instruction::InvokeVirtual { + instructions.push(oomir::Instruction::AddressRetype { dest: Some(dest.clone()), - class_name: oomir::POINTER_CLASS.to_string(), - method_name: "retype".to_string(), - method_ty: oomir::Signature { - params: vec![ - ("self".to_string(), source_ty), - ("view_size".to_string(), oomir::Type::U64), - ("view_codec".to_string(), oomir::Type::java_string()), - ], - ret: Box::new(target_jvm_ty.clone()), - is_static: false, - }, - args: vec![ - oomir::Operand::Constant(oomir::Constant::U64( + source: source, + layout: Box::new(oomir::AddressLayout { + pointer_type: target_jvm_ty.clone(), + size: oomir::Operand::Constant(oomir::Constant::U64( u64::try_from( super::super::types::layout_size_bytes(tcx, pointee_ty).unwrap_or_else( |error| panic!("could not determine pointer view layout: {error}"), @@ -254,11 +170,10 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( ) .expect("pointer view layout exceeds u64"), )), - super::super::types::pointer_view_codec_operand( + codec: super::super::types::pointer_view_codec_operand( pointee_ty, tcx, data_types, instance, ), - ], - operand: source, + }), }); return operand_var(dest, target_jvm_ty.clone()); } @@ -338,7 +253,7 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( && tcx .layout_of(TypingEnv::fully_monomorphized().as_query_input(target_rust_ty)) .is_ok_and(|layout| matches!(layout.backend_repr, BackendRepr::Scalar(_))); - if let Some(oomir::Type::Pointer(inner)) = source.get_type() + if let Some(oomir::Type::Pointer(oomir::Pointee { value: inner, .. })) = source.get_type() && !matches!(target_rust_ty.kind(), TyKind::RawPtr(..) | TyKind::Ref(..)) && !target_is_scalar_adt { @@ -350,55 +265,5 @@ pub(super) fn adapt_mutable_reference_carrier<'tcx>( ); } - if source - .get_type() - .as_ref() - .is_some_and(|source_ty| mutable_reference_chain_contains(source_ty, target_jvm_ty)) - { - return unwrap_mutable_references(source, target_jvm_ty, temp_prefix, instructions); - } - - if let TyKind::Ref(_, pointee_ty, mutability) = target_rust_ty.kind() - && mutability.is_mut() - && let oomir::Type::MutableReference(target_inner) = target_jvm_ty - && target_inner.has_jvm_value() - { - let pointee = adapt_operand_to_rust_type( - source, - *pointee_ty, - &format!("{temp_prefix}_pointee"), - tcx, - instance, - data_types, - instructions, - ); - if pointee.get_type().as_ref() != Some(target_inner.as_ref()) { - return pointee; - } - - let dest = format!("{temp_prefix}_ref"); - instructions.push(oomir::Instruction::NewArray { - dest: dest.clone(), - element_type: target_inner.as_ref().clone(), - size: oomir::Operand::Constant(oomir::Constant::I32(1)), - }); - instructions.push(oomir::Instruction::ArrayStore { - array: oomir::Operand::Variable { - name: dest.clone(), - ty: target_jvm_ty.clone(), - }, - index: oomir::Operand::Constant(oomir::Constant::I32(0)), - value: pointee, - copy_value: false, - }); - return operand_var(dest, target_jvm_ty.clone()); - } - - if matches!(source.get_type(), Some(oomir::Type::MutableReference(_))) - && !matches!(target_rust_ty.kind(), TyKind::RawPtr(..)) - { - return unwrap_mutable_references(source, target_jvm_ty, temp_prefix, instructions); - } - source } diff --git a/src/lower2.rs b/src/lower2.rs index eba971ca..bdffe37e 100644 --- a/src/lower2.rs +++ b/src/lower2.rs @@ -4,8 +4,7 @@ use crate::oomir::{self, DataType}; use jvm_compiler_core::jvm::MethodCode; use jvm_gen::{ create_data_type_classfile_for_class, create_data_type_classfile_for_interface, - create_default_constructor, create_slice_view_classfile, create_utf8_view_classfile, - oomir_type_to_ristretto_field_type, + create_default_constructor, oomir_type_to_ristretto_field_type, }; #[derive(Clone, Copy)] pub(crate) struct DebugInfoOptions { @@ -24,7 +23,6 @@ use rustc_middle::ty::TyCtxt; use std::{ collections::{BTreeMap, BTreeSet}, path::PathBuf, - sync::Arc, }; use jvm::constant_pool; @@ -49,7 +47,6 @@ pub const U128_CLASS: &str = "org/rustlang/runtime/U128"; pub fn oomir_to_jvm_bytecode( mut module: oomir::Module, debug_info: DebugInfoOptions, - emit_runtime_views: bool, registry: &EmittedClassRegistry, ) -> jvm::Result> { let context = if let Some(shared) = &module.shared_context { @@ -58,61 +55,7 @@ pub fn oomir_to_jvm_bytecode( std::borrow::Cow::Owned(oomir::construct::Context::new(&module)) }; - let function_relative_methods = module - .functions - .values() - .filter_map(|function| { - (function.signature.is_static - && function.name != "" - && function.signature.supports_relative_pointer_abi()) - .then(|| { - oomir::FunctionKey::new( - module.owner_class_for_function(function), - &function.name, - &function.signature, - ) - }) - }) - .collect::>(); - for method in function_relative_methods { - if !module.relative_static_methods.contains(&method) { - Arc::make_mut(&mut module.relative_static_methods).insert(method); - } - } - for (class_name, data_type) in &module.data_types { - let oomir::DataType::Class { methods, .. } = data_type else { - continue; - }; - for (method_name, method) in methods { - let Some(signature) = method.function_signature() else { - continue; - }; - if signature.is_static - && method_name != "" - && signature.supports_relative_pointer_abi() - { - let method = oomir::FunctionKey::new(class_name, method_name, signature); - if !module.relative_static_methods.contains(&method) { - Arc::make_mut(&mut module.relative_static_methods).insert(method); - } - } - } - } let mut output = output::ClassOutput::create(&module.name)?; - if emit_runtime_views { - output.emit( - registry, - oomir::SLICE_VIEW_CLASS.to_string(), - create_slice_view_classfile()?, - crate::metrics::ClassOrigin::Runtime, - )?; - output.emit( - registry, - oomir::UTF8_VIEW_CLASS.to_string(), - create_utf8_view_classfile()?, - crate::metrics::ClassOrigin::Runtime, - )?; - } // Consume functions class-by-class so their OOMIR is released as soon as // the corresponding classfile has been serialized. @@ -172,12 +115,16 @@ pub fn oomir_to_jvm_bytecode( .flatten() .or_else(|| source_files.first().map(|file| (*file).to_string())); let mut class_statics = statics_by_class.remove(&class_name_jvm).unwrap_or_default(); + let private_statics = !class_statics.is_empty() + && functions.is_empty() + && class_statics.iter().all(|value| value.is_private); class_statics.sort_by(|left, right| left.field_name.cmp(&right.field_name)); let mut main_cp = InternedConstantPool::default(); let super_class_name_jvm = "java/lang/Object"; // Standard superclass let super_class_index = main_cp.add_class(super_class_name_jvm)?; let this_class_index = main_cp.add_class(&class_name_jvm)?; + main_cp.set_resource_anchor(this_class_index); let mut methods: Vec = Vec::new(); let mut bootstrap_methods: Vec = Vec::new(); @@ -219,7 +166,6 @@ pub fn oomir_to_jvm_bytecode( next_factory: &mut next_factory, owner: &class_name_jvm, kind: jvm_gen::BodyOwner::Module, - relative_methods: &module.relative_static_methods, debug: debug_info, context: &context, } @@ -227,7 +173,7 @@ pub fn oomir_to_jvm_bytecode( } // Add a default constructor if none was provided in OOMIR - if !has_constructor { + if !has_constructor && !class_name_jvm.contains("/mono/Mono_") { methods.push(create_default_constructor(&mut main_cp, super_class_index)?); } @@ -240,6 +186,12 @@ pub fn oomir_to_jvm_bytecode( } else { Vec::new() }; + if (class_name_jvm.contains("/mono/Mono_") && class_statics.is_empty()) || private_statics { + attributes.push(Attribute::Unknown { + name_index: main_cp.add_utf8(jvm::summary::PRIVATE_ATTRIBUTE)?, + info: Vec::new(), + }); + } if !bootstrap_methods.is_empty() { attributes.push(Attribute::BootstrapMethods { name_index: main_cp.add_utf8("BootstrapMethods")?, @@ -247,6 +199,7 @@ pub fn oomir_to_jvm_bytecode( }); } + output.resources(registry, &mut main_cp)?; let class_file = ClassFile { code_source_url: None, version: Version::Java8 { minor: 0 }, @@ -339,6 +292,7 @@ pub fn oomir_to_jvm_bytecode( match data_type { DataType::Class { + kind, is_abstract, super_class, fields, @@ -351,6 +305,7 @@ pub fn oomir_to_jvm_bytecode( let dt_bytecode = create_data_type_classfile_for_class( dt_name_oomir, fields, + *kind, *is_abstract, methods, super_class.as_deref().unwrap_or("java/lang/Object"), @@ -359,8 +314,9 @@ pub fn oomir_to_jvm_bytecode( &subclasses, nest_host.as_deref(), debug_info, - &module.relative_static_methods, &context, + &mut output, + registry, )?; output.emit( registry, @@ -385,8 +341,9 @@ pub fn oomir_to_jvm_bytecode( subclasses, nest_host, debug_info, - &module.relative_static_methods, &context, + &mut output, + registry, )?; output.emit( registry, diff --git a/src/lower2/constants/arrays.rs b/src/lower2/constants/arrays.rs index 6060c6c1..fa79eb25 100644 --- a/src/lower2/constants/arrays.rs +++ b/src/lower2/constants/arrays.rs @@ -61,6 +61,10 @@ pub(super) fn load_array( element_type: &oomir::Type, elements: &[oomir::Constant], ) -> jvm::Result<()> { + if super::resources::eligible(cp, element_type, elements) { + append_empty_array(instructions, cp, element_type, elements.len())?; + return super::resources::fill(instructions, cp, element_type, elements); + } if oomir::is_packed_byte_array(element_type, elements) { append_empty_array(instructions, cp, element_type, elements.len())?; return fill_packed_bytes(instructions, cp, 0, elements.len(), |index| { @@ -95,6 +99,9 @@ pub(super) fn load_bytes( ) -> jvm::Result<()> { if bytes.len() >= 32 { append_empty_array(instructions, cp, &oomir::Type::U8, bytes.len())?; + if super::resources::fill_bytes(instructions, cp, bytes)? { + return Ok(()); + } return fill_packed_bytes(instructions, cp, 0, bytes.len(), |index| bytes[index]); } load_array_with( diff --git a/src/lower2/constants/encoding.rs b/src/lower2/constants/encoding.rs index d5f6ca7e..438be23b 100644 --- a/src/lower2/constants/encoding.rs +++ b/src/lower2/constants/encoding.rs @@ -36,6 +36,42 @@ pub fn load_constant( instructions.push(JI::Dup); instructions.push(JI::Invokespecial(constructor)); } + OC::FunctionHandle { + interface_name, + owner, + name, + descriptor, + interface, + .. + } => { + let signature = cp.add_class(interface_name)?; + let runtime = cp.add_class("org/rustlang/runtime/FunctionPointers")?; + let handles = cp.add_class("java/lang/invoke/MethodHandles")?; + let lookup = cp.add_method_ref( + handles, + "lookup", + "()Ljava/lang/invoke/MethodHandles$Lookup;", + )?; + let bind = cp.add_method_ref( + runtime, + "bind", + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/Class;Ljava/lang/invoke/MethodHandle;)Ljava/lang/Object;", + )?; + let owner = cp.add_class(owner)?; + let method = if *interface { + cp.add_interface_method_ref(owner, name, descriptor)? + } else { + cp.add_method_ref(owner, name, descriptor)? + }; + let handle = cp.add_method_handle(jvm::ReferenceKind::InvokeStatic, method)?; + instructions.extend([ + JI::Invokestatic(lookup), + JI::Ldc_w(signature), + JI::Ldc_w(handle), + JI::Invokestatic(bind), + JI::Checkcast(signature), + ]); + } OC::FactoryCall { owner_class, method_name, @@ -254,6 +290,35 @@ pub fn load_constant( OC::F64(v) => instructions.push(get_double_const_instr(cp, *v)), OC::Boolean(v) => instructions.push(if *v { JI::Iconst_1 } else { JI::Iconst_0 }), OC::Char(v) => instructions.push(get_int_const_instr(cp, *v as i32)), + OC::Str(s) if !jvm::names::literal_fits(s) => { + // The JVM UTF-8 limit includes linker tags. Large Rust strings require byte resources. + let string = cp.add_class("java/lang/String")?; + instructions.extend([JI::New(string), JI::Dup]); + super::arrays::load_bytes(instructions, cp, s.as_bytes())?; + let charsets = cp.add_class("java/nio/charset/StandardCharsets")?; + instructions.push(JI::Getstatic(cp.add_field_ref( + charsets, + "UTF_8", + "Ljava/nio/charset/Charset;", + )?)); + instructions.push(JI::Invokespecial(cp.add_method_ref( + string, + "", + "([BLjava/nio/charset/Charset;)V", + )?)); + // Repeated loads must share the same backing allocation as ldc strings. + instructions.push(JI::Invokevirtual(cp.add_method_ref( + string, + "intern", + "()Ljava/lang/String;", + )?)); + let view = cp.add_class(oomir::UTF8_VIEW_CLASS)?; + instructions.push(JI::Invokestatic(cp.add_method_ref( + view, + "fromJavaString", + "(Ljava/lang/String;)Lorg/rustlang/runtime/Utf8View;", + )?)); + } OC::Str(s) => { let index = cp.add_string(s)?; instructions.push(if let Ok(idx8) = u8::try_from(index) { @@ -266,8 +331,12 @@ pub fn load_constant( let from_java = cp.add_method_ref(view_class, "fromJavaString", descriptor)?; instructions.push(JI::Invokestatic(from_java)); } - OC::String(s) => { - let index = cp.add_name_string(s)?; + OC::String(s) | OC::LiteralString(s) => { + let index = if matches!(constant, OC::LiteralString(_)) { + cp.add_string(s)? + } else { + cp.add_name_string(s)? + }; instructions.push(if let Ok(idx8) = u8::try_from(index) { JI::Ldc(idx8) } else { diff --git a/src/lower2/constants/factories.rs b/src/lower2/constants/factories.rs index 9392b3a6..bca1e659 100644 --- a/src/lower2/constants/factories.rs +++ b/src/lower2/constants/factories.rs @@ -2,7 +2,13 @@ use super::*; use std::borrow::Cow; -fn add_constant_helper_method( +/// The owner distinguishes equal constants when the linker combines private method holders. +pub(super) fn helper_name(kind: char, owner: &str, identity: &str, next: usize) -> String { + let identity = crate::stable_hash::short_hash_value(&(owner, identity), 16); + format!("$c{kind}{identity}_{next:x}") +} + +pub(super) fn add_constant_helper_method( cp: &mut InternedConstantPool, methods: &mut Vec, method_name: &str, @@ -10,27 +16,15 @@ fn add_constant_helper_method( max_locals: u16, instructions: Vec, ) -> jvm::Result<()> { - let initial = jvm_compiler_core::jvm::frames::initial_locals_for_descriptor( - descriptor, true, None, false, - )?; - let max_stack = jvm_compiler_core::jvm::frames::analyze( - &instructions, - &initial, - &[], - usize::from(max_locals), + let code = super::super::jvm_gen::code_attribute_for_descriptor( cp, - method_name, - &[], - )? - .max_stack; - let code = Attribute::Code { - name_index: cp.add_utf8("Code")?, - max_stack, max_locals, - code: instructions, - exception_table: Vec::new(), - attributes: Vec::new(), - }; + instructions, + descriptor, + true, + None, + method_name, + )?; methods.push(jvm::Method { access_flags: MethodAccessFlags::PRIVATE | MethodAccessFlags::STATIC @@ -51,6 +45,60 @@ pub(super) fn create_chunked_array_factory( next_factory: &mut usize, storage_field: Option, ) -> jvm::Result { + if super::resources::eligible(cp, element_type, elements) { + let array_type = oomir::Type::Array(Box::new(element_type.clone())); + let mut code = Vec::new(); + if let Some(field) = storage_field { + code.push(Instruction::Getstatic(field)); + } else { + append_empty_array(&mut code, cp, element_type, elements.len())?; + } + super::resources::fill(&mut code, cp, element_type, elements)?; + code.push(Instruction::Areturn); + let storage_name = storage_field.map(|field| { + let jvm::Constant::FieldRef { + name_and_type_index, + .. + } = cp.try_get(field).unwrap() + else { + unreachable!("static storage field") + }; + let jvm::Constant::NameAndType { name_index, .. } = + cp.try_get(*name_and_type_index).unwrap() + else { + unreachable!("static storage name") + }; + cp.try_get_utf8(*name_index).unwrap().to_string() + }); + let identity = + crate::stable_hash::short_hash_value(&(element_type, elements, storage_name), 16); + let name = helper_name('d', owner_class, &identity, *next_factory); + *next_factory += 1; + add_constant_helper_method( + cp, + methods, + &name, + &format!("(){}", array_type.to_jvm_descriptor()), + 0, + code, + )?; + return Ok(oomir::Constant::FactoryCall { + owner_class: owner_class.into(), + method_name: name, + ty: array_type, + }); + } + if let Some(factory) = super::structured::factory( + cp, + owner_class, + element_type, + elements, + methods, + next_factory, + storage_field, + )? { + return Ok(factory); + } if oomir::is_packed_byte_array(element_type, elements) { return create_byte_array_factory( cp, @@ -94,7 +142,7 @@ pub(super) fn create_chunked_array_factory( end += 1; } - let method_name = format!("_constant_fill_{identity}_{}", *next_factory); + let method_name = helper_name('x', owner_class, &identity, *next_factory); *next_factory += 1; let mut instructions = Vec::new(); for (index, element) in prepared[start..end].iter().enumerate() { @@ -136,7 +184,7 @@ pub(super) fn create_chunked_array_factory( } } - let method_name = format!("_constant_factory_{identity}_{}", *next_factory); + let method_name = helper_name('f', owner_class, &identity, *next_factory); *next_factory += 1; let descriptor = format!("(){array_descriptor}"); let mut instructions = Vec::new(); @@ -182,7 +230,7 @@ fn create_byte_array_factory( } let chunk_size = (MAX_INLINE_CONSTANT_INSTRUCTIONS / 4).max(1) * oomir::PACKED_BYTE_CHUNK; for (chunk_index, chunk) in elements.chunks(chunk_size).enumerate() { - let name = format!("_constant_fill_{identity}_{}", *next_factory); + let name = helper_name('x', owner_class, &identity, *next_factory); *next_factory += 1; let mut fill = vec![Instruction::Aload_0]; super::arrays::fill_packed_bytes( @@ -200,7 +248,7 @@ fn create_byte_array_factory( )); } instructions.push(Instruction::Areturn); - let name = format!("_constant_factory_{identity}_{}", *next_factory); + let name = helper_name('f', owner_class, &identity, *next_factory); *next_factory += 1; add_constant_helper_method(cp, methods, &name, "()[B", 0, instructions)?; Ok(oomir::Constant::FactoryCall { @@ -238,7 +286,8 @@ pub(super) fn create_shared_pointer_factory( methods: &mut Vec, next_factory: &mut usize, ) -> jvm::Result { - let builder = create_constant_factory(cp, owner_class, constant, methods, next_factory)?; + let builder = + create_unshared_constant_factory(cp, owner_class, constant, methods, next_factory)?; let identity = crate::stable_hash::short_hash_value(constant, 16); share_factory(cp, owner_class, &identity, builder, methods, next_factory) } @@ -260,7 +309,7 @@ fn share_factory( unreachable!("shared constant construction requires a factory call"); }; - let method_name = format!("_constant_shared_{constant_identity}_{}", *next_factory); + let method_name = helper_name('s', owner_class, constant_identity, *next_factory); *next_factory += 1; let descriptor = format!("(){}", ty.to_jvm_descriptor()); let identity = format!("{owner_class}#{constant_identity}"); @@ -301,6 +350,22 @@ pub(super) fn create_constant_factory( constant: &oomir::Constant, methods: &mut Vec, next_factory: &mut usize, +) -> jvm::Result { + // Nested references also identify stable CTFE allocations. Without caching, each use + // reconstructs the same table. + if super::prepare::shared_pointer(constant) { + create_shared_pointer_factory(cp, owner_class, constant, methods, next_factory) + } else { + create_unshared_constant_factory(cp, owner_class, constant, methods, next_factory) + } +} + +fn create_unshared_constant_factory( + cp: &mut InternedConstantPool, + owner_class: &str, + constant: &oomir::Constant, + methods: &mut Vec, + next_factory: &mut usize, ) -> jvm::Result { if let oomir::Constant::Array(element_type, elements) = constant && constant_instruction_cost(constant) > MAX_INLINE_CONSTANT_INSTRUCTIONS @@ -451,7 +516,7 @@ pub(super) fn create_constant_factory( let return_type = oomir::Type::from_constant(&prepared); let identity = crate::stable_hash::short_hash_value(constant, 16); - let method_name = format!("_constant_factory_{identity}_{}", *next_factory); + let method_name = helper_name('f', owner_class, &identity, *next_factory); *next_factory += 1; let descriptor = format!("(){}", return_type.to_jvm_descriptor()); let mut instructions = Vec::new(); diff --git a/src/lower2/constants/mod.rs b/src/lower2/constants/mod.rs index 2d8eb4bd..aaabfe80 100644 --- a/src/lower2/constants/mod.rs +++ b/src/lower2/constants/mod.rs @@ -11,6 +11,8 @@ use jvm::{ mod encoding; pub(super) use encoding::*; mod arrays; +mod resources; +mod structured; use arrays::append_empty_array; mod factories; use factories::{create_constant_factory, create_shared_array_factory}; @@ -21,5 +23,7 @@ pub(super) use prepare::{function_needs_constant_preparation, prepare_function_c mod statics; pub(super) use statics::create_static_initializer_method; +#[cfg(test)] +mod test_support; #[cfg(test)] mod tests; diff --git a/src/lower2/constants/prepare.rs b/src/lower2/constants/prepare.rs index 79f1634c..c64dcdd3 100644 --- a/src/lower2/constants/prepare.rs +++ b/src/lower2/constants/prepare.rs @@ -5,7 +5,7 @@ pub(super) const MAX_INLINE_CONSTANT_INSTRUCTIONS: usize = 1_024; use crate::oomir::constant_instruction_cost; -fn shared_pointer(constant: &oomir::Constant) -> bool { +pub(super) fn shared_pointer(constant: &oomir::Constant) -> bool { // These already denote one stable CTFE allocation. Cache the materializer // too, so repeated loads do not build values which the runtime discards. // Tiny values need less bytecode inline than two factory methods. Include @@ -38,7 +38,7 @@ fn shared_arrays(body: &oomir::SsaBody) -> Vec { let parent = match inst.op { Op::Reinterpret(value) | Op::Cast(value) => Some(value), Op::GetField { object, .. } => Some(object), - Op::ArrayGet { array, index } => { + Op::ArrayGet { array, index, .. } => { pending.push(ir.resolve(index)); Some(array) } diff --git a/src/lower2/constants/resources.rs b/src/lower2/constants/resources.rs new file mode 100644 index 00000000..9712e3e9 --- /dev/null +++ b/src/lower2/constants/resources.rs @@ -0,0 +1,103 @@ +//! Large primitive constants are data, never a program of element stores. +use super::*; + +const MIN_BYTES: usize = 1024; + +pub(super) fn element(ty: &oomir::Type, value: &oomir::Constant) -> Option<(u64, usize)> { + use oomir::{Constant as C, Type as T}; + Some(match (ty, value) { + (T::Boolean, C::Boolean(v)) => (u64::from(*v), 1), + (T::I8 | T::U8, C::I8(v)) => (*v as u8 as u64, 1), + (T::I8 | T::U8, C::U8(v)) => (u64::from(*v), 1), + (T::I16, C::I16(v)) => (*v as u16 as u64, 2), + (T::U16, C::U16(v)) | (T::F16, C::F16(v)) => (u64::from(*v), 2), + (T::Char, C::Char(v)) => (u64::from(*v as u16), 2), + (T::I32, C::I32(v)) => (*v as u32 as u64, 4), + (T::U32, C::U32(v)) => (u64::from(*v), 4), + (T::I64, C::I64(v)) => (*v as u64, 8), + (T::U64, C::U64(v)) => (*v, 8), + (T::F32, C::F32(v)) => (u64::from(v.to_bits()), 4), + (T::F64, C::F64(v)) => (v.to_bits(), 8), + _ => return None, + }) +} + +pub(super) fn eligible( + cp: &InternedConstantPool, + ty: &oomir::Type, + values: &[oomir::Constant], +) -> bool { + cp.resource_anchor().is_some() + && values + .first() + .and_then(|v| element(ty, v)) + .is_some_and(|(_, size)| { + values.len().saturating_mul(size) >= MIN_BYTES + && values.iter().all(|v| element(ty, v).is_some()) + }) +} + +pub(super) fn fill( + code: &mut Vec, + cp: &mut InternedConstantPool, + ty: &oomir::Type, + values: &[oomir::Constant], +) -> jvm::Result<()> { + let (_, size) = element(ty, &values[0]).expect("validated primitive constant"); + let per_block = jvm::resources::BLOCK_BYTES / size; + for (block, values) in values.chunks(per_block).enumerate() { + let mut bytes = Vec::with_capacity(values.len() * size); + for value in values { + let (bits, _) = element(ty, value).expect("validated primitive constant"); + bytes.extend_from_slice(&bits.to_le_bytes()[..size]); + } + fill_block(code, cp, block * per_block, values.len(), bytes)?; + } + Ok(()) +} + +pub(super) fn fill_bytes( + code: &mut Vec, + cp: &mut InternedConstantPool, + bytes: &[u8], +) -> jvm::Result { + if bytes.len() < MIN_BYTES || cp.resource_anchor().is_none() { + return Ok(false); + } + for (block, bytes) in bytes.chunks(jvm::resources::BLOCK_BYTES).enumerate() { + fill_block( + code, + cp, + block * jvm::resources::BLOCK_BYTES, + bytes.len(), + bytes.to_vec(), + )?; + } + Ok(true) +} + +fn fill_block( + code: &mut Vec, + cp: &mut InternedConstantPool, + start: usize, + count: usize, + bytes: Vec, +) -> jvm::Result<()> { + let anchor = cp.resource_anchor().expect("resource owner"); + let name = cp.add_resource(bytes)?; + let owner = cp.add_class("org/rustlang/runtime/ConstantData")?; + let fill = cp.add_method_ref( + owner, + "fill", + "(Ljava/lang/Object;IILjava/lang/String;Ljava/lang/Class;)V", + )?; + code.extend([ + Instruction::Dup, + get_int_const_instr(cp, i32::try_from(start)?), + get_int_const_instr(cp, i32::try_from(count)?), + Instruction::Ldc_w(name), + Instruction::Ldc_w(anchor), + Instruction::Invokestatic(fill), + ]); + Ok(()) +} diff --git a/src/lower2/constants/statics.rs b/src/lower2/constants/statics.rs index f17cc36e..efbc0b26 100644 --- a/src/lower2/constants/statics.rs +++ b/src/lower2/constants/statics.rs @@ -24,7 +24,9 @@ pub(in crate::lower2) fn create_static_initializer_method( }); } - if let oomir::Constant::Array(element_type, elements) = &static_value.initializer { + if let oomir::Constant::Array(element_type, elements) = &static_value.initializer + && matches!(static_value.storage_type, oomir::Type::Array(_)) + { append_empty_array(&mut instructions, cp, element_type, elements.len())?; } else { instructions.push(Instruction::Aconst_null); @@ -92,29 +94,29 @@ pub(in crate::lower2) fn create_static_initializer_method( &static_value.field_name, &static_value.storage_type.to_jvm_descriptor(), )?; - let initializer = - if let oomir::Constant::Array(element_type, elements) = &static_value.initializer { - // Fill the published array in place, including large chunked - // constants, without allocating and copying a second array. - factories::create_chunked_array_factory( - cp, - owner_class, - element_type, - elements, - methods, - next_factory, - Some(field_ref), - )? - } else { - instructions.push(Instruction::Getstatic(field_ref)); - create_constant_factory( - cp, - owner_class, - &static_value.initializer, - methods, - next_factory, - )? - }; + let initializer = if let oomir::Constant::Array(element_type, elements) = + &static_value.initializer + && matches!(static_value.storage_type, oomir::Type::Array(_)) + { + factories::create_chunked_array_factory( + cp, + owner_class, + element_type, + elements, + methods, + next_factory, + Some(field_ref), + )? + } else { + instructions.push(Instruction::Getstatic(field_ref)); + create_constant_factory( + cp, + owner_class, + &static_value.initializer, + methods, + next_factory, + )? + }; load_constant(&mut instructions, cp, &initializer)?; if matches!(static_value.storage_type, oomir::Type::Pointer(_)) { let initializer_type = oomir::Type::from_constant(&initializer); diff --git a/src/lower2/constants/structured.rs b/src/lower2/constants/structured.rs new file mode 100644 index 00000000..ea615f78 --- /dev/null +++ b/src/lower2/constants/structured.rs @@ -0,0 +1,259 @@ +//! Repeated constant shapes share a constructor loop. Each row and nested array receives separate +//! storage. +use super::*; +use oomir::{Constant as C, Type as T}; + +#[cfg(test)] +#[path = "structured_tests.rs"] +mod tests; + +enum Shape<'a> { + Scalar(&'a T, usize), + Array(&'a T, usize), + Object(&'a str, &'a [T], Vec>), + Unit, +} + +impl<'a> Shape<'a> { + fn new(ty: &'a T, value: &'a C, budget: &mut usize) -> Option { + *budget = budget.checked_sub(1)?; + if let Some((_, size)) = resources::element(ty, value) { + return Some(Self::Scalar(ty, size)); + } + match value { + C::Unit => Some(Self::Unit), + C::Array(element, _) => { + let size = match element.as_ref() { + T::Boolean | T::I8 | T::U8 => 1, + T::I16 | T::U16 | T::Char | T::F16 => 2, + T::I32 | T::U32 | T::F32 => 4, + T::I64 | T::U64 | T::F64 => 8, + _ => return None, + }; + Some(Self::Array(element, size)) + } + C::Instance { + class_name, + params, + param_types, + } if params.len() == param_types.len() => { + let fields = params + .iter() + .zip(param_types) + .map(|(value, ty)| Self::new(ty, value, budget)) + .collect::>>()?; + Some(Self::Object(class_name, param_types, fields)) + } + _ => None, + } + } + + fn encode(&self, value: &C, bytes: &mut Vec) -> bool { + match (self, value) { + (Self::Scalar(ty, size), value) => { + let Some((bits, _)) = resources::element(ty, value) else { + return false; + }; + bytes.extend_from_slice(&bits.to_le_bytes()[..*size]); + true + } + (Self::Array(ty, size), C::Array(element, values)) if *ty == element.as_ref() => { + let length = values.len().saturating_mul(*size).saturating_add(4); + if bytes.len().saturating_add(length) > jvm::resources::BLOCK_BYTES { + return false; + } + bytes.extend_from_slice(&(values.len() as u32).to_le_bytes()); + values + .iter() + .all(|value| Self::Scalar(ty, *size).encode(value, bytes)) + } + ( + Self::Object(name, types, fields), + C::Instance { + class_name, + param_types, + params, + }, + ) if *name == class_name && *types == param_types && fields.len() == params.len() => { + fields + .iter() + .zip(params) + .all(|(field, value)| field.encode(value, bytes)) + } + (Self::Unit, C::Unit) => true, + _ => false, + } + } + + fn read(&self, code: &mut Vec, cp: &mut InternedConstantPool) -> jvm::Result<()> { + match self { + Self::Unit => {} + Self::Scalar(ty, _) => { + let (name, desc) = match ty { + T::Boolean | T::I8 | T::U8 => ("get", "()B"), + T::I16 | T::F16 => ("getShort", "()S"), + T::Char | T::U16 => ("getChar", "()C"), + T::I32 | T::U32 => ("getInt", "()I"), + T::I64 | T::U64 => ("getLong", "()J"), + T::F32 => ("getFloat", "()F"), + T::F64 => ("getDouble", "()D"), + _ => unreachable!("validated scalar resource"), + }; + let owner = cp.add_class("java/nio/ByteBuffer")?; + code.extend([ + Instruction::Aload_3, + Instruction::Invokevirtual(cp.add_method_ref(owner, name, desc)?), + ]); + } + Self::Array(ty, _) => { + let buffer = cp.add_class("java/nio/ByteBuffer")?; + code.extend([ + Instruction::Aload_3, + Instruction::Invokevirtual(cp.add_method_ref(buffer, "getInt", "()I")?), + Instruction::Newarray(ArrayType::from_bytes(&mut jvm::ByteReader::new(&[ty + .to_jvm_primitive_array_type_code() + .expect("primitive constant array")]))?), + ]); + code.extend([Instruction::Dup, Instruction::Aload_3]); + let runtime = cp.add_class("org/rustlang/runtime/ConstantData")?; + code.push(Instruction::Invokestatic(cp.add_method_ref( + runtime, + "readArray", + "(Ljava/lang/Object;Ljava/nio/ByteBuffer;)V", + )?)); + } + Self::Object(name, types, fields) => { + let owner = cp.add_class(name)?; + code.extend([Instruction::New(owner), Instruction::Dup]); + for field in fields { + field.read(code, cp)?; + } + let descriptor = format!( + "({})V", + types + .iter() + .filter(|ty| ty.has_jvm_value()) + .map(T::to_jvm_descriptor) + .collect::() + ); + code.push(Instruction::Invokespecial( + cp.add_method_ref(owner, "", descriptor)?, + )); + } + } + Ok(()) + } +} + +pub(super) fn factory( + cp: &mut InternedConstantPool, + owner: &str, + element: &T, + values: &[C], + methods: &mut Vec, + next: &mut usize, + storage: Option, +) -> jvm::Result> { + let Some(anchor) = cp.resource_anchor() else { + return Ok(None); + }; + let Some(first) = values.first() else { + return Ok(None); + }; + let Some(shape) = Shape::new(element, first, &mut 64) else { + return Ok(None); + }; + // Validation precedes pool and method changes. Only block boundaries remain in memory. + let mut bytes = Vec::new(); + let mut total = 0usize; + let mut block_bytes = 0usize; + let mut start = 0; + let mut blocks = Vec::new(); + for (index, value) in values.iter().enumerate() { + bytes.clear(); + if !shape.encode(value, &mut bytes) || bytes.len() > jvm::resources::BLOCK_BYTES { + return Ok(None); + } + total = total.saturating_add(bytes.len()); + if block_bytes + bytes.len() > jvm::resources::BLOCK_BYTES { + blocks.push(start..index); + start = index; + block_bytes = 0; + } + block_bytes += bytes.len(); + } + if total < 1024 { + return Ok(None); + } + blocks.push(start..values.len()); + let identity = crate::stable_hash::short_hash_value(&(element, values, storage), 16); + let fill_name = factories::helper_name('r', owner, &identity, *next); + let factory_name = factories::helper_name('d', owner, &identity, *next); + *next += 1; + let array = T::Array(Box::new(element.clone())); + let descriptor = format!("({}IILjava/nio/ByteBuffer;)V", array.to_jvm_descriptor()); + let mut fill = vec![ + Instruction::Iload_1, + Instruction::Iload_2, + Instruction::If_icmpge(0), + Instruction::Aload_0, + Instruction::Iload_1, + ]; + shape.read(&mut fill, cp)?; + fill.extend([ + element + .get_jvm_array_store_instruction() + .expect("resource array"), + Instruction::Iinc(1, 1), + Instruction::Goto(0), + ]); + fill[2] = Instruction::If_icmpge(u16::try_from(fill.len())?); + fill.push(Instruction::Return); + factories::add_constant_helper_method(cp, methods, &fill_name, &descriptor, 4, fill)?; + let owner = cp.add_class(owner)?; + let fill = cp.add_method_ref(owner, &fill_name, &descriptor)?; + let runtime = cp.add_class("org/rustlang/runtime/ConstantData")?; + let buffer = cp.add_method_ref( + runtime, + "buffer", + "(Ljava/lang/String;Ljava/lang/Class;I)Ljava/nio/ByteBuffer;", + )?; + let mut code = Vec::new(); + if let Some(storage) = storage { + code.push(Instruction::Getstatic(storage)); + } else { + append_empty_array(&mut code, cp, element, values.len())?; + } + for range in blocks { + let mut bytes = Vec::new(); + for value in &values[range.clone()] { + assert!(shape.encode(value, &mut bytes)); + } + let length = bytes.len(); + let resource = cp.add_resource(bytes)?; + code.extend([ + Instruction::Dup, + get_int_const_instr(cp, i32::try_from(range.start)?), + get_int_const_instr(cp, i32::try_from(range.end)?), + Instruction::Ldc_w(resource), + Instruction::Ldc_w(anchor), + get_int_const_instr(cp, i32::try_from(length)?), + Instruction::Invokestatic(buffer), + Instruction::Invokestatic(fill), + ]); + } + code.push(Instruction::Areturn); + factories::add_constant_helper_method( + cp, + methods, + &factory_name, + &format!("(){}", array.to_jvm_descriptor()), + 0, + code, + )?; + Ok(Some(C::FactoryCall { + owner_class: cp.try_get_class(owner)?.to_string(), + method_name: factory_name, + ty: array, + })) +} diff --git a/src/lower2/constants/structured_tests.rs b/src/lower2/constants/structured_tests.rs new file mode 100644 index 00000000..7f3aab61 --- /dev/null +++ b/src/lower2/constants/structured_tests.rs @@ -0,0 +1,195 @@ +use super::*; +use std::{fs, process::Command}; + +fn row(index: usize) -> C { + C::Instance { + class_name: "Row".into(), + param_types: vec![ + T::Array(Box::new(T::U8)), + T::I16, + T::U16, + T::I32, + T::U64, + T::F32, + T::F64, + T::Boolean, + ], + params: vec![ + C::Array( + Box::new(T::U8), + (0..index % 17).map(|j| C::U8((index + j) as u8)).collect(), + ), + C::I16(-12345), + C::U16(0xfedc), + C::I32(index as i32), + C::U64(u64::MAX), + C::F32(-0.0), + C::F64(f64::from_bits(0x7ff8_1234_5678_9abc)), + C::Boolean(true), + ], + } +} + +#[test] +fn structured_tables_use_bounded_resources_and_fresh_storage() { + let mut cp = InternedConstantPool::default(); + let this_class = cp.add_class("Data").unwrap(); + cp.set_resource_anchor(this_class); + let super_class = cp.add_class("java/lang/Object").unwrap(); + let mut methods = Vec::new(); + let rows = (0..10000).map(row).collect::>(); + let value = factory( + &mut cp, + "Data", + &T::Class("Row".into()), + &rows, + &mut methods, + &mut 0, + None, + ) + .unwrap() + .unwrap(); + assert_eq!( + methods.len(), + 2, + "table length must not multiply constructor methods" + ); + let mut code = Vec::new(); + load_constant(&mut code, &mut cp, &value).unwrap(); + code.push(Instruction::Areturn); + factories::add_constant_helper_method(&mut cp, &mut methods, "rows", "()[LRow;", 0, code) + .unwrap(); + methods.last_mut().unwrap().access_flags = + MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC; + let resources = cp.take_resources(); + assert!(resources.len() > 1); + assert!( + resources + .iter() + .all(|r| r.bytes.len() <= jvm::resources::BLOCK_BYTES) + ); + let class = jvm::ClassFile { + code_source_url: None, + version: jvm::Version::Java8 { minor: 0 }, + constant_pool: cp.into_inner(), + access_flags: jvm::ClassAccessFlags::PUBLIC | jvm::ClassAccessFlags::SUPER, + this_class, + super_class, + interfaces: vec![], + fields: vec![], + methods, + attributes: vec![], + }; + let mut bytes = Vec::new(); + jvm::encode::class_file(&class, &mut bytes).unwrap(); + assert!( + bytes.len() < 4096, + "constant data expanded into bytecode: {}", + bytes.len() + ); + let directory = + std::env::temp_dir().join(format!("rcj-structured-data-{}", std::process::id())); + fs::create_dir_all(&directory).unwrap(); + let jar = super::super::test_support::link(&directory, &class, resources); + fs::write( + directory.join("Row.java"), + r#" +public class Row { + final byte[] bytes; + final int index; + public Row(byte[] bytes, short s, char c, int index, long l, float f, double d, boolean b) { + if (s != -12345 || c != 0xfedc || l != -1 || Float.floatToRawIntBits(f) != 0x80000000 + || Double.doubleToRawLongBits(d) != 0x7ff8123456789abcL || !b) + throw new AssertionError("scalar decoding"); + this.bytes = bytes; this.index = index; + } +} +"#, + ) + .unwrap(); + let result = Command::new("javac") + .arg(directory.join("Row.java")) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + fs::write( + directory.join("Run.java"), + r#" +class Run { + public static void main(String[] args) { + Row[] a = Data.rows(), b = Data.rows(); + if (a.length != 10000) throw new AssertionError("length"); + for (int i = 0; i < a.length; i++) { + if (a[i].index != i || a[i].bytes.length != i % 17) throw new AssertionError("row " + i); + for (int j = 0; j < a[i].bytes.length; j++) { + if (a[i].bytes[j] != (byte)(i + j)) throw new AssertionError("byte " + i); + } + if (a[i] == b[i] || a[i].bytes == b[i].bytes) throw new AssertionError("shared row"); + } + a[1].bytes[0] = 99; + if (b[1].bytes[0] != 1 || a[257].bytes[0] != 1) throw new AssertionError("aliased bytes"); + } +} +"#, + ) + .unwrap(); + let result = Command::new("javac") + .arg("-cp") + .arg(std::env::join_paths([&jar, &directory]).unwrap()) + .arg(directory.join("Run.java")) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let runtime = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("runtime/build/libs/runtime-0.1.0.jar"); + let result = Command::new("java") + .args(["-Xverify:all", "--class-path"]) + .arg(std::env::join_paths([&jar, &directory, &runtime]).unwrap()) + .arg("Run") + .output() + .unwrap(); + assert!( + result.status.success(), + "{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn incompatible_rows_do_not_partially_emit_a_resource() { + let mut cp = InternedConstantPool::default(); + let owner = cp.add_class("Data").unwrap(); + cp.set_resource_anchor(owner); + let mut rows = (0..100).map(row).collect::>(); + let C::Instance { class_name, .. } = rows.last_mut().unwrap() else { + unreachable!() + }; + *class_name = "Other".into(); + let before = cp.len(); + let mut methods = vec![]; + assert!( + factory( + &mut cp, + "Data", + &T::Class("Row".into()), + &rows, + &mut methods, + &mut 0, + None + ) + .unwrap() + .is_none() + ); + assert!(methods.is_empty() && cp.take_resources().is_empty()); + assert_eq!(cp.len(), before); +} diff --git a/src/lower2/constants/test_support.rs b/src/lower2/constants/test_support.rs new file mode 100644 index 00000000..0c0ab2a3 --- /dev/null +++ b/src/lower2/constants/test_support.rs @@ -0,0 +1,48 @@ +use super::jvm; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +/// Constants contain linker tags even in these hand-built unit-test classes. +pub(super) fn link( + directory: &Path, + class: &jvm::ClassFile<'_>, + resources: Vec, +) -> PathBuf { + let bundle = directory.join("constants.jvmbundle"); + let mut writer = jvm::bundle::Writer::create(&bundle).unwrap(); + let registry = jvm::registry::ClassRegistry::default(); + let mut bytes = Vec::new(); + jvm::encode::class_file(class, &mut bytes).unwrap(); + let name = class.constant_pool.try_get_class(class.this_class).unwrap(); + registry + .emit(&mut writer, &name.to_rust_string(), &bytes) + .unwrap(); + for resource in resources { + registry + .emit( + &mut writer, + &format!("{}{}", jvm::resources::BUNDLE_PREFIX, resource.name), + &resource.bytes, + ) + .unwrap(); + } + writer.finish().unwrap(); + let output = directory.join("constants.jar"); + let linker = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("java-linker/target/release/java-linker") + .with_extension(std::env::consts::EXE_EXTENSION); + let result = Command::new(linker) + .arg(bundle) + .arg("-o") + .arg(&output) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + output +} diff --git a/src/lower2/constants/tests.rs b/src/lower2/constants/tests.rs index fba0d60f..6a328214 100644 --- a/src/lower2/constants/tests.rs +++ b/src/lower2/constants/tests.rs @@ -12,6 +12,7 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { .collect(); let mut cp = InternedConstantPool::default(); let this_class = cp.add_class("PackedConstants").unwrap(); + cp.set_resource_anchor(this_class); let super_class = cp.add_class("java/lang/Object").unwrap(); let mut methods = Vec::new(); let factory = factories::create_shared_array_factory( @@ -23,6 +24,32 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { &mut 0, ) .unwrap(); + let nested = factories::create_constant_factory( + &mut cp, + "PackedConstants", + &oomir::Constant::SliceRef { + backing: Box::new(oomir::Constant::InternedPointer { + identity: "PackedConstants::nested".into(), + value: Box::new(oomir::Constant::Array( + Box::new(oomir::Type::U8), + values.clone(), + )), + array_backed: true, + allocation_size: length as u64, + offset: 0, + view_size: 1, + alignment: 1, + view_codec: Box::new(oomir::Constant::Null(oomir::Type::java_string())), + pointee: Box::new(oomir::Type::U8), + }), + element_type: Box::new(oomir::Type::U8), + offset: 1, + length: (length - 1) as u64, + }, + &mut methods, + &mut 0, + ) + .unwrap(); let pointer = factories::create_shared_pointer_factory( &mut cp, "PackedConstants", @@ -90,16 +117,22 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { ) .unwrap(); for name in [ - "array", "signed", "bytes", "shared", "pointer", "scalar", "another", + "array", "signed", "bytes", "packed", "shared", "pointer", "scalar", "another", "nested", ] { let mut code = Vec::new(); match name { "array" => arrays::load_array(&mut code, &mut cp, &oomir::Type::U8, &values), "signed" => arrays::load_array(&mut code, &mut cp, &oomir::Type::I8, &signed), "bytes" => arrays::load_bytes(&mut code, &mut cp, &bytes), + "packed" => { + let size = oomir::PACKED_BYTE_CHUNK * 2 + 1; + arrays::append_empty_array(&mut code, &mut cp, &oomir::Type::U8, size).unwrap(); + arrays::fill_packed_bytes(&mut code, &mut cp, 0, size, |_| 255) + } "pointer" => load_constant(&mut code, &mut cp, &pointer), "scalar" => load_constant(&mut code, &mut cp, &scalar), "another" => load_constant(&mut code, &mut cp, &another), + "nested" => load_constant(&mut code, &mut cp, &nested), _ => load_constant(&mut code, &mut cp, &factory), } .unwrap(); @@ -111,6 +144,8 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { descriptor_index: cp .add_utf8(if matches!(name, "pointer" | "scalar" | "another") { "()Lorg/rustlang/runtime/Pointer;" + } else if name == "nested" { + "()Lorg/rustlang/runtime/SliceView;" } else { "()[B" }) @@ -125,6 +160,31 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { }], }); } + for (name, value) in [ + ("shortText", "\0😃é".repeat(100)), + ("longText", "\0😃é".repeat(15000)), + ] { + let mut code = Vec::new(); + load_constant(&mut code, &mut cp, &oomir::Constant::Str(value)).unwrap(); + code.push(Instruction::Areturn); + factories::add_constant_helper_method( + &mut cp, + &mut methods, + name, + "()Lorg/rustlang/runtime/Utf8View;", + 0, + code, + ) + .unwrap(); + methods.last_mut().unwrap().access_flags = + MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC; + } + let resources = cp.take_resources(); + assert!(!resources.is_empty()); + assert!( + cp.len() < 256, + "binary payload leaked into the constant pool" + ); let class = jvm::ClassFile { code_source_url: None, version: jvm::Version::Java8 { minor: 0 }, @@ -137,15 +197,23 @@ fn packed_byte_constants_preserve_every_byte_and_array_ownership() { methods, attributes: vec![], }; - let mut encoded = Vec::new(); - jvm::encode::class_file(&class, &mut encoded).unwrap(); let directory = std::env::temp_dir().join(format!("rcj-packed-bytes-{}", std::process::id())); fs::create_dir_all(&directory).unwrap(); - fs::write(directory.join("PackedConstants.class"), encoded).unwrap(); + let jar = super::test_support::link(&directory, &class, resources); fs::write(directory.join("Run.java"), format!(r#" public class Run {{ private static volatile Object sink; public static void main(String[] args) {{ + String piece = "\0\uD83D\uDE03é"; + if (!org.rustlang.runtime.Utf8View.toJavaString(PackedConstants.shortText()).equals(piece.repeat(100))) + throw new AssertionError("short text"); + if (!org.rustlang.runtime.Utf8View.toJavaString(PackedConstants.longText()).equals(piece.repeat(15000))) + throw new AssertionError("long text"); + if (PackedConstants.longText().array != PackedConstants.longText().array) + throw new AssertionError("long text identity"); + byte[] packed = PackedConstants.packed(); + if (packed.length != {packed_length}) throw new AssertionError("packed length"); + for (byte b : packed) if (b != -1) throw new AssertionError("packed byte"); byte[][] arrays = {{ PackedConstants.array(), PackedConstants.signed(), PackedConstants.bytes(), PackedConstants.shared() }}; for (byte[] bytes : arrays) {{ if (bytes.length != {length}) throw new AssertionError("length"); @@ -158,15 +226,26 @@ public class Run {{ org.rustlang.runtime.Pointer pointer = PackedConstants.pointer(); byte[] backing = (byte[]) pointer.backingArray(); if (!java.util.Arrays.equals(backing, arrays[3])) throw new AssertionError("pointer contents"); + org.rustlang.runtime.SliceView nested = PackedConstants.nested(); + if (nested.offset != 1 || nested.length != {length} - 1 + || !java.util.Arrays.equals((byte[]) ((org.rustlang.runtime.Pointer) nested.array).backingArray(), arrays[3])) + throw new AssertionError("nested pointer contents"); + if (PackedConstants.nested().array != nested.array) throw new AssertionError("nested pointer identity"); java.lang.management.ThreadMXBean bean = java.lang.management.ManagementFactory.getThreadMXBean(); if (bean instanceof com.sun.management.ThreadMXBean) {{ com.sun.management.ThreadMXBean allocations = (com.sun.management.ThreadMXBean) bean; if (allocations.isThreadAllocatedMemorySupported()) {{ allocations.setThreadAllocatedMemoryEnabled(true); - for (int i = 0; i < 32; i++) sink = PackedConstants.pointer(); + for (int i = 0; i < 32; i++) {{ + sink = PackedConstants.pointer(); + sink = PackedConstants.nested(); + }} long thread = Thread.currentThread().getId(); long before = allocations.getThreadAllocatedBytes(thread); - for (int i = 0; i < 128; i++) sink = PackedConstants.pointer(); + for (int i = 0; i < 128; i++) {{ + sink = PackedConstants.pointer(); + sink = PackedConstants.nested(); + }} long allocated = allocations.getThreadAllocatedBytes(thread) - before; if (allocated > 1024 * 1024) throw new AssertionError("reconstructed cached pointer: " + allocated); }} @@ -177,13 +256,13 @@ public class Run {{ throw new AssertionError("colliding factory"); }} }} -"#)).unwrap(); +"#, packed_length = oomir::PACKED_BYTE_CHUNK * 2 + 1)).unwrap(); let runtime = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("runtime/build/libs/runtime-0.1.0.jar"); let result = Command::new("java") .arg("-Xverify:all") .arg("--class-path") - .arg(std::env::join_paths([&directory, &runtime]).unwrap()) + .arg(std::env::join_paths([&jar, &directory, &runtime]).unwrap()) .arg(directory.join("Run.java")) .output() .unwrap(); diff --git a/src/lower2/helpers.rs b/src/lower2/helpers.rs index 8f24f94f..84528fff 100644 --- a/src/lower2/helpers.rs +++ b/src/lower2/helpers.rs @@ -30,11 +30,10 @@ pub fn local_kind(ty: &Type) -> Option K::Long, Type::F32 => K::Float, Type::F64 => K::Double, - Type::Reference(_) - | Type::Pointer(_) - | Type::MutableReference(_) + Type::Pointer(_) | Type::Array(_) | Type::Slice(_) + | Type::TaggedI64 | Type::Str | Type::Class(_) | Type::Interface(_) => K::Reference, @@ -182,18 +181,6 @@ pub fn get_cast_instructions( return Ok(vec![JI::Invokevirtual(method)]); } - if let Type::MutableReference(inner) = src { - let inner = inner.as_ref(); - let mut instrs = vec![JI::Iconst_0, JI::Aaload]; - if dest != inner { - if let Some(dest_name) = dest.to_jvm_descriptor_or_internal_name() { - let dest_idx = cp.add_class(&dest_name)?; - instrs.push(JI::Checkcast(dest_idx)); - } - } - return Ok(instrs); - } - // Generic checkcast for all other reference-to-reference // Check if both are reference types AND have valid internal names/descriptors if let (Some(_), Some(dest_name)) = ( @@ -411,13 +398,12 @@ pub(super) fn return_instruction_for_type(ty: &oomir::Type) -> Instruction { oomir::Type::I64 | oomir::Type::U64 => Instruction::Lreturn, oomir::Type::F32 => Instruction::Freturn, oomir::Type::F64 => Instruction::Dreturn, - oomir::Type::Str + oomir::Type::TaggedI64 + | oomir::Type::Str | oomir::Type::Class(_) | oomir::Type::Array(_) | oomir::Type::Slice(_) - | oomir::Type::Reference(_) | oomir::Type::Pointer(_) - | oomir::Type::MutableReference(_) | oomir::Type::Interface(_) => Instruction::Areturn, oomir::Type::Void | oomir::Type::Unit => Instruction::Return, } diff --git a/src/lower2/jvm_gen.rs b/src/lower2/jvm_gen.rs index b82c8a1f..739c9a38 100644 --- a/src/lower2/jvm_gen.rs +++ b/src/lower2/jvm_gen.rs @@ -1,17 +1,14 @@ //! Class assembly and shared JVM method recipes. mod body; +mod records; pub(super) use body::{BodyEmitter, BodyOwner}; mod constructors; +mod enum_shapes; +mod fields; +mod shapes; pub(super) use constructors::create_default_constructor; -use constructors::{ - create_field_constructor, create_managed_copy_method, create_relative_pointer_field_constructor, -}; -mod slices; -pub(super) use slices::create_slice_view_classfile; -mod strings; -pub(super) use strings::create_utf8_view_classfile; +use constructors::{create_field_constructor, create_managed_copy_method}; mod bridges; -pub(super) use bridges::create_relative_pointer_bridge; use bridges::create_static_instance_bridge; mod enum_equality; use enum_equality::create_enum_equality_methods; @@ -23,6 +20,8 @@ mod interfaces; pub(super) use interfaces::create_data_type_classfile_for_interface; mod forward; +mod forward_returns; +mod function_handles; use super::{ DebugInfoOptions, @@ -75,7 +74,7 @@ fn code_attribute_with_stack_maps( }) } -fn code_attribute_for_descriptor( +pub(super) fn code_attribute_for_descriptor( cp: &mut InternedConstantPool, max_locals: u16, code: Vec, @@ -105,11 +104,8 @@ pub(super) fn oomir_type_to_ristretto_field_type(type2: &oomir::Type) -> jvm::Fi oomir::Type::F64 => jvm::FieldType::Base(BaseType::Double), oomir::Type::Boolean => jvm::FieldType::Base(BaseType::Boolean), oomir::Type::Char => jvm::FieldType::Base(BaseType::Char), + oomir::Type::TaggedI64 => jvm::FieldType::Object(oomir::TAGGED_LONG_CLASS.into()), oomir::Type::Str => jvm::FieldType::Object(oomir::UTF8_VIEW_CLASS.into()), - oomir::Type::Reference(ref2) => { - let inner_ty = ref2.as_ref(); - oomir_type_to_ristretto_field_type(inner_ty) - } oomir::Type::Pointer(_) => jvm::FieldType::Object(oomir::POINTER_CLASS.into()), oomir::Type::Array(inner_ty) => { let inner_field_type = if inner_ty.has_jvm_value() { @@ -120,13 +116,6 @@ pub(super) fn oomir_type_to_ristretto_field_type(type2: &oomir::Type) -> jvm::Fi jvm::FieldType::Array(Box::new(inner_field_type)) } oomir::Type::Slice(_) => jvm::FieldType::Object(oomir::SLICE_VIEW_CLASS.into()), - oomir::Type::MutableReference(inner_ty) if !inner_ty.has_jvm_value() => { - jvm::FieldType::Object("java/lang/Object".into()) - } - oomir::Type::MutableReference(inner_ty) => { - let inner_field_type = oomir_type_to_ristretto_field_type(inner_ty); - jvm::FieldType::Array(Box::new(inner_field_type)) - } oomir::Type::Class(name) | oomir::Type::Interface(name) => { jvm::FieldType::Object(name.clone().into()) } diff --git a/src/lower2/jvm_gen/body.rs b/src/lower2/jvm_gen/body.rs index 42748af5..7a08e5b4 100644 --- a/src/lower2/jvm_gen/body.rs +++ b/src/lower2/jvm_gen/body.rs @@ -16,13 +16,29 @@ pub(in crate::lower2) struct BodyEmitter<'a> { pub next_factory: &'a mut usize, pub owner: &'a str, pub kind: BodyOwner, - pub relative_methods: &'a HashSet, pub debug: DebugInfoOptions, pub context: &'a oomir::construct::Context, } impl BodyEmitter<'_> { - pub fn emit_owned(&mut self, function: oomir::Function) -> jvm::Result<()> { + pub fn emit_owned(&mut self, mut function: oomir::Function) -> jvm::Result<()> { + function + .owner_class + .get_or_insert_with(|| self.owner.to_owned()); + if let Some(signature) = + self.context + .record_signature(self.owner, &function.name, &function.signature) + { + self.methods.push(super::records::bridge( + self.cp, + self.context, + self.owner, + &function.name, + &function.signature, + &signature, + )?); + function.name.push_str(oomir::construct::RECORD_ENTRY); + } let function = oomir::construct::seal(function, self.context).map_err(|message| { jvm::Error::VerificationError { context: "SSA construction".into(), @@ -81,32 +97,19 @@ impl BodyEmitter<'_> { let outlined = !interface && name.starts_with(oomir::outline::METHOD_PREFIX); let is_static = module_owner || function.signature.is_static; let signature = &function.signature; - let relative_adapter = !module_owner - && name.ends_with(oomir::RELATIVE_POINTER_METHOD_SUFFIX) - && signature.supports_relative_pointer_abi(); - let relative_static = is_static - && self - .relative_methods - .contains(&oomir::FunctionKey::new(self.owner, name, signature)); - let relative = relative_adapter || relative_static; let body = &function.body; - let code = - crate::lower2::select::compile(body, self.cp, self.bootstrap, self.debug, relative) - .map_err(|error| jvm::Error::VerificationError { - context: format!("Function {}::{name}", self.owner), - message: format!("Failed to translate function: {error:?}"), - })?; + let code = crate::lower2::select::compile( + Some(self.context), + body, + self.cp, + self.bootstrap, + self.debug, + ) + .map_err(|error| jvm::Error::VerificationError { + context: format!("Function {}::{name}", self.owner), + message: format!("Failed to translate function: {error:?}"), + })?; crate::metrics::record_selection_method(body, &code, || format!("{}::{name}", self.owner)); - let emitted_signature = if relative { - Cow::Owned(signature.relative_pointer_abi_signature()) - } else { - Cow::Borrowed(signature) - }; - let emitted_name = if relative_static { - Cow::Owned(format!("{name}{}", oomir::RELATIVE_POINTER_METHOD_SUFFIX)) - } else { - Cow::Borrowed(name) - }; let mut attributes = vec![Attribute::Code { name_index: self.cp.add_utf8("Code")?, max_stack: code.max_stack, @@ -115,9 +118,9 @@ impl BodyEmitter<'_> { exception_table: code.exceptions, attributes: code.attributes, }]; - if !outlined && (interface || name != "") { + if self.debug.local_variables && !outlined && (interface || name != "") { let mut parameters = Vec::new(); - for (name, ty) in emitted_signature.explicit_jvm_params() { + for (name, ty) in signature.explicit_jvm_params() { if ty.has_jvm_value() { parameters.push(jvm::attributes::MethodParameter { name_index: self.cp.add_utf8(name)?, @@ -130,10 +133,7 @@ impl BodyEmitter<'_> { parameters, }); } - let instance_bridge = !module_owner - && !signature.is_static - && !signature.params.is_empty() - && !relative_adapter; + let instance_bridge = !module_owner && !signature.is_static && !signature.params.is_empty(); // Enum-interface static calls select the nominal implementation. These // remaining generated computational bodies still need that static entry. let static_code = (interface && instance_bridge).then(|| attributes[0].clone()); @@ -149,20 +149,11 @@ impl BodyEmitter<'_> { }; self.methods.push(jvm::Method { access_flags, - name_index: self.cp.add_utf8(emitted_name.as_ref())?, - descriptor_index: self.cp.add_utf8(emitted_signature.to_string())?, + name_index: self.cp.add_utf8(name)?, + descriptor_index: self.cp.add_utf8(signature.to_string())?, attributes, }); - if relative_static { - self.methods.push(create_relative_pointer_bridge( - self.cp, - self.owner, - name, - signature, - access_flags, - interface, - )?); - } + if let Some(code) = static_code { let mut signature = signature.clone(); signature.is_static = true; diff --git a/src/lower2/jvm_gen/bridges.rs b/src/lower2/jvm_gen/bridges.rs index 1b1fd501..b4ae25ff 100644 --- a/src/lower2/jvm_gen/bridges.rs +++ b/src/lower2/jvm_gen/bridges.rs @@ -1,60 +1,6 @@ //! Native JVM bridges emission. use super::*; -pub(in crate::lower2) fn create_relative_pointer_bridge( - cp: &mut InternedConstantPool, - class_name: &str, - method_name: &str, - signature: &oomir::Signature, - access_flags: MethodAccessFlags, - owner_is_interface: bool, -) -> jvm::Result { - debug_assert!(signature.is_static); - let relative_signature = signature.relative_pointer_abi_signature(); - let relative_name = format!("{method_name}{}", oomir::RELATIVE_POINTER_METHOD_SUFFIX); - let class_index = cp.add_class(class_name)?; - let target = if owner_is_interface { - cp.add_interface_method_ref(class_index, &relative_name, &relative_signature.to_string())? - } else { - cp.add_method_ref(class_index, &relative_name, &relative_signature.to_string())? - }; - - let mut instructions = Vec::new(); - let mut local = 0u16; - - for (_, ty) in &signature.params { - if !ty.has_jvm_value() { - continue; - } - instructions.push(get_load_instruction(ty, local)?); - let size = get_type_size(ty); - local += size; - - if matches!(ty, Type::Pointer(_)) { - instructions.push(Instruction::Lconst_0); - instructions.push(Instruction::Lconst_0); - } - } - instructions.push(Instruction::Invokestatic(target)); - instructions.push(return_instruction_for_type(&signature.ret)); - - let descriptor = signature.to_string(); - Ok(jvm::Method { - access_flags, - name_index: cp.add_utf8(method_name)?, - descriptor_index: cp.add_utf8(&descriptor)?, - attributes: vec![code_attribute_for_descriptor( - cp, - local, - instructions, - &descriptor, - true, - Some(class_name), - method_name, - )?], - }) -} - pub(super) fn create_static_instance_bridge( cp: &mut InternedConstantPool, class_name_jvm: &str, @@ -118,37 +64,18 @@ pub(super) fn create_static_instance_bridge( } instructions.push(return_instruction_for_type(bridge_signature.ret.as_ref())); - let mut parameters = Vec::new(); - for (param_name, param_ty) in &bridge_signature.params { - if !param_ty.has_jvm_value() { - continue; - } - let name_index = cp.add_utf8(param_name)?; - parameters.push(jvm::attributes::MethodParameter { - name_index, - access_flags: MethodAccessFlags::empty(), - }); - } - let method_parameters_attribute_name_index = cp.add_utf8("MethodParameters")?; - Ok(jvm::Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, name_index, descriptor_index, - attributes: vec![ - code_attribute_for_descriptor( - cp, - next_local, - instructions, - &bridge_descriptor, - true, - None, - method_name, - )?, - Attribute::MethodParameters { - name_index: method_parameters_attribute_name_index, - parameters, - }, - ], + attributes: vec![code_attribute_for_descriptor( + cp, + next_local, + instructions, + &bridge_descriptor, + true, + None, + method_name, + )?], }) } diff --git a/src/lower2/jvm_gen/classes.rs b/src/lower2/jvm_gen/classes.rs index 28dcb4b3..e4a6d0e5 100644 --- a/src/lower2/jvm_gen/classes.rs +++ b/src/lower2/jvm_gen/classes.rs @@ -1,10 +1,12 @@ //! Native JVM classes emission. +use super::enums::object_equality; use super::*; /// Creates a ClassFile (as bytes) for a given OOMIR DataType that's a class pub(in crate::lower2) fn create_data_type_classfile_for_class( class_name_jvm: &str, fields: &[(String, Type)], + kind: oomir::ClassKind, is_abstract: bool, methods: HashMap, super_class_name_jvm: &str, @@ -13,8 +15,9 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( subclasses: &[String], nest_host: Option<&str>, debug_info: DebugInfoOptions, - relative_static_methods: &HashSet, context: &oomir::construct::Context, + output: &mut crate::lower2::output::ClassOutput, + registry: &crate::lower2::EmittedClassRegistry, ) -> jvm::Result> { let source_files = methods .values() @@ -42,6 +45,7 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( let mut cp = InternedConstantPool::default(); let this_class_index = cp.add_class(class_name_jvm)?; + cp.set_resource_anchor(this_class_index); let super_class_index = cp.add_class(super_class_name_jvm)?; @@ -55,41 +59,38 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( let interface_index = cp.add_class(interface_name)?; interface_indices.push(interface_index); } - if !is_abstract { + if kind != oomir::ClassKind::Static && !is_abstract { let rust_copy_interface = "org/rustlang/runtime/RustCopy"; if seen_interfaces.insert(rust_copy_interface) { interface_indices.push(cp.add_class(rust_copy_interface)?); } } + let split_addresses = kind == oomir::ClassKind::Value && oomir::fields::split_borrows(&fields); + let physical_fields = if split_addresses { + oomir::fields::physical(&fields) + } else { + fields.clone() + }; let mut jvm_fields: Vec = Vec::new(); - for (field_name, field_ty) in &fields { + for (field_name, field_ty) in &physical_fields { let name_index = cp.add_utf8(field_name)?; let descriptor = field_ty.to_jvm_descriptor(); // Ensure this method exists on oomir::Type let descriptor_index = cp.add_utf8(&descriptor)?; let field = jvm::Field { - access_flags: FieldAccessFlags::PUBLIC, + access_flags: FieldAccessFlags::PUBLIC + | if field_name.starts_with("$rust$") { + FieldAccessFlags::SYNTHETIC + } else { + FieldAccessFlags::empty() + }, name_index, descriptor_index, field_type: oomir_type_to_ristretto_field_type(field_ty), // Use helper attributes: Vec::new(), }; jvm_fields.push(field); - if matches!(field_ty, Type::Pointer(_)) { - for offset_name in [ - oomir::relative_pointer_element_offset_field(field_name), - oomir::relative_pointer_byte_offset_field(field_name), - ] { - jvm_fields.push(jvm::Field { - access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::SYNTHETIC, - name_index: cp.add_utf8(offset_name)?, - descriptor_index: cp.add_utf8("J")?, - field_type: oomir_type_to_ristretto_field_type(&Type::I64), - attributes: Vec::new(), - }); - } - } breadcrumbs::log!( breadcrumbs::LogLevel::Info, "bytecode-gen", @@ -97,34 +98,80 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( ); } + if split_addresses { + jvm_fields.extend(fields::layout_metadata(&mut cp, &fields)?); + } + // Fielded Rust structs/enums must be initialized with all fields. Only genuinely // fieldless classes keep a no-args constructor. - let constructor = if fields.is_empty() { - create_default_constructor(&mut cp, super_class_index)? + let mut jvm_methods = Vec::new(); + if kind != oomir::ClassKind::Static { + jvm_methods.push(if fields.is_empty() { + create_default_constructor(&mut cp, super_class_index)? + } else { + create_field_constructor( + &mut cp, + this_class_index, + super_class_index, + &physical_fields, + )? + }); + if split_addresses { + jvm_methods.push(fields::constructor_bridge( + &mut cp, + this_class_index, + &fields, + )?); + } } else { - create_field_constructor(&mut cp, this_class_index, super_class_index, &fields)? - }; - let mut jvm_methods = vec![constructor]; - if fields - .iter() - .any(|(_, field_ty)| matches!(field_ty, Type::Pointer(_))) - { - jvm_methods.push(create_relative_pointer_field_constructor( - &mut cp, - this_class_index, - super_class_index, - &fields, - )?); + assert!( + fields.is_empty() && implements_interfaces.is_empty(), + "static helper has instance storage" + ); } - if !is_abstract { + if kind != oomir::ClassKind::Static && !is_abstract { jvm_methods.push(create_managed_copy_method( &mut cp, this_class_index, class_name_jvm, &fields, + split_addresses, + context, )?); } let mut class_attributes = Vec::new(); + if matches!(kind, oomir::ClassKind::Value | oomir::ClassKind::Static) { + class_attributes.push(Attribute::Unknown { + name_index: cp.add_utf8(jvm::summary::PRIVATE_ATTRIBUTE)?, + info: Vec::new(), + }); + } + if let Some(info) = super::shapes::carrier_recipe( + kind, + &fields, + &methods, + implements_interfaces, + super_class_name_jvm, + is_abstract, + |ty| object_equality(module, ty), + ) + .or_else(|| { + super::enum_shapes::variant( + class_name_jvm, + kind, + &fields, + &methods, + implements_interfaces, + super_class_name_jvm, + is_abstract, + module, + ) + }) { + class_attributes.push(Attribute::Unknown { + name_index: cp.add_utf8(jvm::summary::CARRIER_ATTRIBUTE)?, + info, + }); + } let mut bootstrap_methods: Vec = Vec::new(); let mut next_factory = 0; @@ -140,7 +187,6 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( next_factory: &mut next_factory, owner: class_name_jvm, kind: body::BodyOwner::Class, - relative_methods: relative_static_methods, debug: debug_info, context, } @@ -202,7 +248,6 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( method_name, recipe, module, - relative_static_methods, false, )?); } @@ -255,6 +300,7 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( this_class_idx, field_name, field_ty, + split_addresses, )?; } @@ -293,19 +339,20 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( field_ty, } => { let method_desc = format!("(){}", field_ty.to_jvm_descriptor()); - let field_ref = cp.add_field_ref( + let mut code = vec![Instruction::Aload_0]; + fields::load( + &mut cp, + &mut code, this_class_index, field_name, - &field_ty.to_jvm_descriptor(), + field_ty, + split_addresses, )?; + code.push(return_instruction_for_type(field_ty)); let code_attribute = code_attribute_for_descriptor( &mut cp, 1, - vec![ - Instruction::Aload_0, - Instruction::Getfield(field_ref), - return_instruction_for_type(field_ty), - ], + code, &method_desc, false, Some(class_name_jvm), @@ -394,6 +441,7 @@ pub(in crate::lower2) fn create_data_type_classfile_for_class( }); } + output.resources(registry, &mut cp)?; let class_file = ClassFile { code_source_url: None, version: Version::Java8 { minor: 0 }, diff --git a/src/lower2/jvm_gen/constructors.rs b/src/lower2/jvm_gen/constructors.rs index b67be78f..28cc1335 100644 --- a/src/lower2/jvm_gen/constructors.rs +++ b/src/lower2/jvm_gen/constructors.rs @@ -101,93 +101,23 @@ pub(super) fn create_field_constructor( }) } -pub(super) fn create_relative_pointer_field_constructor( - cp: &mut InternedConstantPool, - this_class_index: u16, - super_class_index: u16, - fields: &[(String, Type)], -) -> jvm::Result { - let descriptor = format!( - "({})V", - fields - .iter() - .map(|(_, ty)| { - if matches!(ty, Type::Pointer(_)) { - format!("{}JJ", ty.to_jvm_descriptor()) - } else { - ty.to_jvm_descriptor() - } - }) - .collect::() - ); - let super_init = cp.add_method_ref(super_class_index, "", "()V")?; - let mut instructions = vec![Instruction::Aload_0, Instruction::Invokespecial(super_init)]; - let mut next_local = 1u16; - let mut parameters = Vec::new(); - - for (field_name, field_ty) in fields { - let field = - cp.add_field_ref(this_class_index, field_name, &field_ty.to_jvm_descriptor())?; - instructions.push(Instruction::Aload_0); - instructions.push(get_load_instruction(field_ty, next_local)?); - instructions.push(Instruction::Putfield(field)); - parameters.push(jvm::attributes::MethodParameter { - name_index: cp.add_utf8(field_name)?, - access_flags: MethodAccessFlags::empty(), - }); - next_local += get_type_size(field_ty); - - if matches!(field_ty, Type::Pointer(_)) { - for offset_name in [ - oomir::relative_pointer_element_offset_field(field_name), - oomir::relative_pointer_byte_offset_field(field_name), - ] { - let offset = cp.add_field_ref(this_class_index, &offset_name, "J")?; - instructions.push(Instruction::Aload_0); - instructions.push(get_load_instruction(&Type::I64, next_local)?); - instructions.push(Instruction::Putfield(offset)); - parameters.push(jvm::attributes::MethodParameter { - name_index: cp.add_utf8(offset_name)?, - access_flags: MethodAccessFlags::SYNTHETIC, - }); - next_local += 2; - } - } - } - instructions.push(Instruction::Return); - - Ok(jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::SYNTHETIC, - name_index: cp.add_utf8("")?, - descriptor_index: cp.add_utf8(&descriptor)?, - attributes: vec![ - code_attribute_for_descriptor( - cp, - next_local, - instructions, - &descriptor, - false, - None, - "", - )?, - Attribute::MethodParameters { - name_index: cp.add_utf8("MethodParameters")?, - parameters, - }, - ], - }) -} - pub(super) fn create_managed_copy_method( cp: &mut InternedConstantPool, this_class_index: u16, class_name: &str, fields: &[(String, Type)], + split: bool, + context: &oomir::construct::Context, ) -> jvm::Result { let descriptor = "()Ljava/lang/Object;"; + let physical = if split { + oomir::fields::physical(fields) + } else { + fields.to_vec() + }; let constructor_descriptor = format!( "({})V", - fields + physical .iter() .map(|(_, ty)| ty.to_jvm_descriptor()) .collect::() @@ -203,33 +133,42 @@ pub(super) fn create_managed_copy_method( let mut instructions = vec![Instruction::New(this_class_index), Instruction::Dup]; for (field_name, field_ty) in fields { + if let Some(parts) = oomir::fields::components(field_name, field_ty).filter(|_| split) { + for (name, ty) in parts { + let field = cp.add_field_ref(this_class_index, &name, &ty.to_jvm_descriptor())?; + instructions.extend([Instruction::Aload_0, Instruction::Getfield(field)]); + } + continue; + } let field = cp.add_field_ref(this_class_index, field_name, &field_ty.to_jvm_descriptor())?; instructions.push(Instruction::Aload_0); instructions.push(Instruction::Getfield(field)); - if matches!(field_ty, Type::Pointer(_)) { - for offset_name in [ - oomir::relative_pointer_element_offset_field(field_name), - oomir::relative_pointer_byte_offset_field(field_name), - ] { - let offset = cp.add_field_ref(this_class_index, offset_name, "J")?; - instructions.push(Instruction::Aload_0); - instructions.push(Instruction::Getfield(offset)); - } - let materialize = cp.add_method_ref( - pointer_class, - "materializeRelative", - &format!("(L{};JJ)L{};", oomir::POINTER_CLASS, oomir::POINTER_CLASS), - )?; - instructions.push(Instruction::Invokestatic(materialize)); - } else if field_ty.is_jvm_reference_type() { - instructions.push(Instruction::Invokestatic(copy_managed_value)); + if !matches!(field_ty, Type::Pointer(_)) && field_ty.is_jvm_reference_type() { + let direct = match field_ty { + Type::Class(owner) if context.direct_copy(owner) => Some(owner), + _ => None, + }; + let branch = if let Some(owner) = direct { + let owner = cp.add_class(owner)?; + let copy = cp.add_method_ref(owner, "rustCopy", "()Ljava/lang/Object;")?; + instructions.push(Instruction::Dup); + let branch = instructions.len(); + instructions.extend([Instruction::Ifnull(0), Instruction::Invokevirtual(copy)]); + Some(branch) + } else { + instructions.push(Instruction::Invokestatic(copy_managed_value)); + None + }; instructions.extend(get_cast_instructions( "rustCopy", &object_type, field_ty, cp, )?); + if let Some(branch) = branch { + instructions[branch] = Instruction::Ifnull(instructions.len() as u16); + } } } instructions.push(Instruction::Invokespecial(constructor)); diff --git a/src/lower2/jvm_gen/enum_equality.rs b/src/lower2/jvm_gen/enum_equality.rs index 38ff41db..dc5b3dd0 100644 --- a/src/lower2/jvm_gen/enum_equality.rs +++ b/src/lower2/jvm_gen/enum_equality.rs @@ -114,6 +114,7 @@ fn equality_code( runtime_type_idx, field_name, field_ty, + matches!(module.data_type(&variant.runtime_type), Some(oomir::DataType::Class { kind: oomir::ClassKind::Value, fields, .. }) if oomir::fields::split_borrows(fields)), )?; } } diff --git a/src/lower2/jvm_gen/enum_equality/tests.rs b/src/lower2/jvm_gen/enum_equality/tests.rs index cbd10bc3..55b2757c 100644 --- a/src/lower2/jvm_gen/enum_equality/tests.rs +++ b/src/lower2/jvm_gen/enum_equality/tests.rs @@ -12,7 +12,6 @@ fn large_enum_equality_verifies_and_executes() { suppressed_data_types: HashSet::default(), shared_data_types: None, shared_context: None, - relative_static_methods: Default::default(), external_interfaces: HashSet::default(), statics: HashMap::default(), }; diff --git a/src/lower2/jvm_gen/enum_shapes.rs b/src/lower2/jvm_gen/enum_shapes.rs new file mode 100644 index 00000000..962d76ab --- /dev/null +++ b/src/lower2/jvm_gen/enum_shapes.rs @@ -0,0 +1,284 @@ +//! Private enum storage shares proven shapes. Java exports, codecs, and custom methods keep nominal +//! representations. +use super::*; + +fn reference(recipe: &mut Vec, name: &str) { + recipe.extend_from_slice(format!("0:L{name};;").as_bytes()); +} + +fn fields(recipe: &mut Vec, fields: &[(String, Type)], module: &oomir::Module) -> Option<()> { + recipe.extend(super::shapes::carrier_recipe( + oomir::ClassKind::Value, + fields, + &HashMap::default(), + &[], + "java/lang/Object", + false, + |ty| super::enums::object_equality(module, ty), + )?); + Some(()) +} + +pub(super) fn variant( + name: &str, + kind: oomir::ClassKind, + fields_: &[(String, Type)], + methods: &HashMap, + interfaces: &[String], + superclass: &str, + abstract_class: bool, + module: &oomir::Module, +) -> Option> { + if kind != oomir::ClassKind::Value + || abstract_class + || superclass != "java/lang/Object" + || interfaces.len() != 1 + || methods.len() != 1 + || !matches!( + module.data_type(&interfaces[0]), + Some(oomir::DataType::Interface { is_enum: true, .. }) + ) + { + return None; + } + let DataTypeMethod::SimpleConstantReturn(Type::I64, Some(oomir::Constant::I64(tag))) = + methods.get(oomir::ENUM_TAG_METHOD)? + else { + return None; + }; + let mut recipe = format!( + "carrier-v2;variant-v1;{};{tag};", + jvm::names::inner_name(name) + ) + .into_bytes(); + reference(&mut recipe, &interfaces[0]); + fields(&mut recipe, fields_, module)?; + Some(recipe) +} + +pub(super) fn interface( + methods: &HashMap, + interfaces: &[String], + module: &oomir::Module, + subclasses: &[String], +) -> Option> { + if !interfaces.is_empty() || !methods.contains_key(oomir::ENUM_TAG_METHOD) { + return None; + } + let mut recipe = b"carrier-v2;enum-v1;".to_vec(); + // Kotlin requires the empty Context variant and Poll nested-class names. Dead helper removal + // must preserve them. + for (empty, full) in [("None", "Some"), ("Pending", "Ready")] { + if let Some(variant) = subclasses + .iter() + .find(|n| jvm::names::inner_name(n) == empty) + && subclasses.iter().any(|n| jvm::names::inner_name(n) == full) + { + recipe.extend_from_slice(format!("nested={empty},{full};").as_bytes()); + reference(&mut recipe, variant); + } + } + let mut methods = methods.iter().collect::>(); + methods.sort_unstable_by_key(|(name, _)| *name); + for (name, method) in methods { + recipe.extend_from_slice(format!("method={name};").as_bytes()); + let (variants, compare_fields) = match method { + DataTypeMethod::SimpleConstantReturn(Type::I64, None) + if name == oomir::ENUM_TAG_METHOD => + { + recipe.extend_from_slice(b"abstract-tag;"); + continue; + } + DataTypeMethod::AdtHelperMethod { kind } => match kind { + AdtHelperKind::EnumVariantIndex { + enum_class, + variants, + } => { + recipe.extend_from_slice(b"index;"); + reference(&mut recipe, enum_class); + (variants, false) + } + AdtHelperKind::StaticPartialEqEnum { + enum_class, + variants, + } => { + recipe.extend_from_slice(b"eq;"); + reference(&mut recipe, enum_class); + (variants, true) + } + AdtHelperKind::EnumDiscriminant { + enum_class, + dispatch, + .. + } if dispatch.as_deref() == Some(oomir::ENUM_TAG_METHOD) => { + // The helper calls the receiver tag method. Each variant recipe supplies its + // own discriminant. + recipe.extend_from_slice(b"dispatch-tag;"); + reference(&mut recipe, enum_class); + continue; + } + AdtHelperKind::EnumIsVariant { + enum_class, + runtime_type, + } => { + recipe.extend_from_slice(b"is;"); + reference(&mut recipe, enum_class); + reference(&mut recipe, runtime_type); + continue; + } + _ => return None, + }, + _ => return None, + }; + for variant in variants { + if variant.transparent { + return None; + } + reference(&mut recipe, &variant.runtime_type); + if compare_fields { + fields(&mut recipe, &variant.fields, module)?; + } + } + } + Some(recipe) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> oomir::Module { + oomir::Module { + name: "test".into(), + source_file: None, + functions: HashMap::default(), + data_types: HashMap::default(), + suppressed_data_types: HashSet::default(), + shared_data_types: None, + shared_context: None, + external_interfaces: HashSet::default(), + statics: HashMap::default(), + } + } + + #[test] + fn dispatched_discriminants_do_not_depend_on_variant_layouts_or_values() { + let module = module(); + let recipe = |values| { + interface( + &HashMap::from_iter([ + ( + oomir::ENUM_TAG_METHOD.to_owned(), + DataTypeMethod::SimpleConstantReturn(Type::I64, None), + ), + ( + "_unionDiscriminant".into(), + DataTypeMethod::AdtHelperMethod { + kind: AdtHelperKind::EnumDiscriminant { + enum_class: "test/Enum".into(), + variants: Vec::new(), + values, + dispatch: Some(oomir::ENUM_TAG_METHOD.into()), + }, + }, + ), + ]), + &[], + &module, + &[], + ) + .unwrap() + }; + assert_eq!(recipe(vec![0, 1]), recipe(vec![-19, 101, 999])); + assert!( + String::from_utf8(recipe(vec![])) + .unwrap() + .contains("dispatch-tag;") + ); + } + + #[test] + fn recipes_retain_tags_and_exclude_exports_and_custom_methods() { + let mut module = module(); + module.data_types.insert( + "test/Enum".into(), + oomir::DataType::Interface { + methods: HashMap::default(), + interfaces: vec![], + is_enum: true, + }, + ); + let methods = |tag| { + HashMap::from_iter([( + oomir::ENUM_TAG_METHOD.to_owned(), + DataTypeMethod::SimpleConstantReturn(Type::I64, Some(oomir::Constant::I64(tag))), + )]) + }; + let recipe = |methods: &_, kind| { + variant( + "test/Enum$Value", + kind, + &[("value".into(), Type::I32)], + methods, + &["test/Enum".into()], + "java/lang/Object", + false, + &module, + ) + }; + let a = recipe(&methods(-19), oomir::ClassKind::Value).unwrap(); + let b = recipe(&methods(101), oomir::ClassKind::Value).unwrap(); + assert_ne!(a, b); + assert!(recipe(&methods(-19), oomir::ClassKind::JavaValue).is_none()); + let mut custom = methods(-19); + custom.insert( + "_writeUnionStorage".into(), + DataTypeMethod::SimpleConstantReturn(Type::Void, None), + ); + assert!(recipe(&custom, oomir::ClassKind::Value).is_none()); + let mut interface_methods = HashMap::from_iter([( + oomir::ENUM_TAG_METHOD.to_owned(), + DataTypeMethod::SimpleConstantReturn(Type::I64, None), + )]); + let ordinary = interface(&interface_methods, &[], &module, &[]).unwrap(); + let option = interface( + &interface_methods, + &[], + &module, + &["test/Enum$None".into(), "test/Enum$Some".into()], + ) + .unwrap(); + let other_payload = interface( + &interface_methods, + &[], + &module, + &["test/Enum$None".into(), "different/Payload$Some".into()], + ) + .unwrap(); + let poll = interface( + &interface_methods, + &[], + &module, + &["test/Enum$Pending".into(), "test/Enum$Ready".into()], + ) + .unwrap(); + assert_eq!( + option, other_payload, + "unused payloads must not constrain the protocol" + ); + assert_ne!(ordinary, option); + assert_ne!(option, poll); + assert!( + String::from_utf8(option) + .unwrap() + .contains("0:Ltest/Enum$None;;") + ); + assert!(interface(&interface_methods, &[], &module, &[]).is_some()); + assert!(interface(&interface_methods, &["test/Parent".into()], &module, &[]).is_none()); + interface_methods.insert( + "callback".into(), + DataTypeMethod::SimpleConstantReturn(Type::Void, None), + ); + assert!(interface(&interface_methods, &[], &module, &[]).is_none()); + } +} diff --git a/src/lower2/jvm_gen/enums.rs b/src/lower2/jvm_gen/enums.rs index bec6c1fc..df9fa45d 100644 --- a/src/lower2/jvm_gen/enums.rs +++ b/src/lower2/jvm_gen/enums.rs @@ -23,6 +23,29 @@ fn has_generated_eq(module: &oomir::Module, class_name: &str) -> bool { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ObjectEquality { + Object, + Class, + Interface, + Enum, +} + +pub(super) fn object_equality(module: &oomir::Module, ty: &Type) -> ObjectEquality { + let (Type::Class(name) | Type::Interface(name)) = ty else { + unreachable!("object equality needs a nominal object type") + }; + if !has_generated_eq(module, name) { + return ObjectEquality::Object; + } + match module.data_type(name) { + Some(oomir::DataType::Interface { is_enum: true, .. }) => ObjectEquality::Enum, + Some(oomir::DataType::Interface { .. }) => ObjectEquality::Interface, + _ if matches!(ty, Type::Interface(_)) => ObjectEquality::Interface, + _ => ObjectEquality::Class, + } +} + fn append_boolean_false_check(instructions: &mut Vec, false_fixups: &mut Vec) { false_fixups.push(instructions.len()); instructions.push(Instruction::Ifeq(0)); @@ -36,16 +59,28 @@ pub(super) fn append_field_equality_check( variant_class_idx: u16, field_name: &str, field_ty: &Type, + split_addresses: bool, ) -> jvm::Result<()> { - let field_ref = - cp.add_field_ref(variant_class_idx, field_name, &field_ty.to_jvm_descriptor())?; - instructions.push(Instruction::Aload_0); instructions.push(Instruction::Checkcast(variant_class_idx)); - instructions.push(Instruction::Getfield(field_ref)); + fields::load( + cp, + instructions, + variant_class_idx, + field_name, + field_ty, + split_addresses, + )?; instructions.push(Instruction::Aload_1); instructions.push(Instruction::Checkcast(variant_class_idx)); - instructions.push(Instruction::Getfield(field_ref)); + fields::load( + cp, + instructions, + variant_class_idx, + field_name, + field_ty, + split_addresses, + )?; match field_ty { Type::I64 | Type::U64 => { @@ -75,6 +110,15 @@ pub(super) fn append_field_equality_check( false_fixups.push(instructions.len()); instructions.push(Instruction::If_icmpne(0)); } + Type::TaggedI64 => { + let owner = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + instructions.push(Instruction::Invokestatic(cp.add_method_ref( + owner, + "optionEquals", + "(Lorg/rustlang/runtime/TaggedLong;Lorg/rustlang/runtime/TaggedLong;)Z", + )?)); + append_boolean_false_check(instructions, false_fixups); + } Type::Str => { let view_class = cp.add_class(oomir::UTF8_VIEW_CLASS)?; let descriptor = format!( @@ -93,43 +137,31 @@ pub(super) fn append_field_equality_check( instructions.push(Instruction::Invokevirtual(equals_ref)); append_boolean_false_check(instructions, false_fixups); } - Type::Class(class_name) if has_generated_eq(module, class_name) => { - let class_idx = cp.add_class(class_name)?; - if matches!( - module.data_type(class_name), - Some(oomir::DataType::Interface { is_enum: true, .. }) - ) { - let eq_desc = format!("(L{class_name};L{class_name};)Z"); - let eq_ref = cp.add_interface_method_ref(class_idx, "eq", &eq_desc)?; - instructions.push(Instruction::Invokestatic(eq_ref)); - } else if matches!( - module.data_type(class_name), - Some(oomir::DataType::Interface { .. }) - ) { - let eq_desc = format!("(L{class_name};)Z"); - let eq_ref = cp.add_interface_method_ref(class_idx, "eq", &eq_desc)?; - instructions.push(Instruction::Invokeinterface(eq_ref, 2)); - } else { - let eq_desc = format!("(L{class_name};)Z"); - let eq_ref = cp.add_method_ref(class_idx, "eq", &eq_desc)?; - instructions.push(Instruction::Invokevirtual(eq_ref)); - } - append_boolean_false_check(instructions, false_fixups); - } - Type::Interface(interface_name) if has_generated_eq(module, interface_name) => { - let interface_idx = cp.add_class(interface_name)?; - if matches!( - module.data_type(interface_name), - Some(oomir::DataType::Interface { is_enum: true, .. }) - ) { - let eq_desc = format!("(L{interface_name};L{interface_name};)Z"); - let eq_ref = cp.add_interface_method_ref(interface_idx, "eq", &eq_desc)?; - instructions.push(Instruction::Invokestatic(eq_ref)); - } else { - let eq_desc = format!("(L{interface_name};)Z"); - let eq_ref = cp.add_interface_method_ref(interface_idx, "eq", &eq_desc)?; - instructions.push(Instruction::Invokeinterface(eq_ref, 2)); - } + Type::Class(name) | Type::Interface(name) => { + let owner = cp.add_class(name)?; + let descriptor = format!("(L{name};)Z"); + let instruction = + match object_equality(module, field_ty) { + ObjectEquality::Enum => Instruction::Invokestatic( + cp.add_interface_method_ref(owner, "eq", format!("(L{name};L{name};)Z"))?, + ), + ObjectEquality::Interface => Instruction::Invokeinterface( + cp.add_interface_method_ref(owner, "eq", descriptor)?, + 2, + ), + ObjectEquality::Class => { + Instruction::Invokevirtual(cp.add_method_ref(owner, "eq", descriptor)?) + } + ObjectEquality::Object => { + let object = cp.add_class("java/lang/Object")?; + Instruction::Invokevirtual(cp.add_method_ref( + object, + "equals", + "(Ljava/lang/Object;)Z", + )?) + } + }; + instructions.push(instruction); append_boolean_false_check(instructions, false_fixups); } Type::Array(inner) if matches!(inner.as_ref(), Type::Pointer(_)) => { @@ -142,13 +174,6 @@ pub(super) fn append_field_equality_check( instructions.push(Instruction::Invokestatic(equals_ref)); append_boolean_false_check(instructions, false_fixups); } - Type::Class(_) | Type::Interface(_) => { - let object_class_idx = cp.add_class("java/lang/Object")?; - let equals_ref = - cp.add_method_ref(object_class_idx, "equals", "(Ljava/lang/Object;)Z")?; - instructions.push(Instruction::Invokevirtual(equals_ref)); - append_boolean_false_check(instructions, false_fixups); - } _ => { false_fixups.push(instructions.len()); instructions.push(Instruction::If_acmpne(0)); @@ -190,7 +215,28 @@ pub(super) fn create_enum_adt_helper_method( enum_class, variants, values, + dispatch, } => { + if let Some(dispatch) = dispatch { + let owner = cp.add_class(enum_class)?; + let method = cp.add_interface_method_ref(owner, dispatch, "()J")?; + return enum_helper_method( + cp, + owner_name, + method_name, + &format!("(L{enum_class};)J"), + 1, + vec![ + Instruction::Aload_0, + Instruction::Ifnonnull(4), + Instruction::Lconst_0, + Instruction::Lreturn, + Instruction::Aload_0, + Instruction::Invokeinterface(method, 1), + Instruction::Lreturn, + ], + ); + } let mut instructions = Vec::new(); for (variant, value) in variants.iter().zip(values) { instructions.push(Instruction::Aload_0); @@ -232,11 +278,29 @@ pub(super) fn create_enum_adt_helper_method( } }; + enum_helper_method( + cp, + owner_name, + method_name, + &descriptor, + max_locals, + instructions, + ) +} + +fn enum_helper_method( + cp: &mut InternedConstantPool, + owner_name: &str, + method_name: &str, + descriptor: &str, + max_locals: u16, + instructions: Vec, +) -> jvm::Result { let code = code_attribute_for_descriptor( cp, max_locals, instructions, - &descriptor, + descriptor, true, Some(owner_name), method_name, @@ -244,7 +308,7 @@ pub(super) fn create_enum_adt_helper_method( Ok(jvm::Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, name_index: cp.add_utf8(method_name)?, - descriptor_index: cp.add_utf8(&descriptor)?, + descriptor_index: cp.add_utf8(descriptor)?, attributes: vec![code], }) } diff --git a/src/lower2/jvm_gen/fields.rs b/src/lower2/jvm_gen/fields.rs new file mode 100644 index 00000000..d035ef52 --- /dev/null +++ b/src/lower2/jvm_gen/fields.rs @@ -0,0 +1,180 @@ +//! Logical field access at reflection and Java helper boundaries. +use super::*; + +pub(super) fn load( + cp: &mut InternedConstantPool, + code: &mut Vec, + owner: u16, + name: &str, + ty: &Type, + split: bool, +) -> jvm::Result<()> { + if split && matches!(ty, Type::TaggedI64) { + let names = jvm_compiler_core::jvm::abi::tagged_field_names(name); + let value = cp.add_field_ref(owner, &names[0], "J")?; + let tag = cp.add_field_ref(owner, &names[1], "J")?; + let carrier = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + code.extend([ + Instruction::Dup, + Instruction::Getfield(value), + Instruction::Dup2_x1, + Instruction::Pop2, + Instruction::Getfield(tag), + Instruction::Invokestatic(cp.add_method_ref( + carrier, + "of", + "(JJ)Lorg/rustlang/runtime/TaggedLong;", + )?), + ]); + } else if split && matches!(ty, Type::Pointer(_)) { + let root = cp.add_field_ref(owner, name, "Ljava/lang/Object;")?; + let offset = cp.add_field_ref(owner, &oomir::fields::displacement_name(name, ty), "J")?; + code.extend([ + Instruction::Dup, + Instruction::Getfield(root), + Instruction::Swap, + Instruction::Getfield(offset), + ]); + ty.materialize_address(cp, code)?; + } else if split && matches!(ty, Type::Slice(_) | Type::Str) { + let names = jvm_compiler_core::jvm::abi::view_field_names(name, matches!(ty, Type::Str)); + let root = cp.add_field_ref(owner, &names[0], "Ljava/lang/Object;")?; + let start = cp.add_field_ref(owner, &names[1], "I")?; + let length = cp.add_field_ref(owner, &names[2], "J")?; + let class = cp.add_class(if matches!(ty, Type::Str) { + oomir::UTF8_VIEW_CLASS + } else { + oomir::SLICE_VIEW_CLASS + })?; + let init = cp.add_method_ref(class, "", "(Ljava/lang/Object;IJ)V")?; + code.extend([ + Instruction::New(class), + Instruction::Dup_x1, + Instruction::Swap, + Instruction::Dup, + Instruction::Getfield(root), + Instruction::Swap, + Instruction::Dup, + Instruction::Getfield(start), + Instruction::Swap, + Instruction::Getfield(length), + Instruction::Invokespecial(init), + ]); + } else { + code.push(Instruction::Getfield(cp.add_field_ref( + owner, + name, + &ty.to_jvm_descriptor(), + )?)); + } + Ok(()) +} + +pub(super) fn constructor_bridge( + cp: &mut InternedConstantPool, + owner: u16, + fields: &[(String, Type)], +) -> jvm::Result { + let physical = oomir::fields::physical(fields); + let descriptor = format!( + "({})V", + fields + .iter() + .map(|(_, ty)| ty.to_jvm_descriptor()) + .collect::() + ); + let target = format!( + "({})V", + physical + .iter() + .map(|(_, ty)| ty.to_jvm_descriptor()) + .collect::() + ); + let target = cp.add_method_ref(owner, "", &target)?; + let mut code = vec![Instruction::Aload_0]; + let mut slot = 1; + let mut parameters = Vec::new(); + for (name, ty) in fields { + if matches!(ty, Type::TaggedI64) { + let carrier = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + for part in ["value", "tag"] { + code.extend([ + get_load_instruction(ty, slot)?, + Instruction::Invokestatic(cp.add_method_ref( + carrier, + part, + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?), + ]); + } + } else if matches!(ty, Type::Slice(_) | Type::Str) { + let runtime = cp.add_class("org/rustlang/runtime/RustField")?; + for (method, result) in [ + ("viewRoot", "Ljava/lang/Object;"), + ("viewStart", "I"), + ("viewLength", "J"), + ] { + let method = + cp.add_method_ref(runtime, method, format!("(Ljava/lang/Object;){result}"))?; + code.extend([ + get_load_instruction(ty, slot)?, + Instruction::Invokestatic(method), + ]); + } + } else { + code.push(get_load_instruction(ty, slot)?); + if matches!(ty, Type::Pointer(_)) { + code.push(Instruction::Lconst_0); + } + } + slot += get_type_size(ty); + parameters.push(jvm::attributes::MethodParameter { + name_index: cp.add_utf8(name)?, + access_flags: MethodAccessFlags::empty(), + }); + } + code.extend([Instruction::Invokespecial(target), Instruction::Return]); + Ok(jvm::Method { + access_flags: MethodAccessFlags::PUBLIC, + name_index: cp.add_utf8("")?, + descriptor_index: cp.add_utf8(&descriptor)?, + attributes: vec![ + code_attribute_for_descriptor(cp, slot, code, &descriptor, false, None, "")?, + Attribute::MethodParameters { + name_index: cp.add_utf8("MethodParameters")?, + parameters, + }, + ], + }) +} + +/// Each field has an exact-layout codec constant. Reflection uses it to reconstruct general address +/// carriers. +pub(super) fn layout_metadata( + cp: &mut InternedConstantPool, + fields: &[(String, Type)], +) -> jvm::Result> { + let mut metadata = Vec::new(); + for (name, ty) in fields { + let Type::Pointer(pointee) = ty else { continue }; + let Some(layout) = &pointee.layout else { + continue; + }; + let Some(codec) = &layout.codec else { continue }; + let name = jvm_compiler_core::jvm::abi::address_codec_field_name(name); + metadata.push(jvm::Field { + access_flags: FieldAccessFlags::PUBLIC + | FieldAccessFlags::STATIC + | FieldAccessFlags::FINAL + | FieldAccessFlags::SYNTHETIC, + name_index: cp.add_utf8(name)?, + descriptor_index: cp.add_utf8("Ljava/lang/String;")?, + field_type: oomir_type_to_ristretto_field_type(&Type::java_string()), + attributes: vec![Attribute::ConstantValue { + name_index: cp.add_utf8("ConstantValue")?, + constant_value_index: cp.add_name_string(codec)?, + }], + }); + } + Ok(metadata) +} diff --git a/src/lower2/jvm_gen/forward.rs b/src/lower2/jvm_gen/forward.rs index e5c69227..e48d5c23 100644 --- a/src/lower2/jvm_gen/forward.rs +++ b/src/lower2/jvm_gen/forward.rs @@ -1,7 +1,7 @@ //! Emit ABI forwarders directly. There is no computational OOMIR body and no //! optimizer pass: the recipe already describes straight-line stack code. use super::*; -use std::borrow::Cow; +use jvm_compiler_core::jvm::locals::LocalKind as Kind; pub(super) fn emit( cp: &mut InternedConstantPool, @@ -9,24 +9,16 @@ pub(super) fn emit( name: &str, recipe: &oomir::MethodForwarder, module: &oomir::Module, - relative_methods: &HashSet, interface: bool, ) -> jvm::Result> { - let signature = &recipe.signature; - let source_relative = signature.is_static && signature.supports_relative_pointer_abi(); - let emitted_signature = if source_relative { - Cow::Owned(signature.relative_pointer_abi_signature()) + let emitted = if module.component_method(owner, name) { + recipe.signature.component_signature() } else { - Cow::Borrowed(signature) + recipe.signature.clone() }; - let emitted_name = if source_relative { - Cow::Owned(format!("{name}{}", oomir::RELATIVE_POINTER_METHOD_SUFFIX)) - } else { - Cow::Borrowed(name) - }; - let (instructions, max_locals) = - body(cp, owner, recipe, module, relative_methods, source_relative)?; - let descriptor = emitted_signature.to_string(); + let signature = &emitted; + let (instructions, max_locals) = body(cp, owner, recipe, module, signature)?; + let descriptor = signature.to_string(); let code = code_attribute_for_descriptor( cp, max_locals, @@ -36,15 +28,6 @@ pub(super) fn emit( Some(owner), name, )?; - let mut parameters = Vec::new(); - for (name, ty) in emitted_signature.explicit_jvm_params() { - if ty.has_jvm_value() { - parameters.push(jvm::attributes::MethodParameter { - name_index: cp.add_utf8(name)?, - access_flags: MethodAccessFlags::empty(), - }); - } - } let static_body = (interface && !signature.is_static).then(|| code.clone()); let mut methods = vec![jvm::Method { access_flags: MethodAccessFlags::PUBLIC @@ -53,26 +36,11 @@ pub(super) fn emit( } else { MethodAccessFlags::empty() }, - name_index: cp.add_utf8(emitted_name.as_ref())?, + name_index: cp.add_utf8(name)?, descriptor_index: cp.add_utf8(&descriptor)?, - attributes: vec![ - code, - Attribute::MethodParameters { - name_index: cp.add_utf8("MethodParameters")?, - parameters, - }, - ], + attributes: vec![code], }]; - if source_relative { - methods.push(create_relative_pointer_bridge( - cp, - owner, - name, - signature, - MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - interface, - )?); - } else if let Some(code) = static_body { + if let Some(code) = static_body { // Enum-interface static dispatch must select this canonical body even // when the receiver has a same-named method on a nested enum. let mut signature = signature.clone(); @@ -97,8 +65,7 @@ fn body( owner: &str, recipe: &oomir::MethodForwarder, module: &oomir::Module, - relative_methods: &HashSet, - source_relative: bool, + source_signature: &Signature, ) -> jvm::Result<(Vec, u16)> { let target_signature = &recipe.target_signature; if recipe.signature.params.len() != target_signature.params.len() || !target_signature.is_static @@ -108,13 +75,19 @@ fn body( message: "incompatible canonical signature".into(), }); } - let target_relative = target_signature.supports_relative_pointer_abi() - && (!crate::lower1::naming::is_global_link_symbol_class(&recipe.target_owner) - || relative_methods.contains(&oomir::FunctionKey::new( - &recipe.target_owner, - &recipe.target_name, - target_signature, - ))); + let emitted_target = if module.component_method(&recipe.target_owner, &recipe.target_name) { + target_signature.component_signature() + } else { + target_signature.clone() + }; + let source_components = source_signature.params.len() != recipe.signature.params.len(); + let target_components = emitted_target.params.len() != target_signature.params.len(); + let parameter_slots = source_signature + .params + .iter() + .map(|(_, ty)| get_type_size(ty)) + .sum::(); + let mut max_locals = parameter_slots; let mut code = Vec::new(); let mut local = 0u16; for (index, ((_, source), (_, target))) in recipe @@ -125,6 +98,15 @@ fn body( .enumerate() { let receiver = !recipe.signature.is_static && index == 0; + let source_parts = source_components && !receiver && source.component_shape().is_some(); + let target_parts = target_components && target.component_shape().is_some(); + if source_parts && target_parts && source.component_shape() == target.component_shape() { + for ty in source.components().unwrap() { + code.push(get_load_instruction(&ty, local)?); + local += get_type_size(&ty); + } + continue; + } let actual_source = if receiver { &Type::Class(owner.to_owned()) } else { @@ -133,9 +115,43 @@ fn body( if !actual_source.has_jvm_value() { continue; } - code.push(get_load_instruction(actual_source, local)?); - let pointer_offsets = source_relative && matches!(source, Type::Pointer(_)); - local += get_type_size(actual_source); + if source_parts { + if matches!(source, Type::TaggedI64) { + let owner = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + code.extend([ + Instruction::New(owner), + Instruction::Dup, + Kind::Long.load(local), + Kind::Long.load(local + 2), + Instruction::Invokespecial(cp.add_method_ref(owner, "", "(JJ)V")?), + ]); + local += 4; + } else if matches!(source, Type::Pointer(_)) { + let size = source.address_plan(); + code.extend([Kind::Reference.load(local), Kind::Long.load(local + 1)]); + source.materialize_address(cp, &mut code)?; + local += 3; + } else { + let view_class = cp.add_class(if matches!(source, Type::Str) { + oomir::UTF8_VIEW_CLASS + } else { + oomir::SLICE_VIEW_CLASS + })?; + let init = cp.add_method_ref(view_class, "", "(Ljava/lang/Object;IJ)V")?; + code.extend([ + Instruction::New(view_class), + Instruction::Dup, + Kind::Reference.load(local), + Kind::Int.load(local + 1), + Kind::Long.load(local + 2), + Instruction::Invokespecial(init), + ]); + local += 4; + } + } else { + code.push(get_load_instruction(actual_source, local)?); + local += get_type_size(actual_source); + } if receiver && let Some(receiver) = &recipe.receiver { code.push(get_int_const_instr(cp, receiver.size)); load_constant(&mut code, cp, &receiver.codec)?; @@ -155,61 +171,61 @@ fn body( cp, )?); } - if target_relative && matches!(target, Type::Pointer(_)) { - if pointer_offsets { - code.push(get_load_instruction(&Type::I64, local)?); - code.push(get_load_instruction(&Type::I64, local + 2)?); - } else { - code.extend([Instruction::Lconst_0, Instruction::Lconst_0]); + if target_parts && matches!(target, Type::Pointer(_)) { + code.push(Instruction::Lconst_0); + } else if target_parts && matches!(target, Type::TaggedI64) { + max_locals = parameter_slots + 1; + code.push(Kind::Reference.store(parameter_slots)); + let owner = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + for part in ["value", "tag"] { + code.extend([ + Kind::Reference.load(parameter_slots), + Instruction::Invokestatic(cp.add_method_ref( + owner, + part, + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?), + ]); + } + } else if target_parts { + max_locals = parameter_slots + 1; + code.push(Kind::Reference.store(parameter_slots)); + for index in 0..3 { + code.push(Kind::Reference.load(parameter_slots)); + code.push(jvm_compiler_core::jvm::abi::view_part_access(cp, index)?); } - } else if pointer_offsets { - code.push(get_load_instruction(&Type::I64, local)?); - code.push(get_load_instruction(&Type::I64, local + 2)?); - let class = cp.add_class(oomir::POINTER_CLASS)?; - let method = cp.add_method_ref( - class, - "materializeRelative", - "(Lorg/rustlang/runtime/Pointer;JJ)Lorg/rustlang/runtime/Pointer;", - )?; - code.push(Instruction::Invokestatic(method)); - } - if pointer_offsets { - local += 4; } } - let signature = if target_relative { - Cow::Owned(target_signature.relative_pointer_abi_signature()) - } else { - Cow::Borrowed(target_signature) - }; - let name = if target_relative { - Cow::Owned(format!( - "{}{}", - recipe.target_name, - oomir::RELATIVE_POINTER_METHOD_SUFFIX - )) - } else { - Cow::Borrowed(recipe.target_name.as_str()) - }; + let source_return = source_signature.ret != recipe.signature.ret; + let target_return = emitted_target.ret != target_signature.ret; + let metadata = super::forward_returns::prepare( + cp, + &mut code, + source_return, + target_return, + parameter_slots, + &mut max_locals, + )?; let class = cp.add_class(&recipe.target_owner)?; let method = if matches!( module.data_type(&recipe.target_owner), Some(oomir::DataType::Interface { .. }) ) || module.external_interfaces.contains(&recipe.target_owner) { - cp.add_interface_method_ref(class, name.as_ref(), &signature.to_string())? + cp.add_interface_method_ref(class, &recipe.target_name, &emitted_target.to_string())? } else { - cp.add_method_ref(class, name.as_ref(), &signature.to_string())? + cp.add_method_ref(class, &recipe.target_name, &emitted_target.to_string())? }; code.push(Instruction::Invokestatic(method)); - if !target_signature.ret.same_jvm_type(&recipe.signature.ret) { - code.extend(get_cast_instructions( - &recipe.target_name, - &target_signature.ret, - &recipe.signature.ret, - cp, - )?); - } - code.push(return_instruction_for_type(&recipe.signature.ret)); - Ok((code, local)) + super::forward_returns::finish( + cp, + &mut code, + &recipe.signature.ret, + &target_signature.ret, + source_return, + target_return, + metadata, + &mut max_locals, + )?; + Ok((code, max_locals)) } diff --git a/src/lower2/jvm_gen/forward_returns.rs b/src/lower2/jvm_gen/forward_returns.rs new file mode 100644 index 00000000..3cb1e6dc --- /dev/null +++ b/src/lower2/jvm_gen/forward_returns.rs @@ -0,0 +1,182 @@ +//! Adapt borrowed return components at generated Java/Rust forwarding boundaries. +use super::*; +use jvm_compiler_core::jvm::locals::LocalKind as Kind; + +pub(super) fn prepare( + cp: &mut InternedConstantPool, + code: &mut Vec, + source: bool, + target: bool, + parameter_slots: u16, + locals: &mut u16, +) -> jvm::Result> { + if source { + let slot = parameter_slots - 1; + if target { + code.push(Kind::Reference.load(slot)); + } + Ok(Some(slot)) + } else if target { + let slot = *locals; + *locals += 1; + code.extend([ + get_int_const_instr(cp, 2), + Instruction::Newarray(jvm::attributes::ArrayType::Long), + Instruction::Dup, + Kind::Reference.store(slot), + ]); + Ok(Some(slot)) + } else { + Ok(None) + } +} + +pub(super) fn finish( + cp: &mut InternedConstantPool, + code: &mut Vec, + source: &Type, + target: &Type, + source_parts: bool, + target_parts: bool, + metadata: Option, + locals: &mut u16, +) -> jvm::Result<()> { + if source_parts && target_parts && source.component_shape() == target.component_shape() { + code.push(if matches!(source, Type::TaggedI64) { + Instruction::Lreturn + } else { + Instruction::Areturn + }); + return Ok(()); + } + let temporary = *locals; + *locals += if source_parts || target_parts { 2 } else { 0 }; + if target_parts { + materialize(cp, code, target, metadata.unwrap(), temporary)?; + } + if !target.same_jvm_type(source) { + code.extend(get_cast_instructions( + "return forwarder", + target, + source, + cp, + )?); + } + if source_parts { + code.push(Kind::Reference.store(temporary)); + let metadata = metadata.unwrap(); + match source { + Type::TaggedI64 => { + let owner = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + code.extend([ + Kind::Reference.load(metadata), + Instruction::Iconst_0, + Kind::Reference.load(temporary), + Instruction::Invokestatic(cp.add_method_ref( + owner, + "tag", + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?), + Instruction::Lastore, + Kind::Reference.load(temporary), + Instruction::Invokestatic(cp.add_method_ref( + owner, + "value", + "(Lorg/rustlang/runtime/TaggedLong;)J", + )?), + ]); + } + Type::Pointer(_) => { + code.extend([ + Kind::Reference.load(metadata), + Instruction::Iconst_0, + Instruction::Lconst_0, + Instruction::Lastore, + Kind::Reference.load(temporary), + ]); + } + Type::Slice(_) | Type::Str => { + for index in 0..2 { + code.extend([ + Kind::Reference.load(metadata), + get_int_const_instr(cp, index), + Kind::Reference.load(temporary), + jvm_compiler_core::jvm::abi::view_part_access(cp, index as usize + 1)?, + ]); + if index == 0 { + code.push(Instruction::I2l); + } + code.push(Instruction::Lastore); + } + code.extend([ + Kind::Reference.load(temporary), + jvm_compiler_core::jvm::abi::view_part_access(cp, 0)?, + ]); + } + _ => unreachable!(), + } + code.push(if matches!(source, Type::TaggedI64) { + Instruction::Lreturn + } else { + Instruction::Areturn + }); + } else { + code.push(return_instruction_for_type(source)); + } + Ok(()) +} + +fn materialize( + cp: &mut InternedConstantPool, + code: &mut Vec, + ty: &Type, + metadata: u16, + temporary: u16, +) -> jvm::Result<()> { + if matches!(ty, Type::TaggedI64) { + let owner = cp.add_class(oomir::TAGGED_LONG_CLASS)?; + code.extend([ + Kind::Long.store(temporary), + Instruction::New(owner), + Instruction::Dup, + Kind::Long.load(temporary), + Kind::Reference.load(metadata), + Instruction::Iconst_0, + Instruction::Laload, + Instruction::Invokespecial(cp.add_method_ref(owner, "", "(JJ)V")?), + ]); + } else if matches!(ty, Type::Pointer(_)) { + code.extend([ + Kind::Reference.load(metadata), + Instruction::Iconst_0, + Instruction::Laload, + ]); + let size = ty.address_plan(); + ty.materialize_address(cp, code)?; + } else { + let owner = cp.add_class(if matches!(ty, Type::Str) { + oomir::UTF8_VIEW_CLASS + } else { + oomir::SLICE_VIEW_CLASS + })?; + code.extend([ + Kind::Reference.store(temporary), + Instruction::New(owner), + Instruction::Dup, + Kind::Reference.load(temporary), + Kind::Reference.load(metadata), + Instruction::Iconst_0, + Instruction::Laload, + Instruction::L2i, + Kind::Reference.load(metadata), + Instruction::Iconst_1, + Instruction::Laload, + Instruction::Invokespecial(cp.add_method_ref( + owner, + "", + "(Ljava/lang/Object;IJ)V", + )?), + ]); + } + Ok(()) +} diff --git a/src/lower2/jvm_gen/function_handles.rs b/src/lower2/jvm_gen/function_handles.rs new file mode 100644 index 00000000..01ffd667 --- /dev/null +++ b/src/lower2/jvm_gen/function_handles.rs @@ -0,0 +1,61 @@ +//! One exact invocation body per function ABI, shared by cold code targets. +use super::*; + +/// Function interfaces depend on the physical call ABI. Separate descriptor components permit +/// storage aliases. +pub(super) fn carrier_recipe(signature: &Signature) -> Vec { + let mut recipe = String::from("carrier-v2;function-v1;"); + for (_, ty) in signature.explicit_jvm_params() { + if ty.has_jvm_value() { + recipe.push_str(&format!("param;0:{};", ty.to_jvm_descriptor())); + } + } + recipe.push_str(&format!("return;0:{};", signature.ret.to_jvm_descriptor())); + recipe.into_bytes() +} + +pub(super) fn bridge( + cp: &mut InternedConstantPool, + signature: &Signature, + bootstrap: &mut Vec, +) -> jvm::Result { + let descriptor = signature.to_jvm_descriptor_with_explicit_params(); + let bridge = format!("(Ljava/lang/invoke/MethodHandle;{}", &descriptor[1..]); + let mut code = vec![Instruction::Aload_0]; + let mut local = 1; + for (_, ty) in signature.explicit_jvm_params() { + if ty.has_jvm_value() { + code.push(get_load_instruction(ty, local)?); + local += get_type_size(ty); + } + } + let owner = cp.add_class("org/rustlang/runtime/FunctionCallSite")?; + let method = cp.add_method_ref( + owner, + "bootstrap", + concat!( + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;", + "Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;" + ), + )?; + let bootstrap_method_ref = cp.add_method_handle(jvm::ReferenceKind::InvokeStatic, method)?; + let index = u16::try_from(bootstrap.len())?; + bootstrap.push(BootstrapMethod { + bootstrap_method_ref, + arguments: vec![], + }); + let invoke = cp.add_invoke_dynamic(index, "call", &bridge)?; + code.extend([ + Instruction::Invokedynamic(invoke), + return_instruction_for_type(&signature.ret), + ]); + let body = code_attribute_for_descriptor(cp, local, code, &bridge, true, None, "$rust$invoke")?; + Ok(jvm::Method { + access_flags: MethodAccessFlags::PUBLIC + | MethodAccessFlags::STATIC + | MethodAccessFlags::SYNTHETIC, + name_index: cp.add_utf8("$rust$invoke")?, + descriptor_index: cp.add_utf8(bridge)?, + attributes: vec![body], + }) +} diff --git a/src/lower2/jvm_gen/interfaces.rs b/src/lower2/jvm_gen/interfaces.rs index 86ef16d6..01ddfedd 100644 --- a/src/lower2/jvm_gen/interfaces.rs +++ b/src/lower2/jvm_gen/interfaces.rs @@ -10,8 +10,9 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( subclasses: Vec, nest_host: Option, debug_info: DebugInfoOptions, - relative_static_methods: &HashSet, context: &oomir::construct::Context, + output: &mut crate::lower2::output::ClassOutput, + registry: &crate::lower2::EmittedClassRegistry, ) -> jvm::Result> { let source_files = methods .values() @@ -25,14 +26,48 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( let mut cp = InternedConstantPool::default(); let this_class_index = cp.add_class(interface_name_jvm)?; + cp.set_resource_anchor(this_class_index); // Interfaces always implicitly extend Object, and must specify it in the classfile let super_class_index = cp.add_class("java/lang/Object")?; let mut jvm_methods: Vec = Vec::new(); let mut class_attributes = Vec::new(); + // Public Java enum classes remain linker roots. Private carriers follow their users. + let private = interface_name_jvm.starts_with("org/rustlang/runtime/FnPtr_") + || (matches!( + module.data_type(interface_name_jvm), + Some(oomir::DataType::Interface { is_enum: true, .. }) + ) && !subclasses.is_empty() + && subclasses.iter().all(|name| { + matches!( + module.data_type(name), + Some(oomir::DataType::Class { + kind: oomir::ClassKind::Value, + .. + }) + ) + })); + if private { + class_attributes.push(Attribute::Unknown { + name_index: cp.add_utf8(jvm::summary::PRIVATE_ATTRIBUTE)?, + info: Vec::new(), + }); + if let Some(info) = + super::enum_shapes::interface(&methods, super_interfaces, module, &subclasses) + { + class_attributes.push(Attribute::Unknown { + name_index: cp.add_utf8(jvm::summary::CARRIER_ATTRIBUTE)?, + info, + }); + } + } let mut bootstrap_methods: Vec = Vec::new(); let mut next_factory = 0; + let share_function = methods.len() == 1 + && super_interfaces.is_empty() + && subclasses.is_empty() + && nest_host.is_none(); for (method_name, method) in methods { let method_name = method_name.as_str(); if let DataTypeMethod::Function(mut function) = method { @@ -44,7 +79,6 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( next_factory: &mut next_factory, owner: interface_name_jvm, kind: body::BodyOwner::Interface, - relative_methods: relative_static_methods, debug: debug_info, context, } @@ -64,6 +98,12 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( // Abstract interface signatures contain only their explicit // JVM parameters. There is no OOMIR function body here, so no // synthetic receiver local needs to be stripped. + let mut signature = signature.clone(); + if oomir::component_method(interface_name_jvm, method_name) { + // Abstract signatures contain no implicit receiver. + signature.is_static = true; + signature = signature.component_signature(); + } let descriptor = signature.to_jvm_descriptor_with_explicit_params(); jvm_methods.push(jvm::Method { access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::ABSTRACT, @@ -71,85 +111,18 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( descriptor_index: cp.add_utf8(&descriptor)?, attributes: Vec::new(), }); - if method_name == "call" && interface_name_jvm.starts_with("org/rustlang/runtime/FnPtr_") { - let mut explicit_signature = signature.clone(); - explicit_signature.is_static = true; - if explicit_signature.supports_relative_pointer_abi() { - let relative_signature = - explicit_signature.relative_pointer_abi_signature(); - let relative_descriptor = - relative_signature.to_jvm_descriptor_with_explicit_params(); - let call_descriptor = - explicit_signature.to_jvm_descriptor_with_explicit_params(); - let pointer_class = cp.add_class(oomir::POINTER_CLASS)?; - let materialize = cp.add_method_ref( - pointer_class, - "materializeRelative", - &format!("(L{};JJ)L{};", oomir::POINTER_CLASS, oomir::POINTER_CLASS), - )?; - let call_ref = cp.add_interface_method_ref( - this_class_index, - "call", - &call_descriptor, - )?; - - let mut instructions = vec![Instruction::Aload_0]; - let mut local = 1u16; - - let mut call_slots = 1u16; - for (_, ty) in &explicit_signature.params { - if !ty.has_jvm_value() { - continue; - } - if matches!(ty, Type::Pointer(_)) { - instructions.push(get_load_instruction(ty, local)?); - instructions.push(get_load_instruction(&Type::I64, local + 1)?); - instructions.push(get_load_instruction(&Type::I64, local + 3)?); - - instructions.push(Instruction::Invokestatic(materialize)); - local += 5; - - call_slots += 1; - } else { - let size = get_type_size(ty); - instructions.push(get_load_instruction(ty, local)?); - local += size; - - call_slots += size; - } - } - instructions.push(Instruction::Invokeinterface( - call_ref, - call_slots - .try_into() - .map_err(|_| jvm::Error::VerificationError { - context: format!( - "Relative function-pointer bridge {interface_name_jvm}" - ), - message: "interface call exceeds 255 JVM parameter slots" - .to_string(), - })?, - )); - instructions.push(return_instruction_for_type(&explicit_signature.ret)); - jvm_methods.push(jvm::Method { - access_flags: MethodAccessFlags::PUBLIC, - name_index: cp.add_utf8(&format!( - "call{}", - oomir::RELATIVE_POINTER_METHOD_SUFFIX - ))?, - descriptor_index: cp.add_utf8(&relative_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - local, - instructions, - &relative_descriptor, - false, - Some(interface_name_jvm), - "call$relative", - )?], + jvm_methods.push(super::function_handles::bridge( + &mut cp, + &signature, + &mut bootstrap_methods, + )?); + if share_function { + class_attributes.push(Attribute::Unknown { + name_index: cp.add_utf8(jvm::summary::CARRIER_ATTRIBUTE)?, + info: super::function_handles::carrier_recipe(&signature), }); } } @@ -184,7 +157,6 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( method_name, recipe, module, - relative_static_methods, true, )?); } @@ -271,6 +243,7 @@ pub(in crate::lower2) fn create_data_type_classfile_for_interface( }); } + output.resources(registry, &mut cp)?; let class_file = ClassFile { code_source_url: None, version: Version::Java8 { minor: 0 }, diff --git a/src/lower2/jvm_gen/records.rs b/src/lower2/jvm_gen/records.rs new file mode 100644 index 00000000..29f01368 --- /dev/null +++ b/src/lower2/jvm_gen/records.rs @@ -0,0 +1,76 @@ +//! Keep the object ABI available for indirect calls and existing entry points. +use super::*; + +pub(super) fn bridge( + cp: &mut InternedConstantPool, + context: &oomir::construct::Context, + owner: &str, + name: &str, + original: &Signature, + scalars: &Signature, +) -> jvm::Result { + let source = original.component_signature(); + let target = scalars.component_signature(); + let source_split = &source != original; + let mut code = Vec::new(); + let mut local: u16 = 0; + for (_, ty) in &original.params { + if let Some(fields) = context.scalar_fields(ty) { + let Type::Class(class) = ty else { + unreachable!() + }; + let class = cp.add_class(class)?; + for (name, ty) in fields { + code.push(get_load_instruction( + &Type::Class("java/lang/Object".into()), + local, + )?); + code.push(Instruction::Getfield(cp.add_field_ref( + class, + name, + ty.to_jvm_descriptor(), + )?)); + } + local += 1; + } else if let Some(parts) = ty.components().filter(|_| source_split) { + for part in parts { + code.push(get_load_instruction(&part, local)?); + local += get_type_size(&part); + } + } else if ty.has_jvm_value() { + code.push(get_load_instruction(ty, local)?); + local += get_type_size(ty); + } + } + if source.ret != original.ret { + code.push(get_load_instruction( + &Type::Class("java/lang/Object".into()), + local, + )?); + local += 1; + } + let class = cp.add_class(owner)?; + let method = cp.add_method_ref( + class, + format!("{name}{}", oomir::construct::RECORD_ENTRY), + target.to_string(), + )?; + code.push(Instruction::Invokestatic(method)); + code.push(return_instruction_for_type(&source.ret)); + let descriptor = source.to_string(); + let attributes = vec![code_attribute_for_descriptor( + cp, + local, + code, + &descriptor, + true, + Some(owner), + name, + )?]; + Ok(jvm::Method { + access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, + name_index: cp.add_utf8(name)?, + descriptor_index: cp.add_utf8(descriptor)?, + attributes, + }) +} diff --git a/src/lower2/jvm_gen/shapes.rs b/src/lower2/jvm_gen/shapes.rs new file mode 100644 index 00000000..0c6534d4 --- /dev/null +++ b/src/lower2/jvm_gen/shapes.rs @@ -0,0 +1,231 @@ +//! Private storage classes share complete layout recipes. Java exports, callbacks, and custom +//! methods retain their identity. +use super::enums::ObjectEquality; +use super::*; + +pub(super) fn carrier_recipe( + kind: oomir::ClassKind, + fields: &[(String, Type)], + methods: &HashMap, + interfaces: &[String], + superclass: &str, + abstract_class: bool, + equality: impl Fn(&Type) -> ObjectEquality, +) -> Option> { + if kind != oomir::ClassKind::Value + || abstract_class + || !interfaces.is_empty() + || superclass != "java/lang/Object" + || methods.iter().any(|(name, method)| { + name != "eq" + || !matches!(method, DataTypeMethod::AdtHelperMethod { + kind: AdtHelperKind::PartialEqClass { fields: compared } + } if compared.iter().filter(|(_, ty)| ty.has_jvm_value()) + .eq(fields.iter().filter(|(_, ty)| ty.has_jvm_value()))) + }) + { + return None; + } + let mut recipe = String::from("carrier-v2;"); + recipe.push_str(if methods.is_empty() { "plain;" } else { "eq;" }); + for (name, ty) in fields.iter().filter(|(_, ty)| ty.has_jvm_value()) { + // Copy helpers dispatch through the actual value type. Equality also requires the nominal + // helper and dispatch kind. + match ty { + ty if ty.is_jvm_primitive() => recipe.push_str("scalar;"), + Type::Pointer(_) | Type::Slice(_) | Type::Str => recipe.push_str("borrow;"), + Type::TaggedI64 => recipe.push_str("tagged-long;"), + Type::Class(_) | Type::Interface(_) => { + recipe.push_str(&format!("object:{:?};", equality(ty))); + } + Type::Array(inner) if matches!(inner.as_ref(), Type::Pointer(_)) => { + recipe.push_str("address-array;"); + } + Type::Array(_) => recipe.push_str("array;"), + _ => return None, + } + recipe.push_str(&format!("{}:{name}{};", name.len(), ty.to_jvm_descriptor())); + if let Type::Pointer(p) = ty + && let Some(layout) = &p.layout + { + recipe.push_str(&format!("layout:{}:{:?};", layout.size, layout.codec)); + } + } + for (name, ty) in oomir::fields::physical(fields) + .into_iter() + .filter(|(_, ty)| ty.has_jvm_value()) + { + recipe.push_str(&format!("{}:{name}{};", name.len(), ty.to_jvm_descriptor())); + } + Some(recipe.into_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn erased_fields_do_not_change_the_storage_or_equality_recipe() { + let fields = vec![("field0".into(), Type::I32)]; + let mut with_erased = fields.clone(); + with_erased.push(("field1".into(), Type::Unit)); + let methods = HashMap::from_iter([( + "eq".into(), + DataTypeMethod::AdtHelperMethod { + kind: AdtHelperKind::PartialEqClass { + fields: with_erased.clone(), + }, + }, + )]); + let recipe = |fields: &_, methods: &_| { + carrier_recipe( + oomir::ClassKind::Value, + fields, + methods, + &[], + "java/lang/Object", + false, + |_| ObjectEquality::Object, + ) + }; + let with_eq = recipe(&fields, &methods); + assert!(with_eq.is_some()); + assert_eq!(with_eq, recipe(&with_erased, &methods)); + assert_eq!( + recipe(&fields, &HashMap::default()), + recipe(&with_erased, &HashMap::default()) + ); + assert_ne!(with_eq, recipe(&fields, &HashMap::default())); + let wrong_fields = vec![("different".into(), Type::I32)]; + assert!(recipe(&wrong_fields, &methods).is_none()); + } + + fn recipe(ty: Type) -> Option> { + recipe_with_equality(ty, ObjectEquality::Object) + } + + fn recipe_with_equality(ty: Type, equality: ObjectEquality) -> Option> { + let fields = vec![("value".into(), ty)]; + let methods = HashMap::from_iter([( + "eq".into(), + DataTypeMethod::AdtHelperMethod { + kind: AdtHelperKind::PartialEqClass { + fields: fields.clone(), + }, + }, + )]); + carrier_recipe( + oomir::ClassKind::Value, + &fields, + &methods, + &[], + "java/lang/Object", + false, + |_| equality, + ) + } + + #[test] + fn physical_shape_sharing_keeps_address_layout_and_nominal_methods_separate() { + let exact = |codec: &str| { + Type::pointer(Type::Array(Box::new(Type::U8))) + .with_address_layout(8, Some(codec.into())) + }; + assert_ne!(recipe(exact("a#b#[B#8")), recipe(exact("a#c#[B#8"))); + assert_eq!( + recipe(Type::pointer(Type::I32)), + recipe(Type::pointer(Type::U32)) + ); + assert_ne!( + recipe(Type::pointer(Type::I32)), + recipe(Type::pointer(Type::I64)) + ); + assert_eq!( + recipe(Type::Slice(Box::new(Type::I32))), + recipe(Type::Slice(Box::new(Type::I64))) + ); + assert_ne!(recipe(Type::Str), recipe(Type::Slice(Box::new(Type::U8)))); + assert!(recipe(Type::Class("test/Payload".into())).is_some()); + assert_ne!( + recipe(Type::Class("test/Payload".into())), + recipe(Type::Class("test/Other".into())) + ); + assert_eq!( + recipe(Type::Array(Box::new(Type::I8))), + recipe(Type::Array(Box::new(Type::U8))) + ); + assert_ne!( + recipe(Type::Array(Box::new(Type::I8))), + recipe(Type::Array(Box::new(Type::I16))) + ); + assert!(recipe(Type::TaggedI64).is_some()); + assert!(recipe(Type::Array(Box::new(Type::Class("test/Element".into())))).is_some()); + assert_ne!( + recipe(Type::Array(Box::new(Type::pointer(Type::I32)))), + recipe(Type::Array(Box::new(Type::Class( + oomir::POINTER_CLASS.into() + )))), + "pointer arrays compare addresses, ordinary object arrays compare identity" + ); + for left in [ + ObjectEquality::Object, + ObjectEquality::Class, + ObjectEquality::Interface, + ObjectEquality::Enum, + ] { + for right in [ + ObjectEquality::Object, + ObjectEquality::Class, + ObjectEquality::Interface, + ObjectEquality::Enum, + ] { + let payload = Type::Class("test/Payload".into()); + assert_eq!( + recipe_with_equality(payload.clone(), left) + == recipe_with_equality(payload, right), + left == right + ); + } + } + let methods = HashMap::from_iter([( + "rustDrop".into(), + DataTypeMethod::SimpleConstantReturn(Type::Void, None), + )]); + assert!( + carrier_recipe( + oomir::ClassKind::Value, + &[], + &methods, + &[], + "java/lang/Object", + false, + |_| ObjectEquality::Object + ) + .is_none() + ); + assert!( + carrier_recipe( + oomir::ClassKind::JavaValue, + &[], + &HashMap::default(), + &[], + "java/lang/Object", + false, + |_| ObjectEquality::Object + ) + .is_none() + ); + assert!( + carrier_recipe( + oomir::ClassKind::Value, + &[], + &HashMap::default(), + &["test/Trait".into()], + "java/lang/Object", + false, + |_| ObjectEquality::Object + ) + .is_none() + ); + } +} diff --git a/src/lower2/jvm_gen/slices.rs b/src/lower2/jvm_gen/slices.rs deleted file mode 100644 index 2d7de7dd..00000000 --- a/src/lower2/jvm_gen/slices.rs +++ /dev/null @@ -1,487 +0,0 @@ -//! Native JVM slices emission. -use super::*; - -/// Builds the backend-owned runtime representation used for Rust slices. -pub(in crate::lower2) fn create_slice_view_classfile() -> jvm::Result> { - let mut cp = InternedConstantPool::default(); - let this_class = cp.add_class(oomir::SLICE_VIEW_CLASS)?; - let object_class = cp.add_class("java/lang/Object")?; - - let array_field = cp.add_field_ref(this_class, "array", "Ljava/lang/Object;")?; - let offset_field = cp.add_field_ref(this_class, "offset", "I")?; - let length_field = cp.add_field_ref(this_class, "length", "I")?; - let rust_length_field = cp.add_field_ref(this_class, "rustLength", "J")?; - let object_init = cp.add_method_ref(object_class, "", "()V")?; - - let constructor_descriptor = "(Ljava/lang/Object;II)V"; - let constructor = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC, - name_index: cp.add_utf8("")?, - descriptor_index: cp.add_utf8(constructor_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 4, - vec![ - Instruction::Aload_0, - Instruction::Invokespecial(object_init), - Instruction::Aload_0, - Instruction::Aload_1, - Instruction::Putfield(array_field), - Instruction::Aload_0, - Instruction::Iload_2, - Instruction::Putfield(offset_field), - Instruction::Aload_0, - Instruction::Iload_3, - Instruction::Putfield(length_field), - Instruction::Aload_0, - Instruction::Iload_3, - Instruction::I2l, - Instruction::Putfield(rust_length_field), - Instruction::Return, - ], - constructor_descriptor, - false, - Some(oomir::SLICE_VIEW_CLASS), - "", - )?], - }; - - let get_class = cp.add_method_ref(object_class, "getClass", "()Ljava/lang/Class;")?; - let class_class = cp.add_class("java/lang/Class")?; - let get_component_type = - cp.add_method_ref(class_class, "getComponentType", "()Ljava/lang/Class;")?; - let reflect_array_class = cp.add_class("java/lang/reflect/Array")?; - let new_array = cp.add_method_ref( - reflect_array_class, - "newInstance", - "(Ljava/lang/Class;I)Ljava/lang/Object;", - )?; - let system_class = cp.add_class("java/lang/System")?; - let array_copy = cp.add_method_ref( - system_class, - "arraycopy", - "(Ljava/lang/Object;ILjava/lang/Object;II)V", - )?; - let to_array_descriptor = "()Ljava/lang/Object;"; - let to_array = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::FINAL, - name_index: cp.add_utf8("toArray")?, - descriptor_index: cp.add_utf8(to_array_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 2, - vec![ - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Invokevirtual(get_class), - Instruction::Invokevirtual(get_component_type), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::Invokestatic(new_array), - Instruction::Astore_1, - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Aload_1, - Instruction::Iconst_0, - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::Invokestatic(array_copy), - Instruction::Aload_1, - Instruction::Areturn, - ], - to_array_descriptor, - false, - Some(oomir::SLICE_VIEW_CLASS), - "toArray", - )?], - }; - - let long_constructor_descriptor = "(Ljava/lang/Object;IJ)V"; - let long_constructor = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC, - name_index: cp.add_utf8("")?, - descriptor_index: cp.add_utf8(long_constructor_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 5, - vec![ - Instruction::Aload_0, - Instruction::Invokespecial(object_init), - Instruction::Aload_0, - Instruction::Aload_1, - Instruction::Putfield(array_field), - Instruction::Aload_0, - Instruction::Iload_2, - Instruction::Putfield(offset_field), - Instruction::Aload_0, - Instruction::Lload_3, - Instruction::L2i, - Instruction::Putfield(length_field), - Instruction::Aload_0, - Instruction::Lload_3, - Instruction::Putfield(rust_length_field), - Instruction::Return, - ], - long_constructor_descriptor, - false, - Some(oomir::SLICE_VIEW_CLASS), - "", - )?], - }; - - let standard_charsets = cp.add_class("java/nio/charset/StandardCharsets")?; - let utf8 = cp.add_field_ref(standard_charsets, "UTF_8", "Ljava/nio/charset/Charset;")?; - let string_class = cp.add_class("java/lang/String")?; - let pointer_class = cp.add_class(oomir::POINTER_CLASS)?; - let string_view = cp.add_method_ref( - pointer_class, - "stringView", - "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/Object;", - )?; - let slice_view_name = cp.add_string(oomir::SLICE_VIEW_CLASS)?; - let from_string_descriptor = format!("(Ljava/lang/String;)L{};", oomir::SLICE_VIEW_CLASS); - let from_string = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("fromString")?, - descriptor_index: cp.add_utf8(&from_string_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![ - Instruction::Aload_0, - Instruction::Ldc_w(slice_view_name), - Instruction::Invokestatic(string_view), - Instruction::Checkcast(this_class), - Instruction::Areturn, - ], - &from_string_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "fromString", - )?], - }; - - let slice_to_byte_array = cp.add_method_ref( - pointer_class, - "sliceToByteArray", - "(Ljava/lang/Object;II)[B", - )?; - let string_from_bytes = - cp.add_method_ref(string_class, "", "([BLjava/nio/charset/Charset;)V")?; - let to_utf8_string_descriptor = format!("(L{};)Ljava/lang/String;", oomir::SLICE_VIEW_CLASS); - let to_utf8_string = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("toUtf8String")?, - descriptor_index: cp.add_utf8(&to_utf8_string_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 2, - vec![ - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::Invokestatic(slice_to_byte_array), - Instruction::Astore_1, - Instruction::New(string_class), - Instruction::Dup, - Instruction::Aload_1, - Instruction::Getstatic(utf8), - Instruction::Invokespecial(string_from_bytes), - Instruction::Areturn, - ], - &to_utf8_string_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "toUtf8String", - )?], - }; - - let character_class = cp.add_class("java/lang/Character")?; - let to_chars = cp.add_method_ref(character_class, "toChars", "(I)[C")?; - let string_value_of = cp.add_method_ref(string_class, "valueOf", "([C)Ljava/lang/String;")?; - let from_string_ref = cp.add_method_ref(this_class, "fromString", &from_string_descriptor)?; - let utf8_class = cp.add_class(oomir::UTF8_VIEW_CLASS)?; - let utf8_constructor = cp.add_method_ref(utf8_class, "", "(Ljava/lang/Object;II)V")?; - let encode_utf8_descriptor = format!( - "(IL{};)L{};", - oomir::SLICE_VIEW_CLASS, - oomir::UTF8_VIEW_CLASS - ); - let encode_utf8 = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("encodeUtf8")?, - descriptor_index: cp.add_utf8(&encode_utf8_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 4, - vec![ - Instruction::Iload_0, - Instruction::Invokestatic(to_chars), - Instruction::Invokestatic(string_value_of), - Instruction::Astore_2, - Instruction::Aload_2, - Instruction::Invokestatic(from_string_ref), - Instruction::Astore_3, - Instruction::Aload_3, - Instruction::Getfield(array_field), - Instruction::Aload_3, - Instruction::Getfield(offset_field), - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Aload_3, - Instruction::Getfield(length_field), - Instruction::Invokestatic(array_copy), - Instruction::New(utf8_class), - Instruction::Dup, - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Aload_3, - Instruction::Getfield(length_field), - Instruction::Invokespecial(utf8_constructor), - Instruction::Areturn, - ], - &encode_utf8_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "encodeUtf8", - )?], - }; - - let slice_get_object = cp.add_method_ref( - pointer_class, - "sliceGetObject", - "(Ljava/lang/Object;I)Ljava/lang/Object;", - )?; - let objects_class = cp.add_class("java/util/Objects")?; - let objects_equals = cp.add_method_ref( - objects_class, - "equals", - "(Ljava/lang/Object;Ljava/lang/Object;)Z", - )?; - let starts_with_descriptor = format!( - "(L{};L{};)Z", - oomir::SLICE_VIEW_CLASS, - oomir::SLICE_VIEW_CLASS - ); - let starts_with = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("startsWith")?, - descriptor_index: cp.add_utf8(&starts_with_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 3, - vec![ - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::If_icmpgt(31), - Instruction::Iconst_0, - Instruction::Istore_2, - Instruction::Iload_2, - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::If_icmpge(29), - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_object), - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_object), - Instruction::Invokestatic(objects_equals), - Instruction::Ifeq(31), - Instruction::Iinc(2, 1), - Instruction::Goto(7), - Instruction::Iconst_1, - Instruction::Ireturn, - Instruction::Iconst_0, - Instruction::Ireturn, - ], - &starts_with_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "startsWith", - )?], - }; - - // Byte slices may be backed either directly by a JVM byte array or by a - // runtime Pointer carrying a codec (notably `[MaybeUninit]` during - // optimised UTF-8 construction). Pointer.sliceGetI8 handles both forms. - let slice_get_i8 = cp.add_method_ref(pointer_class, "sliceGetI8", "(Ljava/lang/Object;I)B")?; - let starts_with_i8 = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("startsWithI8")?, - descriptor_index: cp.add_utf8(&starts_with_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 3, - vec![ - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::If_icmpgt(30), - Instruction::Iconst_0, - Instruction::Istore_2, - Instruction::Iload_2, - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::If_icmpge(28), - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i8), - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i8), - Instruction::If_icmpne(30), - Instruction::Iinc(2, 1), - Instruction::Goto(7), - Instruction::Iconst_1, - Instruction::Ireturn, - Instruction::Iconst_0, - Instruction::Ireturn, - ], - &starts_with_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "startsWithI8", - )?], - }; - - // Integer slice views can be backed by either a JVM primitive array or - // encoded Rust allocation storage. Use the typed accessor so equality is - // independent of that physical representation. - let slice_get_i32 = - cp.add_method_ref(pointer_class, "sliceGetI32", "(Ljava/lang/Object;I)I")?; - let starts_with_i32 = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("startsWithI32")?, - descriptor_index: cp.add_utf8(&starts_with_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 3, - vec![ - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::If_icmpgt(30), - Instruction::Iconst_0, - Instruction::Istore_2, - Instruction::Iload_2, - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::If_icmpge(28), - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i32), - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i32), - Instruction::If_icmpne(30), - Instruction::Iinc(2, 1), - Instruction::Goto(7), - Instruction::Iconst_1, - Instruction::Ireturn, - Instruction::Iconst_0, - Instruction::Ireturn, - ], - &starts_with_descriptor, - true, - Some(oomir::SLICE_VIEW_CLASS), - "startsWithI32", - )?], - }; - - let fields = vec![ - jvm::Field { - access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::FINAL, - name_index: cp.add_utf8("array")?, - descriptor_index: cp.add_utf8("Ljava/lang/Object;")?, - field_type: jvm::FieldType::Object("java/lang/Object".into()), - attributes: Vec::new(), - }, - jvm::Field { - access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::FINAL, - name_index: cp.add_utf8("offset")?, - descriptor_index: cp.add_utf8("I")?, - field_type: jvm::FieldType::Base(BaseType::Int), - attributes: Vec::new(), - }, - jvm::Field { - access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::FINAL, - name_index: cp.add_utf8("rustLength")?, - descriptor_index: cp.add_utf8("J")?, - field_type: jvm::FieldType::Base(BaseType::Long), - attributes: Vec::new(), - }, - jvm::Field { - access_flags: FieldAccessFlags::PUBLIC | FieldAccessFlags::FINAL, - name_index: cp.add_utf8("length")?, - descriptor_index: cp.add_utf8("I")?, - field_type: jvm::FieldType::Base(BaseType::Int), - attributes: Vec::new(), - }, - ]; - - let class_file = ClassFile { - code_source_url: None, - version: Version::Java8 { minor: 0 }, - constant_pool: cp.into_inner(), - access_flags: ClassAccessFlags::PUBLIC | ClassAccessFlags::SUPER, - this_class, - super_class: object_class, - interfaces: Vec::new(), - fields, - methods: vec![ - constructor, - long_constructor, - to_array, - from_string, - to_utf8_string, - encode_utf8, - starts_with, - starts_with_i8, - starts_with_i32, - ], - attributes: Vec::new(), - }; - verify_no_duplicate_constants(&class_file)?; - - let mut bytes = Vec::new(); - class_file.to_bytes(&mut bytes)?; - Ok(bytes) -} diff --git a/src/lower2/jvm_gen/strings.rs b/src/lower2/jvm_gen/strings.rs deleted file mode 100644 index bc5435c0..00000000 --- a/src/lower2/jvm_gen/strings.rs +++ /dev/null @@ -1,330 +0,0 @@ -//! Native JVM strings emission. -use super::*; - -/// Builds the UTF-8-valid specialization used for Rust `str` values. -pub(in crate::lower2) fn create_utf8_view_classfile() -> jvm::Result> { - let mut cp = InternedConstantPool::default(); - let this_class = cp.add_class(oomir::UTF8_VIEW_CLASS)?; - let slice_class = cp.add_class(oomir::SLICE_VIEW_CLASS)?; - let constructor_descriptor = "(Ljava/lang/Object;II)V"; - let slice_constructor = cp.add_method_ref(slice_class, "", constructor_descriptor)?; - let long_constructor_descriptor = "(Ljava/lang/Object;IJ)V"; - let long_slice_constructor = - cp.add_method_ref(slice_class, "", long_constructor_descriptor)?; - let array_field = cp.add_field_ref(slice_class, "array", "Ljava/lang/Object;")?; - let offset_field = cp.add_field_ref(slice_class, "offset", "I")?; - let length_field = cp.add_field_ref(slice_class, "length", "I")?; - let utf8_constructor = cp.add_method_ref(this_class, "", constructor_descriptor)?; - - let constructor = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC, - name_index: cp.add_utf8("")?, - descriptor_index: cp.add_utf8(constructor_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 4, - vec![ - Instruction::Aload_0, - Instruction::Aload_1, - Instruction::Iload_2, - Instruction::Iload_3, - Instruction::Invokespecial(slice_constructor), - Instruction::Return, - ], - constructor_descriptor, - false, - Some(oomir::UTF8_VIEW_CLASS), - "", - )?], - }; - - let long_constructor = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC, - name_index: cp.add_utf8("")?, - descriptor_index: cp.add_utf8(long_constructor_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 5, - vec![ - Instruction::Aload_0, - Instruction::Aload_1, - Instruction::Iload_2, - Instruction::Lload_3, - Instruction::Invokespecial(long_slice_constructor), - Instruction::Return, - ], - long_constructor_descriptor, - false, - Some(oomir::UTF8_VIEW_CLASS), - "", - )?], - }; - - let pointer_class = cp.add_class(oomir::POINTER_CLASS)?; - let string_view = cp.add_method_ref( - pointer_class, - "stringView", - "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/Object;", - )?; - let utf8_view_name = cp.add_string(oomir::UTF8_VIEW_CLASS)?; - let from_java_descriptor = format!("(Ljava/lang/String;)L{};", oomir::UTF8_VIEW_CLASS); - let from_java = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("fromJavaString")?, - descriptor_index: cp.add_utf8(&from_java_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![ - Instruction::Aload_0, - Instruction::Ldc_w(utf8_view_name), - Instruction::Invokestatic(string_view), - Instruction::Checkcast(this_class), - Instruction::Areturn, - ], - &from_java_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "fromJavaString", - )?], - }; - - let slice_to_string_descriptor = format!("(L{};)Ljava/lang/String;", oomir::SLICE_VIEW_CLASS); - let slice_to_string = - cp.add_method_ref(slice_class, "toUtf8String", &slice_to_string_descriptor)?; - let to_java_descriptor = format!("(L{};)Ljava/lang/String;", oomir::UTF8_VIEW_CLASS); - let to_java = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("toJavaString")?, - descriptor_index: cp.add_utf8(&to_java_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![ - Instruction::Aload_0, - Instruction::Invokestatic(slice_to_string), - Instruction::Areturn, - ], - &to_java_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "toJavaString", - )?], - }; - - let as_slice_descriptor = format!( - "(L{};)L{};", - oomir::UTF8_VIEW_CLASS, - oomir::SLICE_VIEW_CLASS - ); - let as_slice = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("asSlice")?, - descriptor_index: cp.add_utf8(&as_slice_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![Instruction::Aload_0, Instruction::Areturn], - &as_slice_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "asSlice", - )?], - }; - - let from_slice_descriptor = format!( - "(L{};)L{};", - oomir::SLICE_VIEW_CLASS, - oomir::UTF8_VIEW_CLASS - ); - let from_slice = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("fromSlice")?, - descriptor_index: cp.add_utf8(&from_slice_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![ - Instruction::New(this_class), - Instruction::Dup, - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::Invokespecial(utf8_constructor), - Instruction::Areturn, - ], - &from_slice_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "fromSlice", - )?], - }; - - let len_descriptor = format!("(L{};)J", oomir::UTF8_VIEW_CLASS); - let len = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("len")?, - descriptor_index: cp.add_utf8(&len_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 1, - vec![ - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::I2l, - Instruction::Lreturn, - ], - &len_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "len", - )?], - }; - - let pointer_class = cp.add_class(oomir::POINTER_CLASS)?; - let slice_get_i8 = cp.add_method_ref(pointer_class, "sliceGetI8", "(Ljava/lang/Object;I)B")?; - let starts_with_descriptor = format!( - "(L{};L{};)Z", - oomir::UTF8_VIEW_CLASS, - oomir::UTF8_VIEW_CLASS - ); - let starts_with = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("startsWith")?, - descriptor_index: cp.add_utf8(&starts_with_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 3, - vec![ - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::If_icmpgt(30), - Instruction::Iconst_0, - Instruction::Istore_2, - Instruction::Iload_2, - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::If_icmpge(28), - Instruction::Aload_0, - Instruction::Getfield(array_field), - Instruction::Aload_0, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i8), - Instruction::Aload_1, - Instruction::Getfield(array_field), - Instruction::Aload_1, - Instruction::Getfield(offset_field), - Instruction::Iload_2, - Instruction::Iadd, - Instruction::Invokestatic(slice_get_i8), - Instruction::If_icmpne(30), - Instruction::Iinc(2, 1), - Instruction::Goto(7), - Instruction::Iconst_1, - Instruction::Ireturn, - Instruction::Iconst_0, - Instruction::Ireturn, - ], - &starts_with_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "startsWith", - )?], - }; - - let equals_descriptor = starts_with_descriptor.clone(); - let starts_with_ref = cp.add_method_ref(this_class, "startsWith", &starts_with_descriptor)?; - let equals = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("equals")?, - descriptor_index: cp.add_utf8(&equals_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 2, - vec![ - Instruction::Aload_0, - Instruction::Getfield(length_field), - Instruction::Aload_1, - Instruction::Getfield(length_field), - Instruction::If_icmpne(9), - Instruction::Aload_0, - Instruction::Aload_1, - Instruction::Invokestatic(starts_with_ref), - Instruction::Ireturn, - Instruction::Iconst_0, - Instruction::Ireturn, - ], - &equals_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "equals", - )?], - }; - - let to_java_ref = cp.add_method_ref(this_class, "toJavaString", &to_java_descriptor)?; - let character_class = cp.add_class("java/lang/Character")?; - let to_chars = cp.add_method_ref(character_class, "toChars", "(I)[C")?; - let string_class = cp.add_class("java/lang/String")?; - let string_value_of = cp.add_method_ref(string_class, "valueOf", "([C)Ljava/lang/String;")?; - let java_starts_with = - cp.add_method_ref(string_class, "startsWith", "(Ljava/lang/String;)Z")?; - let starts_with_char_descriptor = format!("(L{};I)Z", oomir::UTF8_VIEW_CLASS); - let starts_with_char = jvm::Method { - access_flags: MethodAccessFlags::PUBLIC | MethodAccessFlags::STATIC, - name_index: cp.add_utf8("startsWithChar")?, - descriptor_index: cp.add_utf8(&starts_with_char_descriptor)?, - attributes: vec![code_attribute_for_descriptor( - &mut cp, - 2, - vec![ - Instruction::Aload_0, - Instruction::Invokestatic(to_java_ref), - Instruction::Iload_1, - Instruction::Invokestatic(to_chars), - Instruction::Invokestatic(string_value_of), - Instruction::Invokevirtual(java_starts_with), - Instruction::Ireturn, - ], - &starts_with_char_descriptor, - true, - Some(oomir::UTF8_VIEW_CLASS), - "startsWithChar", - )?], - }; - - let class_file = ClassFile { - code_source_url: None, - version: Version::Java8 { minor: 0 }, - constant_pool: cp.into_inner(), - access_flags: ClassAccessFlags::PUBLIC | ClassAccessFlags::FINAL | ClassAccessFlags::SUPER, - this_class, - super_class: slice_class, - interfaces: Vec::new(), - fields: Vec::new(), - methods: vec![ - constructor, - long_constructor, - from_java, - to_java, - as_slice, - from_slice, - len, - starts_with, - equals, - starts_with_char, - ], - attributes: Vec::new(), - }; - verify_no_duplicate_constants(&class_file)?; - - let mut bytes = Vec::new(); - class_file.to_bytes(&mut bytes)?; - Ok(bytes) -} diff --git a/src/lower2/output.rs b/src/lower2/output.rs index 0e422d66..08ed30c7 100644 --- a/src/lower2/output.rs +++ b/src/lower2/output.rs @@ -25,6 +25,20 @@ fn io_error(context: &str, error: std::io::Error) -> jvm::Error { } impl ClassOutput { + pub fn resources( + &mut self, + registry: &ClassRegistry, + cp: &mut super::constant_pool::InternedConstantPool, + ) -> jvm::Result<()> { + for resource in cp.take_resources() { + let name = format!("{}{}", jvm::resources::BUNDLE_PREFIX, resource.name); + registry + .emit(&mut self.writer, &name, &resource.bytes) + .map_err(|e| io_error(&name, e))?; + } + Ok(()) + } + pub fn create(module: &str) -> jvm::Result { let ordinal = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); let directory = std::env::temp_dir().join(format!( diff --git a/src/lower2/select.rs b/src/lower2/select.rs index 7192a2ba..4a2ee721 100644 --- a/src/lower2/select.rs +++ b/src/lower2/select.rs @@ -3,11 +3,11 @@ use super::*; use jvm::attributes::Instruction; pub(crate) fn compile( + context: Option<&oomir::construct::Context>, body: &oomir::SsaBody, constant_pool: &mut InternedConstantPool, bootstrap_methods: &mut Vec, debug_info: DebugInfoOptions, - relative_pointer_abi: bool, ) -> jvm::Result { static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); struct Constants<'a>(&'a [oomir::Constant]); @@ -43,13 +43,13 @@ pub(crate) fn compile( &body.types, constant_pool, jvm_compiler_core::jvm::select::Options { + direct_copy: Some(&|owner| context.is_some_and(|context| context.direct_copy(owner))), verify: *VERIFY.get_or_init(|| std::env::var_os("RCGJ_VERIFY_SSA").is_some()), lines: if debug_info.line_numbers { body.lines.as_ref() } else { None }, - relative_pointer_abi, bootstrap: Some(bootstrap_methods), constants: Some(&Constants(&body.constants)), debug: if debug_info.local_variables { diff --git a/src/mono.rs b/src/mono.rs index 7933613f..e0eb55f3 100644 --- a/src/mono.rs +++ b/src/mono.rs @@ -88,17 +88,19 @@ pub(super) fn place_or_insert_mono_function<'tcx>( }; receiver_self != container_ty }); - // The method may be monomorphized in a downstream crate. Emit a - // receiver-class fragment there so the linker can attach it to the - // upstream class definition rather than leaving only a static copy. - let self_oomir_ty = lower1::types::force_define_named_adt( - container_ty, - tcx, - &mut oomir_module.data_types, - instance, - ); - - if !has_arbitrary_self_receiver && let Type::Class(class_name) = self_oomir_ty { + let receiver_surface = matches!(container_ty.kind(), TyKind::Coroutine(..)) + || matches!(container_ty.kind(), TyKind::Adt(def, args) + if lower1::types::adt_class_kind(tcx, def, args) == oomir::ClassKind::JavaValue); + if receiver_surface + && !has_arbitrary_self_receiver + && lower1::types::enum_carrier(container_ty, tcx).is_none() + && let Type::Class(class_name) = lower1::types::force_define_named_adt( + container_ty, + tcx, + &mut oomir_module.data_types, + instance, + ) + { let can_extend_compiled_core_class = lower1::jvm_names::uses_compiled_core(tcx) && (instance.def_id().is_local() || lower1::jvm_names::compiles_external_core_instances(tcx)); diff --git a/src/oomir.rs b/src/oomir.rs index f298cf7d..3d0711c4 100644 --- a/src/oomir.rs +++ b/src/oomir.rs @@ -3,6 +3,7 @@ mod forward; pub use forward::{MethodForwarder, ReceiverPointer}; mod body; pub(crate) mod construct; +pub mod fields; pub(crate) mod outline; pub use body::SsaBody; pub type SsaFunction = Function>; @@ -23,12 +24,13 @@ pub mod scalar; mod visit; pub use jvm_compiler_core::jvm::abi::{ - POINTER_CLASS, RELATIVE_POINTER_METHOD_SUFFIX, SLICE_VIEW_CLASS, UTF8_VIEW_CLASS, - relative_pointer_byte_offset_field, relative_pointer_element_offset_field, + POINTER_CLASS, SLICE_VIEW_CLASS, TAGGED_LONG_CLASS, UTF8_VIEW_CLASS, }; pub const JAVA_STRING_CLASS: &str = "java/lang/String"; pub const CALLER_LOCATION_PARAM_NAME: &str = "__caller_location"; +pub const ENUM_TAG_METHOD: &str = "$rust$tag"; pub use jvm_compiler_core::debug::SourceLocation; +pub use jvm_compiler_core::ir::HeapOp; /// A source-level Rust variable that can be represented by a JVM local slot. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -53,9 +55,6 @@ pub struct Module> { pub shared_data_types: Option>>, /// Canonical emission shards use the same immutable representation tables. pub shared_context: Option>>, - /// Static methods removed into the canonical data-type contribution table - /// that still use the component-carrying internal pointer ABI. - pub relative_static_methods: Arc>, /// JVM interfaces referenced by this shard but defined in another crate. pub external_interfaces: HashSet, pub statics: HashMap, @@ -73,7 +72,6 @@ impl Module { suppressed_data_types: self.suppressed_data_types, shared_data_types: self.shared_data_types, shared_context: self.shared_context, - relative_static_methods: self.relative_static_methods, external_interfaces: self.external_interfaces, statics: self.statics, } @@ -94,6 +92,17 @@ impl Module { } impl Module { + pub fn component_method(&self, owner: &str, name: &str) -> bool { + component_method(owner, name) + || (name == "call" + && self.data_type(owner).is_some_and(|data| { + let (DataType::Class { interfaces, .. } + | DataType::Interface { interfaces, .. }) = data; + interfaces + .iter() + .any(|p| p.starts_with("org/rustlang/runtime/FnPtr_")) + })) + } pub fn data_type(&self, name: &str) -> Option<&DataType> { self.data_types.get(name).or_else(|| { self.shared_data_types @@ -167,6 +176,7 @@ pub struct Static { pub allocation_alignment: usize, pub allocation_codec_class_name: Option, pub is_thread_local: bool, + pub is_private: bool, } impl Static { @@ -215,6 +225,8 @@ pub enum AdtHelperKind { enum_class: String, variants: Vec, values: Vec, + /// Private variants answer directly without naming every sibling class. + dispatch: Option, }, EnumIsVariant { enum_class: String, @@ -233,9 +245,21 @@ pub enum AdtHelperKind { }, } +/// Static method namespaces require no Rust storage, constructors, or copy helpers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ClassKind { + Value, + /// A named public Java value keeps its declared field contract. + JavaValue, + /// DST fields describe decoded views. They do not supply authoritative inline storage. + MemoryView, + Static, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum DataType { Class { + kind: ClassKind, is_abstract: bool, super_class: Option, fields: Vec<(String, Type)>, @@ -254,12 +278,14 @@ impl Hash for DataType { std::mem::discriminant(self).hash(state); match self { Self::Class { + kind, is_abstract, super_class, fields, methods, interfaces, } => { + kind.hash(state); is_abstract.hash(state); super_class.hash(state); fields.hash(state); @@ -288,6 +314,7 @@ impl DataType { pub fn clean_duplicates(&mut self) { match self { DataType::Class { + kind: _, is_abstract: _, super_class: _, fields, diff --git a/src/oomir/constant.rs b/src/oomir/constant.rs index b0be41f7..99867da2 100644 --- a/src/oomir/constant.rs +++ b/src/oomir/constant.rs @@ -13,6 +13,13 @@ pub enum Constant { adapter_class: String, interface_name: String, }, + FunctionHandle { + interface_name: String, + owner: String, + name: String, + descriptor: String, + interface: bool, + }, /// A lower2-generated call to a private factory method. Large static object /// graphs use these to stay within the JVM's per-method bytecode limit. FactoryCall { @@ -94,7 +101,10 @@ pub enum Constant { Boolean(bool), Char(char), Str(String), + /// Compiler-controlled class, method or descriptor used by reflection. String(String), + /// Java text with no method/class naming semantics (including field names). + LiteralString(String), // 0 = the type of elements, 1 = the elements as a vec of constants Array(Box, Vec), Slice(Box, Vec), @@ -141,6 +151,15 @@ impl std::hash::Hash for Constant { adapter_class.hash(state); interface_name.hash(state); } + Constant::FunctionHandle { + interface_name, + owner, + name, + descriptor, + interface, + } => { + (interface_name, owner, name, descriptor, interface).hash(state); + } Constant::FactoryCall { owner_class, method_name, @@ -248,7 +267,7 @@ impl std::hash::Hash for Constant { Constant::Boolean(b) => b.hash(state), Constant::Char(c) => c.hash(state), Constant::Str(s) => s.hash(state), - Constant::String(s) => s.hash(state), + Constant::String(s) | Constant::LiteralString(s) => s.hash(state), Constant::Array(ty, elements) => { ty.hash(state); elements.hash(state); diff --git a/src/oomir/constant_cost.rs b/src/oomir/constant_cost.rs index 69f19e43..3b81f147 100644 --- a/src/oomir/constant_cost.rs +++ b/src/oomir/constant_cost.rs @@ -1,6 +1,7 @@ //! Estimated constant emission work, shared by preparation and outlining. // A byte becomes one Latin-1 character, at most two modified UTF-8 bytes. -pub(crate) const PACKED_BYTE_CHUNK: usize = 32767; +pub(crate) const PACKED_BYTE_CHUNK: usize = + (u16::MAX as usize - jvm_compiler_core::classfile::names::LITERAL_STRING.len()) / 2; pub(crate) fn constant_byte(constant: &super::Constant) -> Option { match constant { @@ -33,6 +34,7 @@ pub(crate) fn constant_instruction_cost(constant: &super::Constant) -> usize { cost.saturating_add(constant_instruction_cost(arg)) }), C::FunctionPointer { .. } => 3, + C::FunctionHandle { .. } => 4, C::PointerAddress { .. } => 3, C::RepeatedBytePointer { .. } => 8, C::ByteArrayPointer { bytes, .. } => byte_array_cost(bytes.len()).saturating_add(8), @@ -49,7 +51,11 @@ pub(crate) fn constant_instruction_cost(constant: &super::Constant) -> usize { | C::Boolean(_) | C::Char(_) | C::String(_) + | C::LiteralString(_) | C::Null(_) => 1, + C::Str(s) if !jvm_compiler_core::classfile::names::literal_fits(s) => { + byte_array_cost(s.len()).saturating_add(6) + } C::Str(_) => 2, C::Array(element_type, elements) | C::Slice(element_type, elements) if is_packed_byte_array(element_type, elements) => diff --git a/src/oomir/constant_eq.rs b/src/oomir/constant_eq.rs index 69861317..07ca0df2 100644 --- a/src/oomir/constant_eq.rs +++ b/src/oomir/constant_eq.rs @@ -20,6 +20,10 @@ mod tests { let array = Constant::Array(Box::new(super::super::Type::F64), vec![nan]); assert_eq!(array, array.clone()); assert!(HashSet::from([array.clone()]).contains(&array)); + let literal = Constant::LiteralString("eq".into()); + assert_eq!(literal, literal.clone()); + assert_ne!(literal, Constant::String("eq".into())); + assert!(HashSet::from([literal.clone()]).contains(&literal)); } } @@ -49,6 +53,22 @@ impl PartialEq for Constant { interface_name: b1, }, ) => a0 == b0 && a1 == b1, + ( + Self::FunctionHandle { + interface_name: a1, + owner: a2, + name: a3, + descriptor: a4, + interface: a5, + }, + Self::FunctionHandle { + interface_name: b1, + owner: b2, + name: b3, + descriptor: b4, + interface: b5, + }, + ) => (a1, a2, a3, a4, a5) == (b1, b2, b3, b4, b5), ( Self::FactoryCall { owner_class: a0, @@ -189,7 +209,8 @@ impl PartialEq for Constant { (Self::Boolean(a0), Self::Boolean(b0)) => a0 == b0, (Self::Char(a0), Self::Char(b0)) => a0 == b0, (Self::Str(a0), Self::Str(b0)) => a0 == b0, - (Self::String(a0), Self::String(b0)) => a0 == b0, + (Self::String(a0), Self::String(b0)) + | (Self::LiteralString(a0), Self::LiteralString(b0)) => a0 == b0, (Self::Array(a0, a1), Self::Array(b0, b1)) => a0 == b0 && a1 == b1, (Self::Slice(a0, a1), Self::Slice(b0, b1)) => a0 == b0 && a1 == b1, ( diff --git a/src/oomir/construct/addresses.rs b/src/oomir/construct/addresses.rs new file mode 100644 index 00000000..8c3e20a5 --- /dev/null +++ b/src/oomir/construct/addresses.rs @@ -0,0 +1,163 @@ +//! Rust address operations carry layout and arithmetic independently of Java. +use super::*; + +fn static_size(value: &oomir::Operand) -> Option { + use oomir::{Constant as C, Operand::Constant}; + let value = match value { + Constant(C::I32(n)) => u32::try_from(*n).ok()?, + Constant(C::U32(n)) => *n, + Constant(C::I64(n)) => u32::try_from(*n).ok()?, + Constant(C::U64(n)) => u32::try_from(*n).ok()?, + _ => return None, + }; + (value <= i32::MAX as u32).then_some(value) +} + +impl Emission<'_> { + pub(super) fn address(&mut self, instruction: oomir::Instruction) -> Result<()> { + use oomir::Instruction::*; + let (dest, value) = match instruction { + AddressOffset { + dest, + source, + count, + ty, + bytes, + wrapping, + subtract, + } => { + let pointer = self.operand(source)?; + let count = self.operand(count)?; + let mut offset = self.adapt(count, self.ty(&oomir::Type::I64))?; + if subtract { + offset = self + .emit(ir::Op::Neg(offset), Some(self.ty(&oomir::Type::I64))) + .unwrap(); + } + let value = self + .emit( + ir::Op::Offset { + pointer, + offset, + bytes, + wrapping, + }, + Some(self.ty(&ty)), + ) + .unwrap(); + (dest, value) + } + AddressRetype { + dest, + source, + layout, + } => (dest, self.address_layout(source, *layout, false)?), + ViewAddress { + dest, + source, + layout, + } => (dest, self.address_layout(source, *layout, true)?), + _ => unreachable!("non-address instruction"), + }; + if let Some(dest) = dest { + self.write(&dest, value)?; + } + Ok(()) + } + + pub(super) fn address_layout( + &mut self, + source: oomir::Operand, + layout: oomir::AddressLayout, + view: bool, + ) -> Result { + let oomir::AddressLayout { + pointer_type, + size, + codec, + } = layout; + let source_type = source.get_type().expect("typed address source"); + let codec_id = match &codec { + oomir::Operand::Constant(oomir::Constant::Null(_)) => Some(None), + oomir::Operand::Constant(oomir::Constant::String(name)) => Some(Some( + self.vocabulary + .types + .find_symbol(name) + .expect("registered address codec"), + )), + _ => None, + }; + let input = self.operand(source)?; + let returns = self.ty(&pointer_type); + // DST casts keep the slice length for later tail construction. Sized casts can discard it. + let thin_view = match &pointer_type { + oomir::Type::Pointer(inner) => match inner.as_ref() { + oomir::Type::Class(name) => self + .context + .fields + .get(name) + .is_some_and(|layout| layout.direct), + oomir::Type::Interface(_) => false, + _ => true, + }, + _ => false, + }; + let typed = matches!(pointer_type, oomir::Type::Pointer(_)) + && if view { + thin_view && matches!(source_type, oomir::Type::Slice(_) | oomir::Type::Str) + } else { + matches!(source_type, oomir::Type::Pointer(_)) + }; + if typed && let (Some(size), Some(codec)) = (static_size(&size), codec_id) { + let op = if view { + let mut values = Vec::with_capacity(3); + for (index, ty) in [types::object(), oomir::Type::I32, oomir::Type::U64] + .iter() + .enumerate() + { + values.push( + self.emit( + ir::Op::ViewPart { + view: input, + index: index as u8, + }, + Some(self.ty(ty)), + ) + .unwrap(), + ); + } + ir::Op::ViewAddress { + parts: self.builder.args(values), + size, + codec, + } + } else if pointer_type.scalar_address_size() == Some(size) && codec.is_none() { + ir::Op::Cast(input) + } else { + ir::Op::RetypeAddress { + pointer: input, + size, + codec, + } + }; + return Ok(self.emit(op, Some(returns)).unwrap()); + } + let source_type = if view { types::object() } else { source_type }; + let size = self.operand(size)?; + let codec = self.operand(codec)?; + Ok(self + .call( + oomir::POINTER_CLASS.into(), + if view { "fromSlice" } else { "retype" }.into(), + vec![ + self.ty(&source_type), + self.ty(&oomir::Type::U64), + self.ty(&oomir::Type::java_string()), + ], + returns, + ir::CallKind::JvmStatic, + vec![input, size, codec], + )? + .unwrap()) + } +} diff --git a/src/oomir/construct/arrays.rs b/src/oomir/construct/arrays.rs index 4b6f2fe1..15fe33f0 100644 --- a/src/oomir/construct/arrays.rs +++ b/src/oomir/construct/arrays.rs @@ -28,7 +28,14 @@ impl Emission<'_> { return self.write(&dest, value); } let value = self - .emit(Op::ArrayGet { array, index }, Some(element)) + .emit( + Op::ArrayGet { + native: false, + array, + index, + }, + Some(element), + ) .unwrap(); self.write(&dest, value) } @@ -48,20 +55,12 @@ impl Emission<'_> { return Ok(()); } if copy && self.vocabulary.types.get(element).unwrap().carrier() == 5 { - value = self - .call( - oomir::POINTER_CLASS.into(), - "copyManagedValue".into(), - vec![self.ty(&types::object())], - self.ty(&types::object()), - CallKind::JvmStatic, - vec![value], - )? - .unwrap(); + value = self.copy_value(value); } let value = self.adapt(value, element)?; self.emit( Op::ArraySet { + native: false, array, index, value, @@ -78,6 +77,16 @@ impl Emission<'_> { ) -> Result<()> { let array = self.operand(array)?; let value = self.operand(value)?; + if let Some(ir::Type::Array(element)) = self + .vocabulary + .types + .get(self.builder.body.value_type(array)) + && ir::StorageSlot::scalar(element, &self.vocabulary.types).is_some() + { + let value = self.adapt(value, element)?; + self.emit(Op::ArrayFill { array, value }, None); + return Ok(()); + } let copy = self.constant(oomir::Constant::Boolean(copy))?; self.call( oomir::POINTER_CLASS.into(), diff --git a/src/oomir/construct/context.rs b/src/oomir/construct/context.rs index 2437751f..026046fa 100644 --- a/src/oomir/construct/context.rs +++ b/src/oomir/construct/context.rs @@ -5,6 +5,8 @@ use super::*; pub(super) struct FieldLayout { pub members: Vec<(String, oomir::Type)>, pub direct: bool, + pub split_borrows: bool, + pub scalar_record: bool, } #[derive(Debug, Clone)] @@ -16,6 +18,24 @@ pub(crate) struct Context { parents: HashMap>, } impl Context { + pub(crate) fn direct_copy(&self, owner: &str) -> bool { + !owner.starts_with("org/rustlang/runtime/") + && self.fields.get(owner).is_some_and(|layout| layout.direct) + && self + .parents + .get(owner) + .is_none_or(|parents| parents.iter().all(|parent| parent == "java/lang/Object")) + } + + pub(super) fn component_method(&self, owner: &str, name: &str) -> bool { + oomir::component_method(owner, name) + || (name == "call" + && self.parents.get(owner).is_some_and(|parents| { + parents + .iter() + .any(|p| p.starts_with("org/rustlang/runtime/FnPtr_")) + })) + } pub(crate) fn new(module: &oomir::Module) -> Self { fn zero( ty: &oomir::Type, @@ -69,9 +89,10 @@ impl Context { } oomir::DataType::Class { fields, + kind, is_abstract: false, .. - } => { + } if *kind != oomir::ClassKind::Static => { context.fields.insert( name.clone(), FieldLayout { @@ -80,7 +101,20 @@ impl Context { .filter(|(_, ty)| ty.has_jvm_value()) .cloned() .collect(), - direct: false, + scalar_record: *kind == oomir::ClassKind::Value + && !name.starts_with("org/rustlang/runtime/") + && (2..=4).contains( + &fields.iter().filter(|(_, ty)| ty.has_jvm_value()).count(), + ) + && fields.iter().filter(|(_, ty)| ty.has_jvm_value()).all( + |(_, ty)| { + oomir::scalar::scalar_type(ty).is_some() + && !ty.is_jvm_reference_type() + }, + ), + direct: *kind != oomir::ClassKind::MemoryView, + split_borrows: *kind == oomir::ClassKind::Value + && oomir::fields::split_borrows(fields), }, ); let count = fields @@ -97,46 +131,8 @@ impl Context { _ => {} } } - // Slice/str tails need byte projection: their enclosing JVM carrier - // may not exist (for example a raw cast from a tuple to a Rust DST). - // Arrays are conservatively retained on the general path too, since - // this representation vocabulary does not distinguish sized tails. - fn direct_tail( - name: &str, - fields: &HashMap, - seen: &mut HashMap, - ) -> bool { - if let Some(&direct) = seen.get(name) { - return direct; - } - seen.insert(name.into(), false); - let direct = fields - .get(name) - .is_some_and(|layout| match layout.members.last() { - None => true, - Some((_, oomir::Type::Class(tail))) => direct_tail(tail, fields, seen), - Some(( - _, - oomir::Type::Slice(_) - | oomir::Type::Str - | oomir::Type::Array(_) - | oomir::Type::MutableReference(_) - | oomir::Type::Interface(_), - )) => false, - _ => true, - }); - *seen.get_mut(name).unwrap() = direct; - direct - } - let mut direct = HashMap::default(); - for name in context.fields.keys() { - direct_tail(name, &context.fields, &mut direct); - } - for (name, direct) in direct { - if let Some(layout) = context.fields.get_mut(&name) { - layout.direct = direct; - } - } + // Stored borrows have fixed size even with unsized pointees. The Rust schema distinguishes + // them from DST tails. context } pub(super) fn is_zero(&self, ty: &oomir::Type) -> bool { diff --git a/src/oomir/construct/heap.rs b/src/oomir/construct/heap.rs new file mode 100644 index 00000000..2b58e7d5 --- /dev/null +++ b/src/oomir/construct/heap.rs @@ -0,0 +1,44 @@ +//! Platform allocation enters SSA as storage operations, before JVM selection. +use super::*; + +impl Emission<'_> { + pub(super) fn heap( + &mut self, + operation: ir::HeapOp, + operands: Vec, + dest: Option, + ) -> Result<()> { + let pointer = self.ty(&oomir::Type::pointer(oomir::Type::U8)); + let object = self.ty(&oomir::Type::Class("java/lang/Object".into())); + let long = self.ty(&oomir::Type::I64); + let mut values = operands + .into_iter() + .map(|op| self.operand(op)) + .collect::>>()?; + let mut args = Vec::new(); + if operation != ir::HeapOp::Allocate { + let address = self.adapt(values.remove(0), pointer)?; + for (index, ty) in [(0, object), (1, long)] { + args.push( + self.emit(ir::Op::AddressPart { address, index }, Some(ty)) + .unwrap(), + ); + } + } + for value in values { + args.push(self.adapt(value, long)?); + } + let returns = operation != ir::HeapOp::Deallocate; + let args = self.builder.args(args); + let root = self.emit(ir::Op::Heap { operation, args }, returns.then_some(object)); + if let (Some(root), Some(dest)) = (root, dest) { + let zero = self.constant(oomir::Constant::I64(0))?; + let parts = self.builder.args([root, zero]); + let address = self + .emit(ir::Op::AddressPack(parts), Some(pointer)) + .unwrap(); + self.write(&dest, address)?; + } + Ok(()) + } +} diff --git a/src/oomir/construct/memory.rs b/src/oomir/construct/memory.rs new file mode 100644 index 00000000..cafffb47 --- /dev/null +++ b/src/oomir/construct/memory.rs @@ -0,0 +1,145 @@ +//! Semantic places and memory accesses become SSA without helper recognition. +use super::*; +use ir::{CallKind, Op}; + +impl Emission<'_> { + pub(super) fn copy_value(&mut self, source: ir::ValueId) -> ir::ValueId { + let ty = self.builder.body.value_type(source); + if matches!( + self.vocabulary.types.get(ty), + Some( + ir::Type::TaggedI64 + | ir::Type::Scalar(_) + | ir::Type::Pointer(_) + | ir::Type::Slice(_) + | ir::Type::Str + ) + ) { + source + } else { + self.emit(Op::CopyValue(source), Some(ty)).unwrap() + } + } + + pub(super) fn memory(&mut self, instruction: oomir::Instruction) -> Result<()> { + use oomir::Instruction::*; + match instruction { + ValueCopy { dest, source } => { + let source = self.operand(source)?; + let value = self.copy_value(source); + self.write(&dest, value)?; + } + MemoryLoad { + dest, + pointer, + pointee, + owned, + } => { + let pointer = self.operand(pointer)?; + let pointer = + self.adapt(pointer, self.ty(&oomir::Type::pointer(pointee.clone())))?; + let value = self + .emit( + if owned && pointee.is_jvm_reference_type() { + Op::LoadCopy(pointer) + } else { + Op::Load(pointer) + }, + Some(self.ty(&pointee)), + ) + .unwrap(); + self.write(&dest, value)?; + } + MemoryStore { + pointer, + pointee, + value, + } => { + let pointer = self.operand(pointer)?; + let pointer = + self.adapt(pointer, self.ty(&oomir::Type::pointer(pointee.clone())))?; + let value = self.operand(value)?; + let value = self.adapt(value, self.ty(&pointee))?; + self.emit(Op::Store { pointer, value }, None); + } + MemoryCommit { pointer } => { + let pointer = self.operand(pointer)?; + self.emit(Op::Commit(pointer), None); + } + MemoryProject { + dest, + base, + projection, + } => { + let oomir::MemoryProjection { + owner, + field, + pointee, + offset, + size, + codec, + } = *projection; + let base_type = oomir::Type::Class(owner.clone()); + let pointer_type = oomir::Type::pointer(base_type.clone()); + let result_type = oomir::Type::pointer(pointee.clone()); + let direct = self.context.fields.get(&owner).is_some_and(|layout| { + layout.direct + && layout + .members + .iter() + .any(|(name, ty)| name == &field && ty == &pointee) + }) && base.get_type().as_ref() == Some(&pointer_type); + let base = self.operand(base)?; + let value = if direct { + let base = self.adapt(base, self.ty(&pointer_type))?; + let field = self.builder.field(ir::FieldRef { + owner: self.ty(&base_type), + name: field, + ty: self.ty(&pointee), + is_static: false, + }); + let projection = self.builder.projection(ir::PointerProjection { + parent: None, + field, + offset, + size, + codec, + }); + self.emit( + Op::Project { base, projection }, + Some(self.ty(&result_type)), + ) + .unwrap() + } else { + let mut args = vec![base]; + for value in [ + oomir::Constant::String(owner), + oomir::Constant::LiteralString(field), + oomir::Constant::U64(offset), + oomir::Constant::U64(size), + codec.map_or_else( + || oomir::Constant::Null(oomir::Type::java_string()), + oomir::Constant::String, + ), + ] { + args.push(self.constant(value)?); + } + let string = self.ty(&oomir::Type::java_string()); + let long = self.ty(&oomir::Type::U64); + self.call( + oomir::POINTER_CLASS.into(), + "projectStructField".into(), + vec![string, string, long, long, string], + self.ty(&result_type), + CallKind::Virtual, + args, + )? + .unwrap() + }; + self.write(&dest, value)?; + } + _ => unreachable!("non-memory instruction"), + } + Ok(()) + } +} diff --git a/src/oomir/construct/mod.rs b/src/oomir/construct/mod.rs index b610cc20..d8e6f353 100644 --- a/src/oomir/construct/mod.rs +++ b/src/oomir/construct/mod.rs @@ -9,12 +9,22 @@ use jvm_compiler_core::{ }; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use std::sync::Arc; +mod addresses; mod arithmetic; mod arrays; mod context; +mod heap; +mod memory; mod operations; +#[cfg(test)] +mod pointer_tests; mod pointers; +mod records; +pub(crate) use records::RECORD_ENTRY; mod types; +#[cfg(test)] +mod view_tests; +mod views; mod wrappers; pub(crate) use context::Context; use types::Vocabulary; @@ -41,8 +51,19 @@ pub(crate) fn seal(function: oomir::Function, context: &Context) -> Result Result Result Result