From c8f89df0d5b91c7e968a0921e9d4fd7dfa530e90 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Fri, 2 Oct 2026 20:18:17 +0200 Subject: [PATCH 01/17] move layout_of_enum to its own file --- compiler/rustc_abi/src/layout.rs | 528 +------------------ compiler/rustc_abi/src/layout/enum_layout.rs | 509 ++++++++++++++++++ 2 files changed, 523 insertions(+), 514 deletions(-) create mode 100644 compiler/rustc_abi/src/layout/enum_layout.rs diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index f2e3f8431c105..fc400ff745d4f 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -1,22 +1,22 @@ +use std::cmp; use std::fmt::{self, Write}; use std::num::NonZero; use std::ops::Deref; -use std::range::{RangeFrom, RangeInclusive, RangeToInclusive}; -use std::{cmp, iter}; +use std::range::{RangeFrom, RangeToInclusive}; use rustc_hashes::Hash64; use rustc_index::Idx; use rustc_index::bit_set::BitMatrix; -use tracing::{debug, trace}; +use tracing::debug; use crate::{ AbiAlign, Align, BackendLaneCount, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, - IndexVec, Integer, LayoutData, Niche, NicheOptimizations, NumScalableVectors, Primitive, - ReprOptions, Scalar, Size, StructKind, TagEncoding, TargetDataLayout, VariantLayout, Variants, - WrappingRange, + IndexVec, Integer, LayoutData, NicheOptimizations, NumScalableVectors, ReprOptions, Scalar, + Size, StructKind, TargetDataLayout, Variants, WrappingRange, }; mod coroutine; +mod enum_layout; mod simple; #[cfg(feature = "nightly")] @@ -390,7 +390,14 @@ impl LayoutCalculator { // structs. (We have also handled univariant enums // that allow representation optimization.) assert!(is_enum); - self.layout_of_enum(repr, variants, discr_range_of_repr, discriminants) + + self::enum_layout::layout_of_enum( + self, + repr, + variants, + discr_range_of_repr, + discriminants, + ) } } @@ -580,513 +587,6 @@ impl LayoutCalculator { Ok(st) } - fn layout_of_enum<'a, FieldIdx, VariantIdx, F>( - &self, - repr: &ReprOptions, - variants: &IndexSlice>, - discr_range_of_repr: impl Fn(RangeFrom, RangeToInclusive) -> (Integer, bool), - discriminants: impl Iterator, - ) -> LayoutCalculatorResult - where - FieldIdx: Idx, - VariantIdx: Idx, - F: Deref> + fmt::Debug + Copy, - { - let dl = self.cx.data_layout(); - // bail if the enum has an incoherent repr that cannot be computed - if repr.packed() { - return Err(LayoutCalculatorError::ReprConflict); - } - - let calculate_niche_filling_layout = || -> Option> { - struct VariantLayoutInfo { - align_abi: Align, - } - - if repr.inhibit_enum_layout_opt() { - return None; - } - - if variants.len() < 2 { - return None; - } - - let mut align = dl.aggregate_align; - let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align; - let mut combined_seed = repr.field_shuffle_seed; - - let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants - .iter() - .map(|v| { - let st = self.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?; - - variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); - - align = align.max(st.align.abi); - max_repr_align = max_repr_align.max(st.max_repr_align); - unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); - combined_seed = combined_seed.wrapping_add(st.randomization_seed); - - Some(VariantLayout::from_layout(st)) - }) - .collect::>>()?; - - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - - let all_indices = variants.indices(); - let needs_disc = - |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); - let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), - }; - - let count = - (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; - - // Use the largest niche in the largest variant. - let niche = variant_layouts[largest_variant_index].largest_niche?; - let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align); - - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); - - if this_offset + layout.size > size { - return false; - } - - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } - - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; - - true - }); - - if !all_variants_fit { - return None; - } - - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair { a: first, b: second, b_offset } => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair { - a: niche_scalar, - b: second.to_union(), - b_offset, - } - } else { - BackendRepr::ScalarPair { - a: first.to_union(), - b: niche_scalar, - b_offset, - } - } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; - - let layout = LayoutData { - variants: Variants::Multiple { - tag: niche_scalar, - tag_encoding: TagEncoding::Niche { - untagged_variant: largest_variant_index, - niche_variants, - niche_start, - }, - tag_field: FieldIdx::new(0), - variants: variant_layouts, - }, - fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - backend_repr: abi, - largest_niche, - uninhabited, - size, - align: AbiAlign::new(align), - max_repr_align, - unadjusted_abi_align, - repr_c: repr.c(), - randomization_seed: combined_seed, - }; - - Some(layout) - }; - - let niche_filling_layout = calculate_niche_filling_layout(); - - let discr_type = repr.discr_type(); - let discr_size = Integer::from_attr(dl, discr_type).size(); - - let necessary_discriminants: Vec = discriminants - .filter(|&(i, _)| repr.c() || variants[i].iter().all(|f| !f.is_uninhabited())) - .map(|(_, val)| val) - .collect(); - - // When picking the integer to use, we respect how the discriminants were written - // in the original rust code, rather than looking only at the bit pattern. - let (min_negative, max_positive): (i128, u128) = if discr_type.is_signed() { - necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold( - (0_i128, 0_u128), - |(min, max), val| { - if let Ok(val) = u128::try_from(val) { - (min, max.max(val)) - } else { - (min.min(val), max) - } - }, - ) - } else { - // We might have no inhabited variants, so pretend there's at least one. - (0, necessary_discriminants.iter().copied().max().unwrap_or(0)) - }; - trace!(?min_negative, ?max_positive); - - let (min_ity, signed) = discr_range_of_repr( - RangeFrom { start: min_negative }, - RangeToInclusive { last: max_positive }, - ); //Integer::discr_range_of_repr(tcx, ty, &repr, min, max); - - let mut align = dl.aggregate_align; - let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align; - let mut combined_seed = repr.field_shuffle_seed; - - let mut size = Size::ZERO; - - // We're interested in the smallest alignment, so start large. - let mut start_align = Align::from_bytes(256).unwrap(); - assert_eq!(Integer::for_align(dl, start_align), None); - - // repr(C) on an enum tells us to make a (tag, union) layout, - // so we need to grow the prefix alignment to be at least - // the alignment of the union. (This value is used both for - // determining the alignment of the overall enum, and the - // determining the alignment of the payload after the tag.) - let mut prefix_align = min_ity.align(dl).abi; - if repr.c() { - for fields in variants { - for field in fields { - prefix_align = prefix_align.max(field.align.abi); - } - } - } - - // Create the set of structs that represent each variant. - let mut layout_variants = variants - .iter() - .map(|field_layouts| { - let st = self.layout_of_univariant( - field_layouts, - repr, - StructKind::Prefixed(min_ity.size(), prefix_align), - )?; - // Find the first field we can't move later - // to make room for a larger discriminant. - for field_idx in st.fields.index_by_increasing_offset() { - let field = &field_layouts[FieldIdx::new(field_idx)]; - if !field.is_1zst() { - start_align = start_align.min(field.align.abi); - break; - } - } - size = cmp::max(size, st.size); - align = align.max(st.align.abi); - max_repr_align = max_repr_align.max(st.max_repr_align); - unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); - combined_seed = combined_seed.wrapping_add(st.randomization_seed); - Ok(VariantLayout::from_layout(st)) - }) - .collect::, _>>()?; - - // Align the maximum variant size to the largest alignment. - size = size.align_to(align); - - // FIXME(oli-obk): deduplicate and harden these checks - if size.bytes() >= dl.obj_size_bound() { - return Err(LayoutCalculatorError::SizeOverflow); - } - - let typeck_ity = Integer::from_attr(dl, repr.discr_type()); - if typeck_ity < min_ity { - // It is a bug if Layout decided on a greater discriminant size than typeck for - // some reason at this point (based on values discriminant can take on). Mostly - // because this discriminant will be loaded, and then stored into variable of - // type calculated by typeck. Consider such case (a bug): typeck decided on - // byte-sized discriminant, but layout thinks we need a 16-bit to store all - // discriminant values. That would be a bug, because then, in codegen, in order - // to store this 16-bit discriminant into 8-bit sized temporary some of the - // space necessary to represent would have to be discarded (or layout is wrong - // on thinking it needs 16 bits) - panic!( - "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})" - ); - // However, it is fine to make discr type however large (as an optimisation) - // after this point – we’ll just truncate the value we load in codegen. - } - - // Check to see if we should use a different type for the - // discriminant. We can safely use a type with the same size - // as the alignment of the first field of each variant. - // We increase the size of the discriminant to avoid LLVM copying - // padding when it doesn't need to. This normally causes unaligned - // load/stores and excessive memcpy/memset operations. By using a - // bigger integer size, LLVM can be sure about its contents and - // won't be so conservative. - - // Use the initial field alignment - let mut ity = if repr.c() || repr.int.is_some() { - min_ity - } else { - Integer::for_align(dl, start_align).unwrap_or(min_ity) - }; - - // If the alignment is not larger than the chosen discriminant size, - // don't use the alignment as the final size. - if ity <= min_ity { - ity = min_ity; - } else { - // Patch up the variants' first few fields. - let old_ity_size = min_ity.size(); - let new_ity_size = ity.size(); - for variant in &mut layout_variants { - for i in &mut variant.field_offsets { - if *i <= old_ity_size { - assert_eq!(*i, old_ity_size); - *i = new_ity_size; - } - } - // We might be making the struct larger. - if variant.size <= old_ity_size { - variant.size = new_ity_size; - } - } - } - - let tag_valid_range = { - let tag_size = ity.size(); - let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d)); - WrappingRange::smallest_range_containing(tags, tag_size) - // We might have no inhabited variants, so pretend there's at least one. - .unwrap_or(WrappingRange { start: 0, end: 0 }) - }; - let tag = Scalar::Initialized { - value: Primitive::Int(ity, signed), - valid_range: tag_valid_range, - }; - let mut abi = BackendRepr::Memory { sized: true }; - - let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited()); - if tag.size(dl) == size { - // Make sure we only use scalar layout when the enum is entirely its - // own tag (i.e. it has no padding nor any non-ZST variant fields). - abi = BackendRepr::Scalar(tag); - } else { - // Try to use a ScalarPair for all tagged enums. - // That's possible only if we can find a common primitive type for all variants. - let mut common_prim = None; - let mut common_prim_initialized_in_all_variants = true; - for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) { - // We skip *all* ZST here and later check if we are good in terms of alignment. - // This lets us handle some cases involving aligned ZST. - let mut fields = iter::zip(field_layouts, &layout_variant.field_offsets) - .filter(|p| !p.0.is_zst()); - let (field, offset) = match (fields.next(), fields.next()) { - (None, None) => { - common_prim_initialized_in_all_variants = false; - continue; - } - (Some(pair), None) => pair, - _ => { - common_prim = None; - break; - } - }; - let prim = match field.backend_repr { - BackendRepr::Scalar(scalar) => { - common_prim_initialized_in_all_variants &= - matches!(scalar, Scalar::Initialized { .. }); - scalar.primitive() - } - _ => { - common_prim = None; - break; - } - }; - if let Some((old_prim, common_offset)) = common_prim { - // All variants must be at the same offset - if offset != common_offset { - common_prim = None; - break; - } - // This is pretty conservative. We could go fancier - // by realising that (u8, u8) could just cohabit with - // u16 or even u32. - let new_prim = match (old_prim, prim) { - // Allow all identical primitives. - (x, y) if x == y => x, - // Allow integers of the same size with differing signedness. - // We arbitrarily choose the signedness of the first variant. - (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p, - // Allow integers mixed with pointers of the same layout. - // We must represent this using a pointer, to avoid - // roundtripping pointers through ptrtoint/inttoptr. - (p @ Primitive::Pointer(_), i @ Primitive::Int(..)) - | (i @ Primitive::Int(..), p @ Primitive::Pointer(_)) - if p.size(dl) == i.size(dl) - && p.default_align(dl) == i.default_align(dl) => - { - p - } - _ => { - common_prim = None; - break; - } - }; - // We may be updating the primitive here, for example from int->ptr. - common_prim = Some((new_prim, common_offset)); - } else { - common_prim = Some((prim, offset)); - } - } - if let Some((prim, offset)) = common_prim { - let prim_scalar = if common_prim_initialized_in_all_variants { - let size = prim.size(dl); - assert!(size.bits() <= 128); - Scalar::Initialized { value: prim, valid_range: WrappingRange::full(size) } - } else { - // Common prim might be uninit. - Scalar::Union { value: prim } - }; - let pair = - LayoutData::::scalar_pair(&self.cx, tag, prim_scalar); - let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { - assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); - offsets - } - _ => panic!("encountered a non-arbitrary layout during enum layout"), - }; - if pair_offsets[FieldIdx::new(0)] == Size::ZERO - && pair_offsets[FieldIdx::new(1)] == *offset - && align == pair.align.abi - && size == pair.size - { - // We can use `ScalarPair` only when it matches our - // already computed layout (including `#[repr(C)]`). - abi = pair.backend_repr; - } - } - } - - // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the - // variants to ensure they are consistent. This is because a downcast is - // semantically a NOP, and thus should not affect layout. - if matches!(abi, BackendRepr::Scalar(..) | BackendRepr::ScalarPair { .. }) { - for variant in &mut layout_variants { - // We only do this for variants with fields; the others are not accessed anyway. - // Also do not overwrite any already existing "clever" ABIs. - if matches!(variant.backend_repr, BackendRepr::Memory { .. } if variant.has_fields()) - { - variant.backend_repr = abi; - // Also need to bump up the size, so that the entire value fits in here. - variant.size = cmp::max(variant.size, size); - } - } - } - - let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag); - - let tagged_layout = LayoutData { - variants: Variants::Multiple { - tag, - tag_encoding: TagEncoding::Direct, - tag_field: FieldIdx::new(0), - variants: layout_variants, - }, - fields: FieldsShape::Arbitrary { - offsets: [Size::ZERO].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - largest_niche, - uninhabited, - backend_repr: abi, - align: AbiAlign::new(align), - size, - max_repr_align, - unadjusted_abi_align, - repr_c: repr.c(), - randomization_seed: combined_seed, - }; - - let best_layout = match (tagged_layout, niche_filling_layout) { - (tl, Some(nl)) => { - // Pick the smaller layout; otherwise, - // pick the layout with the larger niche; otherwise, - // pick tagged as it has simpler codegen. - use cmp::Ordering::*; - let niche_size = |l: &LayoutData| { - l.largest_niche.map_or(0, |n| n.available(dl)) - }; - match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) { - (Greater, _) => nl, - (Equal, Less) => nl, - _ => tl, - } - } - (tl, None) => tl, - }; - - Ok(best_layout) - } - fn layout_of_univariant_biased<'a, FieldIdx, VariantIdx, F>( &self, fields: &IndexSlice, diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs new file mode 100644 index 0000000000000..1a6a8c34f7443 --- /dev/null +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -0,0 +1,509 @@ +use std::ops::Deref; +use std::range::{RangeFrom, RangeInclusive, RangeToInclusive}; +use std::{cmp, fmt, iter}; + +use rustc_index::Idx; +use tracing::trace; + +use super::{LayoutCalculator, LayoutCalculatorError, LayoutCalculatorResult, absent}; +use crate::{ + AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer, + LayoutData, Niche, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding, + VariantLayout, Variants, WrappingRange, +}; + +pub(super) fn layout_of_enum<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( + calculator: &LayoutCalculator, + repr: &ReprOptions, + variants: &IndexSlice>, + discr_range_of_repr: impl Fn(RangeFrom, RangeToInclusive) -> (Integer, bool), + discriminants: impl Iterator, +) -> LayoutCalculatorResult +where + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + let dl = calculator.cx.data_layout(); + // bail if the enum has an incoherent repr that cannot be computed + if repr.packed() { + return Err(LayoutCalculatorError::ReprConflict); + } + + let calculate_niche_filling_layout = || -> Option> { + struct VariantLayoutInfo { + align_abi: Align, + } + + if repr.inhibit_enum_layout_opt() { + return None; + } + + if variants.len() < 2 { + return None; + } + + let mut align = dl.aggregate_align; + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = align; + let mut combined_seed = repr.field_shuffle_seed; + + let mut variants_info = IndexVec::::with_capacity(variants.len()); + let mut variant_layouts = variants + .iter() + .map(|v| { + let st = calculator.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?; + + variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); + + align = align.max(st.align.abi); + max_repr_align = max_repr_align.max(st.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); + combined_seed = combined_seed.wrapping_add(st.randomization_seed); + + Some(VariantLayout::from_layout(st)) + }) + .collect::>>()?; + + let largest_variant_index = variant_layouts + .iter_enumerated() + .max_by_key(|(_i, layout)| layout.size.bytes()) + .map(|(i, _layout)| i)?; + + let all_indices = variants.indices(); + let needs_disc = + |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); + let niche_variants = RangeInclusive { + start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), + last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), + }; + + let count = + (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; + + // Use the largest niche in the largest variant. + let niche = variant_layouts[largest_variant_index].largest_niche?; + let (niche_start, niche_scalar) = niche.reserve(dl, count)?; + let niche_offset = niche.offset; + let niche_size = niche.value.size(dl); + let size = variant_layouts[largest_variant_index].size.align_to(align); + + let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } + + layout.largest_niche = None; + + if layout.size <= niche_offset { + // This variant will fit before the niche. + return true; + } + + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); + + if this_offset + layout.size > size { + return false; + } + + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } + + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; + + true + }); + + if !all_variants_fit { + return None; + } + + let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); + let abi = if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair { a: first, b: second, b_offset } => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset } + } else { + BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset } + } + } + _ => BackendRepr::Memory { sized: true }, + } + } else { + BackendRepr::Memory { sized: true } + }; + + let layout = LayoutData { + variants: Variants::Multiple { + tag: niche_scalar, + tag_encoding: TagEncoding::Niche { + untagged_variant: largest_variant_index, + niche_variants, + niche_start, + }, + tag_field: FieldIdx::new(0), + variants: variant_layouts, + }, + fields: FieldsShape::Arbitrary { + offsets: [niche_offset].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + backend_repr: abi, + largest_niche, + uninhabited, + size, + align: AbiAlign::new(align), + max_repr_align, + unadjusted_abi_align, + repr_c: repr.c(), + randomization_seed: combined_seed, + }; + + Some(layout) + }; + + let niche_filling_layout = calculate_niche_filling_layout(); + + let discr_type = repr.discr_type(); + let discr_size = Integer::from_attr(dl, discr_type).size(); + + let necessary_discriminants: Vec = discriminants + .filter(|&(i, _)| repr.c() || variants[i].iter().all(|f| !f.is_uninhabited())) + .map(|(_, val)| val) + .collect(); + + // When picking the integer to use, we respect how the discriminants were written + // in the original rust code, rather than looking only at the bit pattern. + let (min_negative, max_positive): (i128, u128) = if discr_type.is_signed() { + necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold( + (0_i128, 0_u128), + |(min, max), val| { + if let Ok(val) = u128::try_from(val) { + (min, max.max(val)) + } else { + (min.min(val), max) + } + }, + ) + } else { + // We might have no inhabited variants, so pretend there's at least one. + (0, necessary_discriminants.iter().copied().max().unwrap_or(0)) + }; + trace!(?min_negative, ?max_positive); + + let (min_ity, signed) = discr_range_of_repr( + RangeFrom { start: min_negative }, + RangeToInclusive { last: max_positive }, + ); //Integer::discr_range_of_repr(tcx, ty, &repr, min, max); + + let mut align = dl.aggregate_align; + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = align; + let mut combined_seed = repr.field_shuffle_seed; + + let mut size = Size::ZERO; + + // We're interested in the smallest alignment, so start large. + let mut start_align = Align::from_bytes(256).unwrap(); + assert_eq!(Integer::for_align(dl, start_align), None); + + // repr(C) on an enum tells us to make a (tag, union) layout, + // so we need to grow the prefix alignment to be at least + // the alignment of the union. (This value is used both for + // determining the alignment of the overall enum, and the + // determining the alignment of the payload after the tag.) + let mut prefix_align = min_ity.align(dl).abi; + if repr.c() { + for fields in variants { + for field in fields { + prefix_align = prefix_align.max(field.align.abi); + } + } + } + + // Create the set of structs that represent each variant. + let mut layout_variants = variants + .iter() + .map(|field_layouts| { + let st = calculator.layout_of_univariant( + field_layouts, + repr, + StructKind::Prefixed(min_ity.size(), prefix_align), + )?; + // Find the first field we can't move later + // to make room for a larger discriminant. + for field_idx in st.fields.index_by_increasing_offset() { + let field = &field_layouts[FieldIdx::new(field_idx)]; + if !field.is_1zst() { + start_align = start_align.min(field.align.abi); + break; + } + } + size = cmp::max(size, st.size); + align = align.max(st.align.abi); + max_repr_align = max_repr_align.max(st.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); + combined_seed = combined_seed.wrapping_add(st.randomization_seed); + Ok(VariantLayout::from_layout(st)) + }) + .collect::, _>>()?; + + // Align the maximum variant size to the largest alignment. + size = size.align_to(align); + + // FIXME(oli-obk): deduplicate and harden these checks + if size.bytes() >= dl.obj_size_bound() { + return Err(LayoutCalculatorError::SizeOverflow); + } + + let typeck_ity = Integer::from_attr(dl, repr.discr_type()); + if typeck_ity < min_ity { + // It is a bug if Layout decided on a greater discriminant size than typeck for + // some reason at this point (based on values discriminant can take on). Mostly + // because this discriminant will be loaded, and then stored into variable of + // type calculated by typeck. Consider such case (a bug): typeck decided on + // byte-sized discriminant, but layout thinks we need a 16-bit to store all + // discriminant values. That would be a bug, because then, in codegen, in order + // to store this 16-bit discriminant into 8-bit sized temporary some of the + // space necessary to represent would have to be discarded (or layout is wrong + // on thinking it needs 16 bits) + panic!( + "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})" + ); + // However, it is fine to make discr type however large (as an optimisation) + // after this point – we’ll just truncate the value we load in codegen. + } + + // Check to see if we should use a different type for the + // discriminant. We can safely use a type with the same size + // as the alignment of the first field of each variant. + // We increase the size of the discriminant to avoid LLVM copying + // padding when it doesn't need to. This normally causes unaligned + // load/stores and excessive memcpy/memset operations. By using a + // bigger integer size, LLVM can be sure about its contents and + // won't be so conservative. + + // Use the initial field alignment + let mut ity = if repr.c() || repr.int.is_some() { + min_ity + } else { + Integer::for_align(dl, start_align).unwrap_or(min_ity) + }; + + // If the alignment is not larger than the chosen discriminant size, + // don't use the alignment as the final size. + if ity <= min_ity { + ity = min_ity; + } else { + // Patch up the variants' first few fields. + let old_ity_size = min_ity.size(); + let new_ity_size = ity.size(); + for variant in &mut layout_variants { + for i in &mut variant.field_offsets { + if *i <= old_ity_size { + assert_eq!(*i, old_ity_size); + *i = new_ity_size; + } + } + // We might be making the struct larger. + if variant.size <= old_ity_size { + variant.size = new_ity_size; + } + } + } + + let tag_valid_range = { + let tag_size = ity.size(); + let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d)); + WrappingRange::smallest_range_containing(tags, tag_size) + // We might have no inhabited variants, so pretend there's at least one. + .unwrap_or(WrappingRange { start: 0, end: 0 }) + }; + let tag = + Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }; + let mut abi = BackendRepr::Memory { sized: true }; + + let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited()); + if tag.size(dl) == size { + // Make sure we only use scalar layout when the enum is entirely its + // own tag (i.e. it has no padding nor any non-ZST variant fields). + abi = BackendRepr::Scalar(tag); + } else { + // Try to use a ScalarPair for all tagged enums. + // That's possible only if we can find a common primitive type for all variants. + let mut common_prim = None; + let mut common_prim_initialized_in_all_variants = true; + for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) { + // We skip *all* ZST here and later check if we are good in terms of alignment. + // This lets us handle some cases involving aligned ZST. + let mut fields = + iter::zip(field_layouts, &layout_variant.field_offsets).filter(|p| !p.0.is_zst()); + let (field, offset) = match (fields.next(), fields.next()) { + (None, None) => { + common_prim_initialized_in_all_variants = false; + continue; + } + (Some(pair), None) => pair, + _ => { + common_prim = None; + break; + } + }; + let prim = match field.backend_repr { + BackendRepr::Scalar(scalar) => { + common_prim_initialized_in_all_variants &= + matches!(scalar, Scalar::Initialized { .. }); + scalar.primitive() + } + _ => { + common_prim = None; + break; + } + }; + if let Some((old_prim, common_offset)) = common_prim { + // All variants must be at the same offset + if offset != common_offset { + common_prim = None; + break; + } + // This is pretty conservative. We could go fancier + // by realising that (u8, u8) could just cohabit with + // u16 or even u32. + let new_prim = match (old_prim, prim) { + // Allow all identical primitives. + (x, y) if x == y => x, + // Allow integers of the same size with differing signedness. + // We arbitrarily choose the signedness of the first variant. + (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p, + // Allow integers mixed with pointers of the same layout. + // We must represent this using a pointer, to avoid + // roundtripping pointers through ptrtoint/inttoptr. + (p @ Primitive::Pointer(_), i @ Primitive::Int(..)) + | (i @ Primitive::Int(..), p @ Primitive::Pointer(_)) + if p.size(dl) == i.size(dl) + && p.default_align(dl) == i.default_align(dl) => + { + p + } + _ => { + common_prim = None; + break; + } + }; + // We may be updating the primitive here, for example from int->ptr. + common_prim = Some((new_prim, common_offset)); + } else { + common_prim = Some((prim, offset)); + } + } + if let Some((prim, offset)) = common_prim { + let prim_scalar = if common_prim_initialized_in_all_variants { + let size = prim.size(dl); + assert!(size.bits() <= 128); + Scalar::Initialized { value: prim, valid_range: WrappingRange::full(size) } + } else { + // Common prim might be uninit. + Scalar::Union { value: prim } + }; + let pair = + LayoutData::::scalar_pair(&calculator.cx, tag, prim_scalar); + let pair_offsets = match pair.fields { + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); + offsets + } + _ => panic!("encountered a non-arbitrary layout during enum layout"), + }; + if pair_offsets[FieldIdx::new(0)] == Size::ZERO + && pair_offsets[FieldIdx::new(1)] == *offset + && align == pair.align.abi + && size == pair.size + { + // We can use `ScalarPair` only when it matches our + // already computed layout (including `#[repr(C)]`). + abi = pair.backend_repr; + } + } + } + + // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the + // variants to ensure they are consistent. This is because a downcast is + // semantically a NOP, and thus should not affect layout. + if matches!(abi, BackendRepr::Scalar(..) | BackendRepr::ScalarPair { .. }) { + for variant in &mut layout_variants { + // We only do this for variants with fields; the others are not accessed anyway. + // Also do not overwrite any already existing "clever" ABIs. + if matches!(variant.backend_repr, BackendRepr::Memory { .. } if variant.has_fields()) { + variant.backend_repr = abi; + // Also need to bump up the size, so that the entire value fits in here. + variant.size = cmp::max(variant.size, size); + } + } + } + + let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag); + + let tagged_layout = LayoutData { + variants: Variants::Multiple { + tag, + tag_encoding: TagEncoding::Direct, + tag_field: FieldIdx::new(0), + variants: layout_variants, + }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + largest_niche, + uninhabited, + backend_repr: abi, + align: AbiAlign::new(align), + size, + max_repr_align, + unadjusted_abi_align, + repr_c: repr.c(), + randomization_seed: combined_seed, + }; + + let best_layout = match (tagged_layout, niche_filling_layout) { + (tl, Some(nl)) => { + // Pick the smaller layout; otherwise, + // pick the layout with the larger niche; otherwise, + // pick tagged as it has simpler codegen. + use cmp::Ordering::*; + let niche_size = |l: &LayoutData| { + l.largest_niche.map_or(0, |n| n.available(dl)) + }; + match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) { + (Greater, _) => nl, + (Equal, Less) => nl, + _ => tl, + } + } + (tl, None) => tl, + }; + + Ok(best_layout) +} From 90a6100a33b7c6e839eff30520cf5290773d9e33 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sun, 4 Oct 2026 12:05:32 +0200 Subject: [PATCH 02/17] make calculate_niche_filling_layout a function instead of a closure --- compiler/rustc_abi/src/layout/enum_layout.rs | 319 ++++++++++--------- 1 file changed, 164 insertions(+), 155 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 1a6a8c34f7443..83999e7aa77e6 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -30,161 +30,7 @@ where return Err(LayoutCalculatorError::ReprConflict); } - let calculate_niche_filling_layout = || -> Option> { - struct VariantLayoutInfo { - align_abi: Align, - } - - if repr.inhibit_enum_layout_opt() { - return None; - } - - if variants.len() < 2 { - return None; - } - - let mut align = dl.aggregate_align; - let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align; - let mut combined_seed = repr.field_shuffle_seed; - - let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants - .iter() - .map(|v| { - let st = calculator.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?; - - variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); - - align = align.max(st.align.abi); - max_repr_align = max_repr_align.max(st.max_repr_align); - unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); - combined_seed = combined_seed.wrapping_add(st.randomization_seed); - - Some(VariantLayout::from_layout(st)) - }) - .collect::>>()?; - - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - - let all_indices = variants.indices(); - let needs_disc = - |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); - let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), - }; - - let count = - (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; - - // Use the largest niche in the largest variant. - let niche = variant_layouts[largest_variant_index].largest_niche?; - let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align); - - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); - - if this_offset + layout.size > size { - return false; - } - - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } - - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; - - true - }); - - if !all_variants_fit { - return None; - } - - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair { a: first, b: second, b_offset } => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset } - } else { - BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset } - } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; - - let layout = LayoutData { - variants: Variants::Multiple { - tag: niche_scalar, - tag_encoding: TagEncoding::Niche { - untagged_variant: largest_variant_index, - niche_variants, - niche_start, - }, - tag_field: FieldIdx::new(0), - variants: variant_layouts, - }, - fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - backend_repr: abi, - largest_niche, - uninhabited, - size, - align: AbiAlign::new(align), - max_repr_align, - unadjusted_abi_align, - repr_c: repr.c(), - randomization_seed: combined_seed, - }; - - Some(layout) - }; - - let niche_filling_layout = calculate_niche_filling_layout(); + let niche_filling_layout = calculate_niche_filling_layout(calculator, repr, variants); let discr_type = repr.discr_type(); let discr_size = Integer::from_attr(dl, discr_type).size(); @@ -507,3 +353,166 @@ where Ok(best_layout) } + +struct VariantLayoutInfo { + align_abi: Align, +} + +fn calculate_niche_filling_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( + calculator: &LayoutCalculator, + repr: &ReprOptions, + variants: &IndexSlice>, +) -> Option> +where + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + let dl = calculator.cx.data_layout(); + if repr.inhibit_enum_layout_opt() { + return None; + } + + if variants.len() < 2 { + return None; + } + + let mut align = dl.aggregate_align; + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = align; + let mut combined_seed = repr.field_shuffle_seed; + + let mut variants_info = IndexVec::::with_capacity(variants.len()); + let mut variant_layouts = variants + .iter() + .map(|v| { + let st = calculator.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?; + + variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); + + align = align.max(st.align.abi); + max_repr_align = max_repr_align.max(st.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); + combined_seed = combined_seed.wrapping_add(st.randomization_seed); + + Some(VariantLayout::from_layout(st)) + }) + .collect::>>()?; + + let largest_variant_index = variant_layouts + .iter_enumerated() + .max_by_key(|(_i, layout)| layout.size.bytes()) + .map(|(i, _layout)| i)?; + + let all_indices = variants.indices(); + let needs_disc = + |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); + let niche_variants = RangeInclusive { + start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), + last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), + }; + + let count = (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; + + // Use the largest niche in the largest variant. + let niche = variant_layouts[largest_variant_index].largest_niche?; + let (niche_start, niche_scalar) = niche.reserve(dl, count)?; + let niche_offset = niche.offset; + let niche_size = niche.value.size(dl); + let size = variant_layouts[largest_variant_index].size.align_to(align); + + let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } + + layout.largest_niche = None; + + if layout.size <= niche_offset { + // This variant will fit before the niche. + return true; + } + + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); + + if this_offset + layout.size > size { + return false; + } + + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } + + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; + + true + }); + + if !all_variants_fit { + return None; + } + + let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); + let abi = if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair { a: first, b: second, b_offset } => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset } + } else { + BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset } + } + } + _ => BackendRepr::Memory { sized: true }, + } + } else { + BackendRepr::Memory { sized: true } + }; + + let layout = LayoutData { + variants: Variants::Multiple { + tag: niche_scalar, + tag_encoding: TagEncoding::Niche { + untagged_variant: largest_variant_index, + niche_variants, + niche_start, + }, + tag_field: FieldIdx::new(0), + variants: variant_layouts, + }, + fields: FieldsShape::Arbitrary { + offsets: [niche_offset].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + backend_repr: abi, + largest_niche, + uninhabited, + size, + align: AbiAlign::new(align), + max_repr_align, + unadjusted_abi_align, + repr_c: repr.c(), + randomization_seed: combined_seed, + }; + + Some(layout) +} From e60b3da5c49782ed1c915016dedd61f9fbd97a06 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sun, 4 Oct 2026 12:09:36 +0200 Subject: [PATCH 03/17] calculate_niche_filling_layout: calculate_niche_abi --- compiler/rustc_abi/src/layout/enum_layout.rs | 74 +++++++++++++------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 83999e7aa77e6..7cab02fd3aef6 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -358,6 +358,44 @@ struct VariantLayoutInfo { align_abi: Align, } +fn calculate_niche_abi( + variant_layouts: &IndexSlice>, + variants_info: &IndexSlice, + niche_scalar: Scalar, + largest_variant_index: VariantIdx, + size: Size, + align: Align, + niche_offset: Size, +) -> BackendRepr { + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair { a: first, b: second, b_offset } => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + // + // How can this be nonzero when everything else is a ZST? `others_zst` is true here + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset } + } else { + BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset } + } + } + _ => BackendRepr::Memory { sized: true }, + } + } else { + BackendRepr::Memory { sized: true } + } +} + fn calculate_niche_filling_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( calculator: &LayoutCalculator, repr: &ReprOptions, @@ -460,33 +498,17 @@ where } let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair { a: first, b: second, b_offset } => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair { a: niche_scalar, b: second.to_union(), b_offset } - } else { - BackendRepr::ScalarPair { a: first.to_union(), b: niche_scalar, b_offset } - } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; + + let abi = calculate_niche_abi( + &variant_layouts, + &variants_info, + niche_scalar, + largest_variant_index, + size, + align, + niche_offset, + ); let layout = LayoutData { variants: Variants::Multiple { From dd25f7f026df7545d992d8d205dbc42e1db03b95 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sun, 4 Oct 2026 12:13:54 +0200 Subject: [PATCH 04/17] calculate_niche_filling_layout: try_fixup_non_niche_variants --- compiler/rustc_abi/src/layout/enum_layout.rs | 89 +++++++++++--------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 7cab02fd3aef6..d215a920918ba 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -358,6 +358,48 @@ struct VariantLayoutInfo { align_abi: Align, } +fn try_fixup_non_niche_variants( + mut variant_layouts: IndexVec>, + variants_info: &IndexSlice, + largest_variant_index: VariantIdx, + niche_offset: Size, + niche_size: Size, + size: Size, +) -> Option>> { + for (i, layout) in variant_layouts.iter_enumerated_mut() { + if i == largest_variant_index { + continue; + } + + layout.largest_niche = None; + + if layout.size <= niche_offset { + // This variant will fit before the niche. + continue; + } + + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); + + if this_offset + layout.size > size { + return None; + } + + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } + + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; + } + Some(variant_layouts) +} + fn calculate_niche_abi( variant_layouts: &IndexSlice>, variants_info: &IndexSlice, @@ -421,7 +463,7 @@ where let mut combined_seed = repr.field_shuffle_seed; let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants + let variant_layouts = variants .iter() .map(|v| { let st = calculator.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?; @@ -459,43 +501,14 @@ where let niche_size = niche.value.size(dl); let size = variant_layouts[largest_variant_index].size.align_to(align); - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); - - if this_offset + layout.size > size { - return false; - } - - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } - - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; - - true - }); - - if !all_variants_fit { - return None; - } + let variant_layouts = try_fixup_non_niche_variants( + variant_layouts, + &variants_info, + largest_variant_index, + niche_offset, + niche_size, + size, + )?; let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); From 5066a3673284f3d2cc37dbd4e0d2ba33dedb5128 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sun, 4 Oct 2026 12:18:12 +0200 Subject: [PATCH 05/17] calculate_niche_filling_layout: remove redundant variables --- compiler/rustc_abi/src/layout/enum_layout.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index d215a920918ba..a3cf3c95012d0 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -497,22 +497,17 @@ where // Use the largest niche in the largest variant. let niche = variant_layouts[largest_variant_index].largest_niche?; let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); let size = variant_layouts[largest_variant_index].size.align_to(align); let variant_layouts = try_fixup_non_niche_variants( variant_layouts, &variants_info, largest_variant_index, - niche_offset, - niche_size, + niche.offset, + niche.value.size(dl), size, )?; - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = calculate_niche_abi( &variant_layouts, &variants_info, @@ -520,10 +515,11 @@ where largest_variant_index, size, align, - niche_offset, + niche.offset, ); let layout = LayoutData { + uninhabited: variant_layouts.iter().all(|v| v.is_uninhabited()), variants: Variants::Multiple { tag: niche_scalar, tag_encoding: TagEncoding::Niche { @@ -535,12 +531,11 @@ where variants: variant_layouts, }, fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), + offsets: [niche.offset].into(), in_memory_order: [FieldIdx::new(0)].into(), }, backend_repr: abi, - largest_niche, - uninhabited, + largest_niche: Niche::from_scalar(dl, niche.offset, niche_scalar), size, align: AbiAlign::new(align), max_repr_align, From 065aab5bf477d0dbb9b100a868590f546a3ee78c Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sun, 4 Oct 2026 12:25:36 +0200 Subject: [PATCH 06/17] move tagged layout calculation to calculate_tagged_layout --- compiler/rustc_abi/src/layout/enum_layout.rs | 61 +++++++++++++------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index a3cf3c95012d0..44613d78b1008 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -32,6 +32,43 @@ where let niche_filling_layout = calculate_niche_filling_layout(calculator, repr, variants); + let tagged_layout = + calculate_tagged_layout(calculator, repr, variants, discr_range_of_repr, discriminants)?; + + let best_layout = match (tagged_layout, niche_filling_layout) { + (tl, Some(nl)) => { + // Pick the smaller layout; otherwise, + // pick the layout with the larger niche; otherwise, + // pick tagged as it has simpler codegen. + use cmp::Ordering::*; + let niche_size = |l: &LayoutData| { + l.largest_niche.map_or(0, |n| n.available(dl)) + }; + match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) { + (Greater, _) => nl, + (Equal, Less) => nl, + _ => tl, + } + } + (tl, None) => tl, + }; + + Ok(best_layout) +} + +fn calculate_tagged_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( + calculator: &LayoutCalculator, + repr: &ReprOptions, + variants: &IndexSlice>, + discr_range_of_repr: impl Fn(RangeFrom, RangeToInclusive) -> (Integer, bool), + discriminants: impl Iterator, +) -> LayoutCalculatorResult +where + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + let dl = calculator.cx.data_layout(); let discr_type = repr.discr_type(); let discr_size = Integer::from_attr(dl, discr_type).size(); @@ -311,7 +348,7 @@ where let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag); - let tagged_layout = LayoutData { + Ok(LayoutData { variants: Variants::Multiple { tag, tag_encoding: TagEncoding::Direct, @@ -331,27 +368,7 @@ where unadjusted_abi_align, repr_c: repr.c(), randomization_seed: combined_seed, - }; - - let best_layout = match (tagged_layout, niche_filling_layout) { - (tl, Some(nl)) => { - // Pick the smaller layout; otherwise, - // pick the layout with the larger niche; otherwise, - // pick tagged as it has simpler codegen. - use cmp::Ordering::*; - let niche_size = |l: &LayoutData| { - l.largest_niche.map_or(0, |n| n.available(dl)) - }; - match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) { - (Greater, _) => nl, - (Equal, Less) => nl, - _ => tl, - } - } - (tl, None) => tl, - }; - - Ok(best_layout) + }) } struct VariantLayoutInfo { From 958995fb96641687eb253f606d45222666d6d598 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:10:19 +0200 Subject: [PATCH 07/17] calculate_tagged_layout: constructable_variant --- compiler/rustc_abi/src/layout/enum_layout.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 44613d78b1008..48c0950724d70 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -56,6 +56,15 @@ where Ok(best_layout) } +fn constructable_variant<'a, VariantIdx, FieldIdx, F>(variant: &IndexSlice) -> bool +where + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + variant.iter().all(|f| !f.is_uninhabited()) +} + fn calculate_tagged_layout<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( calculator: &LayoutCalculator, repr: &ReprOptions, @@ -73,7 +82,7 @@ where let discr_size = Integer::from_attr(dl, discr_type).size(); let necessary_discriminants: Vec = discriminants - .filter(|&(i, _)| repr.c() || variants[i].iter().all(|f| !f.is_uninhabited())) + .filter(|&(i, _)| repr.c() || constructable_variant(&variants[i])) .map(|(_, val)| val) .collect(); From 0534877725e5f08afc04562fb171c4a0eb3ea4a3 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:24:48 +0200 Subject: [PATCH 08/17] calculate_tagged_layout: calculate_tagged_abi --- compiler/rustc_abi/src/layout/enum_layout.rs | 262 +++++++++++-------- 1 file changed, 158 insertions(+), 104 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 48c0950724d70..3c262bde5fb3c 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -9,7 +9,7 @@ use super::{LayoutCalculator, LayoutCalculatorError, LayoutCalculatorResult, abs use crate::{ AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer, LayoutData, Niche, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding, - VariantLayout, Variants, WrappingRange, + TargetDataLayout, VariantLayout, Variants, WrappingRange, }; pub(super) fn layout_of_enum<'a, Cx: HasDataLayout, FieldIdx, VariantIdx, F>( @@ -56,6 +56,162 @@ where Ok(best_layout) } +struct CommonPrimitive { + primitive: Primitive, + offset: Size, + initialized_in_all_variants: bool, +} + +fn calculate_common_primitive<'a, VariantIdx, FieldIdx, F>( + dl: &TargetDataLayout, + layout_variants: &IndexSlice>, + variants: &IndexSlice>, +) -> Option +where + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + // Try to use a ScalarPair for all tagged enums. + // That's possible only if we can find a common primitive type for all variants. + let mut common_prim = None; + let mut initialized_in_all_variants = true; + for (field_layouts, layout_variant) in iter::zip(variants, layout_variants) { + // We skip *all* ZST here and later check if we are good in terms of alignment. + // This lets us handle some cases involving aligned ZST. + let mut fields = + iter::zip(field_layouts, &layout_variant.field_offsets).filter(|p| !p.0.is_zst()); + let (field, offset) = match (fields.next(), fields.next()) { + (None, None) => { + initialized_in_all_variants = false; + continue; + } + (Some(pair), None) => pair, + _ => { + return None; + } + }; + let prim = match field.backend_repr { + BackendRepr::Scalar(scalar) => { + initialized_in_all_variants &= matches!(scalar, Scalar::Initialized { .. }); + scalar.primitive() + } + _ => { + return None; + } + }; + if let Some((old_prim, common_offset)) = common_prim { + // All variants must be at the same offset + if offset != common_offset { + return None; + } + // This is pretty conservative. We could go fancier + // by realising that (u8, u8) could just cohabit with + // u16 or even u32. + let new_prim = match (old_prim, prim) { + // Allow all identical primitives. + (x, y) if x == y => x, + // Allow integers of the same size with differing signedness. + // We arbitrarily choose the signedness of the first variant. + (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p, + // Allow integers mixed with pointers of the same layout. + // We must represent this using a pointer, to avoid + // roundtripping pointers through ptrtoint/inttoptr. + (p @ Primitive::Pointer(_), i @ Primitive::Int(..)) + | (i @ Primitive::Int(..), p @ Primitive::Pointer(_)) + if p.size(dl) == i.size(dl) && p.default_align(dl) == i.default_align(dl) => + { + p + } + _ => { + return None; + } + }; + // We may be updating the primitive here, for example from int->ptr. + common_prim = Some((new_prim, common_offset)); + } else { + common_prim = Some((prim, offset)); + } + } + common_prim.map(|(primitive, offset)| CommonPrimitive { + primitive, + offset: *offset, + initialized_in_all_variants, + }) +} + +fn scalar_pair_repr<'a, Cx, VariantIdx, FieldIdx, F>( + calculator: &LayoutCalculator, + layout_variants: &IndexSlice>, + variants: &IndexSlice>, + size: Size, + align: Align, + tag: Scalar, +) -> Option +where + Cx: HasDataLayout, + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + let CommonPrimitive { primitive, offset, initialized_in_all_variants } = + calculate_common_primitive(calculator.cx.data_layout(), layout_variants, variants)?; + let prim_scalar = if initialized_in_all_variants { + let size = primitive.size(calculator.cx.data_layout()); + assert!(size.bits() <= 128); + Scalar::Initialized { value: primitive, valid_range: WrappingRange::full(size) } + } else { + // Common prim might be uninit. + Scalar::Union { value: primitive } + }; + let pair = LayoutData::::scalar_pair(&calculator.cx, tag, prim_scalar); + let pair_offsets = match pair.fields { + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); + offsets + } + _ => panic!("encountered a non-arbitrary layout during enum layout"), + }; + if pair_offsets[FieldIdx::new(0)] == Size::ZERO + && pair_offsets[FieldIdx::new(1)] == offset + && align == pair.align.abi + && size == pair.size + { + // We can use `ScalarPair` only when it matches our + // already computed layout (including `#[repr(C)]`). + Some(pair.backend_repr) + } else { + None + } +} + +fn calculate_tagged_abi<'a, Cx, VariantIdx, FieldIdx, F>( + calculator: &LayoutCalculator, + layout_variants: &IndexSlice>, + variants: &IndexSlice>, + size: Size, + align: Align, + tag: Scalar, +) -> BackendRepr +where + Cx: HasDataLayout, + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, +{ + if tag.size(calculator.cx.data_layout()) == size { + // Make sure we only use scalar layout when the enum is entirely its + // own tag (i.e. it has no padding nor any non-ZST variant fields). + BackendRepr::Scalar(tag) + } else if let Some(repr) = + scalar_pair_repr(calculator, layout_variants, variants, size, align, tag) + { + repr + } else { + BackendRepr::Memory { sized: true } + } +} + fn constructable_variant<'a, VariantIdx, FieldIdx, F>(variant: &IndexSlice) -> bool where FieldIdx: Idx, @@ -235,110 +391,8 @@ where }; let tag = Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }; - let mut abi = BackendRepr::Memory { sized: true }; - let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited()); - if tag.size(dl) == size { - // Make sure we only use scalar layout when the enum is entirely its - // own tag (i.e. it has no padding nor any non-ZST variant fields). - abi = BackendRepr::Scalar(tag); - } else { - // Try to use a ScalarPair for all tagged enums. - // That's possible only if we can find a common primitive type for all variants. - let mut common_prim = None; - let mut common_prim_initialized_in_all_variants = true; - for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) { - // We skip *all* ZST here and later check if we are good in terms of alignment. - // This lets us handle some cases involving aligned ZST. - let mut fields = - iter::zip(field_layouts, &layout_variant.field_offsets).filter(|p| !p.0.is_zst()); - let (field, offset) = match (fields.next(), fields.next()) { - (None, None) => { - common_prim_initialized_in_all_variants = false; - continue; - } - (Some(pair), None) => pair, - _ => { - common_prim = None; - break; - } - }; - let prim = match field.backend_repr { - BackendRepr::Scalar(scalar) => { - common_prim_initialized_in_all_variants &= - matches!(scalar, Scalar::Initialized { .. }); - scalar.primitive() - } - _ => { - common_prim = None; - break; - } - }; - if let Some((old_prim, common_offset)) = common_prim { - // All variants must be at the same offset - if offset != common_offset { - common_prim = None; - break; - } - // This is pretty conservative. We could go fancier - // by realising that (u8, u8) could just cohabit with - // u16 or even u32. - let new_prim = match (old_prim, prim) { - // Allow all identical primitives. - (x, y) if x == y => x, - // Allow integers of the same size with differing signedness. - // We arbitrarily choose the signedness of the first variant. - (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p, - // Allow integers mixed with pointers of the same layout. - // We must represent this using a pointer, to avoid - // roundtripping pointers through ptrtoint/inttoptr. - (p @ Primitive::Pointer(_), i @ Primitive::Int(..)) - | (i @ Primitive::Int(..), p @ Primitive::Pointer(_)) - if p.size(dl) == i.size(dl) - && p.default_align(dl) == i.default_align(dl) => - { - p - } - _ => { - common_prim = None; - break; - } - }; - // We may be updating the primitive here, for example from int->ptr. - common_prim = Some((new_prim, common_offset)); - } else { - common_prim = Some((prim, offset)); - } - } - if let Some((prim, offset)) = common_prim { - let prim_scalar = if common_prim_initialized_in_all_variants { - let size = prim.size(dl); - assert!(size.bits() <= 128); - Scalar::Initialized { value: prim, valid_range: WrappingRange::full(size) } - } else { - // Common prim might be uninit. - Scalar::Union { value: prim } - }; - let pair = - LayoutData::::scalar_pair(&calculator.cx, tag, prim_scalar); - let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { - assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); - offsets - } - _ => panic!("encountered a non-arbitrary layout during enum layout"), - }; - if pair_offsets[FieldIdx::new(0)] == Size::ZERO - && pair_offsets[FieldIdx::new(1)] == *offset - && align == pair.align.abi - && size == pair.size - { - // We can use `ScalarPair` only when it matches our - // already computed layout (including `#[repr(C)]`). - abi = pair.backend_repr; - } - } - } + let abi = calculate_tagged_abi(calculator, &layout_variants, &variants, size, align, tag); // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the // variants to ensure they are consistent. This is because a downcast is From 372bcf23d5f0b11e04d57c82abae61f70d9dac29 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:30:11 +0200 Subject: [PATCH 09/17] calculate_tagged_layout: remove discr_type variable --- compiler/rustc_abi/src/layout/enum_layout.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 3c262bde5fb3c..035ce72840136 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -234,8 +234,7 @@ where F: Deref> + fmt::Debug + Copy, { let dl = calculator.cx.data_layout(); - let discr_type = repr.discr_type(); - let discr_size = Integer::from_attr(dl, discr_type).size(); + let discr_size = Integer::from_attr(dl, repr.discr_type()).size(); let necessary_discriminants: Vec = discriminants .filter(|&(i, _)| repr.c() || constructable_variant(&variants[i])) @@ -244,7 +243,7 @@ where // When picking the integer to use, we respect how the discriminants were written // in the original rust code, rather than looking only at the bit pattern. - let (min_negative, max_positive): (i128, u128) = if discr_type.is_signed() { + let (min_negative, max_positive): (i128, u128) = if repr.discr_type().is_signed() { necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold( (0_i128, 0_u128), |(min, max), val| { From c108cfdb88aa70520f8dd129b22adc23ea5a90eb Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:32:26 +0200 Subject: [PATCH 10/17] calculate_tagged_layout: make size immutable --- compiler/rustc_abi/src/layout/enum_layout.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 035ce72840136..3a7fff46c06b7 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -318,7 +318,7 @@ where .collect::, _>>()?; // Align the maximum variant size to the largest alignment. - size = size.align_to(align); + let size = size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { From d09e9b58405a086e289ac62700611db8d45419a2 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:40:25 +0200 Subject: [PATCH 11/17] calculate_niche_layout: remove all_indices --- compiler/rustc_abi/src/layout/enum_layout.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 3a7fff46c06b7..2c625e303a342 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -563,12 +563,11 @@ where .max_by_key(|(_i, layout)| layout.size.bytes()) .map(|(i, _layout)| i)?; - let all_indices = variants.indices(); let needs_disc = |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), + start: variants.indices().find(|v| needs_disc(*v)).unwrap(), + last: variants.indices().rev().find(|v| needs_disc(*v)).unwrap(), }; let count = (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; From 767a215f6ba4887961f327b3c27cd0917338f03f Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:53:27 +0200 Subject: [PATCH 12/17] calculate_tagged_layout: sanity_check_ity --- compiler/rustc_abi/src/layout/enum_layout.rs | 38 +++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 2c625e303a342..a527ee7f87efc 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -212,6 +212,26 @@ where } } +fn sanity_check_ity(dl: &TargetDataLayout, repr: &ReprOptions, min_ity: Integer) { + let typeck_ity = Integer::from_attr(dl, repr.discr_type()); + if typeck_ity < min_ity { + // It is a bug if Layout decided on a greater discriminant size than typeck for + // some reason at this point (based on values discriminant can take on). Mostly + // because this discriminant will be loaded, and then stored into variable of + // type calculated by typeck. Consider such case (a bug): typeck decided on + // byte-sized discriminant, but layout thinks we need a 16-bit to store all + // discriminant values. That would be a bug, because then, in codegen, in order + // to store this 16-bit discriminant into 8-bit sized temporary some of the + // space necessary to represent would have to be discarded (or layout is wrong + // on thinking it needs 16 bits) + panic!( + "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})" + ); + // However, it is fine to make discr type however large (as an optimisation) + // after this point – we’ll just truncate the value we load in codegen. + } +} + fn constructable_variant<'a, VariantIdx, FieldIdx, F>(variant: &IndexSlice) -> bool where FieldIdx: Idx, @@ -325,23 +345,7 @@ where return Err(LayoutCalculatorError::SizeOverflow); } - let typeck_ity = Integer::from_attr(dl, repr.discr_type()); - if typeck_ity < min_ity { - // It is a bug if Layout decided on a greater discriminant size than typeck for - // some reason at this point (based on values discriminant can take on). Mostly - // because this discriminant will be loaded, and then stored into variable of - // type calculated by typeck. Consider such case (a bug): typeck decided on - // byte-sized discriminant, but layout thinks we need a 16-bit to store all - // discriminant values. That would be a bug, because then, in codegen, in order - // to store this 16-bit discriminant into 8-bit sized temporary some of the - // space necessary to represent would have to be discarded (or layout is wrong - // on thinking it needs 16 bits) - panic!( - "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})" - ); - // However, it is fine to make discr type however large (as an optimisation) - // after this point – we’ll just truncate the value we load in codegen. - } + sanity_check_ity(dl, repr, min_ity); // Check to see if we should use a different type for the // discriminant. We can safely use a type with the same size From 7dfce3afe6b2d05223dfd75d37703b12fb4759f8 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:55:06 +0200 Subject: [PATCH 13/17] calculate_tagged_layout: remove uninhabited --- compiler/rustc_abi/src/layout/enum_layout.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index a527ee7f87efc..12c7d95c2c8a7 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -394,7 +394,6 @@ where }; let tag = Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }; - let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited()); let abi = calculate_tagged_abi(calculator, &layout_variants, &variants, size, align, tag); // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the @@ -415,6 +414,7 @@ where let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag); Ok(LayoutData { + uninhabited: layout_variants.iter().all(|v| v.is_uninhabited()), variants: Variants::Multiple { tag, tag_encoding: TagEncoding::Direct, @@ -426,7 +426,6 @@ where in_memory_order: [FieldIdx::new(0)].into(), }, largest_niche, - uninhabited, backend_repr: abi, align: AbiAlign::new(align), size, From 940c1446dc6a0ed99d83021d3fc619cbfa122f25 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 09:56:21 +0200 Subject: [PATCH 14/17] calculate_tagged_layout: make ity immutable --- compiler/rustc_abi/src/layout/enum_layout.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 12c7d95c2c8a7..fbd3ea587a9bf 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -357,17 +357,15 @@ where // won't be so conservative. // Use the initial field alignment - let mut ity = if repr.c() || repr.int.is_some() { + let ity = if repr.c() || repr.int.is_some() { min_ity } else { - Integer::for_align(dl, start_align).unwrap_or(min_ity) + Integer::for_align(dl, start_align).unwrap_or(min_ity).max(min_ity) }; // If the alignment is not larger than the chosen discriminant size, // don't use the alignment as the final size. - if ity <= min_ity { - ity = min_ity; - } else { + if ity > min_ity { // Patch up the variants' first few fields. let old_ity_size = min_ity.size(); let new_ity_size = ity.size(); From d144ede1e8202af5a31d5a9737950c9403c85cd0 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 10:04:04 +0200 Subject: [PATCH 15/17] calculate_tagged_layout: make prefix_align immutable --- compiler/rustc_abi/src/layout/enum_layout.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index fbd3ea587a9bf..a4bb435bc849a 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -301,14 +301,14 @@ where // the alignment of the union. (This value is used both for // determining the alignment of the overall enum, and the // determining the alignment of the payload after the tag.) - let mut prefix_align = min_ity.align(dl).abi; - if repr.c() { - for fields in variants { - for field in fields { - prefix_align = prefix_align.max(field.align.abi); - } - } - } + let prefix_align = if repr.c() { + variants + .iter() + .flatten() + .fold(min_ity.align(dl).abi, |prefix_align, field| prefix_align.max(field.align.abi)) + } else { + min_ity.align(dl).abi + }; // Create the set of structs that represent each variant. let mut layout_variants = variants From 7739d09a8e32e4bf3ebe7cb576eb418cb7906f57 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 10:22:14 +0200 Subject: [PATCH 16/17] calculate_tagged_layout: calculate_tagged_tag --- compiler/rustc_abi/src/layout/enum_layout.rs | 117 +++++++++++-------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index a4bb435bc849a..1efed918ef8d9 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -232,6 +232,67 @@ fn sanity_check_ity(dl: &TargetDataLayout, repr: &ReprOptions, min_ity: Integer) } } +fn calculate_tagged_tag( + dl: &TargetDataLayout, + repr: &ReprOptions, + min_ity: Integer, + start_align: Align, + signed: bool, + necessary_discriminants: Vec, + layout_variants: &mut IndexSlice>, +) -> Scalar +where + FieldIdx: Idx, + VariantIdx: Idx, +{ + sanity_check_ity(dl, repr, min_ity); + + // Check to see if we should use a different type for the + // discriminant. We can safely use a type with the same size + // as the alignment of the first field of each variant. + // We increase the size of the discriminant to avoid LLVM copying + // padding when it doesn't need to. This normally causes unaligned + // load/stores and excessive memcpy/memset operations. By using a + // bigger integer size, LLVM can be sure about its contents and + // won't be so conservative. + + // Use the initial field alignment + let ity = if repr.c() || repr.int.is_some() { + min_ity + } else { + Integer::for_align(dl, start_align).unwrap_or(min_ity).max(min_ity) + }; + + // If the alignment is not larger than the chosen discriminant size, + // don't use the alignment as the final size. + if ity > min_ity { + // Patch up the variants' first few fields. + let old_ity_size = min_ity.size(); + let new_ity_size = ity.size(); + for variant in layout_variants { + for i in &mut variant.field_offsets { + if *i <= old_ity_size { + assert_eq!(*i, old_ity_size); + *i = new_ity_size; + } + } + // We might be making the struct larger. + if variant.size <= old_ity_size { + variant.size = new_ity_size; + } + } + } + + let tag_valid_range = { + let tag_size = ity.size(); + let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d)); + WrappingRange::smallest_range_containing(tags, tag_size) + // We might have no inhabited variants, so pretend there's at least one. + .unwrap_or(WrappingRange { start: 0, end: 0 }) + }; + Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range } +} + fn constructable_variant<'a, VariantIdx, FieldIdx, F>(variant: &IndexSlice) -> bool where FieldIdx: Idx, @@ -345,53 +406,15 @@ where return Err(LayoutCalculatorError::SizeOverflow); } - sanity_check_ity(dl, repr, min_ity); - - // Check to see if we should use a different type for the - // discriminant. We can safely use a type with the same size - // as the alignment of the first field of each variant. - // We increase the size of the discriminant to avoid LLVM copying - // padding when it doesn't need to. This normally causes unaligned - // load/stores and excessive memcpy/memset operations. By using a - // bigger integer size, LLVM can be sure about its contents and - // won't be so conservative. - - // Use the initial field alignment - let ity = if repr.c() || repr.int.is_some() { - min_ity - } else { - Integer::for_align(dl, start_align).unwrap_or(min_ity).max(min_ity) - }; - - // If the alignment is not larger than the chosen discriminant size, - // don't use the alignment as the final size. - if ity > min_ity { - // Patch up the variants' first few fields. - let old_ity_size = min_ity.size(); - let new_ity_size = ity.size(); - for variant in &mut layout_variants { - for i in &mut variant.field_offsets { - if *i <= old_ity_size { - assert_eq!(*i, old_ity_size); - *i = new_ity_size; - } - } - // We might be making the struct larger. - if variant.size <= old_ity_size { - variant.size = new_ity_size; - } - } - } - - let tag_valid_range = { - let tag_size = ity.size(); - let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d)); - WrappingRange::smallest_range_containing(tags, tag_size) - // We might have no inhabited variants, so pretend there's at least one. - .unwrap_or(WrappingRange { start: 0, end: 0 }) - }; - let tag = - Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }; + let tag = calculate_tagged_tag( + dl, + repr, + min_ity, + start_align, + signed, + necessary_discriminants, + &mut layout_variants, + ); let abi = calculate_tagged_abi(calculator, &layout_variants, &variants, size, align, tag); // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the From 80bc421492f3f9e4f475f25ffc914459b28caf22 Mon Sep 17 00:00:00 2001 From: Matyas Susits Date: Sat, 3 Oct 2026 10:36:00 +0200 Subject: [PATCH 17/17] calculate_tagged_layout: calculate_tagged_min_ity --- compiler/rustc_abi/src/layout/enum_layout.rs | 54 +++++++++++--------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/compiler/rustc_abi/src/layout/enum_layout.rs b/compiler/rustc_abi/src/layout/enum_layout.rs index 1efed918ef8d9..f655cefa1c890 100644 --- a/compiler/rustc_abi/src/layout/enum_layout.rs +++ b/compiler/rustc_abi/src/layout/enum_layout.rs @@ -232,6 +232,35 @@ fn sanity_check_ity(dl: &TargetDataLayout, repr: &ReprOptions, min_ity: Integer) } } +fn calculate_tagged_min_ity( + repr: &ReprOptions, + necessary_discriminants: &[u128], + discr_size: Size, + discr_range_of_repr: impl Fn(RangeFrom, RangeToInclusive) -> (Integer, bool), +) -> (Integer, bool) { + // When picking the integer to use, we respect how the discriminants were written + // in the original rust code, rather than looking only at the bit pattern. + let (min_negative, max_positive): (i128, u128) = if repr.discr_type().is_signed() { + necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold( + (0_i128, 0_u128), + |(min, max), val| { + if let Ok(val) = u128::try_from(val) { + (min, max.max(val)) + } else { + (min.min(val), max) + } + }, + ) + } else { + // We might have no inhabited variants, so pretend there's at least one. + (0, necessary_discriminants.iter().copied().max().unwrap_or(0)) + }; + trace!(?min_negative, ?max_positive); + + // Integer::discr_range_of_repr(tcx, ty, &repr, min, max); + discr_range_of_repr(RangeFrom { start: min_negative }, RangeToInclusive { last: max_positive }) +} + fn calculate_tagged_tag( dl: &TargetDataLayout, repr: &ReprOptions, @@ -322,29 +351,8 @@ where .map(|(_, val)| val) .collect(); - // When picking the integer to use, we respect how the discriminants were written - // in the original rust code, rather than looking only at the bit pattern. - let (min_negative, max_positive): (i128, u128) = if repr.discr_type().is_signed() { - necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold( - (0_i128, 0_u128), - |(min, max), val| { - if let Ok(val) = u128::try_from(val) { - (min, max.max(val)) - } else { - (min.min(val), max) - } - }, - ) - } else { - // We might have no inhabited variants, so pretend there's at least one. - (0, necessary_discriminants.iter().copied().max().unwrap_or(0)) - }; - trace!(?min_negative, ?max_positive); - - let (min_ity, signed) = discr_range_of_repr( - RangeFrom { start: min_negative }, - RangeToInclusive { last: max_positive }, - ); //Integer::discr_range_of_repr(tcx, ty, &repr, min, max); + let (min_ity, signed) = + calculate_tagged_min_ity(repr, &necessary_discriminants, discr_size, discr_range_of_repr); let mut align = dl.aggregate_align; let mut max_repr_align = repr.align;