diff --git a/libwild/src/elf.rs b/libwild/src/elf.rs index da50564fc..cae898afa 100644 --- a/libwild/src/elf.rs +++ b/libwild/src/elf.rs @@ -1457,6 +1457,10 @@ impl platform::Platform for Elf { mem_sizes.increment(part_id::RELA_DYN_GENERAL, elf::RELA_ENTRY_SIZE); } else if flags.is_address() && output_kind.is_relocatable() { if args.is_relr_enabled() { + // TODO: Implement bitmap packing for GOT-based RELR entries. + // Currently counts flat (one entry per GOT slot) to match the + // writer's write_relr_entry_flat. Bitmap packing requires splitting + // the GOT so relative relocations are contiguous. mem_sizes.increment(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); } else { mem_sizes.increment(part_id::RELA_DYN_RELATIVE, elf::RELA_ENTRY_SIZE); @@ -2827,6 +2831,7 @@ fn process_eh_frame_relocations<'data, 'scope, A: Arch, R: Reloc queue, false, scope, + &mut RelrRunState::default(), // eh_frame relocations are never RELR-eligible )?; if rel.symbol().is_none() { @@ -2929,6 +2934,7 @@ fn process_section_exception_frames<'data, 'scope, A: Arch, R: R queue, true, scope, + &mut RelrRunState::default(), // eh_frame relocations are never RELR-eligible )?; } common.format_specific.exception_frame_relocations += @@ -3383,6 +3389,20 @@ impl<'data> platform::DynamicTagValues<'data> for DynamicTagValues<'data> { } } +/// Tracks RELR bitmap packing state within a single input section during layout. +/// Only packs within input sections — avoids cross-section address ordering +/// issues in Wild's parallel layout phase. +#[derive(Default)] +enum RelrRunState { + /// No run in progress. + #[default] + NoRun, + /// In a run. `bitmap_base` is the start of the current bitmap window + /// (addr+8 initially, advances by 63*8 per emitted bitmap). + /// `has_bitmap` tracks whether a bitmap entry is allocated for this window. + InRun { bitmap_base: u64, has_bitmap: bool }, +} + struct SymDebug<'data>(pub(crate) &'data crate::elf::SymtabEntry); impl std::fmt::Display for SymDebug<'_> { @@ -4964,6 +4984,7 @@ fn load_section_relocations<'scope, 'data, A: Arch, R: Relocatio scope: &Scope<'scope>, ) -> Result { let mut modifier = RelocationModifier::Normal; + let mut relr_run = RelrRunState::default(); for rel in relocations { if modifier == RelocationModifier::SkipNextRelocation { modifier = RelocationModifier::Normal; @@ -4982,6 +5003,7 @@ fn load_section_relocations<'scope, 'data, A: Arch, R: Relocatio queue, false, scope, + &mut relr_run, ) .with_context(|| { format!( @@ -5005,6 +5027,7 @@ fn process_relocation<'data, 'scope, A: Arch, R: Relocation>( queue: &mut layout::LocalWorkQueue, is_debug_section: bool, scope: &Scope<'scope>, + relr_run: &mut RelrRunState, ) -> Result { let args = resources.symbol_db.args; let mut next_modifier = RelocationModifier::Normal; @@ -5101,9 +5124,69 @@ fn process_relocation<'data, 'scope, A: Arch, R: Relocation>( && flags.is_address() { if section_is_writable { - // Odd offsets mean bitmaps in RELR, so we need to fall back to RELA for them. + // Odd offsets can't be encoded as RELR address entries (LSB used as + // bitmap marker), so fall back to RELA for them. if resources.symbol_db.args.is_relr_enabled() && rel.offset().is_multiple_of(2) { - common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + let offset = rel.offset(); + const BITMAP_SLOTS: u64 = 63; + const WORD_SIZE: u64 = 8; + *relr_run = match *relr_run { + RelrRunState::NoRun => { + // No run — start one with an address entry. + common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + RelrRunState::InRun { + bitmap_base: offset + WORD_SIZE, + has_bitmap: false, + } + } + RelrRunState::InRun { + mut bitmap_base, + has_bitmap, + } => { + let d = offset.wrapping_sub(bitmap_base); + if d % WORD_SIZE == 0 && d < BITMAP_SLOTS * WORD_SIZE { + // Fits in current bitmap window. + if !has_bitmap { + // First member in this window: allocate bitmap entry. + common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + } + RelrRunState::InRun { + bitmap_base, + has_bitmap: true, + } + } else if has_bitmap && d % WORD_SIZE == 0 { + // Current window has bits — try next window. + // lld only advances to a new bitmap if the current one is + // non-empty (breaks on empty bitmap). Same rule here. + let next_base = bitmap_base + BITMAP_SLOTS * WORD_SIZE; + let d2 = offset.wrapping_sub(next_base); + if d2.is_multiple_of(WORD_SIZE) && d2 < BITMAP_SLOTS * WORD_SIZE { + // Fits in next window — new bitmap entry, advance window. + common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + bitmap_base = next_base; + RelrRunState::InRun { + bitmap_base, + has_bitmap: true, + } + } else { + // Gap too large — start new address entry. + common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + RelrRunState::InRun { + bitmap_base: offset + WORD_SIZE, + has_bitmap: false, + } + } + } else { + // Unaligned, or current window empty and out of range. + // lld breaks on empty bitmap — start new address entry. + common.allocate(part_id::RELR_DYN, elf::RELR_ENTRY_SIZE); + RelrRunState::InRun { + bitmap_base: offset + WORD_SIZE, + has_bitmap: false, + } + } + } + }; } else { common.allocate(part_id::RELA_DYN_RELATIVE, elf::RELA_ENTRY_SIZE); } diff --git a/libwild/src/elf_writer.rs b/libwild/src/elf_writer.rs index 8cf948afa..e39daf315 100644 --- a/libwild/src/elf_writer.rs +++ b/libwild/src/elf_writer.rs @@ -596,6 +596,19 @@ impl<'out> VersionWriter<'out> { } } +/// Writer-phase RELR bitmap packing state. +/// Carries slice indices so bitmap entries can be updated in-place. +#[derive(Default)] +enum RelrWriterState { + #[default] + NoRun, + /// Address entry written, no bitmap entry yet for current window. + /// `bitmap_base` = addr+8 initially, advances by 63*8 per emitted bitmap. + AddressOnly { bitmap_base: u64 }, + /// Bitmap entry written at `bitmap_idx`; `bitmap_base` is current window start. + WithBitmap { bitmap_base: u64, bitmap_idx: usize }, +} + struct TableWriter<'layout, 'out> { output_kind: OutputKind, got: &'out mut [u64], @@ -605,6 +618,10 @@ struct TableWriter<'layout, 'out> { rela_dyn_relative: &'out mut [crate::elf::Rela], rela_dyn_general: &'out mut [crate::elf::Rela], relr_dyn: Option<&'out mut [elf::Relr]>, + /// Index of next free slot in relr_dyn. + relr_dyn_index: usize, + /// Writer-phase RELR run state for bitmap packing. + relr_state: RelrWriterState, dynsym_writer: SymbolTableWriter<'layout, 'out>, debug_symbol_writer: SymbolTableWriter<'layout, 'out>, eh_frame_start_address: u64, @@ -672,6 +689,8 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { relr_dyn: pack_relative_relocs .then(|| slice_from_all_bytes_mut(buffers.take(part_id::RELR_DYN))) .filter(|b| !b.is_empty()), + relr_dyn_index: 0, + relr_state: RelrWriterState::NoRun, dynsym_writer, debug_symbol_writer, eh_frame_start_address, @@ -744,7 +763,7 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { self.write_ifunc_relocation::(res)?; } else { *got_entry = if res.flags.is_address() && self.output_kind.is_relocatable() { - self.write_address_relocation::(got_address, res.raw_value)? + self.write_relr_entry_flat::(got_address, res.raw_value)? } else { res.raw_value }; @@ -762,7 +781,7 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { let got_entry = self.take_next_got_entry()?; let plt_address = res.plt_address()?; *got_entry = if self.output_kind.is_relocatable() { - self.write_address_relocation::(ifunc_got_address, plt_address)? + self.write_relr_entry_flat::(ifunc_got_address, plt_address)? } else { plt_address }; @@ -912,6 +931,12 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { .ok_or_else(|| insufficient_allocation(".got")) } + /// Resets RELR run state between input sections. + /// Layout tracks runs per-section; writer must do the same to stay in sync. + fn reset_relr_run(&mut self) { + self.relr_state = RelrWriterState::NoRun; + } + /// Checks that we used all of the entries that we requested during layout. fn validate_empty(&self, mem_sizes: &OutputSectionPartMap) -> Result { if !self.got.is_empty() { @@ -935,14 +960,15 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { *mem_sizes.get(part_id::RELA_DYN_GENERAL), )); } - if let Some(relr_dyn) = &self.relr_dyn - && !relr_dyn.is_empty() - { - return Err(excessive_allocation( - ".relr.dyn", - relr_dyn.len() as u64 * elf::RELR_ENTRY_SIZE, - *mem_sizes.get(part_id::RELR_DYN), - )); + if let Some(relr_dyn) = &self.relr_dyn { + let remaining = relr_dyn.len() - self.relr_dyn_index; + if remaining != 0 { + return Err(excessive_allocation( + ".relr.dyn", + remaining as u64 * elf::RELR_ENTRY_SIZE, + *mem_sizes.get(part_id::RELR_DYN), + )); + } } self.dynsym_writer.check_exhausted()?; self.debug_symbol_writer.check_exhausted()?; @@ -1044,6 +1070,43 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { ) } + /// Writes a single flat RELR address entry without bitmap packing. + /// Used for GOT-based RELR entries where layout counts flat (one entry per slot). + /// Falls back to rela.dyn.relative when RELR is not enabled. + // TODO: Implement bitmap packing for GOT-based RELR entries. Requires splitting + // the GOT into two parts so relative relocations are contiguous and countable + // during layout. + fn write_relr_entry_flat>( + &mut self, + place: u64, + relative_address: u64, + ) -> Result { + let e = LittleEndian; + if let Some(relr_writer) = &mut self.relr_dyn + && place.is_multiple_of(2) + { + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, place); + self.relr_dyn_index += 1; + Ok(relative_address) + } else { + let rela = self + .rela_dyn_relative + .split_off_first_mut() + .ok_or_else(|| insufficient_allocation(".rela.dyn (relative)"))?; + rela.r_offset.set(e, place); + rela.r_addend.set(e, relative_address as i64); + rela.r_info.set( + e, + A::get_dynamic_relocation_type(DynamicRelocationKind::Relative).into(), + ); + Ok(0) + } + } + #[inline(always)] /// Writes RELA or RELR entry and returns value that should be written at the relocation site. fn write_address_relocation>( @@ -1056,14 +1119,101 @@ impl<'layout, 'out> TableWriter<'layout, 'out> { "write_address_relocation called when output is not relocatable" ); let e = LittleEndian; - // Odd offsets mean bitmaps in RELR, so we need to fall back to RELA for them. + // Odd offsets can't be encoded as RELR address entries (LSB used as bitmap + // marker), so fall back to RELA for them. if let Some(relr_writer) = &mut self.relr_dyn && place.is_multiple_of(2) { - let relr = relr_writer - .split_off_first_mut() - .ok_or_else(|| insufficient_allocation(".relr.dyn"))?; - relr.0.set(LittleEndian, place); + const BITMAP_SLOTS: u64 = 63; + const WORD_SIZE: u64 = 8; + self.relr_state = match self.relr_state { + RelrWriterState::NoRun => { + // No run — write address entry and start run. + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, place); + self.relr_dyn_index += 1; + RelrWriterState::AddressOnly { + bitmap_base: place + WORD_SIZE, + } + } + RelrWriterState::AddressOnly { bitmap_base } + | RelrWriterState::WithBitmap { bitmap_base, .. } => { + let d = place.wrapping_sub(bitmap_base); + if d % WORD_SIZE == 0 && d < BITMAP_SLOTS * WORD_SIZE { + // Fits in current bitmap window. + let bit = d / WORD_SIZE + 1; + if let RelrWriterState::WithBitmap { bitmap_idx, .. } = self.relr_state { + // Bitmap entry exists — OR in the new bit. + let current = relr_writer[bitmap_idx].0.get(LittleEndian); + relr_writer[bitmap_idx] + .0 + .set(LittleEndian, current | (1 << bit)); + RelrWriterState::WithBitmap { + bitmap_base, + bitmap_idx, + } + } else { + // First member in this window — write bitmap entry. + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, 1 | (1 << bit)); + self.relr_dyn_index += 1; + RelrWriterState::WithBitmap { + bitmap_base, + bitmap_idx: idx, + } + } + } else if let RelrWriterState::WithBitmap { .. } = self.relr_state + && d % WORD_SIZE == 0 + { + // Current window has bits — try next window. + let next_base = bitmap_base + BITMAP_SLOTS * WORD_SIZE; + let d2 = place.wrapping_sub(next_base); + if d2.is_multiple_of(WORD_SIZE) && d2 < BITMAP_SLOTS * WORD_SIZE { + // Fits in next window — write new bitmap entry. + let bit = d2 / WORD_SIZE + 1; + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, 1 | (1 << bit)); + self.relr_dyn_index += 1; + RelrWriterState::WithBitmap { + bitmap_base: next_base, + bitmap_idx: idx, + } + } else { + // Gap too large — start new address entry. + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, place); + self.relr_dyn_index += 1; + RelrWriterState::AddressOnly { + bitmap_base: place + WORD_SIZE, + } + } + } else { + // Unaligned or current window empty and out of range. + // Start new address entry. + let idx = self.relr_dyn_index; + if idx >= relr_writer.len() { + return Err(insufficient_allocation(".relr.dyn")); + } + relr_writer[idx].0.set(LittleEndian, place); + self.relr_dyn_index += 1; + RelrWriterState::AddressOnly { + bitmap_base: place + WORD_SIZE, + } + } + } + }; Ok(relative_address) } else { let rela = self @@ -1472,6 +1622,7 @@ fn write_object<'data, A: Arch>( match sec { SectionSlot::Loaded(sec) => { + table_writer.reset_relr_run(); write_object_section::( object, layout, diff --git a/wild/tests/integration_tests.rs b/wild/tests/integration_tests.rs index 9f614722b..1cc08b8fe 100644 --- a/wild/tests/integration_tests.rs +++ b/wild/tests/integration_tests.rs @@ -67,7 +67,13 @@ //! //! Contains:{string} Checks that the output binary does contain the specified string. //! -//! ExpectSection:{section_name} Checks that the specified section exists in the output binary. +//! ExpectSection:{section_name} [properties] Checks that the specified section exists in the +//! output binary. Optional properties: +//! max_entries=N: Asserts the section has at most N entries (uses the section's sh_entsize, +//! or 1 if sh_entsize is 0). +//! +//! RelrCount:N Checks that .relr.dyn encodes at least N total relocations (counting both address +//! entries and bitmap-encoded relocations). //! //! NoSection:{section_name} Checks that the specified section does not exist in the output binary. //! @@ -1446,8 +1452,10 @@ struct Assertions { expected_dynamic_entries: Vec, absent_dynamic_entries: Vec, expected_sections: Vec, + expected_sections_with_assertions: Vec, absent_sections: Vec, expected_section_bytes: Vec, + relr_count: Option, expected_gdb_index_cu_count: Option, expected_gdb_index_symbols: Vec, expected_gdb_index_distinct_addr_cus: Option, @@ -1463,6 +1471,18 @@ struct ExpectedSectionBytes { expected_bytes: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ExpectedSection { + section_name: String, + assertions: SectionAssertions, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct SectionAssertions { + max_entries: Option, +} + #[derive(Debug, Clone)] struct OutputFileMatch { filename: String, @@ -1821,10 +1841,28 @@ fn process_directive( } "DoesNotContain" => config.assertions.does_not_contain.push(arg.to_owned()), "Contains" => config.assertions.contains_strings.push(arg.to_owned()), - "ExpectSection" => config - .assertions - .expected_sections - .push(arg.trim().to_owned()), + "ExpectSection" => { + let arg = arg.trim(); + if let Some((name, props)) = arg.split_once(' ') { + let assertions = serde_keyvalue::from_key_values(props.trim())?; + config + .assertions + .expected_sections_with_assertions + .push(ExpectedSection { + section_name: name.to_owned(), + assertions, + }); + } else { + config.assertions.expected_sections.push(arg.to_owned()); + } + } + "RelrCount" => { + config.assertions.relr_count = Some( + arg.trim() + .parse::() + .with_context(|| format!("Invalid RelrCount: {arg}"))?, + ); + } "NoSection" => config .assertions .absent_sections @@ -4069,6 +4107,7 @@ impl Assertions { self.verify_expected_sections(&obj)?; self.verify_absent_sections(&obj)?; self.verify_section_bytes(&obj)?; + self.verify_relr_count(&obj)?; self.verify_gdb_index_cu_count(&obj)?; self.verify_gdb_index_symbols(&obj)?; self.verify_gdb_index_distinct_addr_cus(&obj)?; @@ -4084,6 +4123,7 @@ impl Assertions { self.verify_symbols_absent(&self.no_sym, elf_obj.dynamic_symbols(), "dynsym")?; self.verify_symbols_absent(&self.no_dynsym, elf_obj.dynamic_symbols(), "dynsym")?; self.verify_program_headers(&elf_obj)?; + self.verify_expected_sections_with_assertions(&elf_obj)?; } object::File::MachO64(_) => { if !self.expected_comments.is_empty() { @@ -4278,6 +4318,62 @@ impl Assertions { Ok(()) } + fn verify_relr_count(&self, obj: &object::File) -> Result { + let Some(expected_count) = self.relr_count else { + return Ok(()); + }; + let Some(section) = obj.section_by_name(".relr.dyn") else { + bail!("RelrCount: section `.relr.dyn` not found"); + }; + let data = section.data()?; + let mut count: u64 = 0; + let mut i = 0; + while i + 8 <= data.len() { + let val = u64::from_le_bytes(data[i..i + 8].try_into().unwrap()); + if val & 1 == 0 { + // Address entry — encodes 1 relocation. + count += 1; + } else { + // Bitmap entry — count set bits excluding LSB marker. + count += (val >> 1).count_ones() as u64; + } + i += 8; + } + ensure!( + count >= expected_count, + "RelrCount: expected at least {} RELR relocations, got {}", + expected_count, + count, + ); + Ok(()) + } + + fn verify_expected_sections_with_assertions( + &self, + obj: &object::read::elf::ElfFile64<'_, object::Endianness>, + ) -> Result { + for expected in &self.expected_sections_with_assertions { + let section = obj + .section_by_name(&expected.section_name) + .with_context(|| format!("Section `{}` not found", expected.section_name))?; + if let Some(max_entries) = expected.assertions.max_entries { + let size = section.size(); + let entry_size = section.elf_section_header().sh_entsize.get(obj.endian()); + let entry_size = if entry_size == 0 { 1 } else { entry_size }; + let actual_entries = size / entry_size; + ensure!( + actual_entries <= max_entries, + "Section `{}` has {} entries, expected at most {}. \ + Bitmap packing may not be working correctly.", + expected.section_name, + actual_entries, + max_entries, + ); + } + } + Ok(()) + } + fn verify_max_thunks(&self, binary_path: &Path) -> Result { let layout_path = linker_layout::layout_path(binary_path); let layout_bytes = std::fs::read(&layout_path); diff --git a/wild/tests/sources/elf/relr-bitmap/relr-bitmap.c b/wild/tests/sources/elf/relr-bitmap/relr-bitmap.c new file mode 100644 index 000000000..a5f62e772 --- /dev/null +++ b/wild/tests/sources/elf/relr-bitmap/relr-bitmap.c @@ -0,0 +1,58 @@ +// Verify that consecutive RELR-eligible relocations are packed into bitmap +// entries rather than emitted as individual address entries. +// +// Three consecutive constructors in .init_array produce function pointers at +// consecutive 8-byte-aligned offsets. Without bitmap packing, each would get +// its own RELR address entry. With packing, they share an address entry and +// a bitmap entry. +//#AbstractConfig:default +//#Object:runtime.c +//#Object:init.c:-fPIC +//#Mode:dynamic +//#LinkArgs:-pie -z now -z pack-relative-relocs --no-gc-sections +//#DiffMatchAny:true +// GNU ld ignores `-z pack-relative-relocs` on RISC-V. +//#ReferenceLinkers:lld +// Verify all 3 constructor pointers are correctly encoded as RELR relocations. +//#RelrCount:3 + +//#Config:x86_64:default +//#Arch:x86_64 +//#ExpectSection:.relr.dyn max_entries=2 + +//#Config:aarch64:default +//#Arch:aarch64 +//#ExpectSection:.relr.dyn max_entries=6 + +//#Config:loongarch64:default +//#Arch:loongarch64 +//#ExpectSection:.relr.dyn max_entries=6 + +//#Config:riscv64:default +//#Arch:riscv64 +//#ExpectSection:.relr.dyn max_entries=7 + +//#Config:ppc64le:default +//#Arch:ppc64le +//#ExpectSection:.relr.dyn max_entries=4 +#include "../common/init.h" +#include "../common/runtime.h" + +static int foo = 0; +static int bar = 0; +static int baz = 0; + +// Three consecutive constructors — their function pointers go into .init_array +// at consecutive 8-byte-aligned offsets, making them RELR bitmap-packable. +__attribute__((constructor)) static void ctor_foo(void) { foo = 1; } +__attribute__((constructor)) static void ctor_bar(void) { bar = 2; } +__attribute__((constructor)) static void ctor_baz(void) { baz = 3; } + +void _start(void) { + runtime_init(); + call_init_functions(); + if (foo != 1) exit_syscall(1); + if (bar != 2) exit_syscall(2); + if (baz != 3) exit_syscall(3); + exit_syscall(42); +}