From e979b80f28b87c62680077515d9fced7992a682c Mon Sep 17 00:00:00 2001 From: Nathan Royer <61582713+NathanRoyer@users.noreply.github.com> Date: Sun, 20 Aug 2023 12:16:48 +0200 Subject: [PATCH 1/4] Improve `pci` abstractions; support MMIO PCI access for aarch64 (#1019) * Support PCI on aarch64 via MMIO access mechanisms. * Redesign the PCI addressing code to be very clear and use more types. Currently some checks are done at runtime with assertions, but these could easily be moved to compile time by introducing traits about register sizes that can be implemented by each `RegisterSpan` within the `PciRegister` type. * Note: PCI currently uses Port I/O on x86 and MMIO on aarch64 to read/write, but x86_64 may also use MMIO-based access in the future, because Port I/O is the legacy method to access the PCI configuration space. * The `device_manager` now initializes and scans the PCI bus on both the aarch64 and x86_64 platforms. * Scanning the PCI bus for the first time also maps the memory behind the configuration space. Co-authored-by: Kevin Boos --- Cargo.lock | 5 +- Makefile | 3 + kernel/arm_boards/src/boards/qemu_virt.rs | 9 +- kernel/arm_boards/src/lib.rs | 8 + kernel/device_manager/Cargo.toml | 2 +- kernel/device_manager/src/lib.rs | 15 +- kernel/pci/Cargo.toml | 9 +- kernel/pci/src/lib.rs | 527 ++++++++++++++++------ 8 files changed, 434 insertions(+), 144 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03d79fc1d3..83c657a18e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2567,9 +2567,9 @@ checksum = "be5e13c266502aadf83426d87d81a0f5d1ef45b8027f5a471c360abfe4bfae92" [[package]] name = "paste" -version = "1.0.5" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "path" @@ -2587,6 +2587,7 @@ dependencies = [ name = "pci" version = "0.1.0" dependencies = [ + "arm_boards", "bit_field 0.7.0", "cpu", "interrupts", diff --git a/Makefile b/Makefile index 2b1d192b03..2ff1f0ce9c 100644 --- a/Makefile +++ b/Makefile @@ -933,6 +933,9 @@ else ifeq ($(ARCH),aarch64) QEMU_FLAGS += -machine virt,gic-version=3 QEMU_FLAGS += -device ramfb QEMU_FLAGS += -cpu cortex-a72 + QEMU_FLAGS += -usb + QEMU_FLAGS += -device usb-ehci,id=ehci + QEMU_FLAGS += -device usb-kbd else QEMU_FLAGS += -cpu Broadwell endif diff --git a/kernel/arm_boards/src/boards/qemu_virt.rs b/kernel/arm_boards/src/boards/qemu_virt.rs index 81de9c88d8..c570c6d8f7 100644 --- a/kernel/arm_boards/src/boards/qemu_virt.rs +++ b/kernel/arm_boards/src/boards/qemu_virt.rs @@ -2,7 +2,7 @@ use super::{ InterruptControllerConfig::GicV3, GicV3InterruptControllerConfig, - BoardConfig, mpidr::DefinedMpidrValue, + BoardConfig, mpidr::DefinedMpidrValue, PciEcamConfig, }; use memory_structs::PhysicalAddress; @@ -38,4 +38,11 @@ pub const BOARD_CONFIG: BoardConfig = BoardConfig { pl011_base_addresses: [ PhysicalAddress::new_canonical(0x09000000) ], pl011_rx_spi: 33, cpu_local_timer_ppi: 30, + + // obtained via internal qemu debugging + // todo: will this always be correct? + pci_ecam: PciEcamConfig { + base_address: PhysicalAddress::new_canonical(0x4010000000), + size_bytes: 0x10000000, + } }; diff --git a/kernel/arm_boards/src/lib.rs b/kernel/arm_boards/src/lib.rs index cd3a1da0d6..584b7f2d43 100644 --- a/kernel/arm_boards/src/lib.rs +++ b/kernel/arm_boards/src/lib.rs @@ -19,6 +19,12 @@ pub struct GicV3InterruptControllerConfig { pub redistributor_base_addresses: [PhysicalAddress; NUM_CPUS], } +#[derive(Debug, Copy, Clone)] +pub struct PciEcamConfig { + pub base_address: PhysicalAddress, + pub size_bytes: usize, +} + #[derive(Debug, Copy, Clone)] pub enum InterruptControllerConfig { GicV3(GicV3InterruptControllerConfig), @@ -46,6 +52,8 @@ pub struct BoardConfig { // // aarch64 manuals define the default timer IRQ number to be 30. pub cpu_local_timer_ppi: u8, + + pub pci_ecam: PciEcamConfig, } // by default & on x86_64, the default.rs file is used diff --git a/kernel/device_manager/Cargo.toml b/kernel/device_manager/Cargo.toml index e0ae226d1e..5f51ab2988 100644 --- a/kernel/device_manager/Cargo.toml +++ b/kernel/device_manager/Cargo.toml @@ -12,6 +12,7 @@ event_types = { path = "../event_types" } serial_port = { path = "../serial_port" } console = { path = "../console" } logger = { path = "../logger" } +pci = { path = "../pci" } derive_more = "0.99.0" mpmc = "0.1.6" log = "0.4.8" @@ -20,7 +21,6 @@ log = "0.4.8" memory = { path = "../memory" } e1000 = { path = "../e1000" } acpi = { path = "../acpi" } -pci = { path = "../pci" } ps2 = { path = "../ps2" } keyboard = { path = "../keyboard" } mouse = { path = "../mouse" } diff --git a/kernel/device_manager/src/lib.rs b/kernel/device_manager/src/lib.rs index 3647c938cf..b0a806a80f 100644 --- a/kernel/device_manager/src/lib.rs +++ b/kernel/device_manager/src/lib.rs @@ -3,11 +3,11 @@ extern crate alloc; -use log::info; +use log::{info, debug}; #[cfg(target_arch = "x86_64")] use { - log::{error, debug, warn}, + log::{error, warn}, mpmc::Queue, event_types::Event, memory::MemoryManagementInfo, @@ -86,19 +86,20 @@ pub fn init( mouse::init(ps2_controller.mouse_ref(), mouse_producer)?; } - // No PCI support on aarch64 at the moment - #[cfg(target_arch = "x86_64")] { // Initialize/scan the PCI bus to discover PCI devices - for dev in pci::pci_device_iter() { - debug!("Found pci device: {:X?}", dev); + for dev in pci::pci_device_iter()? { + debug!("Found PCI device: {:X?}", dev); } + // No NIC support on aarch64 at the moment + #[cfg(target_arch = "x86_64")] { + // store all the initialized ixgbe NICs here to be added to the network interface list let mut ixgbe_devs = Vec::new(); // Iterate over all PCI devices and initialize the drivers for the devices we support. - for dev in pci::pci_device_iter() { + for dev in pci::pci_device_iter()? { // Currently we skip Bridge devices, since we have no use for them yet. if dev.class == 0x06 { continue; diff --git a/kernel/pci/Cargo.toml b/kernel/pci/Cargo.toml index 04a9829788..869d260769 100644 --- a/kernel/pci/Cargo.toml +++ b/kernel/pci/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["Kevin Boos "] name = "pci" -description = "Basic PCI support for Theseus, x86 only." +description = "Basic PCI support for Theseus." version = "0.1.0" edition = "2021" @@ -12,10 +12,15 @@ log = "0.4.8" volatile = "0.2.4" zerocopy = "0.5.0" -port_io = { path = "../../libs/port_io" } memory = { path = "../memory" } cpu = { path = "../cpu" } interrupts = { path = "../interrupts" } +[target.'cfg(target_arch = "x86_64")'.dependencies] +port_io = { path = "../../libs/port_io" } + +[target.'cfg(target_arch = "aarch64")'.dependencies] +arm_boards = { path = "../arm_boards" } + [lib] crate-type = ["rlib"] diff --git a/kernel/pci/src/lib.rs b/kernel/pci/src/lib.rs index 4733b72676..1fd89a0018 100644 --- a/kernel/pci/src/lib.rs +++ b/kernel/pci/src/lib.rs @@ -1,13 +1,17 @@ +//! PCI Configuration Space Access +//! +//! Note: while pci currently uses port-io on x86 and mmio on aarch64, +//! x86 may also support memory-based PCI configuration in the future; +//! port-io is the legacy way to access the config space. + #![no_std] #![allow(dead_code)] extern crate alloc; use log::*; -use core::fmt; -use core::ops::{Deref, DerefMut}; +use core::{fmt, ops::{Deref, DerefMut}, mem::size_of}; use alloc::vec::Vec; -use port_io::Port; use spin::{Once, Mutex}; use memory::{PhysicalAddress, BorrowedSliceMappedPages, Mutable, MappedPages, map_frame_range, MMIO_FLAGS}; use bit_field::BitField; @@ -16,36 +20,106 @@ use zerocopy::FromBytes; use cpu::CpuId; use interrupts::InterruptNumber; +#[cfg(target_arch = "x86_64")] +use port_io::Port; + +#[cfg(target_arch = "aarch64")] +use arm_boards::BOARD_CONFIG; + +#[derive(Debug, Copy, Clone)] +/// The span of bytes within a 4-byte chunk that a PCI register occupies. +/// +/// The PCI configuration space is represented as an array of 4-byte chunks. +/// This struct defines where in a given 4-byte chunk a PCI register exists. +enum RegisterSpan { + /// Bits [0:31] + FullDword, + /// Bits [0:15] + Word0, + /// Bits [16:31] + Word1, + /// Bits [0:7] + Byte0, + /// Bits [8:15] + Byte1, + /// Bits [16:23] + Byte2, + /// Bits [24:31] + Byte3, +} +use RegisterSpan::*; + +/// A definition of a PCI configuration space register. +#[derive(Clone, Copy, Debug)] +struct PciRegister { + /// The location of this register in the PCI configuration space, + /// given as an index into the space as an array of `u32`s (4-byte chunks). + index: u8, + /// The location of this register within the 4-byte chunk. + span: RegisterSpan, +} +impl PciRegister { + const fn from_offset(raw_offset: u8, size_in_bytes: u8) -> Self { + let index = raw_offset >> 2; + match (size_in_bytes, raw_offset & 0b11) { + (1, 0) => PciRegister { index, span: Byte0 }, + (1, 1) => PciRegister { index, span: Byte1 }, + (1, 2) => PciRegister { index, span: Byte2 }, + (1, 3) => PciRegister { index, span: Byte3 }, + (2, 0) => PciRegister { index, span: Word0 }, + (2, 2) => PciRegister { index, span: Word1 }, + (4, 0) => PciRegister { index, span: FullDword }, + // Throw a const panic (compile error) for invalid values. + _ => panic!("Invalid PciRegister specification"), + } + } +} + +/// A macro for easily defining PCI registers using offsets from the PCI spec. +/// +/// The last argument only accepts register sizes of 1, 2, and 4 bytes. +macro_rules! pci_register { + ($name:ident, $offset:expr, 1) => { + const $name: PciRegister = PciRegister::from_offset($offset, 1); + }; + ($name:ident, $offset:expr, 2) => { + const $name: PciRegister = PciRegister::from_offset($offset, 2); + }; + ($name:ident, $offset:expr, 4) => { + const $name: PciRegister = PciRegister::from_offset($offset, 4); + }; +} + // The below constants define the PCI configuration space. // More info here: -const PCI_VENDOR_ID: u8 = 0x0; -const PCI_DEVICE_ID: u8 = 0x2; -const PCI_COMMAND: u8 = 0x4; -const PCI_STATUS: u8 = 0x6; -const PCI_REVISION_ID: u8 = 0x8; -const PCI_PROG_IF: u8 = 0x9; -const PCI_SUBCLASS: u8 = 0xA; -const PCI_CLASS: u8 = 0xB; -const PCI_CACHE_LINE_SIZE: u8 = 0xC; -const PCI_LATENCY_TIMER: u8 = 0xD; -const PCI_HEADER_TYPE: u8 = 0xE; -const PCI_BIST: u8 = 0xF; -const PCI_BAR0: u8 = 0x10; -const PCI_BAR1: u8 = 0x14; -const PCI_BAR2: u8 = 0x18; -const PCI_BAR3: u8 = 0x1C; -const PCI_BAR4: u8 = 0x20; -const PCI_BAR5: u8 = 0x24; -const PCI_CARDBUS_CIS: u8 = 0x28; -const PCI_SUBSYSTEM_VENDOR_ID: u8 = 0x2C; -const PCI_SUBSYSTEM_ID: u8 = 0x2E; -const PCI_EXPANSION_ROM_BASE: u8 = 0x30; -const PCI_CAPABILITIES: u8 = 0x34; +pci_register!(PCI_VENDOR_ID, 0x00, 2); +pci_register!(PCI_DEVICE_ID, 0x02, 2); +pci_register!(PCI_COMMAND, 0x04, 2); +pci_register!(PCI_STATUS, 0x06, 2); +pci_register!(PCI_REVISION_ID, 0x08, 1); +pci_register!(PCI_PROG_IF, 0x09, 1); +pci_register!(PCI_SUBCLASS, 0x0A, 1); +pci_register!(PCI_CLASS, 0x0B, 1); +pci_register!(PCI_CACHE_LINE_SIZE, 0x0C, 1); +pci_register!(PCI_LATENCY_TIMER, 0x0D, 1); +pci_register!(PCI_HEADER_TYPE, 0x0E, 1); +pci_register!(PCI_BIST, 0x0F, 1); +pci_register!(PCI_BAR0, 0x10, 4); +pci_register!(PCI_BAR1, 0x14, 4); +pci_register!(PCI_BAR2, 0x18, 4); +pci_register!(PCI_BAR3, 0x1C, 4); +pci_register!(PCI_BAR4, 0x20, 4); +pci_register!(PCI_BAR5, 0x24, 4); +pci_register!(PCI_CARDBUS_CIS, 0x28, 4); +pci_register!(PCI_SUBSYSTEM_VENDOR_ID, 0x2C, 2); +pci_register!(PCI_SUBSYSTEM_ID, 0x2E, 2); +pci_register!(PCI_EXPANSION_ROM_BASE, 0x30, 4); +pci_register!(PCI_CAPABILITIES, 0x34, 1); // 0x35 through 0x3B are reserved -const PCI_INTERRUPT_LINE: u8 = 0x3C; -const PCI_INTERRUPT_PIN: u8 = 0x3D; -const PCI_MIN_GRANT: u8 = 0x3E; -const PCI_MAX_LATENCY: u8 = 0x3F; +pci_register!(PCI_INTERRUPT_LINE, 0x3C, 1); +pci_register!(PCI_INTERRUPT_PIN, 0x3D, 1); +pci_register!(PCI_MIN_GRANT, 0x3E, 1); +pci_register!(PCI_MAX_LATENCY, 0x3F, 1); #[repr(u8)] pub enum PciCapability { @@ -64,19 +138,35 @@ const MAX_SLOTS_PER_BUS: u8 = 32; /// There is a maximum of 32 functions (devices) on one PCI slot. const MAX_FUNCTIONS_PER_SLOT: u8 = 8; -/// Addresses/offsets into the PCI configuration space should clear the least-significant 2 bits. -const PCI_CONFIG_ADDRESS_OFFSET_MASK: u8 = 0xFC; +/// Addresses/offsets into the PCI configuration space should clear the +/// least-significant 2 bits in order to be 4-byte aligned. +const PCI_CONFIG_ADDRESS_OFFSET_MASK: u8 = 0b11111100; + const CONFIG_ADDRESS: u16 = 0xCF8; const CONFIG_DATA: u16 = 0xCFC; /// This port is used to specify the address in the PCI configuration space /// for the next read/write of the `PCI_CONFIG_DATA_PORT`. +#[cfg(target_arch = "x86_64")] static PCI_CONFIG_ADDRESS_PORT: Mutex> = Mutex::new(Port::new(CONFIG_ADDRESS)); /// This port is used to transfer data to or from the PCI configuration space /// specified by a previous write to the `PCI_CONFIG_ADDRESS_PORT`. +#[cfg(target_arch = "x86_64")] static PCI_CONFIG_DATA_PORT: Mutex> = Mutex::new(Port::new(CONFIG_DATA)); +#[cfg(target_arch = "x86_64")] +const BASE_OFFSET: u32 = 0x8000_0000; + +#[cfg(target_arch = "aarch64")] +type PciConfigSpace = BorrowedSliceMappedPages, Mutable>; + +#[cfg(target_arch = "aarch64")] +static PCI_CONFIG_SPACE: Mutex> = Mutex::new(Once::new()); + +#[cfg(target_arch = "aarch64")] +const BASE_OFFSET: u32 = 0; + pub enum InterruptPin { A, B, @@ -88,32 +178,33 @@ pub enum InterruptPin { /// Returns a list of all PCI buses in this system. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn get_pci_buses() -> &'static Vec { +pub fn get_pci_buses() -> Result<&'static Vec, &'static str> { static PCI_BUSES: Once> = Once::new(); - PCI_BUSES.call_once(scan_pci) + PCI_BUSES.try_call_once(scan_pci) } /// Returns a reference to the `PciDevice` with the given bus, slot, func identifier. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn get_pci_device_bsf(bus: u8, slot: u8, func: u8) -> Option<&'static PciDevice> { - for b in get_pci_buses() { +pub fn get_pci_device_bsf(bus: u8, slot: u8, func: u8) -> Result, &'static str> { + for b in get_pci_buses()? { if b.bus_number == bus { for d in &b.devices { if d.slot == slot && d.func == func { - return Some(d); + return Ok(Some(d)); } } } } - None + + Ok(None) } /// Returns an iterator that iterates over all `PciDevice`s, in no particular guaranteed order. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn pci_device_iter() -> impl Iterator { - get_pci_buses().iter().flat_map(|b| b.devices.iter()) +pub fn pci_device_iter() -> Result, &'static str> { + Ok(get_pci_buses()?.iter().flat_map(|b| b.devices.iter())) } @@ -129,7 +220,18 @@ pub struct PciBus { /// Scans all PCI Buses (brute force iteration) to enumerate PCI Devices on each bus. /// Initializes structures containing this information. -fn scan_pci() -> Vec { +fn scan_pci() -> Result, &'static str> { + #[cfg(target_arch = "aarch64")] + PCI_CONFIG_SPACE.lock().try_call_once(|| { + let config = BOARD_CONFIG.pci_ecam; + let mapped = memory::map_frame_range(config.base_address, config.size_bytes, MMIO_FLAGS)?; + let config_space_u32_len = BOARD_CONFIG.pci_ecam.size_bytes / size_of::(); + match mapped.into_borrowed_slice_mut(0, config_space_u32_len) { + Ok(bsm) => Ok(bsm), + Err((_, msg)) => Err(msg), + } + })?; + let mut buses: Vec = Vec::new(); for bus in 0..MAX_PCI_BUSES { @@ -198,9 +300,34 @@ fn scan_pci() -> Vec { } } - buses + Ok(buses) } +impl RegisterSpan { + const fn get_mask_and_bitshift(self) -> (u32, u8) { + match self { + FullDword => (0xffff_ffff, 0), + Word0 => (0x0000_ffff, 0), + Word1 => (0xffff_0000, 16), + Byte0 => (0x0000_00ff, 0), + Byte1 => (0x0000_ff00, 8), + Byte2 => (0x00ff_0000, 16), + Byte3 => (0xff00_0000, 24), + } + } + + const fn width_in_bytes(self) -> usize { + match self { + FullDword => size_of::(), + Word0 => size_of::(), + Word1 => size_of::(), + Byte0 => size_of::(), + Byte1 => size_of::(), + Byte2 => size_of::(), + Byte3 => size_of::(), + } + } +} /// The bus, slot, and function number of a given PCI device. /// This offers methods for reading and writing the PCI config space. @@ -216,93 +343,194 @@ impl PciLocation { pub fn slot(&self) -> u8 { self.slot } pub fn function(&self) -> u8 { self.func } + /// Read the value of the given `register` in the PCI Configuration Space. + fn pci_read_raw(&self, register: PciRegister) -> u32 { + let PciRegister { index, span } = register; + let (mask, shift) = span.get_mask_and_bitshift(); + const U32_BYTES: u32 = size_of::() as u32; - /// Computes a PCI address from bus, slot, func, and offset. - /// The least two significant bits of offset are masked, so it's 4-byte aligned addressing, - /// which makes sense since we read PCI data (from the configuration space) in 32-bit chunks. - fn pci_address(self, offset: u8) -> u32 { - ((self.bus as u32) << 16) | - ((self.slot as u32) << 11) | - ((self.func as u32) << 8) | - ((offset as u32) & (PCI_CONFIG_ADDRESS_OFFSET_MASK as u32)) | - 0x8000_0000 - } + let dword_address = BASE_OFFSET + | ((self.bus as u32) << 16) + | ((self.slot as u32) << 11) + | ((self.func as u32) << 8) + | ((index as u32) * U32_BYTES); + + let dword_value; - /// read 32-bit data at the specified `offset` from the PCI device specified by the given `bus`, `slot`, `func` set. - fn pci_read_32(&self, offset: u8) -> u32 { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(offset)); + #[cfg(target_arch = "x86_64")] { + unsafe { + PCI_CONFIG_ADDRESS_PORT.lock().write(dword_address); + } + dword_value = PCI_CONFIG_DATA_PORT.lock().read(); } - Self::read_data_port() >> ((offset & (!PCI_CONFIG_ADDRESS_OFFSET_MASK)) * 8) + + #[cfg(target_arch = "aarch64")] { + let config_space = PCI_CONFIG_SPACE.lock(); + let config_space = config_space.get() + .expect("PCI Config Space wasn't mapped yet"); + let dword_index = (dword_address as usize) / size_of::(); + dword_value = config_space[dword_index].read(); + } + + (dword_value & mask) >> shift } - /// Read 16-bit data at the specified `offset` from this PCI device. - fn pci_read_16(&self, offset: u8) -> u16 { - self.pci_read_32(offset) as u16 - } + /// Read a 4-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`FullDword`] + fn pci_read_32(&self, register: PciRegister) -> u32 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_32: register isn't 32-bit wide"); - /// Read 8-bit data at the specified `offset` from the PCI device. - fn pci_read_8(&self, offset: u8) -> u8 { - self.pci_read_32(offset) as u8 + self.pci_read_raw(register) } - /// Write 32-bit data to the specified `offset` for the PCI device. - fn pci_write(&self, offset: u8, value: u32) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(offset)); - Self::write_data_port((value) << ((offset & 2) * 8)); - } + /// Read a 2-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Word0`] / [`Word1`] + fn pci_read_16(&self, register: PciRegister) -> u16 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_16: register isn't 16-bit wide"); + + self.pci_read_raw(register) as _ + } + + /// Read a one-byte register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Byte0`] / [`Byte1`] / [`Byte2`] / [`Byte3`] + fn pci_read_8(&self, register: PciRegister) -> u8 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_16: register isn't 8-bit wide"); + + self.pci_read_raw(register) as _ } - fn write_data_port(value: u32) { - unsafe { - PCI_CONFIG_DATA_PORT.lock().write(value); + /// Writes (part of) the given `value` to the given `register` in the PCI Configuration Space. + /// + /// If the width of the given `register` is less than 4 bytes, this function will first + /// read the initial value from the `register` to ensure we don't ovewrite other + /// unrelated parts of the `u32` value. + fn pci_write_raw(&self, register: PciRegister, value: u32) { + let PciRegister { index, span } = register; + const U32_BYTES: u32 = size_of::() as u32; + let dword_address = BASE_OFFSET + | ((self.bus as u32) << 16) + | ((self.slot as u32) << 11) + | ((self.func as u32) << 8) + | ((index as u32) * U32_BYTES); + + /// A macro that handles the required bitmasking/shifting to calculate the + /// final value that should actually be written to this `register`. + macro_rules! calc_value { + ($read_initial_value:expr) => { + if matches!(span, FullDword) { + value + } else { + let mut dword = $read_initial_value; + let (mask, shift) = span.get_mask_and_bitshift(); + dword &= !mask; + dword |= value << shift; + dword + } + } + } + + #[cfg(target_arch = "x86_64")] { + unsafe { + PCI_CONFIG_ADDRESS_PORT.lock().write(dword_address); + } + let dword = calc_value!(PCI_CONFIG_DATA_PORT.lock().read()); + unsafe { + PCI_CONFIG_DATA_PORT.lock().write(dword); + } + } + + #[cfg(target_arch = "aarch64")] { + let mut config_space = PCI_CONFIG_SPACE.lock(); + let config_space = config_space.get_mut() + .expect("PCI Config Space wasn't mapped yet"); + let dword_index = (dword_address as usize) / size_of::(); + + let dword = calc_value!(config_space[dword_index].read()); + config_space[dword_index].write(dword); } } - fn read_data_port() -> u32 { - PCI_CONFIG_DATA_PORT.lock().read() + /// Write a 4-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`FullDword`] + fn pci_write_32(&self, register: PciRegister, value: u32) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_32: register isn't 32-bit wide"); + + self.pci_write_raw(register, value) + } + + /// Write a 2-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Word0`] / [`Word1`] + fn pci_write_16(&self, register: PciRegister, value: u16) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_16: register isn't 16-bit wide"); + + self.pci_write_raw(register, value as _) + } + + /// Write a one-byte register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Byte0`] / [`Byte1`] / [`Byte2`] / [`Byte3`] + fn pci_write_8(&self, register: PciRegister, value: u8) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_16: register isn't 8-bit wide"); + + self.pci_write_raw(register, value as _) } /// Sets the PCI device's bit 3 in the command portion, which is apparently needed to activate DMA (??) pub fn pci_set_command_bus_master_bit(&self) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(PCI_COMMAND)); - } - let inval = Self::read_data_port(); - trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value: {:#x}", self, inval); - Self::write_data_port(inval | (1 << 2)); + let value = self.pci_read_16(PCI_COMMAND); + trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value: {:#x}", self, value); + + self.pci_write_16(PCI_COMMAND, value | (1 << 2)); + trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value AFTER WRITE CMD: {:#x}", self, - Self::read_data_port() + self.pci_read_16(PCI_COMMAND), ); } /// Sets the PCI device's command bit 10 to disable legacy interrupts pub fn pci_set_interrupt_disable_bit(&self) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(PCI_COMMAND)); - } - let command = Self::read_data_port(); + let command = self.pci_read_16(PCI_COMMAND); trace!("pci_set_interrupt_disable_bit: PciDevice: {}, read value: {:#x}", self, command); - const INTERRUPT_DISABLE: u32 = 1 << 10; - Self::write_data_port(command | INTERRUPT_DISABLE); + + const INTERRUPT_DISABLE: u16 = 1 << 10; + self.pci_write_16(PCI_COMMAND, command | INTERRUPT_DISABLE); + trace!("pci_set_interrupt_disable_bit: PciDevice: {} read value AFTER WRITE CMD: {:#x}", - self, Self::read_data_port()); + self, + self.pci_read_16(PCI_COMMAND), + ); } - /// Explores the PCI config space and returns address of requested capability, if present. - /// PCI capabilities are stored as a linked list in the PCI config space, + /// Explores the PCI config space and returns address of requested capability, if present. + /// PCI capabilities are stored as a linked list in the PCI config space, /// with each capability storing the pointer to the next capability right after its ID. - /// The function returns a None value if capabilities are not valid for this device - /// or if the requested capability is not present. + /// The function returns a None value if capabilities are not valid for this device + /// or if the requested capability is not present. fn find_pci_capability(&self, pci_capability: PciCapability) -> Option { let pci_capability = pci_capability as u8; let status = self.pci_read_16(PCI_STATUS); // capabilities are only valid if bit 4 of status register is set const CAPABILITIES_VALID: u16 = 1 << 4; - if status & CAPABILITIES_VALID != 0 { + if status & CAPABILITIES_VALID != 0 { // retrieve the capabilities pointer from the pci config space let capabilities = self.pci_read_8(PCI_CAPABILITIES); @@ -314,21 +542,24 @@ impl PciLocation { // the last capability will have its next pointer equal to zero let final_capability = 0; - // iterate through the linked list of capabilities until the requested capability is found or the list reaches its end + // iterate through the linked list of capabilities until the requested + // capability is found or the list reaches its end while cap_addr != final_capability { - // the capability header is a 16 bit value which contains the current capability ID and the pointer to the next capability - let cap_header = self.pci_read_16(cap_addr); + // the capability header is a 16 bit value which contains + // the current capability ID and the pointer to the next capability // the id is the lower byte of the header - let cap_id = (cap_header & 0xFF) as u8; - + let cap_id_reg = PciRegister::from_offset(cap_addr, 1); + let cap_id = self.pci_read_8(cap_id_reg); + if cap_id == pci_capability { debug!("Found capability: {:#X} at {:#X}", pci_capability, cap_addr); return Some(cap_addr); } // find address of next capability which is the higher byte of the header - cap_addr = ((cap_header >> 8) & 0xFF) as u8; + let next_cap_ptr_reg = PciRegister::from_offset(cap_addr + 1, 1); + cap_addr = self.pci_read_8(next_cap_ptr_reg); } } None @@ -435,15 +666,18 @@ impl PciDevice { // (4) Bitwise "not" (negate) that value, then add 1. // The resulting value is the size of that BAR's memory region. // (5) Restore the original value to that BAR - let bar_offset = PCI_BAR0 + (bar_index as u8 * 4); + let bar_reg_def = PciRegister { + index: PCI_BAR0.index + (bar_index as u8), + span: FullDword, + }; let original_value = self.bars[bar_index]; - self.pci_write(bar_offset, 0xFFFF_FFFF); // Step 1 - let mut mem_size = self.pci_read_32(bar_offset); // Step 2 - mem_size.set_bits(0..4, 0); // Step 3 - mem_size = !(mem_size); // Step 4 - mem_size += 1; // Step 4 - self.pci_write(bar_offset, original_value); // Step 5 + self.pci_write_32(bar_reg_def, 0xFFFF_FFFF); // Step 1 + let mut mem_size = self.pci_read_32(bar_reg_def); // Step 2 + mem_size.set_bits(0..4, 0); // Step 3 + mem_size = !(mem_size); // Step 4 + mem_size += 1; // Step 4 + self.pci_write_32(bar_reg_def, original_value); // Step 5 mem_size } @@ -455,34 +689,54 @@ impl PciDevice { /// # Arguments /// * `core_id`: core that interrupt will be routed to /// * `int_num`: interrupt number to assign to the MSI vector + /// + /// # Panics + /// + /// This function panics if the MSI capability isn't aligned to 4 bytes pub fn pci_enable_msi(&self, core_id: u8, int_num: u8) -> Result<(), &'static str> { // find out if the device is msi capable let cap_addr = self.find_pci_capability(PciCapability::Msi).ok_or("Device not MSI capable")?; + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msi: Invalid MSI capability address alignment"); + let msi_reg_index = cap_addr >> 2; // offset in the capability space where the message address register is located - const MESSAGE_ADDRESS_REGISTER_OFFSET: u8 = 4; + const MESSAGE_ADDRESS_REGISTER_OFFSET: u8 = 1 /* one dword */; + // the memory region is a constant defined for Intel cpus where MSI messages are written // it should be written to bit 20 of the message address register const MEMORY_REGION: u32 = 0x0FEE << 20; + // the core id tells which cpu the interrupt will be routed to // it should be written to bit 12 of the message address register let core = (core_id as u32) << 12; + // set the core the MSI will be sent to in the Message Address Register (Intel Arch SDM, vol3, 10.11) - self.pci_write(cap_addr + MESSAGE_ADDRESS_REGISTER_OFFSET, MEMORY_REGION| core); + let msg_addr_reg = PciRegister { + index: msi_reg_index + MESSAGE_ADDRESS_REGISTER_OFFSET, + span: FullDword, + }; + self.pci_write_32(msg_addr_reg, MEMORY_REGION | core); // offset in the capability space where the message data register is located - const MESSAGE_DATA_REGISTER_OFFSET: u8 = 12; + const MESSAGE_DATA_REGISTER_OFFSET: u8 = 3 /* dwords */; + // Set the interrupt number for the MSI in the Message Data Register - self.pci_write(cap_addr + MESSAGE_DATA_REGISTER_OFFSET, int_num as u32); + let msg_data_reg = PciRegister { + index: msi_reg_index + MESSAGE_DATA_REGISTER_OFFSET, + span: FullDword, + }; + self.pci_write_32(msg_data_reg, int_num as u32); + + // to enable the MSI capability, we need to set bit 0 of the message control register + const MSI_ENABLE: u16 = 1; - // offset in the capability space where the message control register is located - const MESSAGE_CONTROL_REGISTER_OFFSET: u8 = 2; - // to enable the MSI capability, we need to set it bit 0 of the message control register - const MSI_ENABLE: u32 = 1; - let ctrl = self.pci_read_16(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET) as u32; // enable MSI in the Message Control Register - self.pci_write(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET, ctrl | MSI_ENABLE); + // the message control register corresponds to bits [16:31] of the first dword + let msg_ctrl_reg = PciRegister { index: msi_reg_index, span: Word1 }; + let mut ctrl = self.pci_read_16(msg_ctrl_reg); + ctrl |= MSI_ENABLE; + self.pci_write_16(msg_ctrl_reg, ctrl); Ok(()) } @@ -491,19 +745,21 @@ impl PciDevice { /// Only the enable bit is set and the remaining initialization steps of /// setting the interrupt number and core id should be completed in the device driver. pub fn pci_enable_msix(&self) -> Result<(), &'static str> { - // find out if the device is msi-x capable let cap_addr = self.find_pci_capability(PciCapability::Msix).ok_or("Device not MSI-X capable")?; + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msix: Invalid MSI-X capability address alignment"); + let msix_reg_index = cap_addr >> 2; - // offset in the capability space where the message control register is located - const MESSAGE_CONTROL_REGISTER_OFFSET: u8 = 2; - let ctrl = self.pci_read_16(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET) as u32; + // write to bit 15 of Message Control Register to enable MSI-X + const MSIX_ENABLE: u16 = 1 << 15; - // write to bit 15 of Message Control Register to enable MSI-X - const MSIX_ENABLE: u32 = 1 << 15; - self.pci_write(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET, ctrl | MSIX_ENABLE); + // the message control register corresponds to bits [16:31] of the first dword + let msg_ctrl_reg = PciRegister { index: msix_reg_index, span: Word1 }; + let mut ctrl = self.pci_read_16(msg_ctrl_reg); + ctrl |= MSIX_ENABLE; + self.pci_write_16(msg_ctrl_reg, ctrl); - // let ctrl = pci_read_32(dev.bus, dev.slot, dev.func, cap_addr); + // let ctrl = pci_read_16(msg_ctrl_reg, msix_reg_index); // debug!("MSIX HEADER AFTER ENABLE: {:#X}", ctrl); Ok(()) @@ -516,15 +772,23 @@ impl PciDevice { pub fn pci_mem_map_msix(&self, max_vectors: usize) -> Result { // retreive the address in the pci config space for the msi-x capability let cap_addr = self.find_pci_capability(PciCapability::Msix).ok_or("Device not MSI-X capable")?; - // find the BAR used for msi-x - let vector_table_offset = 4; - let table_offset = self.pci_read_32(cap_addr + vector_table_offset); + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msix: Invalid MSI-X capability address alignment"); + let msix_reg_index = cap_addr >> 2; + + // get physical address of vector table + const VECTOR_TABLE_OFFSET: u8 = 1; + let vector_table_reg = PciRegister { + index: msix_reg_index + VECTOR_TABLE_OFFSET, + span: FullDword, + }; + let table_offset = self.pci_read_32(vector_table_reg); let bar = table_offset & 0x7; let offset = table_offset >> 3; + let addr = self.bars[bar as usize] + offset; + // find the memory base address and size of the area for the vector table - let mem_base = PhysicalAddress::new((self.bars[bar as usize] + offset) as usize) - .ok_or("Invalid BAR content")?; - let mem_size_in_bytes = core::mem::size_of::() * max_vectors; + let mem_base = PhysicalAddress::new(addr as usize).ok_or("Invalid BAR content")?; + let mem_size_in_bytes = size_of::() * max_vectors; // debug!("msi-x vector table bar: {}, base_address: {:#X} and size: {} bytes", bar, mem_base, mem_size_in_bytes); @@ -642,6 +906,7 @@ impl MsixVectorEntry { self.msg_lower_addr.write(address | MSIX_INTERRUPT_REGION | dest_id); // write interrupt number + #[allow(clippy::unnecessary_cast)] self.msg_data.write(int_num as u32); if false { From 4e544231e7ed6c19d562a6c809a8bf54cae4d3f5 Mon Sep 17 00:00:00 2001 From: Kevin Boos <1139460+kevinaboos@users.noreply.github.com> Date: Tue, 22 Aug 2023 19:07:02 -0700 Subject: [PATCH 2/4] Allow page allocation requests to specify a desired alignment (#1029) * The new `AllocationRequest::AlignedAt` variant can accept an alignment value, specified in number of 4K pages (not bytes). * This is needed to support easier allocation of huge pages, which have a large alignment requirement, e.g., a 2MiB huge page requires a contiguous 512-page allocation aligned to that same boundary of 512 normal 4K pages (512 * 4KiB = 2MiB). --- Cargo.lock | 10 +++ Cargo.toml | 1 + .../test_aligned_page_allocation/Cargo.toml | 15 ++++ .../test_aligned_page_allocation/src/lib.rs | 45 ++++++++++++ kernel/memory/src/lib.rs | 4 ++ kernel/memory_structs/src/lib.rs | 10 +++ kernel/page_allocator/src/lib.rs | 68 ++++++++++++++----- theseus_features/Cargo.toml | 2 + 8 files changed, 139 insertions(+), 16 deletions(-) create mode 100644 applications/test_aligned_page_allocation/Cargo.toml create mode 100644 applications/test_aligned_page_allocation/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 83c657a18e..99b617840d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3758,6 +3758,15 @@ dependencies = [ "sync_irq", ] +[[package]] +name = "test_aligned_page_allocation" +version = "0.1.0" +dependencies = [ + "app_io", + "log", + "memory", +] + [[package]] name = "test_async" version = "0.1.0" @@ -4048,6 +4057,7 @@ dependencies = [ "seconds_counter", "shell", "swap", + "test_aligned_page_allocation", "test_async", "test_backtrace", "test_block_io", diff --git a/Cargo.toml b/Cargo.toml index 721c060cb5..56bdae404f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ exclude = [ ## Exclude application crates used for testing specific Theseus functionality. ## TODO: move these to a specific "tests" folder so we can exclude that entire folder. + "applications/test_aligned_page_allocation", "applications/test_backtrace", "applications/test_block_io", "applications/test_channel", diff --git a/applications/test_aligned_page_allocation/Cargo.toml b/applications/test_aligned_page_allocation/Cargo.toml new file mode 100644 index 0000000000..a81b8f9f54 --- /dev/null +++ b/applications/test_aligned_page_allocation/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "test_aligned_page_allocation" +version = "0.1.0" +description = "Tests the `AllocationRequest::AlignedTo` variant, which is needed for huge pages" +authors = ["Kevin Boos "] +edition = "2021" + +[dependencies] +log = "0.4.8" + +[dependencies.memory] +path = "../../kernel/memory" + +[dependencies.app_io] +path = "../../kernel/app_io" diff --git a/applications/test_aligned_page_allocation/src/lib.rs b/applications/test_aligned_page_allocation/src/lib.rs new file mode 100644 index 0000000000..ebe99e7c5f --- /dev/null +++ b/applications/test_aligned_page_allocation/src/lib.rs @@ -0,0 +1,45 @@ +//! A set of basic tests for the [`AllocationRequest::AlignedTo`] variant. + +#![no_std] + +extern crate alloc; + +use alloc::{ + vec::Vec, + string::String, +}; +use app_io::println; +use memory::AllocationRequest; + +static TEST_SET: [usize; 9] = [1, 2, 4, 8, 27, 48, 256, 512, 1024]; + +pub fn main(_args: Vec) -> isize { + match rmain() { + Ok(_) => 0, + Err(e) => { + println!("Error: {}", e); + -1 + } + } +} + +fn rmain() -> Result<(), &'static str> { + for num_pages in TEST_SET.into_iter() { + for alignment in TEST_SET.into_iter() { + println!("Attempting to allocate {num_pages} pages with alignment of {alignment} 4K pages..."); + match memory::allocate_pages_deferred( + AllocationRequest::AlignedTo { alignment_4k_pages: alignment }, + num_pages, + ) { + Ok((ap, _action)) => { + assert_eq!(ap.start().number() % alignment, 0); + assert_eq!(ap.size_in_pages(), num_pages); + println!(" Success: {ap:?}"); + } + Err(e) => println!(" !! FAILURE: {e:?}"), + } + } + } + + Ok(()) +} diff --git a/kernel/memory/src/lib.rs b/kernel/memory/src/lib.rs index 600e3063da..a3161bd58e 100644 --- a/kernel/memory/src/lib.rs +++ b/kernel/memory/src/lib.rs @@ -26,6 +26,8 @@ pub use memory_structs::*; pub use page_allocator::{ AllocatedPages, AllocationRequest, + allocate_pages_deferred, + allocate_pages_by_bytes_deferred, allocate_pages, allocate_pages_at, allocate_pages_by_bytes, @@ -37,6 +39,8 @@ pub use page_allocator::{ pub use frame_allocator::{ AllocatedFrames, UnmappedFrames, + allocate_frames_deferred, + allocate_frames_by_bytes_deferred, allocate_frames, allocate_frames_at, allocate_frames_by_bytes, diff --git a/kernel/memory_structs/src/lib.rs b/kernel/memory_structs/src/lib.rs index d7511fcd80..3563eae66a 100644 --- a/kernel/memory_structs/src/lib.rs +++ b/kernel/memory_structs/src/lib.rs @@ -7,6 +7,7 @@ #![no_std] #![feature(step_trait)] +#![feature(int_roundings)] #![allow(incomplete_features)] #![feature(adt_const_params)] @@ -287,6 +288,15 @@ macro_rules! implement_page_frame { number: addr.value() / PAGE_SIZE, } } + + #[doc = "Returns a new `" $TypeName "` that is aligned up from this \ + `" $TypeName "` to the nearest multiple of `alignment_4k_pages`."] + #[doc(alias = "next_multiple_of")] + pub const fn align_up(&self, alignment_4k_pages: usize) -> $TypeName { + $TypeName { + number: self.number.next_multiple_of(alignment_4k_pages) + } + } } impl fmt::Debug for $TypeName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { diff --git a/kernel/page_allocator/src/lib.rs b/kernel/page_allocator/src/lib.rs index fb632a56a5..cdb476c768 100644 --- a/kernel/page_allocator/src/lib.rs +++ b/kernel/page_allocator/src/lib.rs @@ -38,7 +38,7 @@ use static_array_rb_tree::*; /// Certain regions are pre-designated for special usage, specifically the kernel's initial identity mapping. -/// They will be allocated from if an address within them is specifically requested; +/// They will be allocated from if an address within them is specifically /// otherwise, they will only be allocated from as a "last resort" if all other non-designated address ranges are exhausted. /// /// Any virtual addresses **less than or equal** to this address are considered "designated". @@ -536,10 +536,15 @@ fn find_specific_chunk( /// If no range is specified, this function first attempts to find a suitable chunk /// that is **not** within the designated regions, /// and only allocates from the designated regions as a backup option. +/// +/// If an alignment is specified (in terms of number of 4KiB pages), then the starting page +/// in the allocated range must be aligned to that number of pages. +/// If no specific alignment is needed, the default aligment of 1 page should be used. fn find_any_chunk( list: &mut StaticArrayRBTree, num_pages: usize, within_range: Option<&PageRange>, + alignment_4k_pages: usize, ) -> Result<(AllocatedPages, DeferredAllocAction<'static>), AllocationError> { let designated_low_end = DESIGNATED_PAGES_LOW_END.get() .ok_or(AllocationError::NotInitialized)?; @@ -555,7 +560,8 @@ fn find_any_chunk( if let Some(chunk) = elem { // Use max and min below to ensure that the range of pages we allocate from // is within *both* the current chunk's bounds and the range's bounds. - let lowest_possible_start_page = *max(chunk.start(), range.start()); + let lowest_possible_start_page = max(chunk.start(), range.start()) + .align_up(alignment_4k_pages); let highest_possible_end_page = *min(chunk.end(), range.end()); if lowest_possible_start_page + num_pages <= highest_possible_end_page { return adjust_chosen_chunk( @@ -589,7 +595,8 @@ fn find_any_chunk( while let Some(chunk) = cursor.get().map(|w| w.deref()) { // Use max and min below to ensure that the range of pages we allocate from // is within *both* the current chunk's bounds and the range's bounds. - let lowest_possible_start_page = *max(chunk.start(), range.start()); + let lowest_possible_start_page = max(chunk.start(), range.start()) + .align_up(alignment_4k_pages); let highest_possible_end_page = *min(chunk.end(), range.end()); if lowest_possible_start_page + num_pages <= highest_possible_end_page { return adjust_chosen_chunk( @@ -621,8 +628,14 @@ fn find_any_chunk( Inner::Array(ref mut arr) => { for elem in arr.iter_mut() { if let Some(chunk) = elem { - if num_pages <= chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::Array(elem)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::Array(elem), + ); } } } @@ -644,8 +657,14 @@ fn find_any_chunk( // The first iterates over the lower designated region, from higher addresses to lower, down to zero. let mut cursor = tree.upper_bound_mut(Bound::Included(designated_low_end)); while let Some(chunk) = cursor.get().map(|w| w.deref()) { - if num_pages < chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::RBTree(cursor)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::RBTree(cursor), + ); } cursor.move_prev(); } @@ -657,8 +676,14 @@ fn find_any_chunk( // we already iterated over non-designated pages in the first match statement above, so we're out of memory. break; } - if num_pages < chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::RBTree(cursor)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::RBTree(cursor), + ); } cursor.move_prev(); } @@ -729,23 +754,31 @@ fn adjust_chosen_chunk( } -/// Possible options when requested pages from the page allocator. +/// Possible options when requesting pages from the page allocator. pub enum AllocationRequest<'r> { - /// The allocated pages can be located at any virtual address. - Any, /// The allocated pages must start exactly at the given `VirtualAddress`. AtVirtualAddress(VirtualAddress), + /// The allocated pages may be located at any virtual address, + /// but the starting page must be aligned to a multiple of `alignment_4k_pages`. + /// An alignment of `1` page is equivalent to specifying no alignment requirement. + /// + /// Note: alignment is specified in number of 4KiB pages, not number of bytes. + AlignedTo { alignment_4k_pages: usize }, /// The allocated pages can be located anywhere within the given range. WithinRange(&'r PageRange), + /// The allocated pages can be located at any virtual address + /// and have no special alignment requirements beyond a single page. + Any, } + /// The core page allocation routine that allocates the given number of virtual pages, /// optionally at the requested starting `VirtualAddress`. /// /// This simply reserves a range of virtual addresses, it does not allocate /// actual physical memory frames nor do any memory mapping. /// Thus, the returned `AllocatedPages` aren't directly usable until they are mapped to physical frames. -/// +/// /// Allocation is based on a red-black tree and is thus `O(log(n))`. /// Fragmentation isn't cleaned up until we're out of address space, but that's not really a big deal. /// @@ -780,11 +813,14 @@ pub fn allocate_pages_deferred( AllocationRequest::AtVirtualAddress(vaddr) => { find_specific_chunk(&mut locked_list, Page::containing_address(vaddr), num_pages) } - AllocationRequest::Any => { - find_any_chunk(&mut locked_list, num_pages, None) + AllocationRequest::AlignedTo { alignment_4k_pages } => { + find_any_chunk(&mut locked_list, num_pages, None, alignment_4k_pages) } AllocationRequest::WithinRange(range) => { - find_any_chunk(&mut locked_list, num_pages, Some(range)) + find_any_chunk(&mut locked_list, num_pages, Some(range), 1) + } + AllocationRequest::Any => { + find_any_chunk(&mut locked_list, num_pages, None, 1) } }; res.map_err(From::from) // convert from AllocationError to &str diff --git a/theseus_features/Cargo.toml b/theseus_features/Cargo.toml index 15d3ee2d6c..b4f504aea4 100644 --- a/theseus_features/Cargo.toml +++ b/theseus_features/Cargo.toml @@ -47,6 +47,7 @@ hello = { path = "../applications/hello", optional = true } raw_mode = { path = "../applications/raw_mode", optional = true } print_fault_log = { path = "../applications/print_fault_log", optional = true } seconds_counter = { path = "../applications/seconds_counter", optional = true } +test_aligned_page_allocation = { path = "../applications/test_aligned_page_allocation", optional = true } test_async = { path = "../applications/test_async", optional = true } test_backtrace = { path = "../applications/test_backtrace", optional = true } test_block_io = { path = "../applications/test_block_io", optional = true } @@ -146,6 +147,7 @@ theseus_tests = [ "hello", "raw_mode", "seconds_counter", + "test_aligned_page_allocation", "test_async", "test_backtrace", "test_block_io", From 2d44b3cc845ed397697c61354afbec367a15a9ff Mon Sep 17 00:00:00 2001 From: Nathan Royer <61582713+NathanRoyer@users.noreply.github.com> Date: Wed, 23 Aug 2023 19:21:32 +0200 Subject: [PATCH 3/4] Redesign GIC and interrupt controller to use owned, per-CPU memory mappings (#1026) * The GIC crate now uses MMIO register structs overlaid atop `MappedPages` for safe, easier-to-understand access to GIC features. * GIC-related functions are now implemented as methods on the appropriate register-owning structs. * This is similar to how things are done in x86's `LocalApic`. * GIC now implements two main structures: 1. `ArmGicDistributor`, the system-wide interrupt controller. 2. `ArmGicCpuComponents`: the per-CPU local interrupt controller. There is one of these per CPU; they are currently stored in an array but will be moved into CPU-local storage once that is ready. * These are initialized by the aarch64 module in `interrupt_controller`. Co-authored-by: Kevin Boos --- Cargo.lock | 4 +- kernel/arm_boards/src/lib.rs | 2 + kernel/gic/Cargo.toml | 3 +- kernel/gic/src/gic/cpu_interface_gicv2.rs | 119 +++-- kernel/gic/src/gic/cpu_interface_gicv3.rs | 34 +- kernel/gic/src/gic/dist_interface.rs | 223 +++++---- kernel/gic/src/gic/mod.rs | 550 ++++++++++++--------- kernel/gic/src/gic/redist_interface.rs | 198 +++++--- kernel/gic/src/lib.rs | 6 +- kernel/interrupt_controller/Cargo.toml | 1 + kernel/interrupt_controller/src/aarch64.rs | 221 ++++----- kernel/interrupt_controller/src/lib.rs | 5 + kernel/interrupt_controller/src/x86_64.rs | 8 + kernel/interrupts/src/aarch64/mod.rs | 16 +- 14 files changed, 784 insertions(+), 606 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99b617840d..793010530c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1385,7 +1385,8 @@ dependencies = [ "cpu", "log", "memory", - "static_assertions", + "spin 0.9.4", + "volatile 0.2.7", "zerocopy", ] @@ -1581,6 +1582,7 @@ dependencies = [ "ioapic", "log", "memory", + "spin 0.9.4", "sync_irq", ] diff --git a/kernel/arm_boards/src/lib.rs b/kernel/arm_boards/src/lib.rs index 584b7f2d43..04afb18f36 100644 --- a/kernel/arm_boards/src/lib.rs +++ b/kernel/arm_boards/src/lib.rs @@ -25,6 +25,8 @@ pub struct PciEcamConfig { pub size_bytes: usize, } +/// This excludes GICv2 because there's no way to send IPIs +/// with GICv2 via the current driver API. #[derive(Debug, Copy, Clone)] pub enum InterruptControllerConfig { GicV3(GicV3InterruptControllerConfig), diff --git a/kernel/gic/Cargo.toml b/kernel/gic/Cargo.toml index 6b37e010c5..3f34a945eb 100644 --- a/kernel/gic/Cargo.toml +++ b/kernel/gic/Cargo.toml @@ -6,8 +6,9 @@ edition = "2021" name = "gic" [dependencies] -static_assertions = "1.1.0" zerocopy = "0.5.0" +volatile = "0.2.7" +spin = "0.9.4" log = "0.4.8" memory = { path = "../memory" } diff --git a/kernel/gic/src/gic/cpu_interface_gicv2.rs b/kernel/gic/src/gic/cpu_interface_gicv2.rs index 3a301fede5..58d8c81bca 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv2.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv2.rs @@ -6,17 +6,39 @@ //! - Acknowledging interrupt requests //! - Sending End-Of-Interrupts signals -use super::GicRegisters; use super::Priority; use super::InterruptNumber; -mod offset { - use crate::Offset32; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); - pub(crate) const PMR: Offset32 = Offset32::from_byte_offset(0x04); - pub(crate) const IAR: Offset32 = Offset32::from_byte_offset(0x0C); - pub(crate) const RPR: Offset32 = Offset32::from_byte_offset(0x14); - pub(crate) const EOIR: Offset32 = Offset32::from_byte_offset(0x10); +use volatile::{Volatile, ReadOnly, WriteOnly}; +use zerocopy::FromBytes; + +/// The GICv2 MMIO registers for interfacing with a specific CPU. +/// +/// Methods herein apply to the "current" CPU only, i.e., the CPU +/// on which the code that accesses these registers is currently running. +/// +/// Note: the physical address for this structure is the same for all CPUs, +/// but the actual backing memory refers to physically separate registers. +#[derive(FromBytes)] +#[repr(C)] +pub struct CpuRegsP1 { // base offset + /// CPU Interface Control Register + ctlr: Volatile, // 0x00 + + /// Interrupt Priority Mask Register + prio_mask: Volatile, // 0x04 + + /// Binary Point Register + _unused0: u32, + + /// Interrupt Acknowledge Register + acknowledge: ReadOnly, // 0x0C + + /// End of Interrupt Register + eoi: WriteOnly, // 0x10 + + /// Running Priority Register + running_prio: ReadOnly, // 0x14 } // enable group 0 @@ -25,45 +47,52 @@ mod offset { // enable group 1 const CTLR_ENGRP1: u32 = 0b10; -/// Enables routing of group 1 interrupts for the current CPU -pub fn init(registers: &mut GicRegisters) { - let mut reg = registers.read_volatile(offset::CTLR); - reg |= CTLR_ENGRP1; - registers.write_volatile(offset::CTLR, reg); -} +impl CpuRegsP1 { + /// Enables routing of group 1 interrupts for the current CPU. + pub fn init(&mut self) { + let mut reg = self.ctlr.read(); + reg |= CTLR_ENGRP1; + self.ctlr.write(reg); + } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them -pub fn get_minimum_priority(registers: &GicRegisters) -> Priority { - u8::MAX - (registers.read_volatile(offset::PMR) as u8) -} + /// Retrieves the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. + pub fn get_minimum_priority(&self) -> Priority { + u8::MAX - (self.prio_mask.read() as u8) + } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them -pub fn set_minimum_priority(registers: &mut GicRegisters, priority: Priority) { - registers.write_volatile(offset::PMR, (u8::MAX - priority) as u32); -} + /// Sets the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. + pub fn set_minimum_priority(&mut self, priority: Priority) { + self.prio_mask.write((u8::MAX - priority) as u32); + } -/// Signals to the controller that the currently processed interrupt has -/// been fully handled, by zeroing the current priority level of this CPU. -/// This implies that the CPU is ready to process interrupts again. -pub fn end_of_interrupt(registers: &mut GicRegisters, int: InterruptNumber) { - registers.write_volatile(offset::EOIR, int); -} + /// Signals to the controller that the currently processed interrupt + /// has been fully handled, by zeroing the current priority level of + /// the current CPU. + /// + /// This implies that the CPU is ready to process interrupts again. + pub fn end_of_interrupt(&mut self, int: InterruptNumber) { + self.eoi.write(int); + } + + /// Acknowledge the currently serviced interrupt and fetches its + /// number. + /// + /// This tells the GIC that the requested interrupt is being + /// handled by this CPU. + pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) { + // Reading the interrupt number has the side effect + // of acknowledging the interrupt. + let int_num = self.acknowledge.read() as InterruptNumber; + let priority = self.running_prio.read() as u8; -/// Acknowledge the currently serviced interrupt -/// and fetches its number; this tells the GIC that -/// the requested interrupt is being handled by -/// this CPU. -pub fn acknowledge_interrupt(registers: &mut GicRegisters) -> (InterruptNumber, Priority) { - // Reading the interrupt number has the side effect - // of acknowledging the interrupt. - let int_num = registers.read_volatile(offset::IAR) as InterruptNumber; - let priority = registers.read_volatile(offset::RPR) as u8; - - (int_num, priority) + (int_num, priority) + } } diff --git a/kernel/gic/src/gic/cpu_interface_gicv3.rs b/kernel/gic/src/gic/cpu_interface_gicv3.rs index b5ee7d399b..63072864b2 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv3.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv3.rs @@ -35,37 +35,42 @@ pub fn init() { unsafe { asm!("msr ICC_IGRPEN1_EL1, {}", in(reg) IGRPEN_ENABLED) }; } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them +/// Retrieves the current priority threshold for the current CPU. +/// +/// Interrupts have a priority; if their priority is lower or +/// equal to this threshold, they're queued until the current CPU +/// is ready to handle them. pub fn get_minimum_priority() -> Priority { let mut reg_value: u64; unsafe { asm!("mrs {}, ICC_PMR_EL1", out(reg) reg_value) }; u8::MAX - (reg_value as u8) } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them +/// Sets the current priority threshold for the current CPU. +/// +/// Interrupts have a priority; if their priority is lower or +/// equal to this threshold, they're queued until the current CPU +/// is ready to handle them. pub fn set_minimum_priority(priority: Priority) { let reg_value = (u8::MAX - priority) as u64; unsafe { asm!("msr ICC_PMR_EL1, {}", in(reg) reg_value) }; } -/// Signals to the controller that the currently processed interrupt has -/// been fully handled, by zeroing the current priority level of this CPU. +/// Signals to the controller that the currently processed interrupt +/// has been fully handled, by zeroing the current priority level of +/// the current CPU. +/// /// This implies that the CPU is ready to process interrupts again. pub fn end_of_interrupt(int: InterruptNumber) { let reg_value = int as u64; unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) }; } -/// Acknowledge the currently serviced interrupt -/// and fetches its number; this tells the GIC that -/// the requested interrupt is being handled by -/// this CPU. +/// Acknowledge the currently serviced interrupt and fetches its +/// number. +/// +/// This tells the GIC that the requested interrupt is being +/// handled by this CPU. pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { let int_num: u64; let priority: u64; @@ -82,6 +87,7 @@ pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { (int_num as InterruptNumber, priority as u8) } +/// Generates an interrupt in CPU interfaces of the system pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) { let mut value = match target { IpiTargetCpu::Specific(cpu) => { diff --git a/kernel/gic/src/gic/dist_interface.rs b/kernel/gic/src/gic/dist_interface.rs index 398568aa55..baa7fb7488 100644 --- a/kernel/gic/src/gic/dist_interface.rs +++ b/kernel/gic/src/gic/dist_interface.rs @@ -13,29 +13,65 @@ //! - Getting or setting the target of SPIs based on their numbers //! - Generating software interrupts (GICv2 style) -use super::GicRegisters; use super::IpiTargetCpu; use super::SpiDestination; use super::InterruptNumber; use super::Enabled; use super::Priority; use super::TargetList; -use super::ArmGicV3; +use super::read_array_volatile; +use super::write_array_volatile; +use volatile::{Volatile, ReadOnly}; +use zerocopy::FromBytes; use cpu::MpidrValue; -mod offset { - use crate::{Offset32, Offset64}; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x000); - pub(crate) const IIDR: Offset32 = Offset32::from_byte_offset(0x008); - pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x080); - pub(crate) const ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); - pub(crate) const ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); - pub(crate) const IPRIORITYR: Offset32 = Offset32::from_byte_offset(0x400); - pub(crate) const ITARGETSR: Offset32 = Offset32::from_byte_offset(0x800); - pub(crate) const SGIR: Offset32 = Offset32::from_byte_offset(0xf00); - /// This one is on the 6th page - pub(crate) const P6IROUTER: Offset64 = Offset64::from_byte_offset(0x100); +/// First page of distributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct DistRegsP1 { // base offset + /// Distributor Control Register + ctlr: Volatile, // 0x000 + + /// Interrupt Controller Type Register + typer: ReadOnly, // 0x004 + + /// Distributor Implementer Identification Register + ident: ReadOnly, // 0x008 + + _unused0: [u8; 0x074], + + /// Interrupt Group Registers + group: [Volatile; 0x020], // 0x080 + + /// Interrupt Set-Enable Registers + set_enable: [Volatile; 0x020], // 0x100 + + /// Interrupt Clear-Enable Registers + clear_enable: [Volatile; 0x020], // 0x180 + + _unused1: [u8; 0x200], + + /// Interrupt Priority Registers + priority: [Volatile; 0x100], // 0x400 + + /// Interrupt Processor Targets Registers + target: [Volatile; 0x100], // 0x800 + + _unused2: [u8; 0x300], + + /// Software Generated Interrupt Register + sgir: Volatile, // 0xf00 +} + +/// Sixth page of distributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct DistRegsP6 { // base offset + _unused: [u8; 0x100], + + /// Interrupt Routing Registers + route: Volatile, // 0x100 } // enable group 0 @@ -66,75 +102,78 @@ const GROUP_1: u32 = 1; // bit 15: which interrupt group to target const SGIR_NSATT_GRP1: u32 = 1 << 15; -/// Initializes the distributor by enabling forwarding -/// of group 1 interrupts and allowing the GIC to pick -/// a core that is asleep for "1 of N" interrupts. -/// -/// Return value: whether or not affinity routing is -/// currently enabled for both secure and non-secure -/// states. -pub fn init(registers: &mut GicRegisters) -> Enabled { - let mut reg = registers.read_volatile(offset::CTLR); - reg |= CTLR_ENGRP1; - reg |= CTLR_E1NWF; - registers.write_volatile(offset::CTLR, reg); - - // Return value: whether or not affinity routing is - // currently enabled for both secure and non-secure - // states. - reg & CTLR_ARE_NS > 0 -} - -/// Returns whether the given SPI (shared peripheral interrupt) will be -/// forwarded by the distributor -pub fn is_spi_enabled(registers: &GicRegisters, int: InterruptNumber) -> Enabled { - // enabled? - registers.read_array_volatile::<32>(offset::ISENABLER, int) > 0 - && - // part of group 1? - registers.read_array_volatile::<32>(offset::IGROUPR, int) == GROUP_1 -} +impl DistRegsP1 { + /// Initializes the distributor by enabling forwarding + /// of group 1 interrupts and allowing the GIC to pick + /// a core that is asleep for "1 of N" interrupts. + /// + /// Return value: whether or not affinity routing is + /// currently enabled for both secure and non-secure + /// states. + pub fn init(&mut self) -> Enabled { + let mut reg = self.ctlr.read(); + reg |= CTLR_ENGRP1; + reg |= CTLR_E1NWF; + self.ctlr.write(reg); + + // Return value: whether or not affinity routing is + // currently enabled for both secure and non-secure + // states. + reg & CTLR_ARE_NS > 0 + } -/// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt) -pub fn enable_spi(registers: &mut GicRegisters, int: InterruptNumber, enabled: Enabled) { - let reg_base = match enabled { - true => offset::ISENABLER, - false => offset::ICENABLER, - }; - registers.write_array_volatile::<32>(reg_base, int, 1); - - // whether we're enabling or disabling, - // set as part of group 1 - registers.write_array_volatile::<32>(reg_base, int, GROUP_1); -} + /// Returns whether the given SPI (shared peripheral interrupt) will be + /// forwarded by the distributor + pub fn is_spi_enabled(&self, int: InterruptNumber) -> Enabled { + // enabled? + read_array_volatile::<32>(&self.set_enable, int) > 0 + && + // part of group 1? + read_array_volatile::<32>(&self.group, int) == GROUP_1 + } -/// Returns the priority of an SPI. -pub fn get_spi_priority(registers: &GicRegisters, int: InterruptNumber) -> Priority { - u8::MAX - (registers.read_array_volatile::<4>(offset::IPRIORITYR, int) as u8) -} + /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt) + pub fn enable_spi(&mut self, int: InterruptNumber, enabled: Enabled) { + let reg_base = match enabled { + true => &mut self.set_enable, + false => &mut self.clear_enable, + }; + write_array_volatile::<32>(reg_base, int, 1); + + // whether we're enabling or disabling, + // set as part of group 1 + write_array_volatile::<32>(&mut self.group, int, GROUP_1); + } -/// Sets the priority of an SPI. -pub fn set_spi_priority(registers: &mut GicRegisters, int: InterruptNumber, prio: Priority) { - registers.write_array_volatile::<4>(offset::IPRIORITYR, int, (u8::MAX - prio) as u32); -} + /// Returns the priority of an SPI. + pub fn get_spi_priority(&self, int: InterruptNumber) -> Priority { + u8::MAX - (read_array_volatile::<4>(&self.priority, int) as u8) + } -/// Sends an Inter-Processor-Interrupt -/// -/// legacy / GICv2 method -/// int_num must be less than 16 -pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: IpiTargetCpu) { - if let IpiTargetCpu::Specific(cpu) = &target { - assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + /// Sets the priority of an SPI. + pub fn set_spi_priority(&mut self, int: InterruptNumber, prio: Priority) { + write_array_volatile::<4>(&mut self.priority, int, (u8::MAX - prio) as u32); } - let target_list = match target { - IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16, - IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, - IpiTargetCpu::GICv2TargetList(list) => (list.0 as u32) << 16, - }; + /// Sends an Inter-Processor-Interrupt + /// + /// legacy / GICv2 method + /// int_num must be less than 16 + #[allow(dead_code)] + pub fn send_ipi_gicv2(&mut self, int_num: u32, target: IpiTargetCpu) { + if let IpiTargetCpu::Specific(cpu) = &target { + assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + } + + let target_list = match target { + IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16, + IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, + IpiTargetCpu::GICv2TargetList(list) => (list.0 as u32) << 16, + }; - let value: u32 = int_num | target_list | SGIR_NSATT_GRP1; - registers.write_volatile(offset::SGIR, value); + let value: u32 = int_num | target_list | SGIR_NSATT_GRP1; + self.sgir.write(value); + } } /// Deserialized content of the `IIDR` distributor register @@ -147,23 +186,23 @@ pub struct Implementer { pub implementer_jep106: u16, } -impl super::ArmGic { - pub(crate) fn distributor(&self) -> &GicRegisters { +impl super::ArmGicDistributor { + pub(crate) fn distributor(&self) -> &DistRegsP1 { match self { - Self::V2(v2) => &v2.distributor, - Self::V3(v3) => &v3.distributor, + Self::V2 { registers } => registers, + Self::V3 { v2_regs, .. } => v2_regs, } } - pub(crate) fn distributor_mut(&mut self) -> &mut GicRegisters { + pub(crate) fn distributor_mut(&mut self) -> &mut DistRegsP1 { match self { - Self::V2(v2) => &mut v2.distributor, - Self::V3(v3) => &mut v3.distributor, + Self::V2 { registers } => registers, + Self::V3 { v2_regs, .. } => v2_regs, } } pub fn implementer(&self) -> Implementer { - let raw = self.distributor().read_volatile(offset::IIDR); + let raw = self.distributor().ident.read(); Implementer { product_id: (raw >> 24) as _, version: ((raw >> 12) & 0xff) as _, @@ -179,8 +218,8 @@ impl super::ArmGic { pub fn get_spi_target(&self, int: InterruptNumber) -> Result { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); match self { - Self::V2(_) | Self::V3(ArmGicV3 { affinity_routing: false, .. }) => { - let flags = self.distributor().read_array_volatile::<4>(offset::ITARGETSR, int); + Self::V2 { .. } | Self::V3 { affinity_routing: false, .. } => { + let flags = read_array_volatile::<4>(&self.distributor().target, int); for i in 0..8 { let target = 1 << i; @@ -193,8 +232,8 @@ impl super::ArmGic { Ok(SpiDestination::GICv2TargetList(TargetList(flags as u8)).canonicalize()) }, - Self::V3(ArmGicV3 { affinity_routing: true, dist_extended, .. }) => { - let reg = dist_extended.read_volatile_64(offset::P6IROUTER); + Self::V3 { affinity_routing: true, v3_regs, .. } => { + let reg = v3_regs.route.read(); // bit 31: Interrupt Routing Mode // value of 1 to target any available cpu @@ -214,7 +253,7 @@ impl super::ArmGic { pub fn set_spi_target(&mut self, int: InterruptNumber, target: SpiDestination) { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); match self { - Self::V2(_) | Self::V3(ArmGicV3 { affinity_routing: false, .. }) => { + Self::V2 { .. } | Self::V3 { affinity_routing: false, .. } => { if let SpiDestination::Specific(cpu) = &target { assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); } @@ -229,9 +268,9 @@ impl super::ArmGic { SpiDestination::GICv2TargetList(list) => list.0 as u32, }; - self.distributor_mut().write_array_volatile::<4>(offset::ITARGETSR, int, value); + write_array_volatile::<4>(&mut self.distributor_mut().target, int, value); }, - Self::V3(ArmGicV3 { affinity_routing: true, dist_extended, .. }) => { + Self::V3 { affinity_routing: true, v3_regs, .. } => { let value = match target { SpiDestination::Specific(cpu) => MpidrValue::from(cpu).value(), // bit 31: Interrupt Routing Mode @@ -242,7 +281,7 @@ impl super::ArmGic { }, }; - dist_extended.write_volatile_64(offset::P6IROUTER, value); + v3_regs.route.write(value); } } } diff --git a/kernel/gic/src/gic/mod.rs b/kernel/gic/src/gic/mod.rs index 00f233f104..66f8247ac9 100644 --- a/kernel/gic/src/gic/mod.rs +++ b/kernel/gic/src/gic/mod.rs @@ -1,19 +1,24 @@ -use core::convert::AsMut; - -use cpu::{CpuId, MpidrValue}; use arm_boards::{BOARD_CONFIG, NUM_CPUS}; +use cpu::{CpuId, MpidrValue}; +use volatile::Volatile; +use spin::Once; + use memory::{ - PageTable, BorrowedMappedPages, Mutable, PhysicalAddress, - allocate_pages, allocate_frames_at, MMIO_FLAGS, + BorrowedMappedPages, Mutable, PhysicalAddress, MappedPages, + AllocatedFrames, get_kernel_mmi_ref, allocate_frames_at, + allocate_pages, map_frame_range, PAGE_SIZE, MMIO_FLAGS, }; -use static_assertions::const_assert_eq; mod cpu_interface_gicv3; mod cpu_interface_gicv2; mod dist_interface; mod redist_interface; +use dist_interface::{DistRegsP1, DistRegsP6}; +use cpu_interface_gicv2::CpuRegsP1; +use redist_interface::{RedistRegsP1, RedistRegsSgiPpi}; + /// Boolean pub type Enabled = bool; @@ -118,336 +123,395 @@ impl SpiDestination { const U32BITS: usize = u32::BITS as usize; -#[derive(Copy, Clone)] -pub(crate) struct Offset32(usize); +// Reads one item of an array spanning across +// multiple u32s. +// +// The maximum item size is 32 bits, and the items are always aligned to 2**N bits. +// The array spans multiple adjacent u32s but there is always a integer number of +// items in a single u32. +// +// - `int` is the index +// - `INTS_PER_U32` = how many array slots per u32 in this array +fn read_array_volatile(slice: &[Volatile], int: InterruptNumber) -> u32 { + let int = int as usize; + let bits_per_int: usize = U32BITS / INTS_PER_U32; + let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); + + let offset = int / INTS_PER_U32; + let reg_index = int & (INTS_PER_U32 - 1); + let shift = reg_index * bits_per_int; + + let reg = slice[offset].read(); + (reg >> shift) & mask +} -#[derive(Copy, Clone)] -pub(crate) struct Offset64(usize); +// Writes one item of an array spanning across +// multiple u32s. +// +// The maximum item size is 32 bits, and the items are always aligned to 2**N bits. +// The array spans multiple adjacent u32s but there is always a integer number of +// items in a single u32. +// +// - `int` is the index +// - `INTS_PER_U32` = how many array slots per u32 in this array +// - `value` is the value to write +fn write_array_volatile(slice: &mut [Volatile], int: InterruptNumber, value: u32) { + let int = int as usize; + let bits_per_int: usize = U32BITS / INTS_PER_U32; + let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); + + let offset = int / INTS_PER_U32; + let reg_index = int & (INTS_PER_U32 - 1); + let shift = reg_index * bits_per_int; + + let mut reg = slice[offset].read(); + reg &= !(mask << shift); + reg |= (value & mask) << shift; + slice[offset].write(reg); +} -impl Offset32 { - pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { - Self(byte_offset / core::mem::size_of::()) - } +const REDIST_SGIPPI_OFFSET: usize = 0x10000; +const DIST_P6_OFFSET: usize = 0x6000; + +pub struct ArmGicV3RedistPages { + pub redistributor: BorrowedMappedPages, + pub redist_sgippi: BorrowedMappedPages, } -impl Offset64 { - pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { - Self(byte_offset / core::mem::size_of::()) +pub enum Version { + InitV2 { + dist: PhysicalAddress, + cpu: PhysicalAddress, + }, + InitV3 { + dist: PhysicalAddress, + redist: [PhysicalAddress; NUM_CPUS], } } -#[repr(C)] -#[derive(zerocopy::FromBytes)] -pub struct GicRegisters { - inner: [u32; 0x400], +pub enum ArmGicDistributor { + V2 { + registers: BorrowedMappedPages, + }, + V3 { + affinity_routing: Enabled, + v2_regs: BorrowedMappedPages, + v3_regs: BorrowedMappedPages, + }, } -impl GicRegisters { - fn read_volatile(&self, offset: Offset32) -> u32 { - unsafe { (&self.inner[offset.0] as *const u32).read_volatile() } - } +impl ArmGicDistributor { + pub fn init(version: &Version) -> Result { + match version { + Version::InitV2 { dist, .. } => { + let mapped = map_frame_range(*dist, PAGE_SIZE, MMIO_FLAGS)?; + let mut registers = mapped + .into_borrowed_mut::(0) + .map_err(|(_, e)| e)?; - fn write_volatile(&mut self, offset: Offset32, value: u32) { - unsafe { (&mut self.inner[offset.0] as *mut u32).write_volatile(value) } - } + registers.init(); - fn read_volatile_64(&self, offset: Offset64) -> u64 { - unsafe { (self.inner.as_ptr() as *const u64).add(offset.0).read_volatile() } - } + Ok(Self::V2 { + registers, + }) + }, + Version::InitV3 { dist, .. } => { + let mapped = map_frame_range(*dist, PAGE_SIZE, MMIO_FLAGS)?; + let mut v2_regs = mapped + .into_borrowed_mut::(0) + .map_err(|(_, e)| e)?; + + let v3_regs: BorrowedMappedPages = { + let paddr = *dist + DIST_P6_OFFSET; + let mapped = map_frame_range(paddr, PAGE_SIZE, MMIO_FLAGS)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; - fn write_volatile_64(&mut self, offset: Offset64, value: u64) { - unsafe { (self.inner.as_mut_ptr() as *mut u64).add(offset.0).write_volatile(value) } - } + let affinity_routing = v2_regs.init(); - // Reads one item of an array spanning across - // multiple u32s. - // - // The maximum item size is 32 bits, and the items are always aligned to 2**N bits. - // The array spans multiple adjacent u32s but there is always a integer number of - // items in a single u32. - // - // - `int` is the index - // - `offset` tells the beginning of the array - // - `INTS_PER_U32` = how many array slots per u32 in this array - fn read_array_volatile(&self, offset: Offset32, int: InterruptNumber) -> u32 { - let int = int as usize; - let bits_per_int: usize = U32BITS / INTS_PER_U32; - let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - - let offset = Offset32(offset.0 + (int / INTS_PER_U32)); - let reg_index = int & (INTS_PER_U32 - 1); - let shift = reg_index * bits_per_int; - - let reg = self.read_volatile(offset); - (reg >> shift) & mask + Ok(Self::V3 { + affinity_routing, + v2_regs, + v3_regs, + }) + }, + } } - // Writes one item of an array spanning across - // multiple u32s. - // - // The maximum item size is 32 bits, and the items are always aligned to 2**N bits. - // The array spans multiple adjacent u32s but there is always a integer number of - // items in a single u32. - // - // - `int` is the index - // - `offset` tells the beginning of the array - // - `INTS_PER_U32` = how many array slots per u32 in this array - // - `value` is the value to write - fn write_array_volatile(&mut self, offset: Offset32, int: InterruptNumber, value: u32) { - let int = int as usize; - let bits_per_int: usize = U32BITS / INTS_PER_U32; - let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - - let offset = Offset32(offset.0 + (int / INTS_PER_U32)); - let reg_index = int & (INTS_PER_U32 - 1); - let shift = reg_index * bits_per_int; - - let mut reg = self.read_volatile(offset); - reg &= !(mask << shift); - reg |= (value & mask) << shift; - self.write_volatile(offset, reg); + /// Returns whether the given interrupt is forwarded by the distributor. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn get_spi_state(&self, int: InterruptNumber) -> Enabled { + assert!(int >= 32, "get_spi_state: `int` must be >= 32"); + self.distributor().is_spi_enabled(int) } -} - -const_assert_eq!(core::mem::size_of::(), 0x1000); - -/// Returns the index to the redistributor base address for this CPU -/// in the array of register base addresses. -/// -/// This is defined in `arm_boards::INTERRUPT_CONTROLLER_CONFIG`. -fn get_current_cpu_redist_index() -> usize { - let cpu_id = cpu::current_cpu(); - BOARD_CONFIG.cpu_ids.iter() - .position(|mpidr| CpuId::from(*mpidr) == cpu_id) - .expect("BUG: get_current_cpu_redist_index: unexpected CpuId for current CPU") -} - -const REDIST_SGIPPI_OFFSET: usize = 0x10000; -const DIST_P6_OFFSET: usize = 0x6000; -pub struct ArmGicV2 { - pub distributor: BorrowedMappedPages, - pub processor: BorrowedMappedPages, -} + /// Enables or disables the forwarding of the given interrupt + /// by the distributor. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn set_spi_state(&mut self, int: InterruptNumber, enabled: Enabled) { + assert!(int >= 32, "set_spi_state: `int` must be >= 32"); + self.distributor_mut().enable_spi(int, enabled) + } -pub struct ArmGicV3RedistPages { - pub redistributor: BorrowedMappedPages, - pub redist_sgippi: BorrowedMappedPages, -} + /// Returns the priority of the given interrupt. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn get_spi_priority(&self, int: InterruptNumber) -> Priority { + assert!(int >= 32, "get_spi_priority: `int` must be >= 32"); + self.distributor().get_spi_priority(int) + } -pub struct ArmGicV3 { - pub affinity_routing: Enabled, - pub distributor: BorrowedMappedPages, - pub dist_extended: BorrowedMappedPages, - pub redistributors: [ArmGicV3RedistPages; NUM_CPUS], + /// Sets the priority of the given interrupt. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn set_spi_priority(&mut self, int: InterruptNumber, enabled: Priority) { + assert!(int >= 32, "set_spi_priority: `int` must be >= 32"); + self.distributor_mut().set_spi_priority(int, enabled) + } } -/// Arm Generic Interrupt Controller -/// -/// The GIC is an extension to ARMv8 which -/// allows routing and filtering interrupts -/// in a single or multi-core system. -#[allow(clippy::large_enum_variant)] -pub enum ArmGic { - V2(ArmGicV2), - V3(ArmGicV3), +pub enum ArmGicCpuComponents { + V2 { + registers: BorrowedMappedPages, + cpu_index: u16, + }, + V3 { + redist_regs: ArmGicV3RedistPages, + }, } -pub enum Version { - InitV2 { - dist: PhysicalAddress, - cpu: PhysicalAddress, - }, - InitV3 { - dist: PhysicalAddress, - redist: [PhysicalAddress; NUM_CPUS], +/// Map the physical frames containing the GICv2 CPU Interface's MMIO registers into the given `page_table`. +fn map_gicv2_cpu_iface(cpu_iface: PhysicalAddress) -> Result { + static CPU_IFACE_FRAME: Once = Once::new(); + + let frame = if let Some(cpu_iface) = CPU_IFACE_FRAME.get() { + cpu_iface + } else { + let cpu_iface = allocate_frames_at(cpu_iface, 1)?; + CPU_IFACE_FRAME.call_once(|| cpu_iface) + }; + + let new_page = allocate_pages(1).ok_or("out of virtual address space!")?; + let mmi = get_kernel_mmi_ref().ok_or("map_gicv2_cpu_iface(): uninitialized KERNEL_MMI")?; + let mut mmi = mmi.lock(); + + // The CPU Interface frame is the same actual physical address across all CPU cores, + // but they're actually completely independent pieces of hardware that share one address. + // Therefore, there's no way to represent that to the Rust language or + // MappedPages/AllocatedFrames types, so we must use unsafe code, at least for now. + unsafe { + memory::Mapper::map_to_non_exclusive( + &mut mmi.page_table, + new_page, + frame, + MMIO_FLAGS, + ) } } -impl ArmGic { - pub fn init(page_table: &mut PageTable, version: Version) -> Result { - let mut map_dist = |gicd_base| -> Result, &'static str> { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the distributor interface")?; - let frames = allocate_frames_at(gicd_base, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e) - }; +impl ArmGicCpuComponents { + pub fn init(cpu_id: CpuId, version: &Version) -> Result { + let cpu_index = BOARD_CONFIG.cpu_ids.iter() + .position(|mpidr| CpuId::from(*mpidr) == cpu_id) + .expect("BUG: invalid CpuId in ArmGicCpuComponents::init"); match version { - Version::InitV2 { dist, cpu } => { - let mut distributor = map_dist(dist)?; - - let mut processor: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the CPU interface")?; - let frames = allocate_frames_at(cpu, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; + Version::InitV2 { cpu, .. } => { + let mut registers: BorrowedMappedPages = { + let mapped = map_gicv2_cpu_iface(*cpu)?; mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - cpu_interface_gicv2::init(processor.as_mut()); - dist_interface::init(distributor.as_mut()); + registers.init(); - Ok(Self::V2(ArmGicV2 { distributor, processor })) + Ok(Self::V2 { + registers, + cpu_index: cpu_index as u16, + }) }, - Version::InitV3 { dist, redist } => { - let mut distributor = map_dist(dist)?; + Version::InitV3 { redist, .. } => { + let phys_addr = redist[cpu_index]; - let dist_extended: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended distributor interface")?; - let frames = allocate_frames_at(dist + DIST_P6_OFFSET, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; + let mut redistributor: BorrowedMappedPages = { + let mapped = map_frame_range(phys_addr, PAGE_SIZE, MMIO_FLAGS)?; mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - let redistributors: [ArmGicV3RedistPages; NUM_CPUS] = core::array::try_from_fn(|i| { - let phys_addr = redist[i]; - - let mut redistributor: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the redistributor interface")?; - let frames = allocate_frames_at(phys_addr, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; - - redist_interface::init(redistributor.as_mut())?; + let redist_sgippi = { + let rso_paddr = phys_addr + REDIST_SGIPPI_OFFSET; + let mapped = map_frame_range(rso_paddr, PAGE_SIZE, MMIO_FLAGS)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; - let redist_sgippi = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended redistributor interface")?; - let frames = allocate_frames_at(phys_addr + REDIST_SGIPPI_OFFSET, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; + redistributor.init()?; + cpu_interface_gicv3::init(); - Ok::(ArmGicV3RedistPages { + Ok(Self::V3 { + redist_regs: ArmGicV3RedistPages { redistributor, redist_sgippi, - }) - })?; - - // this cannot fail as we pushed exactly `arm_boards::CPUS` items - // let redistributors = redistributors.into_inner().unwrap(); - - cpu_interface_gicv3::init(); - let affinity_routing = dist_interface::init(distributor.as_mut()); - - Ok(Self::V3(ArmGicV3 { distributor, dist_extended, redistributors, affinity_routing })) + }, + }) }, } } pub fn init_secondary_cpu_interface(&mut self) { match self { - Self::V2(v2) => cpu_interface_gicv2::init(v2.processor.as_mut()), - Self::V3( _) => cpu_interface_gicv3::init(), + Self::V2 { registers, .. } => registers.init(), + Self::V3 { .. } => cpu_interface_gicv3::init(), } } - /// Sends an inter processor interrupt (IPI), - /// also called software generated interrupt (SGI). + /// Sends an Inter-Processor Interrupt (IPI) with the given interrupt number + /// to the given target CPU(s). + /// + /// This is also referred to as a Software-Generated Interrupt (SGI). /// - /// note: on Aarch64, IPIs must have a number below 16 on ARMv8 - pub fn send_ipi(&mut self, int_num: InterruptNumber, target: IpiTargetCpu) { - assert!(int_num < 16, "IPIs must have a number below 16 on ARMv8"); + /// Panics if `int` is greater than or equal to 16; + /// on aarch64, IPIs much be sent to an interrupt number less than 16. + pub fn send_ipi(&mut self, int: InterruptNumber, target: IpiTargetCpu) { + assert!(int < 16, "IPIs must have a number below 16 on ARMv8"); - match self { - Self::V2(v2) => dist_interface::send_ipi_gicv2(&mut v2.distributor, int_num, target), - Self::V3( _) => cpu_interface_gicv3::send_ipi(int_num, target), + if let Self::V3 { .. } = self { + cpu_interface_gicv3::send_ipi(int, target) + } else { + // we don't have access to the distributor... code would be: + // dist_interface::send_ipi_gicv2(&mut dist_regs, int, target) + // workaround: caller could check is this must be done in the dist + // and then get the SystemInterruptController and call a dedicated + // method on it, like `sys_ctlr.send_ipi_gicv2()` + + panic!("GICv2 doesn't support sending IPIs (need distributor)"); } } - /// Acknowledge the currently serviced interrupt - /// and fetches its number + /// Acknowledge the currently-serviced interrupt. + /// + /// This tells the GIC that the current interrupt is in the midst of + /// being handled by this CPU. + /// + /// Returns a tuple of the interrupt's number and priority. pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) { match self { - Self::V2(v2) => cpu_interface_gicv2::acknowledge_interrupt(&mut v2.processor), - Self::V3( _) => cpu_interface_gicv3::acknowledge_interrupt(), + Self::V2 { registers, .. } => registers.acknowledge_interrupt(), + Self::V3 { .. } => cpu_interface_gicv3::acknowledge_interrupt(), } } - /// Performs priority drop for the specified interrupt + /// Signals to the controller that the currently processed interrupt + /// has been fully handled, by zeroing the current priority level of + /// the current CPU. + /// + /// This implies that the CPU is ready to process interrupts again. pub fn end_of_interrupt(&mut self, int: InterruptNumber) { match self { - Self::V2(v2) => cpu_interface_gicv2::end_of_interrupt(&mut v2.processor, int), - Self::V3( _) => cpu_interface_gicv3::end_of_interrupt(int), + Self::V2 { registers, .. } => registers.end_of_interrupt(int), + Self::V3 { .. } => cpu_interface_gicv3::end_of_interrupt(int), } } - /// Will that interrupt be forwarded by the distributor? + /// Returns whether the given local interrupt will be received by the current CPU. + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn get_interrupt_state(&self, int: InterruptNumber) -> Enabled { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::is_sgippi_enabled(&v3.redistributors[i].redist_sgippi, int) - }, - (_, this) => dist_interface::is_spi_enabled(this.distributor(), int), + assert!(int < 32, "get_interrupt_state: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.is_sgippi_enabled(int) + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support enabling/disabling local interrupt"); + + // should we panic? + true } } - /// Enables or disables the forwarding of - /// a particular interrupt in the distributor + /// Enables or disables the receiving of a local interrupt in the distributor. + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn set_interrupt_state(&mut self, int: InterruptNumber, enabled: Enabled) { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::enable_sgippi(&mut v3.redistributors[i].redist_sgippi, int, enabled); - }, - (_, this) => dist_interface::enable_spi(this.distributor_mut(), int, enabled), - }; + assert!(int < 32, "set_interrupt_state: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.enable_sgippi(int, enabled); + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support enabling/disabling local interrupt"); + } } - /// Returns the priority of an interrupt + /// Returns the priority of a local interrupt + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn get_interrupt_priority(&self, int: InterruptNumber) -> Priority { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::get_sgippi_priority(&v3.redistributors[i].redist_sgippi, int) - }, - (_, this) => dist_interface::get_spi_priority(this.distributor(), int), + assert!(int < 32, "get_interrupt_priority: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.get_sgippi_priority(int) + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support setting local interrupt priority"); + + // should we panic? + 128 } } - /// Sets the priority of an interrupt (0-255) + /// Sets the priority of a local interrupt (prio: 0-255) + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn set_interrupt_priority(&mut self, int: InterruptNumber, enabled: Priority) { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::set_sgippi_priority(&mut v3.redistributors[i].redist_sgippi, int, enabled); - }, - (_, this) => dist_interface::set_spi_priority(this.distributor_mut(), int, enabled), - }; + assert!(int < 32, "set_interrupt_priority: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.set_sgippi_priority(int, enabled); + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support setting local interrupt priority"); + } } - /// Interrupts have a priority; if their priority - /// is lower or equal to this one, they're queued - /// until this CPU or another one is ready to handle - /// them + /// Retrieves the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. pub fn get_minimum_priority(&self) -> Priority { match self { - Self::V2(v2) => cpu_interface_gicv2::get_minimum_priority(&v2.processor), - Self::V3( _) => cpu_interface_gicv3::get_minimum_priority(), + Self::V2 { registers, .. } => registers.get_minimum_priority(), + Self::V3 { .. } => cpu_interface_gicv3::get_minimum_priority(), } } - /// Interrupts have a priority; if their priority - /// is lower or equal to this one, they're queued - /// until this CPU or another one is ready to handle - /// them + /// Sets the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. pub fn set_minimum_priority(&mut self, priority: Priority) { match self { - Self::V2(v2) => cpu_interface_gicv2::set_minimum_priority(&mut v2.processor, priority), - Self::V3( _) => cpu_interface_gicv3::set_minimum_priority(priority), + Self::V2 { registers, .. } => registers.set_minimum_priority(priority), + Self::V3 { .. } => cpu_interface_gicv3::set_minimum_priority(priority), } } /// Returns the internal ID of the redistributor (GICv3) /// - /// Note #2: this is only provided for debugging purposes - /// Note #1: as a compatibility feature, on GICv2, the CPU index is returned. + /// ## Notes + /// * As a compatibility feature, on GICv2, the CPU index is returned. + /// * This is only provided for debugging purposes. pub fn get_cpu_interface_id(&self) -> u16 { - let i = get_current_cpu_redist_index(); match self { - Self::V3(v3) => redist_interface::get_internal_id(&v3.redistributors[i].redistributor), - _ => i as _, + Self::V3 { redist_regs } => redist_regs.redistributor.get_internal_id(), + Self::V2 { cpu_index, .. } => *cpu_index, } } } diff --git a/kernel/gic/src/gic/redist_interface.rs b/kernel/gic/src/gic/redist_interface.rs index 7f14ae2907..02461b343a 100644 --- a/kernel/gic/src/gic/redist_interface.rs +++ b/kernel/gic/src/gic/redist_interface.rs @@ -9,20 +9,58 @@ //! - Enabling or disabling the forwarding of PPIs & SGIs based on their numbers //! - Getting or setting the priority of PPIs & SGIs based on their numbers -use super::GicRegisters; use super::InterruptNumber; use super::Enabled; use super::Priority; +use super::read_array_volatile; +use super::write_array_volatile; -mod offset { - use crate::{Offset32, Offset64}; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); - pub(crate) const TYPER: Offset64 = Offset64::from_byte_offset(0x08); - pub(crate) const WAKER: Offset32 = Offset32::from_byte_offset(0x14); - pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x80); - pub(crate) const SGIPPI_ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); - pub(crate) const SGIPPI_ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); - pub(crate) const SGIPPI_IPRIORITYR: Offset32 = Offset32::from_byte_offset(0x400); +use volatile::{Volatile, ReadOnly}; +use zerocopy::FromBytes; + +/// General redistributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct RedistRegsP1 { // base offset + /// Redistributor Control Register + ctlr: Volatile, // 0x00 + + /// Implementer Identification Register + _unused0: u32, + + /// Redistributor Type Register + ident: ReadOnly, // 0x08 + + /// Error Reporting Status Register, optional + _unused1: u32, + + /// Redistributor Wake Register + waker: Volatile, // 0x14 +} + +/// Redistributor registers for SGIs & PPIs +#[derive(FromBytes)] +#[repr(C)] +pub struct RedistRegsSgiPpi { // base offset + _reserved0: [u8; 0x80], + + /// Interrupt Group Register 0 + group: [Volatile; 0x01], // 0x080 + _reserved1: [u32; 0x1f], + + /// Interrupt Set-Enable Registers + set_enable: [Volatile; 0x01], // 0x100 + _reserved2: [u32; 0x1f], + + /// Interrupt Clear-Enable Registers + clear_enable: [Volatile; 0x01], // 0x180 + _reserved3: [u32; 0x1f], + + /// Interrupt Set & Clear Pending / Active Registers + _unused0: [u32; 0x80], + + /// Interrupt Priority Registers + priority: [Volatile; 0x08], // 0x400 } const WAKER_PROCESSOR_SLEEP: u32 = 1 << 1; @@ -57,86 +95,90 @@ const GROUP_1: u32 = 1; /// developers (or directly submit a PR). const TIMEOUT_ITERATIONS: usize = 0xffff; -/// Initializes the redistributor by waking it up and waiting for it to awaken. -/// -/// Returns an error if a timeout occurs while waiting. -pub fn init(registers: &mut GicRegisters) -> Result<(), &'static str> { - let mut reg = registers.read_volatile(offset::WAKER); - - // Wake the redistributor - reg &= !WAKER_PROCESSOR_SLEEP; - registers.write_volatile(offset::WAKER, reg); - - // Then, wait for the children to wake up, timing out if it never happens. - let children_asleep = || { - registers.read_volatile(offset::WAKER) & WAKER_CHLIDREN_ASLEEP > 0 - }; - let mut counter = 0; - while children_asleep() { - counter += 1; +impl RedistRegsP1 { + /// Initializes the redistributor by waking it up and waiting for it to awaken. + /// + /// Returns an error if a timeout occurs while waiting. + pub fn init(&mut self) -> Result<(), &'static str> { + let mut reg = self.waker.read(); + + // Wake the redistributor + reg &= !WAKER_PROCESSOR_SLEEP; + self.waker.write(reg); + + // Then, wait for the children to wake up, timing out if it never happens. + let children_asleep = || { + self.waker.read() & WAKER_CHLIDREN_ASLEEP > 0 + }; + let mut counter = 0; + while children_asleep() { + counter += 1; + if counter >= TIMEOUT_ITERATIONS { + break; + } + } + if counter >= TIMEOUT_ITERATIONS { - break; + return Err("BUG: gic driver: The redistributor didn't wake up in time."); } - } - if counter >= TIMEOUT_ITERATIONS { - return Err("BUG: gic driver: The redistributor didn't wake up in time."); - } + if self.ident.read() & TYPER_DPGS != 0 { + // DPGS bits are supported in GICR_CTLR + let mut reg = self.ctlr.read(); - if registers.read_volatile_64(offset::TYPER) & TYPER_DPGS != 0 { - // DPGS bits are supported in GICR_CTLR - let mut reg = registers.read_volatile(offset::CTLR); + // Enable PE selection for non-secure group 1 SPIs + reg &= !CTLR_DPG1NS; - // Enable PE selection for non-secure group 1 SPIs - reg &= !CTLR_DPG1NS; + // Disable PE selection for group 0 & secure group 1 SPIs + reg |= CTLR_DPG0; + reg |= CTLR_DPG1S; - // Disable PE selection for group 0 & secure group 1 SPIs - reg |= CTLR_DPG0; - reg |= CTLR_DPG1S; + self.ctlr.write(reg); + } - registers.write_volatile(offset::CTLR, reg); + Ok(()) } - Ok(()) -} - -/// Returns whether the given SGI (software generated interrupts) or -/// PPI (private peripheral interrupts) will be forwarded by the redistributor -pub fn is_sgippi_enabled(registers: &GicRegisters, int: InterruptNumber) -> Enabled { - registers.read_array_volatile::<32>(offset::SGIPPI_ISENABLER, int) > 0 - && - // part of group 1? - registers.read_array_volatile::<32>(offset::IGROUPR, int) == GROUP_1 + /// Returns the internal ID of the redistributor + /// + /// Note: this is only provided for debugging purposes + pub fn get_internal_id(&self) -> u16 { + (self.ident.read() >> 8) as _ + } } -/// Enables or disables the forwarding of a particular -/// SGI (software generated interrupts) or PPI (private -/// peripheral interrupts) -pub fn enable_sgippi(registers: &mut GicRegisters, int: InterruptNumber, enabled: Enabled) { - let reg = match enabled { - true => offset::SGIPPI_ISENABLER, - false => offset::SGIPPI_ICENABLER, - }; - registers.write_array_volatile::<32>(reg, int, 1); - - // whether we're enabling or disabling, - // set as part of group 1 - registers.write_array_volatile::<32>(offset::IGROUPR, int, GROUP_1); -} +impl RedistRegsSgiPpi { + /// Returns whether the given SGI (software generated interrupts) or + /// PPI (private peripheral interrupts) will be forwarded by the redistributor + pub fn is_sgippi_enabled(&self, int: InterruptNumber) -> Enabled { + read_array_volatile::<32>(&self.set_enable, int) > 0 + && + // part of group 1? + read_array_volatile::<32>(&self.group, int) == GROUP_1 + } -/// Returns the priority of an SGI/PPI. -pub fn get_sgippi_priority(registers: &GicRegisters, int: InterruptNumber) -> Priority { - u8::MAX - (registers.read_array_volatile::<4>(offset::SGIPPI_IPRIORITYR, int) as u8) -} + /// Enables or disables the forwarding of a particular + /// SGI (software generated interrupts) or PPI (private + /// peripheral interrupts) + pub fn enable_sgippi(&mut self, int: InterruptNumber, enabled: Enabled) { + let reg = match enabled { + true => &mut self.set_enable, + false => &mut self.clear_enable, + }; + write_array_volatile::<32>(reg, int, 1); + + // whether we're enabling or disabling, + // set as part of group 1 + write_array_volatile::<32>(&mut self.group, int, GROUP_1); + } -/// Sets the priority of an SGI/PPI. -pub fn set_sgippi_priority(registers: &mut GicRegisters, int: InterruptNumber, prio: Priority) { - registers.write_array_volatile::<4>(offset::SGIPPI_IPRIORITYR, int, (u8::MAX - prio) as u32); -} + /// Returns the priority of an SGI/PPI. + pub fn get_sgippi_priority(&self, int: InterruptNumber) -> Priority { + u8::MAX - (read_array_volatile::<4>(&self.priority, int) as u8) + } -/// Returns the internal ID of the redistributor -/// -/// Note: this is only provided for debugging purposes -pub fn get_internal_id(registers: &GicRegisters) -> u16 { - (registers.read_volatile_64(offset::TYPER) >> 8) as _ + /// Sets the priority of an SGI/PPI. + pub fn set_sgippi_priority(&mut self, int: InterruptNumber, prio: Priority) { + write_array_volatile::<4>(&mut self.priority, int, (u8::MAX - prio) as u32); + } } diff --git a/kernel/gic/src/lib.rs b/kernel/gic/src/lib.rs index 2861978bdf..c07e40ec24 100644 --- a/kernel/gic/src/lib.rs +++ b/kernel/gic/src/lib.rs @@ -1,4 +1,7 @@ -//! Allows configuring the Generic Interrupt Controller +//! Arm Generic Interrupt Controller Support +//! +//! The GIC is an extension to ARMv8 which allows routing and +//! filtering interrupts in a single or multi-core system. //! //! The term "Forwarding" is sometimes used in this crate. //! This is because the Distributor, Redistributor and CPU interface are @@ -8,7 +11,6 @@ #![no_std] #![feature(doc_cfg)] -#![feature(array_try_from_fn)] #[cfg(target_arch = "aarch64")] mod gic; diff --git a/kernel/interrupt_controller/Cargo.toml b/kernel/interrupt_controller/Cargo.toml index 61a8ccf902..8d237fea46 100644 --- a/kernel/interrupt_controller/Cargo.toml +++ b/kernel/interrupt_controller/Cargo.toml @@ -14,6 +14,7 @@ sync_irq = { path = "../../libs/sync_irq" } arm_boards = { path = "../arm_boards" } memory = { path = "../memory" } gic = { path = "../gic" } +spin = "0.9.4" [target.'cfg(target_arch = "x86_64")'.dependencies] apic = { path = "../apic" } diff --git a/kernel/interrupt_controller/src/aarch64.rs b/kernel/interrupt_controller/src/aarch64.rs index 5988c9f97d..6da4f16714 100644 --- a/kernel/interrupt_controller/src/aarch64.rs +++ b/kernel/interrupt_controller/src/aarch64.rs @@ -1,9 +1,10 @@ use { - gic::{ArmGic, SpiDestination, IpiTargetCpu, Version as GicVersion}, - arm_boards::{BOARD_CONFIG, InterruptControllerConfig}, + gic::{ArmGicDistributor, ArmGicCpuComponents, SpiDestination, IpiTargetCpu, Version as GicVersion}, + arm_boards::{NUM_CPUS, BOARD_CONFIG, InterruptControllerConfig}, + core::array::try_from_fn, sync_irq::IrqSafeMutex, - memory::get_kernel_mmi_ref, - core::ops::DerefMut, + cpu::current_cpu, + spin::Once, }; use super::*; @@ -17,65 +18,72 @@ pub struct SystemInterruptControllerId(pub u8); #[derive(Debug, Copy, Clone)] pub struct LocalInterruptControllerId(pub u16); -/// The private global Generic Interrupt Controller singleton -pub(crate) static INTERRUPT_CONTROLLER: IrqSafeMutex> = IrqSafeMutex::new(None); +/// Per-CPU local interrupt controller +/// +/// To get the controller for a specific CPU: +/// a. Find the position of its CpuId in `BOARD_CONFIG.cpu_ids` +/// b. Index into this array using that position +static LOCAL_INT_CTRL: Once<[LocalInterruptController; NUM_CPUS]> = Once::new(); + +/// System-wide interrupt controller +static SYSTEM_WIDE_INT_CTRL: Once = Once::new(); /// Initializes the interrupt controller, on aarch64 pub fn init() -> Result<(), &'static str> { - let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); - if int_ctrl.is_some() { - Err("The interrupt controller has already been initialized!") - } else { - match BOARD_CONFIG.interrupt_controller { - InterruptControllerConfig::GicV3(gicv3_cfg) => { - let kernel_mmi_ref = get_kernel_mmi_ref() - .ok_or("interrupts::aarch64::init: couldn't get kernel MMI ref")?; - - let mut mmi = kernel_mmi_ref.lock(); - let page_table = &mut mmi.deref_mut().page_table; - - *int_ctrl = Some(ArmGic::init( - page_table, - GicVersion::InitV3 { - dist: gicv3_cfg.distributor_base_address, - redist: gicv3_cfg.redistributor_base_addresses, - }, - )?); - }, - } - - Ok(()) - } + match BOARD_CONFIG.interrupt_controller { + InterruptControllerConfig::GicV3(gicv3_cfg) => { + let version = GicVersion::InitV3 { + dist: gicv3_cfg.distributor_base_address, + redist: gicv3_cfg.redistributor_base_addresses, + }; + + SYSTEM_WIDE_INT_CTRL.try_call_once(|| -> Result<_, &'static str> { + let distrib = ArmGicDistributor::init(&version)?; + let mutex = IrqSafeMutex::new(distrib); + Ok(SystemInterruptController(mutex)) + })?; + + LOCAL_INT_CTRL.try_call_once(|| -> Result<_, &'static str> { + let cpu_ctlrs: [ArmGicCpuComponents; NUM_CPUS] = try_from_fn(|i| { + let cpu_id = BOARD_CONFIG.cpu_ids[i].into(); + ArmGicCpuComponents::init(cpu_id, &version) + })?; + + Ok(cpu_ctlrs.map(|ctlr| { + let mutex = IrqSafeMutex::new(ctlr); + LocalInterruptController(mutex) + })) + })?; + }, + } + + Ok(()) } /// Structure representing a top-level/system-wide interrupt controller chip, /// responsible for routing interrupts between peripherals and CPU cores. /// /// On aarch64 w/ GIC, this corresponds to the Distributor. -pub struct SystemInterruptController; +pub struct SystemInterruptController(IrqSafeMutex); /// Struct representing per-cpu-core interrupt controller chips. /// /// On aarch64 w/ GIC, this corresponds to a Redistributor & CPU interface. -pub struct LocalInterruptController; +pub struct LocalInterruptController(IrqSafeMutex); impl SystemInterruptControllerApi for SystemInterruptController { - fn id(&self) -> SystemInterruptControllerId { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: id(): INTERRUPT_CONTROLLER was uninitialized"); + fn get() -> &'static Self { + SYSTEM_WIDE_INT_CTRL.get().expect("interrupt_controller wasn't initialized") + } - SystemInterruptControllerId(int_ctlr.implementer().product_id) + fn id(&self) -> SystemInterruptControllerId { + let dist = self.0.lock(); + SystemInterruptControllerId(dist.implementer().product_id) } fn version(&self) -> SystemInterruptControllerVersion { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: version(): INTERRUPT_CONTROLLER was uninitialized"); - - SystemInterruptControllerVersion(int_ctlr.implementer().version) + let dist = self.0.lock(); + SystemInterruptControllerVersion(dist.implementer().version) } fn get_destination( @@ -83,13 +91,10 @@ impl SystemInterruptControllerApi for SystemInterruptController { interrupt_num: InterruptNumber, ) -> Result<(Vec, Priority), &'static str> { assert!(interrupt_num >= 32, "shared peripheral interrupts have a number >= 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_destination(): INTERRUPT_CONTROLLER was uninitialized"); + let dist = self.0.lock(); - let priority = int_ctlr.get_interrupt_priority(interrupt_num as _); - let vec = match int_ctlr.get_spi_target(interrupt_num as _)?.canonicalize() { + let priority = dist.get_spi_priority(interrupt_num as _); + let vec = match dist.get_spi_target(interrupt_num as _)?.canonicalize() { SpiDestination::Specific(cpu) => [cpu].to_vec(), SpiDestination::AnyCpuAvailable => BOARD_CONFIG.cpu_ids.map(|mpidr| mpidr.into()).to_vec(), SpiDestination::GICv2TargetList(list) => { @@ -111,126 +116,98 @@ impl SystemInterruptControllerApi for SystemInterruptController { priority: Priority, ) -> Result<(), &'static str> { assert!(sys_int_num >= 32, "shared peripheral interrupts have a number >= 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_destination(): INTERRUPT_CONTROLLER was uninitialized"); + let mut dist = self.0.lock(); - int_ctlr.set_spi_target(sys_int_num as _, SpiDestination::Specific(destination)); - int_ctlr.set_interrupt_priority(sys_int_num as _, priority); + dist.set_spi_target(sys_int_num as _, SpiDestination::Specific(destination)); + dist.set_spi_priority(sys_int_num as _, priority); Ok(()) } } impl LocalInterruptControllerApi for LocalInterruptController { - fn init_secondary_cpu_interface(&self) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: init_secondary_cpu_interface(): INTERRUPT_CONTROLLER was uninitialized"); + fn get() -> &'static Self { + // how this function works: + // a. get the current CpuId: this CpuId of the current CPU + // b. iterate on all valid CpuIds, find the index of the current CpuId. + // This is used as a current CPU index. + // c. get the global array of interrupt controllers + // d. index into this array based on the current CPU index - int_ctlr.init_secondary_cpu_interface(); + let cpu_id = current_cpu(); + // While we're waiting for cpu-local-storage, this loop will work as fine as an AtomicMap + let index = BOARD_CONFIG.cpu_ids.iter().position(|mpidr| cpu_id == (*mpidr).into()); + let index = index.expect("Invalid CpuId returned by current_cpu()"); + + let ctrls = LOCAL_INT_CTRL.get(); + let ctrls = ctrls.expect("interrupt_controller wasn't initialized"); + + &ctrls[index] } - fn id(&self) -> LocalInterruptControllerId { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: id(): INTERRUPT_CONTROLLER was uninitialized"); + fn init_secondary_cpu_interface(&self) { + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.init_secondary_cpu_interface(); + } - LocalInterruptControllerId(int_ctlr.get_cpu_interface_id()) + fn id(&self) -> LocalInterruptControllerId { + let cpu_ctrl = self.0.lock(); + LocalInterruptControllerId(cpu_ctrl.get_cpu_interface_id()) } fn get_local_interrupt_priority(&self, num: InterruptNumber) -> Priority { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_local_interrupt_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_interrupt_priority(num as _) + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_interrupt_priority(num as _) } fn set_local_interrupt_priority(&self, num: InterruptNumber, priority: Priority) { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_local_interrupt_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_interrupt_priority(num as _, priority); + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_interrupt_priority(num as _, priority); } fn is_local_interrupt_enabled(&self, num: InterruptNumber) -> bool { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: is_local_interrupt_enabled(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_interrupt_state(num as _) + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_interrupt_state(num as _) } fn enable_local_interrupt(&self, num: InterruptNumber, enabled: bool) { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: enable_local_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_interrupt_state(num as _, enabled); + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_interrupt_state(num as _, enabled); } fn send_ipi(&self, num: InterruptNumber, dest: InterruptDestination) { use InterruptDestination::*; assert!(num < 16, "IPIs have a number < 16"); + let mut cpu_ctrl = self.0.lock(); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: send_ipi(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.send_ipi(num as _, match dest { + cpu_ctrl.send_ipi(num as _, match dest { SpecificCpu(cpu) => IpiTargetCpu::Specific(cpu), AllOtherCpus => IpiTargetCpu::AllOtherCpus, }); } fn get_minimum_priority(&self) -> Priority { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_minimum_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_minimum_priority() + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_minimum_priority() } fn set_minimum_priority(&self, priority: Priority) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_minimum_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_minimum_priority(priority) + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_minimum_priority(priority) } fn acknowledge_interrupt(&self) -> (InterruptNumber, Priority) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: acknowledge_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - let (num, prio) = int_ctlr.acknowledge_interrupt(); + let mut cpu_ctrl = self.0.lock(); + let (num, prio) = cpu_ctrl.acknowledge_interrupt(); (num as _, prio) } fn end_of_interrupt(&self, number: InterruptNumber) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: end_of_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.end_of_interrupt(number as _) + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.end_of_interrupt(number as _) } } diff --git a/kernel/interrupt_controller/src/lib.rs b/kernel/interrupt_controller/src/lib.rs index 7d8c69f459..55c08fe697 100644 --- a/kernel/interrupt_controller/src/lib.rs +++ b/kernel/interrupt_controller/src/lib.rs @@ -1,5 +1,6 @@ #![no_std] #![allow(unused_variables, unused_mut)] +#![feature(array_try_from_fn)] extern crate alloc; @@ -38,6 +39,8 @@ pub enum InterruptDestination { } pub trait SystemInterruptControllerApi { + fn get() -> &'static Self; + fn id(&self) -> SystemInterruptControllerId; fn version(&self) -> SystemInterruptControllerVersion; @@ -55,6 +58,8 @@ pub trait SystemInterruptControllerApi { } pub trait LocalInterruptControllerApi { + fn get() -> &'static Self; + /// Aarch64-specific way to initialize the secondary CPU interfaces. /// /// Must be called once from every secondary CPU. diff --git a/kernel/interrupt_controller/src/x86_64.rs b/kernel/interrupt_controller/src/x86_64.rs index 1b4ca9cfb9..917062e3f4 100644 --- a/kernel/interrupt_controller/src/x86_64.rs +++ b/kernel/interrupt_controller/src/x86_64.rs @@ -31,6 +31,10 @@ pub struct SystemInterruptController { pub struct LocalInterruptController; impl SystemInterruptControllerApi for SystemInterruptController { + fn get() -> &'static Self { + unimplemented!() + } + fn id(&self) -> SystemInterruptControllerId { let mut int_ctlr = get_ioapic(self.id).expect("BUG: id(): get_ioapic() returned None"); SystemInterruptControllerId(int_ctlr.id()) @@ -65,6 +69,10 @@ impl SystemInterruptControllerApi for SystemInterruptController { impl LocalInterruptControllerApi for LocalInterruptController { + fn get() -> &'static Self { + unimplemented!() + } + fn init_secondary_cpu_interface(&self) { panic!("This must not be used on x86_64") } diff --git a/kernel/interrupts/src/aarch64/mod.rs b/kernel/interrupts/src/aarch64/mod.rs index 892d78cd2d..716dd42506 100644 --- a/kernel/interrupts/src/aarch64/mod.rs +++ b/kernel/interrupts/src/aarch64/mod.rs @@ -140,7 +140,7 @@ pub fn init_ap() { set_vbar_el1(); // Enable the CPU-local timer - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.init_secondary_cpu_interface(); int_ctrl.set_minimum_priority(0); @@ -160,7 +160,7 @@ pub fn init() -> Result<(), &'static str> { set_vbar_el1(); - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.set_minimum_priority(0); Ok(()) @@ -178,7 +178,7 @@ pub fn init_timer(timer_tick_handler: InterruptHandler) -> Result<(), &'static s // Route the IRQ to this core (implicit as IRQ < 32) & Enable the interrupt. { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); // enable routing of this interrupt int_ctrl.enable_local_interrupt(CPU_LOCAL_TIMER_IRQ, true); @@ -198,7 +198,7 @@ pub fn setup_ipi_handler(handler: InterruptHandler, local_num: InterruptNumber) } { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); // enable routing of this interrupt int_ctrl.enable_local_interrupt(local_num, true); @@ -209,7 +209,7 @@ pub fn setup_ipi_handler(handler: InterruptHandler, local_num: InterruptNumber) /// Enables the PL011 "RX" SPI and routes it to the current CPU. pub fn init_pl011_rx_interrupt() -> Result<(), &'static str> { - let int_ctrl = SystemInterruptController; + let int_ctrl = SystemInterruptController::get(); int_ctrl.set_destination(PL011_RX_SPI, current_cpu(), u8::MAX) } @@ -295,14 +295,14 @@ pub fn deregister_interrupt(int_num: InterruptNumber, func: InterruptHandler) -> /// Broadcast an Inter-Processor Interrupt to all other /// cores in the system pub fn send_ipi_to_all_other_cpus(irq_num: InterruptNumber) { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.send_ipi(irq_num, InterruptDestination::AllOtherCpus); } /// Send an "end of interrupt" signal, notifying the interrupt chip that /// the given interrupt request `irq` has been serviced. pub fn eoi(irq_num: InterruptNumber) { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.end_of_interrupt(irq_num); } @@ -417,7 +417,7 @@ extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { #[no_mangle] extern "C" fn current_elx_irq(exc: &mut ExceptionContext) { let (irq_num, _priority) = { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.acknowledge_interrupt() }; From 27429e57120be7b6725101fde4582dfd48b936d0 Mon Sep 17 00:00:00 2001 From: Klim Tsoutsman Date: Fri, 25 Aug 2023 09:40:46 +1000 Subject: [PATCH 4/4] format: remove unnecessary whitespace in `mod_mgmt/src/lib.rs` (#1030) No changes in functionality. Intended to reduce the number of changes in #1028. Signed-off-by: Klimenty Tsoutsman --- kernel/mod_mgmt/src/lib.rs | 380 ++++++++++++++++++------------------- 1 file changed, 188 insertions(+), 192 deletions(-) diff --git a/kernel/mod_mgmt/src/lib.rs b/kernel/mod_mgmt/src/lib.rs index 5439308ec0..edd569705e 100644 --- a/kernel/mod_mgmt/src/lib.rs +++ b/kernel/mod_mgmt/src/lib.rs @@ -8,8 +8,8 @@ use core::{fmt, ops::{Deref, Range}}; use alloc::{ - collections::{BTreeMap, btree_map, BTreeSet}, - string::{String, ToString}, + collections::{BTreeMap, btree_map, BTreeSet}, + string::{String, ToString}, sync::{Arc, Weak}, vec::Vec }; use spin::{Mutex, Once}; @@ -59,15 +59,15 @@ pub fn get_namespaces_directory() -> Option { /// The thread-local storage (TLS) area "image" that is used as the initial data for each `Task`. /// When spawning a new task, the new task will create its own local TLS area /// with this `TlsInitializer` as the initial data values. -/// +/// /// # Implementation Notes/Shortcomings /// Currently, a single system-wide `TlsInitializer` instance is shared across all namespaces. /// In the future, each namespace should hold its own TLS sections in its TlsInitializer area. -/// +/// /// However, this is quite complex because each namespace must be aware of the TLS sections /// in BOTH its underlying recursive namespace AND its (multiple) "parent" namespace(s) /// that recursively depend on it, since no two TLS sections can conflict (have the same offset). -/// +/// /// Thus, we stick with a singleton `TlsInitializer` instance, which makes sense /// because it behaves much like an allocator, in that it reserves space (index ranges) in the TLS area. static TLS_INITIALIZER: Mutex = Mutex::new(TlsInitializer::empty()); @@ -76,10 +76,10 @@ static TLS_INITIALIZER: Mutex = Mutex::new(TlsInitializer::empty /// Create a new application `CrateNamespace` that uses the default application directory /// and is structured atop the given `recursive_namespace`. /// If no `recursive_namespace` is provided, the default initial kernel namespace will be used. -/// +/// /// # Return /// The returned `CrateNamespace` will itself be empty, having no crates and no symbols in its map. -/// +/// pub fn create_application_namespace(recursive_namespace: Option>) -> Result, &'static str> { // (1) use the initial kernel CrateNamespace as the new app namespace's recursive namespace if none was provided. let recursive_namespace = recursive_namespace @@ -119,10 +119,10 @@ pub fn init( /// and placing them into namespace-specific directories according to their name prefix, e.g., "k#", "ksse#". /// This function does not create any namespaces, it just populates the files and directories /// such that namespaces can be created based on those files. -/// +/// /// If a file does not have an expected crate prefix according to [`CrateType::from_module_name()`], /// then it is treated as part of "extra_files"; see [`parse_extra_file()`] for more. -/// +/// /// Returns a tuple of: /// * the top-level root "namespaces" directory that contains all other namespace directories, /// * the directory of the default kernel crate namespace. @@ -232,14 +232,14 @@ fn parse_bootloader_modules_into_files( } /// Adds the given extra file to the directory of extra files -/// +/// /// See the top-level Makefile target "extra_files" for an explanation of how these work. /// Basically, they are arbitrary files that are included by the bootloader as modules /// (files that exist as areas of pre-loaded memory). -/// +/// /// Their file paths are encoded by flattening directory hierarchies into a the file name, /// using `'!'` (exclamation marks) to replace the directory delimiter `'/'`. -/// +/// /// Thus, for example, a file named `"foo!bar!me!test.txt"` will be placed at the path /// `/extra_files/foo/bar/me/test.txt`. fn parse_extra_file( @@ -248,9 +248,8 @@ fn parse_extra_file( extra_file_mp: MappedPages, extra_files_dir: DirRef ) -> Result { - let mut file_name = extra_file_name; - + let mut parent_dir = extra_files_dir; let mut iter = extra_file_name.split(EXTRA_FILES_DIRECTORY_DELIMITER).peekable(); while let Some(path_component) = iter.next() { @@ -278,7 +277,7 @@ fn parse_extra_file( -/// A "symbol map" from a fully-qualified demangled symbol String +/// A "symbol map" from a fully-qualified demangled symbol String /// to weak reference to a `LoadedSection`. /// This is used for relocations, and for looking up function names. pub type SymbolMap = Trie; @@ -286,9 +285,9 @@ pub type SymbolMap = Trie; /// A wrapper around a `Directory` reference that offers special convenience functions /// for getting and inserting crate object files into a directory. -/// +/// /// Auto-derefs into a `DirRef`. -#[derive(Clone)] +#[derive(Clone)] pub struct NamespaceDir(DirRef); impl Deref for NamespaceDir { @@ -312,10 +311,10 @@ impl NamespaceDir { /// Creates a new `NamespaceDir` that wraps the given `DirRef`. pub fn new(dir: DirRef) -> NamespaceDir { NamespaceDir(dir) - } + } /// Finds the single file in this directory whose name starts with the given `prefix`. - /// + /// /// # Return /// If a single file matches, then that file is returned. /// Otherwise, if no files or multiple files match, then `None` is returned. @@ -347,11 +346,11 @@ impl NamespaceDir { } /// Gets the given object file based on its crate name prefix. - /// + /// /// # Arguments /// * `crate_object_file_name`: the name of the object file to be returned, /// with or without a preceding `CrateType` prefix. - /// + /// /// # Examples /// * The name "k#keyboard-36be916209949cef.o" will look for and return the file "keyboard-36be916209949cef.o". /// * The name "keyboard-36be916209949cef.o" will look for and return the file "keyboard-36be916209949cef.o". @@ -362,12 +361,12 @@ impl NamespaceDir { } /// Insert the given crate object file based on its crate type prefix. - /// + /// /// # Arguments /// * `crate_object_file_name`: the name of the object file to be inserted, /// with a preceding `CrateType` prefix. /// * `content`: the bytes that will be written into the file. - /// + /// /// # Examples /// * The file "k#keyboard-36be916209949cef.o" will be written to "./keyboard-36be916209949cef.o". /// * The file "a#ps.o" will be placed into "./ps.o". @@ -381,7 +380,7 @@ impl NamespaceDir { /// A type that can be converted into a crate object file. -/// +/// /// We use an enum rather than implement `TryInto` because we need additional information /// to resolve a `Prefix`, namely the `CrateNamespace` in which to search for the prefix. pub enum IntoCrateObjectFile { @@ -410,9 +409,9 @@ impl fmt::Debug for IntoCrateObjectFile { /// An application crate that has been loaded into a `CrateNamespace`. -/// +/// /// This type auto-derefs into the application's `StrongCrateRef`. -/// +/// /// When dropped, the application crate will be removed /// from the `CrateNamespace` into which it was originally loaded. pub struct AppCrateRef { @@ -455,11 +454,11 @@ impl Drop for AppCrateRef { /// that have all been loaded and linked against each other, /// completely separate and in isolation from any other crate namespace /// (although a given crate may be shared across multiple namespaces). -/// +/// /// Each `CrateNamespace` can be treated as a separate OS personality, /// but are significantly more efficient than library OS-style personalities. /// A `CrateNamespace` is also useful to create a process (task group) abstraction. -/// +/// /// `CrateNamespace`s can also optionally be recursive. /// For example, a namespace that holds just application crates and symbols /// can recursively rely upon (link against) the crates and symbols in a lower-level namespace @@ -499,7 +498,7 @@ pub struct CrateNamespace { /// that is spawned and runs within this `CrateNamespace`. /// When spawning a new task, the new task will create its own local TLS area /// with this `tls_initializer` as the local data. - /// + /// /// NOTE: this is currently a global system-wide singleton. See the static [`static@TLS_INITIALIZER`] for more. tls_initializer: &'static Mutex, @@ -509,7 +508,7 @@ pub struct CrateNamespace { /// Fuzzy matching should only be successful if there is just a single matching symbol; /// if there are multiple matches (e.g., `my_crate::foo::h456` and `my_crate::foo::h789` both exist), /// then the dependency should fail to be resolved. - /// + /// /// This is a potentially dangerous setting because it overrides the compiler-chosen dependency links. /// Thus, it is false by default, and should only be enabled with expert knowledge, /// ideally only temporarily in order to manually load a given crate. @@ -533,7 +532,7 @@ impl CrateNamespace { symbol_map: Mutex::new(SymbolMap::new()), fuzzy_symbol_matching: false, } - } + } /// Returns the name of this `CrateNamespace`, which is just used for debugging purposes. pub fn name(&self) -> &str { @@ -593,7 +592,7 @@ impl CrateNamespace { /// Iterates over all crates in this namespace and calls the given function `f` on each crate. /// If `recursive` is true, crates in recursive namespaces are included in the iteration as well. - /// + /// /// The function `f` is called with two arguments: the name of the crate, and a reference to the crate. /// The function `f` must return a boolean value that indicates whether to continue iterating; /// if `true`, the iteration will continue, if `false`, the iteration will stop. @@ -619,7 +618,7 @@ impl CrateNamespace { /// Acquires the lock on this `CrateNamespace`'s crate list and returns the crate /// that matches the given `crate_name`, if it exists in this namespace. /// If it does not exist in this namespace, then the recursive namespace is searched as well. - /// + /// /// # Important note about Return value /// Returns a `StrongCrateReference` that **has not** been marked as a shared crate reference, /// so if the caller wants to keep the returned `StrongCrateRef` as a shared crate @@ -638,7 +637,7 @@ impl CrateNamespace { /// This function is similar to the [`get_crate`](#method.get_crate) method, /// but it also returns the `CrateNamespace` in which the crate was found. /// It is an associated function rather than a method so it can operate on `Arc`s. - /// + /// /// # Important note about Return value /// Returns a `StrongCrateReference` that **has not** been marked as a shared crate reference, /// so if the caller wants to keep the returned `StrongCrateRef` as a shared crate @@ -654,18 +653,18 @@ impl CrateNamespace { } /// Finds the `LoadedCrate`s whose names start with the given `crate_name_prefix`. - /// + /// /// # Return /// Returns a list of matching crates, in the form of a tuple containing the crate's name, /// a shallow-cloned reference to the crate, and a reference to the namespace in which the matching crate was found. /// If you want to add the returned crate to another namespace, /// you MUST fully `clone()` the returned crate reference in order to mark that crate as shared across namespaces. - /// + /// /// # Important Usage Note /// To avoid greedily matching more crates than expected, you may wish to end the `crate_name_prefix` with "`-`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`-`". /// This works because the delimiter between a crate name and its trailing hash value is "`-`". - /// + /// /// # Example /// * This `CrateNamespace` contains the crates `my_crate-843a613894da0c24` and /// `my_crate_new-933a635894ce0f12`. @@ -673,7 +672,7 @@ impl CrateNamespace { pub fn get_crates_starting_with<'n>( namespace: &'n Arc, crate_name_prefix: &str - ) -> Vec<(StrRef, StrongCrateRef, &'n Arc)> { + ) -> Vec<(StrRef, StrongCrateRef, &'n Arc)> { // First, we make a list of matching crates in this namespace. let crates = namespace.crate_tree.lock(); let mut crates_in_this_namespace = crates.iter_prefix(crate_name_prefix.as_bytes()) @@ -687,24 +686,24 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. crates_in_this_namespace.append(&mut crates_in_recursive_namespace); - crates_in_this_namespace + crates_in_this_namespace } /// Finds the `LoadedCrate` whose name starts with the given `crate_name_prefix`, /// *if and only if* there is a single matching crate in this namespace or any of its recursive namespaces. /// This is a convenience wrapper around the [`get_crates_starting_with()`](#method.get_crates_starting_with) method. - /// + /// /// # Return /// Returns a tuple containing the crate's name, a shallow-cloned reference to the crate, /// and a reference to the namespace in which the matching crate was found. /// If you want to add the returned crate to another namespace, /// you MUST fully `clone()` the returned crate reference in order to mark that crate as shared across namespaces. - /// + /// /// # Important Usage Note /// To avoid greedily matching more crates than expected, you may wish to end the `crate_name_prefix` with "`-`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`-`". /// This works because the delimiter between a crate name and its trailing hash value is "`-`". - /// + /// /// # Example /// * This `CrateNamespace` contains the crates `my_crate-843a613894da0c24` and /// `my_crate_new-933a635894ce0f12`. @@ -714,20 +713,20 @@ impl CrateNamespace { pub fn get_crate_starting_with<'n>( namespace: &'n Arc, crate_name_prefix: &str - ) -> Option<(StrRef, StrongCrateRef, &'n Arc)> { + ) -> Option<(StrRef, StrongCrateRef, &'n Arc)> { let mut crates_iter = Self::get_crates_starting_with(namespace, crate_name_prefix).into_iter(); crates_iter.next().filter(|_| crates_iter.next().is_none()) // ensure single element } /// Like [`get_crates_starting_with()`](#method.get_crates_starting_with), /// but for crate *object file*s instead of loaded crates. - /// + /// /// Returns a list of matching object files and the namespace in which they were found, /// inclusive of recursive namespaces. pub fn get_crate_object_files_starting_with<'n>( namespace: &'n Arc, file_name_prefix: &str - ) -> Vec<(FileRef, &'n Arc)> { + ) -> Vec<(FileRef, &'n Arc)> { // First, we make a list of matching files in this namespace. let mut files = namespace.dir .get_files_starting_with(file_name_prefix) @@ -742,18 +741,18 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. files.append(&mut files_in_recursive_namespace); - files + files } /// Like [`get_crate_starting_with()`](#method.get_crate_starting_with), /// but for crate *object file*s instead of loaded crates. - /// + /// /// Returns the matching object file and the namespace in which it was found, /// if and only if there was a single match (inclusive of recursive namespaces). pub fn get_crate_object_file_starting_with<'n>( namespace: &'n Arc, file_name_prefix: &str - ) -> Option<(FileRef, &'n Arc)> { + ) -> Option<(FileRef, &'n Arc)> { let mut files_iter = Self::get_crate_object_files_starting_with(namespace, file_name_prefix).into_iter(); files_iter.next().filter(|_| files_iter.next().is_none()) // ensure single element } @@ -762,13 +761,13 @@ impl CrateNamespace { /// Same as `get_crate_object_files_starting_with()`, /// but is a method instead of an associated function, /// and also returns `&CrateNamespace` instead of `&Arc`. - /// + /// /// This is only necessary because I can't figure out how to make a generic function /// that accepts and returns either `&CrateNamespace` or `&Arc`. pub fn method_get_crate_object_files_starting_with( &self, file_name_prefix: &str - ) -> Vec<(FileRef, &CrateNamespace)> { + ) -> Vec<(FileRef, &CrateNamespace)> { // First, we make a list of matching files in this namespace. let mut files = self.dir .get_files_starting_with(file_name_prefix) @@ -783,37 +782,37 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. files.append(&mut files_in_recursive_namespace); - files + files } /// Same as `get_crate_object_file_starting_with()`, /// but is a method instead of an associated function, /// and also returns `&CrateNamespace` instead of `&Arc`. - /// + /// /// This is only necessary because I can't figure out how to make a generic function /// that accepts and returns either `&CrateNamespace` or `&Arc`. pub fn method_get_crate_object_file_starting_with( &self, file_name_prefix: &str - ) -> Option<(FileRef, &CrateNamespace)> { + ) -> Option<(FileRef, &CrateNamespace)> { let mut files_iter = self.method_get_crate_object_files_starting_with(file_name_prefix).into_iter(); files_iter.next().filter(|_| files_iter.next().is_none()) // ensure single element } /// Loads the specified application crate into this `CrateNamespace`, allowing it to be run. - /// + /// /// The new application crate's public symbols are added to this `CrateNamespace`'s symbol map, /// allowing other crates in this namespace to depend upon it. - /// + /// /// Application crates are added to the CrateNamespace just like kernel crates, /// so to load an application crate multiple times to spawn multiple instances of it, /// you can create a new top-level namespace to hold that application crate. - /// + /// /// Returns a Result containing the newly-loaded application crate itself. pub fn load_crate_as_application( namespace: &Arc, - crate_object_file: &FileRef, - kernel_mmi_ref: &MmiRef, + crate_object_file: &FileRef, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result { debug!("load_crate_as_application(): trying to load application crate at {:?}", crate_object_file.lock().get_absolute_path()); @@ -836,7 +835,7 @@ impl CrateNamespace { /// Loads the specified crate into memory, allowing it to be invoked. /// Returns a Result containing the number of symbols that were added to the symbol map /// as a result of loading this crate. - /// + /// /// # Arguments /// * `crate_object_file`: the crate object file that will be loaded into this `CrateNamespace`. /// * `temp_backup_namespace`: the `CrateNamespace` that should be searched for missing symbols @@ -848,21 +847,20 @@ impl CrateNamespace { pub fn load_crate( &self, crate_object_file: &FileRef, - temp_backup_namespace: Option<&CrateNamespace>, - kernel_mmi_ref: &MmiRef, + temp_backup_namespace: Option<&CrateNamespace>, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result<(StrongCrateRef, usize), &'static str> { - #[cfg(not(loscd_eval))] debug!("load_crate: trying to load crate at {:?}", crate_object_file.lock().get_absolute_path()); let new_crate_ref = self.load_crate_internal(crate_object_file, temp_backup_namespace, kernel_mmi_ref, verbose_log)?; - + let (new_crate_name, _num_sections, new_syms) = { let new_crate = new_crate_ref.lock_as_ref(); let new_syms = self.add_symbols(new_crate.sections.values(), verbose_log); (new_crate.crate_name.clone(), new_crate.sections.len(), new_syms) }; - + #[cfg(not(loscd_eval))] info!("loaded new crate {:?}, num sections: {}, added {} new symbols.", new_crate_name, _num_sections, new_syms); self.crate_tree.lock().insert(new_crate_name, new_crate_ref.clone_shallow()); @@ -875,8 +873,8 @@ impl CrateNamespace { /// See [`load_crate`](#method.load_crate) and [`load_crate_as_application`](#fn.load_crate_as_application). fn load_crate_internal(&self, crate_object_file: &FileRef, - temp_backup_namespace: Option<&CrateNamespace>, - kernel_mmi_ref: &MmiRef, + temp_backup_namespace: Option<&CrateNamespace>, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result { let cf = crate_object_file.lock(); @@ -885,13 +883,13 @@ impl CrateNamespace { Ok(new_crate_ref) } - + /// This function first loads all of the given crates' sections and adds them to the symbol map, /// and only after *all* crates are loaded does it move on to linking/relocation calculations. - /// + /// /// This allows multiple object files with circular dependencies on one another /// to be loaded all at once, as if they were a single entity. - /// + /// /// # Example /// If crate `A` depends on crate `B`, and crate `B` depends on crate `A`, /// this function will load both crate `A` and `B` before trying to resolve their dependencies individually. @@ -901,7 +899,7 @@ impl CrateNamespace { temp_backup_namespace: Option<&CrateNamespace>, kernel_mmi_ref: &MmiRef, verbose_log: bool, - ) -> Result<(), &'static str> + ) -> Result<(), &'static str> where I: Iterator { // First, lock all of the crate object files. @@ -911,13 +909,13 @@ impl CrateNamespace { } // Second, do all of the section parsing and loading, and add all public symbols to the symbol map. - let mut partially_loaded_crates: Vec<(StrongCrateRef, ElfFile)> = Vec::with_capacity(locked_crate_files.len()); - for locked_crate_file in &locked_crate_files { + let mut partially_loaded_crates: Vec<(StrongCrateRef, ElfFile)> = Vec::with_capacity(locked_crate_files.len()); + for locked_crate_file in &locked_crate_files { let (new_crate_ref, elf_file) = self.load_crate_sections(locked_crate_file.deref(), kernel_mmi_ref, verbose_log)?; let _new_syms = self.add_symbols(new_crate_ref.lock_as_ref().sections.values(), verbose_log); partially_loaded_crates.push((new_crate_ref, elf_file)); } - + // Finally, we do all of the relocations. for (new_crate_ref, elf_file) in partially_loaded_crates { self.perform_relocations(&elf_file, &new_crate_ref, temp_backup_namespace, kernel_mmi_ref, verbose_log)?; @@ -932,11 +930,11 @@ impl CrateNamespace { /// Duplicates this `CrateNamespace` into a new `CrateNamespace`, /// but uses a copy-on-write/clone-on-write semantic that creates /// a special shared reference to each crate that indicates it is shared across multiple namespaces. - /// + /// /// In other words, crates in the new namespace returned by this fucntions /// are fully shared with crates in *this* namespace, /// until either namespace attempts to modify a shared crate in the future. - /// + /// /// When modifying crates in the new namespace, e.g., swapping crates, /// any crates in the new namespace that are still shared with the old namespace /// must be deeply copied into a new crate that is exclusively owned, @@ -949,7 +947,7 @@ impl CrateNamespace { /// that now depend on `A2` instead of `A`. /// The existing versions of `B` and `C` would still depend on `A`, /// but they would no longer be part of the new namespace. - /// + /// pub fn clone_on_write(&self) -> CrateNamespace { CrateNamespace { name: self.name.clone(), @@ -972,7 +970,6 @@ impl CrateNamespace { new_section: &StrongSectionRef, kernel_mmi_ref: &MmiRef ) -> Result<(), &'static str> { - for weak_dep in &old_section.inner.read().sections_dependent_on_me { let target_sec = weak_dep.section.upgrade().ok_or("couldn't upgrade WeakDependent.section")?; let relocation_entry = weak_dep.relocation; @@ -1001,7 +998,7 @@ impl CrateNamespace { target_sec_mapped_pages.remap(&mut kernel_mmi_ref.lock().page_table, target_sec_initial_flags)?; }; } - + // Tell the new source_sec that the existing target_sec depends on it. // Note that we don't need to do this if we're re-swapping in a cached crate, // because that crate's sections' dependents are already properly set up from when it was first swapped in. @@ -1036,13 +1033,13 @@ impl CrateNamespace { /// The primary internal routine for parsing and loading all sections in a crate object file. /// This does not perform any relocations or linking, so the crate **is not yet ready to use after this function**, /// since its sections are totally incomplete and non-executable. - /// + /// /// However, it does add all of the newly-loaded crate sections to the symbol map (yes, even before relocation/linking), /// since we can use them to resolve missing symbols for relocations. - /// + /// /// Parses each section in the given `crate_file` object file and copies its contents to each section. /// Returns a tuple of a reference to the new `LoadedCrate` and the crate's ELF file (to avoid having to re-parse it). - /// + /// /// # Arguments /// * `crate_file`: the object file for the crate that will be loaded into this `CrateNamespace`. /// * `kernel_mmi_ref`: the kernel's MMI struct, for memory mapping use. @@ -1053,7 +1050,6 @@ impl CrateNamespace { kernel_mmi_ref: &MmiRef, _verbose_log: bool ) -> Result<(StrongCrateRef, ElfFile<'f>), &'static str> { - let mapped_pages = crate_file.as_mapping()?; let size_in_bytes = crate_file.len(); let abs_path = Path::new(crate_file.get_absolute_path()); @@ -1070,7 +1066,7 @@ impl CrateNamespace { // It's probably better to pass in the actual crate file reference so we can use it here, // but since we don't currently do that, we just get another reference to the crate object file via its Path. let crate_object_file = match Path::get_absolute(&abs_path) { - Some(FileOrDir::File(f)) => f, + Some(FileOrDir::File(f)) => f, _ => return Err("BUG: load_crate_sections(): couldn't get crate object file path"), }; @@ -1104,7 +1100,7 @@ impl CrateNamespace { let new_crate = CowArc::new(LoadedCrate { crate_name, debug_symbols_file: Arc::downgrade(&crate_object_file), - object_file: crate_object_file, + object_file: crate_object_file, sections: HashMap::new(), text_pages: text_pages.clone(), rodata_pages: rodata_pages.clone(), @@ -1116,7 +1112,7 @@ impl CrateNamespace { }); let new_crate_weak_ref = CowArc::downgrade(&new_crate); - let load_sections_fn = if sections_are_merged { + let load_sections_fn = if sections_are_merged { Self::load_crate_with_merged_sections } else { Self::load_crate_with_separate_sections @@ -1152,10 +1148,10 @@ impl CrateNamespace { /// An internal routine to load and populate the sections of a crate's object file /// if those sections have already been merged. - /// + /// /// This is the "new, acclerated" way to load sections, and is used by `load_crate_sections()` /// for object files that **have** been modified by Theseus's special partial relinking script. - /// + /// /// This works by iterating over all symbols in the object file /// and creating section entries for each one of those symbols only. /// The actual section data can be loaded quickly because they have been merged into top-level sections, @@ -1168,7 +1164,7 @@ impl CrateNamespace { rodata_pages: Option<(Arc>, Range)>, data_pages: Option<(Arc>, Range)>, ) -> Result { - + let mut text_pages_locked = text_pages .as_ref().map(|(tp, tp_range)| (tp.clone(), tp.lock(), tp_range.start)); let mut read_only_pages_locked = rodata_pages.as_ref().map(|(rp, rp_range)| (rp.clone(), rp.lock(), rp_range.start)); let mut read_write_pages_locked = data_pages .as_ref().map(|(dp, dp_range)| (dp.clone(), dp.lock(), dp_range.start)); @@ -1178,7 +1174,7 @@ impl CrateNamespace { let mut read_only_offset: Option = None; // The section header offset of the first read-write section, which is .data or .bss let mut read_write_offset: Option = None; - + // We need to track various section `shndx`s to differentiate between // the different types of "OBJECT" symbols and "TLS" symbols. // @@ -1194,7 +1190,7 @@ impl CrateNamespace { let mut tbss_shndx_and_section: Option<(Shndx, StrongSectionRef)> = None; // The set of `LoadedSections` that will be parsed and populated into this `new_crate`. - let mut loaded_sections: HashMap = HashMap::new(); + let mut loaded_sections: HashMap = HashMap::new(); let mut data_sections: BTreeSet = BTreeSet::new(); let mut tls_sections: BTreeSet = BTreeSet::new(); let mut last_shndx = 0; @@ -1207,7 +1203,7 @@ impl CrateNamespace { let sec_flags = sec.flags(); // Skip non-allocated sections, because they don't appear in the loaded object file. if sec_flags & SHF_ALLOC == 0 { - continue; + continue; } // get the relevant section info, i.e., size, alignment, and data contents @@ -1290,7 +1286,7 @@ impl CrateNamespace { .map(|(rp_ref, rp, _)| (rp_ref, rp)) .ok_or("BUG: ELF file contained a .tdata/.tbss section, but no rodata_pages were allocated")?; // Use a placeholder vaddr; it will be replaced in `add_new_dynamic_tls_section()` below. - virt_addr = VirtualAddress::zero(); + virt_addr = VirtualAddress::zero(); tls_sections.insert(shndx); } @@ -1453,9 +1449,9 @@ impl CrateNamespace { typ = SectionType::Rodata; mapped_pages = rp_ref; // no additional offset below, because .rodata is always the first read-only section. - mapped_pages_offset = sec_value; + mapped_pages_offset = sec_value; virt_addr = rp_start_vaddr + mapped_pages_offset; - } + } // Handle .data/.bss symbol else { data_sections.insert(last_shndx); @@ -1538,7 +1534,7 @@ impl CrateNamespace { if is_global { global_sections.insert(last_shndx); } - + last_shndx += 1; } // end of iterating over all symbol table entries @@ -1581,7 +1577,7 @@ impl CrateNamespace { // } // } - Ok(SectionMetadata { + Ok(SectionMetadata { loaded_sections, global_sections, tls_sections, @@ -1592,10 +1588,10 @@ impl CrateNamespace { /// An internal routine to load and populate the sections of a crate's object file /// if those sections have not been merged. - /// + /// /// This is the "legacy" way to load sections, and is used by `load_crate_sections()` /// for object files that have **not** been modified by Theseus's special partial relinking script. - /// + /// /// This works by iterating over all section headers in the object file /// and extracting the symbol names from each section name, which is quite slow. fn load_crate_with_separate_sections( @@ -1605,8 +1601,8 @@ impl CrateNamespace { text_pages: Option<(Arc>, Range)>, rodata_pages: Option<(Arc>, Range)>, data_pages: Option<(Arc>, Range)>, - ) -> Result { - + ) -> Result { + // Check the symbol table to get the set of sections that are global (publicly visible). let global_sections: BTreeSet = { // For us to properly load the ELF file, it must NOT have been fully stripped, @@ -1619,7 +1615,7 @@ impl CrateNamespace { // Include all symbols with "GLOBAL" binding, regardless of visibility. if entry.get_binding() == Ok(xmas_elf::symbol_table::Binding::Global) { match entry.get_type() { - Ok(xmas_elf::symbol_table::Type::Func + Ok(xmas_elf::symbol_table::Type::Func | xmas_elf::symbol_table::Type::Object | xmas_elf::symbol_table::Type::Tls) => { globals.insert(entry.shndx() as Shndx); @@ -1628,7 +1624,7 @@ impl CrateNamespace { } } } - globals + globals }; // Since .text sections come at the beginning of the object file, @@ -1655,7 +1651,7 @@ impl CrateNamespace { // we copy them into their respective pages individually on a per-section basis, // keeping track of the offset into each of their MappedPages as we go. let (mut rodata_offset, mut data_offset) = (0 , 0); - + const TEXT_PREFIX: &str = ".text."; const UNLIKELY_PREFIX: &str = "unlikely."; // the full section prefix is ".text.unlikely." const RODATA_PREFIX: &str = ".rodata."; @@ -1669,12 +1665,12 @@ impl CrateNamespace { /// A convenient macro to obtain the rest of the symbol name after its prefix, /// i.e., the characters after '.text', '.rodata', '.data', etc. - /// + /// /// * If the name isn't long enough, the macro prints and returns an error str. /// * If the name isn't long enough but is an empty section (e.g., just ".text", ".rodata", etc) /// this macro `continue`s to the next iteration of the loop. /// * The `$prefix` argument must be `const` so it can be `concat!()`-ed into a const &str. - /// + /// /// Note: I'd prefer this to be a const function that accepts the prefix as a const &'static str, /// but Rust does not support concat!()-ing const generic parameters yet. macro_rules! try_get_symbol_name_after_prefix { @@ -1702,7 +1698,7 @@ impl CrateNamespace { } // this maps section header index (shndx) to LoadedSection - let mut loaded_sections: HashMap = HashMap::new(); + let mut loaded_sections: HashMap = HashMap::new(); // the set of Shndxes for .data and .bss sections let mut data_sections: BTreeSet = BTreeSet::new(); // the set of Shndxes for TLS sections (.tdata, .tbss) @@ -1718,7 +1714,7 @@ impl CrateNamespace { let sec_flags = sec.flags(); // Skip non-allocated sections, because they don't appear in the loaded object file. if sec_flags & SHF_ALLOC == 0 { - continue; + continue; } // Even if we're using the next section's data (for a zero-sized section, as handled below), @@ -1791,7 +1787,7 @@ impl CrateNamespace { let dest_vaddr = tp_range.start + text_offset; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( SectionType::Text, demangled, @@ -1853,7 +1849,7 @@ impl CrateNamespace { new_crate.clone(), ); // trace!("Loaded new TLS section: {:?}", new_tls_section); - + // Add the new TLS section to this namespace's initial TLS area, // which will reserve/obtain a new offset into that TLS area which holds this section's data. // This will also update the section's virtual address field to hold that offset value, @@ -1891,7 +1887,7 @@ impl CrateNamespace { // }) }; let demangled = demangle(name).to_string().as_str().into(); - + if let Some((ref dp_ref, ref mut dp)) = read_write_pages_locked { // here: we're ready to copy the data/bss section to the proper address let dest_vaddr = dp.address_at_offset(data_offset) @@ -1905,7 +1901,7 @@ impl CrateNamespace { return Err("couldn't get section data in .data section"); } } - + loaded_sections.insert( shndx, Arc::new(LoadedSection::new( @@ -1946,9 +1942,9 @@ impl CrateNamespace { return Err("couldn't get section data in .rodata section"); } } - + loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( SectionType::Rodata, demangled, @@ -1993,7 +1989,7 @@ impl CrateNamespace { let typ = SectionType::GccExceptTable; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( typ, section_name_str_ref(&typ), @@ -2032,7 +2028,7 @@ impl CrateNamespace { let typ = SectionType::EhFrame; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( typ, section_name_str_ref(&typ), @@ -2063,7 +2059,7 @@ impl CrateNamespace { } } - Ok(SectionMetadata { + Ok(SectionMetadata { loaded_sections, global_sections, tls_sections, @@ -2071,7 +2067,7 @@ impl CrateNamespace { }) } - + /// The second stage of parsing and loading a new kernel crate, /// filling in the missing relocation information in the already-loaded sections. /// It also remaps the `new_crate`'s MappedPages according to each of their section permissions. @@ -2092,9 +2088,9 @@ impl CrateNamespace { // Iterate over every non-zero relocation section in the file for sec in elf_file.section_iter().filter(|sec| sec.get_type() == Ok(ShType::Rela) && sec.size() != 0) { use xmas_elf::sections::SectionData::Rela64; - if verbose_log { + if verbose_log { trace!("Found Rela section name: {:?}, type: {:?}, target_sec_index: {:?}", - sec.get_name(elf_file), sec.get_type(), sec.info()); + sec.get_name(elf_file), sec.get_type(), sec.info()); } // Debug sections are handled separately @@ -2109,23 +2105,23 @@ impl CrateNamespace { _ => { error!("Found Rela section that wasn't able to be parsed as Rela64: {:?}", sec); return Err("Found Rela section that wasn't able to be parsed as Rela64"); - } + } }; // The target section is where we write the relocation data to. // The source section is where we get the data from. // There is one target section per rela section (`rela_array`), and one source section per rela_entry in this rela section. // The "info" field in the Rela section specifies which section is the target of the relocation. - + // Get the target section (that we already loaded) for this rela_array Rela section. let target_sec_shndx = sec.info() as usize; let target_sec = new_crate.sections.get(&target_sec_shndx).ok_or_else(|| { error!("ELF file error: target section was not loaded for Rela section {:?}!", sec.get_name(elf_file)); "target section was not loaded for Rela section" - })?; + })?; let mut target_sec_data_was_modified = false; - + let mut target_sec_dependencies: Vec = Vec::new(); #[cfg(internal_deps)] let mut target_sec_internal_dependencies: Vec = Vec::new(); @@ -2138,16 +2134,16 @@ impl CrateNamespace { // iterate through each relocation entry in the relocation array for the target_sec for rela_entry in rela_array { - if verbose_log { + if verbose_log { trace!(" Rela64 offset: {:#X}, addend: {:#X}, symtab_index: {}, type: {:#X}", rela_entry.get_offset(), rela_entry.get_addend(), rela_entry.get_symbol_table_index(), rela_entry.get_type()); } use xmas_elf::symbol_table::Entry; let source_sec_entry = &symtab[rela_entry.get_symbol_table_index() as usize]; - let source_sec_shndx = source_sec_entry.shndx() as usize; + let source_sec_shndx = source_sec_entry.shndx() as usize; let source_sec_value = source_sec_entry.value() as usize; - if verbose_log { + if verbose_log { let source_sec_header_name = source_sec_entry.get_section_header(elf_file, rela_entry.get_symbol_table_index() as usize) .and_then(|s| s.get_name(elf_file)); trace!(" relevant section [{}]: {:?}, value: {:#X}", source_sec_shndx, source_sec_header_name, source_sec_value); @@ -2156,7 +2152,7 @@ impl CrateNamespace { // source_sec_entry.get_other(), source_sec_entry.get_binding(), source_sec_entry.get_type(), // source_sec_entry.shndx(), source_sec_entry.value(), source_sec_entry.size()); } - + let mut source_and_target_in_same_crate = false; // We first try to get the source section from loaded_sections, which works if the section is in the crate currently being loaded. @@ -2219,13 +2215,13 @@ impl CrateNamespace { relocation: relocation_entry, }; source_sec.inner.write().sections_dependent_on_me.push(weak_dep); - + // tell the target_sec that it has a strong dependency on the source_sec let strong_dep = StrongDependency { section: Arc::clone(&source_sec), relocation: relocation_entry, }; - target_sec_dependencies.push(strong_dep); + target_sec_dependencies.push(strong_dep); } } } @@ -2233,7 +2229,7 @@ impl CrateNamespace { // If the target section of the relocation was a TLS section, // that TLS section's initializer data has now changed. // Thus, we need to invalidate the TLS initializer area's cached data. - if target_sec_data_was_modified && + if target_sec_data_was_modified && (target_sec.typ == SectionType::TlsData || target_sec.typ == SectionType::TlsBss) { // debug!("Invalidating TlsInitializer due to relocation written to section {:?}", &*target_sec); @@ -2253,7 +2249,7 @@ impl CrateNamespace { // We need to remap each section's mapped pages with the proper permission bits, // since we initially mapped them all as writable. - if let Some(ref tp) = new_crate.text_pages { + if let Some(ref tp) = new_crate.text_pages { tp.0.lock().remap(&mut kernel_mmi_ref.lock().page_table, TEXT_SECTION_FLAGS)?; } if let Some(ref rp) = new_crate.rodata_pages { @@ -2265,13 +2261,13 @@ impl CrateNamespace { // By default, we can safely remove the metadata for all private (non-global) .rodata sections // that do not have any strong dependencies (its `sections_i_depend_on` list is empty). // If you want all sections to be kept, e.g., for debugging, you can set the below cfg option. - #[cfg(not(keep_private_rodata))] + #[cfg(not(keep_private_rodata))] { new_crate.sections.retain(|_shndx, sec| { - let should_remove = !sec.global + let should_remove = !sec.global && sec.typ == SectionType::Rodata && sec.inner.read().sections_i_depend_on.is_empty(); - + // For an element to be removed, this closure should return `false`. !should_remove }); @@ -2280,7 +2276,7 @@ impl CrateNamespace { Ok(()) } - + /// Adds the given symbol to this namespace's symbol map. /// If the symbol already exists in the symbol map, this replaces the existing symbol with the new one, warning if they differ in size. /// Returns true if the symbol was added, and false if it already existed and thus was merely replaced. @@ -2297,7 +2293,7 @@ impl CrateNamespace { // debug!(" add_symbol(): replacing section: old: {:?}, new: {:?}", old_sec, new_section); if new_section.size != old_sec.size { warn!("Unexpectedly replacing differently-sized section: old: ({}B) {:?}, new: ({}B) {:?}", old_sec.size, old_sec.name, new_section.size, new_section.name); - } + } else { warn!("Replacing new symbol already present: old {:?}, new: {:?}", old_sec.name, new_section.name); } @@ -2307,7 +2303,7 @@ impl CrateNamespace { false } qp_trie::Entry::Vacant(new_entry) => { - if log_replacements { + if log_replacements { debug!(" add_symbol(): Adding brand new symbol: new: {:?}", new_section); } new_entry.insert(Arc::downgrade(new_section)); @@ -2317,12 +2313,12 @@ impl CrateNamespace { } /// Adds only *global* symbols in the given `sections` iterator to this namespace's symbol map, - /// + /// /// If a symbol already exists in the symbol map, this replaces the existing symbol but does not count it as a newly-added one. - /// + /// /// Returns the number of *new* unique symbols added. pub fn add_symbols<'a, I>( - &self, + &self, sections: I, _log_replacements: bool, ) -> usize @@ -2334,12 +2330,12 @@ impl CrateNamespace { /// Adds symbols in the given `sections` iterator to this namespace's symbol map, /// but only sections that are *global* AND for which the given `filter_func` returns true. - /// + /// /// If a symbol already exists in the symbol map, this replaces the existing symbol but does not count it as a newly-added one. - /// + /// /// Returns the number of *new* unique symbols added. fn add_symbols_filtered<'a, I, F>( - &self, + &self, sections: I, filter_func: F, log_replacements: bool, @@ -2361,34 +2357,34 @@ impl CrateNamespace { } } } - + count } - + /// Finds the crate that contains the given `VirtualAddress` in its loaded code. - /// + /// /// By default, only executable sections (`.text`) are searched, since typically the only use case /// for this function is to search for an instruction pointer (program counter) address. /// However, if `search_all_section_types` is `true`, both the read-only and read-write sections /// will be included in the search, e.g., `.rodata`, `.data`, `.bss`. - /// + /// /// # Usage /// This is mostly useful for printing symbol names for a stack trace (backtrace). /// It is also similar in functionality to the tool `addr2line`, /// but gives the section itself rather than the line of code. - /// + /// /// # Locking /// This can obtain the lock on every crate and every section, /// so to avoid deadlock, please ensure that the caller task does not hold any such locks. /// It does *not* need to obtain locks on the underlying `MappedPages` regions. - /// + /// /// # Note /// This is a slow procedure because, in the worst case, /// it will iterate through **every** loaded crate in this namespace (and its recursive namespace). pub fn get_crate_containing_address( - &self, - virt_addr: VirtualAddress, + &self, + virt_addr: VirtualAddress, search_all_section_types: bool, ) -> Option { @@ -2396,7 +2392,7 @@ impl CrateNamespace { let crate_contains_vaddr = |crate_ref: &StrongCrateRef| { let krate = crate_ref.lock_as_ref(); if let Some(ref tp) = krate.text_pages { - if tp.1.contains(&virt_addr) { + if tp.1.contains(&virt_addr) { return true; } } @@ -2414,7 +2410,7 @@ impl CrateNamespace { } false }; - + let mut found_crate = None; // Here, we didn't find the symbol when searching from the starting crate, @@ -2434,29 +2430,29 @@ impl CrateNamespace { /// Finds the section that contains the given `VirtualAddress` in its loaded code. - /// + /// /// By default, only executable sections (`.text`) are searched, since the typical use case /// for this function is to search for an instruction pointer (program counter) address. /// However, if `search_all_section_types` is `true`, both the read-only and read-write sections /// will be included in the search, e.g., `.rodata`, `.data`, `.bss`. - /// + /// /// # Usage /// This is mostly useful for printing symbol names for a stack trace (backtrace). /// It is also similar in functionality to the tool `addr2line`, /// but gives the section itself rather than the line of code. - /// + /// /// # Locking /// This can obtain the lock on every crate in this namespace and its recursive namespaces, /// so to avoid deadlock, please ensure that the caller task does not hold any such locks. - /// + /// /// # Note /// This is a slow procedure because, in the worst case, /// it will iterate through **every** section in **every** loaded crate /// in this namespace (and its recursive namespace), /// not just the publicly-visible (global) sections. pub fn get_section_containing_address( - &self, - virt_addr: VirtualAddress, + &self, + virt_addr: VirtualAddress, search_all_section_types: bool, ) -> Option<(StrongSectionRef, usize)> { @@ -2475,7 +2471,7 @@ impl CrateNamespace { for sec in crate_locked.sections.values() { // .text sections are always included, other sections are included if requested. let eligible_section = sec.typ == SectionType::Text || search_all_section_types; - + // If the section's address bounds contain the address, then we've found it. // Only a single section can contain the address, so it's safe to stop once we've found a match. if eligible_section @@ -2484,7 +2480,7 @@ impl CrateNamespace { { let offset = virt_addr.value() - sec.virt_addr.value(); merged_section_and_offset = Some((sec.clone(), offset)); - + if sec.name.as_str() == sec.typ.name() { // If this section is a merged section, it will have the standard name // for its section type, e.g., ".text", ".data", ".rodata", etc. @@ -2527,23 +2523,23 @@ impl CrateNamespace { /// similar to the simpler function `get_symbol()`, but takes the additional step of trying to /// automatically find and/or load the crate containing that symbol /// (and does so recursively for any of its crate dependencies). - /// + /// /// (1) First, it recursively searches this namespace's and its recursive namespaces' symbol maps, /// and returns the symbol if already loaded. - /// + /// /// (2) Second, if the symbol is missing from this namespace, it looks in the `temp_backup_namespace`. /// If we find it there, then we add that symbol and its containing crate as a shared crate in this namespace. - /// + /// /// (3) Third, if this namespace has `fuzzy_symbol_matching` enabled, it searches the backup namespace /// for symbols that match the given `demangled_full_symbol` without the hash suffix. - /// + /// /// (4) Fourth, if the missing symbol isn't in the backup namespace either, /// try to load its containing crate from the object file. /// This can only be done for symbols that have a leading crate name, such as "my_crate::foo"; /// if a symbol was given the `no_mangle` attribute, then we will not be able to find it, /// and that symbol's containing crate should be manually loaded before invoking this. - /// - /// + /// + /// /// # Arguments /// * `demangled_full_symbol`: a fully-qualified symbol string, e.g., "my_crate::MyStruct::foo::h843a9ea794da0c24". /// * `temp_backup_namespace`: the `CrateNamespace` that should be temporarily searched (just during this call) @@ -2551,9 +2547,9 @@ impl CrateNamespace { /// If `temp_backup_namespace` is `None`, then only this namespace (and its recursive namespaces) will be searched. /// * `kernel_mmi_ref`: a reference to the kernel's `MemoryManagementInfo`, which must not be locked. pub fn get_symbol_or_load( - &self, - demangled_full_symbol: &str, - temp_backup_namespace: Option<&CrateNamespace>, + &self, + demangled_full_symbol: &str, + temp_backup_namespace: Option<&CrateNamespace>, kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> WeakSectionRef { @@ -2597,7 +2593,7 @@ impl CrateNamespace { /// Looks for the given `demangled_full_symbol` in the `temp_backup_namespace` and returns a reference to the matching section. - /// + /// /// This is the second and third attempts to find a symbol within [`get_symbol_or_load()`](#method.get_symbol_or_load). fn get_symbol_from_backup_namespace( &self, @@ -2636,7 +2632,7 @@ impl CrateNamespace { })?; // Here, we found the matching section in the temp_backup_namespace. - let parent_crate_ref = { + let parent_crate_ref = { sec.parent_crate.upgrade().or_else(|| { error!("BUG: Found symbol \"{}\" in backup namespace, but unexpectedly couldn't get its parent crate!", demangled_full_symbol); None @@ -2652,7 +2648,7 @@ impl CrateNamespace { self.add_symbols(Some(sec.clone()).iter(), verbose_log); parent_crate.crate_name.clone() }; - + #[cfg(not(loscd_eval))] info!("Symbol {:?} not initially found, using {}symbol {} from crate {:?} in backup namespace {:?} in new namespace {:?}", demangled_full_symbol, @@ -2671,24 +2667,24 @@ impl CrateNamespace { /// Attempts to find and load the crate that may contain the given `demangled_full_symbol`. - /// + /// /// If successful, the new crate is loaded into this `CrateNamespace` and the symbol's section is returned. /// If this namespace does not contain any matching crates, its recursive namespaces are searched as well. - /// + /// /// This approach only works for mangled symbols that contain a crate name, such as "my_crate::foo". /// If "foo()" was marked no_mangle, then we don't know which crate to load because there is no "my_crate::" prefix before it. - /// + /// /// Note: while attempting to find the missing `demangled_full_symbol`, this function may end up /// loading *multiple* crates into this `CrateNamespace` or its recursive namespaces, due to two reasons: /// 1. The `demangled_full_symbol` may have multiple crate prefixes within it. /// * For example, `::drop::h55e0a4c312ccdd63` /// contains two possible crate prefixes: `page_allocator` and `core`. /// 2. There may be multiple versions of a single crate. - /// + /// /// Possible crates are iteratively loaded and searched until the missing symbol is found. /// Currently, crates that were loaded but did *not* contain the missing symbol are *not* unloaded, /// but you could manually unload them later with no adverse effects to reclaim memory. - /// + /// /// This is the final attempt to find a symbol within [`CrateNamespace::get_symbol_or_load()`]. fn load_crate_for_missing_symbol( &self, @@ -2700,7 +2696,7 @@ impl CrateNamespace { // Some symbols may have multiple potential containing crates, so we try to load each one to find the missing symbol. for potential_crate_name in get_containing_crate_name(demangled_full_symbol) { let potential_crate_name = format!("{potential_crate_name}-"); - + // Try to find and load the missing crate object file from this namespace's directory or its recursive namespace's directory, // (or from the backup namespace's directory set). // The object files from the recursive namespace(s) are appended after the files in the initial namespace, @@ -2733,7 +2729,7 @@ impl CrateNamespace { // We *could* return an error here, but we might as well continue on to trying to load other crates. } } - } + } } warn!("Couldn't find/load crate(s) that may contain the missing symbol {:?}", demangled_full_symbol); @@ -2744,16 +2740,16 @@ impl CrateNamespace { /// Returns a copied list of the corresponding `LoadedSection`s /// with names that start with the given `symbol_prefix`. /// This will also search the recursive namespace's symbol map. - /// + /// /// This method causes allocation because it creates a copy /// of the matching entries in the symbol map. - /// + /// /// # Example /// The symbol map contains `my_crate::foo::h843a613894da0c24` and /// `my_crate::foo::h933a635894ce0f12`. /// Calling `find_symbols_starting_with("my_crate::foo")` will return /// a vector containing both sections, which can then be iterated through. - pub fn find_symbols_starting_with(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef)> { + pub fn find_symbols_starting_with(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef)> { let mut syms: Vec<(String, WeakSectionRef)> = self.symbol_map.lock() .iter_prefix(symbol_prefix.as_bytes()) .map(|(k, v)| (String::from(k.as_str()), v.clone())) @@ -2769,7 +2765,7 @@ impl CrateNamespace { /// Similar to `find_symbols_starting_with`, but also includes a reference to the exact `CrateNamespace` /// where the matching symbol was found. - pub fn find_symbols_starting_with_and_namespace(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef, &CrateNamespace)> { + pub fn find_symbols_starting_with_and_namespace(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef, &CrateNamespace)> { let mut syms: Vec<(String, WeakSectionRef, &CrateNamespace)> = self.symbol_map.lock() .iter_prefix(symbol_prefix.as_bytes()) .map(|(k, v)| (String::from(k.as_str()), v.clone(), self)) @@ -2786,12 +2782,12 @@ impl CrateNamespace { /// Returns a weak reference to the `LoadedSection` whose name beings with the given `symbol_prefix`, /// *if and only if* the symbol map only contains a single possible matching symbol. /// This will also search the recursive namespace's symbol map. - /// + /// /// # Important Usage Note /// To avoid greedily matching more symbols than expected, you may wish to end the `symbol_prefix` with "`::`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`::`". /// This works because the delimiter between a symbol and its trailing hash value is "`::`". - /// + /// /// # Example /// * The symbol map contains `my_crate::foo::h843a613894da0c24` /// and no other symbols that start with `my_crate::foo`. @@ -2808,21 +2804,21 @@ impl CrateNamespace { /// because it will match both `foo` and `foo_new`. /// To match only `foo`, call this function as `get_symbol_starting_with("my_crate::foo::")` /// (note the trailing "`::`"). - pub fn get_symbol_starting_with(&self, symbol_prefix: &str) -> WeakSectionRef { + pub fn get_symbol_starting_with(&self, symbol_prefix: &str) -> WeakSectionRef { self.get_symbol_starting_with_internal(symbol_prefix) .unwrap_or_default() } /// This is an internal version of method: [`get_symbol_starting_with()`](#method.get_symbol_starting_with) /// that returns an Option to allow easier recursive use. - fn get_symbol_starting_with_internal(&self, symbol_prefix: &str) -> Option { + fn get_symbol_starting_with_internal(&self, symbol_prefix: &str) -> Option { // First, we see if there's a single matching symbol in this namespace. let map = self.symbol_map.lock(); let mut iter = map.iter_prefix(symbol_prefix.as_bytes()).map(|tuple| tuple.1); let symbol_in_this_namespace = iter.next() .filter(|_| iter.next().is_none()) // ensure single element .cloned(); - + // Second, we see if there's a single matching symbol in the recursive namespace. let symbol_in_recursive_namespace = self.recursive_namespace.as_ref().and_then(|r_ns| r_ns.get_symbol_starting_with_internal(symbol_prefix)); @@ -2830,7 +2826,7 @@ impl CrateNamespace { symbol_in_this_namespace.xor(symbol_in_recursive_namespace) } - + /// Simple debugging function that returns the entire symbol map as a String. /// This includes only symbols from this namespace, and excludes symbols from recursive namespaces. pub fn dump_symbol_map(&self) -> String { @@ -2990,7 +2986,7 @@ fn allocate_section_pages(elf_file: &ElfFile, kernel_mmi_ref: &MmiRef) -> Result allocate_pages_by_bytes(size_in_bytes) .ok_or("Couldn't allocate pages for new section")? }; - + kernel_mmi_ref.lock().page_table.map_allocated_pages( allocated_pages, flags.valid(true).writable(true) @@ -3059,7 +3055,7 @@ fn dump_weak_dependents(sec: &LoadedSection, prefix: String) { /// Returns a reference to the symbol table in the given `ElfFile`. -pub fn find_symbol_table<'e>(elf_file: &'e ElfFile) +pub fn find_symbol_table<'e>(elf_file: &'e ElfFile) -> Result<&'e [xmas_elf::symbol_table::Entry64], &'static str> { use xmas_elf::sections::SectionData::SymbolTable64;