diff --git a/Cargo.lock b/Cargo.lock index 56f007e742..d79c3f88ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1385,7 +1385,8 @@ dependencies = [ "cpu", "log", "memory", - "static_assertions", + "spin 0.9.4", + "volatile 0.2.7", "zerocopy", ] @@ -1581,6 +1582,7 @@ dependencies = [ "ioapic", "log", "memory", + "spin 0.9.4", "sync_irq", ] @@ -2567,9 +2569,9 @@ checksum = "be5e13c266502aadf83426d87d81a0f5d1ef45b8027f5a471c360abfe4bfae92" [[package]] name = "paste" -version = "1.0.5" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "path" @@ -2587,6 +2589,7 @@ dependencies = [ name = "pci" version = "0.1.0" dependencies = [ + "arm_boards", "bit_field 0.7.0", "cpu", "interrupts", @@ -3762,6 +3765,15 @@ name = "test_1gb_huge_pages" version = "0.1.0" dependencies = [ "app_io", + "memory", +] + +[[package]] +name = "test_aligned_page_allocation" +version = "0.1.0" +dependencies = [ + "app_io", + "log", "memory", ] @@ -4065,6 +4077,7 @@ dependencies = [ "seconds_counter", "shell", "swap", + "test_aligned_page_allocation", "test_async", "test_backtrace", "test_block_io", diff --git a/Cargo.toml b/Cargo.toml index 721c060cb5..56bdae404f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ exclude = [ ## Exclude application crates used for testing specific Theseus functionality. ## TODO: move these to a specific "tests" folder so we can exclude that entire folder. + "applications/test_aligned_page_allocation", "applications/test_backtrace", "applications/test_block_io", "applications/test_channel", diff --git a/Makefile b/Makefile index 2b1d192b03..2ff1f0ce9c 100644 --- a/Makefile +++ b/Makefile @@ -933,6 +933,9 @@ else ifeq ($(ARCH),aarch64) QEMU_FLAGS += -machine virt,gic-version=3 QEMU_FLAGS += -device ramfb QEMU_FLAGS += -cpu cortex-a72 + QEMU_FLAGS += -usb + QEMU_FLAGS += -device usb-ehci,id=ehci + QEMU_FLAGS += -device usb-kbd else QEMU_FLAGS += -cpu Broadwell endif diff --git a/applications/test_aligned_page_allocation/Cargo.toml b/applications/test_aligned_page_allocation/Cargo.toml new file mode 100644 index 0000000000..a81b8f9f54 --- /dev/null +++ b/applications/test_aligned_page_allocation/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "test_aligned_page_allocation" +version = "0.1.0" +description = "Tests the `AllocationRequest::AlignedTo` variant, which is needed for huge pages" +authors = ["Kevin Boos "] +edition = "2021" + +[dependencies] +log = "0.4.8" + +[dependencies.memory] +path = "../../kernel/memory" + +[dependencies.app_io] +path = "../../kernel/app_io" diff --git a/applications/test_aligned_page_allocation/src/lib.rs b/applications/test_aligned_page_allocation/src/lib.rs new file mode 100644 index 0000000000..ebe99e7c5f --- /dev/null +++ b/applications/test_aligned_page_allocation/src/lib.rs @@ -0,0 +1,45 @@ +//! A set of basic tests for the [`AllocationRequest::AlignedTo`] variant. + +#![no_std] + +extern crate alloc; + +use alloc::{ + vec::Vec, + string::String, +}; +use app_io::println; +use memory::AllocationRequest; + +static TEST_SET: [usize; 9] = [1, 2, 4, 8, 27, 48, 256, 512, 1024]; + +pub fn main(_args: Vec) -> isize { + match rmain() { + Ok(_) => 0, + Err(e) => { + println!("Error: {}", e); + -1 + } + } +} + +fn rmain() -> Result<(), &'static str> { + for num_pages in TEST_SET.into_iter() { + for alignment in TEST_SET.into_iter() { + println!("Attempting to allocate {num_pages} pages with alignment of {alignment} 4K pages..."); + match memory::allocate_pages_deferred( + AllocationRequest::AlignedTo { alignment_4k_pages: alignment }, + num_pages, + ) { + Ok((ap, _action)) => { + assert_eq!(ap.start().number() % alignment, 0); + assert_eq!(ap.size_in_pages(), num_pages); + println!(" Success: {ap:?}"); + } + Err(e) => println!(" !! FAILURE: {e:?}"), + } + } + } + + Ok(()) +} diff --git a/kernel/arm_boards/src/boards/qemu_virt.rs b/kernel/arm_boards/src/boards/qemu_virt.rs index 81de9c88d8..c570c6d8f7 100644 --- a/kernel/arm_boards/src/boards/qemu_virt.rs +++ b/kernel/arm_boards/src/boards/qemu_virt.rs @@ -2,7 +2,7 @@ use super::{ InterruptControllerConfig::GicV3, GicV3InterruptControllerConfig, - BoardConfig, mpidr::DefinedMpidrValue, + BoardConfig, mpidr::DefinedMpidrValue, PciEcamConfig, }; use memory_structs::PhysicalAddress; @@ -38,4 +38,11 @@ pub const BOARD_CONFIG: BoardConfig = BoardConfig { pl011_base_addresses: [ PhysicalAddress::new_canonical(0x09000000) ], pl011_rx_spi: 33, cpu_local_timer_ppi: 30, + + // obtained via internal qemu debugging + // todo: will this always be correct? + pci_ecam: PciEcamConfig { + base_address: PhysicalAddress::new_canonical(0x4010000000), + size_bytes: 0x10000000, + } }; diff --git a/kernel/arm_boards/src/lib.rs b/kernel/arm_boards/src/lib.rs index cd3a1da0d6..04afb18f36 100644 --- a/kernel/arm_boards/src/lib.rs +++ b/kernel/arm_boards/src/lib.rs @@ -19,6 +19,14 @@ pub struct GicV3InterruptControllerConfig { pub redistributor_base_addresses: [PhysicalAddress; NUM_CPUS], } +#[derive(Debug, Copy, Clone)] +pub struct PciEcamConfig { + pub base_address: PhysicalAddress, + pub size_bytes: usize, +} + +/// This excludes GICv2 because there's no way to send IPIs +/// with GICv2 via the current driver API. #[derive(Debug, Copy, Clone)] pub enum InterruptControllerConfig { GicV3(GicV3InterruptControllerConfig), @@ -46,6 +54,8 @@ pub struct BoardConfig { // // aarch64 manuals define the default timer IRQ number to be 30. pub cpu_local_timer_ppi: u8, + + pub pci_ecam: PciEcamConfig, } // by default & on x86_64, the default.rs file is used diff --git a/kernel/device_manager/Cargo.toml b/kernel/device_manager/Cargo.toml index e0ae226d1e..5f51ab2988 100644 --- a/kernel/device_manager/Cargo.toml +++ b/kernel/device_manager/Cargo.toml @@ -12,6 +12,7 @@ event_types = { path = "../event_types" } serial_port = { path = "../serial_port" } console = { path = "../console" } logger = { path = "../logger" } +pci = { path = "../pci" } derive_more = "0.99.0" mpmc = "0.1.6" log = "0.4.8" @@ -20,7 +21,6 @@ log = "0.4.8" memory = { path = "../memory" } e1000 = { path = "../e1000" } acpi = { path = "../acpi" } -pci = { path = "../pci" } ps2 = { path = "../ps2" } keyboard = { path = "../keyboard" } mouse = { path = "../mouse" } diff --git a/kernel/device_manager/src/lib.rs b/kernel/device_manager/src/lib.rs index 3647c938cf..b0a806a80f 100644 --- a/kernel/device_manager/src/lib.rs +++ b/kernel/device_manager/src/lib.rs @@ -3,11 +3,11 @@ extern crate alloc; -use log::info; +use log::{info, debug}; #[cfg(target_arch = "x86_64")] use { - log::{error, debug, warn}, + log::{error, warn}, mpmc::Queue, event_types::Event, memory::MemoryManagementInfo, @@ -86,19 +86,20 @@ pub fn init( mouse::init(ps2_controller.mouse_ref(), mouse_producer)?; } - // No PCI support on aarch64 at the moment - #[cfg(target_arch = "x86_64")] { // Initialize/scan the PCI bus to discover PCI devices - for dev in pci::pci_device_iter() { - debug!("Found pci device: {:X?}", dev); + for dev in pci::pci_device_iter()? { + debug!("Found PCI device: {:X?}", dev); } + // No NIC support on aarch64 at the moment + #[cfg(target_arch = "x86_64")] { + // store all the initialized ixgbe NICs here to be added to the network interface list let mut ixgbe_devs = Vec::new(); // Iterate over all PCI devices and initialize the drivers for the devices we support. - for dev in pci::pci_device_iter() { + for dev in pci::pci_device_iter()? { // Currently we skip Bridge devices, since we have no use for them yet. if dev.class == 0x06 { continue; diff --git a/kernel/gic/Cargo.toml b/kernel/gic/Cargo.toml index 6b37e010c5..3f34a945eb 100644 --- a/kernel/gic/Cargo.toml +++ b/kernel/gic/Cargo.toml @@ -6,8 +6,9 @@ edition = "2021" name = "gic" [dependencies] -static_assertions = "1.1.0" zerocopy = "0.5.0" +volatile = "0.2.7" +spin = "0.9.4" log = "0.4.8" memory = { path = "../memory" } diff --git a/kernel/gic/src/gic/cpu_interface_gicv2.rs b/kernel/gic/src/gic/cpu_interface_gicv2.rs index 3a301fede5..58d8c81bca 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv2.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv2.rs @@ -6,17 +6,39 @@ //! - Acknowledging interrupt requests //! - Sending End-Of-Interrupts signals -use super::GicRegisters; use super::Priority; use super::InterruptNumber; -mod offset { - use crate::Offset32; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); - pub(crate) const PMR: Offset32 = Offset32::from_byte_offset(0x04); - pub(crate) const IAR: Offset32 = Offset32::from_byte_offset(0x0C); - pub(crate) const RPR: Offset32 = Offset32::from_byte_offset(0x14); - pub(crate) const EOIR: Offset32 = Offset32::from_byte_offset(0x10); +use volatile::{Volatile, ReadOnly, WriteOnly}; +use zerocopy::FromBytes; + +/// The GICv2 MMIO registers for interfacing with a specific CPU. +/// +/// Methods herein apply to the "current" CPU only, i.e., the CPU +/// on which the code that accesses these registers is currently running. +/// +/// Note: the physical address for this structure is the same for all CPUs, +/// but the actual backing memory refers to physically separate registers. +#[derive(FromBytes)] +#[repr(C)] +pub struct CpuRegsP1 { // base offset + /// CPU Interface Control Register + ctlr: Volatile, // 0x00 + + /// Interrupt Priority Mask Register + prio_mask: Volatile, // 0x04 + + /// Binary Point Register + _unused0: u32, + + /// Interrupt Acknowledge Register + acknowledge: ReadOnly, // 0x0C + + /// End of Interrupt Register + eoi: WriteOnly, // 0x10 + + /// Running Priority Register + running_prio: ReadOnly, // 0x14 } // enable group 0 @@ -25,45 +47,52 @@ mod offset { // enable group 1 const CTLR_ENGRP1: u32 = 0b10; -/// Enables routing of group 1 interrupts for the current CPU -pub fn init(registers: &mut GicRegisters) { - let mut reg = registers.read_volatile(offset::CTLR); - reg |= CTLR_ENGRP1; - registers.write_volatile(offset::CTLR, reg); -} +impl CpuRegsP1 { + /// Enables routing of group 1 interrupts for the current CPU. + pub fn init(&mut self) { + let mut reg = self.ctlr.read(); + reg |= CTLR_ENGRP1; + self.ctlr.write(reg); + } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them -pub fn get_minimum_priority(registers: &GicRegisters) -> Priority { - u8::MAX - (registers.read_volatile(offset::PMR) as u8) -} + /// Retrieves the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. + pub fn get_minimum_priority(&self) -> Priority { + u8::MAX - (self.prio_mask.read() as u8) + } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them -pub fn set_minimum_priority(registers: &mut GicRegisters, priority: Priority) { - registers.write_volatile(offset::PMR, (u8::MAX - priority) as u32); -} + /// Sets the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. + pub fn set_minimum_priority(&mut self, priority: Priority) { + self.prio_mask.write((u8::MAX - priority) as u32); + } -/// Signals to the controller that the currently processed interrupt has -/// been fully handled, by zeroing the current priority level of this CPU. -/// This implies that the CPU is ready to process interrupts again. -pub fn end_of_interrupt(registers: &mut GicRegisters, int: InterruptNumber) { - registers.write_volatile(offset::EOIR, int); -} + /// Signals to the controller that the currently processed interrupt + /// has been fully handled, by zeroing the current priority level of + /// the current CPU. + /// + /// This implies that the CPU is ready to process interrupts again. + pub fn end_of_interrupt(&mut self, int: InterruptNumber) { + self.eoi.write(int); + } + + /// Acknowledge the currently serviced interrupt and fetches its + /// number. + /// + /// This tells the GIC that the requested interrupt is being + /// handled by this CPU. + pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) { + // Reading the interrupt number has the side effect + // of acknowledging the interrupt. + let int_num = self.acknowledge.read() as InterruptNumber; + let priority = self.running_prio.read() as u8; -/// Acknowledge the currently serviced interrupt -/// and fetches its number; this tells the GIC that -/// the requested interrupt is being handled by -/// this CPU. -pub fn acknowledge_interrupt(registers: &mut GicRegisters) -> (InterruptNumber, Priority) { - // Reading the interrupt number has the side effect - // of acknowledging the interrupt. - let int_num = registers.read_volatile(offset::IAR) as InterruptNumber; - let priority = registers.read_volatile(offset::RPR) as u8; - - (int_num, priority) + (int_num, priority) + } } diff --git a/kernel/gic/src/gic/cpu_interface_gicv3.rs b/kernel/gic/src/gic/cpu_interface_gicv3.rs index b5ee7d399b..63072864b2 100644 --- a/kernel/gic/src/gic/cpu_interface_gicv3.rs +++ b/kernel/gic/src/gic/cpu_interface_gicv3.rs @@ -35,37 +35,42 @@ pub fn init() { unsafe { asm!("msr ICC_IGRPEN1_EL1, {}", in(reg) IGRPEN_ENABLED) }; } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them +/// Retrieves the current priority threshold for the current CPU. +/// +/// Interrupts have a priority; if their priority is lower or +/// equal to this threshold, they're queued until the current CPU +/// is ready to handle them. pub fn get_minimum_priority() -> Priority { let mut reg_value: u64; unsafe { asm!("mrs {}, ICC_PMR_EL1", out(reg) reg_value) }; u8::MAX - (reg_value as u8) } -/// Interrupts have a priority; if their priority -/// is lower or equal to this one, they're queued -/// until this CPU or another one is ready to handle -/// them +/// Sets the current priority threshold for the current CPU. +/// +/// Interrupts have a priority; if their priority is lower or +/// equal to this threshold, they're queued until the current CPU +/// is ready to handle them. pub fn set_minimum_priority(priority: Priority) { let reg_value = (u8::MAX - priority) as u64; unsafe { asm!("msr ICC_PMR_EL1, {}", in(reg) reg_value) }; } -/// Signals to the controller that the currently processed interrupt has -/// been fully handled, by zeroing the current priority level of this CPU. +/// Signals to the controller that the currently processed interrupt +/// has been fully handled, by zeroing the current priority level of +/// the current CPU. +/// /// This implies that the CPU is ready to process interrupts again. pub fn end_of_interrupt(int: InterruptNumber) { let reg_value = int as u64; unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) }; } -/// Acknowledge the currently serviced interrupt -/// and fetches its number; this tells the GIC that -/// the requested interrupt is being handled by -/// this CPU. +/// Acknowledge the currently serviced interrupt and fetches its +/// number. +/// +/// This tells the GIC that the requested interrupt is being +/// handled by this CPU. pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { let int_num: u64; let priority: u64; @@ -82,6 +87,7 @@ pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) { (int_num as InterruptNumber, priority as u8) } +/// Generates an interrupt in CPU interfaces of the system pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) { let mut value = match target { IpiTargetCpu::Specific(cpu) => { diff --git a/kernel/gic/src/gic/dist_interface.rs b/kernel/gic/src/gic/dist_interface.rs index 398568aa55..baa7fb7488 100644 --- a/kernel/gic/src/gic/dist_interface.rs +++ b/kernel/gic/src/gic/dist_interface.rs @@ -13,29 +13,65 @@ //! - Getting or setting the target of SPIs based on their numbers //! - Generating software interrupts (GICv2 style) -use super::GicRegisters; use super::IpiTargetCpu; use super::SpiDestination; use super::InterruptNumber; use super::Enabled; use super::Priority; use super::TargetList; -use super::ArmGicV3; +use super::read_array_volatile; +use super::write_array_volatile; +use volatile::{Volatile, ReadOnly}; +use zerocopy::FromBytes; use cpu::MpidrValue; -mod offset { - use crate::{Offset32, Offset64}; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x000); - pub(crate) const IIDR: Offset32 = Offset32::from_byte_offset(0x008); - pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x080); - pub(crate) const ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); - pub(crate) const ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); - pub(crate) const IPRIORITYR: Offset32 = Offset32::from_byte_offset(0x400); - pub(crate) const ITARGETSR: Offset32 = Offset32::from_byte_offset(0x800); - pub(crate) const SGIR: Offset32 = Offset32::from_byte_offset(0xf00); - /// This one is on the 6th page - pub(crate) const P6IROUTER: Offset64 = Offset64::from_byte_offset(0x100); +/// First page of distributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct DistRegsP1 { // base offset + /// Distributor Control Register + ctlr: Volatile, // 0x000 + + /// Interrupt Controller Type Register + typer: ReadOnly, // 0x004 + + /// Distributor Implementer Identification Register + ident: ReadOnly, // 0x008 + + _unused0: [u8; 0x074], + + /// Interrupt Group Registers + group: [Volatile; 0x020], // 0x080 + + /// Interrupt Set-Enable Registers + set_enable: [Volatile; 0x020], // 0x100 + + /// Interrupt Clear-Enable Registers + clear_enable: [Volatile; 0x020], // 0x180 + + _unused1: [u8; 0x200], + + /// Interrupt Priority Registers + priority: [Volatile; 0x100], // 0x400 + + /// Interrupt Processor Targets Registers + target: [Volatile; 0x100], // 0x800 + + _unused2: [u8; 0x300], + + /// Software Generated Interrupt Register + sgir: Volatile, // 0xf00 +} + +/// Sixth page of distributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct DistRegsP6 { // base offset + _unused: [u8; 0x100], + + /// Interrupt Routing Registers + route: Volatile, // 0x100 } // enable group 0 @@ -66,75 +102,78 @@ const GROUP_1: u32 = 1; // bit 15: which interrupt group to target const SGIR_NSATT_GRP1: u32 = 1 << 15; -/// Initializes the distributor by enabling forwarding -/// of group 1 interrupts and allowing the GIC to pick -/// a core that is asleep for "1 of N" interrupts. -/// -/// Return value: whether or not affinity routing is -/// currently enabled for both secure and non-secure -/// states. -pub fn init(registers: &mut GicRegisters) -> Enabled { - let mut reg = registers.read_volatile(offset::CTLR); - reg |= CTLR_ENGRP1; - reg |= CTLR_E1NWF; - registers.write_volatile(offset::CTLR, reg); - - // Return value: whether or not affinity routing is - // currently enabled for both secure and non-secure - // states. - reg & CTLR_ARE_NS > 0 -} - -/// Returns whether the given SPI (shared peripheral interrupt) will be -/// forwarded by the distributor -pub fn is_spi_enabled(registers: &GicRegisters, int: InterruptNumber) -> Enabled { - // enabled? - registers.read_array_volatile::<32>(offset::ISENABLER, int) > 0 - && - // part of group 1? - registers.read_array_volatile::<32>(offset::IGROUPR, int) == GROUP_1 -} +impl DistRegsP1 { + /// Initializes the distributor by enabling forwarding + /// of group 1 interrupts and allowing the GIC to pick + /// a core that is asleep for "1 of N" interrupts. + /// + /// Return value: whether or not affinity routing is + /// currently enabled for both secure and non-secure + /// states. + pub fn init(&mut self) -> Enabled { + let mut reg = self.ctlr.read(); + reg |= CTLR_ENGRP1; + reg |= CTLR_E1NWF; + self.ctlr.write(reg); + + // Return value: whether or not affinity routing is + // currently enabled for both secure and non-secure + // states. + reg & CTLR_ARE_NS > 0 + } -/// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt) -pub fn enable_spi(registers: &mut GicRegisters, int: InterruptNumber, enabled: Enabled) { - let reg_base = match enabled { - true => offset::ISENABLER, - false => offset::ICENABLER, - }; - registers.write_array_volatile::<32>(reg_base, int, 1); - - // whether we're enabling or disabling, - // set as part of group 1 - registers.write_array_volatile::<32>(reg_base, int, GROUP_1); -} + /// Returns whether the given SPI (shared peripheral interrupt) will be + /// forwarded by the distributor + pub fn is_spi_enabled(&self, int: InterruptNumber) -> Enabled { + // enabled? + read_array_volatile::<32>(&self.set_enable, int) > 0 + && + // part of group 1? + read_array_volatile::<32>(&self.group, int) == GROUP_1 + } -/// Returns the priority of an SPI. -pub fn get_spi_priority(registers: &GicRegisters, int: InterruptNumber) -> Priority { - u8::MAX - (registers.read_array_volatile::<4>(offset::IPRIORITYR, int) as u8) -} + /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt) + pub fn enable_spi(&mut self, int: InterruptNumber, enabled: Enabled) { + let reg_base = match enabled { + true => &mut self.set_enable, + false => &mut self.clear_enable, + }; + write_array_volatile::<32>(reg_base, int, 1); + + // whether we're enabling or disabling, + // set as part of group 1 + write_array_volatile::<32>(&mut self.group, int, GROUP_1); + } -/// Sets the priority of an SPI. -pub fn set_spi_priority(registers: &mut GicRegisters, int: InterruptNumber, prio: Priority) { - registers.write_array_volatile::<4>(offset::IPRIORITYR, int, (u8::MAX - prio) as u32); -} + /// Returns the priority of an SPI. + pub fn get_spi_priority(&self, int: InterruptNumber) -> Priority { + u8::MAX - (read_array_volatile::<4>(&self.priority, int) as u8) + } -/// Sends an Inter-Processor-Interrupt -/// -/// legacy / GICv2 method -/// int_num must be less than 16 -pub fn send_ipi_gicv2(registers: &mut GicRegisters, int_num: u32, target: IpiTargetCpu) { - if let IpiTargetCpu::Specific(cpu) = &target { - assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + /// Sets the priority of an SPI. + pub fn set_spi_priority(&mut self, int: InterruptNumber, prio: Priority) { + write_array_volatile::<4>(&mut self.priority, int, (u8::MAX - prio) as u32); } - let target_list = match target { - IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16, - IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, - IpiTargetCpu::GICv2TargetList(list) => (list.0 as u32) << 16, - }; + /// Sends an Inter-Processor-Interrupt + /// + /// legacy / GICv2 method + /// int_num must be less than 16 + #[allow(dead_code)] + pub fn send_ipi_gicv2(&mut self, int_num: u32, target: IpiTargetCpu) { + if let IpiTargetCpu::Specific(cpu) = &target { + assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); + } + + let target_list = match target { + IpiTargetCpu::Specific(cpu) => (1 << cpu.value()) << 16, + IpiTargetCpu::AllOtherCpus => SGIR_TARGET_ALL_OTHER_PE, + IpiTargetCpu::GICv2TargetList(list) => (list.0 as u32) << 16, + }; - let value: u32 = int_num | target_list | SGIR_NSATT_GRP1; - registers.write_volatile(offset::SGIR, value); + let value: u32 = int_num | target_list | SGIR_NSATT_GRP1; + self.sgir.write(value); + } } /// Deserialized content of the `IIDR` distributor register @@ -147,23 +186,23 @@ pub struct Implementer { pub implementer_jep106: u16, } -impl super::ArmGic { - pub(crate) fn distributor(&self) -> &GicRegisters { +impl super::ArmGicDistributor { + pub(crate) fn distributor(&self) -> &DistRegsP1 { match self { - Self::V2(v2) => &v2.distributor, - Self::V3(v3) => &v3.distributor, + Self::V2 { registers } => registers, + Self::V3 { v2_regs, .. } => v2_regs, } } - pub(crate) fn distributor_mut(&mut self) -> &mut GicRegisters { + pub(crate) fn distributor_mut(&mut self) -> &mut DistRegsP1 { match self { - Self::V2(v2) => &mut v2.distributor, - Self::V3(v3) => &mut v3.distributor, + Self::V2 { registers } => registers, + Self::V3 { v2_regs, .. } => v2_regs, } } pub fn implementer(&self) -> Implementer { - let raw = self.distributor().read_volatile(offset::IIDR); + let raw = self.distributor().ident.read(); Implementer { product_id: (raw >> 24) as _, version: ((raw >> 12) & 0xff) as _, @@ -179,8 +218,8 @@ impl super::ArmGic { pub fn get_spi_target(&self, int: InterruptNumber) -> Result { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); match self { - Self::V2(_) | Self::V3(ArmGicV3 { affinity_routing: false, .. }) => { - let flags = self.distributor().read_array_volatile::<4>(offset::ITARGETSR, int); + Self::V2 { .. } | Self::V3 { affinity_routing: false, .. } => { + let flags = read_array_volatile::<4>(&self.distributor().target, int); for i in 0..8 { let target = 1 << i; @@ -193,8 +232,8 @@ impl super::ArmGic { Ok(SpiDestination::GICv2TargetList(TargetList(flags as u8)).canonicalize()) }, - Self::V3(ArmGicV3 { affinity_routing: true, dist_extended, .. }) => { - let reg = dist_extended.read_volatile_64(offset::P6IROUTER); + Self::V3 { affinity_routing: true, v3_regs, .. } => { + let reg = v3_regs.route.read(); // bit 31: Interrupt Routing Mode // value of 1 to target any available cpu @@ -214,7 +253,7 @@ impl super::ArmGic { pub fn set_spi_target(&mut self, int: InterruptNumber, target: SpiDestination) { assert!(int >= 32, "interrupts number below 32 (SGIs & PPIs) don't have a target CPU"); match self { - Self::V2(_) | Self::V3(ArmGicV3 { affinity_routing: false, .. }) => { + Self::V2 { .. } | Self::V3 { affinity_routing: false, .. } => { if let SpiDestination::Specific(cpu) = &target { assert!(cpu.value() < 8, "affinity routing is disabled; cannot target a CPU with id >= 8"); } @@ -229,9 +268,9 @@ impl super::ArmGic { SpiDestination::GICv2TargetList(list) => list.0 as u32, }; - self.distributor_mut().write_array_volatile::<4>(offset::ITARGETSR, int, value); + write_array_volatile::<4>(&mut self.distributor_mut().target, int, value); }, - Self::V3(ArmGicV3 { affinity_routing: true, dist_extended, .. }) => { + Self::V3 { affinity_routing: true, v3_regs, .. } => { let value = match target { SpiDestination::Specific(cpu) => MpidrValue::from(cpu).value(), // bit 31: Interrupt Routing Mode @@ -242,7 +281,7 @@ impl super::ArmGic { }, }; - dist_extended.write_volatile_64(offset::P6IROUTER, value); + v3_regs.route.write(value); } } } diff --git a/kernel/gic/src/gic/mod.rs b/kernel/gic/src/gic/mod.rs index 00f233f104..66f8247ac9 100644 --- a/kernel/gic/src/gic/mod.rs +++ b/kernel/gic/src/gic/mod.rs @@ -1,19 +1,24 @@ -use core::convert::AsMut; - -use cpu::{CpuId, MpidrValue}; use arm_boards::{BOARD_CONFIG, NUM_CPUS}; +use cpu::{CpuId, MpidrValue}; +use volatile::Volatile; +use spin::Once; + use memory::{ - PageTable, BorrowedMappedPages, Mutable, PhysicalAddress, - allocate_pages, allocate_frames_at, MMIO_FLAGS, + BorrowedMappedPages, Mutable, PhysicalAddress, MappedPages, + AllocatedFrames, get_kernel_mmi_ref, allocate_frames_at, + allocate_pages, map_frame_range, PAGE_SIZE, MMIO_FLAGS, }; -use static_assertions::const_assert_eq; mod cpu_interface_gicv3; mod cpu_interface_gicv2; mod dist_interface; mod redist_interface; +use dist_interface::{DistRegsP1, DistRegsP6}; +use cpu_interface_gicv2::CpuRegsP1; +use redist_interface::{RedistRegsP1, RedistRegsSgiPpi}; + /// Boolean pub type Enabled = bool; @@ -118,336 +123,395 @@ impl SpiDestination { const U32BITS: usize = u32::BITS as usize; -#[derive(Copy, Clone)] -pub(crate) struct Offset32(usize); +// Reads one item of an array spanning across +// multiple u32s. +// +// The maximum item size is 32 bits, and the items are always aligned to 2**N bits. +// The array spans multiple adjacent u32s but there is always a integer number of +// items in a single u32. +// +// - `int` is the index +// - `INTS_PER_U32` = how many array slots per u32 in this array +fn read_array_volatile(slice: &[Volatile], int: InterruptNumber) -> u32 { + let int = int as usize; + let bits_per_int: usize = U32BITS / INTS_PER_U32; + let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); + + let offset = int / INTS_PER_U32; + let reg_index = int & (INTS_PER_U32 - 1); + let shift = reg_index * bits_per_int; + + let reg = slice[offset].read(); + (reg >> shift) & mask +} -#[derive(Copy, Clone)] -pub(crate) struct Offset64(usize); +// Writes one item of an array spanning across +// multiple u32s. +// +// The maximum item size is 32 bits, and the items are always aligned to 2**N bits. +// The array spans multiple adjacent u32s but there is always a integer number of +// items in a single u32. +// +// - `int` is the index +// - `INTS_PER_U32` = how many array slots per u32 in this array +// - `value` is the value to write +fn write_array_volatile(slice: &mut [Volatile], int: InterruptNumber, value: u32) { + let int = int as usize; + let bits_per_int: usize = U32BITS / INTS_PER_U32; + let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); + + let offset = int / INTS_PER_U32; + let reg_index = int & (INTS_PER_U32 - 1); + let shift = reg_index * bits_per_int; + + let mut reg = slice[offset].read(); + reg &= !(mask << shift); + reg |= (value & mask) << shift; + slice[offset].write(reg); +} -impl Offset32 { - pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { - Self(byte_offset / core::mem::size_of::()) - } +const REDIST_SGIPPI_OFFSET: usize = 0x10000; +const DIST_P6_OFFSET: usize = 0x6000; + +pub struct ArmGicV3RedistPages { + pub redistributor: BorrowedMappedPages, + pub redist_sgippi: BorrowedMappedPages, } -impl Offset64 { - pub(crate) const fn from_byte_offset(byte_offset: usize) -> Self { - Self(byte_offset / core::mem::size_of::()) +pub enum Version { + InitV2 { + dist: PhysicalAddress, + cpu: PhysicalAddress, + }, + InitV3 { + dist: PhysicalAddress, + redist: [PhysicalAddress; NUM_CPUS], } } -#[repr(C)] -#[derive(zerocopy::FromBytes)] -pub struct GicRegisters { - inner: [u32; 0x400], +pub enum ArmGicDistributor { + V2 { + registers: BorrowedMappedPages, + }, + V3 { + affinity_routing: Enabled, + v2_regs: BorrowedMappedPages, + v3_regs: BorrowedMappedPages, + }, } -impl GicRegisters { - fn read_volatile(&self, offset: Offset32) -> u32 { - unsafe { (&self.inner[offset.0] as *const u32).read_volatile() } - } +impl ArmGicDistributor { + pub fn init(version: &Version) -> Result { + match version { + Version::InitV2 { dist, .. } => { + let mapped = map_frame_range(*dist, PAGE_SIZE, MMIO_FLAGS)?; + let mut registers = mapped + .into_borrowed_mut::(0) + .map_err(|(_, e)| e)?; - fn write_volatile(&mut self, offset: Offset32, value: u32) { - unsafe { (&mut self.inner[offset.0] as *mut u32).write_volatile(value) } - } + registers.init(); - fn read_volatile_64(&self, offset: Offset64) -> u64 { - unsafe { (self.inner.as_ptr() as *const u64).add(offset.0).read_volatile() } - } + Ok(Self::V2 { + registers, + }) + }, + Version::InitV3 { dist, .. } => { + let mapped = map_frame_range(*dist, PAGE_SIZE, MMIO_FLAGS)?; + let mut v2_regs = mapped + .into_borrowed_mut::(0) + .map_err(|(_, e)| e)?; + + let v3_regs: BorrowedMappedPages = { + let paddr = *dist + DIST_P6_OFFSET; + let mapped = map_frame_range(paddr, PAGE_SIZE, MMIO_FLAGS)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; - fn write_volatile_64(&mut self, offset: Offset64, value: u64) { - unsafe { (self.inner.as_mut_ptr() as *mut u64).add(offset.0).write_volatile(value) } - } + let affinity_routing = v2_regs.init(); - // Reads one item of an array spanning across - // multiple u32s. - // - // The maximum item size is 32 bits, and the items are always aligned to 2**N bits. - // The array spans multiple adjacent u32s but there is always a integer number of - // items in a single u32. - // - // - `int` is the index - // - `offset` tells the beginning of the array - // - `INTS_PER_U32` = how many array slots per u32 in this array - fn read_array_volatile(&self, offset: Offset32, int: InterruptNumber) -> u32 { - let int = int as usize; - let bits_per_int: usize = U32BITS / INTS_PER_U32; - let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - - let offset = Offset32(offset.0 + (int / INTS_PER_U32)); - let reg_index = int & (INTS_PER_U32 - 1); - let shift = reg_index * bits_per_int; - - let reg = self.read_volatile(offset); - (reg >> shift) & mask + Ok(Self::V3 { + affinity_routing, + v2_regs, + v3_regs, + }) + }, + } } - // Writes one item of an array spanning across - // multiple u32s. - // - // The maximum item size is 32 bits, and the items are always aligned to 2**N bits. - // The array spans multiple adjacent u32s but there is always a integer number of - // items in a single u32. - // - // - `int` is the index - // - `offset` tells the beginning of the array - // - `INTS_PER_U32` = how many array slots per u32 in this array - // - `value` is the value to write - fn write_array_volatile(&mut self, offset: Offset32, int: InterruptNumber, value: u32) { - let int = int as usize; - let bits_per_int: usize = U32BITS / INTS_PER_U32; - let mask: u32 = u32::MAX >> (U32BITS - bits_per_int); - - let offset = Offset32(offset.0 + (int / INTS_PER_U32)); - let reg_index = int & (INTS_PER_U32 - 1); - let shift = reg_index * bits_per_int; - - let mut reg = self.read_volatile(offset); - reg &= !(mask << shift); - reg |= (value & mask) << shift; - self.write_volatile(offset, reg); + /// Returns whether the given interrupt is forwarded by the distributor. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn get_spi_state(&self, int: InterruptNumber) -> Enabled { + assert!(int >= 32, "get_spi_state: `int` must be >= 32"); + self.distributor().is_spi_enabled(int) } -} - -const_assert_eq!(core::mem::size_of::(), 0x1000); - -/// Returns the index to the redistributor base address for this CPU -/// in the array of register base addresses. -/// -/// This is defined in `arm_boards::INTERRUPT_CONTROLLER_CONFIG`. -fn get_current_cpu_redist_index() -> usize { - let cpu_id = cpu::current_cpu(); - BOARD_CONFIG.cpu_ids.iter() - .position(|mpidr| CpuId::from(*mpidr) == cpu_id) - .expect("BUG: get_current_cpu_redist_index: unexpected CpuId for current CPU") -} - -const REDIST_SGIPPI_OFFSET: usize = 0x10000; -const DIST_P6_OFFSET: usize = 0x6000; -pub struct ArmGicV2 { - pub distributor: BorrowedMappedPages, - pub processor: BorrowedMappedPages, -} + /// Enables or disables the forwarding of the given interrupt + /// by the distributor. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn set_spi_state(&mut self, int: InterruptNumber, enabled: Enabled) { + assert!(int >= 32, "set_spi_state: `int` must be >= 32"); + self.distributor_mut().enable_spi(int, enabled) + } -pub struct ArmGicV3RedistPages { - pub redistributor: BorrowedMappedPages, - pub redist_sgippi: BorrowedMappedPages, -} + /// Returns the priority of the given interrupt. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn get_spi_priority(&self, int: InterruptNumber) -> Priority { + assert!(int >= 32, "get_spi_priority: `int` must be >= 32"); + self.distributor().get_spi_priority(int) + } -pub struct ArmGicV3 { - pub affinity_routing: Enabled, - pub distributor: BorrowedMappedPages, - pub dist_extended: BorrowedMappedPages, - pub redistributors: [ArmGicV3RedistPages; NUM_CPUS], + /// Sets the priority of the given interrupt. + /// + /// Panics if `int` is not in the SPI range (>= 32). + pub fn set_spi_priority(&mut self, int: InterruptNumber, enabled: Priority) { + assert!(int >= 32, "set_spi_priority: `int` must be >= 32"); + self.distributor_mut().set_spi_priority(int, enabled) + } } -/// Arm Generic Interrupt Controller -/// -/// The GIC is an extension to ARMv8 which -/// allows routing and filtering interrupts -/// in a single or multi-core system. -#[allow(clippy::large_enum_variant)] -pub enum ArmGic { - V2(ArmGicV2), - V3(ArmGicV3), +pub enum ArmGicCpuComponents { + V2 { + registers: BorrowedMappedPages, + cpu_index: u16, + }, + V3 { + redist_regs: ArmGicV3RedistPages, + }, } -pub enum Version { - InitV2 { - dist: PhysicalAddress, - cpu: PhysicalAddress, - }, - InitV3 { - dist: PhysicalAddress, - redist: [PhysicalAddress; NUM_CPUS], +/// Map the physical frames containing the GICv2 CPU Interface's MMIO registers into the given `page_table`. +fn map_gicv2_cpu_iface(cpu_iface: PhysicalAddress) -> Result { + static CPU_IFACE_FRAME: Once = Once::new(); + + let frame = if let Some(cpu_iface) = CPU_IFACE_FRAME.get() { + cpu_iface + } else { + let cpu_iface = allocate_frames_at(cpu_iface, 1)?; + CPU_IFACE_FRAME.call_once(|| cpu_iface) + }; + + let new_page = allocate_pages(1).ok_or("out of virtual address space!")?; + let mmi = get_kernel_mmi_ref().ok_or("map_gicv2_cpu_iface(): uninitialized KERNEL_MMI")?; + let mut mmi = mmi.lock(); + + // The CPU Interface frame is the same actual physical address across all CPU cores, + // but they're actually completely independent pieces of hardware that share one address. + // Therefore, there's no way to represent that to the Rust language or + // MappedPages/AllocatedFrames types, so we must use unsafe code, at least for now. + unsafe { + memory::Mapper::map_to_non_exclusive( + &mut mmi.page_table, + new_page, + frame, + MMIO_FLAGS, + ) } } -impl ArmGic { - pub fn init(page_table: &mut PageTable, version: Version) -> Result { - let mut map_dist = |gicd_base| -> Result, &'static str> { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the distributor interface")?; - let frames = allocate_frames_at(gicd_base, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e) - }; +impl ArmGicCpuComponents { + pub fn init(cpu_id: CpuId, version: &Version) -> Result { + let cpu_index = BOARD_CONFIG.cpu_ids.iter() + .position(|mpidr| CpuId::from(*mpidr) == cpu_id) + .expect("BUG: invalid CpuId in ArmGicCpuComponents::init"); match version { - Version::InitV2 { dist, cpu } => { - let mut distributor = map_dist(dist)?; - - let mut processor: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the CPU interface")?; - let frames = allocate_frames_at(cpu, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; + Version::InitV2 { cpu, .. } => { + let mut registers: BorrowedMappedPages = { + let mapped = map_gicv2_cpu_iface(*cpu)?; mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - cpu_interface_gicv2::init(processor.as_mut()); - dist_interface::init(distributor.as_mut()); + registers.init(); - Ok(Self::V2(ArmGicV2 { distributor, processor })) + Ok(Self::V2 { + registers, + cpu_index: cpu_index as u16, + }) }, - Version::InitV3 { dist, redist } => { - let mut distributor = map_dist(dist)?; + Version::InitV3 { redist, .. } => { + let phys_addr = redist[cpu_index]; - let dist_extended: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended distributor interface")?; - let frames = allocate_frames_at(dist + DIST_P6_OFFSET, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; + let mut redistributor: BorrowedMappedPages = { + let mapped = map_frame_range(phys_addr, PAGE_SIZE, MMIO_FLAGS)?; mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? }; - let redistributors: [ArmGicV3RedistPages; NUM_CPUS] = core::array::try_from_fn(|i| { - let phys_addr = redist[i]; - - let mut redistributor: BorrowedMappedPages = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the redistributor interface")?; - let frames = allocate_frames_at(phys_addr, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; - - redist_interface::init(redistributor.as_mut())?; + let redist_sgippi = { + let rso_paddr = phys_addr + REDIST_SGIPPI_OFFSET; + let mapped = map_frame_range(rso_paddr, PAGE_SIZE, MMIO_FLAGS)?; + mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? + }; - let redist_sgippi = { - let pages = allocate_pages(1).ok_or("couldn't allocate pages for the extended redistributor interface")?; - let frames = allocate_frames_at(phys_addr + REDIST_SGIPPI_OFFSET, 1)?; - let mapped = page_table.map_allocated_pages_to(pages, frames, MMIO_FLAGS)?; - mapped.into_borrowed_mut(0).map_err(|(_, e)| e)? - }; + redistributor.init()?; + cpu_interface_gicv3::init(); - Ok::(ArmGicV3RedistPages { + Ok(Self::V3 { + redist_regs: ArmGicV3RedistPages { redistributor, redist_sgippi, - }) - })?; - - // this cannot fail as we pushed exactly `arm_boards::CPUS` items - // let redistributors = redistributors.into_inner().unwrap(); - - cpu_interface_gicv3::init(); - let affinity_routing = dist_interface::init(distributor.as_mut()); - - Ok(Self::V3(ArmGicV3 { distributor, dist_extended, redistributors, affinity_routing })) + }, + }) }, } } pub fn init_secondary_cpu_interface(&mut self) { match self { - Self::V2(v2) => cpu_interface_gicv2::init(v2.processor.as_mut()), - Self::V3( _) => cpu_interface_gicv3::init(), + Self::V2 { registers, .. } => registers.init(), + Self::V3 { .. } => cpu_interface_gicv3::init(), } } - /// Sends an inter processor interrupt (IPI), - /// also called software generated interrupt (SGI). + /// Sends an Inter-Processor Interrupt (IPI) with the given interrupt number + /// to the given target CPU(s). + /// + /// This is also referred to as a Software-Generated Interrupt (SGI). /// - /// note: on Aarch64, IPIs must have a number below 16 on ARMv8 - pub fn send_ipi(&mut self, int_num: InterruptNumber, target: IpiTargetCpu) { - assert!(int_num < 16, "IPIs must have a number below 16 on ARMv8"); + /// Panics if `int` is greater than or equal to 16; + /// on aarch64, IPIs much be sent to an interrupt number less than 16. + pub fn send_ipi(&mut self, int: InterruptNumber, target: IpiTargetCpu) { + assert!(int < 16, "IPIs must have a number below 16 on ARMv8"); - match self { - Self::V2(v2) => dist_interface::send_ipi_gicv2(&mut v2.distributor, int_num, target), - Self::V3( _) => cpu_interface_gicv3::send_ipi(int_num, target), + if let Self::V3 { .. } = self { + cpu_interface_gicv3::send_ipi(int, target) + } else { + // we don't have access to the distributor... code would be: + // dist_interface::send_ipi_gicv2(&mut dist_regs, int, target) + // workaround: caller could check is this must be done in the dist + // and then get the SystemInterruptController and call a dedicated + // method on it, like `sys_ctlr.send_ipi_gicv2()` + + panic!("GICv2 doesn't support sending IPIs (need distributor)"); } } - /// Acknowledge the currently serviced interrupt - /// and fetches its number + /// Acknowledge the currently-serviced interrupt. + /// + /// This tells the GIC that the current interrupt is in the midst of + /// being handled by this CPU. + /// + /// Returns a tuple of the interrupt's number and priority. pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) { match self { - Self::V2(v2) => cpu_interface_gicv2::acknowledge_interrupt(&mut v2.processor), - Self::V3( _) => cpu_interface_gicv3::acknowledge_interrupt(), + Self::V2 { registers, .. } => registers.acknowledge_interrupt(), + Self::V3 { .. } => cpu_interface_gicv3::acknowledge_interrupt(), } } - /// Performs priority drop for the specified interrupt + /// Signals to the controller that the currently processed interrupt + /// has been fully handled, by zeroing the current priority level of + /// the current CPU. + /// + /// This implies that the CPU is ready to process interrupts again. pub fn end_of_interrupt(&mut self, int: InterruptNumber) { match self { - Self::V2(v2) => cpu_interface_gicv2::end_of_interrupt(&mut v2.processor, int), - Self::V3( _) => cpu_interface_gicv3::end_of_interrupt(int), + Self::V2 { registers, .. } => registers.end_of_interrupt(int), + Self::V3 { .. } => cpu_interface_gicv3::end_of_interrupt(int), } } - /// Will that interrupt be forwarded by the distributor? + /// Returns whether the given local interrupt will be received by the current CPU. + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn get_interrupt_state(&self, int: InterruptNumber) -> Enabled { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::is_sgippi_enabled(&v3.redistributors[i].redist_sgippi, int) - }, - (_, this) => dist_interface::is_spi_enabled(this.distributor(), int), + assert!(int < 32, "get_interrupt_state: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.is_sgippi_enabled(int) + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support enabling/disabling local interrupt"); + + // should we panic? + true } } - /// Enables or disables the forwarding of - /// a particular interrupt in the distributor + /// Enables or disables the receiving of a local interrupt in the distributor. + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn set_interrupt_state(&mut self, int: InterruptNumber, enabled: Enabled) { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::enable_sgippi(&mut v3.redistributors[i].redist_sgippi, int, enabled); - }, - (_, this) => dist_interface::enable_spi(this.distributor_mut(), int, enabled), - }; + assert!(int < 32, "set_interrupt_state: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.enable_sgippi(int, enabled); + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support enabling/disabling local interrupt"); + } } - /// Returns the priority of an interrupt + /// Returns the priority of a local interrupt + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn get_interrupt_priority(&self, int: InterruptNumber) -> Priority { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::get_sgippi_priority(&v3.redistributors[i].redist_sgippi, int) - }, - (_, this) => dist_interface::get_spi_priority(this.distributor(), int), + assert!(int < 32, "get_interrupt_priority: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.get_sgippi_priority(int) + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support setting local interrupt priority"); + + // should we panic? + 128 } } - /// Sets the priority of an interrupt (0-255) + /// Sets the priority of a local interrupt (prio: 0-255) + /// + /// Panics if `int` is greater than or equal to 32, which is beyond the range + /// of local interrupt numbers. pub fn set_interrupt_priority(&mut self, int: InterruptNumber, enabled: Priority) { - match (int, self) { - (0..=31, Self::V3(v3)) => { - let i = get_current_cpu_redist_index(); - redist_interface::set_sgippi_priority(&mut v3.redistributors[i].redist_sgippi, int, enabled); - }, - (_, this) => dist_interface::set_spi_priority(this.distributor_mut(), int, enabled), - }; + assert!(int < 32, "set_interrupt_priority: `int` doesn't lie in the SGI/PPI (local interrupt) range"); + + if let Self::V3 { redist_regs } = self { + redist_regs.redist_sgippi.set_sgippi_priority(int, enabled); + } else { + // there is no redistributor and we don't have access to the distributor + log::error!("GICv2 doesn't support setting local interrupt priority"); + } } - /// Interrupts have a priority; if their priority - /// is lower or equal to this one, they're queued - /// until this CPU or another one is ready to handle - /// them + /// Retrieves the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. pub fn get_minimum_priority(&self) -> Priority { match self { - Self::V2(v2) => cpu_interface_gicv2::get_minimum_priority(&v2.processor), - Self::V3( _) => cpu_interface_gicv3::get_minimum_priority(), + Self::V2 { registers, .. } => registers.get_minimum_priority(), + Self::V3 { .. } => cpu_interface_gicv3::get_minimum_priority(), } } - /// Interrupts have a priority; if their priority - /// is lower or equal to this one, they're queued - /// until this CPU or another one is ready to handle - /// them + /// Sets the current priority threshold for the current CPU. + /// + /// Interrupts have a priority; if their priority is lower or + /// equal to this threshold, they're queued until the current CPU + /// is ready to handle them. pub fn set_minimum_priority(&mut self, priority: Priority) { match self { - Self::V2(v2) => cpu_interface_gicv2::set_minimum_priority(&mut v2.processor, priority), - Self::V3( _) => cpu_interface_gicv3::set_minimum_priority(priority), + Self::V2 { registers, .. } => registers.set_minimum_priority(priority), + Self::V3 { .. } => cpu_interface_gicv3::set_minimum_priority(priority), } } /// Returns the internal ID of the redistributor (GICv3) /// - /// Note #2: this is only provided for debugging purposes - /// Note #1: as a compatibility feature, on GICv2, the CPU index is returned. + /// ## Notes + /// * As a compatibility feature, on GICv2, the CPU index is returned. + /// * This is only provided for debugging purposes. pub fn get_cpu_interface_id(&self) -> u16 { - let i = get_current_cpu_redist_index(); match self { - Self::V3(v3) => redist_interface::get_internal_id(&v3.redistributors[i].redistributor), - _ => i as _, + Self::V3 { redist_regs } => redist_regs.redistributor.get_internal_id(), + Self::V2 { cpu_index, .. } => *cpu_index, } } } diff --git a/kernel/gic/src/gic/redist_interface.rs b/kernel/gic/src/gic/redist_interface.rs index 7f14ae2907..02461b343a 100644 --- a/kernel/gic/src/gic/redist_interface.rs +++ b/kernel/gic/src/gic/redist_interface.rs @@ -9,20 +9,58 @@ //! - Enabling or disabling the forwarding of PPIs & SGIs based on their numbers //! - Getting or setting the priority of PPIs & SGIs based on their numbers -use super::GicRegisters; use super::InterruptNumber; use super::Enabled; use super::Priority; +use super::read_array_volatile; +use super::write_array_volatile; -mod offset { - use crate::{Offset32, Offset64}; - pub(crate) const CTLR: Offset32 = Offset32::from_byte_offset(0x00); - pub(crate) const TYPER: Offset64 = Offset64::from_byte_offset(0x08); - pub(crate) const WAKER: Offset32 = Offset32::from_byte_offset(0x14); - pub(crate) const IGROUPR: Offset32 = Offset32::from_byte_offset(0x80); - pub(crate) const SGIPPI_ISENABLER: Offset32 = Offset32::from_byte_offset(0x100); - pub(crate) const SGIPPI_ICENABLER: Offset32 = Offset32::from_byte_offset(0x180); - pub(crate) const SGIPPI_IPRIORITYR: Offset32 = Offset32::from_byte_offset(0x400); +use volatile::{Volatile, ReadOnly}; +use zerocopy::FromBytes; + +/// General redistributor registers +#[derive(FromBytes)] +#[repr(C)] +pub struct RedistRegsP1 { // base offset + /// Redistributor Control Register + ctlr: Volatile, // 0x00 + + /// Implementer Identification Register + _unused0: u32, + + /// Redistributor Type Register + ident: ReadOnly, // 0x08 + + /// Error Reporting Status Register, optional + _unused1: u32, + + /// Redistributor Wake Register + waker: Volatile, // 0x14 +} + +/// Redistributor registers for SGIs & PPIs +#[derive(FromBytes)] +#[repr(C)] +pub struct RedistRegsSgiPpi { // base offset + _reserved0: [u8; 0x80], + + /// Interrupt Group Register 0 + group: [Volatile; 0x01], // 0x080 + _reserved1: [u32; 0x1f], + + /// Interrupt Set-Enable Registers + set_enable: [Volatile; 0x01], // 0x100 + _reserved2: [u32; 0x1f], + + /// Interrupt Clear-Enable Registers + clear_enable: [Volatile; 0x01], // 0x180 + _reserved3: [u32; 0x1f], + + /// Interrupt Set & Clear Pending / Active Registers + _unused0: [u32; 0x80], + + /// Interrupt Priority Registers + priority: [Volatile; 0x08], // 0x400 } const WAKER_PROCESSOR_SLEEP: u32 = 1 << 1; @@ -57,86 +95,90 @@ const GROUP_1: u32 = 1; /// developers (or directly submit a PR). const TIMEOUT_ITERATIONS: usize = 0xffff; -/// Initializes the redistributor by waking it up and waiting for it to awaken. -/// -/// Returns an error if a timeout occurs while waiting. -pub fn init(registers: &mut GicRegisters) -> Result<(), &'static str> { - let mut reg = registers.read_volatile(offset::WAKER); - - // Wake the redistributor - reg &= !WAKER_PROCESSOR_SLEEP; - registers.write_volatile(offset::WAKER, reg); - - // Then, wait for the children to wake up, timing out if it never happens. - let children_asleep = || { - registers.read_volatile(offset::WAKER) & WAKER_CHLIDREN_ASLEEP > 0 - }; - let mut counter = 0; - while children_asleep() { - counter += 1; +impl RedistRegsP1 { + /// Initializes the redistributor by waking it up and waiting for it to awaken. + /// + /// Returns an error if a timeout occurs while waiting. + pub fn init(&mut self) -> Result<(), &'static str> { + let mut reg = self.waker.read(); + + // Wake the redistributor + reg &= !WAKER_PROCESSOR_SLEEP; + self.waker.write(reg); + + // Then, wait for the children to wake up, timing out if it never happens. + let children_asleep = || { + self.waker.read() & WAKER_CHLIDREN_ASLEEP > 0 + }; + let mut counter = 0; + while children_asleep() { + counter += 1; + if counter >= TIMEOUT_ITERATIONS { + break; + } + } + if counter >= TIMEOUT_ITERATIONS { - break; + return Err("BUG: gic driver: The redistributor didn't wake up in time."); } - } - if counter >= TIMEOUT_ITERATIONS { - return Err("BUG: gic driver: The redistributor didn't wake up in time."); - } + if self.ident.read() & TYPER_DPGS != 0 { + // DPGS bits are supported in GICR_CTLR + let mut reg = self.ctlr.read(); - if registers.read_volatile_64(offset::TYPER) & TYPER_DPGS != 0 { - // DPGS bits are supported in GICR_CTLR - let mut reg = registers.read_volatile(offset::CTLR); + // Enable PE selection for non-secure group 1 SPIs + reg &= !CTLR_DPG1NS; - // Enable PE selection for non-secure group 1 SPIs - reg &= !CTLR_DPG1NS; + // Disable PE selection for group 0 & secure group 1 SPIs + reg |= CTLR_DPG0; + reg |= CTLR_DPG1S; - // Disable PE selection for group 0 & secure group 1 SPIs - reg |= CTLR_DPG0; - reg |= CTLR_DPG1S; + self.ctlr.write(reg); + } - registers.write_volatile(offset::CTLR, reg); + Ok(()) } - Ok(()) -} - -/// Returns whether the given SGI (software generated interrupts) or -/// PPI (private peripheral interrupts) will be forwarded by the redistributor -pub fn is_sgippi_enabled(registers: &GicRegisters, int: InterruptNumber) -> Enabled { - registers.read_array_volatile::<32>(offset::SGIPPI_ISENABLER, int) > 0 - && - // part of group 1? - registers.read_array_volatile::<32>(offset::IGROUPR, int) == GROUP_1 + /// Returns the internal ID of the redistributor + /// + /// Note: this is only provided for debugging purposes + pub fn get_internal_id(&self) -> u16 { + (self.ident.read() >> 8) as _ + } } -/// Enables or disables the forwarding of a particular -/// SGI (software generated interrupts) or PPI (private -/// peripheral interrupts) -pub fn enable_sgippi(registers: &mut GicRegisters, int: InterruptNumber, enabled: Enabled) { - let reg = match enabled { - true => offset::SGIPPI_ISENABLER, - false => offset::SGIPPI_ICENABLER, - }; - registers.write_array_volatile::<32>(reg, int, 1); - - // whether we're enabling or disabling, - // set as part of group 1 - registers.write_array_volatile::<32>(offset::IGROUPR, int, GROUP_1); -} +impl RedistRegsSgiPpi { + /// Returns whether the given SGI (software generated interrupts) or + /// PPI (private peripheral interrupts) will be forwarded by the redistributor + pub fn is_sgippi_enabled(&self, int: InterruptNumber) -> Enabled { + read_array_volatile::<32>(&self.set_enable, int) > 0 + && + // part of group 1? + read_array_volatile::<32>(&self.group, int) == GROUP_1 + } -/// Returns the priority of an SGI/PPI. -pub fn get_sgippi_priority(registers: &GicRegisters, int: InterruptNumber) -> Priority { - u8::MAX - (registers.read_array_volatile::<4>(offset::SGIPPI_IPRIORITYR, int) as u8) -} + /// Enables or disables the forwarding of a particular + /// SGI (software generated interrupts) or PPI (private + /// peripheral interrupts) + pub fn enable_sgippi(&mut self, int: InterruptNumber, enabled: Enabled) { + let reg = match enabled { + true => &mut self.set_enable, + false => &mut self.clear_enable, + }; + write_array_volatile::<32>(reg, int, 1); + + // whether we're enabling or disabling, + // set as part of group 1 + write_array_volatile::<32>(&mut self.group, int, GROUP_1); + } -/// Sets the priority of an SGI/PPI. -pub fn set_sgippi_priority(registers: &mut GicRegisters, int: InterruptNumber, prio: Priority) { - registers.write_array_volatile::<4>(offset::SGIPPI_IPRIORITYR, int, (u8::MAX - prio) as u32); -} + /// Returns the priority of an SGI/PPI. + pub fn get_sgippi_priority(&self, int: InterruptNumber) -> Priority { + u8::MAX - (read_array_volatile::<4>(&self.priority, int) as u8) + } -/// Returns the internal ID of the redistributor -/// -/// Note: this is only provided for debugging purposes -pub fn get_internal_id(registers: &GicRegisters) -> u16 { - (registers.read_volatile_64(offset::TYPER) >> 8) as _ + /// Sets the priority of an SGI/PPI. + pub fn set_sgippi_priority(&mut self, int: InterruptNumber, prio: Priority) { + write_array_volatile::<4>(&mut self.priority, int, (u8::MAX - prio) as u32); + } } diff --git a/kernel/gic/src/lib.rs b/kernel/gic/src/lib.rs index 2861978bdf..c07e40ec24 100644 --- a/kernel/gic/src/lib.rs +++ b/kernel/gic/src/lib.rs @@ -1,4 +1,7 @@ -//! Allows configuring the Generic Interrupt Controller +//! Arm Generic Interrupt Controller Support +//! +//! The GIC is an extension to ARMv8 which allows routing and +//! filtering interrupts in a single or multi-core system. //! //! The term "Forwarding" is sometimes used in this crate. //! This is because the Distributor, Redistributor and CPU interface are @@ -8,7 +11,6 @@ #![no_std] #![feature(doc_cfg)] -#![feature(array_try_from_fn)] #[cfg(target_arch = "aarch64")] mod gic; diff --git a/kernel/interrupt_controller/Cargo.toml b/kernel/interrupt_controller/Cargo.toml index 61a8ccf902..8d237fea46 100644 --- a/kernel/interrupt_controller/Cargo.toml +++ b/kernel/interrupt_controller/Cargo.toml @@ -14,6 +14,7 @@ sync_irq = { path = "../../libs/sync_irq" } arm_boards = { path = "../arm_boards" } memory = { path = "../memory" } gic = { path = "../gic" } +spin = "0.9.4" [target.'cfg(target_arch = "x86_64")'.dependencies] apic = { path = "../apic" } diff --git a/kernel/interrupt_controller/src/aarch64.rs b/kernel/interrupt_controller/src/aarch64.rs index 5988c9f97d..6da4f16714 100644 --- a/kernel/interrupt_controller/src/aarch64.rs +++ b/kernel/interrupt_controller/src/aarch64.rs @@ -1,9 +1,10 @@ use { - gic::{ArmGic, SpiDestination, IpiTargetCpu, Version as GicVersion}, - arm_boards::{BOARD_CONFIG, InterruptControllerConfig}, + gic::{ArmGicDistributor, ArmGicCpuComponents, SpiDestination, IpiTargetCpu, Version as GicVersion}, + arm_boards::{NUM_CPUS, BOARD_CONFIG, InterruptControllerConfig}, + core::array::try_from_fn, sync_irq::IrqSafeMutex, - memory::get_kernel_mmi_ref, - core::ops::DerefMut, + cpu::current_cpu, + spin::Once, }; use super::*; @@ -17,65 +18,72 @@ pub struct SystemInterruptControllerId(pub u8); #[derive(Debug, Copy, Clone)] pub struct LocalInterruptControllerId(pub u16); -/// The private global Generic Interrupt Controller singleton -pub(crate) static INTERRUPT_CONTROLLER: IrqSafeMutex> = IrqSafeMutex::new(None); +/// Per-CPU local interrupt controller +/// +/// To get the controller for a specific CPU: +/// a. Find the position of its CpuId in `BOARD_CONFIG.cpu_ids` +/// b. Index into this array using that position +static LOCAL_INT_CTRL: Once<[LocalInterruptController; NUM_CPUS]> = Once::new(); + +/// System-wide interrupt controller +static SYSTEM_WIDE_INT_CTRL: Once = Once::new(); /// Initializes the interrupt controller, on aarch64 pub fn init() -> Result<(), &'static str> { - let mut int_ctrl = INTERRUPT_CONTROLLER.lock(); - if int_ctrl.is_some() { - Err("The interrupt controller has already been initialized!") - } else { - match BOARD_CONFIG.interrupt_controller { - InterruptControllerConfig::GicV3(gicv3_cfg) => { - let kernel_mmi_ref = get_kernel_mmi_ref() - .ok_or("interrupts::aarch64::init: couldn't get kernel MMI ref")?; - - let mut mmi = kernel_mmi_ref.lock(); - let page_table = &mut mmi.deref_mut().page_table; - - *int_ctrl = Some(ArmGic::init( - page_table, - GicVersion::InitV3 { - dist: gicv3_cfg.distributor_base_address, - redist: gicv3_cfg.redistributor_base_addresses, - }, - )?); - }, - } - - Ok(()) - } + match BOARD_CONFIG.interrupt_controller { + InterruptControllerConfig::GicV3(gicv3_cfg) => { + let version = GicVersion::InitV3 { + dist: gicv3_cfg.distributor_base_address, + redist: gicv3_cfg.redistributor_base_addresses, + }; + + SYSTEM_WIDE_INT_CTRL.try_call_once(|| -> Result<_, &'static str> { + let distrib = ArmGicDistributor::init(&version)?; + let mutex = IrqSafeMutex::new(distrib); + Ok(SystemInterruptController(mutex)) + })?; + + LOCAL_INT_CTRL.try_call_once(|| -> Result<_, &'static str> { + let cpu_ctlrs: [ArmGicCpuComponents; NUM_CPUS] = try_from_fn(|i| { + let cpu_id = BOARD_CONFIG.cpu_ids[i].into(); + ArmGicCpuComponents::init(cpu_id, &version) + })?; + + Ok(cpu_ctlrs.map(|ctlr| { + let mutex = IrqSafeMutex::new(ctlr); + LocalInterruptController(mutex) + })) + })?; + }, + } + + Ok(()) } /// Structure representing a top-level/system-wide interrupt controller chip, /// responsible for routing interrupts between peripherals and CPU cores. /// /// On aarch64 w/ GIC, this corresponds to the Distributor. -pub struct SystemInterruptController; +pub struct SystemInterruptController(IrqSafeMutex); /// Struct representing per-cpu-core interrupt controller chips. /// /// On aarch64 w/ GIC, this corresponds to a Redistributor & CPU interface. -pub struct LocalInterruptController; +pub struct LocalInterruptController(IrqSafeMutex); impl SystemInterruptControllerApi for SystemInterruptController { - fn id(&self) -> SystemInterruptControllerId { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: id(): INTERRUPT_CONTROLLER was uninitialized"); + fn get() -> &'static Self { + SYSTEM_WIDE_INT_CTRL.get().expect("interrupt_controller wasn't initialized") + } - SystemInterruptControllerId(int_ctlr.implementer().product_id) + fn id(&self) -> SystemInterruptControllerId { + let dist = self.0.lock(); + SystemInterruptControllerId(dist.implementer().product_id) } fn version(&self) -> SystemInterruptControllerVersion { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: version(): INTERRUPT_CONTROLLER was uninitialized"); - - SystemInterruptControllerVersion(int_ctlr.implementer().version) + let dist = self.0.lock(); + SystemInterruptControllerVersion(dist.implementer().version) } fn get_destination( @@ -83,13 +91,10 @@ impl SystemInterruptControllerApi for SystemInterruptController { interrupt_num: InterruptNumber, ) -> Result<(Vec, Priority), &'static str> { assert!(interrupt_num >= 32, "shared peripheral interrupts have a number >= 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_destination(): INTERRUPT_CONTROLLER was uninitialized"); + let dist = self.0.lock(); - let priority = int_ctlr.get_interrupt_priority(interrupt_num as _); - let vec = match int_ctlr.get_spi_target(interrupt_num as _)?.canonicalize() { + let priority = dist.get_spi_priority(interrupt_num as _); + let vec = match dist.get_spi_target(interrupt_num as _)?.canonicalize() { SpiDestination::Specific(cpu) => [cpu].to_vec(), SpiDestination::AnyCpuAvailable => BOARD_CONFIG.cpu_ids.map(|mpidr| mpidr.into()).to_vec(), SpiDestination::GICv2TargetList(list) => { @@ -111,126 +116,98 @@ impl SystemInterruptControllerApi for SystemInterruptController { priority: Priority, ) -> Result<(), &'static str> { assert!(sys_int_num >= 32, "shared peripheral interrupts have a number >= 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_destination(): INTERRUPT_CONTROLLER was uninitialized"); + let mut dist = self.0.lock(); - int_ctlr.set_spi_target(sys_int_num as _, SpiDestination::Specific(destination)); - int_ctlr.set_interrupt_priority(sys_int_num as _, priority); + dist.set_spi_target(sys_int_num as _, SpiDestination::Specific(destination)); + dist.set_spi_priority(sys_int_num as _, priority); Ok(()) } } impl LocalInterruptControllerApi for LocalInterruptController { - fn init_secondary_cpu_interface(&self) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: init_secondary_cpu_interface(): INTERRUPT_CONTROLLER was uninitialized"); + fn get() -> &'static Self { + // how this function works: + // a. get the current CpuId: this CpuId of the current CPU + // b. iterate on all valid CpuIds, find the index of the current CpuId. + // This is used as a current CPU index. + // c. get the global array of interrupt controllers + // d. index into this array based on the current CPU index - int_ctlr.init_secondary_cpu_interface(); + let cpu_id = current_cpu(); + // While we're waiting for cpu-local-storage, this loop will work as fine as an AtomicMap + let index = BOARD_CONFIG.cpu_ids.iter().position(|mpidr| cpu_id == (*mpidr).into()); + let index = index.expect("Invalid CpuId returned by current_cpu()"); + + let ctrls = LOCAL_INT_CTRL.get(); + let ctrls = ctrls.expect("interrupt_controller wasn't initialized"); + + &ctrls[index] } - fn id(&self) -> LocalInterruptControllerId { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: id(): INTERRUPT_CONTROLLER was uninitialized"); + fn init_secondary_cpu_interface(&self) { + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.init_secondary_cpu_interface(); + } - LocalInterruptControllerId(int_ctlr.get_cpu_interface_id()) + fn id(&self) -> LocalInterruptControllerId { + let cpu_ctrl = self.0.lock(); + LocalInterruptControllerId(cpu_ctrl.get_cpu_interface_id()) } fn get_local_interrupt_priority(&self, num: InterruptNumber) -> Priority { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_local_interrupt_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_interrupt_priority(num as _) + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_interrupt_priority(num as _) } fn set_local_interrupt_priority(&self, num: InterruptNumber, priority: Priority) { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_local_interrupt_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_interrupt_priority(num as _, priority); + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_interrupt_priority(num as _, priority); } fn is_local_interrupt_enabled(&self, num: InterruptNumber) -> bool { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: is_local_interrupt_enabled(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_interrupt_state(num as _) + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_interrupt_state(num as _) } fn enable_local_interrupt(&self, num: InterruptNumber, enabled: bool) { assert!(num < 32, "local interrupts have a number < 32"); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: enable_local_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_interrupt_state(num as _, enabled); + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_interrupt_state(num as _, enabled); } fn send_ipi(&self, num: InterruptNumber, dest: InterruptDestination) { use InterruptDestination::*; assert!(num < 16, "IPIs have a number < 16"); + let mut cpu_ctrl = self.0.lock(); - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: send_ipi(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.send_ipi(num as _, match dest { + cpu_ctrl.send_ipi(num as _, match dest { SpecificCpu(cpu) => IpiTargetCpu::Specific(cpu), AllOtherCpus => IpiTargetCpu::AllOtherCpus, }); } fn get_minimum_priority(&self) -> Priority { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_ref() - .expect("BUG: get_minimum_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.get_minimum_priority() + let cpu_ctrl = self.0.lock(); + cpu_ctrl.get_minimum_priority() } fn set_minimum_priority(&self, priority: Priority) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: set_minimum_priority(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.set_minimum_priority(priority) + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.set_minimum_priority(priority) } fn acknowledge_interrupt(&self) -> (InterruptNumber, Priority) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: acknowledge_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - let (num, prio) = int_ctlr.acknowledge_interrupt(); + let mut cpu_ctrl = self.0.lock(); + let (num, prio) = cpu_ctrl.acknowledge_interrupt(); (num as _, prio) } fn end_of_interrupt(&self, number: InterruptNumber) { - let mut int_ctlr = INTERRUPT_CONTROLLER.lock(); - let int_ctlr = int_ctlr - .as_mut() - .expect("BUG: end_of_interrupt(): INTERRUPT_CONTROLLER was uninitialized"); - - int_ctlr.end_of_interrupt(number as _) + let mut cpu_ctrl = self.0.lock(); + cpu_ctrl.end_of_interrupt(number as _) } } diff --git a/kernel/interrupt_controller/src/lib.rs b/kernel/interrupt_controller/src/lib.rs index 7d8c69f459..55c08fe697 100644 --- a/kernel/interrupt_controller/src/lib.rs +++ b/kernel/interrupt_controller/src/lib.rs @@ -1,5 +1,6 @@ #![no_std] #![allow(unused_variables, unused_mut)] +#![feature(array_try_from_fn)] extern crate alloc; @@ -38,6 +39,8 @@ pub enum InterruptDestination { } pub trait SystemInterruptControllerApi { + fn get() -> &'static Self; + fn id(&self) -> SystemInterruptControllerId; fn version(&self) -> SystemInterruptControllerVersion; @@ -55,6 +58,8 @@ pub trait SystemInterruptControllerApi { } pub trait LocalInterruptControllerApi { + fn get() -> &'static Self; + /// Aarch64-specific way to initialize the secondary CPU interfaces. /// /// Must be called once from every secondary CPU. diff --git a/kernel/interrupt_controller/src/x86_64.rs b/kernel/interrupt_controller/src/x86_64.rs index 1b4ca9cfb9..917062e3f4 100644 --- a/kernel/interrupt_controller/src/x86_64.rs +++ b/kernel/interrupt_controller/src/x86_64.rs @@ -31,6 +31,10 @@ pub struct SystemInterruptController { pub struct LocalInterruptController; impl SystemInterruptControllerApi for SystemInterruptController { + fn get() -> &'static Self { + unimplemented!() + } + fn id(&self) -> SystemInterruptControllerId { let mut int_ctlr = get_ioapic(self.id).expect("BUG: id(): get_ioapic() returned None"); SystemInterruptControllerId(int_ctlr.id()) @@ -65,6 +69,10 @@ impl SystemInterruptControllerApi for SystemInterruptController { impl LocalInterruptControllerApi for LocalInterruptController { + fn get() -> &'static Self { + unimplemented!() + } + fn init_secondary_cpu_interface(&self) { panic!("This must not be used on x86_64") } diff --git a/kernel/interrupts/src/aarch64/mod.rs b/kernel/interrupts/src/aarch64/mod.rs index 892d78cd2d..716dd42506 100644 --- a/kernel/interrupts/src/aarch64/mod.rs +++ b/kernel/interrupts/src/aarch64/mod.rs @@ -140,7 +140,7 @@ pub fn init_ap() { set_vbar_el1(); // Enable the CPU-local timer - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.init_secondary_cpu_interface(); int_ctrl.set_minimum_priority(0); @@ -160,7 +160,7 @@ pub fn init() -> Result<(), &'static str> { set_vbar_el1(); - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.set_minimum_priority(0); Ok(()) @@ -178,7 +178,7 @@ pub fn init_timer(timer_tick_handler: InterruptHandler) -> Result<(), &'static s // Route the IRQ to this core (implicit as IRQ < 32) & Enable the interrupt. { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); // enable routing of this interrupt int_ctrl.enable_local_interrupt(CPU_LOCAL_TIMER_IRQ, true); @@ -198,7 +198,7 @@ pub fn setup_ipi_handler(handler: InterruptHandler, local_num: InterruptNumber) } { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); // enable routing of this interrupt int_ctrl.enable_local_interrupt(local_num, true); @@ -209,7 +209,7 @@ pub fn setup_ipi_handler(handler: InterruptHandler, local_num: InterruptNumber) /// Enables the PL011 "RX" SPI and routes it to the current CPU. pub fn init_pl011_rx_interrupt() -> Result<(), &'static str> { - let int_ctrl = SystemInterruptController; + let int_ctrl = SystemInterruptController::get(); int_ctrl.set_destination(PL011_RX_SPI, current_cpu(), u8::MAX) } @@ -295,14 +295,14 @@ pub fn deregister_interrupt(int_num: InterruptNumber, func: InterruptHandler) -> /// Broadcast an Inter-Processor Interrupt to all other /// cores in the system pub fn send_ipi_to_all_other_cpus(irq_num: InterruptNumber) { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.send_ipi(irq_num, InterruptDestination::AllOtherCpus); } /// Send an "end of interrupt" signal, notifying the interrupt chip that /// the given interrupt request `irq` has been serviced. pub fn eoi(irq_num: InterruptNumber) { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.end_of_interrupt(irq_num); } @@ -417,7 +417,7 @@ extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { #[no_mangle] extern "C" fn current_elx_irq(exc: &mut ExceptionContext) { let (irq_num, _priority) = { - let int_ctrl = LocalInterruptController; + let int_ctrl = LocalInterruptController::get(); int_ctrl.acknowledge_interrupt() }; diff --git a/kernel/memory/src/lib.rs b/kernel/memory/src/lib.rs index 600e3063da..a3161bd58e 100644 --- a/kernel/memory/src/lib.rs +++ b/kernel/memory/src/lib.rs @@ -26,6 +26,8 @@ pub use memory_structs::*; pub use page_allocator::{ AllocatedPages, AllocationRequest, + allocate_pages_deferred, + allocate_pages_by_bytes_deferred, allocate_pages, allocate_pages_at, allocate_pages_by_bytes, @@ -37,6 +39,8 @@ pub use page_allocator::{ pub use frame_allocator::{ AllocatedFrames, UnmappedFrames, + allocate_frames_deferred, + allocate_frames_by_bytes_deferred, allocate_frames, allocate_frames_at, allocate_frames_by_bytes, diff --git a/kernel/memory_structs/src/lib.rs b/kernel/memory_structs/src/lib.rs index 41afb6c745..a83afa0f20 100644 --- a/kernel/memory_structs/src/lib.rs +++ b/kernel/memory_structs/src/lib.rs @@ -7,6 +7,7 @@ #![no_std] #![feature(step_trait)] +#![feature(int_roundings)] #![allow(incomplete_features)] #![feature(adt_const_params)] @@ -322,6 +323,15 @@ macro_rules! implement_page_frame { size: PhantomData::, } } + + #[doc = "Returns a new `" $TypeName "` that is aligned up from this \ + `" $TypeName "` to the nearest multiple of `alignment_4k_pages`."] + #[doc(alias = "next_multiple_of")] + pub const fn align_up(&self, alignment_4k_pages: usize) -> $TypeName { + $TypeName { + number: self.number.next_multiple_of(alignment_4k_pages) + } + } } impl

$TypeName

where P: PageSize + 'static { #[doc = "Returns the `" $address "` at the start of this `" $TypeName "`."] diff --git a/kernel/mod_mgmt/src/lib.rs b/kernel/mod_mgmt/src/lib.rs index 5439308ec0..edd569705e 100644 --- a/kernel/mod_mgmt/src/lib.rs +++ b/kernel/mod_mgmt/src/lib.rs @@ -8,8 +8,8 @@ use core::{fmt, ops::{Deref, Range}}; use alloc::{ - collections::{BTreeMap, btree_map, BTreeSet}, - string::{String, ToString}, + collections::{BTreeMap, btree_map, BTreeSet}, + string::{String, ToString}, sync::{Arc, Weak}, vec::Vec }; use spin::{Mutex, Once}; @@ -59,15 +59,15 @@ pub fn get_namespaces_directory() -> Option { /// The thread-local storage (TLS) area "image" that is used as the initial data for each `Task`. /// When spawning a new task, the new task will create its own local TLS area /// with this `TlsInitializer` as the initial data values. -/// +/// /// # Implementation Notes/Shortcomings /// Currently, a single system-wide `TlsInitializer` instance is shared across all namespaces. /// In the future, each namespace should hold its own TLS sections in its TlsInitializer area. -/// +/// /// However, this is quite complex because each namespace must be aware of the TLS sections /// in BOTH its underlying recursive namespace AND its (multiple) "parent" namespace(s) /// that recursively depend on it, since no two TLS sections can conflict (have the same offset). -/// +/// /// Thus, we stick with a singleton `TlsInitializer` instance, which makes sense /// because it behaves much like an allocator, in that it reserves space (index ranges) in the TLS area. static TLS_INITIALIZER: Mutex = Mutex::new(TlsInitializer::empty()); @@ -76,10 +76,10 @@ static TLS_INITIALIZER: Mutex = Mutex::new(TlsInitializer::empty /// Create a new application `CrateNamespace` that uses the default application directory /// and is structured atop the given `recursive_namespace`. /// If no `recursive_namespace` is provided, the default initial kernel namespace will be used. -/// +/// /// # Return /// The returned `CrateNamespace` will itself be empty, having no crates and no symbols in its map. -/// +/// pub fn create_application_namespace(recursive_namespace: Option>) -> Result, &'static str> { // (1) use the initial kernel CrateNamespace as the new app namespace's recursive namespace if none was provided. let recursive_namespace = recursive_namespace @@ -119,10 +119,10 @@ pub fn init( /// and placing them into namespace-specific directories according to their name prefix, e.g., "k#", "ksse#". /// This function does not create any namespaces, it just populates the files and directories /// such that namespaces can be created based on those files. -/// +/// /// If a file does not have an expected crate prefix according to [`CrateType::from_module_name()`], /// then it is treated as part of "extra_files"; see [`parse_extra_file()`] for more. -/// +/// /// Returns a tuple of: /// * the top-level root "namespaces" directory that contains all other namespace directories, /// * the directory of the default kernel crate namespace. @@ -232,14 +232,14 @@ fn parse_bootloader_modules_into_files( } /// Adds the given extra file to the directory of extra files -/// +/// /// See the top-level Makefile target "extra_files" for an explanation of how these work. /// Basically, they are arbitrary files that are included by the bootloader as modules /// (files that exist as areas of pre-loaded memory). -/// +/// /// Their file paths are encoded by flattening directory hierarchies into a the file name, /// using `'!'` (exclamation marks) to replace the directory delimiter `'/'`. -/// +/// /// Thus, for example, a file named `"foo!bar!me!test.txt"` will be placed at the path /// `/extra_files/foo/bar/me/test.txt`. fn parse_extra_file( @@ -248,9 +248,8 @@ fn parse_extra_file( extra_file_mp: MappedPages, extra_files_dir: DirRef ) -> Result { - let mut file_name = extra_file_name; - + let mut parent_dir = extra_files_dir; let mut iter = extra_file_name.split(EXTRA_FILES_DIRECTORY_DELIMITER).peekable(); while let Some(path_component) = iter.next() { @@ -278,7 +277,7 @@ fn parse_extra_file( -/// A "symbol map" from a fully-qualified demangled symbol String +/// A "symbol map" from a fully-qualified demangled symbol String /// to weak reference to a `LoadedSection`. /// This is used for relocations, and for looking up function names. pub type SymbolMap = Trie; @@ -286,9 +285,9 @@ pub type SymbolMap = Trie; /// A wrapper around a `Directory` reference that offers special convenience functions /// for getting and inserting crate object files into a directory. -/// +/// /// Auto-derefs into a `DirRef`. -#[derive(Clone)] +#[derive(Clone)] pub struct NamespaceDir(DirRef); impl Deref for NamespaceDir { @@ -312,10 +311,10 @@ impl NamespaceDir { /// Creates a new `NamespaceDir` that wraps the given `DirRef`. pub fn new(dir: DirRef) -> NamespaceDir { NamespaceDir(dir) - } + } /// Finds the single file in this directory whose name starts with the given `prefix`. - /// + /// /// # Return /// If a single file matches, then that file is returned. /// Otherwise, if no files or multiple files match, then `None` is returned. @@ -347,11 +346,11 @@ impl NamespaceDir { } /// Gets the given object file based on its crate name prefix. - /// + /// /// # Arguments /// * `crate_object_file_name`: the name of the object file to be returned, /// with or without a preceding `CrateType` prefix. - /// + /// /// # Examples /// * The name "k#keyboard-36be916209949cef.o" will look for and return the file "keyboard-36be916209949cef.o". /// * The name "keyboard-36be916209949cef.o" will look for and return the file "keyboard-36be916209949cef.o". @@ -362,12 +361,12 @@ impl NamespaceDir { } /// Insert the given crate object file based on its crate type prefix. - /// + /// /// # Arguments /// * `crate_object_file_name`: the name of the object file to be inserted, /// with a preceding `CrateType` prefix. /// * `content`: the bytes that will be written into the file. - /// + /// /// # Examples /// * The file "k#keyboard-36be916209949cef.o" will be written to "./keyboard-36be916209949cef.o". /// * The file "a#ps.o" will be placed into "./ps.o". @@ -381,7 +380,7 @@ impl NamespaceDir { /// A type that can be converted into a crate object file. -/// +/// /// We use an enum rather than implement `TryInto` because we need additional information /// to resolve a `Prefix`, namely the `CrateNamespace` in which to search for the prefix. pub enum IntoCrateObjectFile { @@ -410,9 +409,9 @@ impl fmt::Debug for IntoCrateObjectFile { /// An application crate that has been loaded into a `CrateNamespace`. -/// +/// /// This type auto-derefs into the application's `StrongCrateRef`. -/// +/// /// When dropped, the application crate will be removed /// from the `CrateNamespace` into which it was originally loaded. pub struct AppCrateRef { @@ -455,11 +454,11 @@ impl Drop for AppCrateRef { /// that have all been loaded and linked against each other, /// completely separate and in isolation from any other crate namespace /// (although a given crate may be shared across multiple namespaces). -/// +/// /// Each `CrateNamespace` can be treated as a separate OS personality, /// but are significantly more efficient than library OS-style personalities. /// A `CrateNamespace` is also useful to create a process (task group) abstraction. -/// +/// /// `CrateNamespace`s can also optionally be recursive. /// For example, a namespace that holds just application crates and symbols /// can recursively rely upon (link against) the crates and symbols in a lower-level namespace @@ -499,7 +498,7 @@ pub struct CrateNamespace { /// that is spawned and runs within this `CrateNamespace`. /// When spawning a new task, the new task will create its own local TLS area /// with this `tls_initializer` as the local data. - /// + /// /// NOTE: this is currently a global system-wide singleton. See the static [`static@TLS_INITIALIZER`] for more. tls_initializer: &'static Mutex, @@ -509,7 +508,7 @@ pub struct CrateNamespace { /// Fuzzy matching should only be successful if there is just a single matching symbol; /// if there are multiple matches (e.g., `my_crate::foo::h456` and `my_crate::foo::h789` both exist), /// then the dependency should fail to be resolved. - /// + /// /// This is a potentially dangerous setting because it overrides the compiler-chosen dependency links. /// Thus, it is false by default, and should only be enabled with expert knowledge, /// ideally only temporarily in order to manually load a given crate. @@ -533,7 +532,7 @@ impl CrateNamespace { symbol_map: Mutex::new(SymbolMap::new()), fuzzy_symbol_matching: false, } - } + } /// Returns the name of this `CrateNamespace`, which is just used for debugging purposes. pub fn name(&self) -> &str { @@ -593,7 +592,7 @@ impl CrateNamespace { /// Iterates over all crates in this namespace and calls the given function `f` on each crate. /// If `recursive` is true, crates in recursive namespaces are included in the iteration as well. - /// + /// /// The function `f` is called with two arguments: the name of the crate, and a reference to the crate. /// The function `f` must return a boolean value that indicates whether to continue iterating; /// if `true`, the iteration will continue, if `false`, the iteration will stop. @@ -619,7 +618,7 @@ impl CrateNamespace { /// Acquires the lock on this `CrateNamespace`'s crate list and returns the crate /// that matches the given `crate_name`, if it exists in this namespace. /// If it does not exist in this namespace, then the recursive namespace is searched as well. - /// + /// /// # Important note about Return value /// Returns a `StrongCrateReference` that **has not** been marked as a shared crate reference, /// so if the caller wants to keep the returned `StrongCrateRef` as a shared crate @@ -638,7 +637,7 @@ impl CrateNamespace { /// This function is similar to the [`get_crate`](#method.get_crate) method, /// but it also returns the `CrateNamespace` in which the crate was found. /// It is an associated function rather than a method so it can operate on `Arc`s. - /// + /// /// # Important note about Return value /// Returns a `StrongCrateReference` that **has not** been marked as a shared crate reference, /// so if the caller wants to keep the returned `StrongCrateRef` as a shared crate @@ -654,18 +653,18 @@ impl CrateNamespace { } /// Finds the `LoadedCrate`s whose names start with the given `crate_name_prefix`. - /// + /// /// # Return /// Returns a list of matching crates, in the form of a tuple containing the crate's name, /// a shallow-cloned reference to the crate, and a reference to the namespace in which the matching crate was found. /// If you want to add the returned crate to another namespace, /// you MUST fully `clone()` the returned crate reference in order to mark that crate as shared across namespaces. - /// + /// /// # Important Usage Note /// To avoid greedily matching more crates than expected, you may wish to end the `crate_name_prefix` with "`-`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`-`". /// This works because the delimiter between a crate name and its trailing hash value is "`-`". - /// + /// /// # Example /// * This `CrateNamespace` contains the crates `my_crate-843a613894da0c24` and /// `my_crate_new-933a635894ce0f12`. @@ -673,7 +672,7 @@ impl CrateNamespace { pub fn get_crates_starting_with<'n>( namespace: &'n Arc, crate_name_prefix: &str - ) -> Vec<(StrRef, StrongCrateRef, &'n Arc)> { + ) -> Vec<(StrRef, StrongCrateRef, &'n Arc)> { // First, we make a list of matching crates in this namespace. let crates = namespace.crate_tree.lock(); let mut crates_in_this_namespace = crates.iter_prefix(crate_name_prefix.as_bytes()) @@ -687,24 +686,24 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. crates_in_this_namespace.append(&mut crates_in_recursive_namespace); - crates_in_this_namespace + crates_in_this_namespace } /// Finds the `LoadedCrate` whose name starts with the given `crate_name_prefix`, /// *if and only if* there is a single matching crate in this namespace or any of its recursive namespaces. /// This is a convenience wrapper around the [`get_crates_starting_with()`](#method.get_crates_starting_with) method. - /// + /// /// # Return /// Returns a tuple containing the crate's name, a shallow-cloned reference to the crate, /// and a reference to the namespace in which the matching crate was found. /// If you want to add the returned crate to another namespace, /// you MUST fully `clone()` the returned crate reference in order to mark that crate as shared across namespaces. - /// + /// /// # Important Usage Note /// To avoid greedily matching more crates than expected, you may wish to end the `crate_name_prefix` with "`-`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`-`". /// This works because the delimiter between a crate name and its trailing hash value is "`-`". - /// + /// /// # Example /// * This `CrateNamespace` contains the crates `my_crate-843a613894da0c24` and /// `my_crate_new-933a635894ce0f12`. @@ -714,20 +713,20 @@ impl CrateNamespace { pub fn get_crate_starting_with<'n>( namespace: &'n Arc, crate_name_prefix: &str - ) -> Option<(StrRef, StrongCrateRef, &'n Arc)> { + ) -> Option<(StrRef, StrongCrateRef, &'n Arc)> { let mut crates_iter = Self::get_crates_starting_with(namespace, crate_name_prefix).into_iter(); crates_iter.next().filter(|_| crates_iter.next().is_none()) // ensure single element } /// Like [`get_crates_starting_with()`](#method.get_crates_starting_with), /// but for crate *object file*s instead of loaded crates. - /// + /// /// Returns a list of matching object files and the namespace in which they were found, /// inclusive of recursive namespaces. pub fn get_crate_object_files_starting_with<'n>( namespace: &'n Arc, file_name_prefix: &str - ) -> Vec<(FileRef, &'n Arc)> { + ) -> Vec<(FileRef, &'n Arc)> { // First, we make a list of matching files in this namespace. let mut files = namespace.dir .get_files_starting_with(file_name_prefix) @@ -742,18 +741,18 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. files.append(&mut files_in_recursive_namespace); - files + files } /// Like [`get_crate_starting_with()`](#method.get_crate_starting_with), /// but for crate *object file*s instead of loaded crates. - /// + /// /// Returns the matching object file and the namespace in which it was found, /// if and only if there was a single match (inclusive of recursive namespaces). pub fn get_crate_object_file_starting_with<'n>( namespace: &'n Arc, file_name_prefix: &str - ) -> Option<(FileRef, &'n Arc)> { + ) -> Option<(FileRef, &'n Arc)> { let mut files_iter = Self::get_crate_object_files_starting_with(namespace, file_name_prefix).into_iter(); files_iter.next().filter(|_| files_iter.next().is_none()) // ensure single element } @@ -762,13 +761,13 @@ impl CrateNamespace { /// Same as `get_crate_object_files_starting_with()`, /// but is a method instead of an associated function, /// and also returns `&CrateNamespace` instead of `&Arc`. - /// + /// /// This is only necessary because I can't figure out how to make a generic function /// that accepts and returns either `&CrateNamespace` or `&Arc`. pub fn method_get_crate_object_files_starting_with( &self, file_name_prefix: &str - ) -> Vec<(FileRef, &CrateNamespace)> { + ) -> Vec<(FileRef, &CrateNamespace)> { // First, we make a list of matching files in this namespace. let mut files = self.dir .get_files_starting_with(file_name_prefix) @@ -783,37 +782,37 @@ impl CrateNamespace { // Third, we combine the lists into one list that spans all namespaces. files.append(&mut files_in_recursive_namespace); - files + files } /// Same as `get_crate_object_file_starting_with()`, /// but is a method instead of an associated function, /// and also returns `&CrateNamespace` instead of `&Arc`. - /// + /// /// This is only necessary because I can't figure out how to make a generic function /// that accepts and returns either `&CrateNamespace` or `&Arc`. pub fn method_get_crate_object_file_starting_with( &self, file_name_prefix: &str - ) -> Option<(FileRef, &CrateNamespace)> { + ) -> Option<(FileRef, &CrateNamespace)> { let mut files_iter = self.method_get_crate_object_files_starting_with(file_name_prefix).into_iter(); files_iter.next().filter(|_| files_iter.next().is_none()) // ensure single element } /// Loads the specified application crate into this `CrateNamespace`, allowing it to be run. - /// + /// /// The new application crate's public symbols are added to this `CrateNamespace`'s symbol map, /// allowing other crates in this namespace to depend upon it. - /// + /// /// Application crates are added to the CrateNamespace just like kernel crates, /// so to load an application crate multiple times to spawn multiple instances of it, /// you can create a new top-level namespace to hold that application crate. - /// + /// /// Returns a Result containing the newly-loaded application crate itself. pub fn load_crate_as_application( namespace: &Arc, - crate_object_file: &FileRef, - kernel_mmi_ref: &MmiRef, + crate_object_file: &FileRef, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result { debug!("load_crate_as_application(): trying to load application crate at {:?}", crate_object_file.lock().get_absolute_path()); @@ -836,7 +835,7 @@ impl CrateNamespace { /// Loads the specified crate into memory, allowing it to be invoked. /// Returns a Result containing the number of symbols that were added to the symbol map /// as a result of loading this crate. - /// + /// /// # Arguments /// * `crate_object_file`: the crate object file that will be loaded into this `CrateNamespace`. /// * `temp_backup_namespace`: the `CrateNamespace` that should be searched for missing symbols @@ -848,21 +847,20 @@ impl CrateNamespace { pub fn load_crate( &self, crate_object_file: &FileRef, - temp_backup_namespace: Option<&CrateNamespace>, - kernel_mmi_ref: &MmiRef, + temp_backup_namespace: Option<&CrateNamespace>, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result<(StrongCrateRef, usize), &'static str> { - #[cfg(not(loscd_eval))] debug!("load_crate: trying to load crate at {:?}", crate_object_file.lock().get_absolute_path()); let new_crate_ref = self.load_crate_internal(crate_object_file, temp_backup_namespace, kernel_mmi_ref, verbose_log)?; - + let (new_crate_name, _num_sections, new_syms) = { let new_crate = new_crate_ref.lock_as_ref(); let new_syms = self.add_symbols(new_crate.sections.values(), verbose_log); (new_crate.crate_name.clone(), new_crate.sections.len(), new_syms) }; - + #[cfg(not(loscd_eval))] info!("loaded new crate {:?}, num sections: {}, added {} new symbols.", new_crate_name, _num_sections, new_syms); self.crate_tree.lock().insert(new_crate_name, new_crate_ref.clone_shallow()); @@ -875,8 +873,8 @@ impl CrateNamespace { /// See [`load_crate`](#method.load_crate) and [`load_crate_as_application`](#fn.load_crate_as_application). fn load_crate_internal(&self, crate_object_file: &FileRef, - temp_backup_namespace: Option<&CrateNamespace>, - kernel_mmi_ref: &MmiRef, + temp_backup_namespace: Option<&CrateNamespace>, + kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> Result { let cf = crate_object_file.lock(); @@ -885,13 +883,13 @@ impl CrateNamespace { Ok(new_crate_ref) } - + /// This function first loads all of the given crates' sections and adds them to the symbol map, /// and only after *all* crates are loaded does it move on to linking/relocation calculations. - /// + /// /// This allows multiple object files with circular dependencies on one another /// to be loaded all at once, as if they were a single entity. - /// + /// /// # Example /// If crate `A` depends on crate `B`, and crate `B` depends on crate `A`, /// this function will load both crate `A` and `B` before trying to resolve their dependencies individually. @@ -901,7 +899,7 @@ impl CrateNamespace { temp_backup_namespace: Option<&CrateNamespace>, kernel_mmi_ref: &MmiRef, verbose_log: bool, - ) -> Result<(), &'static str> + ) -> Result<(), &'static str> where I: Iterator { // First, lock all of the crate object files. @@ -911,13 +909,13 @@ impl CrateNamespace { } // Second, do all of the section parsing and loading, and add all public symbols to the symbol map. - let mut partially_loaded_crates: Vec<(StrongCrateRef, ElfFile)> = Vec::with_capacity(locked_crate_files.len()); - for locked_crate_file in &locked_crate_files { + let mut partially_loaded_crates: Vec<(StrongCrateRef, ElfFile)> = Vec::with_capacity(locked_crate_files.len()); + for locked_crate_file in &locked_crate_files { let (new_crate_ref, elf_file) = self.load_crate_sections(locked_crate_file.deref(), kernel_mmi_ref, verbose_log)?; let _new_syms = self.add_symbols(new_crate_ref.lock_as_ref().sections.values(), verbose_log); partially_loaded_crates.push((new_crate_ref, elf_file)); } - + // Finally, we do all of the relocations. for (new_crate_ref, elf_file) in partially_loaded_crates { self.perform_relocations(&elf_file, &new_crate_ref, temp_backup_namespace, kernel_mmi_ref, verbose_log)?; @@ -932,11 +930,11 @@ impl CrateNamespace { /// Duplicates this `CrateNamespace` into a new `CrateNamespace`, /// but uses a copy-on-write/clone-on-write semantic that creates /// a special shared reference to each crate that indicates it is shared across multiple namespaces. - /// + /// /// In other words, crates in the new namespace returned by this fucntions /// are fully shared with crates in *this* namespace, /// until either namespace attempts to modify a shared crate in the future. - /// + /// /// When modifying crates in the new namespace, e.g., swapping crates, /// any crates in the new namespace that are still shared with the old namespace /// must be deeply copied into a new crate that is exclusively owned, @@ -949,7 +947,7 @@ impl CrateNamespace { /// that now depend on `A2` instead of `A`. /// The existing versions of `B` and `C` would still depend on `A`, /// but they would no longer be part of the new namespace. - /// + /// pub fn clone_on_write(&self) -> CrateNamespace { CrateNamespace { name: self.name.clone(), @@ -972,7 +970,6 @@ impl CrateNamespace { new_section: &StrongSectionRef, kernel_mmi_ref: &MmiRef ) -> Result<(), &'static str> { - for weak_dep in &old_section.inner.read().sections_dependent_on_me { let target_sec = weak_dep.section.upgrade().ok_or("couldn't upgrade WeakDependent.section")?; let relocation_entry = weak_dep.relocation; @@ -1001,7 +998,7 @@ impl CrateNamespace { target_sec_mapped_pages.remap(&mut kernel_mmi_ref.lock().page_table, target_sec_initial_flags)?; }; } - + // Tell the new source_sec that the existing target_sec depends on it. // Note that we don't need to do this if we're re-swapping in a cached crate, // because that crate's sections' dependents are already properly set up from when it was first swapped in. @@ -1036,13 +1033,13 @@ impl CrateNamespace { /// The primary internal routine for parsing and loading all sections in a crate object file. /// This does not perform any relocations or linking, so the crate **is not yet ready to use after this function**, /// since its sections are totally incomplete and non-executable. - /// + /// /// However, it does add all of the newly-loaded crate sections to the symbol map (yes, even before relocation/linking), /// since we can use them to resolve missing symbols for relocations. - /// + /// /// Parses each section in the given `crate_file` object file and copies its contents to each section. /// Returns a tuple of a reference to the new `LoadedCrate` and the crate's ELF file (to avoid having to re-parse it). - /// + /// /// # Arguments /// * `crate_file`: the object file for the crate that will be loaded into this `CrateNamespace`. /// * `kernel_mmi_ref`: the kernel's MMI struct, for memory mapping use. @@ -1053,7 +1050,6 @@ impl CrateNamespace { kernel_mmi_ref: &MmiRef, _verbose_log: bool ) -> Result<(StrongCrateRef, ElfFile<'f>), &'static str> { - let mapped_pages = crate_file.as_mapping()?; let size_in_bytes = crate_file.len(); let abs_path = Path::new(crate_file.get_absolute_path()); @@ -1070,7 +1066,7 @@ impl CrateNamespace { // It's probably better to pass in the actual crate file reference so we can use it here, // but since we don't currently do that, we just get another reference to the crate object file via its Path. let crate_object_file = match Path::get_absolute(&abs_path) { - Some(FileOrDir::File(f)) => f, + Some(FileOrDir::File(f)) => f, _ => return Err("BUG: load_crate_sections(): couldn't get crate object file path"), }; @@ -1104,7 +1100,7 @@ impl CrateNamespace { let new_crate = CowArc::new(LoadedCrate { crate_name, debug_symbols_file: Arc::downgrade(&crate_object_file), - object_file: crate_object_file, + object_file: crate_object_file, sections: HashMap::new(), text_pages: text_pages.clone(), rodata_pages: rodata_pages.clone(), @@ -1116,7 +1112,7 @@ impl CrateNamespace { }); let new_crate_weak_ref = CowArc::downgrade(&new_crate); - let load_sections_fn = if sections_are_merged { + let load_sections_fn = if sections_are_merged { Self::load_crate_with_merged_sections } else { Self::load_crate_with_separate_sections @@ -1152,10 +1148,10 @@ impl CrateNamespace { /// An internal routine to load and populate the sections of a crate's object file /// if those sections have already been merged. - /// + /// /// This is the "new, acclerated" way to load sections, and is used by `load_crate_sections()` /// for object files that **have** been modified by Theseus's special partial relinking script. - /// + /// /// This works by iterating over all symbols in the object file /// and creating section entries for each one of those symbols only. /// The actual section data can be loaded quickly because they have been merged into top-level sections, @@ -1168,7 +1164,7 @@ impl CrateNamespace { rodata_pages: Option<(Arc>, Range)>, data_pages: Option<(Arc>, Range)>, ) -> Result { - + let mut text_pages_locked = text_pages .as_ref().map(|(tp, tp_range)| (tp.clone(), tp.lock(), tp_range.start)); let mut read_only_pages_locked = rodata_pages.as_ref().map(|(rp, rp_range)| (rp.clone(), rp.lock(), rp_range.start)); let mut read_write_pages_locked = data_pages .as_ref().map(|(dp, dp_range)| (dp.clone(), dp.lock(), dp_range.start)); @@ -1178,7 +1174,7 @@ impl CrateNamespace { let mut read_only_offset: Option = None; // The section header offset of the first read-write section, which is .data or .bss let mut read_write_offset: Option = None; - + // We need to track various section `shndx`s to differentiate between // the different types of "OBJECT" symbols and "TLS" symbols. // @@ -1194,7 +1190,7 @@ impl CrateNamespace { let mut tbss_shndx_and_section: Option<(Shndx, StrongSectionRef)> = None; // The set of `LoadedSections` that will be parsed and populated into this `new_crate`. - let mut loaded_sections: HashMap = HashMap::new(); + let mut loaded_sections: HashMap = HashMap::new(); let mut data_sections: BTreeSet = BTreeSet::new(); let mut tls_sections: BTreeSet = BTreeSet::new(); let mut last_shndx = 0; @@ -1207,7 +1203,7 @@ impl CrateNamespace { let sec_flags = sec.flags(); // Skip non-allocated sections, because they don't appear in the loaded object file. if sec_flags & SHF_ALLOC == 0 { - continue; + continue; } // get the relevant section info, i.e., size, alignment, and data contents @@ -1290,7 +1286,7 @@ impl CrateNamespace { .map(|(rp_ref, rp, _)| (rp_ref, rp)) .ok_or("BUG: ELF file contained a .tdata/.tbss section, but no rodata_pages were allocated")?; // Use a placeholder vaddr; it will be replaced in `add_new_dynamic_tls_section()` below. - virt_addr = VirtualAddress::zero(); + virt_addr = VirtualAddress::zero(); tls_sections.insert(shndx); } @@ -1453,9 +1449,9 @@ impl CrateNamespace { typ = SectionType::Rodata; mapped_pages = rp_ref; // no additional offset below, because .rodata is always the first read-only section. - mapped_pages_offset = sec_value; + mapped_pages_offset = sec_value; virt_addr = rp_start_vaddr + mapped_pages_offset; - } + } // Handle .data/.bss symbol else { data_sections.insert(last_shndx); @@ -1538,7 +1534,7 @@ impl CrateNamespace { if is_global { global_sections.insert(last_shndx); } - + last_shndx += 1; } // end of iterating over all symbol table entries @@ -1581,7 +1577,7 @@ impl CrateNamespace { // } // } - Ok(SectionMetadata { + Ok(SectionMetadata { loaded_sections, global_sections, tls_sections, @@ -1592,10 +1588,10 @@ impl CrateNamespace { /// An internal routine to load and populate the sections of a crate's object file /// if those sections have not been merged. - /// + /// /// This is the "legacy" way to load sections, and is used by `load_crate_sections()` /// for object files that have **not** been modified by Theseus's special partial relinking script. - /// + /// /// This works by iterating over all section headers in the object file /// and extracting the symbol names from each section name, which is quite slow. fn load_crate_with_separate_sections( @@ -1605,8 +1601,8 @@ impl CrateNamespace { text_pages: Option<(Arc>, Range)>, rodata_pages: Option<(Arc>, Range)>, data_pages: Option<(Arc>, Range)>, - ) -> Result { - + ) -> Result { + // Check the symbol table to get the set of sections that are global (publicly visible). let global_sections: BTreeSet = { // For us to properly load the ELF file, it must NOT have been fully stripped, @@ -1619,7 +1615,7 @@ impl CrateNamespace { // Include all symbols with "GLOBAL" binding, regardless of visibility. if entry.get_binding() == Ok(xmas_elf::symbol_table::Binding::Global) { match entry.get_type() { - Ok(xmas_elf::symbol_table::Type::Func + Ok(xmas_elf::symbol_table::Type::Func | xmas_elf::symbol_table::Type::Object | xmas_elf::symbol_table::Type::Tls) => { globals.insert(entry.shndx() as Shndx); @@ -1628,7 +1624,7 @@ impl CrateNamespace { } } } - globals + globals }; // Since .text sections come at the beginning of the object file, @@ -1655,7 +1651,7 @@ impl CrateNamespace { // we copy them into their respective pages individually on a per-section basis, // keeping track of the offset into each of their MappedPages as we go. let (mut rodata_offset, mut data_offset) = (0 , 0); - + const TEXT_PREFIX: &str = ".text."; const UNLIKELY_PREFIX: &str = "unlikely."; // the full section prefix is ".text.unlikely." const RODATA_PREFIX: &str = ".rodata."; @@ -1669,12 +1665,12 @@ impl CrateNamespace { /// A convenient macro to obtain the rest of the symbol name after its prefix, /// i.e., the characters after '.text', '.rodata', '.data', etc. - /// + /// /// * If the name isn't long enough, the macro prints and returns an error str. /// * If the name isn't long enough but is an empty section (e.g., just ".text", ".rodata", etc) /// this macro `continue`s to the next iteration of the loop. /// * The `$prefix` argument must be `const` so it can be `concat!()`-ed into a const &str. - /// + /// /// Note: I'd prefer this to be a const function that accepts the prefix as a const &'static str, /// but Rust does not support concat!()-ing const generic parameters yet. macro_rules! try_get_symbol_name_after_prefix { @@ -1702,7 +1698,7 @@ impl CrateNamespace { } // this maps section header index (shndx) to LoadedSection - let mut loaded_sections: HashMap = HashMap::new(); + let mut loaded_sections: HashMap = HashMap::new(); // the set of Shndxes for .data and .bss sections let mut data_sections: BTreeSet = BTreeSet::new(); // the set of Shndxes for TLS sections (.tdata, .tbss) @@ -1718,7 +1714,7 @@ impl CrateNamespace { let sec_flags = sec.flags(); // Skip non-allocated sections, because they don't appear in the loaded object file. if sec_flags & SHF_ALLOC == 0 { - continue; + continue; } // Even if we're using the next section's data (for a zero-sized section, as handled below), @@ -1791,7 +1787,7 @@ impl CrateNamespace { let dest_vaddr = tp_range.start + text_offset; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( SectionType::Text, demangled, @@ -1853,7 +1849,7 @@ impl CrateNamespace { new_crate.clone(), ); // trace!("Loaded new TLS section: {:?}", new_tls_section); - + // Add the new TLS section to this namespace's initial TLS area, // which will reserve/obtain a new offset into that TLS area which holds this section's data. // This will also update the section's virtual address field to hold that offset value, @@ -1891,7 +1887,7 @@ impl CrateNamespace { // }) }; let demangled = demangle(name).to_string().as_str().into(); - + if let Some((ref dp_ref, ref mut dp)) = read_write_pages_locked { // here: we're ready to copy the data/bss section to the proper address let dest_vaddr = dp.address_at_offset(data_offset) @@ -1905,7 +1901,7 @@ impl CrateNamespace { return Err("couldn't get section data in .data section"); } } - + loaded_sections.insert( shndx, Arc::new(LoadedSection::new( @@ -1946,9 +1942,9 @@ impl CrateNamespace { return Err("couldn't get section data in .rodata section"); } } - + loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( SectionType::Rodata, demangled, @@ -1993,7 +1989,7 @@ impl CrateNamespace { let typ = SectionType::GccExceptTable; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( typ, section_name_str_ref(&typ), @@ -2032,7 +2028,7 @@ impl CrateNamespace { let typ = SectionType::EhFrame; loaded_sections.insert( - shndx, + shndx, Arc::new(LoadedSection::new( typ, section_name_str_ref(&typ), @@ -2063,7 +2059,7 @@ impl CrateNamespace { } } - Ok(SectionMetadata { + Ok(SectionMetadata { loaded_sections, global_sections, tls_sections, @@ -2071,7 +2067,7 @@ impl CrateNamespace { }) } - + /// The second stage of parsing and loading a new kernel crate, /// filling in the missing relocation information in the already-loaded sections. /// It also remaps the `new_crate`'s MappedPages according to each of their section permissions. @@ -2092,9 +2088,9 @@ impl CrateNamespace { // Iterate over every non-zero relocation section in the file for sec in elf_file.section_iter().filter(|sec| sec.get_type() == Ok(ShType::Rela) && sec.size() != 0) { use xmas_elf::sections::SectionData::Rela64; - if verbose_log { + if verbose_log { trace!("Found Rela section name: {:?}, type: {:?}, target_sec_index: {:?}", - sec.get_name(elf_file), sec.get_type(), sec.info()); + sec.get_name(elf_file), sec.get_type(), sec.info()); } // Debug sections are handled separately @@ -2109,23 +2105,23 @@ impl CrateNamespace { _ => { error!("Found Rela section that wasn't able to be parsed as Rela64: {:?}", sec); return Err("Found Rela section that wasn't able to be parsed as Rela64"); - } + } }; // The target section is where we write the relocation data to. // The source section is where we get the data from. // There is one target section per rela section (`rela_array`), and one source section per rela_entry in this rela section. // The "info" field in the Rela section specifies which section is the target of the relocation. - + // Get the target section (that we already loaded) for this rela_array Rela section. let target_sec_shndx = sec.info() as usize; let target_sec = new_crate.sections.get(&target_sec_shndx).ok_or_else(|| { error!("ELF file error: target section was not loaded for Rela section {:?}!", sec.get_name(elf_file)); "target section was not loaded for Rela section" - })?; + })?; let mut target_sec_data_was_modified = false; - + let mut target_sec_dependencies: Vec = Vec::new(); #[cfg(internal_deps)] let mut target_sec_internal_dependencies: Vec = Vec::new(); @@ -2138,16 +2134,16 @@ impl CrateNamespace { // iterate through each relocation entry in the relocation array for the target_sec for rela_entry in rela_array { - if verbose_log { + if verbose_log { trace!(" Rela64 offset: {:#X}, addend: {:#X}, symtab_index: {}, type: {:#X}", rela_entry.get_offset(), rela_entry.get_addend(), rela_entry.get_symbol_table_index(), rela_entry.get_type()); } use xmas_elf::symbol_table::Entry; let source_sec_entry = &symtab[rela_entry.get_symbol_table_index() as usize]; - let source_sec_shndx = source_sec_entry.shndx() as usize; + let source_sec_shndx = source_sec_entry.shndx() as usize; let source_sec_value = source_sec_entry.value() as usize; - if verbose_log { + if verbose_log { let source_sec_header_name = source_sec_entry.get_section_header(elf_file, rela_entry.get_symbol_table_index() as usize) .and_then(|s| s.get_name(elf_file)); trace!(" relevant section [{}]: {:?}, value: {:#X}", source_sec_shndx, source_sec_header_name, source_sec_value); @@ -2156,7 +2152,7 @@ impl CrateNamespace { // source_sec_entry.get_other(), source_sec_entry.get_binding(), source_sec_entry.get_type(), // source_sec_entry.shndx(), source_sec_entry.value(), source_sec_entry.size()); } - + let mut source_and_target_in_same_crate = false; // We first try to get the source section from loaded_sections, which works if the section is in the crate currently being loaded. @@ -2219,13 +2215,13 @@ impl CrateNamespace { relocation: relocation_entry, }; source_sec.inner.write().sections_dependent_on_me.push(weak_dep); - + // tell the target_sec that it has a strong dependency on the source_sec let strong_dep = StrongDependency { section: Arc::clone(&source_sec), relocation: relocation_entry, }; - target_sec_dependencies.push(strong_dep); + target_sec_dependencies.push(strong_dep); } } } @@ -2233,7 +2229,7 @@ impl CrateNamespace { // If the target section of the relocation was a TLS section, // that TLS section's initializer data has now changed. // Thus, we need to invalidate the TLS initializer area's cached data. - if target_sec_data_was_modified && + if target_sec_data_was_modified && (target_sec.typ == SectionType::TlsData || target_sec.typ == SectionType::TlsBss) { // debug!("Invalidating TlsInitializer due to relocation written to section {:?}", &*target_sec); @@ -2253,7 +2249,7 @@ impl CrateNamespace { // We need to remap each section's mapped pages with the proper permission bits, // since we initially mapped them all as writable. - if let Some(ref tp) = new_crate.text_pages { + if let Some(ref tp) = new_crate.text_pages { tp.0.lock().remap(&mut kernel_mmi_ref.lock().page_table, TEXT_SECTION_FLAGS)?; } if let Some(ref rp) = new_crate.rodata_pages { @@ -2265,13 +2261,13 @@ impl CrateNamespace { // By default, we can safely remove the metadata for all private (non-global) .rodata sections // that do not have any strong dependencies (its `sections_i_depend_on` list is empty). // If you want all sections to be kept, e.g., for debugging, you can set the below cfg option. - #[cfg(not(keep_private_rodata))] + #[cfg(not(keep_private_rodata))] { new_crate.sections.retain(|_shndx, sec| { - let should_remove = !sec.global + let should_remove = !sec.global && sec.typ == SectionType::Rodata && sec.inner.read().sections_i_depend_on.is_empty(); - + // For an element to be removed, this closure should return `false`. !should_remove }); @@ -2280,7 +2276,7 @@ impl CrateNamespace { Ok(()) } - + /// Adds the given symbol to this namespace's symbol map. /// If the symbol already exists in the symbol map, this replaces the existing symbol with the new one, warning if they differ in size. /// Returns true if the symbol was added, and false if it already existed and thus was merely replaced. @@ -2297,7 +2293,7 @@ impl CrateNamespace { // debug!(" add_symbol(): replacing section: old: {:?}, new: {:?}", old_sec, new_section); if new_section.size != old_sec.size { warn!("Unexpectedly replacing differently-sized section: old: ({}B) {:?}, new: ({}B) {:?}", old_sec.size, old_sec.name, new_section.size, new_section.name); - } + } else { warn!("Replacing new symbol already present: old {:?}, new: {:?}", old_sec.name, new_section.name); } @@ -2307,7 +2303,7 @@ impl CrateNamespace { false } qp_trie::Entry::Vacant(new_entry) => { - if log_replacements { + if log_replacements { debug!(" add_symbol(): Adding brand new symbol: new: {:?}", new_section); } new_entry.insert(Arc::downgrade(new_section)); @@ -2317,12 +2313,12 @@ impl CrateNamespace { } /// Adds only *global* symbols in the given `sections` iterator to this namespace's symbol map, - /// + /// /// If a symbol already exists in the symbol map, this replaces the existing symbol but does not count it as a newly-added one. - /// + /// /// Returns the number of *new* unique symbols added. pub fn add_symbols<'a, I>( - &self, + &self, sections: I, _log_replacements: bool, ) -> usize @@ -2334,12 +2330,12 @@ impl CrateNamespace { /// Adds symbols in the given `sections` iterator to this namespace's symbol map, /// but only sections that are *global* AND for which the given `filter_func` returns true. - /// + /// /// If a symbol already exists in the symbol map, this replaces the existing symbol but does not count it as a newly-added one. - /// + /// /// Returns the number of *new* unique symbols added. fn add_symbols_filtered<'a, I, F>( - &self, + &self, sections: I, filter_func: F, log_replacements: bool, @@ -2361,34 +2357,34 @@ impl CrateNamespace { } } } - + count } - + /// Finds the crate that contains the given `VirtualAddress` in its loaded code. - /// + /// /// By default, only executable sections (`.text`) are searched, since typically the only use case /// for this function is to search for an instruction pointer (program counter) address. /// However, if `search_all_section_types` is `true`, both the read-only and read-write sections /// will be included in the search, e.g., `.rodata`, `.data`, `.bss`. - /// + /// /// # Usage /// This is mostly useful for printing symbol names for a stack trace (backtrace). /// It is also similar in functionality to the tool `addr2line`, /// but gives the section itself rather than the line of code. - /// + /// /// # Locking /// This can obtain the lock on every crate and every section, /// so to avoid deadlock, please ensure that the caller task does not hold any such locks. /// It does *not* need to obtain locks on the underlying `MappedPages` regions. - /// + /// /// # Note /// This is a slow procedure because, in the worst case, /// it will iterate through **every** loaded crate in this namespace (and its recursive namespace). pub fn get_crate_containing_address( - &self, - virt_addr: VirtualAddress, + &self, + virt_addr: VirtualAddress, search_all_section_types: bool, ) -> Option { @@ -2396,7 +2392,7 @@ impl CrateNamespace { let crate_contains_vaddr = |crate_ref: &StrongCrateRef| { let krate = crate_ref.lock_as_ref(); if let Some(ref tp) = krate.text_pages { - if tp.1.contains(&virt_addr) { + if tp.1.contains(&virt_addr) { return true; } } @@ -2414,7 +2410,7 @@ impl CrateNamespace { } false }; - + let mut found_crate = None; // Here, we didn't find the symbol when searching from the starting crate, @@ -2434,29 +2430,29 @@ impl CrateNamespace { /// Finds the section that contains the given `VirtualAddress` in its loaded code. - /// + /// /// By default, only executable sections (`.text`) are searched, since the typical use case /// for this function is to search for an instruction pointer (program counter) address. /// However, if `search_all_section_types` is `true`, both the read-only and read-write sections /// will be included in the search, e.g., `.rodata`, `.data`, `.bss`. - /// + /// /// # Usage /// This is mostly useful for printing symbol names for a stack trace (backtrace). /// It is also similar in functionality to the tool `addr2line`, /// but gives the section itself rather than the line of code. - /// + /// /// # Locking /// This can obtain the lock on every crate in this namespace and its recursive namespaces, /// so to avoid deadlock, please ensure that the caller task does not hold any such locks. - /// + /// /// # Note /// This is a slow procedure because, in the worst case, /// it will iterate through **every** section in **every** loaded crate /// in this namespace (and its recursive namespace), /// not just the publicly-visible (global) sections. pub fn get_section_containing_address( - &self, - virt_addr: VirtualAddress, + &self, + virt_addr: VirtualAddress, search_all_section_types: bool, ) -> Option<(StrongSectionRef, usize)> { @@ -2475,7 +2471,7 @@ impl CrateNamespace { for sec in crate_locked.sections.values() { // .text sections are always included, other sections are included if requested. let eligible_section = sec.typ == SectionType::Text || search_all_section_types; - + // If the section's address bounds contain the address, then we've found it. // Only a single section can contain the address, so it's safe to stop once we've found a match. if eligible_section @@ -2484,7 +2480,7 @@ impl CrateNamespace { { let offset = virt_addr.value() - sec.virt_addr.value(); merged_section_and_offset = Some((sec.clone(), offset)); - + if sec.name.as_str() == sec.typ.name() { // If this section is a merged section, it will have the standard name // for its section type, e.g., ".text", ".data", ".rodata", etc. @@ -2527,23 +2523,23 @@ impl CrateNamespace { /// similar to the simpler function `get_symbol()`, but takes the additional step of trying to /// automatically find and/or load the crate containing that symbol /// (and does so recursively for any of its crate dependencies). - /// + /// /// (1) First, it recursively searches this namespace's and its recursive namespaces' symbol maps, /// and returns the symbol if already loaded. - /// + /// /// (2) Second, if the symbol is missing from this namespace, it looks in the `temp_backup_namespace`. /// If we find it there, then we add that symbol and its containing crate as a shared crate in this namespace. - /// + /// /// (3) Third, if this namespace has `fuzzy_symbol_matching` enabled, it searches the backup namespace /// for symbols that match the given `demangled_full_symbol` without the hash suffix. - /// + /// /// (4) Fourth, if the missing symbol isn't in the backup namespace either, /// try to load its containing crate from the object file. /// This can only be done for symbols that have a leading crate name, such as "my_crate::foo"; /// if a symbol was given the `no_mangle` attribute, then we will not be able to find it, /// and that symbol's containing crate should be manually loaded before invoking this. - /// - /// + /// + /// /// # Arguments /// * `demangled_full_symbol`: a fully-qualified symbol string, e.g., "my_crate::MyStruct::foo::h843a9ea794da0c24". /// * `temp_backup_namespace`: the `CrateNamespace` that should be temporarily searched (just during this call) @@ -2551,9 +2547,9 @@ impl CrateNamespace { /// If `temp_backup_namespace` is `None`, then only this namespace (and its recursive namespaces) will be searched. /// * `kernel_mmi_ref`: a reference to the kernel's `MemoryManagementInfo`, which must not be locked. pub fn get_symbol_or_load( - &self, - demangled_full_symbol: &str, - temp_backup_namespace: Option<&CrateNamespace>, + &self, + demangled_full_symbol: &str, + temp_backup_namespace: Option<&CrateNamespace>, kernel_mmi_ref: &MmiRef, verbose_log: bool ) -> WeakSectionRef { @@ -2597,7 +2593,7 @@ impl CrateNamespace { /// Looks for the given `demangled_full_symbol` in the `temp_backup_namespace` and returns a reference to the matching section. - /// + /// /// This is the second and third attempts to find a symbol within [`get_symbol_or_load()`](#method.get_symbol_or_load). fn get_symbol_from_backup_namespace( &self, @@ -2636,7 +2632,7 @@ impl CrateNamespace { })?; // Here, we found the matching section in the temp_backup_namespace. - let parent_crate_ref = { + let parent_crate_ref = { sec.parent_crate.upgrade().or_else(|| { error!("BUG: Found symbol \"{}\" in backup namespace, but unexpectedly couldn't get its parent crate!", demangled_full_symbol); None @@ -2652,7 +2648,7 @@ impl CrateNamespace { self.add_symbols(Some(sec.clone()).iter(), verbose_log); parent_crate.crate_name.clone() }; - + #[cfg(not(loscd_eval))] info!("Symbol {:?} not initially found, using {}symbol {} from crate {:?} in backup namespace {:?} in new namespace {:?}", demangled_full_symbol, @@ -2671,24 +2667,24 @@ impl CrateNamespace { /// Attempts to find and load the crate that may contain the given `demangled_full_symbol`. - /// + /// /// If successful, the new crate is loaded into this `CrateNamespace` and the symbol's section is returned. /// If this namespace does not contain any matching crates, its recursive namespaces are searched as well. - /// + /// /// This approach only works for mangled symbols that contain a crate name, such as "my_crate::foo". /// If "foo()" was marked no_mangle, then we don't know which crate to load because there is no "my_crate::" prefix before it. - /// + /// /// Note: while attempting to find the missing `demangled_full_symbol`, this function may end up /// loading *multiple* crates into this `CrateNamespace` or its recursive namespaces, due to two reasons: /// 1. The `demangled_full_symbol` may have multiple crate prefixes within it. /// * For example, `::drop::h55e0a4c312ccdd63` /// contains two possible crate prefixes: `page_allocator` and `core`. /// 2. There may be multiple versions of a single crate. - /// + /// /// Possible crates are iteratively loaded and searched until the missing symbol is found. /// Currently, crates that were loaded but did *not* contain the missing symbol are *not* unloaded, /// but you could manually unload them later with no adverse effects to reclaim memory. - /// + /// /// This is the final attempt to find a symbol within [`CrateNamespace::get_symbol_or_load()`]. fn load_crate_for_missing_symbol( &self, @@ -2700,7 +2696,7 @@ impl CrateNamespace { // Some symbols may have multiple potential containing crates, so we try to load each one to find the missing symbol. for potential_crate_name in get_containing_crate_name(demangled_full_symbol) { let potential_crate_name = format!("{potential_crate_name}-"); - + // Try to find and load the missing crate object file from this namespace's directory or its recursive namespace's directory, // (or from the backup namespace's directory set). // The object files from the recursive namespace(s) are appended after the files in the initial namespace, @@ -2733,7 +2729,7 @@ impl CrateNamespace { // We *could* return an error here, but we might as well continue on to trying to load other crates. } } - } + } } warn!("Couldn't find/load crate(s) that may contain the missing symbol {:?}", demangled_full_symbol); @@ -2744,16 +2740,16 @@ impl CrateNamespace { /// Returns a copied list of the corresponding `LoadedSection`s /// with names that start with the given `symbol_prefix`. /// This will also search the recursive namespace's symbol map. - /// + /// /// This method causes allocation because it creates a copy /// of the matching entries in the symbol map. - /// + /// /// # Example /// The symbol map contains `my_crate::foo::h843a613894da0c24` and /// `my_crate::foo::h933a635894ce0f12`. /// Calling `find_symbols_starting_with("my_crate::foo")` will return /// a vector containing both sections, which can then be iterated through. - pub fn find_symbols_starting_with(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef)> { + pub fn find_symbols_starting_with(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef)> { let mut syms: Vec<(String, WeakSectionRef)> = self.symbol_map.lock() .iter_prefix(symbol_prefix.as_bytes()) .map(|(k, v)| (String::from(k.as_str()), v.clone())) @@ -2769,7 +2765,7 @@ impl CrateNamespace { /// Similar to `find_symbols_starting_with`, but also includes a reference to the exact `CrateNamespace` /// where the matching symbol was found. - pub fn find_symbols_starting_with_and_namespace(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef, &CrateNamespace)> { + pub fn find_symbols_starting_with_and_namespace(&self, symbol_prefix: &str) -> Vec<(String, WeakSectionRef, &CrateNamespace)> { let mut syms: Vec<(String, WeakSectionRef, &CrateNamespace)> = self.symbol_map.lock() .iter_prefix(symbol_prefix.as_bytes()) .map(|(k, v)| (String::from(k.as_str()), v.clone(), self)) @@ -2786,12 +2782,12 @@ impl CrateNamespace { /// Returns a weak reference to the `LoadedSection` whose name beings with the given `symbol_prefix`, /// *if and only if* the symbol map only contains a single possible matching symbol. /// This will also search the recursive namespace's symbol map. - /// + /// /// # Important Usage Note /// To avoid greedily matching more symbols than expected, you may wish to end the `symbol_prefix` with "`::`". /// This may provide results more in line with the caller's expectations; see the last example below about a trailing "`::`". /// This works because the delimiter between a symbol and its trailing hash value is "`::`". - /// + /// /// # Example /// * The symbol map contains `my_crate::foo::h843a613894da0c24` /// and no other symbols that start with `my_crate::foo`. @@ -2808,21 +2804,21 @@ impl CrateNamespace { /// because it will match both `foo` and `foo_new`. /// To match only `foo`, call this function as `get_symbol_starting_with("my_crate::foo::")` /// (note the trailing "`::`"). - pub fn get_symbol_starting_with(&self, symbol_prefix: &str) -> WeakSectionRef { + pub fn get_symbol_starting_with(&self, symbol_prefix: &str) -> WeakSectionRef { self.get_symbol_starting_with_internal(symbol_prefix) .unwrap_or_default() } /// This is an internal version of method: [`get_symbol_starting_with()`](#method.get_symbol_starting_with) /// that returns an Option to allow easier recursive use. - fn get_symbol_starting_with_internal(&self, symbol_prefix: &str) -> Option { + fn get_symbol_starting_with_internal(&self, symbol_prefix: &str) -> Option { // First, we see if there's a single matching symbol in this namespace. let map = self.symbol_map.lock(); let mut iter = map.iter_prefix(symbol_prefix.as_bytes()).map(|tuple| tuple.1); let symbol_in_this_namespace = iter.next() .filter(|_| iter.next().is_none()) // ensure single element .cloned(); - + // Second, we see if there's a single matching symbol in the recursive namespace. let symbol_in_recursive_namespace = self.recursive_namespace.as_ref().and_then(|r_ns| r_ns.get_symbol_starting_with_internal(symbol_prefix)); @@ -2830,7 +2826,7 @@ impl CrateNamespace { symbol_in_this_namespace.xor(symbol_in_recursive_namespace) } - + /// Simple debugging function that returns the entire symbol map as a String. /// This includes only symbols from this namespace, and excludes symbols from recursive namespaces. pub fn dump_symbol_map(&self) -> String { @@ -2990,7 +2986,7 @@ fn allocate_section_pages(elf_file: &ElfFile, kernel_mmi_ref: &MmiRef) -> Result allocate_pages_by_bytes(size_in_bytes) .ok_or("Couldn't allocate pages for new section")? }; - + kernel_mmi_ref.lock().page_table.map_allocated_pages( allocated_pages, flags.valid(true).writable(true) @@ -3059,7 +3055,7 @@ fn dump_weak_dependents(sec: &LoadedSection, prefix: String) { /// Returns a reference to the symbol table in the given `ElfFile`. -pub fn find_symbol_table<'e>(elf_file: &'e ElfFile) +pub fn find_symbol_table<'e>(elf_file: &'e ElfFile) -> Result<&'e [xmas_elf::symbol_table::Entry64], &'static str> { use xmas_elf::sections::SectionData::SymbolTable64; diff --git a/kernel/page_allocator/src/lib.rs b/kernel/page_allocator/src/lib.rs index ef01aeed00..45c1f0890c 100644 --- a/kernel/page_allocator/src/lib.rs +++ b/kernel/page_allocator/src/lib.rs @@ -38,7 +38,7 @@ use static_array_rb_tree::*; /// Certain regions are pre-designated for special usage, specifically the kernel's initial identity mapping. -/// They will be allocated from if an address within them is specifically requested; +/// They will be allocated from if an address within them is specifically /// otherwise, they will only be allocated from as a "last resort" if all other non-designated address ranges are exhausted. /// /// Any virtual addresses **less than or equal** to this address are considered "designated". @@ -803,10 +803,15 @@ fn find_specific_chunk( /// If no range is specified, this function first attempts to find a suitable chunk /// that is **not** within the designated regions, /// and only allocates from the designated regions as a backup option. +/// +/// If an alignment is specified (in terms of number of 4KiB pages), then the starting page +/// in the allocated range must be aligned to that number of pages. +/// If no specific alignment is needed, the default aligment of 1 page should be used. fn find_any_chunk( list: &mut StaticArrayRBTree, num_pages: usize, within_range: Option<&PageRange>, + alignment_4k_pages: usize, ) -> Result<(AllocatedPages, DeferredAllocAction<'static>), AllocationError> { let designated_low_end = DESIGNATED_PAGES_LOW_END.get() .ok_or(AllocationError::NotInitialized)?; @@ -822,7 +827,8 @@ fn find_any_chunk( if let Some(chunk) = elem { // Use max and min below to ensure that the range of pages we allocate from // is within *both* the current chunk's bounds and the range's bounds. - let lowest_possible_start_page = *max(chunk.start(), range.start()); + let lowest_possible_start_page = max(chunk.start(), range.start()) + .align_up(alignment_4k_pages); let highest_possible_end_page = *min(chunk.end(), range.end()); if lowest_possible_start_page + num_pages <= highest_possible_end_page { return adjust_chosen_chunk( @@ -856,7 +862,8 @@ fn find_any_chunk( while let Some(chunk) = cursor.get().map(|w| w.deref()) { // Use max and min below to ensure that the range of pages we allocate from // is within *both* the current chunk's bounds and the range's bounds. - let lowest_possible_start_page = *max(chunk.start(), range.start()); + let lowest_possible_start_page = max(chunk.start(), range.start()) + .align_up(alignment_4k_pages); let highest_possible_end_page = *min(chunk.end(), range.end()); if lowest_possible_start_page + num_pages <= highest_possible_end_page { return adjust_chosen_chunk( @@ -888,8 +895,14 @@ fn find_any_chunk( Inner::Array(ref mut arr) => { for elem in arr.iter_mut() { if let Some(chunk) = elem { - if num_pages <= chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::Array(elem)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::Array(elem), + ); } } } @@ -911,8 +924,14 @@ fn find_any_chunk( // The first iterates over the lower designated region, from higher addresses to lower, down to zero. let mut cursor = tree.upper_bound_mut(Bound::Included(designated_low_end)); while let Some(chunk) = cursor.get().map(|w| w.deref()) { - if num_pages < chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::RBTree(cursor)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::RBTree(cursor), + ); } cursor.move_prev(); } @@ -924,8 +943,14 @@ fn find_any_chunk( // we already iterated over non-designated pages in the first match statement above, so we're out of memory. break; } - if num_pages < chunk.size_in_pages() { - return adjust_chosen_chunk(*chunk.start(), num_pages, &chunk.clone(), ValueRefMut::RBTree(cursor)); + let lowest_possible_start_page = chunk.start().align_up(alignment_4k_pages); + if lowest_possible_start_page + num_pages <= *chunk.end() { + return adjust_chosen_chunk( + lowest_possible_start_page, + num_pages, + &chunk.clone(), + ValueRefMut::RBTree(cursor), + ); } cursor.move_prev(); } @@ -996,23 +1021,31 @@ fn adjust_chosen_chunk( } -/// Possible options when requested pages from the page allocator. +/// Possible options when requesting pages from the page allocator. pub enum AllocationRequest<'r> { - /// The allocated pages can be located at any virtual address. - Any, /// The allocated pages must start exactly at the given `VirtualAddress`. AtVirtualAddress(VirtualAddress), + /// The allocated pages may be located at any virtual address, + /// but the starting page must be aligned to a multiple of `alignment_4k_pages`. + /// An alignment of `1` page is equivalent to specifying no alignment requirement. + /// + /// Note: alignment is specified in number of 4KiB pages, not number of bytes. + AlignedTo { alignment_4k_pages: usize }, /// The allocated pages can be located anywhere within the given range. WithinRange(&'r PageRange), + /// The allocated pages can be located at any virtual address + /// and have no special alignment requirements beyond a single page. + Any, } + /// The core page allocation routine that allocates the given number of virtual pages, /// optionally at the requested starting `VirtualAddress`. /// /// This simply reserves a range of virtual addresses, it does not allocate /// actual physical memory frames nor do any memory mapping. /// Thus, the returned `AllocatedPages` aren't directly usable until they are mapped to physical frames. -/// +/// /// Allocation is based on a red-black tree and is thus `O(log(n))`. /// Fragmentation isn't cleaned up until we're out of address space, but that's not really a big deal. /// @@ -1047,11 +1080,14 @@ pub fn allocate_pages_deferred( AllocationRequest::AtVirtualAddress(vaddr) => { find_specific_chunk(&mut locked_list, Page::containing_address(vaddr), num_pages) } - AllocationRequest::Any => { - find_any_chunk(&mut locked_list, num_pages, None) + AllocationRequest::AlignedTo { alignment_4k_pages } => { + find_any_chunk(&mut locked_list, num_pages, None, alignment_4k_pages) } AllocationRequest::WithinRange(range) => { - find_any_chunk(&mut locked_list, num_pages, Some(range)) + find_any_chunk(&mut locked_list, num_pages, Some(range), 1) + } + AllocationRequest::Any => { + find_any_chunk(&mut locked_list, num_pages, None, 1) } }; res.map_err(From::from) // convert from AllocationError to &str diff --git a/kernel/pci/Cargo.toml b/kernel/pci/Cargo.toml index 04a9829788..869d260769 100644 --- a/kernel/pci/Cargo.toml +++ b/kernel/pci/Cargo.toml @@ -1,7 +1,7 @@ [package] authors = ["Kevin Boos "] name = "pci" -description = "Basic PCI support for Theseus, x86 only." +description = "Basic PCI support for Theseus." version = "0.1.0" edition = "2021" @@ -12,10 +12,15 @@ log = "0.4.8" volatile = "0.2.4" zerocopy = "0.5.0" -port_io = { path = "../../libs/port_io" } memory = { path = "../memory" } cpu = { path = "../cpu" } interrupts = { path = "../interrupts" } +[target.'cfg(target_arch = "x86_64")'.dependencies] +port_io = { path = "../../libs/port_io" } + +[target.'cfg(target_arch = "aarch64")'.dependencies] +arm_boards = { path = "../arm_boards" } + [lib] crate-type = ["rlib"] diff --git a/kernel/pci/src/lib.rs b/kernel/pci/src/lib.rs index 4733b72676..1fd89a0018 100644 --- a/kernel/pci/src/lib.rs +++ b/kernel/pci/src/lib.rs @@ -1,13 +1,17 @@ +//! PCI Configuration Space Access +//! +//! Note: while pci currently uses port-io on x86 and mmio on aarch64, +//! x86 may also support memory-based PCI configuration in the future; +//! port-io is the legacy way to access the config space. + #![no_std] #![allow(dead_code)] extern crate alloc; use log::*; -use core::fmt; -use core::ops::{Deref, DerefMut}; +use core::{fmt, ops::{Deref, DerefMut}, mem::size_of}; use alloc::vec::Vec; -use port_io::Port; use spin::{Once, Mutex}; use memory::{PhysicalAddress, BorrowedSliceMappedPages, Mutable, MappedPages, map_frame_range, MMIO_FLAGS}; use bit_field::BitField; @@ -16,36 +20,106 @@ use zerocopy::FromBytes; use cpu::CpuId; use interrupts::InterruptNumber; +#[cfg(target_arch = "x86_64")] +use port_io::Port; + +#[cfg(target_arch = "aarch64")] +use arm_boards::BOARD_CONFIG; + +#[derive(Debug, Copy, Clone)] +/// The span of bytes within a 4-byte chunk that a PCI register occupies. +/// +/// The PCI configuration space is represented as an array of 4-byte chunks. +/// This struct defines where in a given 4-byte chunk a PCI register exists. +enum RegisterSpan { + /// Bits [0:31] + FullDword, + /// Bits [0:15] + Word0, + /// Bits [16:31] + Word1, + /// Bits [0:7] + Byte0, + /// Bits [8:15] + Byte1, + /// Bits [16:23] + Byte2, + /// Bits [24:31] + Byte3, +} +use RegisterSpan::*; + +/// A definition of a PCI configuration space register. +#[derive(Clone, Copy, Debug)] +struct PciRegister { + /// The location of this register in the PCI configuration space, + /// given as an index into the space as an array of `u32`s (4-byte chunks). + index: u8, + /// The location of this register within the 4-byte chunk. + span: RegisterSpan, +} +impl PciRegister { + const fn from_offset(raw_offset: u8, size_in_bytes: u8) -> Self { + let index = raw_offset >> 2; + match (size_in_bytes, raw_offset & 0b11) { + (1, 0) => PciRegister { index, span: Byte0 }, + (1, 1) => PciRegister { index, span: Byte1 }, + (1, 2) => PciRegister { index, span: Byte2 }, + (1, 3) => PciRegister { index, span: Byte3 }, + (2, 0) => PciRegister { index, span: Word0 }, + (2, 2) => PciRegister { index, span: Word1 }, + (4, 0) => PciRegister { index, span: FullDword }, + // Throw a const panic (compile error) for invalid values. + _ => panic!("Invalid PciRegister specification"), + } + } +} + +/// A macro for easily defining PCI registers using offsets from the PCI spec. +/// +/// The last argument only accepts register sizes of 1, 2, and 4 bytes. +macro_rules! pci_register { + ($name:ident, $offset:expr, 1) => { + const $name: PciRegister = PciRegister::from_offset($offset, 1); + }; + ($name:ident, $offset:expr, 2) => { + const $name: PciRegister = PciRegister::from_offset($offset, 2); + }; + ($name:ident, $offset:expr, 4) => { + const $name: PciRegister = PciRegister::from_offset($offset, 4); + }; +} + // The below constants define the PCI configuration space. // More info here: -const PCI_VENDOR_ID: u8 = 0x0; -const PCI_DEVICE_ID: u8 = 0x2; -const PCI_COMMAND: u8 = 0x4; -const PCI_STATUS: u8 = 0x6; -const PCI_REVISION_ID: u8 = 0x8; -const PCI_PROG_IF: u8 = 0x9; -const PCI_SUBCLASS: u8 = 0xA; -const PCI_CLASS: u8 = 0xB; -const PCI_CACHE_LINE_SIZE: u8 = 0xC; -const PCI_LATENCY_TIMER: u8 = 0xD; -const PCI_HEADER_TYPE: u8 = 0xE; -const PCI_BIST: u8 = 0xF; -const PCI_BAR0: u8 = 0x10; -const PCI_BAR1: u8 = 0x14; -const PCI_BAR2: u8 = 0x18; -const PCI_BAR3: u8 = 0x1C; -const PCI_BAR4: u8 = 0x20; -const PCI_BAR5: u8 = 0x24; -const PCI_CARDBUS_CIS: u8 = 0x28; -const PCI_SUBSYSTEM_VENDOR_ID: u8 = 0x2C; -const PCI_SUBSYSTEM_ID: u8 = 0x2E; -const PCI_EXPANSION_ROM_BASE: u8 = 0x30; -const PCI_CAPABILITIES: u8 = 0x34; +pci_register!(PCI_VENDOR_ID, 0x00, 2); +pci_register!(PCI_DEVICE_ID, 0x02, 2); +pci_register!(PCI_COMMAND, 0x04, 2); +pci_register!(PCI_STATUS, 0x06, 2); +pci_register!(PCI_REVISION_ID, 0x08, 1); +pci_register!(PCI_PROG_IF, 0x09, 1); +pci_register!(PCI_SUBCLASS, 0x0A, 1); +pci_register!(PCI_CLASS, 0x0B, 1); +pci_register!(PCI_CACHE_LINE_SIZE, 0x0C, 1); +pci_register!(PCI_LATENCY_TIMER, 0x0D, 1); +pci_register!(PCI_HEADER_TYPE, 0x0E, 1); +pci_register!(PCI_BIST, 0x0F, 1); +pci_register!(PCI_BAR0, 0x10, 4); +pci_register!(PCI_BAR1, 0x14, 4); +pci_register!(PCI_BAR2, 0x18, 4); +pci_register!(PCI_BAR3, 0x1C, 4); +pci_register!(PCI_BAR4, 0x20, 4); +pci_register!(PCI_BAR5, 0x24, 4); +pci_register!(PCI_CARDBUS_CIS, 0x28, 4); +pci_register!(PCI_SUBSYSTEM_VENDOR_ID, 0x2C, 2); +pci_register!(PCI_SUBSYSTEM_ID, 0x2E, 2); +pci_register!(PCI_EXPANSION_ROM_BASE, 0x30, 4); +pci_register!(PCI_CAPABILITIES, 0x34, 1); // 0x35 through 0x3B are reserved -const PCI_INTERRUPT_LINE: u8 = 0x3C; -const PCI_INTERRUPT_PIN: u8 = 0x3D; -const PCI_MIN_GRANT: u8 = 0x3E; -const PCI_MAX_LATENCY: u8 = 0x3F; +pci_register!(PCI_INTERRUPT_LINE, 0x3C, 1); +pci_register!(PCI_INTERRUPT_PIN, 0x3D, 1); +pci_register!(PCI_MIN_GRANT, 0x3E, 1); +pci_register!(PCI_MAX_LATENCY, 0x3F, 1); #[repr(u8)] pub enum PciCapability { @@ -64,19 +138,35 @@ const MAX_SLOTS_PER_BUS: u8 = 32; /// There is a maximum of 32 functions (devices) on one PCI slot. const MAX_FUNCTIONS_PER_SLOT: u8 = 8; -/// Addresses/offsets into the PCI configuration space should clear the least-significant 2 bits. -const PCI_CONFIG_ADDRESS_OFFSET_MASK: u8 = 0xFC; +/// Addresses/offsets into the PCI configuration space should clear the +/// least-significant 2 bits in order to be 4-byte aligned. +const PCI_CONFIG_ADDRESS_OFFSET_MASK: u8 = 0b11111100; + const CONFIG_ADDRESS: u16 = 0xCF8; const CONFIG_DATA: u16 = 0xCFC; /// This port is used to specify the address in the PCI configuration space /// for the next read/write of the `PCI_CONFIG_DATA_PORT`. +#[cfg(target_arch = "x86_64")] static PCI_CONFIG_ADDRESS_PORT: Mutex> = Mutex::new(Port::new(CONFIG_ADDRESS)); /// This port is used to transfer data to or from the PCI configuration space /// specified by a previous write to the `PCI_CONFIG_ADDRESS_PORT`. +#[cfg(target_arch = "x86_64")] static PCI_CONFIG_DATA_PORT: Mutex> = Mutex::new(Port::new(CONFIG_DATA)); +#[cfg(target_arch = "x86_64")] +const BASE_OFFSET: u32 = 0x8000_0000; + +#[cfg(target_arch = "aarch64")] +type PciConfigSpace = BorrowedSliceMappedPages, Mutable>; + +#[cfg(target_arch = "aarch64")] +static PCI_CONFIG_SPACE: Mutex> = Mutex::new(Once::new()); + +#[cfg(target_arch = "aarch64")] +const BASE_OFFSET: u32 = 0; + pub enum InterruptPin { A, B, @@ -88,32 +178,33 @@ pub enum InterruptPin { /// Returns a list of all PCI buses in this system. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn get_pci_buses() -> &'static Vec { +pub fn get_pci_buses() -> Result<&'static Vec, &'static str> { static PCI_BUSES: Once> = Once::new(); - PCI_BUSES.call_once(scan_pci) + PCI_BUSES.try_call_once(scan_pci) } /// Returns a reference to the `PciDevice` with the given bus, slot, func identifier. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn get_pci_device_bsf(bus: u8, slot: u8, func: u8) -> Option<&'static PciDevice> { - for b in get_pci_buses() { +pub fn get_pci_device_bsf(bus: u8, slot: u8, func: u8) -> Result, &'static str> { + for b in get_pci_buses()? { if b.bus_number == bus { for d in &b.devices { if d.slot == slot && d.func == func { - return Some(d); + return Ok(Some(d)); } } } } - None + + Ok(None) } /// Returns an iterator that iterates over all `PciDevice`s, in no particular guaranteed order. /// If the PCI bus hasn't been initialized, this initializes the PCI bus & scans it to enumerates devices. -pub fn pci_device_iter() -> impl Iterator { - get_pci_buses().iter().flat_map(|b| b.devices.iter()) +pub fn pci_device_iter() -> Result, &'static str> { + Ok(get_pci_buses()?.iter().flat_map(|b| b.devices.iter())) } @@ -129,7 +220,18 @@ pub struct PciBus { /// Scans all PCI Buses (brute force iteration) to enumerate PCI Devices on each bus. /// Initializes structures containing this information. -fn scan_pci() -> Vec { +fn scan_pci() -> Result, &'static str> { + #[cfg(target_arch = "aarch64")] + PCI_CONFIG_SPACE.lock().try_call_once(|| { + let config = BOARD_CONFIG.pci_ecam; + let mapped = memory::map_frame_range(config.base_address, config.size_bytes, MMIO_FLAGS)?; + let config_space_u32_len = BOARD_CONFIG.pci_ecam.size_bytes / size_of::(); + match mapped.into_borrowed_slice_mut(0, config_space_u32_len) { + Ok(bsm) => Ok(bsm), + Err((_, msg)) => Err(msg), + } + })?; + let mut buses: Vec = Vec::new(); for bus in 0..MAX_PCI_BUSES { @@ -198,9 +300,34 @@ fn scan_pci() -> Vec { } } - buses + Ok(buses) } +impl RegisterSpan { + const fn get_mask_and_bitshift(self) -> (u32, u8) { + match self { + FullDword => (0xffff_ffff, 0), + Word0 => (0x0000_ffff, 0), + Word1 => (0xffff_0000, 16), + Byte0 => (0x0000_00ff, 0), + Byte1 => (0x0000_ff00, 8), + Byte2 => (0x00ff_0000, 16), + Byte3 => (0xff00_0000, 24), + } + } + + const fn width_in_bytes(self) -> usize { + match self { + FullDword => size_of::(), + Word0 => size_of::(), + Word1 => size_of::(), + Byte0 => size_of::(), + Byte1 => size_of::(), + Byte2 => size_of::(), + Byte3 => size_of::(), + } + } +} /// The bus, slot, and function number of a given PCI device. /// This offers methods for reading and writing the PCI config space. @@ -216,93 +343,194 @@ impl PciLocation { pub fn slot(&self) -> u8 { self.slot } pub fn function(&self) -> u8 { self.func } + /// Read the value of the given `register` in the PCI Configuration Space. + fn pci_read_raw(&self, register: PciRegister) -> u32 { + let PciRegister { index, span } = register; + let (mask, shift) = span.get_mask_and_bitshift(); + const U32_BYTES: u32 = size_of::() as u32; - /// Computes a PCI address from bus, slot, func, and offset. - /// The least two significant bits of offset are masked, so it's 4-byte aligned addressing, - /// which makes sense since we read PCI data (from the configuration space) in 32-bit chunks. - fn pci_address(self, offset: u8) -> u32 { - ((self.bus as u32) << 16) | - ((self.slot as u32) << 11) | - ((self.func as u32) << 8) | - ((offset as u32) & (PCI_CONFIG_ADDRESS_OFFSET_MASK as u32)) | - 0x8000_0000 - } + let dword_address = BASE_OFFSET + | ((self.bus as u32) << 16) + | ((self.slot as u32) << 11) + | ((self.func as u32) << 8) + | ((index as u32) * U32_BYTES); + + let dword_value; - /// read 32-bit data at the specified `offset` from the PCI device specified by the given `bus`, `slot`, `func` set. - fn pci_read_32(&self, offset: u8) -> u32 { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(offset)); + #[cfg(target_arch = "x86_64")] { + unsafe { + PCI_CONFIG_ADDRESS_PORT.lock().write(dword_address); + } + dword_value = PCI_CONFIG_DATA_PORT.lock().read(); } - Self::read_data_port() >> ((offset & (!PCI_CONFIG_ADDRESS_OFFSET_MASK)) * 8) + + #[cfg(target_arch = "aarch64")] { + let config_space = PCI_CONFIG_SPACE.lock(); + let config_space = config_space.get() + .expect("PCI Config Space wasn't mapped yet"); + let dword_index = (dword_address as usize) / size_of::(); + dword_value = config_space[dword_index].read(); + } + + (dword_value & mask) >> shift } - /// Read 16-bit data at the specified `offset` from this PCI device. - fn pci_read_16(&self, offset: u8) -> u16 { - self.pci_read_32(offset) as u16 - } + /// Read a 4-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`FullDword`] + fn pci_read_32(&self, register: PciRegister) -> u32 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_32: register isn't 32-bit wide"); - /// Read 8-bit data at the specified `offset` from the PCI device. - fn pci_read_8(&self, offset: u8) -> u8 { - self.pci_read_32(offset) as u8 + self.pci_read_raw(register) } - /// Write 32-bit data to the specified `offset` for the PCI device. - fn pci_write(&self, offset: u8, value: u32) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(offset)); - Self::write_data_port((value) << ((offset & 2) * 8)); - } + /// Read a 2-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Word0`] / [`Word1`] + fn pci_read_16(&self, register: PciRegister) -> u16 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_16: register isn't 16-bit wide"); + + self.pci_read_raw(register) as _ + } + + /// Read a one-byte register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Byte0`] / [`Byte1`] / [`Byte2`] / [`Byte3`] + fn pci_read_8(&self, register: PciRegister) -> u8 { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_read_16: register isn't 8-bit wide"); + + self.pci_read_raw(register) as _ } - fn write_data_port(value: u32) { - unsafe { - PCI_CONFIG_DATA_PORT.lock().write(value); + /// Writes (part of) the given `value` to the given `register` in the PCI Configuration Space. + /// + /// If the width of the given `register` is less than 4 bytes, this function will first + /// read the initial value from the `register` to ensure we don't ovewrite other + /// unrelated parts of the `u32` value. + fn pci_write_raw(&self, register: PciRegister, value: u32) { + let PciRegister { index, span } = register; + const U32_BYTES: u32 = size_of::() as u32; + let dword_address = BASE_OFFSET + | ((self.bus as u32) << 16) + | ((self.slot as u32) << 11) + | ((self.func as u32) << 8) + | ((index as u32) * U32_BYTES); + + /// A macro that handles the required bitmasking/shifting to calculate the + /// final value that should actually be written to this `register`. + macro_rules! calc_value { + ($read_initial_value:expr) => { + if matches!(span, FullDword) { + value + } else { + let mut dword = $read_initial_value; + let (mask, shift) = span.get_mask_and_bitshift(); + dword &= !mask; + dword |= value << shift; + dword + } + } + } + + #[cfg(target_arch = "x86_64")] { + unsafe { + PCI_CONFIG_ADDRESS_PORT.lock().write(dword_address); + } + let dword = calc_value!(PCI_CONFIG_DATA_PORT.lock().read()); + unsafe { + PCI_CONFIG_DATA_PORT.lock().write(dword); + } + } + + #[cfg(target_arch = "aarch64")] { + let mut config_space = PCI_CONFIG_SPACE.lock(); + let config_space = config_space.get_mut() + .expect("PCI Config Space wasn't mapped yet"); + let dword_index = (dword_address as usize) / size_of::(); + + let dword = calc_value!(config_space[dword_index].read()); + config_space[dword_index].write(dword); } } - fn read_data_port() -> u32 { - PCI_CONFIG_DATA_PORT.lock().read() + /// Write a 4-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`FullDword`] + fn pci_write_32(&self, register: PciRegister, value: u32) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_32: register isn't 32-bit wide"); + + self.pci_write_raw(register, value) + } + + /// Write a 2-bytes register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Word0`] / [`Word1`] + fn pci_write_16(&self, register: PciRegister, value: u16) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_16: register isn't 16-bit wide"); + + self.pci_write_raw(register, value as _) + } + + /// Write a one-byte register from the PCI Configuration Space. + /// + /// Panics if the register isn't a [`Byte0`] / [`Byte1`] / [`Byte2`] / [`Byte3`] + fn pci_write_8(&self, register: PciRegister, value: u8) { + let reg_width = register.span.width_in_bytes(); + let output_width = size_of::(); + assert_eq!(reg_width, output_width, "pci_write_16: register isn't 8-bit wide"); + + self.pci_write_raw(register, value as _) } /// Sets the PCI device's bit 3 in the command portion, which is apparently needed to activate DMA (??) pub fn pci_set_command_bus_master_bit(&self) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(PCI_COMMAND)); - } - let inval = Self::read_data_port(); - trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value: {:#x}", self, inval); - Self::write_data_port(inval | (1 << 2)); + let value = self.pci_read_16(PCI_COMMAND); + trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value: {:#x}", self, value); + + self.pci_write_16(PCI_COMMAND, value | (1 << 2)); + trace!("pci_set_command_bus_master_bit: PciDevice: {}, read value AFTER WRITE CMD: {:#x}", self, - Self::read_data_port() + self.pci_read_16(PCI_COMMAND), ); } /// Sets the PCI device's command bit 10 to disable legacy interrupts pub fn pci_set_interrupt_disable_bit(&self) { - unsafe { - PCI_CONFIG_ADDRESS_PORT.lock().write(self.pci_address(PCI_COMMAND)); - } - let command = Self::read_data_port(); + let command = self.pci_read_16(PCI_COMMAND); trace!("pci_set_interrupt_disable_bit: PciDevice: {}, read value: {:#x}", self, command); - const INTERRUPT_DISABLE: u32 = 1 << 10; - Self::write_data_port(command | INTERRUPT_DISABLE); + + const INTERRUPT_DISABLE: u16 = 1 << 10; + self.pci_write_16(PCI_COMMAND, command | INTERRUPT_DISABLE); + trace!("pci_set_interrupt_disable_bit: PciDevice: {} read value AFTER WRITE CMD: {:#x}", - self, Self::read_data_port()); + self, + self.pci_read_16(PCI_COMMAND), + ); } - /// Explores the PCI config space and returns address of requested capability, if present. - /// PCI capabilities are stored as a linked list in the PCI config space, + /// Explores the PCI config space and returns address of requested capability, if present. + /// PCI capabilities are stored as a linked list in the PCI config space, /// with each capability storing the pointer to the next capability right after its ID. - /// The function returns a None value if capabilities are not valid for this device - /// or if the requested capability is not present. + /// The function returns a None value if capabilities are not valid for this device + /// or if the requested capability is not present. fn find_pci_capability(&self, pci_capability: PciCapability) -> Option { let pci_capability = pci_capability as u8; let status = self.pci_read_16(PCI_STATUS); // capabilities are only valid if bit 4 of status register is set const CAPABILITIES_VALID: u16 = 1 << 4; - if status & CAPABILITIES_VALID != 0 { + if status & CAPABILITIES_VALID != 0 { // retrieve the capabilities pointer from the pci config space let capabilities = self.pci_read_8(PCI_CAPABILITIES); @@ -314,21 +542,24 @@ impl PciLocation { // the last capability will have its next pointer equal to zero let final_capability = 0; - // iterate through the linked list of capabilities until the requested capability is found or the list reaches its end + // iterate through the linked list of capabilities until the requested + // capability is found or the list reaches its end while cap_addr != final_capability { - // the capability header is a 16 bit value which contains the current capability ID and the pointer to the next capability - let cap_header = self.pci_read_16(cap_addr); + // the capability header is a 16 bit value which contains + // the current capability ID and the pointer to the next capability // the id is the lower byte of the header - let cap_id = (cap_header & 0xFF) as u8; - + let cap_id_reg = PciRegister::from_offset(cap_addr, 1); + let cap_id = self.pci_read_8(cap_id_reg); + if cap_id == pci_capability { debug!("Found capability: {:#X} at {:#X}", pci_capability, cap_addr); return Some(cap_addr); } // find address of next capability which is the higher byte of the header - cap_addr = ((cap_header >> 8) & 0xFF) as u8; + let next_cap_ptr_reg = PciRegister::from_offset(cap_addr + 1, 1); + cap_addr = self.pci_read_8(next_cap_ptr_reg); } } None @@ -435,15 +666,18 @@ impl PciDevice { // (4) Bitwise "not" (negate) that value, then add 1. // The resulting value is the size of that BAR's memory region. // (5) Restore the original value to that BAR - let bar_offset = PCI_BAR0 + (bar_index as u8 * 4); + let bar_reg_def = PciRegister { + index: PCI_BAR0.index + (bar_index as u8), + span: FullDword, + }; let original_value = self.bars[bar_index]; - self.pci_write(bar_offset, 0xFFFF_FFFF); // Step 1 - let mut mem_size = self.pci_read_32(bar_offset); // Step 2 - mem_size.set_bits(0..4, 0); // Step 3 - mem_size = !(mem_size); // Step 4 - mem_size += 1; // Step 4 - self.pci_write(bar_offset, original_value); // Step 5 + self.pci_write_32(bar_reg_def, 0xFFFF_FFFF); // Step 1 + let mut mem_size = self.pci_read_32(bar_reg_def); // Step 2 + mem_size.set_bits(0..4, 0); // Step 3 + mem_size = !(mem_size); // Step 4 + mem_size += 1; // Step 4 + self.pci_write_32(bar_reg_def, original_value); // Step 5 mem_size } @@ -455,34 +689,54 @@ impl PciDevice { /// # Arguments /// * `core_id`: core that interrupt will be routed to /// * `int_num`: interrupt number to assign to the MSI vector + /// + /// # Panics + /// + /// This function panics if the MSI capability isn't aligned to 4 bytes pub fn pci_enable_msi(&self, core_id: u8, int_num: u8) -> Result<(), &'static str> { // find out if the device is msi capable let cap_addr = self.find_pci_capability(PciCapability::Msi).ok_or("Device not MSI capable")?; + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msi: Invalid MSI capability address alignment"); + let msi_reg_index = cap_addr >> 2; // offset in the capability space where the message address register is located - const MESSAGE_ADDRESS_REGISTER_OFFSET: u8 = 4; + const MESSAGE_ADDRESS_REGISTER_OFFSET: u8 = 1 /* one dword */; + // the memory region is a constant defined for Intel cpus where MSI messages are written // it should be written to bit 20 of the message address register const MEMORY_REGION: u32 = 0x0FEE << 20; + // the core id tells which cpu the interrupt will be routed to // it should be written to bit 12 of the message address register let core = (core_id as u32) << 12; + // set the core the MSI will be sent to in the Message Address Register (Intel Arch SDM, vol3, 10.11) - self.pci_write(cap_addr + MESSAGE_ADDRESS_REGISTER_OFFSET, MEMORY_REGION| core); + let msg_addr_reg = PciRegister { + index: msi_reg_index + MESSAGE_ADDRESS_REGISTER_OFFSET, + span: FullDword, + }; + self.pci_write_32(msg_addr_reg, MEMORY_REGION | core); // offset in the capability space where the message data register is located - const MESSAGE_DATA_REGISTER_OFFSET: u8 = 12; + const MESSAGE_DATA_REGISTER_OFFSET: u8 = 3 /* dwords */; + // Set the interrupt number for the MSI in the Message Data Register - self.pci_write(cap_addr + MESSAGE_DATA_REGISTER_OFFSET, int_num as u32); + let msg_data_reg = PciRegister { + index: msi_reg_index + MESSAGE_DATA_REGISTER_OFFSET, + span: FullDword, + }; + self.pci_write_32(msg_data_reg, int_num as u32); + + // to enable the MSI capability, we need to set bit 0 of the message control register + const MSI_ENABLE: u16 = 1; - // offset in the capability space where the message control register is located - const MESSAGE_CONTROL_REGISTER_OFFSET: u8 = 2; - // to enable the MSI capability, we need to set it bit 0 of the message control register - const MSI_ENABLE: u32 = 1; - let ctrl = self.pci_read_16(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET) as u32; // enable MSI in the Message Control Register - self.pci_write(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET, ctrl | MSI_ENABLE); + // the message control register corresponds to bits [16:31] of the first dword + let msg_ctrl_reg = PciRegister { index: msi_reg_index, span: Word1 }; + let mut ctrl = self.pci_read_16(msg_ctrl_reg); + ctrl |= MSI_ENABLE; + self.pci_write_16(msg_ctrl_reg, ctrl); Ok(()) } @@ -491,19 +745,21 @@ impl PciDevice { /// Only the enable bit is set and the remaining initialization steps of /// setting the interrupt number and core id should be completed in the device driver. pub fn pci_enable_msix(&self) -> Result<(), &'static str> { - // find out if the device is msi-x capable let cap_addr = self.find_pci_capability(PciCapability::Msix).ok_or("Device not MSI-X capable")?; + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msix: Invalid MSI-X capability address alignment"); + let msix_reg_index = cap_addr >> 2; - // offset in the capability space where the message control register is located - const MESSAGE_CONTROL_REGISTER_OFFSET: u8 = 2; - let ctrl = self.pci_read_16(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET) as u32; + // write to bit 15 of Message Control Register to enable MSI-X + const MSIX_ENABLE: u16 = 1 << 15; - // write to bit 15 of Message Control Register to enable MSI-X - const MSIX_ENABLE: u32 = 1 << 15; - self.pci_write(cap_addr + MESSAGE_CONTROL_REGISTER_OFFSET, ctrl | MSIX_ENABLE); + // the message control register corresponds to bits [16:31] of the first dword + let msg_ctrl_reg = PciRegister { index: msix_reg_index, span: Word1 }; + let mut ctrl = self.pci_read_16(msg_ctrl_reg); + ctrl |= MSIX_ENABLE; + self.pci_write_16(msg_ctrl_reg, ctrl); - // let ctrl = pci_read_32(dev.bus, dev.slot, dev.func, cap_addr); + // let ctrl = pci_read_16(msg_ctrl_reg, msix_reg_index); // debug!("MSIX HEADER AFTER ENABLE: {:#X}", ctrl); Ok(()) @@ -516,15 +772,23 @@ impl PciDevice { pub fn pci_mem_map_msix(&self, max_vectors: usize) -> Result { // retreive the address in the pci config space for the msi-x capability let cap_addr = self.find_pci_capability(PciCapability::Msix).ok_or("Device not MSI-X capable")?; - // find the BAR used for msi-x - let vector_table_offset = 4; - let table_offset = self.pci_read_32(cap_addr + vector_table_offset); + assert_eq!(cap_addr & 0b11, 0, "pci_enable_msix: Invalid MSI-X capability address alignment"); + let msix_reg_index = cap_addr >> 2; + + // get physical address of vector table + const VECTOR_TABLE_OFFSET: u8 = 1; + let vector_table_reg = PciRegister { + index: msix_reg_index + VECTOR_TABLE_OFFSET, + span: FullDword, + }; + let table_offset = self.pci_read_32(vector_table_reg); let bar = table_offset & 0x7; let offset = table_offset >> 3; + let addr = self.bars[bar as usize] + offset; + // find the memory base address and size of the area for the vector table - let mem_base = PhysicalAddress::new((self.bars[bar as usize] + offset) as usize) - .ok_or("Invalid BAR content")?; - let mem_size_in_bytes = core::mem::size_of::() * max_vectors; + let mem_base = PhysicalAddress::new(addr as usize).ok_or("Invalid BAR content")?; + let mem_size_in_bytes = size_of::() * max_vectors; // debug!("msi-x vector table bar: {}, base_address: {:#X} and size: {} bytes", bar, mem_base, mem_size_in_bytes); @@ -642,6 +906,7 @@ impl MsixVectorEntry { self.msg_lower_addr.write(address | MSIX_INTERRUPT_REGION | dest_id); // write interrupt number + #[allow(clippy::unnecessary_cast)] self.msg_data.write(int_num as u32); if false { diff --git a/theseus_features/Cargo.toml b/theseus_features/Cargo.toml index 15d3ee2d6c..b4f504aea4 100644 --- a/theseus_features/Cargo.toml +++ b/theseus_features/Cargo.toml @@ -47,6 +47,7 @@ hello = { path = "../applications/hello", optional = true } raw_mode = { path = "../applications/raw_mode", optional = true } print_fault_log = { path = "../applications/print_fault_log", optional = true } seconds_counter = { path = "../applications/seconds_counter", optional = true } +test_aligned_page_allocation = { path = "../applications/test_aligned_page_allocation", optional = true } test_async = { path = "../applications/test_async", optional = true } test_backtrace = { path = "../applications/test_backtrace", optional = true } test_block_io = { path = "../applications/test_block_io", optional = true } @@ -146,6 +147,7 @@ theseus_tests = [ "hello", "raw_mode", "seconds_counter", + "test_aligned_page_allocation", "test_async", "test_backtrace", "test_block_io",