diff --git a/hal_aarch64/src/lib.rs b/hal_aarch64/src/lib.rs index 95a5622..279a3b3 100644 --- a/hal_aarch64/src/lib.rs +++ b/hal_aarch64/src/lib.rs @@ -41,3 +41,18 @@ unsafe extern "C" fn _start() -> ! { ", ); } + +pub struct Aarch64CoreInfo; + +impl hal_core::CoreInfo for Aarch64CoreInfo { + fn init(_core_id: usize) { + // We just read MPIDR_EL1 on aarch64. + } + + fn core_id() -> usize { + let mpidr = MPIDR_EL1.get() as usize; + assert!(mpidr < usize::MAX); + + mpidr + } +} diff --git a/hal_aarch64/src/mm/mod.rs b/hal_aarch64/src/mm/mod.rs index 721ad93..793f4bb 100644 --- a/hal_aarch64/src/mm/mod.rs +++ b/hal_aarch64/src/mm/mod.rs @@ -1,47 +1 @@ -use hal_core::{ - mm::{self, PageAlloc, PageMap}, - AddressRange, Error, -}; - pub mod pgt48; - -use pgt48::PageTable; - -pub type EntryType = usize; - -pub const PAGE_SIZE: usize = PageTable::PAGE_SIZE; - -use core::cell::OnceCell; - -static mut GPT: OnceCell<&'static mut PageTable> = OnceCell::new(); - -pub fn is_pagetable_installed() -> bool { - unsafe { GPT.get_mut().is_some() } -} - -pub fn prefill_pagetable( - r: impl Iterator, - rw: impl Iterator, - rwx: impl Iterator, - pre_allocated: impl Iterator, - allocator: &impl PageAlloc, -) -> Result<(), Error> { - let pt = hal_core::mm::prefill_pagetable::(r, rw, rwx, pre_allocated, allocator)?; - - // TODO: put into into the hal_core::Error - unsafe { - if GPT.set(pt).is_err() { - panic!("GPT is already set ?"); - } - }; - - Ok(()) -} - -pub fn align_up(addr: usize) -> usize { - mm::align_up(addr, PAGE_SIZE) -} - -pub fn align_down(addr: usize) -> usize { - mm::align_down(addr, PAGE_SIZE) -} diff --git a/hal_core/src/hal.rs b/hal_core/src/hal.rs index a4c41c5..a6c2a24 100644 --- a/hal_core/src/hal.rs +++ b/hal_core/src/hal.rs @@ -1,23 +1,28 @@ use super::mm::{self, Mmu, PageAlloc, PageMap}; use super::once_lock::OnceLock; use super::AddressRange; +use super::CoreInfo; use super::Error; use super::ReentrantSpinlock; use super::{IrqOps, TimerCallbackFn}; -pub struct Hal { - kpt: OnceLock>, +pub struct Hal { + kpt: OnceLock>, irq_ops: I, } -impl Hal { - pub const fn new(irq_ops: I) -> Hal { +impl Hal { + pub const fn new(irq_ops: I) -> Hal { Self { kpt: OnceLock::new(), irq_ops, } } + pub fn init_core_info(&self, core_id: usize) { + G::init(core_id) + } + pub fn init_irqs(&'static self) { self.irq_ops.init(); } @@ -91,7 +96,7 @@ impl Hal { mm::align_down(val, self.page_size()) } - pub fn kpt(&'static self) -> &ReentrantSpinlock<&'static mut P> { + pub fn kpt(&'static self) -> &ReentrantSpinlock { self.kpt.get().unwrap() } } diff --git a/hal_core/src/lib.rs b/hal_core/src/lib.rs index 9812eee..3516aab 100644 --- a/hal_core/src/lib.rs +++ b/hal_core/src/lib.rs @@ -13,6 +13,11 @@ pub mod once_lock; mod reentrant_spinlock; pub use reentrant_spinlock::ReentrantSpinlock; +pub trait CoreInfo { + fn init(core_id: usize); + fn core_id() -> usize; +} + pub trait IrqOps { fn init(&'static self); fn init_irq_chip(&self, allocator: &impl mm::PageAlloc) -> Result<(), Error>; diff --git a/hal_core/src/reentrant_spinlock.rs b/hal_core/src/reentrant_spinlock.rs index e575a95..8d84598 100644 --- a/hal_core/src/reentrant_spinlock.rs +++ b/hal_core/src/reentrant_spinlock.rs @@ -1,28 +1,25 @@ +use core::marker::PhantomData; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use lock_api::{GuardSend, RawMutex}; -pub struct RawReentrantSpinlock { +use crate::CoreInfo; + +pub struct RawReentrantSpinlock { user: AtomicUsize, lock: AtomicBool, + _marker: PhantomData, } -fn core_id() -> usize { - let mut id: u64; - - unsafe { core::arch::asm!("mrs {:x}, mpidr_el1", out(reg) id) }; - - id as usize -} - -unsafe impl RawMutex for RawReentrantSpinlock { +unsafe impl RawMutex for RawReentrantSpinlock { // The underlying const with interior mutability is fine because it is only used for // construction. // Clippy recommends using a const fn for constructors but I don't have that freedom of choice // since we depend on lock_api. #[allow(clippy::declare_interior_mutable_const)] - const INIT: RawReentrantSpinlock = RawReentrantSpinlock { + const INIT: RawReentrantSpinlock = RawReentrantSpinlock:: { user: AtomicUsize::new(usize::MAX), lock: AtomicBool::new(false), + _marker: PhantomData, }; // A spinlock guard can be sent to another thread and unlocked there @@ -35,7 +32,7 @@ unsafe impl RawMutex for RawReentrantSpinlock { } fn try_lock(&self) -> bool { - let my_id = core_id(); + let my_id = G::core_id(); if self.user.load(Ordering::Acquire) == my_id { assert!(self.lock.load(Ordering::Relaxed)); @@ -54,7 +51,7 @@ unsafe impl RawMutex for RawReentrantSpinlock { .is_ok() && self .user - .compare_exchange(usize::MAX, core_id(), Ordering::Acquire, Ordering::Relaxed) + .compare_exchange(usize::MAX, my_id, Ordering::Acquire, Ordering::Relaxed) .is_ok() } @@ -64,4 +61,4 @@ unsafe impl RawMutex for RawReentrantSpinlock { } } -pub type ReentrantSpinlock = lock_api::Mutex; +pub type ReentrantSpinlock = lock_api::Mutex, T>; diff --git a/hal_riscv64/Cargo.toml b/hal_riscv64/Cargo.toml index 3b6c527..1ce8b94 100644 --- a/hal_riscv64/Cargo.toml +++ b/hal_riscv64/Cargo.toml @@ -10,3 +10,4 @@ hal_core = { path = "../hal_core" } modular-bitfield = "0.11" sbi = "0.2.0" riscv = "0.10" +log = "0.4" diff --git a/hal_riscv64/src/cpu.rs b/hal_riscv64/src/cpu.rs deleted file mode 100644 index 5f6e471..0000000 --- a/hal_riscv64/src/cpu.rs +++ /dev/null @@ -1,12 +0,0 @@ -use super::registers; - -pub fn unmask_interrupts() { - registers::set_sstatus_sie(); - registers::set_sie_ssie(); - registers::set_sie_seie(); - registers::set_sie_stie(); -} - -pub fn clear_physical_timer() { - sbi::timer::set_timer(u64::MAX).unwrap(); -} diff --git a/hal_riscv64/src/irq.rs b/hal_riscv64/src/irq.rs index 9e2a4e6..8e6090b 100644 --- a/hal_riscv64/src/irq.rs +++ b/hal_riscv64/src/irq.rs @@ -1,54 +1,96 @@ -use hal_core::{ - mm::{PageAlloc, PageMap, Permissions, VAddr}, - Error, TimerCallbackFn, -}; +use hal_core::{mm::PageAlloc, once_lock::OnceLock, Error, IrqOps, TimerCallbackFn}; + +use log; -use super::mm; use super::plic::Plic; use super::registers; -use core::arch::asm; +use core::arch::naked_asm; use core::ptr; use core::sync::atomic::{AtomicPtr, Ordering}; use riscv; use sbi; -pub fn init_exception_handlers() { - registers::set_stvec(trap_handler as usize); +static IRQS: OnceLock<&Riscv64Irqs> = OnceLock::new(); + +#[derive(Debug)] +pub struct Riscv64Irqs { + irq_chip: OnceLock, + timer_callback: AtomicPtr, } -static mut IRQ_CHIP: Option = None; - -pub fn init_irq_chip(_dt_node: (), allocator: &impl PageAlloc) -> Result<(), Error> { - // TODO map the dt_node - let base = 0xc000000; - let max_offset = 0x3FFFFFC; - - mm::current().identity_map_range( - VAddr::new(base), - max_offset / mm::PAGE_SIZE + 1, - Permissions::READ | Permissions::WRITE, - allocator, - )?; - unsafe { - IRQ_CHIP = Some(Plic::new(base)); +unsafe impl Sync for Riscv64Irqs {} + +impl IrqOps for Riscv64Irqs { + fn init(&'static self) { + registers::set_stvec(asm_trap_handler as usize); + IRQS.set(self).expect( + "Looks like Riscv64Irqs::init has already been called, must only be called once !", + ); } - Ok(()) -} + fn unmask_interrupts(&self) { + registers::set_sstatus_sie(); + registers::set_sie_ssie(); + registers::set_sie_seie(); + registers::set_sie_stie(); + } + + fn init_irq_chip(&self, _allocator: &impl PageAlloc) -> Result<(), Error> { + let base = 0xc000000; + self.irq_chip + .set(Plic::new(base)) + .expect("Riscv64Irqs has already been called"); -static TIMER_CALLBACK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + Ok(()) + } -pub fn set_timer_handler(h: TimerCallbackFn) { - TIMER_CALLBACK.store(h as *mut _, Ordering::Relaxed); + fn set_timer_handler(&self, h: TimerCallbackFn) { + self.timer_callback.store(h as *mut _, Ordering::Relaxed); + } + + fn set_timer(&self, ticks: usize) -> Result<(), Error> { + let target_time = riscv::register::time::read() + ticks; + sbi::timer::set_timer(target_time as u64).unwrap(); + + Ok(()) + } + + fn clear_timer(&self) { + sbi::timer::set_timer(u64::MAX).unwrap(); + } } -pub fn set_timer(ticks: usize) -> Result<(), Error> { - let target_time = riscv::register::time::read() + ticks; - sbi::timer::set_timer(target_time as u64).unwrap(); +impl Riscv64Irqs { + pub const fn new() -> Self { + Self { + irq_chip: OnceLock::new(), + timer_callback: AtomicPtr::new(ptr::null_mut()), + } + } + + fn trap_dispatch(&self, exception_code: u64) { + match InterruptType::from(exception_code) { + InterruptType::SupervisorTimer => { + let timer_cb = self.timer_callback.load(Ordering::Relaxed); + if !timer_cb.is_null() { + unsafe { + // Cannot simply dereference TIMER_CALLBACK here. + // We are using an AtomicPtr and TIMER_CALLBACK already holds the fn(). + core::mem::transmute::<_, fn()>(timer_cb)(); + } + } - Ok(()) + // Clear the timer or it will trigger again. + self.clear_timer(); + } + InterruptType::SupervisorExternal => { + log::trace!("caught an external interrupt"); + } + e => panic!("getting caught by unhandler exception {:?}", e), + } + } } #[derive(Debug, Copy, Clone)] @@ -159,27 +201,16 @@ impl From for TrapType { } } -static mut INTERRUPT_VECTOR: &[extern "C" fn()] = &[ - undefined_handler, - undefined_handler, - undefined_handler, - undefined_handler, - undefined_handler, - timer_handler, - undefined_handler, - undefined_handler, - undefined_handler, - supervisor_external_interrupt_handler, -]; - /// Dispatch interrupts and exceptions /// Returns 0 if it was synchronous, 1 otherwise #[no_mangle] -extern "C" fn trap_dispatch(cause: u64) -> u64 { +extern "C" fn c_trap_dispatch(cause: u64) -> u64 { match TrapType::from(cause) { TrapType::Interrupt(itype) => { let exception_code: u64 = itype.into(); - unsafe { INTERRUPT_VECTOR[exception_code as usize]() }; + IRQS.get() + .expect("no one has init'ed the rscv64 hal yet...") + .trap_dispatch(exception_code); if itype.is_asynchronous() { 1 @@ -193,28 +224,11 @@ extern "C" fn trap_dispatch(cause: u64) -> u64 { } } -extern "C" fn supervisor_external_interrupt_handler() { - todo!("fwd the external int to the irq_chip or smthing..."); -} - -extern "C" fn undefined_handler() { - panic!("Interruption is not handled yet"); -} - -extern "C" fn timer_handler() { - let timer_cb = TIMER_CALLBACK.load(Ordering::Relaxed); - if !timer_cb.is_null() { - unsafe { - core::mem::transmute::<_, fn()>(timer_cb)(); - } - } -} - #[naked] #[no_mangle] #[repr(align(4))] -unsafe extern "C" fn trap_handler() { - asm!( +unsafe extern "C" fn asm_trap_handler() { + naked_asm!( " addi sp, sp, -0x100 @@ -257,7 +271,7 @@ unsafe extern "C" fn trap_handler() { // csrr a5, sstatus csrr a0, scause - jal trap_dispatch + jal c_trap_dispatch bne a0, x0, 1f @@ -265,7 +279,7 @@ unsafe extern "C" fn trap_handler() { addi t0, t0, 4 csrw sepc, t0 -1: + 1: ld x1, 0x0(sp) ld x2, 0x8(sp) ld x3, 0x10(sp) @@ -301,7 +315,6 @@ unsafe extern "C" fn trap_handler() { addi sp, sp, 0x100 sret", - options(noreturn) ); // Obviously this isn't done, we need to jump back to the previous context before the // interrupt using mpp/spp and mepc/sepc. diff --git a/hal_riscv64/src/lib.rs b/hal_riscv64/src/lib.rs index b83db9c..42c919f 100644 --- a/hal_riscv64/src/lib.rs +++ b/hal_riscv64/src/lib.rs @@ -2,18 +2,30 @@ #![feature(fn_align)] #![feature(naked_functions)] -pub mod cpu; pub mod irq; pub mod mm; mod plic; mod registers; -use core::arch::asm; +use core::arch::naked_asm; pub fn panic_info() {} #[naked] #[no_mangle] unsafe extern "C" fn _start() -> ! { - asm!("la sp, STACK_START", "call k_main", options(noreturn)); + naked_asm!("la sp, STACK_START", "call k_main"); +} + +pub struct Riscv64CoreInfo; + +impl hal_core::CoreInfo for Riscv64CoreInfo { + fn init(core_id: usize) { + // The core_id is the value in the mhartid CSR we got from the machine-level firmware. + registers::set_sscratch(core_id); + } + fn core_id() -> usize { + // Early kernel code called Self::init and putthe core_id argument into the sscratch. + registers::get_sscratch() + } } diff --git a/hal_riscv64/src/mm/mod.rs b/hal_riscv64/src/mm/mod.rs index 0da7729..e128b8c 100644 --- a/hal_riscv64/src/mm/mod.rs +++ b/hal_riscv64/src/mm/mod.rs @@ -1,62 +1 @@ -use core::arch::asm; -use core::cell::OnceCell; -use hal_core::{ - mm::{self, PageAlloc, PageMap}, - AddressRange, Error, -}; - -mod sv39; -use sv39::{PageTable, Satp, SatpMode}; - -pub const PAGE_SIZE: usize = PageTable::PAGE_SIZE; - -static mut GPT: OnceCell<&'static mut PageTable> = OnceCell::new(); - -pub fn current() -> &'static mut PageTable { - unsafe { GPT.get_mut().unwrap() } -} - -pub fn prefill_pagetable( - r: impl Iterator, - rw: impl Iterator, - rwx: impl Iterator, - pre_allocated: impl Iterator, - allocator: &impl PageAlloc, -) -> Result<(), Error> { - let pt = hal_core::mm::prefill_pagetable::(r, rw, rwx, pre_allocated, allocator)?; - - // TODO: put into into the hal_core::Error - unsafe { - if GPT.set(pt).is_err() { - panic!("GPT is already set ?"); - } - }; - - Ok(()) -} - -pub fn enable_paging() { - unsafe { - load_pagetable(current()); - } -} - -unsafe fn load_pagetable(pt: &'static mut PageTable) { - let pt_addr = pt as *mut PageTable as usize; - let ppn = pt_addr >> 12; - - let satp = Satp::with_values(ppn as u64, 0, SatpMode::Sv39); - - unsafe { - asm!("csrw satp, {}", in(reg)u64::from(satp)); - asm!("sfence.vma"); - } -} - -pub fn align_down(addr: usize) -> usize { - mm::align_down(addr, PageTable::PAGE_SIZE) -} - -pub fn align_up(addr: usize) -> usize { - mm::align_up(addr, PageTable::PAGE_SIZE) -} +pub mod sv39; diff --git a/hal_riscv64/src/mm/sv39.rs b/hal_riscv64/src/mm/sv39.rs index 65aaeb5..9046352 100644 --- a/hal_riscv64/src/mm/sv39.rs +++ b/hal_riscv64/src/mm/sv39.rs @@ -1,6 +1,7 @@ use modular_bitfield::{bitfield, prelude::*}; -use hal_core::mm::{self, PageAlloc, PageEntry, PageMap}; +use core::arch; +use hal_core::mm::{self, Mmu, PageAlloc, PageEntry, PageMap}; use hal_core::Error; #[repr(C)] @@ -177,6 +178,10 @@ impl PageMap for PageTable { Ok(page_table) } + fn ptr(&self) -> *const () { + self as *const Self as *const () + } + fn map( &mut self, va: mm::VAddr, @@ -220,6 +225,20 @@ impl PageMap for PageTable { } } +impl Mmu for PageTable { + fn mmu_on(pagetable: &P) { + let pt_addr = pagetable.ptr() as usize; + let ppn = pt_addr >> 12; + + let satp = Satp::with_values(ppn as u64, 0, SatpMode::Sv39); + + unsafe { + arch::asm!("csrw satp, {}", in(reg)u64::from(satp)); + arch::asm!("sfence.vma"); + } + } +} + #[repr(u8)] pub(crate) enum SatpMode { _Bare = 0, diff --git a/hal_riscv64/src/plic.rs b/hal_riscv64/src/plic.rs index df10ec0..645a4b8 100644 --- a/hal_riscv64/src/plic.rs +++ b/hal_riscv64/src/plic.rs @@ -9,6 +9,7 @@ const PLIC_NUMBER_SOURCE_REGISTER: u16 = const PLIC_MAX_CONTEXT: u16 = 0x3e00; const PLIC_CLAIM_OFFSET: usize = 0x201004; +#[derive(Debug)] pub struct Plic { base_register_address: usize, } diff --git a/hal_riscv64/src/registers.rs b/hal_riscv64/src/registers.rs index 7dcea18..324035a 100644 --- a/hal_riscv64/src/registers.rs +++ b/hal_riscv64/src/registers.rs @@ -29,3 +29,19 @@ pub fn set_stvec(addr: usize) { asm!("csrw stvec, {}", in(reg)(addr)); } } + +pub fn set_sscratch(val: usize) { + unsafe { + asm!("csrw sscratch, {}", in(reg)(val)); + } +} + +pub fn get_sscratch() -> usize { + let mut val: usize; + + unsafe { + asm!("csrr {}, sscratch", out(reg)(val)); + } + + val +} diff --git a/kernel/src/generic_main.rs b/kernel/src/generic_main.rs index 3f372cf..80d9a5a 100644 --- a/kernel/src/generic_main.rs +++ b/kernel/src/generic_main.rs @@ -28,10 +28,10 @@ pub fn generic_main(dt: DeviceTree, hacky_devices: &[& // Driver stuff // let _drvmgr = DriverManager::with_devices(&dt).unwrap(); - log::trace!("mapping gic pages"); - cfg_if::cfg_if! { if #[cfg(target_arch = "aarch64")] { + // XXX: Ideally we'd read the device tree but we're not doing for now... + log::trace!("mapping gic pages"); let (gicd_base, gicc_base) = (0x800_0000, 0x801_0000); HAL.kpt().lock().identity_map_range( VAddr::new(gicd_base), @@ -45,6 +45,15 @@ pub fn generic_main(dt: DeviceTree, hacky_devices: &[& Permissions::READ | Permissions::WRITE, &globals::PHYSICAL_MEMORY_MANAGER ).unwrap(); + } else if #[cfg(target_arch = "riscv64")] { + let base = 0xc000000; + let max_offset = 0x3FFFFFC; + HAL.kpt().lock().identity_map_range( + VAddr::new(base), + max_offset / HAL.page_size() + 1, + Permissions::READ | Permissions::WRITE, + &globals::PHYSICAL_MEMORY_MANAGER, + ).unwrap(); } } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 3457880..cd3d56d 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -38,11 +38,18 @@ cfg_if::cfg_if! { use hal_aarch64::{ mm::pgt48, irq::Aarch64Irqs, + Aarch64CoreInfo, }; use hal_core::Hal; - pub static HAL: Hal = Hal::new(Aarch64Irqs::new()); + pub static HAL: Hal = Hal::new(Aarch64Irqs::new()); } else if #[cfg(target_arch = "riscv64")] { pub type ConsoleImpl = drivers::ns16550::Ns16550; - pub use hal_riscv64 as hal; + use hal_riscv64::{ + mm::sv39, + irq::Riscv64Irqs, + Riscv64CoreInfo, + }; + use hal_core::Hal; + pub static HAL: Hal = Hal::new(Riscv64Irqs::new()); } } diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index bfdc886..53ff96b 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -137,7 +137,6 @@ pub fn map_address_space<'a, I: Iterator>( pre_allocated_entries.into_iter(), &globals::PHYSICAL_MEMORY_MANAGER, )?; - // All pmm pages are located in DRAM so they are already in the pagetable (they are part of // the pre_allocated_entries). // Therefore no allocations will be made, pass the NullPageAllocator. diff --git a/riscv64_qemuvirt/src/main.rs b/riscv64_qemuvirt/src/main.rs index c5f3950..4531dba 100644 --- a/riscv64_qemuvirt/src/main.rs +++ b/riscv64_qemuvirt/src/main.rs @@ -15,16 +15,21 @@ pub const UART_INTERRUPT_NUMBER: u16 = 10; const LAUNCH_TESTS: bool = cfg!(feature = "launch_tests"); #[no_mangle] -extern "C" fn k_main(_core_id: usize, device_tree_ptr: usize) -> ! { +extern "C" fn k_main(core_id: usize, device_tree_ptr: usize) -> ! { + kernel::HAL.init_core_info(core_id); + kernel::HAL.init_irqs(); + static NS16550: Ns16550 = Ns16550::new(UART_ADDR); kernel::kernel_console::set_earlyinit_console(&NS16550); kernel::kernel_console::init_logging().unwrap(); + assert_eq!( + core_id, 0, + "Kernel must be booted on the first core with id == 0" + ); info!("GoOSe is booting"); - kernel::hal::irq::init_exception_handlers(); - let device_tree = kernel::device_tree::DeviceTree::new(device_tree_ptr).unwrap(); kernel::generic_main::generic_main::(device_tree, &[&NS16550]); }