Smp (#23)

* Fire up multiple processors

* Use IPIs to wake up secondary processors

* Much better exception information

* Modifications to show more information on fault

* WIP: Use real libstd

* Add TLS (not complete)

* Add random function, export getpid, cleanup

* Do not spin APs until new context

* Update rust

* Update rust

* Use rd/wrfsbase

* Implement TLS

* Implement compiler builtins and update rust

* Update rust

* Back to Redox libstd

* Update rust

arch/x86_64/Cargo.toml

@@ -7,6 +7,7 @@ bitflags = "*"
 hole_list_allocator = { path = "../../crates/hole_list_allocator/" }
 io = { path = "../../crates/io/" }
 spin = "*"
+syscall = { path = "../../syscall/" }
 
 [dependencies.x86]
 version = "0.7"

arch/x86_64/src/acpi/local_apic.rs

@@ -1,77 +0,0 @@
-use core::intrinsics::{volatile_load, volatile_store};
-use x86::cpuid::CpuId;
-use x86::msr::*;
-
-use memory::Frame;
-use paging::{entry, ActivePageTable, PhysicalAddress};
-
-/// Local APIC
-pub struct LocalApic {
-    pub address: u32,
-    pub x2: bool
-}
-
-impl LocalApic {
-    pub fn new(active_table: &mut ActivePageTable) -> Self {
-        let mut apic = LocalApic {
-            address: (unsafe { rdmsr(IA32_APIC_BASE) as u32 } & 0xFFFF0000),
-            x2: false
-        };
-
-        if CpuId::new().get_feature_info().unwrap().has_x2apic() {
-            unsafe { wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10) };
-            apic.x2 = true;
-        } else {
-            active_table.identity_map(Frame::containing_address(PhysicalAddress::new(apic.address as usize)), entry::PRESENT | entry::WRITABLE | entry::NO_EXECUTE);
-        }
-
-        apic
-    }
-
-    unsafe fn read(&self, reg: u32) -> u32 {
-        volatile_load((self.address + reg) as *const u32)
-    }
-
-    unsafe fn write(&self, reg: u32, value: u32) {
-        volatile_store((self.address + reg) as *mut u32, value);
-    }
-
-    pub fn id(&self) -> u32 {
-        if self.x2 {
-            unsafe { rdmsr(IA32_X2APIC_APICID) as u32 }
-        } else {
-            unsafe { self.read(0x20) }
-        }
-    }
-
-    pub fn version(&self) -> u32 {
-        if self.x2 {
-            unsafe { rdmsr(IA32_X2APIC_VERSION) as u32 }
-        } else {
-            unsafe { self.read(0x30) }
-        }
-    }
-
-    pub fn icr(&self) -> u64 {
-        if self.x2 {
-            unsafe { rdmsr(IA32_X2APIC_ICR) }
-        } else {
-            unsafe {
-                (self.read(0x310) as u64) << 32 | self.read(0x300) as u64
-            }
-        }
-    }
-
-    pub fn set_icr(&mut self, value: u64) {
-        if self.x2 {
-            unsafe { wrmsr(IA32_X2APIC_ICR, value); }
-        } else {
-            unsafe {
-                while self.read(0x300) & 1 << 12 == 1 << 12 {}
-                self.write(0x310, (value >> 32) as u32);
-                self.write(0x300, value as u32);
-                while self.read(0x300) & 1 << 12 == 1 << 12 {}
-            }
-        }
-    }
-}

arch/x86_64/src/acpi/mod.rs

@@ -4,20 +4,19 @@
 use core::intrinsics::{atomic_load, atomic_store};
 use core::sync::atomic::Ordering;
 
+use device::local_apic::LOCAL_APIC;
 use interrupt;
 use memory::{allocate_frames, Frame};
 use paging::{entry, ActivePageTable, Page, PhysicalAddress, VirtualAddress};
 use start::{kstart_ap, CPU_COUNT, AP_READY};
 
 use self::dmar::{Dmar, DmarEntry};
-use self::local_apic::LocalApic;
 use self::madt::{Madt, MadtEntry};
 use self::rsdt::Rsdt;
 use self::sdt::Sdt;
 use self::xsdt::Xsdt;
 
 pub mod dmar;
-pub mod local_apic;
 pub mod madt;
 pub mod rsdt;
 pub mod sdt;
@@ -35,7 +34,7 @@ pub fn init_sdt(sdt: &'static Sdt, active_table: &mut ActivePageTable) {
     if let Some(madt) = Madt::new(sdt) {
         println!(": {:>08X}: {}", madt.local_address, madt.flags);
 
-        let mut local_apic = LocalApic::new(active_table);
+        let mut local_apic = unsafe { &mut LOCAL_APIC };
 
         let me = local_apic.id() as u8;
 
@@ -60,7 +59,7 @@ pub fn init_sdt(sdt: &'static Sdt, active_table: &mut ActivePageTable) {
                 } else {
                     if ap_local_apic.flags & 1 == 1 {
                         // Increase CPU ID
-                        let cpu_id = CPU_COUNT.fetch_add(1, Ordering::SeqCst);
+                        CPU_COUNT.fetch_add(1, Ordering::SeqCst);
 
                         // Allocate a stack
                         let stack_start = allocate_frames(64).expect("no more frames in acpi stack_start").start_address().get() + ::KERNEL_OFFSET;
@@ -75,7 +74,7 @@ pub fn init_sdt(sdt: &'static Sdt, active_table: &mut ActivePageTable) {
 
                         // Set the ap_ready to 0, volatile
                         unsafe { atomic_store(ap_ready, 0) };
-                        unsafe { atomic_store(ap_cpu_id, cpu_id as u64) };
+                        unsafe { atomic_store(ap_cpu_id, ap_local_apic.id as u64) };
                         unsafe { atomic_store(ap_page_table, active_table.address() as u64) };
                         unsafe { atomic_store(ap_stack_start, stack_start as u64) };
                         unsafe { atomic_store(ap_stack_end, stack_end as u64) };

arch/x86_64/src/context.rs

@@ -1,4 +1,4 @@
-use core::sync::atomic::{AtomicBool, ATOMIC_BOOL_INIT, Ordering};
+use core::sync::atomic::{AtomicBool, ATOMIC_BOOL_INIT};
 
 /// This must be used by the kernel to ensure that context switches are done atomically
 /// Compare and exchange this to true when beginning a context switch on any CPU
@@ -106,13 +106,5 @@ impl Context {
 
         asm!("mov $0, rbp" : "=r"(self.rbp) : : "memory" : "intel", "volatile");
         asm!("mov rbp, $0" : : "r"(next.rbp) : "memory" : "intel", "volatile");
-
-        asm!("call context_switch_unlock" : : : "memory" : "intel", "volatile");
     }
 }
-
-/// Unset global lock, set inside of kernel
-#[no_mangle]
-pub extern fn context_switch_unlock(){
-    CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
-}

arch/x86_64/src/device/local_apic.rs

@@ -0,0 +1,114 @@
+use core::intrinsics::{volatile_load, volatile_store};
+use x86::cpuid::CpuId;
+use x86::msr::*;
+
+use memory::Frame;
+use paging::{entry, ActivePageTable, PhysicalAddress, Page, VirtualAddress};
+
+pub static mut LOCAL_APIC: LocalApic = LocalApic {
+    address: 0,
+    x2: false
+};
+
+pub unsafe fn init(active_table: &mut ActivePageTable) {
+    LOCAL_APIC.init(active_table);
+}
+
+pub unsafe fn init_ap() {
+    LOCAL_APIC.init_ap();
+}
+
+/// Local APIC
+pub struct LocalApic {
+    pub address: usize,
+    pub x2: bool
+}
+
+impl LocalApic {
+    unsafe fn init(&mut self, active_table: &mut ActivePageTable) {
+        self.address = (rdmsr(IA32_APIC_BASE) as usize & 0xFFFF0000) + ::KERNEL_OFFSET;
+        self.x2 = CpuId::new().get_feature_info().unwrap().has_x2apic();
+
+        if ! self.x2 {
+            let page = Page::containing_address(VirtualAddress::new(self.address));
+            let frame = Frame::containing_address(PhysicalAddress::new(self.address - ::KERNEL_OFFSET));
+            active_table.map_to(page, frame, entry::PRESENT | entry::WRITABLE | entry::NO_EXECUTE);
+        }
+
+        self.init_ap();
+    }
+
+    unsafe fn init_ap(&mut self) {
+        if self.x2 {
+            wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10);
+            wrmsr(IA32_X2APIC_SIVR, 0x100);
+        } else {
+            self.write(0xF0, 0x100);
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        volatile_load((self.address + reg as usize) as *const u32)
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        volatile_store((self.address + reg as usize) as *mut u32, value);
+    }
+
+    pub fn id(&self) -> u32 {
+        if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_APICID) as u32 }
+        } else {
+            unsafe { self.read(0x20) }
+        }
+    }
+
+    pub fn version(&self) -> u32 {
+        if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_VERSION) as u32 }
+        } else {
+            unsafe { self.read(0x30) }
+        }
+    }
+
+    pub fn icr(&self) -> u64 {
+        if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_ICR) }
+        } else {
+            unsafe {
+                (self.read(0x310) as u64) << 32 | self.read(0x300) as u64
+            }
+        }
+    }
+
+    pub fn set_icr(&mut self, value: u64) {
+        if self.x2 {
+            unsafe { wrmsr(IA32_X2APIC_ICR, value); }
+        } else {
+            unsafe {
+                while self.read(0x300) & 1 << 12 == 1 << 12 {}
+                self.write(0x310, (value >> 32) as u32);
+                self.write(0x300, value as u32);
+                while self.read(0x300) & 1 << 12 == 1 << 12 {}
+            }
+        }
+    }
+
+    pub fn ipi(&mut self, apic_id: usize) {
+        let mut icr = 0x4040;
+        if self.x2 {
+            icr |= (apic_id as u64) << 32;
+        } else {
+            icr |= (apic_id as u64) << 56;
+        }
+        self.set_icr(icr);
+    }
+
+    pub unsafe fn eoi(&mut self) {
+        if self.x2 {
+            wrmsr(IA32_X2APIC_EOI, 0);
+        } else {
+            self.write(0xB0, 0);
+        }
+    }
+}

arch/x86_64/src/device/mod.rs

@@ -1,7 +1,15 @@
+use paging::ActivePageTable;
+
+pub mod local_apic;
 pub mod rtc;
 pub mod serial;
 
-pub unsafe fn init(){
+pub unsafe fn init(active_table: &mut ActivePageTable){
+    local_apic::init(active_table);
     rtc::init();
     serial::init();
 }
+
+pub unsafe fn init_ap() {
+    local_apic::init_ap();
+}

arch/x86_64/src/idt.rs

@@ -58,6 +58,9 @@ pub unsafe fn init() {
     IDT[46].set_func(irq::ata1);
     IDT[47].set_func(irq::ata2);
 
+    // Set IPI handler (null)
+    IDT[0x40].set_func(ipi::ipi);
+
     // Set syscall function
     IDT[0x80].set_func(syscall::syscall);
     IDT[0x80].set_flags(IDT_PRESENT | IDT_RING_3 | IDT_INTERRUPT);

arch/x86_64/src/interrupt/exception.rs

@@ -1,115 +1,140 @@
 use super::{halt, stack_trace};
 
-interrupt!(divide_by_zero, {
-    print!("Divide by zero fault\n");
+use syscall::flag::*;
+
+extern {
+    fn ksignal(signal: usize);
+}
+
+interrupt_stack!(divide_by_zero, stack, {
+    println!("Divide by zero fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGFPE);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(debug, {
-    print!("Debug trap\n");
+interrupt_stack!(debug, stack, {
+    println!("Debug trap at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGTRAP);
 });
 
-interrupt!(non_maskable, {
-    print!("Non-maskable interrupt\n");
+interrupt_stack!(non_maskable, stack, {
+    println!("Non-maskable interrupt at {:>02X}:{:>016X}", stack.cs, stack.rip);
 });
 
-interrupt!(breakpoint, {
-    print!("Breakpoint trap\n");
+interrupt_stack!(breakpoint, stack, {
+    println!("Breakpoint trap at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGTRAP);
 });
 
-interrupt!(overflow, {
-    print!("Overflow trap\n");
+interrupt_stack!(overflow, stack, {
+    println!("Overflow trap at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGFPE);
 });
 
-interrupt!(bound_range, {
-    print!("Bound range exceeded fault\n");
+interrupt_stack!(bound_range, stack, {
+    println!("Bound range exceeded fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(invalid_opcode, {
-    print!("Invalid opcode fault\n");
+interrupt_stack!(invalid_opcode, stack, {
+    println!("Invalid opcode fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGILL);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(device_not_available, {
-    print!("Device not available fault\n");
+interrupt_stack!(device_not_available, stack, {
+    println!("Device not available fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGILL);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(double_fault, {
-    print!("Double fault\n");
+interrupt_error!(double_fault, stack, {
+    println!("Double fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(invalid_tss, {
-    print!("Invalid TSS fault\n");
+interrupt_error!(invalid_tss, stack, {
+    println!("Invalid TSS fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(segment_not_present, {
-    print!("Segment not present fault\n");
+interrupt_error!(segment_not_present, stack, {
+    println!("Segment not present fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(stack_segment, {
-    print!("Stack segment fault\n");
+interrupt_error!(stack_segment, stack, {
+    println!("Stack segment fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(protection, {
-    print!("Protection fault\n");
+interrupt_error!(protection, stack, {
+    println!("Protection fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(page, {
+interrupt_error!(page, stack, {
     let cr2: usize;
     asm!("mov rax, cr2" : "={rax}"(cr2) : : : "intel", "volatile");
-    println!("Page fault: {:>016X}", cr2);
+    println!("Page fault: {:>02X}:{:>016X} at {:>02X}:{:>016X}", stack.code, cr2, stack.cs, stack.rip);
+    ksignal(SIGSEGV);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(fpu, {
-    print!("FPU floating point fault\n");
+interrupt_stack!(fpu, stack, {
+    println!("FPU floating point fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGFPE);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(alignment_check, {
-    print!("Alignment check fault\n");
+interrupt_error!(alignment_check, stack, {
+    println!("Alignment check fault: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGBUS);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(machine_check, {
-    print!("Machine check fault\n");
+interrupt_stack!(machine_check, stack, {
+    println!("Machine check fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGBUS);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(simd, {
-    print!("SIMD floating point fault\n");
+interrupt_stack!(simd, stack, {
+    println!("SIMD floating point fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGFPE);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt!(virtualization, {
-    print!("Virtualization fault\n");
+interrupt_stack!(virtualization, stack, {
+    println!("Virtualization fault at {:>02X}:{:>016X}", stack.cs, stack.rip);
+    ksignal(SIGBUS);
     stack_trace();
     loop { halt(); }
 });
 
-interrupt_error!(security, {
-    print!("Security exception\n");
+interrupt_error!(security, stack, {
+    println!("Security exception: {:X} at {:>02X}:{:>016X}", stack.code, stack.cs, stack.rip);
+    ksignal(SIGBUS);
     stack_trace();
     loop { halt(); }
 });

arch/x86_64/src/interrupt/ipi.rs

@@ -0,0 +1,5 @@
+use device::local_apic::LOCAL_APIC;
+
+interrupt!(ipi, {
+    LOCAL_APIC.eoi();
+});

arch/x86_64/src/interrupt/mod.rs

@@ -5,6 +5,7 @@ use core::mem;
 use paging::{ActivePageTable, VirtualAddress};
 
 pub mod exception;
+pub mod ipi;
 pub mod irq;
 pub mod syscall;
 

arch/x86_64/src/interrupt/syscall.rs

@@ -23,17 +23,21 @@ pub unsafe extern fn syscall() {
         asm!("" : : "{rax}"(a) : : "intel", "volatile");
     }
 
-    asm!("push fs
-        push rax
-        mov rax, 0x18
-        mov fs, ax
-        pop rax"
+    asm!("push r15
+        rdfsbase r15
+        push r15
+        push fs
+        mov r15, 0x18
+        mov fs, r15"
         : : : : "intel", "volatile");
 
     inner();
 
     // Interrupt return
     asm!("pop fs
+        pop r15
+        wrfsbase r15
+        pop r15
         iretq"
         : : : : "intel", "volatile");
 }

arch/x86_64/src/lib.rs

@@ -17,6 +17,7 @@ extern crate hole_list_allocator as allocator;
 extern crate bitflags;
 extern crate io;
 extern crate spin;
+extern crate syscall;
 pub extern crate x86;
 
 // Because the memory map is so important to not be aliased, it is defined here, in one place
@@ -40,7 +41,7 @@ pub extern crate x86;
 
     /// Offset to kernel percpu variables
     //TODO: Use 64-bit fs offset to enable this pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
-    pub const KERNEL_PERCPU_OFFSET: usize = 0xC0000000;
+    pub const KERNEL_PERCPU_OFFSET: usize = 0xC000_0000;
     /// Size of kernel percpu variables
     pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB
 
@@ -61,8 +62,11 @@ pub extern crate x86;
     /// Size of user stack
     pub const USER_STACK_SIZE: usize = 1024 * 1024; // 1 MB
 
+    /// Offset to user TLS
+    pub const USER_TLS_OFFSET: usize = USER_STACK_OFFSET + PML4_SIZE;
+
     /// Offset to user temporary image (used when cloning)
-    pub const USER_TMP_OFFSET: usize = USER_STACK_OFFSET + PML4_SIZE;
+    pub const USER_TMP_OFFSET: usize = USER_TLS_OFFSET + PML4_SIZE;
 
     /// Offset to user temporary heap (used when cloning)
     pub const USER_TMP_HEAP_OFFSET: usize = USER_TMP_OFFSET + PML4_SIZE;
@@ -73,6 +77,9 @@ pub extern crate x86;
     /// Offset to user temporary stack (used when cloning)
     pub const USER_TMP_STACK_OFFSET: usize = USER_TMP_GRANT_OFFSET + PML4_SIZE;
 
+    /// Offset to user temporary tls (used when cloning)
+    pub const USER_TMP_TLS_OFFSET: usize = USER_TMP_STACK_OFFSET + PML4_SIZE;
+
 
 /// Print to console
 #[macro_export]
@@ -111,6 +118,8 @@ macro_rules! interrupt {
                 push r9
                 push r10
                 push r11
+                rdfsbase rax
+                push rax
                 push fs
                 mov rax, 0x18
                 mov fs, ax"
@@ -121,6 +130,8 @@ macro_rules! interrupt {
 
             // Pop scratch registers and return
             asm!("pop fs
+                pop rax
+                wrfsbase rax
                 pop r11
                 pop r10
                 pop r9
@@ -136,13 +147,101 @@ macro_rules! interrupt {
     };
 }
 
+#[repr(packed)]
+pub struct InterruptStack {
+    fs: usize,
+    r11: usize,
+    r10: usize,
+    r9: usize,
+    r8: usize,
+    rsi: usize,
+    rdi: usize,
+    rdx: usize,
+    rcx: usize,
+    rax: usize,
+    rip: usize,
+    cs: usize,
+    rflags: usize,
+}
+
 #[macro_export]
-macro_rules! interrupt_error {
-    ($name:ident, $func:block) => {
+macro_rules! interrupt_stack {
+    ($name:ident, $stack: ident, $func:block) => {
         #[naked]
         pub unsafe extern fn $name () {
             #[inline(never)]
-            unsafe fn inner() {
+            unsafe fn inner($stack: &$crate::InterruptStack) {
+                $func
+            }
+
+            // Push scratch registers
+            asm!("push rax
+                push rcx
+                push rdx
+                push rdi
+                push rsi
+                push r8
+                push r9
+                push r10
+                push r11
+                rdfsbase rax
+                push rax
+                push fs
+                mov rax, 0x18
+                mov fs, ax"
+                : : : : "intel", "volatile");
+
+            // Get reference to stack variables
+            let rsp: usize;
+            asm!("" : "={rsp}"(rsp) : : : "intel", "volatile");
+
+            // Call inner rust function
+            inner(&*(rsp as *const $crate::InterruptStack));
+
+            // Pop scratch registers and return
+            asm!("pop fs
+                pop rax
+                wrfsbase rax
+                pop r11
+                pop r10
+                pop r9
+                pop r8
+                pop rsi
+                pop rdi
+                pop rdx
+                pop rcx
+                pop rax
+                iretq"
+                : : : : "intel", "volatile");
+        }
+    };
+}
+
+#[repr(packed)]
+pub struct InterruptErrorStack {
+    fs: usize,
+    r11: usize,
+    r10: usize,
+    r9: usize,
+    r8: usize,
+    rsi: usize,
+    rdi: usize,
+    rdx: usize,
+    rcx: usize,
+    rax: usize,
+    code: usize,
+    rip: usize,
+    cs: usize,
+    rflags: usize,
+}
+
+#[macro_export]
+macro_rules! interrupt_error {
+    ($name:ident, $stack:ident, $func:block) => {
+        #[naked]
+        pub unsafe extern fn $name () {
+            #[inline(never)]
+            unsafe fn inner($stack: &$crate::InterruptErrorStack) {
                 $func
             }
 
@@ -162,8 +261,12 @@ macro_rules! interrupt_error {
                 mov fs, ax"
                 : : : : "intel", "volatile");
 
+            // Get reference to stack variables
+            let rsp: usize;
+            asm!("" : "={rsp}"(rsp) : : : "intel", "volatile");
+
             // Call inner rust function
-            inner();
+            inner(&*(rsp as *const $crate::InterruptErrorStack));
 
             // Pop scratch registers, error code, and return
             asm!("pop fs

arch/x86_64/src/start.rs

@@ -109,7 +109,7 @@ pub unsafe extern fn kstart() -> ! {
         }
 
         // Initialize devices
-        device::init();
+        device::init(&mut active_table);
 
         // Read ACPI tables, starts APs
         acpi::init(&mut active_table);
@@ -145,6 +145,9 @@ pub unsafe extern fn kstart_ap(cpu_id: usize, bsp_table: usize, stack_start: usi
             assert_eq!(TDATA_TEST_NONZERO, 0xFFFFFFFFFFFFFFFE);
         }
 
+        // Initialize devices (for AP)
+        device::init_ap();
+
         AP_READY.store(true, Ordering::SeqCst);
     }
 
@@ -155,24 +158,27 @@ pub unsafe extern fn kstart_ap(cpu_id: usize, bsp_table: usize, stack_start: usi
     kmain_ap(cpu_id);
 }
 
-pub unsafe fn usermode(ip: usize, sp: usize) -> ! {
+pub unsafe fn usermode(ip: usize, sp: usize, fs: usize) -> ! {
     // Go to usermode
-    asm!("mov ds, ax
+    asm!("xchg bx, bx
+        mov ds, ax
         mov es, ax
         mov fs, ax
         mov gs, ax
+        wrfsbase rbx
         push rax
-        push rbx
         push rcx
         push rdx
         push rsi
+        push rdi
         iretq"
         : // No output because it never returns
-        :   "{rax}"(gdt::GDT_USER_DATA << 3 | 3), // Stack segment
-            "{rbx}"(sp), // Stack pointer
-            "{rcx}"(3 << 12 | 1 << 9), // Flags - Set IOPL and interrupt enable flag
-            "{rdx}"(gdt::GDT_USER_CODE << 3 | 3), // Code segment
-            "{rsi}"(ip) // IP
+        :   "{rax}"(gdt::GDT_USER_DATA << 3 | 3), // Data segment
+            "{rbx}"(fs), // TLS segment
+            "{rcx}"(sp), // Stack pointer
+            "{rdx}"(3 << 12 | 1 << 9), // Flags - Set IOPL and interrupt enable flag
+            "{rsi}"(gdt::GDT_USER_CODE << 3 | 3), // Code segment
+            "{rdi}"(ip) // IP
         : // No clobers because it never returns
         : "intel", "volatile");
     unreachable!();