From 9a64f5c17c0589fe2a93475441787ff908914dec Mon Sep 17 00:00:00 2001
From: Stefan Kerkmann <karlk90@pm.me>
Date: Sat, 17 Apr 2021 19:34:27 +0200
Subject: [PATCH] Force machine mode on interrupt exit for context switches

The first attempt to solve illegal instruction expections was made in commit
b875108cd0388aed17d8ce3889659cb49db3cbde
 It seemed as this "fixed" the issue, but merely added delays in the code
 which prevented the error to appear in lucky circumstances. Interesting that this code worked in the first place.

Root cause for the expections where write attempts to mstatus in
user privilege mode which raised the illegal instruction exception which
is in spec with the risc-v privileged isa and documented in the
bumbleebee core architecture manual by nucleisys. The solution is
to never enter user mode by forceing mcause.mpp to 0x3
before calling mret when exiting the interrupt handler
for context switching.
---
 .../RISCV-ECLIC/compilers/GCC/chcoreasm.S     | 83 +++++++++----------
 1 file changed, 39 insertions(+), 44 deletions(-)

diff --git a/os/common/ports/RISCV-ECLIC/compilers/GCC/chcoreasm.S b/os/common/ports/RISCV-ECLIC/compilers/GCC/chcoreasm.S
index 2625fa91..db896902 100644
--- a/os/common/ports/RISCV-ECLIC/compilers/GCC/chcoreasm.S
+++ b/os/common/ports/RISCV-ECLIC/compilers/GCC/chcoreasm.S
@@ -64,20 +64,6 @@
     csrs CSR_MSTATUS, MSTATUS_MIE
 .endm
 
-# Clear previous machine interrupt enable bit in mstatus (mstatus.mpie).
-# On machine return (mret) mstatus.mie is assigned this value.
-# Clearing this bit disables interrupts when leaving interrupt processing mode.
-.macro DISABLE_MPIE
-    li a0, MSTATUS_MPIE
-    csrc CSR_MSTATUS, a0
-.endm
-
-# Set previous machine interrupt enable bit in mstatus (mstatus.mpie).
-.macro ENABLE_MPIE
-    li a0, MSTATUS_MPIE
-    csrs CSR_MSTATUS, a0
-.endm
-
 # --------------------------------------------------------------------------
 # Interrupt context save macro. Saves all caller save registers
 # and status csr registers on the stack.
@@ -174,6 +160,31 @@
     mret
 .option pop
 
+# --------------------------------------------------------------------------
+# Start a thread by invoking its work function.
+# 
+# Threads execution starts here, the code leaves the system critical zone
+# and then jumps into the thread function passed in register S0. The
+# register S1 contains the thread parameter. The function chThdExit() is
+# called on thread function return.
+# --------------------------------------------------------------------------
+.globl  _port_thread_start
+_port_thread_start:
+#if CH_DBG_SYSTEM_STATE_CHECK
+    jal     ra, _dbg_check_unlock
+#endif
+#if CH_DBG_STATISTICS
+    jal     ra, _stats_stop_measure_crit_thd
+#endif
+    ENABLE_MIE
+    mv      a0, s1
+    jalr    ra, s0
+    li      a0, 0              # MSG_OK
+    jal     ra, chThdExit
+
+_zombies:
+       j       _zombies
+
 # --------------------------------------------------------------------------
 # Performs a context switch between two threads.
 # a0 = ntp, a1 = otp
@@ -233,31 +244,6 @@ _port_switch:
     ret
 .option pop
 
-# --------------------------------------------------------------------------
-# Start a thread by invoking its work function.
-# 
-# Threads execution starts here, the code leaves the system critical zone
-# and then jumps into the thread function passed in register S0. The
-# register S1 contains the thread parameter. The function chThdExit() is
-# called on thread function return.
-# --------------------------------------------------------------------------
-.globl  _port_thread_start
-_port_thread_start:
-#if CH_DBG_SYSTEM_STATE_CHECK
-    jal     ra, _dbg_check_unlock
-#endif
-#if CH_DBG_STATISTICS
-    jal     ra, _stats_stop_measure_crit_thd
-#endif
-    ENABLE_MIE
-    mv      a0, s1
-    jalr    ra, s0
-    li      a0, 0              # MSG_OK
-    jal     ra, chThdExit
-
-_zombies:
-       j       _zombies
-
 # --------------------------------------------------------------------------
 # IRQ entry point
 # --------------------------------------------------------------------------
@@ -285,8 +271,20 @@ _irq_handler:
     la      a0, _port_switch_from_isr
     csrw    mepc, a0
 
+    # Interrupt handling and context restoring is handled differently in nucleisys cores.
+    # mstatus.mpie and mstatus.mpp are mirror fields of mcause.mpie and mcause.mpp.
+    # Therefore we directly set the bits in mcause and not mstatus.
+    # See https://doc.nucleisys.com/nuclei_spec/isa/core_csr.html#mcause
+
     # Context switch is a critical section, so disable interrupts on return.
-    DISABLE_MPIE
+    # Clear mcause.mpie.
+    li a0, 0x8000000
+    csrc mcause, a0
+
+    # Set previous privelege mode to machine mode to enforce it on return.
+    # Set mcause.mpp to 0x3 (== machine mode).
+    li a0, 0x30000000
+    csrs mcause, a0
 
     mret
 
@@ -323,15 +321,12 @@ _port_switch_from_isr:
     jal     ra, _stats_stop_measure_crit_thd
 #endif
 
-    # Enable interrupts after leaving the interrupt handler
-    ENABLE_MPIE
-
 .globl _port_exit_from_isr
 _port_exit_from_isr:
     # Restore caller registers and csr registers from the thread stack
     RESTORE_CONTEXT
 
-    # Leave machine mode and return to address stored in mepc
+    # Leave interrupt handling and return to address stored in mepc.
     mret
 
 .option pop