From 3b1f10d4fc4acc946136d7e0ac4f0b534f001739 Mon Sep 17 00:00:00 2001 From: ChinYikMing Date: Sat, 26 Oct 2024 16:20:11 +0800 Subject: [PATCH] Preliminary support for MMU emulation To boot a 32-bit RISC-V Linux with MMU, MMU emulation support is essential. The virtual memory scheme required is SV32. Major changes in this commit include implementing the MMU-related riscv_io_t interface and binding it during RISC-V instance initialization. To reuse the riscv_io_t interface, its prototype is modified to allow access to the RISC-V core instance as the first parameter, since MMU-enabled I/O requires access to the SATP CSR. Additionally, a trap_handler callback is added to the riscv_io_t interface to route the actual trap handler. This approach keeps the dispatch_table and TRAP_HANDLER_IMPL static within emulate.c, aligning the schema with other handlers like ebreak_handler and ecall_handler. The SET_CAUSE_AND_TVAL_THEN_TRAP macro is introduced to simplify the dispatch process when invoking a trap. For each memory access, the page table is walked to get the corresponding PTE. Depending on the PTE retrieval, several page faults may need handling. Thus, three exception handlers have been introduced: insn_pgfault, load_pgfault, and store_pgfault, used in MMU_CHECK_FAULT. This commit does not fully handle access faults since they are related to PMA and PMP, which may not be necessary for booting 32-bit RISC-V Linux (possibly supported in the future). Since Linux has not been booted yet, a test suite is needed to test the MMU emulation. This commit includes a test suite that implements a simple kernel space supervisor and a user space application. The supervisor prepares the page table and then passes control to the user space application to test the three aforementioned page faults. Related: #310 --- Makefile | 4 + src/decode.c | 5 +- src/emulate.c | 150 ++++++++----- src/feature.h | 5 + src/gdbstub.c | 4 +- src/main.c | 9 + src/riscv.c | 77 +++++-- src/riscv.h | 183 +++++++++++++++- src/rv32_template.c | 136 ++++++------ src/syscall.c | 11 +- src/system.c | 286 ++++++++++++++++++++++++ src/system.h | 47 ++++ tests/system/mmu/Makefile | 36 ++++ tests/system/mmu/linker.ld | 31 +++ tests/system/mmu/main.c | 102 +++++++++ tests/system/mmu/setup.S | 162 ++++++++++++++ tests/system/mmu/vm_setup.c | 420 ++++++++++++++++++++++++++++++++++++ 17 files changed, 1508 insertions(+), 160 deletions(-) create mode 100644 src/system.c create mode 100644 src/system.h create mode 100644 tests/system/mmu/Makefile create mode 100644 tests/system/mmu/linker.ld create mode 100644 tests/system/mmu/main.c create mode 100644 tests/system/mmu/setup.S create mode 100644 tests/system/mmu/vm_setup.c diff --git a/Makefile b/Makefile index 58a41ae40..a32e46517 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,10 @@ CFLAGS += $(CFLAGS_NO_CET) OBJS_EXT := +ifeq ($(call has, SYSTEM), 1) +OBJS_EXT += system.o +endif + # Integer Multiplication and Division instructions ENABLE_EXT_M ?= 1 $(call set-feature, EXT_M) diff --git a/src/decode.c b/src/decode.c index 4dbce5bd2..248db4796 100644 --- a/src/decode.c +++ b/src/decode.c @@ -907,9 +907,8 @@ static inline bool op_system(rv_insn_t *ir, const uint32_t insn) default: /* illegal instruction */ return false; } - if (!csr_is_writable(ir->imm) && ir->rs1 != rv_reg_zero) - return false; - return true; + + return csr_is_writable(ir->imm) || (ir->rs1 == rv_reg_zero); } /* MISC-MEM: I-type diff --git a/src/emulate.c b/src/emulate.c index 52e1f0375..10e46f84c 100644 --- a/src/emulate.c +++ b/src/emulate.c @@ -14,6 +14,10 @@ #include #endif +#if RV32_HAS(SYSTEM) +#include "system.h" +#endif /* RV32_HAS(SYSTEM) */ + #if RV32_HAS(EXT_F) #include #include "softfloat.h" @@ -41,47 +45,33 @@ extern struct target_ops gdbstub_ops; #define IF_rs2(i, r) (i->rs2 == rv_reg_##r) #define IF_imm(i, v) (i->imm == v) -/* RISC-V exception code list */ +/* RISC-V trap code list */ /* clang-format off */ -#define RV_TRAP_LIST \ - IIF(RV32_HAS(EXT_C))(, \ - _(insn_misaligned, 0) /* Instruction address misaligned */ \ - ) \ - _(illegal_insn, 2) /* Illegal instruction */ \ - _(breakpoint, 3) /* Breakpoint */ \ - _(load_misaligned, 4) /* Load address misaligned */ \ - _(store_misaligned, 6) /* Store/AMO address misaligned */ \ - IIF(RV32_HAS(SYSTEM))(, \ - _(ecall_M, 11) /* Environment call from M-mode */ \ +#define RV_TRAP_LIST \ + IIF(RV32_HAS(EXT_C))(, \ + _(insn_misaligned) /* Instruction address misaligned */ \ + ) \ + _(illegal_insn) /* Illegal instruction */ \ + _(breakpoint) /* Breakpoint */ \ + _(load_misaligned) /* Load address misaligned */ \ + _(store_misaligned) /* Store/AMO address misaligned */ \ + IIF(RV32_HAS(SYSTEM))( \ + _(pagefault_insn) /* Instruction page fault */ \ + _(pagefault_load) /* Load page fault */ \ + _(pagefault_store), /* Store page fault */ \ + _(ecall_M) /* Environment call from M-mode */ \ ) /* clang-format on */ -enum { -#define _(type, code) rv_trap_code_##type = code, - RV_TRAP_LIST -#undef _ -}; - static void rv_trap_default_handler(riscv_t *rv) { rv->csr_mepc += rv->compressed ? 2 : 4; rv->PC = rv->csr_mepc; /* mret */ } -/* - * Trap might occurs during block emulation. For instance, page fault. - * In order to handle trap, we have to escape from block and execute - * registered trap handler. This trap_handler function helps to execute - * the registered trap handler, PC by PC. Once the trap is handled, - * resume the previous execution flow where cause the trap. - * - * Since the system emulation has not yet included in rv32emu, the page - * fault is not practical in current test suite. Instead, we try to - * emulate the misaligned handling in the test suite. - */ #if RV32_HAS(SYSTEM) -static void trap_handler(riscv_t *rv); -#endif +static void __trap_handler(riscv_t *rv); +#endif /* SYSTEM */ /* When a trap occurs in M-mode/S-mode, m/stval is either initialized to zero or * populated with exception-specific details to assist software in managing @@ -99,11 +89,9 @@ static void trap_handler(riscv_t *rv); * it is worth noting that a future standard could redefine how m/stval is * handled for different types of traps. * - * For simplicity and clarity, abstracting stval and mtval into a single - * identifier called tval, as both are handled by TRAP_HANDLER_IMPL. */ -#define TRAP_HANDLER_IMPL(type, code) \ - static void rv_trap_##type(riscv_t *rv, uint32_t tval) \ +#define TRAP_HANDLER_IMPL(type) \ + static void rv_trap_##type(riscv_t *rv) \ { \ /* m/stvec (Machine/Supervisor Trap-Vector Base Address Register) \ * m/stvec[MXLEN-1:2]: vector base address \ @@ -113,9 +101,12 @@ static void trap_handler(riscv_t *rv); * m/scause (Machine/Supervisor Cause Register): store exception code \ * m/sstatus (Machine/Supervisor Status Register): keep track of and \ * controls the hart’s current operating state \ + * \ + * m/stval and m/scause are set in SET_CAUSE_AND_TVAL_THEN_TRAP \ */ \ uint32_t base; \ uint32_t mode; \ + uint32_t cause; \ /* user or supervisor */ \ if (RV_PRIV_IS_U_OR_S_MODE()) { \ const uint32_t sstatus_sie = \ @@ -126,9 +117,8 @@ static void trap_handler(riscv_t *rv); rv->priv_mode = RV_PRIV_S_MODE; \ base = rv->csr_stvec & ~0x3; \ mode = rv->csr_stvec & 0x3; \ + cause = rv->csr_scause; \ rv->csr_sepc = rv->PC; \ - rv->csr_stval = tval; \ - rv->csr_scause = code; \ } else { /* machine */ \ const uint32_t mstatus_mie = \ (rv->csr_mstatus & MSTATUS_MIE) >> MSTATUS_MIE_SHIFT; \ @@ -138,9 +128,8 @@ static void trap_handler(riscv_t *rv); rv->priv_mode = RV_PRIV_M_MODE; \ base = rv->csr_mtvec & ~0x3; \ mode = rv->csr_mtvec & 0x3; \ + cause = rv->csr_mcause; \ rv->csr_mepc = rv->PC; \ - rv->csr_mtval = tval; \ - rv->csr_mcause = code; \ if (!rv->csr_mtvec) { /* in case CSR is not configured */ \ rv_trap_default_handler(rv); \ return; \ @@ -155,14 +144,14 @@ static void trap_handler(riscv_t *rv); case 1: \ /* MSB of code is used to indicate whether the trap is interrupt \ * or exception, so it is not considered as the 'real' code */ \ - rv->PC = base + 4 * (code & MASK(31)); \ + rv->PC = base + 4 * (cause & MASK(31)); \ break; \ } \ - IIF(RV32_HAS(SYSTEM))(if (rv->is_trapped) trap_handler(rv);, ) \ + IIF(RV32_HAS(SYSTEM))(if (rv->is_trapped) __trap_handler(rv);, ) \ } /* RISC-V exception handlers */ -#define _(type, code) TRAP_HANDLER_IMPL(type, code) +#define _(type) TRAP_HANDLER_IMPL(type) RV_TRAP_LIST #undef _ @@ -180,8 +169,8 @@ RV_TRAP_LIST rv->compressed = compress; \ rv->csr_cycle = cycle; \ rv->PC = PC; \ - IIF(RV32_HAS(SYSTEM))(rv->is_trapped = true, ); \ - rv_trap_##type##_misaligned(rv, IIF(IO)(addr, mask_or_pc)); \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, type##_MISALIGNED, \ + IIF(IO)(addr, mask_or_pc)); \ return false; \ } @@ -531,8 +520,8 @@ static bool do_fuse3(riscv_t *rv, rv_insn_t *ir, uint64_t cycle, uint32_t PC) */ for (int i = 0; i < ir->imm2; i++) { uint32_t addr = rv->X[fuse[i].rs1] + fuse[i].imm; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->X[fuse[i].rs2]); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->X[fuse[i].rs2]); } PC += ir->imm2 * 4; if (unlikely(RVOP_NO_NEXT(ir))) { @@ -555,8 +544,8 @@ static bool do_fuse4(riscv_t *rv, rv_insn_t *ir, uint64_t cycle, uint32_t PC) */ for (int i = 0; i < ir->imm2; i++) { uint32_t addr = rv->X[fuse[i].rs1] + fuse[i].imm; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - rv->X[fuse[i].rd] = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + rv->X[fuse[i].rd] = rv->io.mem_read_w(rv, addr); } PC += ir->imm2 * 4; if (unlikely(RVOP_NO_NEXT(ir))) { @@ -666,12 +655,12 @@ static void block_translate(riscv_t *rv, block_t *block) prev_ir->next = ir; /* fetch the next instruction */ - const uint32_t insn = rv->io.mem_ifetch(block->pc_end); + const uint32_t insn = rv->io.mem_ifetch(rv, block->pc_end); /* decode the instruction */ if (!rv_decode(ir, insn)) { rv->compressed = is_compressed(insn); - rv_trap_illegal_insn(rv, insn); + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, ILLEGAL_INSN, insn); break; } ir->impl = dispatch_table[ir->opcode]; @@ -1122,15 +1111,14 @@ void rv_step(void *arg) } #if RV32_HAS(SYSTEM) -static void trap_handler(riscv_t *rv) +static void __trap_handler(riscv_t *rv) { rv_insn_t *ir = mpool_alloc(rv->block_ir_mp); assert(ir); - /* set to false by sret/mret implementation */ - uint32_t insn; + /* set to false by sret implementation */ while (rv->is_trapped && !rv_has_halted(rv)) { - insn = rv->io.mem_ifetch(rv->PC); + uint32_t insn = rv->io.mem_ifetch(rv, rv->PC); assert(insn); rv_decode(ir, insn); @@ -1139,12 +1127,62 @@ static void trap_handler(riscv_t *rv) ir->impl(rv, ir, rv->csr_cycle, rv->PC); } } -#endif +#endif /* SYSTEM */ + +static void _trap_handler(riscv_t *rv) +{ + uint32_t cause = RV_PRIV_IS_U_OR_S_MODE() ? rv->csr_scause : rv->csr_mcause; + + switch (cause) { +#if !RV32_HAS(EXT_C) + case INSN_MISALIGNED: + rv_trap_insn_misaligned(rv); + break; +#endif /* EXT_C */ + case ILLEGAL_INSN: + rv_trap_illegal_insn(rv); + break; + case BREAKPOINT: + rv_trap_breakpoint(rv); + break; + case LOAD_MISALIGNED: + rv_trap_load_misaligned(rv); + break; + case STORE_MISALIGNED: + rv_trap_store_misaligned(rv); + break; +#if RV32_HAS(SYSTEM) + case PAGEFAULT_INSN: + rv_trap_pagefault_insn(rv); + break; + case PAGEFAULT_LOAD: + rv_trap_pagefault_load(rv); + break; + case PAGEFAULT_STORE: + rv_trap_pagefault_store(rv); + break; +#endif /* SYSTEM */ +#if !RV32_HAS(SYSTEM) + case ECALL_M: + rv_trap_ecall_M(rv); + break; +#endif /* SYSTEM */ + default: + __UNREACHABLE; + break; + } +} + +void trap_handler(riscv_t *rv) +{ + assert(rv); + _trap_handler(rv); +} void ebreak_handler(riscv_t *rv) { assert(rv); - rv_trap_breakpoint(rv, rv->PC); + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, BREAKPOINT, rv->PC); } void ecall_handler(riscv_t *rv) @@ -1154,7 +1192,7 @@ void ecall_handler(riscv_t *rv) syscall_handler(rv); rv->PC += 4; #else - rv_trap_ecall_M(rv, 0); + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, ECALL_M, 0); syscall_handler(rv); #endif } diff --git a/src/feature.h b/src/feature.h index ee27936ac..93eb6160d 100644 --- a/src/feature.h +++ b/src/feature.h @@ -63,5 +63,10 @@ #define RV32_FEATURE_T2C 0 #endif +/* System */ +#ifndef RV32_FEATURE_SYSTEM +#define RV32_FEATURE_SYSTEM 0 +#endif + /* Feature test macro */ #define RV32_HAS(x) RV32_FEATURE_##x diff --git a/src/gdbstub.c b/src/gdbstub.c index 663458d62..d9d635b7e 100644 --- a/src/gdbstub.c +++ b/src/gdbstub.c @@ -55,7 +55,7 @@ static int rv_read_mem(void *args, size_t addr, size_t len, void *val) * an invalid address. We may have to do error handling in the * mem_read_* function directly. */ - *((uint8_t *) val + i) = rv->io.mem_read_b(addr + i); + *((uint8_t *) val + i) = rv->io.mem_read_b(rv, addr + i); } return err; @@ -66,7 +66,7 @@ static int rv_write_mem(void *args, size_t addr, size_t len, void *val) riscv_t *rv = (riscv_t *) args; for (size_t i = 0; i < len; i++) - rv->io.mem_write_b(addr + i, *((uint8_t *) val + i)); + rv->io.mem_write_b(rv, addr + i, *((uint8_t *) val + i)); return 0; } diff --git a/src/main.c b/src/main.c index 58480e07c..d9b261e6f 100644 --- a/src/main.c +++ b/src/main.c @@ -217,12 +217,21 @@ int main(int argc, char **args) .log_level = 0, .run_flag = run_flag, .profile_output_file = prof_out_file, +#if RV32_HAS(SYSTEM) + .data.system = malloc(sizeof(vm_system_t)), +#else .data.user = malloc(sizeof(vm_user_t)), +#endif .cycle_per_step = CYCLE_PER_STEP, .allow_misalign = opt_misaligned, }; +#if RV32_HAS(SYSTEM) + assert(attr.data.system); + attr.data.system->elf_program = opt_prog_name; +#else assert(attr.data.user); attr.data.user->elf_program = opt_prog_name; +#endif /* create the RISC-V runtime */ rv = rv_create(&attr); diff --git a/src/riscv.c b/src/riscv.c index ce77a5a88..c87dce552 100644 --- a/src/riscv.c +++ b/src/riscv.c @@ -165,8 +165,10 @@ void rv_remap_stdstream(riscv_t *rv, fd_stream_pair_t *fsp, uint32_t fsp_size) #define MEMIO(op) on_mem_##op #define IO_HANDLER_IMPL(type, op, RW) \ static IIF(RW)( \ - /* W */ void MEMIO(op)(riscv_word_t addr, riscv_##type##_t data), \ - /* R */ riscv_##type##_t MEMIO(op)(riscv_word_t addr)) \ + /* W */ void MEMIO(op)(UNUSED riscv_t * rv, riscv_word_t addr, \ + riscv_##type##_t data), \ + /* R */ riscv_##type##_t MEMIO(op)(UNUSED riscv_t * rv, \ + riscv_word_t addr)) \ { \ IIF(RW) \ (memory_##op(addr, (uint8_t *) &data), return memory_##op(addr)); \ @@ -258,11 +260,40 @@ riscv_t *rv_create(riscv_user_t rv_attr) .on_ebreak = ebreak_handler, .on_memcpy = memcpy_handler, .on_memset = memset_handler, + .on_trap = trap_handler, }; memcpy(&rv->io, &io, sizeof(riscv_io_t)); - } else { - /* TODO: system emulator */ } +#if RV32_HAS(SYSTEM) + else { + /* + * TODO: system emulator + * e.g., kernel image, dtb, rootfs + * + * The test suite is compiled into a single ELF file, so load it as + * an ELF executable, just like a userspace ELF. + * + */ + elf_t *elf = elf_new(); + assert(elf && elf_open(elf, (attr->data.system)->elf_program)); + + const struct Elf32_Sym *end; + if ((end = elf_get_symbol(elf, "_end"))) + attr->break_addr = end->st_value; + + assert(elf_load(elf, attr->mem)); + + /* set the entry pc */ + const struct Elf32_Ehdr *hdr = get_elf_header(elf); + assert(rv_set_pc(rv, hdr->e_entry)); + + elf_delete(elf); + + /* this variable has external linkage to mmu_io defined in system.c */ + extern riscv_io_t mmu_io; + memcpy(&rv->io, &mmu_io, sizeof(riscv_io_t)); + } +#endif /* SYSTEM */ /* default standard stream. * rv_remap_stdstream can be called to overwrite them @@ -303,20 +334,6 @@ riscv_t *rv_create(riscv_user_t rv_attr) #endif #endif -#if RV32_HAS(SYSTEM) - /* - * System simulation defaults to S-mode as - * it does not rely on M-mode software like OpenSBI. - */ - rv->priv_mode = RV_PRIV_S_MODE; - - /* not being trapped */ - rv->is_trapped = false; -#else - /* ISA simulation defaults to M-mode */ - rv->priv_mode = RV_PRIV_M_MODE; -#endif - return rv; } @@ -356,7 +373,13 @@ void rv_run(riscv_t *rv) assert(rv); vm_attr_t *attr = PRIV(rv); - assert(attr && attr->data.user && attr->data.user->elf_program); + assert(attr && +#if RV32_HAS(SYSTEM) + attr->data.system && attr->data.system->elf_program +#else + attr->data.user && attr->data.user->elf_program +#endif + ); if (attr->run_flag & RV_RUN_TRACE) rv_run_and_trace(rv); @@ -516,6 +539,21 @@ void rv_reset(riscv_t *rv, riscv_word_t pc) /* reset sp pointing to argc */ rv->X[rv_reg_sp] = stack_top; + /* reset privilege mode */ +#if RV32_HAS(SYSTEM) + /* + * System simulation defaults to S-mode as + * it does not rely on M-mode software like OpenSBI. + */ + rv->priv_mode = RV_PRIV_S_MODE; + + /* not being trapped */ + rv->is_trapped = false; +#else + /* ISA simulation defaults to M-mode */ + rv->priv_mode = RV_PRIV_M_MODE; +#endif + /* reset the csrs */ rv->csr_mtvec = 0; rv->csr_cycle = 0; @@ -541,7 +579,6 @@ void rv_reset(riscv_t *rv, riscv_word_t pc) rv->csr_misa |= MISA_M; #endif - rv->halt = false; } diff --git a/src/riscv.h b/src/riscv.h index 30d6369ba..58e3eba3d 100644 --- a/src/riscv.h +++ b/src/riscv.h @@ -91,10 +91,117 @@ enum { #define MISA_A (1 << ('A' - 'A')) #define MISA_F (1 << ('F' - 'A')) #define MISA_C (1 << ('C' - 'A')) + +/* The mstatus register keeps track of and controls the hart’s current operating + * state */ +#define MSTATUS_SIE_SHIFT 1 +#define MSTATUS_MIE_SHIFT 3 +#define MSTATUS_SPIE_SHIFT 5 +#define MSTATUS_UBE_SHIFT 6 #define MSTATUS_MPIE_SHIFT 7 +#define MSTATUS_SPP_SHIFT 8 #define MSTATUS_MPP_SHIFT 11 +#define MSTATUS_MPRV_SHIFT 17 +#define MSTATUS_SUM_SHIFT 18 +#define MSTATUS_MXR_SHIFT 18 +#define MSTATUS_TVM_SHIFT 20 +#define MSTATUS_TW_SHIFT 21 +#define MSTATUS_TSR_SHIFT 22 +#define MSTATUS_SIE (1 << MSTATUS_SIE_SHIFT) +#define MSTATUS_MIE (1 << MSTATUS_MIE_SHIFT) +#define MSTATUS_SPIE (1 << MSTATUS_SPIE_SHIFT) +#define MSTATUS_UBE (1 << MSTATUS_UBE_SHIFT) #define MSTATUS_MPIE (1 << MSTATUS_MPIE_SHIFT) +#define MSTATUS_SPP (1 << MSTATUS_SPP_SHIFT) #define MSTATUS_MPP (3 << MSTATUS_MPP_SHIFT) +#define MSTATUS_MPRV (1 << MSTATUS_MPRV_SHIFT) +#define MSTATUS_SUM (1 << MSTATUS_SUM_SHIFT) +#define MSTATUS_MXR (1 << MSTATUS_MXR_SHIFT) +#define MSTATUS_TVM (1 << MSTATUS_TVM_SHIFT) +#define MSTATUS_TW (1 << MSTATUS_TW_SHIFT) +#define MSTATUS_TSR (1 << MSTATUS_TSR_SHIFT) + +/* A restricted view of mstatus */ +#define SSTATUS_SIE_SHIFT 1 +#define SSTATUS_SPIE_SHIFT 5 +#define SSTATUS_UBE_SHIFT 6 +#define SSTATUS_SPP_SHIFT 8 +#define SSTATUS_SUM_SHIFT 18 +#define SSTATUS_MXR_SHIFT 19 +#define SSTATUS_SIE (1 << SSTATUS_SIE_SHIFT) +#define SSTATUS_SPIE (1 << SSTATUS_SPIE_SHIFT) +#define SSTATUS_UBE (1 << SSTATUS_UBE_SHIFT) +#define SSTATUS_SPP (1 << SSTATUS_SPP_SHIFT) +#define SSTATUS_SUM (1 << SSTATUS_SUM_SHIFT) +#define SSTATUS_MXR (1 << SSTATUS_MXR_SHIFT) + +#define SIP_SSIP_SHIFT 1 +#define SIP_STIP_SHIFT 5 +#define SIP_SEIP_SHIFT 9 +#define SIP_SSIP (1 << SIP_SSIP_SHIFT) +#define SIP_STIP (1 << SIP_STIP_SHIFT) +#define SIP_SEIP (1 << SIP_SEIP_SHIFT) + +#define RV_PG_SHIFT 12 +#define RV_PG_SIZE (1 << RV_PG_SHIFT) + +#define RV_PRIV_U_MODE 0 +#define RV_PRIV_S_MODE 1 +#define RV_PRIV_M_MODE 3 + +#define PTE_V (1U) +#define PTE_R (1U << 1) +#define PTE_W (1U << 2) +#define PTE_X (1U << 3) +#define PTE_U (1U << 4) +#define PTE_G (1U << 5) +#define PTE_A (1U << 6) +#define PTE_D (1U << 7) + +/* + * SBI functions must return a pair of values: + * + * struct sbiret { + * long error; + * long value; + * }; + * + * The error and value field will be set to register a0 and a1 respectively + * after the SBI function return. The error field indicate whether the + * SBI call is success or not. SBI_SUCCESS indicates success and + * SBI_ERR_NOT_SUPPORTED indicates not supported failure. The value field is + * the information based on the extension ID(EID) and SBI function ID(FID). + * + * SBI reference: https://github.com/riscv-non-isa/riscv-sbi-doc + * + */ +#define SBI_SUCCESS 0 +#define SBI_ERR_NOT_SUPPORTED -2 + +/* + * All of the functions in the base extension must be supported by + * all SBI implementations. + */ +#define SBI_EID_BASE 0x10 +#define SBI_BASE_GET_SBI_SPEC_VERSION 0 +#define SBI_BASE_GET_SBI_IMPL_ID 1 +#define SBI_BASE_GET_SBI_IMPL_VERSION 2 +#define SBI_BASE_PROBE_EXTENSION 3 +#define SBI_BASE_GET_MVENDORID 4 +#define SBI_BASE_GET_MARCHID 5 +#define SBI_BASE_GET_MIMPID 6 + +/* Make supervisor to schedule the clock for next timer event. */ +#define SBI_EID_TIMER 0x54494D45 +#define SBI_TIMER_SET_TIMER 0 + +/* Allows the supervisor to request system-level reboot or shutdown. */ +#define SBI_EID_RST 0x53525354 +#define SBI_RST_SYSTEM_RESET 0 + +#define RV_MVENDORID 0x12345678 +#define RV_MARCHID ((1ULL << 31) | 1) +#define RV_MIMPID 1 /* The mstatus register keeps track of and controls the hart’s current operating * state */ @@ -151,6 +258,44 @@ enum { #define RV_PRIV_M_MODE 3 #define RV_PRIV_IS_U_OR_S_MODE() (rv->priv_mode <= RV_PRIV_S_MODE) +enum TRAP_CODE { + INSN_MISALIGNED = 0, + ILLEGAL_INSN = 2, + BREAKPOINT = 3, + LOAD_MISALIGNED = 4, + STORE_MISALIGNED = 6, + PAGEFAULT_INSN = 12, + PAGEFAULT_LOAD = 13, + PAGEFAULT_STORE = 15, + ECALL_M = 11, +}; + +/* + * For simplicity and clarity, abstracting m/scause and m/stval + * into a cause and tval identifier respectively. + * + */ +/* clang-format off */ +#define SET_CAUSE_AND_TVAL_THEN_TRAP(rv, cause, tval) \ + { \ + /* \ + * To align rv32emu behavior with Spike \ + * \ + * If not in system mode, the __trap_handler \ + * should be be invoked \ + */ \ + IIF(RV32_HAS(SYSTEM))(rv->is_trapped = true;, ); \ + if(RV_PRIV_IS_U_OR_S_MODE()){ \ + rv->csr_scause = cause; \ + rv->csr_stval = tval; \ + } else { \ + rv->csr_mcause = cause; \ + rv->csr_mtval = tval; \ + } \ + rv->io.on_trap(rv); \ + } +/* clang-format on */ + /* * SBI functions must return a pair of values: * @@ -211,21 +356,28 @@ typedef softfloat_float32_t riscv_float_t; #endif /* memory read handlers */ -typedef riscv_word_t (*riscv_mem_ifetch)(riscv_word_t addr); -typedef riscv_word_t (*riscv_mem_read_w)(riscv_word_t addr); -typedef riscv_half_t (*riscv_mem_read_s)(riscv_word_t addr); -typedef riscv_byte_t (*riscv_mem_read_b)(riscv_word_t addr); +typedef riscv_word_t (*riscv_mem_ifetch)(riscv_t *rv, riscv_word_t addr); +typedef riscv_word_t (*riscv_mem_read_w)(riscv_t *rv, riscv_word_t addr); +typedef riscv_half_t (*riscv_mem_read_s)(riscv_t *rv, riscv_word_t addr); +typedef riscv_byte_t (*riscv_mem_read_b)(riscv_t *rv, riscv_word_t addr); /* memory write handlers */ -typedef void (*riscv_mem_write_w)(riscv_word_t addr, riscv_word_t data); -typedef void (*riscv_mem_write_s)(riscv_word_t addr, riscv_half_t data); -typedef void (*riscv_mem_write_b)(riscv_word_t addr, riscv_byte_t data); +typedef void (*riscv_mem_write_w)(riscv_t *rv, + riscv_word_t addr, + riscv_word_t data); +typedef void (*riscv_mem_write_s)(riscv_t *rv, + riscv_word_t addr, + riscv_half_t data); +typedef void (*riscv_mem_write_b)(riscv_t *rv, + riscv_word_t addr, + riscv_byte_t data); /* system instruction handlers */ typedef void (*riscv_on_ecall)(riscv_t *rv); typedef void (*riscv_on_ebreak)(riscv_t *rv); typedef void (*riscv_on_memset)(riscv_t *rv); typedef void (*riscv_on_memcpy)(riscv_t *rv); +typedef void (*riscv_on_trap)(riscv_t *rv); /* RISC-V emulator I/O interface */ typedef struct { /* memory read interface */ @@ -246,6 +398,7 @@ typedef struct { riscv_on_ebreak on_ebreak; riscv_on_memset on_memset; riscv_on_memcpy on_memcpy; + riscv_on_trap on_trap; } riscv_io_t; /* run emulation */ @@ -294,6 +447,9 @@ void syscall_handler(riscv_t *rv); /* environment call handler */ void ecall_handler(riscv_t *rv); +/* trap handler */ +void trap_handler(riscv_t *rv); + /* memset handler */ void memset_handler(riscv_t *rv); @@ -328,9 +484,20 @@ typedef struct { char *elf_program; } vm_user_t; +/* FIXME: replace with kernel image, dtb, etc */ +#if RV32_HAS(SYSTEM) +typedef struct { + char *elf_program; +} vm_system_t; +#endif /* SYSTEM */ + typedef struct { vm_user_t *user; - /* TODO: system emulator stuff */ + +#if RV32_HAS(SYSTEM) + vm_system_t *system; +#endif /* SYSTEM */ + } vm_data_t; typedef struct { diff --git a/src/rv32_template.c b/src/rv32_template.c index e8f9ee99b..0093b804a 100644 --- a/src/rv32_template.c +++ b/src/rv32_template.c @@ -162,7 +162,7 @@ RVOP( rv->X[ir->rd] = pc + 4; /* check instruction misaligned */ #if !RV32_HAS(EXT_C) - RV_EXC_MISALIGN_HANDLER(pc, insn, false, 0); + RV_EXC_MISALIGN_HANDLER(pc, INSN, false, 0); #endif struct rv_insn *taken = ir->branch_taken; if (taken) { @@ -266,7 +266,7 @@ RVOP( rv->X[ir->rd] = pc + 4; /* check instruction misaligned */ #if !RV32_HAS(EXT_C) - RV_EXC_MISALIGN_HANDLER(pc, insn, false, 0); + RV_EXC_MISALIGN_HANDLER(pc, INSN, false, 0); #endif LOOKUP_OR_UPDATE_BRANCH_HISTORY_TABLE(); goto end_op; @@ -318,7 +318,7 @@ RVOP( PC += ir->imm; \ /* check instruction misaligned */ \ IIF(RV32_HAS(EXT_C)) \ - (, RV_EXC_MISALIGN_HANDLER(pc, insn, false, 0);); \ + (, RV_EXC_MISALIGN_HANDLER(pc, INSN, false, 0);); \ struct rv_insn *taken = ir->branch_taken; \ if (taken) { \ IIF(RV32_HAS(JIT)) \ @@ -513,7 +513,7 @@ RVOP( lb, { rv->X[ir->rd] = - sign_extend_b(rv->io.mem_read_b(rv->X[ir->rs1] + ir->imm)); + sign_extend_b(rv->io.mem_read_b(rv, rv->X[ir->rs1] + ir->imm)); }, GEN({ mem; @@ -529,8 +529,8 @@ RVOP( lh, { const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(1, load, false, 1); - rv->X[ir->rd] = sign_extend_h(rv->io.mem_read_s(addr)); + RV_EXC_MISALIGN_HANDLER(1, LOAD, false, 1); + rv->X[ir->rd] = sign_extend_h(rv->io.mem_read_s(rv, addr)); }, GEN({ mem; @@ -546,8 +546,8 @@ RVOP( lw, { const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - rv->X[ir->rd] = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + rv->X[ir->rd] = rv->io.mem_read_w(rv, addr); }, GEN({ mem; @@ -561,7 +561,7 @@ RVOP( /* LBU: Load Byte Unsigned */ RVOP( lbu, - { rv->X[ir->rd] = rv->io.mem_read_b(rv->X[ir->rs1] + ir->imm); }, + { rv->X[ir->rd] = rv->io.mem_read_b(rv, rv->X[ir->rs1] + ir->imm); }, GEN({ mem; rald, VR0, rs1; @@ -576,8 +576,8 @@ RVOP( lhu, { const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(1, load, false, 1); - rv->X[ir->rd] = rv->io.mem_read_s(addr); + RV_EXC_MISALIGN_HANDLER(1, LOAD, false, 1); + rv->X[ir->rd] = rv->io.mem_read_s(rv, addr); }, GEN({ mem; @@ -597,7 +597,7 @@ RVOP( /* SB: Store Byte */ RVOP( sb, - { rv->io.mem_write_b(rv->X[ir->rs1] + ir->imm, rv->X[ir->rs2]); }, + { rv->io.mem_write_b(rv, rv->X[ir->rs1] + ir->imm, rv->X[ir->rs2]); }, GEN({ mem; rald, VR0, rs1; @@ -612,8 +612,8 @@ RVOP( sh, { const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(1, store, false, 1); - rv->io.mem_write_s(addr, rv->X[ir->rs2]); + RV_EXC_MISALIGN_HANDLER(1, STORE, false, 1); + rv->io.mem_write_s(rv, addr, rv->X[ir->rs2]); }, GEN({ mem; @@ -629,8 +629,8 @@ RVOP( sw, { const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->X[ir->rs2]); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->X[ir->rs2]); }, GEN({ mem; @@ -1041,7 +1041,7 @@ RVOP( assert; /* FIXME: Implement */ })) -/* MRET: return from traps in U-mode */ +/* MRET: return from traps in M-mode */ RVOP( mret, { @@ -1371,9 +1371,9 @@ RVOP( lrw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); if (ir->rd) - rv->X[ir->rd] = rv->io.mem_read_w(addr); + rv->X[ir->rd] = rv->io.mem_read_w(rv, addr); /* skip registration of the 'reservation set' * FIXME: unimplemented */ @@ -1390,8 +1390,8 @@ RVOP( * FIXME: unimplemented */ const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->X[ir->rs2]); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->X[ir->rs2]); rv->X[ir->rd] = 0; }, GEN({ @@ -1403,12 +1403,12 @@ RVOP( amoswapw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; - rv->io.mem_write_w(addr, value2); + rv->io.mem_write_w(rv, addr, value2); }, GEN({ assert; /* FIXME: Implement */ @@ -1419,13 +1419,13 @@ RVOP( amoaddw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t res = value1 + value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1436,13 +1436,13 @@ RVOP( amoxorw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t res = value1 ^ value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1453,13 +1453,13 @@ RVOP( amoandw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t res = value1 & value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1470,13 +1470,13 @@ RVOP( amoorw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t res = value1 | value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1487,15 +1487,15 @@ RVOP( amominw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const int32_t a = value1; const int32_t b = value2; const uint32_t res = a < b ? value1 : value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1506,15 +1506,15 @@ RVOP( amomaxw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const int32_t a = value1; const int32_t b = value2; const uint32_t res = a > b ? value1 : value2; - rv->io.mem_write_w(addr, res); + rv->io.mem_write_w(rv, addr, res); }, GEN({ assert; /* FIXME: Implement */ @@ -1525,13 +1525,13 @@ RVOP( amominuw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t ures = value1 < value2 ? value1 : value2; - rv->io.mem_write_w(addr, ures); + rv->io.mem_write_w(rv, addr, ures); }, GEN({ assert; /* FIXME: Implement */ @@ -1542,13 +1542,13 @@ RVOP( amomaxuw, { const uint32_t addr = rv->X[ir->rs1]; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - const uint32_t value1 = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + const uint32_t value1 = rv->io.mem_read_w(rv, addr); const uint32_t value2 = rv->X[ir->rs2]; if (ir->rd) rv->X[ir->rd] = value1; const uint32_t ures = value1 > value2 ? value1 : value2; - rv->io.mem_write_w(addr, ures); + rv->io.mem_write_w(rv, addr, ures); }, GEN({ assert; /* FIXME: Implement */ @@ -1564,8 +1564,8 @@ RVOP( { /* copy into the float register */ const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - rv->F[ir->rd].v = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + rv->F[ir->rd].v = rv->io.mem_read_w(rv, addr); }, GEN({ assert; /* FIXME: Implement */ @@ -1577,8 +1577,8 @@ RVOP( { /* copy from float registers */ const uint32_t addr = rv->X[ir->rs1] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->F[ir->rs2].v); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->F[ir->rs2].v); }, GEN({ assert; /* FIXME: Implement */ @@ -1938,8 +1938,8 @@ RVOP( clw, { const uint32_t addr = rv->X[ir->rs1] + (uint32_t) ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, true, 1); - rv->X[ir->rd] = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, true, 1); + rv->X[ir->rd] = rv->io.mem_read_w(rv, addr); }, GEN({ mem; @@ -1959,8 +1959,8 @@ RVOP( csw, { const uint32_t addr = rv->X[ir->rs1] + (uint32_t) ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, true, 1); - rv->io.mem_write_w(addr, rv->X[ir->rs2]); + RV_EXC_MISALIGN_HANDLER(3, STORE, true, 1); + rv->io.mem_write_w(rv, addr, rv->X[ir->rs2]); }, GEN({ mem; @@ -2312,8 +2312,8 @@ RVOP( clwsp, { const uint32_t addr = rv->X[rv_reg_sp] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, true, 1); - rv->X[ir->rd] = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, true, 1); + rv->X[ir->rd] = rv->io.mem_read_w(rv, addr); }, GEN({ mem; @@ -2421,8 +2421,8 @@ RVOP( cswsp, { const uint32_t addr = rv->X[rv_reg_sp] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, true, 1); - rv->io.mem_write_w(addr, rv->X[ir->rs2]); + RV_EXC_MISALIGN_HANDLER(3, STORE, true, 1); + rv->io.mem_write_w(rv, addr, rv->X[ir->rs2]); }, GEN({ mem; @@ -2440,8 +2440,8 @@ RVOP( cflwsp, { const uint32_t addr = rv->X[rv_reg_sp] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - rv->F[ir->rd].v = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + rv->F[ir->rd].v = rv->io.mem_read_w(rv, addr); }, GEN({ assert; /* FIXME: Implement */ @@ -2452,8 +2452,8 @@ RVOP( cfswsp, { const uint32_t addr = rv->X[rv_reg_sp] + ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->F[ir->rs2].v); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->F[ir->rs2].v); }, GEN({ assert; /* FIXME: Implement */ @@ -2464,8 +2464,8 @@ RVOP( cflw, { const uint32_t addr = rv->X[ir->rs1] + (uint32_t) ir->imm; - RV_EXC_MISALIGN_HANDLER(3, load, false, 1); - rv->F[ir->rd].v = rv->io.mem_read_w(addr); + RV_EXC_MISALIGN_HANDLER(3, LOAD, false, 1); + rv->F[ir->rd].v = rv->io.mem_read_w(rv, addr); }, GEN({ assert; /* FIXME: Implement */ @@ -2476,8 +2476,8 @@ RVOP( cfsw, { const uint32_t addr = rv->X[ir->rs1] + (uint32_t) ir->imm; - RV_EXC_MISALIGN_HANDLER(3, store, false, 1); - rv->io.mem_write_w(addr, rv->F[ir->rs2].v); + RV_EXC_MISALIGN_HANDLER(3, STORE, false, 1); + rv->io.mem_write_w(rv, addr, rv->F[ir->rs2].v); }, GEN({ assert; /* FIXME: Implement */ diff --git a/src/syscall.c b/src/syscall.c index 1c25408a0..010212185 100644 --- a/src/syscall.c +++ b/src/syscall.c @@ -34,9 +34,11 @@ _(brk, 214) \ _(clock_gettime, 403) \ _(open, 1024) \ - _(sbi_base, 0x10) \ - _(sbi_timer, 0x54494D45) \ - _(sbi_rst, 0x53525354) \ + IIF(RV32_HAS(SYSTEM))( \ + _(sbi_base, 0x10) \ + _(sbi_timer, 0x54494D45) \ + _(sbi_rst, 0x53525354) \ + ) \ IIF(RV32_HAS(SDL))( \ _(draw_frame, 0xBEEF) \ _(setup_queue, 0xC0DE) \ @@ -372,6 +374,8 @@ extern void syscall_setup_audio(riscv_t *rv); extern void syscall_control_audio(riscv_t *rv); #endif +#if RV32_HAS(SYSTEM) + /* SBI related system calls */ static void syscall_sbi_timer(riscv_t *rv) { @@ -459,6 +463,7 @@ static void syscall_sbi_rst(riscv_t *rv) break; } } +#endif /* SYSTEM */ void syscall_handler(riscv_t *rv) { diff --git a/src/system.c b/src/system.c new file mode 100644 index 000000000..f1be8bf82 --- /dev/null +++ b/src/system.c @@ -0,0 +1,286 @@ +/* + * rv32emu is freely redistributable under the MIT License. See the file + * "LICENSE" for information on usage and redistribution of this file. + */ + +#include "system.h" + +static bool ppn_is_valid(riscv_t *rv, uint32_t ppn) +{ + vm_attr_t *attr = PRIV(rv); + const uint32_t nr_pg_max = attr->mem_size / RV_PG_SIZE; + return ppn < nr_pg_max; +} + +#define PAGE_TABLE(ppn) \ + ppn_is_valid(rv, ppn) \ + ? (uint32_t *) (attr->mem->mem_base + (ppn << (RV_PG_SHIFT))) \ + : NULL + +/* Walk through page tables and get the corresponding PTE by virtual address if + * exists + * @rv: RISC-V emulator + * @addr: virtual address + * @level: the level of which the PTE is located + * @return: NULL if a not found or fault else the corresponding PTE + */ +static uint32_t *mmu_walk(riscv_t *rv, const uint32_t addr, uint32_t *level) +{ + vm_attr_t *attr = PRIV(rv); + uint32_t ppn = rv->csr_satp & MASK(22); + + /* root page table */ + uint32_t *page_table = PAGE_TABLE(ppn); + if (!page_table) + return NULL; + + for (int i = 1; i >= 0; i--) { + *level = 2 - i; + uint32_t vpn = + (addr >> RV_PG_SHIFT >> (i * (RV_PG_SHIFT - 2))) & MASK(10); + uint32_t *pte = page_table + vpn; + + uint8_t XWRV_bit = (*pte & MASK(4)); + switch (XWRV_bit) { + case NEXT_PG_TBL: /* next level of the page table */ + ppn = (*pte >> (RV_PG_SHIFT - 2)); + page_table = PAGE_TABLE(ppn); + if (!page_table) + return NULL; + break; + case RO_PAGE: + case RW_PAGE: + case EO_PAGE: + case RX_PAGE: + case RWX_PAGE: + ppn = (*pte >> (RV_PG_SHIFT - 2)); + if (*level == 1 && + unlikely(ppn & MASK(10))) /* misaligned superpage */ + return NULL; + return pte; /* leaf PTE */ + case RESRV_PAGE1: + case RESRV_PAGE2: + default: + return NULL; + } + } + + return NULL; +} + +/* Verify the PTE and generate corresponding faults if needed + * @op: the operation + * @rv: RISC-V emulator + * @pte: to be verified pte + * @addr: the corresponding virtual address to cause fault + * @return: false if a any fault is generated which caused by violating the + * access permission else true + */ +/* FIXME: handle access fault, addr out of range check */ +#define MMU_FAULT_CHECK(op, rv, pte, addr, access_bits) \ + mmu_##op##_fault_check(rv, pte, addr, access_bits) +#define MMU_FAULT_CHECK_IMPL(op, pgfault) \ + static bool mmu_##op##_fault_check(riscv_t *rv, uint32_t *pte, \ + uint32_t addr, uint32_t access_bits) \ + { \ + uint32_t scause; \ + uint32_t stval = addr; \ + switch (access_bits) { \ + case PTE_R: \ + scause = PAGEFAULT_LOAD; \ + break; \ + case PTE_W: \ + scause = PAGEFAULT_STORE; \ + break; \ + case PTE_X: \ + scause = PAGEFAULT_INSN; \ + break; \ + default: \ + __UNREACHABLE; \ + break; \ + } \ + if (pte && (!(*pte & PTE_V))) { \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, scause, stval); \ + return false; \ + } \ + if (!(pte && (*pte & access_bits))) { \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, scause, stval); \ + return false; \ + } \ + /* \ + * (1) When MXR=0, only loads from pages marked readable (R=1) will \ + * succeed. \ + * \ + * (2) When MXR=1, loads from pages marked either readable or \ + * executable (R=1 or X=1) will succeed. \ + */ \ + if (pte && ((!(SSTATUS_MXR & rv->csr_sstatus) && !(*pte & PTE_R) && \ + (access_bits == PTE_R)) || \ + ((SSTATUS_MXR & rv->csr_sstatus) && \ + !((*pte & PTE_R) | (*pte & PTE_X)) && \ + (access_bits == PTE_R)))) { \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, scause, stval); \ + return false; \ + } \ + /* \ + * When SUM=0, S-mode memory accesses to pages that are accessible by \ + * U-mode will fault. \ + */ \ + if (pte && rv->priv_mode == RV_PRIV_S_MODE && \ + !(SSTATUS_SUM & rv->csr_sstatus) && (*pte & PTE_U)) { \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, scause, stval); \ + return false; \ + } \ + /* PTE not found, map it in handler */ \ + if (!pte) { \ + SET_CAUSE_AND_TVAL_THEN_TRAP(rv, scause, stval); \ + return false; \ + } \ + /* valid PTE */ \ + return true; \ + } + +MMU_FAULT_CHECK_IMPL(ifetch, pagefault_insn) +MMU_FAULT_CHECK_IMPL(read, pagefault_load) +MMU_FAULT_CHECK_IMPL(write, pagefault_store) + +#define get_ppn_and_offset(ppn, offset) \ + uint32_t ppn; \ + uint32_t offset; \ + do { \ + ppn = *pte >> (RV_PG_SHIFT - 2) << RV_PG_SHIFT; \ + offset = level == 1 ? addr & MASK((RV_PG_SHIFT + 10)) \ + : addr & MASK(RV_PG_SHIFT); \ + } while (0) + +uint32_t mmu_ifetch(riscv_t *rv, const uint32_t addr) +{ + if (!rv->csr_satp) + return memory_ifetch(addr); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(ifetch, rv, pte, addr, PTE_X); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + return memory_ifetch(ppn | offset); +} + +uint32_t mmu_read_w(riscv_t *rv, const uint32_t addr) +{ + if (!rv->csr_satp) + return memory_read_w(addr); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(read, rv, pte, addr, PTE_R); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + return memory_read_w(ppn | offset); +} + +uint16_t mmu_read_s(riscv_t *rv, const uint32_t addr) +{ + if (!rv->csr_satp) + return memory_read_s(addr); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(read, rv, pte, addr, PTE_R); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + return memory_read_s(ppn | offset); +} + +uint8_t mmu_read_b(riscv_t *rv, const uint32_t addr) +{ + if (!rv->csr_satp) + return memory_read_b(addr); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(read, rv, pte, addr, PTE_R); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + return memory_read_b(ppn | offset); +} + +void mmu_write_w(riscv_t *rv, const uint32_t addr, const uint32_t val) +{ + if (!rv->csr_satp) + return memory_write_w(addr, (uint8_t *) &val); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(write, rv, pte, addr, PTE_W); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + memory_write_w(ppn | offset, (uint8_t *) &val); +} + +void mmu_write_s(riscv_t *rv, const uint32_t addr, const uint16_t val) +{ + if (!rv->csr_satp) + return memory_write_s(addr, (uint8_t *) &val); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(write, rv, pte, addr, PTE_W); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + memory_write_s(ppn | offset, (uint8_t *) &val); +} + +void mmu_write_b(riscv_t *rv, const uint32_t addr, const uint8_t val) +{ + if (!rv->csr_satp) + return memory_write_b(addr, (uint8_t *) &val); + + uint32_t level; + uint32_t *pte = mmu_walk(rv, addr, &level); + bool ok = MMU_FAULT_CHECK(write, rv, pte, addr, PTE_W); + if (unlikely(!ok)) { + pte = mmu_walk(rv, addr, &level); + } + + get_ppn_and_offset(ppn, offset); + memory_write_b(ppn | offset, (uint8_t *) &val); +} + +riscv_io_t mmu_io = { + /* memory read interface */ + .mem_ifetch = mmu_ifetch, + .mem_read_w = mmu_read_w, + .mem_read_s = mmu_read_s, + .mem_read_b = mmu_read_b, + + /* memory write interface */ + .mem_write_w = mmu_write_w, + .mem_write_s = mmu_write_s, + .mem_write_b = mmu_write_b, + + /* system services or essential routines */ + .on_ecall = ecall_handler, + .on_ebreak = ebreak_handler, + .on_memcpy = memcpy_handler, + .on_memset = memset_handler, + .on_trap = trap_handler, +}; diff --git a/src/system.h b/src/system.h new file mode 100644 index 000000000..c4780c383 --- /dev/null +++ b/src/system.h @@ -0,0 +1,47 @@ +/* + * rv32emu is freely redistributable under the MIT License. See the file + * "LICENSE" for information on usage and redistribution of this file. + */ + +#pragma once + +#include "riscv_private.h" + +/* PTE XWRV bit in order */ +enum SV32_PTE_PERM { + NEXT_PG_TBL = 0b0001, + RO_PAGE = 0b0011, + RW_PAGE = 0b0111, + EO_PAGE = 0b1001, + RX_PAGE = 0b1011, + RWX_PAGE = 0b1111, + RESRV_PAGE1 = 0b0101, + RESRV_PAGE2 = 0b1101, +}; + +/* + * The IO handler that operates when the Memory Management Unit (MMU) + * is enabled during system emulation is responsible for managing + * input/output operations. These callbacks are designed to implement + * the riscv_io_t interface, ensuring compatibility and consistency to + * the structure required by the interface. As a result, the riscv_io_t + * interface can be reused. + */ +uint32_t mmu_ifetch(riscv_t *rv, const uint32_t addr); +uint32_t mmu_read_w(riscv_t *rv, const uint32_t addr); +uint16_t mmu_read_s(riscv_t *rv, const uint32_t addr); +uint8_t mmu_read_b(riscv_t *rv, const uint32_t addr); +void mmu_write_w(riscv_t *rv, const uint32_t addr, const uint32_t val); +void mmu_write_s(riscv_t *rv, const uint32_t addr, const uint16_t val); +void mmu_write_b(riscv_t *rv, const uint32_t addr, const uint8_t val); + +/* + * Trap might occurs during block emulation. For instance, page fault. + * In order to handle trap, we have to escape from block and execute + * registered trap handler. This trap_handler function helps to execute + * the registered trap handler, PC by PC. Once the trap is handled, + * resume the previous execution flow where cause the trap. + * + * Now, rv32emu supports misaligned access and page fault handling. + */ +void trap_handler(riscv_t *rv); diff --git a/tests/system/mmu/Makefile b/tests/system/mmu/Makefile new file mode 100644 index 000000000..259f3eefe --- /dev/null +++ b/tests/system/mmu/Makefile @@ -0,0 +1,36 @@ +PREFIX ?= riscv-none-elf- +ARCH = -march=rv32izicsr +LINKER_SCRIPT = linker.ld + +DEBUG_CFLAGS = -g +CFLAGS = -c -march=rv32i_zicsr +LDFLAGS = -T +EXEC = vm.elf + +CC = $(PREFIX)gcc +AS = $(PREFIX)as +LD = $(PREFIX)ld +OBJDUMP = $(PREFIX)objdump + +# Locate libgcc.a dynamically, as the itoa function uses division and modulo operators, +# which rely on __udivsi3 and __umodsi3, both provided by libgcc +LIBGCC = $(shell $(CC) -print-libgcc-file-name) +LIBGCC_PATH = $(shell dirname $(LIBGCC)) + +deps = main.o setup.o vm_setup.o + +all: + $(CC) $(DEBUG_CLAGS) $(CFLAGS) main.c + $(CC) $(DEBUG_CLAGS) $(CFLAGS) vm_setup.c + $(AS) $(DEBUG_CLAGS) $(ARCH) setup.S -o setup.o + $(LD) $(LDFLAGS) $(LINKER_SCRIPT) -o $(EXEC) $(deps) -L$(LIBGCC_PATH) -lgcc + +dump: + $(OBJDUMP) -DS $(EXEC) | less + +# __udivsi3 and __umodsi3 can be found +nm: + $(NM) $(LIBGCC) | less + +clean: + rm $(EXEC) $(deps) diff --git a/tests/system/mmu/linker.ld b/tests/system/mmu/linker.ld new file mode 100644 index 000000000..74546df25 --- /dev/null +++ b/tests/system/mmu/linker.ld @@ -0,0 +1,31 @@ +OUTPUT_ARCH( "riscv" ) + +ENTRY(_start) + +SECTIONS +{ + . = 0x00000000; + .text.main : { *(.text.main) } + . = ALIGN(0x1000); + .mystring : { *(.mystring) } + .data.main : { *(.data.main) } + .bss.main : { *(.bss.main) } + + .text : { + . = 0x7fffeffc; + *(.text.setup) + *(.text.vm_setup) + } + . = ALIGN(0x1000); + .data : { + *(.data.setup) + *(.data.vm_setup) + } + .bss : { + *(.bss.setup) + *(.bss.vm_setup) + } + + . = ALIGN(0x1000); + _end = .; +} diff --git a/tests/system/mmu/main.c b/tests/system/mmu/main.c new file mode 100644 index 000000000..7036a3aa8 --- /dev/null +++ b/tests/system/mmu/main.c @@ -0,0 +1,102 @@ +#define SECTION_TEXT_MAIN __attribute__((section(".text.main"))) +#define SECTION_DATA_MAIN __attribute__((section(".data.main"))) +#define SECTION_BSS_MAIN __attribute__((section(".bss.main"))) + +#define printstr(ptr, length) \ + do { \ + asm volatile( \ + "add a7, x0, 0x40;" \ + "add a0, x0, 0x1;" /* stdout */ \ + "add a1, x0, %0;" \ + "mv a2, %1;" /* length character */ \ + "ecall;" \ + : \ + : "r"(ptr), "r"(length) \ + : "a0", "a1", "a2", "a7"); \ + } while (0) + +#define TEST_OUTPUT(msg, length) printstr(msg, length) + +#define TEST_LOGGER(msg) \ + { \ + char _msg[] = msg; \ + TEST_OUTPUT(_msg, sizeof(_msg) - 1); \ + } + +#define SUCCESS 0 +#define FAIL 1 + +__attribute__((section(".mystring"))) const char pagefault_load_str[] = + "rv32emu"; + +extern void _exit(int status); + +int SECTION_TEXT_MAIN main() +{ + /* instruction fetch page fault test */ + int x = 100; /* trigger instruction page fault */ + int y = 200; + int z = x + y; + TEST_LOGGER("INSTRUCTION FETCH PAGE FAULT TEST PASSED!\n"); + + char buf[8]; + /* data load page fault test */ + /* Clear buffer */ + for (int i = 0; i < 8; i++) { + buf[i] = 0; + } + + char *qtr = (char *) 0x1000; /* first data page */ + /* should trigger load page fault and load pagefault_load_str */ + for (int i = 0; i < 8; i++) { + qtr = (char *) 0x1000; /* FIXME: weird result without this */ + buf[i] = *(qtr + i); + } + + for (int i = 0; i < 8; i++) { /* should not trigger load page fault */ + if (buf[i] != *(qtr + i)) { + TEST_LOGGER("[LOAD PAGE FAULT TEST] rv32emu string not match\n") + _exit(FAIL); + } + } + TEST_LOGGER("LOAD PAGE FAULT TEST PASSED!\n"); + + /* data store page fault test */ + /* Clear buffer */ + for (int i = 0; i < 8; i++) { + buf[i] = 0; + } + + char *ptr = (char *) 0x2000; /* second data page */ + *ptr = 'r'; /* trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 1) = 'v'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 2) = '3'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 3) = '2'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 4) = 'e'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 5) = 'm'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 6) = 'u'; /* should not trigger store page fault */ + ptr = (char *) 0x2000; /* FIXME: weird result without this */ + *(ptr + 7) = '\0'; /* should not trigger store page fault */ + + /* should not trigger load page fault */ + for (int i = 0; i < 8; i++) { + buf[i] = *(ptr + i); + } + + /* should not trigger load page fault */ + for (int i = 0; i < 8; i++) { + if (buf[i] != *(ptr + i)) { + TEST_LOGGER("[STORE PAGE FAULT TEST] rv32emu string not match\n") + _exit(FAIL); + } + } + TEST_LOGGER("STORE PAGE FAULT TEST PASSED!\n"); + + _exit(SUCCESS); +} diff --git a/tests/system/mmu/setup.S b/tests/system/mmu/setup.S new file mode 100644 index 000000000..177cdfb2a --- /dev/null +++ b/tests/system/mmu/setup.S @@ -0,0 +1,162 @@ +.section .text.setup +TRAPFRAME_SIZE = 35 * 4 +PG_SIZE = 4096 +STACK_TOP = _end + PG_SIZE + +# FIXME: implement proper machine trap vector +# Since I assume that all interrupts and exceptions are +# handled by S-mode software, so the machine trap +# vector is not that much important in here +machine_trap_vector: + j _exit; + +.globl _start +_start: + # init regs + li x1, 0 + li x2, 0 + li x3, 0 + li x4, 0 + li x5, 0 + li x6, 0 + li x7, 0 + li x8, 0 + li x9, 0 + li x10, 0 + li x11, 0 + li x12, 0 + li x13, 0 + li x14, 0 + li x15, 0 + li x16, 0 + li x17, 0 + li x18, 0 + li x19, 0 + li x20, 0 + li x21, 0 + li x22, 0 + li x23, 0 + li x24, 0 + li x25, 0 + li x26, 0 + li x27, 0 + li x28, 0 + li x29, 0 + li x30, 0 + li x31, 0 + + la sp, STACK_TOP - TRAPFRAME_SIZE + csrw mscratch, sp; + + #la t0, machine_trap_vector + #csrw mtvec, t0 + #csrr t0, mtvec # for debugging + + # init virtual memory + j vm_boot + +.globl _exit +_exit: + li a7, 93 + ecall + +.globl user_entry +user_entry: + la sp, STACK_TOP - TRAPFRAME_SIZE + jalr x0, 0x4 # jump to user space main + +.globl supervisor_trap_entry +supervisor_trap_entry: + # get trapframe pointer (save a0 into scratch) + csrrw a0, sscratch, a0; + + # push regs into trapframe + sw x1, 0*4(a0); + sw x2, 1*4(a0); + sw x3, 2*4(a0); + sw x4, 3*4(a0); + sw x5, 4*4(a0); + sw x6, 5*4(a0); + sw x7, 6*4(a0); + sw x8, 7*4(a0); + sw x9, 8*4(a0); + sw x11, 10*4(a0); + sw x12, 11*4(a0); + sw x13, 12*4(a0); + sw x14, 13*4(a0); + sw x15, 14*4(a0); + sw x16, 15*4(a0); + sw x17, 16*4(a0); + sw x18, 17*4(a0); + sw x19, 18*4(a0); + sw x20, 19*4(a0); + sw x21, 20*4(a0); + sw x22, 21*4(a0); + sw x23, 22*4(a0); + sw x24, 23*4(a0); + sw x25, 24*4(a0); + sw x26, 25*4(a0); + sw x27, 26*4(a0); + sw x28, 27*4(a0); + sw x29, 28*4(a0); + sw x20, 29*4(a0); + sw x31, 30*4(a0); + + # load stack pointer and save trapframe pointer into scratch + csrrw t0, sscratch, a0; + sw t0, 9*4(a0); + + # push status, epc, badaddr, cause + csrr t0, sstatus; + sw t0, 31*4(a0); + csrr t0, sepc; + sw t0, 32*4(a0); + csrr t0, stval; + sw t0, 33*4(a0); + csrr t0, scause; + sw t0, 34*4(a0); + + csrr sp, sscratch; + j handle_trap; + +.globl pop_tf +pop_tf: + # a0 need to save trapframe pointer + # pop epc and regs from trapframe + lw t0, 32*4(a0) + csrw sepc, t0 + lw x1, 0*4(a0) + lw x2, 1*4(a0) + lw x3, 2*4(a0) + lw x4, 3*4(a0) + lw x5, 4*4(a0) + lw x6, 5*4(a0) + lw x7, 6*4(a0) + lw x8, 7*4(a0) + lw x9, 8*4(a0) + lw x11, 10*4(a0) + lw x12, 11*4(a0) + lw x13, 12*4(a0) + lw x14, 13*4(a0) + lw x15, 14*4(a0) + lw x16, 15*4(a0) + lw x17, 16*4(a0) + lw x18, 17*4(a0) + lw x19, 18*4(a0) + lw x20, 19*4(a0) + lw x21, 20*4(a0) + lw x22, 21*4(a0) + lw x23, 22*4(a0) + lw x24, 23*4(a0) + lw x25, 24*4(a0) + lw x26, 25*4(a0) + lw x27, 26*4(a0) + lw x28, 27*4(a0) + lw x29, 28*4(a0) + lw x20, 29*4(a0) + lw x31, 30*4(a0) + + # save trapframe pointer to sscratch + csrrw a0, sscratch, a0; + + sret diff --git a/tests/system/mmu/vm_setup.c b/tests/system/mmu/vm_setup.c new file mode 100644 index 000000000..015c270d1 --- /dev/null +++ b/tests/system/mmu/vm_setup.c @@ -0,0 +1,420 @@ +/* + * Reference: + * 1. https://github.com/sifive/example-vm-test + * 2. https://github.com/yutongshen/RISC-V-Simulator + */ + +#include +#include +#include + +#if !defined(__GNUC__) || !defined(__riscv) || 32 != __riscv_xlen +#error "GNU Toolchain for 32-bit RISC-V is required" +#endif + +#define SECTION_TEXT_VMSETUP __attribute__((section(".text.vm_setup"))) +#define SECTION_DATA_VMSETUP __attribute__((section(".data.vm_setup"))) +#define SECTION_BSS_VMSETUP __attribute__((section(".bss.vm_setup"))) + +#define assert(x) \ + do { \ + if (!x) { \ + printf("Assertion failed '%s' at line %d of '%s'\n", #x); \ + _exit(1); \ + } \ + } while (0) + +#define read_csr(reg) \ + ({ \ + uint32_t __tmp; \ + asm volatile("csrr %0, " #reg : "=r"(__tmp)); \ + __tmp; \ + }) + +#define write_csr(reg, val) ({ asm volatile("csrw " #reg ", %0" ::"rK"(val)); }) + +#define swap_csr(reg, val) \ + ({ \ + uint32_t __tmp; \ + asm volatile("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "rK"(val)); \ + __tmp; \ + }) + +#define set_csr(reg, bit) \ + ({ \ + uint32_t __tmp; \ + asm volatile("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); \ + __tmp; \ + }) + +#define clear_csr(reg, bit) \ + ({ \ + uint32_t __tmp; \ + asm volatile("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); \ + __tmp; \ + }) + +typedef struct { + uint32_t ra; + uint32_t sp; + uint32_t gp; + uint32_t tp; + uint32_t t0; + uint32_t t1; + uint32_t t2; + uint32_t s0; + uint32_t s1; + uint32_t a0; + uint32_t a1; + uint32_t a2; + uint32_t a3; + uint32_t a4; + uint32_t a5; + uint32_t a6; + uint32_t a7; + uint32_t s2; + uint32_t s3; + uint32_t s4; + uint32_t s5; + uint32_t s6; + uint32_t s7; + uint32_t s8; + uint32_t s9; + uint32_t s10; + uint32_t s11; + uint32_t t3; + uint32_t t4; + uint32_t t5; + uint32_t t6; + uint32_t status; + uint32_t epc; + uint32_t badvaddr; + uint32_t cause; +} trapframe_t; + +#define SV32_MODE 0x80000000 +#define BARE_MODE 0x00000000 +#define PG_SHIFT 12 +#define PG_SIZE (1U << 12) +#define FREE_FRAME_BASE 0x400000 +#define MAX_TEST_PG 32 /* FREE_FRAME_BASE - 0x41ffff */ +#define MEGA_PG (PG_SIZE << 10) /* 4 MiB */ + +#define CAUSE_USER_ECALL (1U << 8) +#define CAUSE_SUPERVISOR_ECALL (1U << 9) +#define CAUSE_FETCH_PAGE_FAULT (1U << 12) +#define CAUSE_LOAD_PAGE_FAULT (1U << 13) +#define CAUSE_STORE_PAGE_FAULT (1U << 15) + +#define SSTATUS_SUM (1U << 18) + +#define PTE_V (1U) +#define PTE_R (1U << 1) +#define PTE_W (1U << 2) +#define PTE_X (1U << 3) +#define PTE_U (1U << 4) +#define PTE_G (1U << 5) +#define PTE_A (1U << 6) +#define PTE_D (1U << 7) + +extern void main(); +extern void _start(); +extern int user_entry(); +extern void supervisor_trap_entry(); +extern void pop_tf(trapframe_t *); +extern void _exit(int status); + +typedef uint32_t pte_t; +typedef struct { + pte_t addr; + void *next; +} freelist_t; + +freelist_t freelist_nodes[MAX_TEST_PG] SECTION_BSS_VMSETUP; +freelist_t *SECTION_BSS_VMSETUP freelist_head; +freelist_t *SECTION_BSS_VMSETUP freelist_tail; + +#define l1pt pt[0] +#define user_l1pt pt[1] +#define NPT 2 +#define PTES_PER_PT (1U << 10) +#define PTE_PPN_SHIFT 10 +pte_t pt[NPT][PTES_PER_PT] SECTION_BSS_VMSETUP + __attribute__((aligned(PG_SIZE))); + +#define MASK(n) (~((~0U << (n)))) +#define pa2kva(x) (((pte_t) (x)) - ((pte_t) (_start)) - MEGA_PG) +#define uva2kva(x) (((pte_t) (x)) - MEGA_PG) + +void *SECTION_TEXT_VMSETUP memcpy(void *ptr, void *src, size_t len) +{ + uint32_t *word_ptr = ptr; + uint32_t *word_src = src; + + while (len >= 4) { + *word_ptr++ = *word_src++; + len -= 4; + } + + char *byte_ptr = (char *) word_ptr; + char *byte_src = (char *) word_src; + while (len--) { + *byte_ptr++ = *byte_src++; + } + + return ptr; +} + +void *SECTION_TEXT_VMSETUP memset(void *ptr, int val, size_t len) +{ + uint32_t *word_ptr = ptr; + uint32_t write_word = (char) val; + write_word |= write_word << 8; + write_word |= write_word << 16; + + while (len >= 4) { + *word_ptr++ = write_word; + len -= 4; + } + + char *byte_ptr = (char *) word_ptr; + while (len--) { + *byte_ptr++ = val; + } + + return ptr; +} + +#define SYSCALL_WRITE 64 +int SECTION_TEXT_VMSETUP putchar(int ch) +{ + int syscall_nr = SYSCALL_WRITE; + asm volatile( + "mv a7, %0;" + "li a0, 0x1;" /* stdout */ + "add a1, x0, %1;" + "li a2, 0x1;" /* one character */ + "ecall;" + : + : "r"(syscall_nr), "r"(&ch) + : "a0", "a1", "a2", "a7"); + + return 0; +} + +int SECTION_TEXT_VMSETUP _puts(const char *s) +{ + int res = 0; + while (*s) { + putchar(*(s++)); + ++res; + } + + return res; +} + +int SECTION_TEXT_VMSETUP puts(const char *s) +{ + int res = 1; + res += _puts(s); + putchar('\n'); + + return res; +} + +char *SECTION_TEXT_VMSETUP itoa(uint32_t value, + int base, + int min_len, + char fill_char) +{ + static char digitals[16] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + static char str[64]; + char *ptr = str + 63; + int tmp; + *ptr = 0; + do { + if (!value) + *--ptr = fill_char; + else + *--ptr = digitals[value % base]; + value /= base; + } while (--min_len > 0 || value); + + return ptr; +} + +int SECTION_TEXT_VMSETUP printf(const char *format, ...) +{ + const char *ptr = format; + int res = 0; + + va_list va; + va_start(va, format); + + while (*ptr) { + if (*ptr == '%') { + int min_len = 0; + char fill_char = 0; + loop1: + switch (*(++ptr)) { + case 'c': + assert(!(min_len || fill_char)); + ++res; + putchar(va_arg(va, int)); + break; + case 's': + assert(!(min_len || fill_char)); + res += _puts(va_arg(va, const char *)); + break; + case 'x': { + assert(!(!fill_char ^ !min_len)); + uint32_t n = va_arg(va, uint32_t); + res += _puts(itoa(n, 16, min_len, min_len ? fill_char : '0')); + } break; + case 'd': { + assert(!(!fill_char ^ !min_len)); + int64_t n = va_arg(va, int64_t); + if (n < 0) { + ++res; + putchar('-'); + n = -n; + } + res += _puts(itoa(n, 10, min_len, min_len ? fill_char : '0')); + } break; + case '%': + ++res; + putchar('%'); + break; + case '1': + assert(fill_char); + min_len *= 10; + min_len += 1; + goto loop1; + case '2': + assert(fill_char); + min_len *= 10; + min_len += 2; + goto loop1; + case '3': + assert(fill_char); + min_len *= 10; + min_len += 3; + goto loop1; + case '4': + assert(fill_char); + min_len *= 10; + min_len += 4; + goto loop1; + case '5': + assert(fill_char); + min_len *= 10; + min_len += 5; + goto loop1; + case '6': + assert(fill_char); + min_len *= 10; + min_len += 6; + goto loop1; + case '7': + assert(fill_char); + min_len *= 10; + min_len += 7; + goto loop1; + case '8': + assert(fill_char); + min_len *= 10; + min_len += 8; + goto loop1; + case '9': + assert(fill_char); + min_len *= 10; + min_len += 9; + goto loop1; + default: + assert(!min_len); + fill_char = *ptr; + goto loop1; + } + } else { + ++res; + putchar(*ptr); + } + ++ptr; + } + return res; +} + +void SECTION_TEXT_VMSETUP handle_fault(uint32_t addr, uint32_t cause) +{ + addr = addr >> PG_SHIFT << PG_SHIFT; /* round down page */ + pte_t *pte = user_l1pt + ((addr >> PG_SHIFT) & MASK(10)); + + /* create a new pte */ + assert(freelist_head); + pte_t pa = freelist_head->addr; + freelist_head = freelist_head->next; + *pte = (pa >> PG_SHIFT << PTE_PPN_SHIFT) | PTE_A | PTE_U | PTE_R | PTE_W | + PTE_X | PTE_V; + if ((1U << cause) == CAUSE_STORE_PAGE_FAULT) + *pte |= PTE_D; + + /* temporarily allow kernel to access user memory to copy data */ + set_csr(sstatus, SSTATUS_SUM); + /* page table is updated, so main should not cause trap here */ + memcpy((uint32_t *) addr, (uint32_t *) uva2kva(addr), PG_SIZE); + /* disallow kernel to access user memory */ + clear_csr(sstatus, SSTATUS_SUM); +} + +void SECTION_TEXT_VMSETUP handle_trap(trapframe_t *tf) +{ + if ((1U << tf->cause) == CAUSE_FETCH_PAGE_FAULT || + (1U << tf->cause) == CAUSE_LOAD_PAGE_FAULT || + (1U << tf->cause) == CAUSE_STORE_PAGE_FAULT) { + handle_fault(tf->badvaddr, tf->cause); + } else + assert(!"Unknown exception"); + + pop_tf(tf); +} + +void SECTION_TEXT_VMSETUP vm_boot() +{ + /* map first page table entry to a next level page table for user */ + l1pt[0] = ((pte_t) user_l1pt >> PG_SHIFT << PTE_PPN_SHIFT) | PTE_V; + /* map last page table leaf entry for kernel which direct maps for user + * virtual memory, note that this is a trick of 2's complement, e.g., 0 - + * MEGA_PG = 0b1111111111xxx...x when page fault occurs after entering main + */ + l1pt[PTES_PER_PT - 1] = ((pte_t) main >> PG_SHIFT << PTE_PPN_SHIFT) | + PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; + + /* direct map kernel virtual memory */ + l1pt[PTES_PER_PT >> 1] = ((pte_t) _start >> PG_SHIFT << PTE_PPN_SHIFT) | + PTE_V | PTE_R | PTE_W | PTE_X | PTE_A | PTE_D; + + /* Enable paging */ + uintptr_t satp_val = ((pte_t) &l1pt >> PG_SHIFT) | SV32_MODE; + write_csr(satp, satp_val); + + /* set up supervisor trap handler */ + write_csr(stvec, supervisor_trap_entry); + write_csr(sscratch, read_csr(mscratch)); + /* No delegation */ + /*write_csr(medeleg, CAUSE_FETCH_PAGE_FAULT | CAUSE_LOAD_PAGE_FAULT | + CAUSE_STORE_PAGE_FAULT);*/ + + freelist_head = (void *) &freelist_nodes[0]; + freelist_tail = &freelist_nodes[MAX_TEST_PG - 1]; + for (uint32_t i = 0; i < MAX_TEST_PG; i++) { + freelist_nodes[i].addr = FREE_FRAME_BASE + i * PG_SIZE; + freelist_nodes[i].next = &freelist_nodes[i + 1]; + } + freelist_nodes[MAX_TEST_PG - 1].next = 0; + + trapframe_t tf; + memset(&tf, 0, sizeof(tf)); + tf.epc = (uint32_t) &user_entry; + pop_tf(&tf); +}