[feat] support passthrough device, refactor vmconfig, support boot Linux

README.md

@@ -20,15 +20,41 @@ Your also need to install [musl-gcc](http://musl.cc/x86_64-linux-musl-cross.tgz)
 
 ### Configuration files
 
+Since guest VM configuration is a complex process, ArceOS-Hypervisor chooses to use toml files to manage guest VM configuration, 
+including vm id, vm name, vm type, number of CPU cores, memory size, virtual devices and pass-through devices, etc. 
+
 We provide several configuration file [templates](arceos-vmm/configs) for setting up guest VMs. 
 
 These configuration files are read and parsed by the `init_guest_vms()` in the [vmm/config](arceos-vmm/src/vmm/config.rs) mod, and are used to configure the guest VMs.
 
+You can also use [axvmconfig](https://github.com/arceos-hypervisor/axvmconfig) tool to generate a custom config.toml.
+
+For more information about VM configuration, visit [axvmconfig](https://arceos-hypervisor.github.io/axvmconfig/axvmconfig/index.html) for details.
+
 ### [Supported guest VMs](doc/GuestVMs.md)
 
-WIP on Linux guests.
+* [ArceOS](https://github.com/arceos-org/arceos)
+* [Starry-OS](https://github.com/Starry-OS)
+* [NimbOS](https://github.com/equation314/nimbos)
+* Linux
+  * currently only Linux with passthrough device on aarch64 is tested.
+  * single core: [config.toml](arceos-vmm/configs/linux-qemu-aarch64.toml) | [dts](arceos-vmm/configs/linux-qemu.dts)
+  * smp: [config.toml](arceos-vmm/configs/linux-qemu-aarch64-smp2.toml) | [dts](arceos-vmm/configs/linux-qemu-smp2.dts)
+
+### Loading Guest VM images
+
+Currently, arceos-hypervisor supports loading guest VM images from arceos' fat file system, or binding guest VM images to hypervisor images through a static compilation manner (`include_bytes`).
 
-## Build File System image
+* load from file system
+  * specify `image_location="fs"` in the `config.toml` file.
+  * `kernel_path` in `config.toml` refers to the location of the kernel image in the arceos rootfs (e.g. `disk.img`).
+  * Note: `"fs"` feature is required for arceos-umhv, this can be configured via environment variables `APP_FEATURES=fs`.
+* load from memory
+  * specify `image_location="memory"` in the `config.toml` file.
+  * `kernel_path` in `config.toml` refers to the relative/absolute path of the kernel image in your workspace when compiling arceos-vmm.
+  * Note that the current method of binding guest VM images through static compilation only supports the loading of one guest VM image at most (usually we use this method to start Linux as a guest VM).
+
+### Build File System image
 
 ```console
 $ cd arceos-vmm
@@ -46,17 +72,28 @@ $ # Arceos-VMM will load the image binaries from the first configuration in the
 
 ## Build & Run Hypervisor
 
-First, you need to prepare your configuration file for the guest VM (several examples are provided in the [configs](arceos-vmm/configs) directory), and then run the hypervisor with the following command:
+### Example build commands
 
 ```console
 $ cd arceos-vmm
-# x86_64
-$ make ACCEL=y ARCH=x86_64 [LOG=warn|info|debug|trace] VM_CONFIGS=/PATH/TO/CONFIG/FILE run
-# aarch64
-$ make ACCEL=n ARCH=aarch64 [LOG=warn|info|debug|trace] VM_CONFIGS=/PATH/TO/CONFIG/FILE run
-# riscv64
-$ make ACCEL=n ARCH=riscv64 [LOG=warn|info|debug|trace] VM_CONFIGS=/PATH/TO/CONFIG/FILE run
+# x86_64 for nimbos
+# [LOG=warn|info|debug|trace]
+$ make ACCEL=y ARCH=x86_64 LOG=info VM_CONFIGS=configs/nimbos-x86_64.toml APP_FEATURES=fs run
+# aarch64 for nimbos
+$ make ACCEL=n ARCH=aarch64 LOG=info VM_CONFIGS=configs/nimbos-aarch64.toml APP_FEATURES=fs run
+# riscv64 for nimbos
+$ make ACCEL=n ARCH=riscv64 LOG=info VM_CONFIGS=configs/nimbos-riscv64.toml APP_FEATURES=fs run
+# aarch64 for Linux
+$ make ARCH=aarch64 VM_CONFIGS=configs/linux-qemu-aarch64.toml LOG=debug BUS=mmio NET=y DISK_IMG=ubuntu-22.04-rootfs_ext4.img run
+# aarch64 for Linux SMP=2
+$ make ARCH=aarch64 VM_CONFIGS=configs/linux-qemu-aarch64-smp2.toml LOG=debug BUS=mmio NET=y DISK_IMG=ubuntu-22.04-rootfs_ext4.img BLK=y SMP=2 run
+```
+
+### Demo Output
 
+```console
+$ cd arceos-vmm
+$ make ACCEL=y ARCH=x86_64 LOG=warn VM_CONFIGS=configs/nimbos-x86_64.toml APP_FEATURES=fs run
 ......
 Booting from ROM..
 Initialize IDT & GDT...

arceos-vmm/Cargo.toml

@@ -5,6 +5,9 @@ edition = "2021"
 authors = ["Keyang Hu <[email protected]>"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+fs = ["axstd/fs"]
+
 [dependencies]
 log = "=0.4.21"
 bitflags = "2.2"
@@ -14,7 +17,7 @@ spin = "0.9"
 axstd = { workspace = true, features = [
     "alloc",
     "paging",
-    "fs",
+    # "fs",
     "irq", # It has to be disabed in riscv64. Todo: fix it.
     "hv",
     "multitask",
@@ -30,22 +33,9 @@ axaddrspace = { workspace = true }
 crate_interface = "0.1"
 axerrno = "0.1.0"
 memory_addr = "0.3"
-page_table_entry = { version = "0.4.1", features = ["arm-el2"] }
-page_table_multiarch = "0.4.1"
+page_table_entry = { version = "0.4.2", features = ["arm-el2"] }
+page_table_multiarch = "0.4.2"
 percpu = { version = "0.1.4", features = ["arm-el2"] }
 
-[target.'cfg(target_arch = "x86_64")'.dependencies]
-x86 = "0.52"
-x86_64 = "0.14"
-raw-cpuid = "11.0"
-
-[target.'cfg(target_arch = "riscv64")'.dependencies]
-fdt = { version = "0.1.5" }
-arrayvec = { version = "0.7.2", default-features = false }
-
-[target.'cfg(target_arch = "aarch64")'.dependencies]
-fdt = { version = "0.1.5" }
-arrayvec = { version = "0.7.2", default-features = false }
-
 [build-dependencies]
 toml = { git = "https://github.com/arceos-hypervisor/toml.git", branch = "no_std" }

arceos-vmm/configs/arceos-aarch64-smp.toml

@@ -9,16 +9,24 @@ kernel_load_addr = 0x4008_0000
 # ramdisk_path = ""
 # ramdisk_load_addr = 0
 # disk_path = "disk.img"
-# Memory regions with format (`base_paddr`, `size`, `flags`).
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
 memory_regions = [
-    [0x4000_0000, 0x100_0000, 0x7], # Low RAM		16M 0b00111 R|W|EXECUTE
-    [0x800_0000, 0x10_000, 0x17],   # intc 		    4K  0b10111 R|W|DEVICE
-    [0x801_0000, 0x10_000, 0x17],   # intc 		    4K  0b10111 R|W|DEVICE
-    [0x900_0000, 0x1000, 0x17],     # pl011 (Uart)	4K  0b10111 R|W|DEVICE
-    [0x901_0000, 0x1000, 0x17],     # pl031 (timer)	4K  0b10111 R|W|DEVICE
-    [0xfed0_0000, 0x1000, 0x17],    # HPET 		    4K  0b10111 R|W|DEVICE
+    [0x4000_0000, 0x100_0000, 0x7, 0], # Low RAM  16M 0b00111 R|W|EXECUTE MAP_ALLOC
 ]
 
 # Emu_devices
 # Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig
 emu_devices = []
+
+# Pass-through devices
+# Name Base-Ipa Base-Pa Length Alloc-Irq
+passthrough_devices = [
+    ["intc@8000000", 0x800_0000, 0x800_0000, 0x50_000, 0x1],
+    ["pl011@9000000", 0x900_0000, 0x900_0000, 0x1000, 0x1],
+    ["pl031@9010000", 0x901_0000, 0x901_0000, 0x1000, 0x1],
+    ["pl061@9030000", 0x903_0000, 0x903_0000, 0x1000, 0x1],
+    # a003000.virtio_mmio virtio_mmio@a003000 
+    # a003200.virtio_mmio virtio_mmio@a003200
+    ["virtio_mmio", 0xa00_0000, 0xa00_0000, 0x4000, 0x1],
+]

arceos-vmm/configs/arceos-aarch64.toml

@@ -4,22 +4,31 @@ vm_type = 1
 cpu_num = 1
 phys_cpu_sets = [1]
 entry_point = 0x4008_0000
+image_location = "fs"
 kernel_path = "arceos-aarch64.bin"
+# kernel_path = "linux-6.6.62.bin"
 kernel_load_addr = 0x4008_0000
 # ramdisk_path = ""
 # ramdisk_load_addr = 0
 # disk_path = "disk.img"
-# Memory regions with format (`base_paddr`, `size`, `flags`).
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
 memory_regions = [
-    [0x4000_0000, 0x100_0000, 0x7],   # Low RAM		    16M 0b00111 R|W|EXECUTE
-    [0x800_0000, 0x10_000, 0x17],     # intc 		    4K  0b10111 R|W|DEVICE
-    [0x801_0000, 0x10_000, 0x17],     # intc 		    4K  0b10111 R|W|DEVICE
-    [0x900_0000, 0x1000, 0x17],       # pl011 (Uart)	4K  0b10111 R|W|DEVICE
-    [0x901_0000, 0x1000, 0x17],       # pl031 (timer)	4K  0b10111 R|W|DEVICE
-    [0xfed0_0000, 0x1000, 0x17],      # HPET 		    4K  0b10111 R|W|DEVICE
+    [0x4000_0000, 0x100_0000, 0x7, 0], # Low RAM		    16M 0b00111 R|W|EXECUTE
 ]
 
 # Emu_devices
 # Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig
-emu_devices = [
+emu_devices = []
+
+# Pass-through devices
+# Name Base-Ipa Base-Pa Length Alloc-Irq
+passthrough_devices = [
+    ["intc@8000000", 0x800_0000, 0x800_0000, 0x50_000, 0x1],
+    ["pl011@9000000", 0x900_0000, 0x900_0000, 0x1000, 0x1],
+    ["pl031@9010000", 0x901_0000, 0x901_0000, 0x1000, 0x1],
+    ["pl061@9030000", 0x903_0000, 0x903_0000, 0x1000, 0x1],
+    # a003000.virtio_mmio virtio_mmio@a003000 
+    # a003200.virtio_mmio virtio_mmio@a003200
+    ["virtio_mmio", 0xa00_0000, 0xa00_0000, 0x4000, 0x1],
 ]

arceos-vmm/configs/arceos-riscv64-smp.toml

@@ -4,16 +4,34 @@ vm_type = 1
 cpu_num = 2
 phys_cpu_sets = [1, 2]
 entry_point = 0x8020_0000
+image_location = "fs"
 kernel_path = "arceos-riscv64-smp.bin"
 kernel_load_addr = 0x8020_0000
-# Memory regions with format (`base_paddr`, `size`, `flags`).
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
 memory_regions = [
-    [0x8000_0000, 0x100_0000, 0xf], # Low RAM		16M	0b1111  R|W|EXECUTE|U
-    [0x0c00_0000, 0x21_0000, 0x13], # PLIC  0x21_0000	0b10011 R|W|DEVICE
-    [0x1000_0000, 0x1000, 0x13],    # UART	     4K	0b10011 R|W|DEVICE
+    [0x8000_0000, 0x100_0000, 0xf, 0], # Low RAM		16M	0b1111  R|W|EXECUTE|U
 ]
 
 # Emu_devices
 # Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig
-emu_devices = [
+emu_devices = []
+
+# Pass-through devices
+# Name Base-Ipa Base-Pa Length Alloc-Irq
+passthrough_devices = [
+    [
+        "PLIC@c000000",
+        0x0c00_0000,
+        0x0c00_0000,
+        0x21_0000,
+        0x1,
+    ],
+    [
+        "UART@10000000",
+        0x1000_0000,
+        0x1000_0000,
+        0x1000,
+        0x1,
+    ],
 ]

arceos-vmm/configs/arceos-riscv64.toml

@@ -4,16 +4,34 @@ vm_type = 1
 cpu_num = 1
 phys_cpu_sets = [1]
 entry_point = 0x8020_0000
+image_location = "fs"
 kernel_path = "arceos-riscv64.bin"
 kernel_load_addr = 0x8020_0000
-# Memory regions with format (`base_paddr`, `size`, `flags`).
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
 memory_regions = [
-    [0x8000_0000, 0x100_0000, 0xf], # Low RAM		16M	0b1111  R|W|EXECUTE|U
-    [0x0c00_0000, 0x21_0000, 0x13], # PLIC  0x21_0000	0b10011 R|W|DEVICE
-    [0x1000_0000, 0x1000, 0x13],    # UART	     4K	0b10011 R|W|DEVICE
+    [0x8000_0000, 0x100_0000, 0xf, 0], # Low RAM		16M	0b1111  R|W|EXECUTE|U
 ]
 
 # Emu_devices
 # Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig
-emu_devices = [
+emu_devices = []
+
+# Pass-through devices
+# Name Base-Ipa Base-Pa Length Alloc-Irq
+passthrough_devices = [
+    [
+        "PLIC@c000000",
+        0x0c00_0000,
+        0x0c00_0000,
+        0x21_0000,
+        0x1,
+    ],
+    [
+        "UART@10000000",
+        0x1000_0000,
+        0x1000_0000,
+        0x1000,
+        0x1,
+    ],
 ]

arceos-vmm/configs/arceos-x86_64.toml

@@ -4,22 +4,46 @@ vm_type = 1
 cpu_num = 1
 phys_cpu_sets = [1]
 entry_point = 0x8000
+image_location = "fs"
 bios_path = "rvm-bios.bin"
 bios_load_addr = 0x8000
 kernel_path = "arceos-x86_64.bin"
 kernel_load_addr = 0x20_0000
 # ramdisk_path = ""
 # ramdisk_load_addr = 0
 # disk_path = "disk.img"
-# Memory regions with format (`base_paddr`, `size`, `flags`).
+# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`).
+# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`.
 memory_regions = [
-    [0x0000_0000, 0x100_0000, 0x7],   # Low RAM		16M	0b111
-    [0xfec0_0000, 0x1000, 0x17],      # IO APIC		4K	0b10111
-    [0xfee0_0000, 0x1000, 0x17],      # Local APIC	4K	0b10111
-    [0xfed0_0000, 0x1000, 0x17],      # HPET 		4K  0b10111
+    [0x0000_0000, 0x100_0000, 0x7, 0], # Low RAM		16M	0b111
 ]
 
 # Emu_devices
 # Name Base-Ipa Ipa_len Alloc-Irq Emu-Type EmuConfig
-emu_devices = [
+emu_devices = []
+
+# Pass-through devices
+# Name Base-Ipa Base-Pa Length Alloc-Irq
+passthrough_devices = [
+    [
+        "IO APIC",
+        0xfec0_0000,
+        0xfec0_0000,
+        0x1000,
+        0x1,
+    ],
+    [
+        "Local APIC",
+        0xfee0_0000,
+        0xfee0_0000,
+        0x1000,
+        0x1,
+    ],
+    [
+        "HPET",
+        0xfed0_0000,
+        0xfed0_0000,
+        0x1000,
+        0x1,
+    ],
 ]