Optimize memory handling (#1963)

* optimize ram block handling

Save the last element of the ram_list. This allows to
faster find where to add new elements when they are not
bigger then page size.

* save ram_list freed

this keeps the optimization for find_ram_offset() intact after snapshot
restore.

* cow only clear the tlb of affected pages

* update flatview when possible

Building each flatview new when the memory has changed is quite
expensive when many MemoryRegions are used. This is an issue when using
snapshots.

* update benchmark for new api

* save flatview in context

this avoids rebuilding the flatview when restore a context.

* init context flatview with zero

* address_space_dispatch_clear remove subpage with higher priority

* docutemnt the options for UC_CTL_CONTEXT_MODE

Specialy stress that with UC_CTL_CONTEXT_MEMORY it is not possible to
use the context with a different unicorn object.

include/qemu.h

@@ -43,6 +43,7 @@ typedef struct {
 typedef struct RAMList {
     bool freed;
     RAMBlock *mru_block;
+    RAMBlock *last_block;
     QLIST_HEAD(, RAMBlock) blocks;
 } RAMList;
 

include/uc_priv.h

@@ -108,6 +108,8 @@ typedef MemoryRegion *(*uc_memory_mapping_t)(struct uc_struct *, hwaddr addr);
 
 typedef void (*uc_memory_filter_t)(MemoryRegion *, int32_t);
 
+typedef bool (*uc_flatview_copy_t)(struct uc_struct *, FlatView *, FlatView *, bool);
+
 typedef void (*uc_readonly_mem_t)(MemoryRegion *mr, bool readonly);
 
 typedef int (*uc_cpus_init)(struct uc_struct *, const char *);
@@ -288,6 +290,7 @@ struct uc_struct {
     uc_args_uc_ram_size_ptr_t memory_map_ptr;
     uc_memory_mapping_t memory_mapping;
     uc_memory_filter_t memory_filter_subregions;
+    uc_flatview_copy_t flatview_copy;
     uc_mem_unmap_t memory_unmap;
     uc_mem_unmap_t memory_moveout;
     uc_mem_unmap_t memory_movein;
@@ -427,6 +430,9 @@ struct uc_context {
     uc_mode mode;        // the mode of this context
     uc_arch arch;        // the arch of this context
     int snapshot_level;  // the memory snapshot level to restore
+    bool ramblock_freed; // wheter there was a some ramblock freed
+    RAMBlock *last_block;// The last element of the ramblock list
+    FlatView *fv;        // The current flatview of the memory
     char data[0];        // context
 };
 

include/unicorn/unicorn.h

@@ -1018,6 +1018,16 @@ struct uc_tlb_entry {
     uc_prot perms;
 };
 
+/*
+ Variables to control which state should be stored in the context.
+ Defaults to UC_CTL_CONTEXT_CPU. The options are used in a bitfield
+ so to enable more then one content the binary or of the required
+ contents can be use.
+ The UC_CTL_CONTEXT_MEMORY stores some pointers to internal allocated
+ memory. Therefor it's not possible to use this context with another
+ unicorn object.
+*/
+
 typedef enum uc_context_content {
     UC_CTL_CONTEXT_CPU = 1,
     UC_CTL_CONTEXT_MEMORY = 2,

qemu/aarch64.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_aarch64
 #define address_space_dispatch_compact address_space_dispatch_compact_aarch64
 #define flatview_translate flatview_translate_aarch64
+#define flatview_copy flatview_copy_aarch64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_aarch64
 #define qemu_get_cpu qemu_get_cpu_aarch64
 #define cpu_address_space_init cpu_address_space_init_aarch64
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_aarch64
 #define address_space_dispatch_new address_space_dispatch_new_aarch64
 #define address_space_dispatch_free address_space_dispatch_free_aarch64
+#define address_space_dispatch_clear address_space_dispatch_clear_aarch64
 #define flatview_read_continue flatview_read_continue_aarch64
 #define address_space_read_full address_space_read_full_aarch64
 #define address_space_write address_space_write_aarch64

qemu/arm.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_arm
 #define address_space_dispatch_compact address_space_dispatch_compact_arm
 #define flatview_translate flatview_translate_arm
+#define flatview_copy flatview_copy_arm
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_arm
 #define qemu_get_cpu qemu_get_cpu_arm
 #define cpu_address_space_init cpu_address_space_init_arm
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_arm
 #define address_space_dispatch_new address_space_dispatch_new_arm
 #define address_space_dispatch_free address_space_dispatch_free_arm
+#define address_space_dispatch_clear address_space_dispatch_clear_arm
 #define flatview_read_continue flatview_read_continue_arm
 #define address_space_read_full address_space_read_full_arm
 #define address_space_write address_space_write_arm

qemu/exec.c

@@ -957,12 +957,10 @@ void flatview_add_to_dispatch(struct uc_struct *uc, FlatView *fv, MemoryRegionSe
 
 static ram_addr_t find_ram_offset_last(struct uc_struct *uc, ram_addr_t size)
 {
-    RAMBlock *block;
     ram_addr_t result = 0;
+    RAMBlock *block = uc->ram_list.last_block;
 
-    RAMBLOCK_FOREACH(block) {
-        result = MAX(block->offset + block->max_length, result);
-    }
+    result = block->offset + block->max_length;
 
     if (result + size > RAM_ADDR_MAX) {
         abort();
@@ -1076,18 +1074,26 @@ static void ram_block_add(struct uc_struct *uc, RAMBlock *new_block)
      * QLIST (which has an RCU-friendly variant) does not have insertion at
      * tail, so save the last element in last_block.
      */
-    RAMBLOCK_FOREACH(block) {
-        last_block = block;
-        if (block->max_length < new_block->max_length) {
-            break;
+    if (uc->ram_list.freed || new_block->max_length > uc->target_page_size) {
+        RAMBLOCK_FOREACH(block) {
+            last_block = block;
+            if (block->max_length < new_block->max_length) {
+                break;
+            }
         }
+    } else {
+        last_block = uc->ram_list.last_block;
+        block = NULL;
     }
+
     if (block) {
         QLIST_INSERT_BEFORE_RCU(block, new_block, next);
     } else if (last_block) {
         QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
+        uc->ram_list.last_block = new_block;
     } else { /* list is empty */
         QLIST_INSERT_HEAD_RCU(&uc->ram_list.blocks, new_block, next);
+        uc->ram_list.last_block = new_block;
     }
     uc->ram_list.mru_block = NULL;
 
@@ -1165,6 +1171,7 @@ void qemu_ram_free(struct uc_struct *uc, RAMBlock *block)
     QLIST_REMOVE_RCU(block, next);
     uc->ram_list.mru_block = NULL;
     uc->ram_list.freed = true;
+    uc->ram_list.last_block = NULL;
     /* Write list before version */
     //smp_wmb();
     // call_rcu(block, reclaim_ramblock, rcu);
@@ -1388,6 +1395,7 @@ static subpage_t *subpage_init(struct uc_struct *uc, FlatView *fv, hwaddr base)
     memory_region_init_io(fv->root->uc, &mmio->iomem, &subpage_ops, mmio,
                           TARGET_PAGE_SIZE);
     mmio->iomem.subpage = true;
+    mmio->iomem.priority = uc->snapshot_level;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
            mmio, base, TARGET_PAGE_SIZE);
@@ -1448,6 +1456,22 @@ AddressSpaceDispatch *address_space_dispatch_new(struct uc_struct *uc, FlatView
     return d;
 }
 
+void address_space_dispatch_clear(AddressSpaceDispatch *d)
+{
+    MemoryRegionSection *section;
+    struct uc_struct *uc = d->uc;
+    while (d->map.sections_nb > 0) {
+        d->map.sections_nb--;
+        section = &d->map.sections[d->map.sections_nb];
+        if (section->mr->priority > uc->snapshot_level) {
+            phys_section_destroy(section->mr);
+        }
+    }
+    g_free(d->map.sections);
+    g_free(d->map.nodes);
+    g_free(d);
+}
+
 void address_space_dispatch_free(AddressSpaceDispatch *d)
 {
     phys_sections_free(&d->map);

qemu/include/exec/memory-internal.h

@@ -45,6 +45,7 @@ void flatview_add_to_dispatch(struct uc_struct *uc, FlatView *fv, MemoryRegionSe
 AddressSpaceDispatch *address_space_dispatch_new(struct uc_struct *uc, FlatView *fv);
 void address_space_dispatch_compact(AddressSpaceDispatch *d);
 void address_space_dispatch_free(AddressSpaceDispatch *d);
+void address_space_dispatch_clear(AddressSpaceDispatch *d);
 
 void mtree_print_dispatch(struct AddressSpaceDispatch *d,
                           MemoryRegion *root);

qemu/include/exec/memory.h

@@ -1220,5 +1220,6 @@ void memory_unmap(struct uc_struct *uc, MemoryRegion *mr);
 void memory_moveout(struct uc_struct *uc, MemoryRegion *mr);
 void memory_movein(struct uc_struct *uc, MemoryRegion *mr);
 int memory_free(struct uc_struct *uc);
+bool flatview_copy(struct uc_struct *uc, FlatView *dst, FlatView *src, bool update_dispatcher);
 
 #endif

qemu/m68k.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_m68k
 #define address_space_dispatch_compact address_space_dispatch_compact_m68k
 #define flatview_translate flatview_translate_m68k
+#define flatview_copy flatview_copy_m68k
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_m68k
 #define qemu_get_cpu qemu_get_cpu_m68k
 #define cpu_address_space_init cpu_address_space_init_m68k
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_m68k
 #define address_space_dispatch_new address_space_dispatch_new_m68k
 #define address_space_dispatch_free address_space_dispatch_free_m68k
+#define address_space_dispatch_clear address_space_dispatch_clear_m68k
 #define flatview_read_continue flatview_read_continue_m68k
 #define address_space_read_full address_space_read_full_m68k
 #define address_space_write address_space_write_m68k

qemu/mips.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_mips
 #define address_space_dispatch_compact address_space_dispatch_compact_mips
 #define flatview_translate flatview_translate_mips
+#define flatview_copy flatview_copy_mips
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips
 #define qemu_get_cpu qemu_get_cpu_mips
 #define cpu_address_space_init cpu_address_space_init_mips
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_mips
 #define address_space_dispatch_new address_space_dispatch_new_mips
 #define address_space_dispatch_free address_space_dispatch_free_mips
+#define address_space_dispatch_clear address_space_dispatch_clear_mips
 #define flatview_read_continue flatview_read_continue_mips
 #define address_space_read_full address_space_read_full_mips
 #define address_space_write address_space_write_mips

qemu/mips64.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_mips64
 #define address_space_dispatch_compact address_space_dispatch_compact_mips64
 #define flatview_translate flatview_translate_mips64
+#define flatview_copy flatview_copy_mips64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64
 #define qemu_get_cpu qemu_get_cpu_mips64
 #define cpu_address_space_init cpu_address_space_init_mips64
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_mips64
 #define address_space_dispatch_new address_space_dispatch_new_mips64
 #define address_space_dispatch_free address_space_dispatch_free_mips64
+#define address_space_dispatch_clear address_space_dispatch_clear_mips64
 #define flatview_read_continue flatview_read_continue_mips64
 #define address_space_read_full address_space_read_full_mips64
 #define address_space_write address_space_write_mips64

qemu/mips64el.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_mips64el
 #define address_space_dispatch_compact address_space_dispatch_compact_mips64el
 #define flatview_translate flatview_translate_mips64el
+#define flatview_copy flatview_copy_mips64el
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mips64el
 #define qemu_get_cpu qemu_get_cpu_mips64el
 #define cpu_address_space_init cpu_address_space_init_mips64el
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_mips64el
 #define address_space_dispatch_new address_space_dispatch_new_mips64el
 #define address_space_dispatch_free address_space_dispatch_free_mips64el
+#define address_space_dispatch_clear address_space_dispatch_clear_mips64el
 #define flatview_read_continue flatview_read_continue_mips64el
 #define address_space_read_full address_space_read_full_mips64el
 #define address_space_write address_space_write_mips64el

qemu/mipsel.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_mipsel
 #define address_space_dispatch_compact address_space_dispatch_compact_mipsel
 #define flatview_translate flatview_translate_mipsel
+#define flatview_copy flatview_copy_mipsel
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_mipsel
 #define qemu_get_cpu qemu_get_cpu_mipsel
 #define cpu_address_space_init cpu_address_space_init_mipsel
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_mipsel
 #define address_space_dispatch_new address_space_dispatch_new_mipsel
 #define address_space_dispatch_free address_space_dispatch_free_mipsel
+#define address_space_dispatch_clear address_space_dispatch_clear_mipsel
 #define flatview_read_continue flatview_read_continue_mipsel
 #define address_space_read_full address_space_read_full_mipsel
 #define address_space_write address_space_write_mipsel

qemu/ppc.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_ppc
 #define address_space_dispatch_compact address_space_dispatch_compact_ppc
 #define flatview_translate flatview_translate_ppc
+#define flatview_copy flatview_copy_ppc
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_ppc
 #define qemu_get_cpu qemu_get_cpu_ppc
 #define cpu_address_space_init cpu_address_space_init_ppc
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_ppc
 #define address_space_dispatch_new address_space_dispatch_new_ppc
 #define address_space_dispatch_free address_space_dispatch_free_ppc
+#define address_space_dispatch_clear address_space_dispatch_clear_ppc
 #define flatview_read_continue flatview_read_continue_ppc
 #define address_space_read_full address_space_read_full_ppc
 #define address_space_write address_space_write_ppc

qemu/ppc64.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_ppc64
 #define address_space_dispatch_compact address_space_dispatch_compact_ppc64
 #define flatview_translate flatview_translate_ppc64
+#define flatview_copy flatview_copy_ppc64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_ppc64
 #define qemu_get_cpu qemu_get_cpu_ppc64
 #define cpu_address_space_init cpu_address_space_init_ppc64
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_ppc64
 #define address_space_dispatch_new address_space_dispatch_new_ppc64
 #define address_space_dispatch_free address_space_dispatch_free_ppc64
+#define address_space_dispatch_clear address_space_dispatch_clear_ppc64
 #define flatview_read_continue flatview_read_continue_ppc64
 #define address_space_read_full address_space_read_full_ppc64
 #define address_space_write address_space_write_ppc64

qemu/riscv32.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_riscv32
 #define address_space_dispatch_compact address_space_dispatch_compact_riscv32
 #define flatview_translate flatview_translate_riscv32
+#define flatview_copy flatview_copy_riscv32
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_riscv32
 #define qemu_get_cpu qemu_get_cpu_riscv32
 #define cpu_address_space_init cpu_address_space_init_riscv32
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_riscv32
 #define address_space_dispatch_new address_space_dispatch_new_riscv32
 #define address_space_dispatch_free address_space_dispatch_free_riscv32
+#define address_space_dispatch_clear address_space_dispatch_clear_riscv32
 #define flatview_read_continue flatview_read_continue_riscv32
 #define address_space_read_full address_space_read_full_riscv32
 #define address_space_write address_space_write_riscv32

qemu/riscv64.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_riscv64
 #define address_space_dispatch_compact address_space_dispatch_compact_riscv64
 #define flatview_translate flatview_translate_riscv64
+#define flatview_copy flatview_copy_riscv64
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_riscv64
 #define qemu_get_cpu qemu_get_cpu_riscv64
 #define cpu_address_space_init cpu_address_space_init_riscv64
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_riscv64
 #define address_space_dispatch_new address_space_dispatch_new_riscv64
 #define address_space_dispatch_free address_space_dispatch_free_riscv64
+#define address_space_dispatch_clear address_space_dispatch_clear_riscv64
 #define flatview_read_continue flatview_read_continue_riscv64
 #define address_space_read_full address_space_read_full_riscv64
 #define address_space_write address_space_write_riscv64

qemu/s390x.h

@@ -54,6 +54,7 @@
 #define vm_start vm_start_s390x
 #define address_space_dispatch_compact address_space_dispatch_compact_s390x
 #define flatview_translate flatview_translate_s390x
+#define flatview_copy flatview_copy_s390x
 #define address_space_translate_for_iotlb address_space_translate_for_iotlb_s390x
 #define qemu_get_cpu qemu_get_cpu_s390x
 #define cpu_address_space_init cpu_address_space_init_s390x
@@ -90,6 +91,7 @@
 #define iotlb_to_section iotlb_to_section_s390x
 #define address_space_dispatch_new address_space_dispatch_new_s390x
 #define address_space_dispatch_free address_space_dispatch_free_s390x
+#define address_space_dispatch_clear address_space_dispatch_clear_s390x
 #define flatview_read_continue flatview_read_continue_s390x
 #define address_space_read_full address_space_read_full_s390x
 #define address_space_write address_space_write_s390x