1G HugeTLB optimizations

src/core/memory.lua

@@ -81,24 +81,21 @@ function get_huge_page_size ()
 end
 
 base_page_size = 4096
+-- Huge page size in bytes
 huge_page_size = get_huge_page_size()
+-- Address bits per huge page (2MB = 21 bits; 1GB = 30 bits)
+huge_page_bits = math.log(huge_page_size, 2)
 
 --- ### Physical address translation
 
-local tlb = require("lib.tlb")
-local phystlb = tlb.new(math.log(huge_page_size, 2))
-
 function virtual_to_physical (virt_addr)
-   local cached = tlb.lookup(phystlb, virt_addr)
-   if cached then return cached end
    virt_addr = ffi.cast("uint64_t", virt_addr)
    local virt_page = tonumber(virt_addr / base_page_size)
    local phys_page = C.phys_page(virt_page) * base_page_size
    if phys_page == 0 then
       error("Failed to resolve physical address of "..tostring(virt_addr))
    end
    local phys_addr = ffi.cast("uint64_t", phys_page + virt_addr % base_page_size)
-   tlb.add(phystlb, virt_addr, phys_addr)
    return phys_addr
 end
 

src/lib/virtio/net_device.lua

@@ -290,34 +290,28 @@ function VirtioNetDevice:rx_signal_used()
    end
 end
 
-function VirtioNetDevice:translate_physical_addr (addr)
-   -- Assuming no-IOMMU
-   return memory.virtual_to_physical(addr)
-end
+local pagebits = memory.huge_page_bits
 
--- Address space remapping.
-local host2guest = tlb.new(21)
-function VirtioNetDevice:map_to_guest (addr)
-   local result = tlb.lookup(host2guest, addr)
-   if result ~= nil then return result end
-   for i = 0, table.getn(self.mem_table) do
-      local m = self.mem_table[i]
-      if addr >= m.snabb and addr < m.snabb + m.size then
-         result = addr + m.guest - m.snabb
-         tlb.add(host2guest, addr, result)
-         break
-      end
-   end
-   if not result then
-      error("mapping to guest address failed")
+-- Cache of the latest referenced physical page.
+local last_virt_page = false
+local last_virt_offset = false
+function VirtioNetDevice:translate_physical_addr (addr)
+   local page = bit.rshift(addr, pagebits)
+   if page == last_virt_page then
+      return addr + last_virt_offset
    end
-   return result
+   local phys = memory.virtual_to_physical(addr)
+   last_virt_page = page
+   last_virt_offset = phys - addr
+   return phys
 end
 
-local guest2host = tlb.new(21)
+local last_guest_page = false
+local last_guest_offset = false
 function VirtioNetDevice:map_from_guest (addr)
-   local result = tlb.lookup(guest2host, addr)
-   if result ~= nil then return result end
+   local page = bit.rshift(addr, pagebits)
+   if page == last_guest_page then return addr + last_guest_offset end
+   local result
    for i = 0, table.getn(self.mem_table) do
       local m = self.mem_table[i]
       if addr >= m.guest and addr < m.guest + m.size then
@@ -326,7 +320,8 @@ function VirtioNetDevice:map_from_guest (addr)
             self.mem_table[0] = m
          end
          result = addr + m.snabb - m.guest
-         tlb.add(guest2host, addr, result)
+         last_guest_page = page
+         last_guest_offset = m.snabb - m.guest
          break
       end
    end
@@ -336,15 +331,12 @@ function VirtioNetDevice:map_from_guest (addr)
    return result
 end
 
-local qemu2host = tlb.new(21)
 function VirtioNetDevice:map_from_qemu (addr)
-   local result = tlb.lookup(qemu2host, addr)
-   if result ~= nil then return result end
+   local result = nil
    for i = 0, table.getn(self.mem_table) do
       local m = self.mem_table[i]
       if addr >= m.qemu and addr < m.qemu + m.size then
          result = addr + m.snabb - m.qemu
-         tlb.add(qemu2host, addr, result)
          break
       end
    end

src/scripts/bench_env/include_checks.sh

@@ -100,13 +100,6 @@ PAGES=`cat /proc/meminfo | grep HugePages_Free | awk  '{ print $2; }'`
 PAGES=`expr $PAGES \* 2`
 
 TOTAL_MEM=`expr $GUEST_MEM \* $GUESTS`
-if [ "$PAGES" -lt "$TOTAL_MEM" ] ; then
-    printf "Exiting: Free HugePages are too low!\n"
-    printf "Increase /proc/sys/vm/nr_hugepages\n"
-    printf "and/or /proc/sys/vm/nr_hugepages_mempolicy\n"
-    printf "Hugepages should be set at: (guests memory / 2) + QEMU bookkeeping.\n"
-    exit 1
-fi
 
 # setup a trap hook
 trap on_exit EXIT HUP INT QUIT TERM