From bf2d58ab8049fd24591aa20914738a4b1f1b3798 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Wed, 20 Jan 2016 10:39:40 +0100
Subject: [PATCH 1/8] Add lib.ctable module.

---
 src/lib/ctable.lua | 444 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 444 insertions(+)
 create mode 100644 src/lib/ctable.lua

diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua
new file mode 100644
index 0000000000..b59ba4c62e
--- /dev/null
+++ b/src/lib/ctable.lua
@@ -0,0 +1,444 @@
+module(..., package.seeall)
+
+local ffi = require("ffi")
+local C = ffi.C
+local S = require("syscall")
+local bit = require("bit")
+local bxor, bnot = bit.bxor, bit.bnot
+local tobit, lshift, rshift = bit.tobit, bit.lshift, bit.rshift
+local max, floor, ceil = math.max, math.floor, math.ceil
+
+CTable = {}
+
+local HASH_MAX = 0xFFFFFFFF
+
+local function make_entry_type(key_type, value_type)
+   return ffi.typeof([[struct {
+         uint32_t hash;
+         $ key;
+         $ value;
+      } __attribute__((packed))]],
+      key_type,
+      value_type)
+end
+
+local function make_entries_type(entry_type)
+   return ffi.typeof('$[?]', entry_type)
+end
+
+-- hash := [0,HASH_MAX); scale := size/HASH_MAX
+local function hash_to_index(hash, scale)
+   return floor(hash*scale + 0.5)
+end
+
+local function make_equal_fn(key_type)
+   local size = ffi.sizeof(key_type)
+   local cast = ffi.cast
+   if tonumber(ffi.new(key_type)) then
+      return function (a, b)
+         return a == b
+      end
+   elseif size == 4 then
+      local uint32_ptr_t = ffi.typeof('uint32_t*')
+      return function (a, b)
+         return cast(uint32_ptr_t, a)[0] == cast(uint32_ptr_t, b)[0]
+      end
+   elseif size == 8 then
+      local uint64_ptr_t = ffi.typeof('uint64_t*')
+      return function (a, b)
+         return cast(uint64_ptr_t, a)[0] == cast(uint64_ptr_t, b)[0]
+      end
+   else
+      return function (a, b)
+         return C.memcmp(a, b, size) == 0
+      end
+   end
+end
+
+local function set(...)
+   local ret = {}
+   for k, v in pairs({...}) do ret[v] = true end
+   return ret
+end
+
+local function parse_params(params, required, optional)
+   local ret = {}
+   for k, _ in pairs(required) do
+      if params[k] == nil then error('missing required option ' .. k) end
+   end
+   for k, v in pairs(params) do
+      if not required[k] and optional[k] == nil then
+         error('unrecognized option ' .. k)
+      end
+      ret[k] = v
+   end
+   for k, v in pairs(optional) do
+      if ret[k] == nil then ret[k] = v end
+   end
+   return ret
+end
+
+-- FIXME: For now the value_type option is required, but in the future
+-- we should allow for a nil value type to create a set instead of a
+-- map.
+local required_params = set('key_type', 'value_type', 'hash_fn')
+local optional_params = {
+   initial_size = 8,
+   max_occupancy_rate = 0.9,
+   min_occupancy_rate = 0.0
+}
+
+function new(params)
+   local ctab = {}   
+   local params = parse_params(params, required_params, optional_params)
+   ctab.entry_type = make_entry_type(params.key_type, params.value_type)
+   ctab.type = make_entries_type(ctab.entry_type)
+   ctab.hash_fn = params.hash_fn
+   ctab.equal_fn = make_equal_fn(params.key_type)
+   ctab.size = 0
+   ctab.occupancy = 0
+   ctab.max_occupancy_rate = params.max_occupancy_rate
+   ctab.min_occupancy_rate = params.min_occupancy_rate
+   ctab = setmetatable(ctab, { __index = CTable })
+   ctab:resize(params.initial_size)
+   return ctab
+end
+
+-- FIXME: There should be a library to help allocate anonymous
+-- hugepages, not this code.
+local try_huge_pages = true
+local huge_page_threshold = 1e6
+local function calloc(t, count)
+   local byte_size = ffi.sizeof(t) * count
+   local mem, err
+   if try_huge_pages and byte_size > huge_page_threshold then
+      mem, err = S.mmap(nil, byte_size, 'read, write',
+                        'private, anonymous, hugetlb')
+      if not mem then
+         print("hugetlb mmap failed ("..tostring(err)..'), falling back.')
+         -- FIXME: Increase vm.nr_hugepages.  See
+         -- core.memory.reserve_new_page().
+      end
+   end
+   if not mem then
+      mem, err = S.mmap(nil, byte_size, 'read, write',
+                        'private, anonymous')
+      if not mem then error("mmap failed: " .. tostring(err)) end
+   end
+   local ret = ffi.cast(ffi.typeof('$*', t), mem)
+   ffi.gc(ret, function (ptr) S.munmap(ptr, byte_size) end)
+   return ret
+end
+
+function CTable:resize(size)
+   assert(size >= (self.occupancy / self.max_occupancy_rate))
+   local old_entries = self.entries
+   local old_size = self.size
+
+   -- Allocate double the requested number of entries to make sure there
+   -- is sufficient displacement if all hashes map to the last bucket.
+   self.entries = calloc(self.entry_type, size * 2)
+   self.size = size
+   self.scale = self.size / HASH_MAX
+   self.occupancy = 0
+   self.max_displacement = 0
+   self.occupancy_hi = ceil(self.size * self.max_occupancy_rate)
+   self.occupancy_lo = floor(self.size * self.min_occupancy_rate)
+   for i=0,self.size*2-1 do self.entries[i].hash = HASH_MAX end
+
+   for i=0,old_size*2-1 do
+      if old_entries[i].hash ~= HASH_MAX then
+         self:insert(old_entries[i].hash, old_entries[i].key, old_entries[i].value)
+      end
+   end
+end
+
+function CTable:insert(hash, key, value, updates_allowed)
+   if self.occupancy + 1 > self.occupancy_hi then
+      self:resize(self.size * 2)
+   end
+
+   local entries = self.entries
+   local scale = self.scale
+   local start_index = hash_to_index(hash, self.scale)
+   local index = start_index
+
+   while entries[index].hash < hash do
+      index = index + 1
+   end
+
+   while entries[index].hash == hash do
+      if self.equal_fn(key, entries[index].key) then
+         assert(updates_allowed, "key is already present in ctable")
+         entries[index].key = key
+         entries[index].value = value
+         return index
+      end
+      index = index + 1
+   end
+
+   assert(updates_allowed ~= 'required', "key not found in ctable")
+
+   self.max_displacement = max(self.max_displacement, index - start_index)
+
+   if entries[index].hash ~= HASH_MAX then
+      -- In a robin hood hash, we seek to spread the wealth around among
+      -- the members of the table.  An entry that can be stored exactly
+      -- where hash_to_index() maps it is a most wealthy entry.  The
+      -- farther from that initial position, the less wealthy.  Here we
+      -- have found an entry whose hash is greater than our hash,
+      -- meaning it has travelled less far, so we steal its position,
+      -- displacing it by one.  We might have to displace other entries
+      -- as well.
+      local empty = index;
+      while entries[empty].hash ~= HASH_MAX do empty = empty + 1 end
+      while empty > index do
+         entries[empty] = entries[empty - 1]
+         local displacement = empty - hash_to_index(entries[empty].hash, scale)
+         self.max_displacement = max(self.max_displacement, displacement)
+         empty = empty - 1;
+      end
+   end
+           
+   self.occupancy = self.occupancy + 1
+   entries[index].hash = hash
+   entries[index].key = key
+   entries[index].value = value
+   return index
+end
+
+function CTable:add(key, value, updates_allowed)
+   local hash = self.hash_fn(key)
+   assert(hash >= 0)
+   assert(hash < HASH_MAX)
+   return self:insert(hash, key, value, updates_allowed)
+end
+
+function CTable:update(key, value)
+   return self:add(key, value, 'required')
+end
+
+function CTable:lookup_ptr(key)
+   local hash = self.hash_fn(key)
+   local entry = self.entries + hash_to_index(hash, self.scale)
+
+   -- Fast path in case we find it directly.
+   if hash == entry.hash and self.equal_fn(key, entry.key) then
+      return entry
+   end
+
+   while entry.hash < hash do entry = entry + 1 end
+
+   while entry.hash == hash do
+      if self.equal_fn(key, entry.key) then return entry end
+      -- Otherwise possibly a collision.
+      entry = entry + 1
+   end
+
+   -- Not found.
+   return nil
+end
+
+function CTable:lookup_and_copy(key, entry)
+   local entry_ptr = self:lookup_ptr(key)
+   if not ptr then return false end
+   entry = entry_ptr
+   return true
+end
+
+function CTable:remove_ptr(entry)
+   local scale = self.scale
+   local index = entry - self.entries
+   assert(index >= 0)
+   assert(index <= self.size + self.max_displacement)
+   assert(entry.hash ~= HASH_MAX)
+
+   self.occupancy = self.occupancy - 1
+   entry.hash = HASH_MAX
+
+   while true do
+      entry = entry + 1
+      index = index + 1
+      if entry.hash == HASH_MAX then break end
+      if hash_to_index(entry.hash, scale) == index then break end
+      -- Give to the poor.
+      entry[-1] = entry[0]
+      entry.hash = HASH_MAX
+   end
+
+   if self.occupancy < self.occupancy_lo then
+      self:resize(self.size / 2)
+   end
+end
+
+-- FIXME: Does NOT shrink max_displacement
+function CTable:remove(key, missing_allowed)
+   local ptr = self:lookup_ptr(key)
+   if not ptr then
+      assert(missing_allowed, "key not found in ctable")
+      return false
+   end
+   self:remove_ptr(ptr)
+   return true
+end
+
+function CTable:selfcheck()
+   local occupancy = 0
+   local max_displacement = 0
+
+   local function fail(expected, op, found, what, where)
+      if where then where = 'at '..where..': ' else where = '' end
+      error(where..what..' check: expected '..expected..op..'found '..found)
+   end
+   local function expect_eq(expected, found, what, where)
+      if expected ~= found then fail(expected, '==', found, what, where) end
+   end
+   local function expect_le(expected, found, what, where)
+      if expected > found then fail(expected, '<=', found, what, where) end
+   end
+
+   local prev = 0
+   for i = 0,self.size*2-1 do
+      local entry = self.entries[i]
+      local hash = entry.hash
+      if hash ~= 0xffffffff then
+         expect_eq(self.hash_fn(entry.key), hash, 'hash', i)
+         local index = hash_to_index(hash, self.scale)
+         if prev == 0xffffffff then
+            expect_eq(index, i, 'undisplaced index', i)
+         else
+            expect_le(prev, hash, 'displaced hash', i)
+         end
+         occupancy = occupancy + 1
+         max_displacement = max(max_displacement, i - index)
+      end
+      prev = hash
+   end
+
+   expect_eq(occupancy, self.occupancy, 'occupancy')
+   -- Compare using <= because remove_at doesn't update max_displacement.
+   expect_le(max_displacement, self.max_displacement, 'max_displacement')
+end
+
+function CTable:dump()
+   local function dump_one(index)
+      io.write(index..':')
+      local entry = self.entries[index]
+      if (entry.hash == HASH_MAX) then
+         io.write('\n')
+      else
+         local distance = index - hash_to_index(entry.hash, self.scale)
+         io.write(' hash: '..entry.hash..' (distance: '..distance..')\n')
+         io.write('    key: '..tostring(entry.key)..'\n')
+         io.write('  value: '..tostring(entry.value)..'\n')
+      end
+   end
+   for index=0,self.size-1 do dump_one(index) end
+   for index=self.size,self.size*2-1 do
+      if self.entries[index].hash == HASH_MAX then break end
+      dump_one(index)
+   end
+end
+
+function CTable:iterate()
+   local max_entry = self.entries + self.size + self.max_displacement
+   local function next_entry(max_entry, entry)
+      while entry <= max_entry do
+         entry = entry + 1
+         if entry.hash ~= HASH_MAX then return entry end
+      end
+   end
+   return next_entry, max_entry, self.entries - 1
+end
+
+-- One of Bob Jenkins' hashes from
+-- http://burtleburtle.net/bob/hash/integer.html.  It's about twice as
+-- fast as MurmurHash3_x86_32 and seems to do just as good a job --
+-- tables using this hash function seem to have the same max
+-- displacement as tables using the murmur hash.
+--
+-- TODO: Switch to a hash function with good security properties,
+-- perhaps by using the DynASM support for AES.
+local uint32_cast = ffi.new('uint32_t[1]')
+function hash_i32(i32)
+   i32 = tobit(i32)
+   i32 = i32 + bnot(lshift(i32, 15))
+   i32 = bxor(i32, (rshift(i32, 10)))
+   i32 = i32 + lshift(i32, 3)
+   i32 = bxor(i32, rshift(i32, 6))
+   i32 = i32 + bnot(lshift(i32, 11))
+   i32 = bxor(i32, rshift(i32, 16))
+
+   -- Unset the low bit, to distinguish valid hashes from HASH_MAX.
+   i32 = lshift(i32, 1)
+
+   -- Project result to u32 range.
+   uint32_cast[0] = i32
+   return uint32_cast[0]
+end
+
+function selftest()
+   print("selftest: ctable")
+
+   -- 32-byte entries
+   local occupancy = 2e6
+   local params = {
+      key_type = ffi.typeof('uint32_t'),
+      value_type = ffi.typeof('int32_t[6]'),
+      hash_fn = hash_i32,
+      max_occupancy_rate = 0.4,
+      initial_size = ceil(occupancy / 0.4)
+   }
+   local ctab = new(params)
+   ctab:resize(occupancy / 0.4 + 1)
+
+   -- Fill with i -> { bnot(i), ... }.
+   local v = ffi.new('int32_t[6]');
+   for i = 1,occupancy do
+      for j=0,5 do v[j] = bnot(i) end
+      ctab:add(i, v)
+   end
+
+   -- In this case we know max_displacement is 8.  Assert here so that
+   -- we can detect any future deviation or regression.
+   assert(ctab.max_displacement == 8)
+
+   ctab:selfcheck()
+
+   for i = 1, occupancy do
+      local value = ctab:lookup_ptr(i).value[0]
+      assert(value == bnot(i))
+   end
+   ctab:selfcheck()
+
+   local iterated = 0
+   for entry in ctab:iterate() do iterated = iterated + 1 end
+   assert(iterated == occupancy)
+
+   -- OK, all looking good with our ctab.
+
+   -- A check that our equality functions work as intended.
+   local numbers_equal = make_equal_fn(ffi.typeof('int'))
+   local four_byte = ffi.typeof('uint32_t[1]')
+   local eight_byte = ffi.typeof('uint32_t[2]')
+   local twelve_byte = ffi.typeof('uint32_t[3]')
+   local four_byte_equal = make_equal_fn(four_byte)
+   local eight_byte_equal = make_equal_fn(eight_byte)
+   local twelve_byte_equal = make_equal_fn(twelve_byte)
+   assert(numbers_equal(1,1))
+   assert(not numbers_equal(1,2))
+   assert(four_byte_equal(ffi.new(four_byte, {1}),
+                          ffi.new(four_byte, {1})))
+   assert(not four_byte_equal(ffi.new(four_byte, {1}),
+                              ffi.new(four_byte, {2})))
+   assert(eight_byte_equal(ffi.new(eight_byte, {1,1}),
+                           ffi.new(eight_byte, {1,1})))
+   assert(not eight_byte_equal(ffi.new(eight_byte, {1,1}),
+                               ffi.new(eight_byte, {1,2})))
+   assert(twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}),
+                            ffi.new(twelve_byte, {1,1,1})))
+   assert(not twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}),
+                                ffi.new(twelve_byte, {1,1,2})))
+
+   print("selftest: ok")
+end

From 9666348f1a09ae8d8edf58c65382ac6b21b4a14a Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 2 Feb 2016 14:47:00 +0100
Subject: [PATCH 2/8] Makefile can process any .md.src to an .md, not just
 README.md

* src/Makefile (MDSRC): Rename from RMSRC, and search for any .md.src
  file.  Update users.
  (MDOBJS): Rename from RMOBJS.
---
 src/Makefile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 938ab86543..bfec34ee3a 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -16,7 +16,7 @@ CSRC   = $(shell find . -regex '[^\#]*\.c' -not -regex './arch/.*' -printf '%P '
 CHDR   = $(shell find . -regex '[^\#]*\.h' -printf '%P ')
 ASM    = $(shell find . -regex '[^\#]*\.dasl' -printf '%P ')
 ARCHSRC= $(shell find . -regex '^./arch/[^\#]*\.c' -printf '%P ')
-RMSRC  = $(shell find . -name  README.md.src -printf '%P ')
+MDSRC  = $(shell find . -regex '[^\#]*\.md.src' -printf '%P ')
 # regexp is to include program/foo but not program/foo/bar
 PROGRAM = $(shell find program -regex '^[^/]+/[^/]+' -type d -printf '%P ')
 # sort to eliminate potential duplicate of programs.inc
@@ -30,7 +30,7 @@ ARCHOBJ:= $(patsubst %.c,obj/%_c.o,    $(ARCHSRC))
 ASMOBJ := $(patsubst %.dasl,obj/%_dasl.o,   $(ASM))
 JITOBJS:= $(patsubst %,obj/jit_%.o,$(JITSRC))
 EXTRAOBJS := obj/jit_tprof.o obj/jit_vmprof.o obj/strict.o
-RMOBJS := $(patsubst %.src,%,$(RMSRC))
+MDOBJS := $(patsubst %.src,%,$(MDSRC))
 INCOBJ := $(patsubst %.inc,obj/%_inc.o, $(INCSRC))
 EXE    := bin/snabb $(patsubst %,bin/%,$(PROGRAM))
 
@@ -69,7 +69,7 @@ $(EXE): snabb bin
 	@echo -n "BINARY    "
 	@ls -sh $@
 
-markdown: $(RMOBJS)
+markdown: $(MDOBJS)
 
 test: $(TESTMODS) $(TESTSCRIPTS)
 
@@ -152,7 +152,7 @@ $(JITOBJS): obj/jit_%.o: ../lib/luajit/src/jit/%.lua $(OBJDIR)
 	$(Q) luajit -bg -n $(patsubst obj/jit_%.o, jit.%, $@) $< $@
 
 
-$(RMOBJS): %: %.src
+$(MDOBJS): %: %.src
 	$(E) "MARKDOWN  $@"
 	$(Q) scripts/process-markdown $< > $@
 
@@ -206,8 +206,8 @@ clean:
 	$(Q)-rm -rf $(CLEAN)
 
 mrproper: clean
-	$(E) "RM        $(RMOBJS)"
-	$(Q)-rm -rf $(RMOBJS)
+	$(E) "RM        $(MDOBJS)"
+	$(Q)-rm -rf $(MDOBJS)
 
 benchmarks:
 	$(Q) (scripts/bench.sh)

From bea68b18641dde89477b85ff9e770b016046342d Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 2 Feb 2016 14:48:55 +0100
Subject: [PATCH 3/8] Better equality procedures for 2-byte/6-byte keys; more
 standard hash functions

* src/lib/ctable.lua (make_equal_fn): Add specialized implementations
  for 2-byte and 6-byte keys as well.
  (hash_32): Rename from hash_i32, as it works on any 32-bit number.
  (hashv_32, hashv_48, hashv_64): New functions.
  (selftest): Update to test the new equal functions.
---
 src/lib/ctable.lua | 66 ++++++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 22 deletions(-)

diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua
index b59ba4c62e..53e71c32db 100644
--- a/src/lib/ctable.lua
+++ b/src/lib/ctable.lua
@@ -11,6 +11,9 @@ local max, floor, ceil = math.max, math.floor, math.ceil
 CTable = {}
 
 local HASH_MAX = 0xFFFFFFFF
+local uint16_ptr_t = ffi.typeof('uint16_t*')
+local uint32_ptr_t = ffi.typeof('uint32_t*')
+local uint64_ptr_t = ffi.typeof('uint64_t*')
 
 local function make_entry_type(key_type, value_type)
    return ffi.typeof([[struct {
@@ -38,13 +41,20 @@ local function make_equal_fn(key_type)
       return function (a, b)
          return a == b
       end
+   elseif size == 2 then
+      return function (a, b)
+         return cast(uint16_ptr_t, a)[0] == cast(uint16_ptr_t, b)[0]
+      end
    elseif size == 4 then
-      local uint32_ptr_t = ffi.typeof('uint32_t*')
       return function (a, b)
          return cast(uint32_ptr_t, a)[0] == cast(uint32_ptr_t, b)[0]
       end
+   elseif size == 6 then
+      return function (a, b)
+         return (cast(uint32_ptr_t, a)[0] == cast(uint32_ptr_t, b)[0] and
+                 cast(uint16_ptr_t, a)[2] == cast(uint16_ptr_t, b)[2])
+      end
    elseif size == 8 then
-      local uint64_ptr_t = ffi.typeof('uint64_t*')
       return function (a, b)
          return cast(uint64_ptr_t, a)[0] == cast(uint64_ptr_t, b)[0]
       end
@@ -360,7 +370,7 @@ end
 -- TODO: Switch to a hash function with good security properties,
 -- perhaps by using the DynASM support for AES.
 local uint32_cast = ffi.new('uint32_t[1]')
-function hash_i32(i32)
+function hash_32(i32)
    i32 = tobit(i32)
    i32 = i32 + bnot(lshift(i32, 15))
    i32 = bxor(i32, (rshift(i32, 10)))
@@ -377,6 +387,25 @@ function hash_i32(i32)
    return uint32_cast[0]
 end
 
+function hashv_32(key)
+   return hash_32(cast(uint32_ptr_t, key)[0])
+end
+
+function hashv_48(key)
+   local hi = cast(uint32_ptr_t, key)[0]
+   local lo = cast(uint16_ptr_t, key)[2]
+   -- Extend lo to the upper half too so that the hash function isn't
+   -- spreading around needless zeroes.
+   lo = bor(lo, lshift(lo, 16))
+   return hash_32(bxor(hi, hash_32(lo)))
+end
+
+function hashv_64(key)
+   local hi = cast(uint32_ptr_t, key)[0]
+   local lo = cast(uint32_ptr_t, key)[1]
+   return hash_32(bxor(hi, hash_32(lo)))
+end
+
 function selftest()
    print("selftest: ctable")
 
@@ -385,7 +414,7 @@ function selftest()
    local params = {
       key_type = ffi.typeof('uint32_t'),
       value_type = ffi.typeof('int32_t[6]'),
-      hash_fn = hash_i32,
+      hash_fn = hash_32,
       max_occupancy_rate = 0.4,
       initial_size = ceil(occupancy / 0.4)
    }
@@ -419,26 +448,19 @@ function selftest()
 
    -- A check that our equality functions work as intended.
    local numbers_equal = make_equal_fn(ffi.typeof('int'))
-   local four_byte = ffi.typeof('uint32_t[1]')
-   local eight_byte = ffi.typeof('uint32_t[2]')
-   local twelve_byte = ffi.typeof('uint32_t[3]')
-   local four_byte_equal = make_equal_fn(four_byte)
-   local eight_byte_equal = make_equal_fn(eight_byte)
-   local twelve_byte_equal = make_equal_fn(twelve_byte)
    assert(numbers_equal(1,1))
    assert(not numbers_equal(1,2))
-   assert(four_byte_equal(ffi.new(four_byte, {1}),
-                          ffi.new(four_byte, {1})))
-   assert(not four_byte_equal(ffi.new(four_byte, {1}),
-                              ffi.new(four_byte, {2})))
-   assert(eight_byte_equal(ffi.new(eight_byte, {1,1}),
-                           ffi.new(eight_byte, {1,1})))
-   assert(not eight_byte_equal(ffi.new(eight_byte, {1,1}),
-                               ffi.new(eight_byte, {1,2})))
-   assert(twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}),
-                            ffi.new(twelve_byte, {1,1,1})))
-   assert(not twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}),
-                                ffi.new(twelve_byte, {1,1,2})))
+
+   local function check_bytes_equal(type, a, b)
+      local equal_fn = make_equal_fn(type)
+      assert(equal_fn(ffi.new(type, a), ffi.new(type, a)))
+      assert(not equal_fn(ffi.new(type, a), ffi.new(type, b)))
+   end
+   check_bytes_equal(ffi.typeof('uint16_t[1]'), {1}, {2})         -- 2 byte
+   check_bytes_equal(ffi.typeof('uint32_t[1]'), {1}, {2})         -- 4 byte
+   check_bytes_equal(ffi.typeof('uint16_t[3]'), {1,1,1}, {1,1,2}) -- 6 byte
+   check_bytes_equal(ffi.typeof('uint32_t[2]'), {1,1}, {1,2})     -- 8 byte
+   check_bytes_equal(ffi.typeof('uint32_t[3]'), {1,1,1}, {1,1,2}) -- 12 byte
 
    print("selftest: ok")
 end

From fe70ccf327739d4b79e9e9c8a8b288efd7932ded Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 2 Feb 2016 14:50:40 +0100
Subject: [PATCH 4/8] Add documentation for lib.ctable.

* src/doc/genbook.sh: Add section for specialized data structures.
* src/lib/README.ctable.md: New file.
---
 src/doc/genbook.sh       |   4 +
 src/lib/README.ctable.md | 198 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 src/lib/README.ctable.md

diff --git a/src/doc/genbook.sh b/src/doc/genbook.sh
index 0746e96657..7b13d6081f 100755
--- a/src/doc/genbook.sh
+++ b/src/doc/genbook.sh
@@ -61,6 +61,10 @@ $(cat ../lib/hardware/README.md)
 
 $(cat ../lib/protocol/README.md)
 
+## Specialized data structures
+
+$(cat ../lib/README.ctable.md)
+
 ## Snabb NFV
 
 $(cat ../program/snabbnfv/README.md)
diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md
new file mode 100644
index 0000000000..62759840b9
--- /dev/null
+++ b/src/lib/README.ctable.md
@@ -0,0 +1,198 @@
+### `ctable` (lib.ctable)
+
+A ctable is a hash table whose keys and values are instances of FFI
+data types.  In Lua parlance, an FFI value is a "cdata" value, hence the
+name "ctable".
+
+A ctable is parameterized the specific types for its keys and values.
+This allows for the table to be stored in an efficient manner.  Adding
+an entry to a ctable will copy the value into the table.  Logically, the
+table "owns" the value.  Lookup can either return a pointer to the value
+in the table, or copy the value into a user-supplied buffer, depending
+on what is most convenient for the user.
+
+As an implementation detail, the table is stored as an open-addressed
+robin-hood hash table with linear probing.  This means that to look up a
+key in the table, we take its hash value (using a user-supplied hash
+function), map that hash value to an index into the table by scaling the
+hash to the table size, and then scan forward in the table until we find
+an entry whose hash value that is greater than or equal to the hash in
+question.  Each entry stores its hash value, and empty entries have a
+hash of `0xFFFFFFFF`.  If the entry's hash matches and the entry's key
+is equal to the one we are looking for, then we have our match.  If the
+entry's hash is greater than our hash, then we have a failure.  Hash
+collisions are possible as well of course; in that case we continue
+scanning forward.
+
+The distance travelled while scanning for the matching hash is known as
+the /displacement/.  The table measures its maximum displacement, for a
+number of purposes, but you might be interested to know that a maximum
+displacement for a table with 2 million entries and a 40% load factor is
+around 8 or 9.  Smaller tables will have smaller maximum displacements.
+
+The ctable has two lookup interfaces.  One will perform the lookup as
+described above, scanning through the hash table in place.  The other
+will fetch all entries within the maximum displacement into a buffer,
+then do a branchless binary search over that buffer.  This second
+streaming lookup can also fetch entries for multiple keys in one go.
+This can amortize the cost of a round-trip to RAM, in the case where you
+expect to miss cache for every lookup.
+
+To create a ctable, first create a parameters table specifying the key
+and value types, along with any other options.  Then call `ctable.new`
+on those parameters.  For example:
+
+```lua
+local ctable = require('lib.ctable')
+local ffi = require('ffi')
+local params = {
+   key_type = ffi.typeof('uint32_t'),
+   value_type = ffi.typeof('int32_t[6]'),
+   hash_fn = ctable.hash_i32,
+   max_occupancy_rate = 0.4,
+   initial_size = math.ceil(occupancy / 0.4)
+}
+local ctab = ctable.new(params)
+```
+
+— Function **ctable.new** *params*
+
+Create a new ctable.  *params* is a table of key/value pairs.  The
+following keys are required:
+
+ * `key_type`: An FFI type for keys in this table.
+ * `value_type`: An FFI type for values in this table.  (In the future,
+   `value_type` will be optional; a nil `value_type` will create a
+   set).
+ * `hash_fn`: A function that takes a key and returns a hash value.
+
+Hash values are unsigned 32-bit integers in the range `[0,
+0xFFFFFFFF)`.  That is to say, `0xFFFFFFFF` is the only unsigned 32-bit
+integer that is not a valid hash value.  The `hash_fn` must return a
+hash value in the correct range.
+
+Optional entries that may be present in the *params* table include:
+
+ * `initial_size`: The initial size of the hash table, including free
+   space.  Defaults to 8 slots.
+ * `max_occupancy_rate`: The maximum ratio of `occupancy/size`, where
+   `occupancy` denotes the number of entries in the table, and `size` is
+   the total table size including free entries.  Trying to add an entry
+   to a "full" table will cause the table to grow in size by a factor of
+   2.  Defaults to 0.9, for a 90% maximum occupancy ratio.
+ * `min_occupancy_rate`: Minimum ratio of `occupancy/size`.  Removing an
+   entry from an "empty" table will shrink the table.
+
+#### Methods
+
+Users interact with a ctable through methods.  In these method
+descriptions, the object on the left-hand-side of the method invocation
+should be a ctable.
+
+— Method **:resize** *size*
+
+Resize the ctable to have *size* total entries, including empty space.
+
+— Method **:insert** *hash* *key* *value* *updates_allowed*
+
+An internal helper method that does the bulk of updates to hash table.
+*hash* is the hash of *key*.  This method takes the hash as an explicit
+parameter because it is used when resizing the table, and that way we
+avoid calling the hash function in that case.  *key* and *value* are FFI
+values for the key and the value, of course.
+
+*updates_allowed* is an optional parameter.  If not present or false,
+then the `:insert` method will raise an error if the *key* is already
+present in the table.  If *updates_allowed* is the string `"required"`,
+then an error will be raised if *key* is /not/ already in the table.
+Any other true value allows updates but does not require them.  An
+update will replace the existing entry in the table.
+
+Returns the index of the inserted entry.
+
+— Method **:add** *key* *value* *updates_allowed*
+
+Add an entry to the ctable, returning the index of the added entry.  See
+the documentation for `:insert` for a description of the parameters.
+
+— Method **:update** *key* *value*
+
+Update the entry in a ctable with the key *key* to have the new value
+*value*.  Throw an error if *key* is not present in the table.
+
+— Method **:lookup_ptr** *key*
+
+Look up *key* in the table, and if found return a pointer to the entry.
+Return nil if the value is not found.
+
+An entry pointer has three fields: the `hash` value, which must not be
+modified; the `key' itself; and the `value`.  Access them as usual in
+Lua:
+
+```lua
+local ptr = ctab:lookup(key)
+if ptr then print(ptr.value) end
+```
+
+Note that pointers are only valid until the next modification of a
+table.
+
+— Method **:lookup_and_copy** *key* *entry*
+
+Look up *key* in the table, and if found, copy that entry into *entry*
+and return true.  Otherwise return false.
+
+— Method **:remove_ptr** *entry*
+
+Remove an entry from a ctable.  *entry* should be a pointer that points
+into the table.  Note that pointers are only valid until the next
+modification of a table.
+
+— Method **:remove** *key* *missing_allowed*
+
+Remove an entry from a ctable, keyed by *key*.
+
+Return true if we actually do find a value and remove it.  Otherwise if
+no entry is found in the table and *missing_allowed* is true, then
+return false.  Otherwise raise an error.
+
+— Method **:selfcheck**
+
+Run an expensive internal diagnostic to verify that the table's internal
+invariants are fulfilled.
+
+— Method **:dump**
+
+Print out the entries in a table.  Can be expensive if the table is
+large.
+
+— Method **:iterate**
+
+Return an iterator for use by `for in`.  For example:
+
+```lua
+for entry in ctab:iterate() do
+   print(entry.key, entry.value)
+end
+```
+
+#### Hash functions
+
+Any hash function will do, as long as it produces values in the right
+range.  In practice we include some functions for hashing byte sequences
+of some common small lengths.
+
+— Function **ctable.hash_32** *number*
+Hash a 32-bit integer.  As a `hash_fn` parameter, this will only work if
+your key type's Lua representation is a Lua number.  For example, use
+`hash_32` on `ffi.typeof('uint32_t')`, but use `hashv_32` on
+`ffi.typeof('uint8_t[4]')`.
+
+— Function **ctable.hashv_32** *ptr*
+Hash the first 32 bits of a byte sequence.
+
+— Function **ctable.hashv_48** *ptr*
+Hash the first 48 bits of a byte sequence.
+
+— Function **ctable.hashv_64** *ptr*
+Hash the first 64 bits of a byte sequence.

From 27b82d5be1dcf9612b79e013b6324e40342709b5 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 2 Feb 2016 15:49:12 +0100
Subject: [PATCH 5/8] Fix typo.

---
 src/lib/README.ctable.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md
index 62759840b9..c9c3da476b 100644
--- a/src/lib/README.ctable.md
+++ b/src/lib/README.ctable.md
@@ -4,12 +4,12 @@ A ctable is a hash table whose keys and values are instances of FFI
 data types.  In Lua parlance, an FFI value is a "cdata" value, hence the
 name "ctable".
 
-A ctable is parameterized the specific types for its keys and values.
-This allows for the table to be stored in an efficient manner.  Adding
-an entry to a ctable will copy the value into the table.  Logically, the
-table "owns" the value.  Lookup can either return a pointer to the value
-in the table, or copy the value into a user-supplied buffer, depending
-on what is most convenient for the user.
+A ctable is parameterized for the specific types for its keys and
+values.  This allows for the table to be stored in an efficient manner.
+Adding an entry to a ctable will copy the value into the table.
+Logically, the table "owns" the value.  Lookup can either return a
+pointer to the value in the table, or copy the value into a
+user-supplied buffer, depending on what is most convenient for the user.
 
 As an implementation detail, the table is stored as an open-addressed
 robin-hood hash table with linear probing.  This means that to look up a

From c5877283c86072beef9330ffd9947dbb82a39a23 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 2 Feb 2016 15:58:40 +0100
Subject: [PATCH 6/8] Fix another typo.

Thanks to Kristian Larsson for catching it.
---
 src/lib/README.ctable.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md
index c9c3da476b..0392d3c1ae 100644
--- a/src/lib/README.ctable.md
+++ b/src/lib/README.ctable.md
@@ -16,7 +16,7 @@ robin-hood hash table with linear probing.  This means that to look up a
 key in the table, we take its hash value (using a user-supplied hash
 function), map that hash value to an index into the table by scaling the
 hash to the table size, and then scan forward in the table until we find
-an entry whose hash value that is greater than or equal to the hash in
+an entry whose hash value is greater than or equal to the hash in
 question.  Each entry stores its hash value, and empty entries have a
 hash of `0xFFFFFFFF`.  If the entry's hash matches and the entry's key
 is equal to the one we are looking for, then we have our match.  If the

From bc2fda2cd6c2a1c383331a79b282853d1cc3b7d3 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Thu, 4 Feb 2016 09:52:57 +0100
Subject: [PATCH 7/8] Revert "Makefile can process any .md.src to an .md, not
 just README.md"

This reverts commit 9666348f1a09ae8d8edf58c65382ac6b21b4a14a.
---
 src/Makefile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index bfec34ee3a..938ab86543 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -16,7 +16,7 @@ CSRC   = $(shell find . -regex '[^\#]*\.c' -not -regex './arch/.*' -printf '%P '
 CHDR   = $(shell find . -regex '[^\#]*\.h' -printf '%P ')
 ASM    = $(shell find . -regex '[^\#]*\.dasl' -printf '%P ')
 ARCHSRC= $(shell find . -regex '^./arch/[^\#]*\.c' -printf '%P ')
-MDSRC  = $(shell find . -regex '[^\#]*\.md.src' -printf '%P ')
+RMSRC  = $(shell find . -name  README.md.src -printf '%P ')
 # regexp is to include program/foo but not program/foo/bar
 PROGRAM = $(shell find program -regex '^[^/]+/[^/]+' -type d -printf '%P ')
 # sort to eliminate potential duplicate of programs.inc
@@ -30,7 +30,7 @@ ARCHOBJ:= $(patsubst %.c,obj/%_c.o,    $(ARCHSRC))
 ASMOBJ := $(patsubst %.dasl,obj/%_dasl.o,   $(ASM))
 JITOBJS:= $(patsubst %,obj/jit_%.o,$(JITSRC))
 EXTRAOBJS := obj/jit_tprof.o obj/jit_vmprof.o obj/strict.o
-MDOBJS := $(patsubst %.src,%,$(MDSRC))
+RMOBJS := $(patsubst %.src,%,$(RMSRC))
 INCOBJ := $(patsubst %.inc,obj/%_inc.o, $(INCSRC))
 EXE    := bin/snabb $(patsubst %,bin/%,$(PROGRAM))
 
@@ -69,7 +69,7 @@ $(EXE): snabb bin
 	@echo -n "BINARY    "
 	@ls -sh $@
 
-markdown: $(MDOBJS)
+markdown: $(RMOBJS)
 
 test: $(TESTMODS) $(TESTSCRIPTS)
 
@@ -152,7 +152,7 @@ $(JITOBJS): obj/jit_%.o: ../lib/luajit/src/jit/%.lua $(OBJDIR)
 	$(Q) luajit -bg -n $(patsubst obj/jit_%.o, jit.%, $@) $< $@
 
 
-$(MDOBJS): %: %.src
+$(RMOBJS): %: %.src
 	$(E) "MARKDOWN  $@"
 	$(Q) scripts/process-markdown $< > $@
 
@@ -206,8 +206,8 @@ clean:
 	$(Q)-rm -rf $(CLEAN)
 
 mrproper: clean
-	$(E) "RM        $(MDOBJS)"
-	$(Q)-rm -rf $(MDOBJS)
+	$(E) "RM        $(RMOBJS)"
+	$(Q)-rm -rf $(RMOBJS)
 
 benchmarks:
 	$(Q) (scripts/bench.sh)

From 5b9400a739f3f800a2bbe52c5dcd461c9ee15fe7 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Thu, 4 Feb 2016 10:33:07 +0100
Subject: [PATCH 8/8] Move ctable documentation to src/lib/README.md, fix nits

Thanks to Max Rottenkolber for review.
---
 src/doc/genbook.sh                      |  6 ++--
 src/lib/{README.ctable.md => README.md} | 44 ++++++++++++++-----------
 2 files changed, 26 insertions(+), 24 deletions(-)
 rename src/lib/{README.ctable.md => README.md} (86%)

diff --git a/src/doc/genbook.sh b/src/doc/genbook.sh
index 7b13d6081f..0a0e76ac96 100755
--- a/src/doc/genbook.sh
+++ b/src/doc/genbook.sh
@@ -53,6 +53,8 @@ $(cat ../apps/socket/README.md)
 
 # Libraries
 
+$(cat ../lib/README.md)
+
 ## Hardware
 
 $(cat ../lib/hardware/README.md)
@@ -61,10 +63,6 @@ $(cat ../lib/hardware/README.md)
 
 $(cat ../lib/protocol/README.md)
 
-## Specialized data structures
-
-$(cat ../lib/README.ctable.md)
-
 ## Snabb NFV
 
 $(cat ../program/snabbnfv/README.md)
diff --git a/src/lib/README.ctable.md b/src/lib/README.md
similarity index 86%
rename from src/lib/README.ctable.md
rename to src/lib/README.md
index 0392d3c1ae..c6b218ab35 100644
--- a/src/lib/README.ctable.md
+++ b/src/lib/README.md
@@ -1,6 +1,8 @@
-### `ctable` (lib.ctable)
+## Specialized data structures
 
-A ctable is a hash table whose keys and values are instances of FFI
+### Ctable (lib.ctable)
+
+A *ctable* is a hash table whose keys and values are instances of FFI
 data types.  In Lua parlance, an FFI value is a "cdata" value, hence the
 name "ctable".
 
@@ -25,7 +27,7 @@ collisions are possible as well of course; in that case we continue
 scanning forward.
 
 The distance travelled while scanning for the matching hash is known as
-the /displacement/.  The table measures its maximum displacement, for a
+the *displacement*.  The table measures its maximum displacement, for a
 number of purposes, but you might be interested to know that a maximum
 displacement for a table with 2 million entries and a 40% load factor is
 around 8 or 9.  Smaller tables will have smaller maximum displacements.
@@ -55,15 +57,13 @@ local params = {
 local ctab = ctable.new(params)
 ```
 
-— Function **ctable.new** *params*
+— Function **ctable.new** *parameters*
 
-Create a new ctable.  *params* is a table of key/value pairs.  The
+Create a new ctable.  *parameters* is a table of key/value pairs.  The
 following keys are required:
 
- * `key_type`: An FFI type for keys in this table.
- * `value_type`: An FFI type for values in this table.  (In the future,
-   `value_type` will be optional; a nil `value_type` will create a
-   set).
+ * `key_type`: An FFI type (LuaJIT "ctype") for keys in this table.
+ * `value_type`: An FFI type (LuaJT "ctype") for values in this table.
  * `hash_fn`: A function that takes a key and returns a hash value.
 
 Hash values are unsigned 32-bit integers in the range `[0,
@@ -71,7 +71,7 @@ Hash values are unsigned 32-bit integers in the range `[0,
 integer that is not a valid hash value.  The `hash_fn` must return a
 hash value in the correct range.
 
-Optional entries that may be present in the *params* table include:
+Optional entries that may be present in the *parameters* table include:
 
  * `initial_size`: The initial size of the hash table, including free
    space.  Defaults to 8 slots.
@@ -93,7 +93,7 @@ should be a ctable.
 
 Resize the ctable to have *size* total entries, including empty space.
 
-— Method **:insert** *hash* *key* *value* *updates_allowed*
+— Method **:insert** *hash*, *key*, *value*, *updates_allowed*
 
 An internal helper method that does the bulk of updates to hash table.
 *hash* is the hash of *key*.  This method takes the hash as an explicit
@@ -104,18 +104,18 @@ values for the key and the value, of course.
 *updates_allowed* is an optional parameter.  If not present or false,
 then the `:insert` method will raise an error if the *key* is already
 present in the table.  If *updates_allowed* is the string `"required"`,
-then an error will be raised if *key* is /not/ already in the table.
+then an error will be raised if *key* is *not* already in the table.
 Any other true value allows updates but does not require them.  An
 update will replace the existing entry in the table.
 
 Returns the index of the inserted entry.
 
-— Method **:add** *key* *value* *updates_allowed*
+— Method **:add** *key*, *value*, *updates_allowed*
 
 Add an entry to the ctable, returning the index of the added entry.  See
 the documentation for `:insert` for a description of the parameters.
 
-— Method **:update** *key* *value*
+— Method **:update** *key*, *value*
 
 Update the entry in a ctable with the key *key* to have the new value
 *value*.  Throw an error if *key* is not present in the table.
@@ -126,7 +126,7 @@ Look up *key* in the table, and if found return a pointer to the entry.
 Return nil if the value is not found.
 
 An entry pointer has three fields: the `hash` value, which must not be
-modified; the `key' itself; and the `value`.  Access them as usual in
+modified; the `key` itself; and the `value`.  Access them as usual in
 Lua:
 
 ```lua
@@ -137,7 +137,7 @@ if ptr then print(ptr.value) end
 Note that pointers are only valid until the next modification of a
 table.
 
-— Method **:lookup_and_copy** *key* *entry*
+— Method **:lookup_and_copy** *key*, *entry*
 
 Look up *key* in the table, and if found, copy that entry into *entry*
 and return true.  Otherwise return false.
@@ -148,7 +148,7 @@ Remove an entry from a ctable.  *entry* should be a pointer that points
 into the table.  Note that pointers are only valid until the next
 modification of a table.
 
-— Method **:remove** *key* *missing_allowed*
+— Method **:remove** *key*, *missing_allowed*
 
 Remove an entry from a ctable, keyed by *key*.
 
@@ -178,21 +178,25 @@ end
 
 #### Hash functions
 
-Any hash function will do, as long as it produces values in the right
-range.  In practice we include some functions for hashing byte sequences
-of some common small lengths.
+Any hash function will do, as long as it produces values in the `[0,
+0xFFFFFFFF)` range.  In practice we include some functions for hashing
+byte sequences of some common small lengths.
 
 — Function **ctable.hash_32** *number*
+
 Hash a 32-bit integer.  As a `hash_fn` parameter, this will only work if
 your key type's Lua representation is a Lua number.  For example, use
 `hash_32` on `ffi.typeof('uint32_t')`, but use `hashv_32` on
 `ffi.typeof('uint8_t[4]')`.
 
 — Function **ctable.hashv_32** *ptr*
+
 Hash the first 32 bits of a byte sequence.
 
 — Function **ctable.hashv_48** *ptr*
+
 Hash the first 48 bits of a byte sequence.
 
 — Function **ctable.hashv_64** *ptr*
+
 Hash the first 64 bits of a byte sequence.