Vf/4018/env variables

src/Makefile

@@ -4,6 +4,8 @@ E= @echo
 #Q=
 #E= @:
 
+TEST_SKIPPED="43"
+
 SRCDIR = $(shell find . -type d -not -regex './obj.*' -printf '%P ')
 OBJDIR = $(patsubst %,obj/%,$(SRCDIR))
 
@@ -47,7 +49,18 @@ test: $(TESTMODS)
 
 $(TESTMODS): testlog snabbswitch
 	$(E) "TEST      $@"
-	$(Q) ./snabbswitch -t $@ > testlog/$@ || (echo "*** EXIT CODE: $?" >> testlog/$@; echo "ERROR     testlog/$@")
+	$(Q) ./snabbswitch -t $@ > testlog/$@ || ( \
+		EXITCODE="$$?"; \
+		[ "$$EXITCODE" -eq $(TEST_SKIPPED) ] \
+		&& ( \
+			echo "SKIPPED   testlog/$@"; \
+			echo "EXITCODE: $$EXITCODE" >> testlog/$@; \
+		) \
+		|| ( \
+			echo "ERROR     testlog/$@"; \
+			echo "EXITCODE: $$EXITCODE" >> testlog/$@; \
+		) \
+	)
 
 $(OBJDIR) testlog:
 	$(E) "DIR       $@"

src/apps/intel/intel_app.lua

@@ -72,20 +72,35 @@ function Intel82599:report ()
    register.dump(self.dev.s, true)
 end
 
+function getTestPCIID()
+   return os.getenv("SNABB_TEST_PCI_ID")
+end
+
 function selftest ()
+   print("selftest: intel_app")
+   if not vfio.is_vfio_available() then
+      print("VFIO not available\nTest skipped")
+      os.exit(app.test_skipped_code)
+   end
+
+   local pciid = getTestPCIID()
+   if not pciid then
+      print("SNABB_sPCI_ID was not set\nTest skipped")
+      os.exit(app.test_skipped_code)
+   end
    -- Create a pieline:
    --   Source --> Intel82599(loopback) --> Sink
    -- and push packets through it.
-   vfio.bind_device_to_vfio("0000:01:00.0")
+   vfio.bind_device_to_vfio(pciid)
    local c = config.new()
-   config.app(c, "intel10g", Intel82599, "0000:01:00.0")
+   config.app(c, "intel10g", Intel82599, pciid)
    config.app(c, "source", basic_apps.Source)
    config.app(c, "sink", basic_apps.Sink)
    config.link(c, "source.out -> intel10g.rx")
    config.link(c, "intel10g.tx -> sink.in")
    app.configure(c)
 --[[
-   app.apps.intel10g = Intel82599:new("0000:01:00.0")
+   app.apps.intel10g = Intel82599:new(pciid)
    app.apps.source = app.new(basic_apps.Source)
    app.apps.sink   = app.new(basic_apps.Sink)
    app.connect("source", "out", "intel10g", "rx")

src/apps/vhost/vhost_user.lua

@@ -416,8 +416,20 @@ function map_from_qemu (addr, mem_table)
    error("mapping to host address failed" .. tostring(ffi.cast("void*",addr)))
 end
 
+function getTestPCIID()
+   return os.getenv("SNABB_TEST_PCI_ID")
+end
+
+function getVhostUserSocketFile()
+   return os.getenv("SNABB_VHOST_USER_SOCKET_FILE")
+end
+
 function selftest ()
    print("selftest: vhost_user")
+   if not vfio.is_vfio_available() then
+      print("VFIO not available\nTest skipped")
+      os.exit(app.test_skipped_code)
+   end
    -- Create an app network that proxies packets between a vhost_user
    -- port (qemu) and an Intel port (in loopback mode). Create
    -- separate pcap traces for packets received from vhost and intel.
@@ -435,14 +447,26 @@ function selftest ()
    --           v
    --       intel pcap
    -- 
-   pci.unbind_device_from_linux('0000:01:00.0')
-   vfio.setup_vfio('0000:01:00.0')
-   vfio.bind_device_to_vfio("0000:01:00.0")
+   local pciid = getTestPCIID()
+   if not pciid then
+      print("SNABB_TEST_PCI_ID was not set\nTest skipped")
+      os.exit(app.test_skipped_code)
+   end
+
+   local vhost_user_sock = getVhostUserSocketFile()
+   if not vhost_user_sock then
+      print("SNABB_VHOST_USER_SOCKET_FILE was not set\nTest skipped")
+      os.exit(app.test_skipped_code)
+   end
+
+   pci.unbind_device_from_linux(pciid)
+   vfio.setup_vfio(pciid)
+   vfio.bind_device_to_vfio(pciid)
    local c = config.new()
-   config.app(c, "vhost_user", VhostUser, "vhost_user_test.sock")
+   config.app(c, "vhost_user", VhostUser, vhost_user_sock)
    config.app(c, "vhost_dump", pcap.PcapWriter, "vhost_vm_dump.cap")
    config.app(c, "vhost_tee", basic_apps.Tee)
-   config.app(c, "intel", intel_app.Intel82599, "0000:01:00.0")
+   config.app(c, "intel", intel_app.Intel82599, pciid)
    config.app(c, "intel_dump", pcap.PcapWriter, "vhost_nic_dump.cap")
    config.app(c, "intel_tee", basic_apps.Tee)
    config.link(c, "vhost_user.tx -> vhost_tee.input")

src/core/app.lua

@@ -8,6 +8,8 @@ local config = require("core.config")
 local timer  = require("core.timer")
 require("core.packet_h")
 
+test_skipped_code = 43
+
 -- The set of all active apps and links in the system.
 -- Indexed both by name (in a table) and by number (in an array).
 app_table,  app_array  = {}, {}

src/core/memory.lua

@@ -6,6 +6,10 @@ local C = ffi.C
 local lib = require("core.lib")
 require("core.memory_h")
 
+-- This path is used if the "SNABB_HUGEPAGES" environment variable is not defined
+DEFAULT_HUGEPAGES_PATH = "/proc/sys/vm/nr_hugepages"
+-- This path is used if the "SNABB_MEMINFO" environment variable is not defined
+DEFAULT_MEMINFO_PATH = "/proc/meminfo"
 
 -- hook variables
 
@@ -59,18 +63,28 @@ function reserve_new_page ()
    set_hugepages(get_hugepages() + 1)
 end
 
+function get_hugepages_path()
+   return os.getenv("SNABB_HUGEPAGES") or DEFAULT_HUGEPAGES_PATH
+end
+
+function get_meminfo_path()
+   return os.getenv("SNABB_MEMINFO") or DEFAULT_MEMINFO_PATH
+end
+
 function get_hugepages ()
-   return lib.readfile("/proc/sys/vm/nr_hugepages", "*n")
+   return lib.readfile(get_hugepages_path(), "*n")
 end
 
 function set_hugepages (n)
-   lib.writefile("/proc/sys/vm/nr_hugepages", tostring(n))
+   lib.writefile(get_hugepages_path(), tostring(n))
 end
 
 function get_huge_page_size ()
-   local meminfo = lib.readfile("/proc/meminfo", "*a")
+   local meminfo = lib.readfile(get_meminfo_path(), "*a")
    local _,_,hugesize = meminfo:find("Hugepagesize: +([0-9]+) kB")
-   return tonumber(hugesize) * 1024
+   return hugesize
+      and tonumber(hugesize) * 1024
+       or 2048 -- A typical x86 system will have a Huge Page Size of 2048 kBytes
 end
 
 base_page_size = 4096
@@ -95,7 +109,7 @@ end
 function selftest (options)
    print("selftest: memory")
    require("lib.hardware.bus")
-   print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages())
+   print("HugeTLB pages (" .. get_hugepages_path() .. "): " .. get_hugepages())
    for i = 1, 4 do
       io.write("  Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB: ")
       io.flush()
@@ -105,7 +119,7 @@ function selftest (options)
       ffi.cast("uint32_t*", dmaptr)[0] = 0xdeadbeef -- try a write
       assert(dmaptr ~= nil and dmalen == huge_page_size)
    end
-   print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages())
+   print("HugeTLB pages (" .. get_hugepages_path() .. "): " .. get_hugepages())
    print("HugeTLB page allocation OK.")
 end
 

src/doc/hacking.md

@@ -0,0 +1,36 @@
+Several environment variables can be set for snabbswitch code:
+
+* SNABB_VFIO_DRIVER
+  Default value "/sys/bus/pci/drivers/vfio-pci"
+
+* SNABB_VFIO_IOMMU_GROUPS
+  Default value "/sys/kernel/iommu_groups"
+
+* SNABB_PCI_DEVICE
+  Default value "/sys/bus/pci/devices"
+
+* SNABB_HUGEPAGES
+  Default value "/proc/sys/vm/nr_hugepages"
+
+* SNABB_MEMINFO
+  Default value "/proc/meminfo"
+
+* SNABB_VHOST_USER_SOCKET_FILE
+  No default value
+
+* SNABB_TEST_PCI_ID
+  No default value
+
+You can run tests defining some of the variables:
+
+    cd src; sudo SNABB_TEST_PCI_ID="0000:01:00.0" \
+      SNABB_VHOST_USER_SOCKET_FILE="vhost_user_test.sock" make test;
+
+if a test can't find resource needed it will usually skip and return code 43
+(TEST_SKIPPED_CODE).
+
+Also separate commands can utilize environment virables changes:
+
+    sudo SNABB_HUGEPAGES=/proc/sys/vm/nr_hugepages snabbswitch -l designs.basic.basic
+
+FIXME: add some sane examples and explanatory notes to variables.

src/lib/hardware/bus.lua

@@ -16,7 +16,7 @@ devices = {}
 map_devices = {}
 
 function scan_devices ()
-    for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do
+    for _,device in ipairs(lib.files_in_directory(pci.get_pci_device_path())) do
         local info = device_info(device)
         if info.driver and not map_devices[device] then
             table.insert(devices, info)
@@ -26,7 +26,7 @@ function scan_devices ()
 end
 
 function host_has_vfio()
-    local files = lib.files_in_directory('/sys/kernel/iommu_groups/')
+    local files = lib.files_in_directory(vfio.get_iommu_groups_path())
     return files and #files > 0
 end
 

src/lib/hardware/pci.lua

@@ -7,6 +7,9 @@ local lib = require("core.lib")
 
 require("lib.hardware.pci_h")
 
+-- This path is used if the "SNABB_PCI_DEVICE" environment variable is not defined
+pci_default_device_path = "/sys/bus/pci/devices"
+
 --- ### Hardware device information
 
 devices = {}
@@ -25,7 +28,7 @@ devices = {}
 
 --- Initialize (or re-initialize) the `devices` table.
 function scan_devices ()
-   for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do
+   for _,device in ipairs(lib.files_in_directory(get_pci_device_path())) do
       local info = device_info(device)
       if info.driver then table.insert(devices, info) end
    end
@@ -46,8 +49,14 @@ function device_info (pciaddress)
    return info
 end
 
+function get_pci_device_path()
+   return os.getenv("SNABB_PCI_DEVICE") or pci_default_device_path
+end
+
 --- Return the path to the sysfs directory for `pcidev`.
-function path(pcidev) return "/sys/bus/pci/devices/"..pcidev end
+function path(pcidev)
+   return get_pci_device_path() .. "/" .. pcidev
+end
 
 -- Return the name of the Lua module that implements support for this device.
 function which_driver (vendor, device)

src/lib/hardware/vfio.lua

@@ -11,6 +11,11 @@ require("lib.hardware.vfio_h")
 -- Is VFIO initialized yet?
 initialized = false
 
+-- This path is used if the "SNABB_VFIO_DRIVER" environment variable is not defined
+vfio_default_driver_path = "/sys/bus/pci/drivers/vfio-pci"
+-- This path is used if the "SNABB_VFIO_IOMMU_GROUPS" environment variable is not defined
+vfio_default_iommu_group_path = "/sys/kernel/iommu_groups"
+
 -- Array of mappings that were requested before vfio was initialized.
 -- 
 -- These must then be mapped at initialization time.
@@ -69,17 +74,31 @@ function device_group(pciaddress)
     return lib.basename(lib.readlink(pci.path(pciaddress)..'/iommu_group'))
 end
 
+function get_iommu_groups_path()
+   return os.getenv("SNABB_VFIO_IOMMU_GROUPS") or vfio_default_iommu_group_path
+end
+
 -- Return all the devices that belong to the same group
 function group_devices(group)
-    if not group then return {} end
-    return lib.files_in_directory('/sys/kernel/iommu_groups/'..group..'/devices/')
+   if not group then return {} end
+   return lib.files_in_directory(
+         get_iommu_groups_path() .. '/' .. group .. '/devices/'
+      )
 end
 
 --- ### Device manipulation.
 
+function get_driver_path ()
+   return os.getenv("SNABB_VFIO_DRIVER") or vfio_default_driver_path
+end
+
 --- add a device to the vfio-pci driver
 function bind_device_to_vfio (pciaddress)
-    lib.writefile("/sys/bus/pci/drivers/vfio-pci/bind", pciaddress)
+   lib.writefile(get_driver_path() .. "/bind", pciaddress)
+end
+
+function is_vfio_available()
+   return lib.can_write(get_driver_path() .. "/bind")
 end
 
 function setup_vfio(pciaddress, do_group)