diff --git a/src/Makefile b/src/Makefile index c8d9c93288..de52a2181a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,6 +4,8 @@ E= @echo #Q= #E= @: +TEST_SKIPPED="43" + SRCDIR = $(shell find . -type d -not -regex './obj.*' -printf '%P ') OBJDIR = $(patsubst %,obj/%,$(SRCDIR)) @@ -47,7 +49,18 @@ test: $(TESTMODS) $(TESTMODS): testlog snabbswitch $(E) "TEST $@" - $(Q) ./snabbswitch -t $@ > testlog/$@ || (echo "*** EXIT CODE: $?" >> testlog/$@; echo "ERROR testlog/$@") + $(Q) ./snabbswitch -t $@ > testlog/$@ || ( \ + EXITCODE="$$?"; \ + [ "$$EXITCODE" -eq $(TEST_SKIPPED) ] \ + && ( \ + echo "SKIPPED testlog/$@"; \ + echo "EXITCODE: $$EXITCODE" >> testlog/$@; \ + ) \ + || ( \ + echo "ERROR testlog/$@"; \ + echo "EXITCODE: $$EXITCODE" >> testlog/$@; \ + ) \ + ) $(OBJDIR) testlog: $(E) "DIR $@" diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 18213b9e2d..21b93c2abe 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -72,20 +72,35 @@ function Intel82599:report () register.dump(self.dev.s, true) end +function getTestPCIID() + return os.getenv("SNABB_TEST_PCI_ID") +end + function selftest () + print("selftest: intel_app") + if not vfio.is_vfio_available() then + print("VFIO not available\nTest skipped") + os.exit(app.test_skipped_code) + end + + local pciid = getTestPCIID() + if not pciid then + print("SNABB_sPCI_ID was not set\nTest skipped") + os.exit(app.test_skipped_code) + end -- Create a pieline: -- Source --> Intel82599(loopback) --> Sink -- and push packets through it. - vfio.bind_device_to_vfio("0000:01:00.0") + vfio.bind_device_to_vfio(pciid) local c = config.new() - config.app(c, "intel10g", Intel82599, "0000:01:00.0") + config.app(c, "intel10g", Intel82599, pciid) config.app(c, "source", basic_apps.Source) config.app(c, "sink", basic_apps.Sink) config.link(c, "source.out -> intel10g.rx") config.link(c, "intel10g.tx -> sink.in") app.configure(c) --[[ - app.apps.intel10g = Intel82599:new("0000:01:00.0") + app.apps.intel10g = Intel82599:new(pciid) app.apps.source = app.new(basic_apps.Source) app.apps.sink = app.new(basic_apps.Sink) app.connect("source", "out", "intel10g", "rx") diff --git a/src/apps/vhost/vhost_user.lua b/src/apps/vhost/vhost_user.lua index d5517d301e..18d5a86b4c 100644 --- a/src/apps/vhost/vhost_user.lua +++ b/src/apps/vhost/vhost_user.lua @@ -416,8 +416,20 @@ function map_from_qemu (addr, mem_table) error("mapping to host address failed" .. tostring(ffi.cast("void*",addr))) end +function getTestPCIID() + return os.getenv("SNABB_TEST_PCI_ID") +end + +function getVhostUserSocketFile() + return os.getenv("SNABB_VHOST_USER_SOCKET_FILE") +end + function selftest () print("selftest: vhost_user") + if not vfio.is_vfio_available() then + print("VFIO not available\nTest skipped") + os.exit(app.test_skipped_code) + end -- Create an app network that proxies packets between a vhost_user -- port (qemu) and an Intel port (in loopback mode). Create -- separate pcap traces for packets received from vhost and intel. @@ -435,14 +447,26 @@ function selftest () -- v -- intel pcap -- - pci.unbind_device_from_linux('0000:01:00.0') - vfio.setup_vfio('0000:01:00.0') - vfio.bind_device_to_vfio("0000:01:00.0") + local pciid = getTestPCIID() + if not pciid then + print("SNABB_TEST_PCI_ID was not set\nTest skipped") + os.exit(app.test_skipped_code) + end + + local vhost_user_sock = getVhostUserSocketFile() + if not vhost_user_sock then + print("SNABB_VHOST_USER_SOCKET_FILE was not set\nTest skipped") + os.exit(app.test_skipped_code) + end + + pci.unbind_device_from_linux(pciid) + vfio.setup_vfio(pciid) + vfio.bind_device_to_vfio(pciid) local c = config.new() - config.app(c, "vhost_user", VhostUser, "vhost_user_test.sock") + config.app(c, "vhost_user", VhostUser, vhost_user_sock) config.app(c, "vhost_dump", pcap.PcapWriter, "vhost_vm_dump.cap") config.app(c, "vhost_tee", basic_apps.Tee) - config.app(c, "intel", intel_app.Intel82599, "0000:01:00.0") + config.app(c, "intel", intel_app.Intel82599, pciid) config.app(c, "intel_dump", pcap.PcapWriter, "vhost_nic_dump.cap") config.app(c, "intel_tee", basic_apps.Tee) config.link(c, "vhost_user.tx -> vhost_tee.input") diff --git a/src/core/app.lua b/src/core/app.lua index 4c950a2421..0612abe255 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -8,6 +8,8 @@ local config = require("core.config") local timer = require("core.timer") require("core.packet_h") +test_skipped_code = 43 + -- The set of all active apps and links in the system. -- Indexed both by name (in a table) and by number (in an array). app_table, app_array = {}, {} diff --git a/src/core/memory.lua b/src/core/memory.lua index f6fb5da7b4..e790637814 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -6,6 +6,10 @@ local C = ffi.C local lib = require("core.lib") require("core.memory_h") +-- This path is used if the "SNABB_HUGEPAGES" environment variable is not defined +DEFAULT_HUGEPAGES_PATH = "/proc/sys/vm/nr_hugepages" +-- This path is used if the "SNABB_MEMINFO" environment variable is not defined +DEFAULT_MEMINFO_PATH = "/proc/meminfo" -- hook variables @@ -59,18 +63,28 @@ function reserve_new_page () set_hugepages(get_hugepages() + 1) end +function get_hugepages_path() + return os.getenv("SNABB_HUGEPAGES") or DEFAULT_HUGEPAGES_PATH +end + +function get_meminfo_path() + return os.getenv("SNABB_MEMINFO") or DEFAULT_MEMINFO_PATH +end + function get_hugepages () - return lib.readfile("/proc/sys/vm/nr_hugepages", "*n") + return lib.readfile(get_hugepages_path(), "*n") end function set_hugepages (n) - lib.writefile("/proc/sys/vm/nr_hugepages", tostring(n)) + lib.writefile(get_hugepages_path(), tostring(n)) end function get_huge_page_size () - local meminfo = lib.readfile("/proc/meminfo", "*a") + local meminfo = lib.readfile(get_meminfo_path(), "*a") local _,_,hugesize = meminfo:find("Hugepagesize: +([0-9]+) kB") - return tonumber(hugesize) * 1024 + return hugesize + and tonumber(hugesize) * 1024 + or 2048 -- A typical x86 system will have a Huge Page Size of 2048 kBytes end base_page_size = 4096 @@ -95,7 +109,7 @@ end function selftest (options) print("selftest: memory") require("lib.hardware.bus") - print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages()) + print("HugeTLB pages (" .. get_hugepages_path() .. "): " .. get_hugepages()) for i = 1, 4 do io.write(" Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB: ") io.flush() @@ -105,7 +119,7 @@ function selftest (options) ffi.cast("uint32_t*", dmaptr)[0] = 0xdeadbeef -- try a write assert(dmaptr ~= nil and dmalen == huge_page_size) end - print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages()) + print("HugeTLB pages (" .. get_hugepages_path() .. "): " .. get_hugepages()) print("HugeTLB page allocation OK.") end diff --git a/src/doc/hacking.md b/src/doc/hacking.md new file mode 100644 index 0000000000..4923cd0045 --- /dev/null +++ b/src/doc/hacking.md @@ -0,0 +1,36 @@ +Several environment variables can be set for snabbswitch code: + +* SNABB_VFIO_DRIVER + Default value "/sys/bus/pci/drivers/vfio-pci" + +* SNABB_VFIO_IOMMU_GROUPS + Default value "/sys/kernel/iommu_groups" + +* SNABB_PCI_DEVICE + Default value "/sys/bus/pci/devices" + +* SNABB_HUGEPAGES + Default value "/proc/sys/vm/nr_hugepages" + +* SNABB_MEMINFO + Default value "/proc/meminfo" + +* SNABB_VHOST_USER_SOCKET_FILE + No default value + +* SNABB_TEST_PCI_ID + No default value + +You can run tests defining some of the variables: + + cd src; sudo SNABB_TEST_PCI_ID="0000:01:00.0" \ + SNABB_VHOST_USER_SOCKET_FILE="vhost_user_test.sock" make test; + +if a test can't find resource needed it will usually skip and return code 43 +(TEST_SKIPPED_CODE). + +Also separate commands can utilize environment virables changes: + + sudo SNABB_HUGEPAGES=/proc/sys/vm/nr_hugepages snabbswitch -l designs.basic.basic + +FIXME: add some sane examples and explanatory notes to variables. diff --git a/src/lib/hardware/bus.lua b/src/lib/hardware/bus.lua index 9dbe1c1846..452302d649 100644 --- a/src/lib/hardware/bus.lua +++ b/src/lib/hardware/bus.lua @@ -16,7 +16,7 @@ devices = {} map_devices = {} function scan_devices () - for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do + for _,device in ipairs(lib.files_in_directory(pci.get_pci_device_path())) do local info = device_info(device) if info.driver and not map_devices[device] then table.insert(devices, info) @@ -26,7 +26,7 @@ function scan_devices () end function host_has_vfio() - local files = lib.files_in_directory('/sys/kernel/iommu_groups/') + local files = lib.files_in_directory(vfio.get_iommu_groups_path()) return files and #files > 0 end diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index 8b5cc5f79c..a4fe0d3804 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -7,6 +7,9 @@ local lib = require("core.lib") require("lib.hardware.pci_h") +-- This path is used if the "SNABB_PCI_DEVICE" environment variable is not defined +pci_default_device_path = "/sys/bus/pci/devices" + --- ### Hardware device information devices = {} @@ -25,7 +28,7 @@ devices = {} --- Initialize (or re-initialize) the `devices` table. function scan_devices () - for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do + for _,device in ipairs(lib.files_in_directory(get_pci_device_path())) do local info = device_info(device) if info.driver then table.insert(devices, info) end end @@ -46,8 +49,14 @@ function device_info (pciaddress) return info end +function get_pci_device_path() + return os.getenv("SNABB_PCI_DEVICE") or pci_default_device_path +end + --- Return the path to the sysfs directory for `pcidev`. -function path(pcidev) return "/sys/bus/pci/devices/"..pcidev end +function path(pcidev) + return get_pci_device_path() .. "/" .. pcidev +end -- Return the name of the Lua module that implements support for this device. function which_driver (vendor, device) diff --git a/src/lib/hardware/vfio.lua b/src/lib/hardware/vfio.lua index 7254efbc56..2d6179e592 100644 --- a/src/lib/hardware/vfio.lua +++ b/src/lib/hardware/vfio.lua @@ -11,6 +11,11 @@ require("lib.hardware.vfio_h") -- Is VFIO initialized yet? initialized = false +-- This path is used if the "SNABB_VFIO_DRIVER" environment variable is not defined +vfio_default_driver_path = "/sys/bus/pci/drivers/vfio-pci" +-- This path is used if the "SNABB_VFIO_IOMMU_GROUPS" environment variable is not defined +vfio_default_iommu_group_path = "/sys/kernel/iommu_groups" + -- Array of mappings that were requested before vfio was initialized. -- -- These must then be mapped at initialization time. @@ -69,17 +74,31 @@ function device_group(pciaddress) return lib.basename(lib.readlink(pci.path(pciaddress)..'/iommu_group')) end +function get_iommu_groups_path() + return os.getenv("SNABB_VFIO_IOMMU_GROUPS") or vfio_default_iommu_group_path +end + -- Return all the devices that belong to the same group function group_devices(group) - if not group then return {} end - return lib.files_in_directory('/sys/kernel/iommu_groups/'..group..'/devices/') + if not group then return {} end + return lib.files_in_directory( + get_iommu_groups_path() .. '/' .. group .. '/devices/' + ) end --- ### Device manipulation. +function get_driver_path () + return os.getenv("SNABB_VFIO_DRIVER") or vfio_default_driver_path +end + --- add a device to the vfio-pci driver function bind_device_to_vfio (pciaddress) - lib.writefile("/sys/bus/pci/drivers/vfio-pci/bind", pciaddress) + lib.writefile(get_driver_path() .. "/bind", pciaddress) +end + +function is_vfio_available() + return lib.can_write(get_driver_path() .. "/bind") end function setup_vfio(pciaddress, do_group)