From 7d4486596f80b0953a2707fcff2f6eeaba742c87 Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Mon, 20 Nov 2017 14:06:52 +0100 Subject: [PATCH 001/100] [wip] use socker server mode --- src/program/packetblaster/lwaftr/lwaftr.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/packetblaster/lwaftr/lwaftr.lua b/src/program/packetblaster/lwaftr/lwaftr.lua index c22e5d831d..5212bae601 100644 --- a/src/program/packetblaster/lwaftr/lwaftr.lua +++ b/src/program/packetblaster/lwaftr/lwaftr.lua @@ -220,7 +220,7 @@ function run (args) config.app(c, "int", raw.RawSocket, int_interface) input, output = "int.rx", "int.tx" elseif sock_interface then - config.app(c, "virtio", VhostUser, { socket_path=sock_interface } ) + config.app(c, "virtio", VhostUser, { socket_path=sock_interface,is_server=true } ) input, output = "virtio.rx", "virtio.tx" else config.app(c, "pcap", pcap.PcapWriter, pcap_file) From 7fce5941c488cc6f6ee7ff5f343fd6f9fda0f08f Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Mon, 20 Nov 2017 14:29:09 +0100 Subject: [PATCH 002/100] undo previous wip --- src/program/packetblaster/lwaftr/lwaftr.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/packetblaster/lwaftr/lwaftr.lua b/src/program/packetblaster/lwaftr/lwaftr.lua index 5212bae601..c22e5d831d 100644 --- a/src/program/packetblaster/lwaftr/lwaftr.lua +++ b/src/program/packetblaster/lwaftr/lwaftr.lua @@ -220,7 +220,7 @@ function run (args) config.app(c, "int", raw.RawSocket, int_interface) input, output = "int.rx", "int.tx" elseif sock_interface then - config.app(c, "virtio", VhostUser, { socket_path=sock_interface,is_server=true } ) + config.app(c, "virtio", VhostUser, { socket_path=sock_interface } ) input, output = "virtio.rx", "virtio.tx" else config.app(c, "pcap", pcap.PcapWriter, pcap_file) From 3e7d2b5e1864959186c393ea65f2e9f3a364a7d3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 13:53:13 +0100 Subject: [PATCH 003/100] Fix "snabb lwaftr run" after lib.scheduling refactor --- src/program/lwaftr/run/run.lua | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 5b53e78aa4..3f09c45505 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -119,13 +119,10 @@ function parse_args(args) if opts.mirror then assert(opts["on-a-stick"], "Mirror option is only valid in on-a-stick mode") end - if opts["on-a-stick"] then - scheduling.pci_addrs = { v4 } - return opts, scheduling, conf_file, v4 - else - scheduling.pci_addrs = { v4, v6 } - return opts, scheduling, conf_file, v4, v6 + if opts["on-a-stick"] and v6 then + fatal("Options --on-a-stick and --v6 are mutually exclusive.") end + return opts, scheduling, conf_file, v4, v6 end -- Requires a V4V6 splitter if running in on-a-stick mode and VLAN tag values From 47588b97928702c25c3d2b8f30d00f525f62b458 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 10:14:54 +0100 Subject: [PATCH 004/100] Split YANG getter-at-path out to own module We'll also start moving utilities from lib.ptree.ptree into lib.yang.path_data. --- src/lib/ptree/ptree.lua | 21 +- src/lib/ptree/support.lua | 5 +- src/lib/ptree/support/snabb-softwire-v2.lua | 7 +- src/lib/yang/path.lua | 205 +-------------- src/lib/yang/path_data.lua | 243 ++++++++++++++++++ src/lib/yang/state.lua | 1 - src/program/config/common.lua | 2 +- .../migrate_configuration.lua | 1 - 8 files changed, 263 insertions(+), 222 deletions(-) create mode 100644 src/lib/yang/path_data.lua diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index dfa6a4515b..16ed507aee 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -20,6 +20,7 @@ local schema = require("lib.yang.schema") local rpc = require("lib.yang.rpc") local state = require("lib.yang.state") local path_mod = require("lib.yang.path") +local path_data = require("lib.yang.path_data") local action_codec = require("lib.ptree.action_codec") local alarm_codec = require("lib.ptree.alarm_codec") local support = require("lib.ptree.support") @@ -259,7 +260,7 @@ function Manager:rpc_get_schema (args) end local function path_printer_for_grammar(grammar, path, format, print_default) - local getter, subgrammar = path_mod.resolver(grammar, path) + local getter, subgrammar = path_data.resolver(grammar, path) local printer if format == "xpath" then printer = data.xpath_printer_from_grammar(subgrammar, print_default, path) @@ -340,7 +341,7 @@ end local function path_parser_for_grammar(grammar, path) - local getter, subgrammar = path_mod.resolver(grammar, path) + local getter, subgrammar = path_data.resolver(grammar, path) return data.data_parser_from_grammar(subgrammar) end @@ -362,7 +363,7 @@ local function path_setter_for_grammar(grammar, path) local tail_name, query = tail_path[1].name, tail_path[1].query if lib.equal(query, {}) then -- No query; the simple case. - local getter, grammar = path_mod.resolver(grammar, head) + local getter, grammar = path_data.resolver(grammar, head) assert(grammar.type == 'struct') local tail_id = data.normalize_id(tail_name) return function(config, subconfig) @@ -373,9 +374,9 @@ local function path_setter_for_grammar(grammar, path) -- Otherwise the path ends in a query; it must denote an array or -- table item. - local getter, grammar = path_mod.resolver(grammar, head..'/'..tail_name) + local getter, grammar = path_data.resolver(grammar, head..'/'..tail_name) if grammar.type == 'array' then - local idx = path_mod.prepare_array_lookup(query) + local idx = path_data.prepare_array_lookup(query) return function(config, subconfig) local array = getter(config) assert(idx <= #array) @@ -383,7 +384,7 @@ local function path_setter_for_grammar(grammar, path) return config end elseif grammar.type == 'table' then - local key = path_mod.prepare_table_lookup(grammar.keys, + local key = path_data.prepare_table_lookup(grammar.keys, grammar.key_ctype, query) if grammar.string_key then key = key[data.normalize_id(grammar.string_key)] @@ -433,7 +434,7 @@ end local function path_adder_for_grammar(grammar, path) local top_grammar = grammar - local getter, grammar = path_mod.resolver(grammar, path) + local getter, grammar = path_data.resolver(grammar, path) if grammar.type == 'array' then if grammar.ctype then -- It's an FFI array; have to create a fresh one, sadly. @@ -519,11 +520,11 @@ local function path_remover_for_grammar(grammar, path) local tail_path = path_mod.parse_path(tail) local tail_name, query = tail_path[1].name, tail_path[1].query local head_and_tail_name = head..'/'..tail_name - local getter, grammar = path_mod.resolver(grammar, head_and_tail_name) + local getter, grammar = path_data.resolver(grammar, head_and_tail_name) if grammar.type == 'array' then if grammar.ctype then -- It's an FFI array; have to create a fresh one, sadly. - local idx = path_mod.prepare_array_lookup(query) + local idx = path_data.prepare_array_lookup(query) local setter = path_setter_for_grammar(top_grammar, head_and_tail_name) local elt_t = data.typeof(grammar.ctype) local array_t = ffi.typeof('$[?]', elt_t) @@ -546,7 +547,7 @@ local function path_remover_for_grammar(grammar, path) return config end elseif grammar.type == 'table' then - local key = path_mod.prepare_table_lookup(grammar.keys, + local key = path_data.prepare_table_lookup(grammar.keys, grammar.key_ctype, query) if grammar.string_key then key = key[data.normalize_id(grammar.string_key)] diff --git a/src/lib/ptree/support.lua b/src/lib/ptree/support.lua index a1f45ad523..7d6baaf2f7 100644 --- a/src/lib/ptree/support.lua +++ b/src/lib/ptree/support.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local app = require("core.app") local app_graph_mod = require("core.config") local path_mod = require("lib.yang.path") +local path_data = require("lib.yang.path_data") local yang = require("lib.yang.yang") local data = require("lib.yang.data") local cltable = require("lib.cltable") @@ -78,9 +79,9 @@ local function compute_objects_maybe_updated_in_place (schema_name, config, for _,path in ipairs(compute_parent_paths(changed_path)) do -- Calling the getter is avg O(N) in depth, so that makes the -- loop O(N^2), though it is generally bounded at a shallow - -- level so perhaps it's OK. path_mod.resolver is O(N) too but + -- level so perhaps it's OK. path_data.resolver is O(N) too but -- memoization makes it O(1). - getter, subgrammar = path_mod.resolver(grammar, path) + getter, subgrammar = path_data.resolver(grammar, path) -- Scalars can't be updated in place. if subgrammar.type == 'scalar' then return objs end table.insert(objs, getter(config)) diff --git a/src/lib/ptree/support/snabb-softwire-v2.lua b/src/lib/ptree/support/snabb-softwire-v2.lua index 7f83d788a4..999c2da85b 100644 --- a/src/lib/ptree/support/snabb-softwire-v2.lua +++ b/src/lib/ptree/support/snabb-softwire-v2.lua @@ -13,6 +13,7 @@ local yang = require('lib.yang.yang') local ctable = require('lib.ctable') local cltable = require('lib.cltable') local path_mod = require('lib.yang.path') +local path_data = require('lib.yang.path_data') local generic = require('lib.ptree.support').generic_schema_config_support local binding_table = require("apps.lwaftr.binding_table") @@ -90,7 +91,7 @@ local function remove_softwire_entry_actions(app_graph, path) assert(app_graph.apps['lwaftr']) path = path_mod.parse_path(path) local grammar = get_softwire_grammar() - local key = path_mod.prepare_table_lookup( + local key = path_data.prepare_table_lookup( grammar.keys, grammar.key_ctype, path[#path].query) local args = {'lwaftr', 'remove_softwire_entry', key} -- If it's the last softwire for the corresponding psid entry, remove it. @@ -216,7 +217,7 @@ end local function schema_getter(schema_name, path) local schema = yang.load_schema_by_name(schema_name) local grammar = data.config_grammar_from_schema(schema) - return path_mod.resolver(grammar, path) + return path_data.resolver(grammar, path) end local function snabb_softwire_getter(path) @@ -603,7 +604,7 @@ local function ietf_softwire_br_translator () local value = snabb_softwire_getter(path)(native_config) for _,instance in cltable.pairs(br_instance) do local grammar = get_ietf_softwire_grammar() - local key = path_mod.prepare_table_lookup( + local key = path_data.prepare_table_lookup( grammar.keys, grammar.key_ctype, {['binding-ipv6info']='::'}) key.binding_ipv6info = value.b4_ipv6 assert(instance.binding_table.binding_entry[key] ~= nil) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 46e9451e30..d2c7a03587 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -26,10 +26,7 @@ module(..., package.seeall) local equal = require("core.lib").equal -local schemalib = require("lib.yang.schema") local datalib = require("lib.yang.data") -local valuelib = require("lib.yang.value") -local util = require("lib.yang.util") local normalize_id = datalib.normalize_id local function table_keys(t) @@ -154,138 +151,9 @@ function normalize_path(path) return '/'..table.concat(ret, '/') end -function prepare_array_lookup(query) - if not equal(table_keys(query), {"position()"}) then - error("Arrays can only be indexed by position.") - end - local idx = tonumber(query["position()"]) - if idx < 1 or idx ~= math.floor(idx) then - error("Arrays can only be indexed by positive integers.") - end - return idx -end - -function prepare_table_lookup(keys, ctype, query) - local static_key = ctype and datalib.typeof(ctype)() or {} - for k,_ in pairs(query) do - if not keys[k] then error("'"..k.."' is not a table key") end - end - for k,grammar in pairs(keys) do - local v = query[k] or grammar.default - if v == nil then - error("Table query missing required key '"..k.."'") - end - local key_primitive_type = grammar.argument_type.primitive_type - local parser = valuelib.types[key_primitive_type].parse - static_key[normalize_id(k)] = parser(v, 'path query value') - end - return static_key -end - --- Returns a resolver for a particular schema and *lua* path. -function resolver(grammar, path_string) - local function ctable_getter(key, getter) - return function(data) - local data = getter(data):lookup_ptr(key) - if data == nil then error("Not found") end - return data.value - end - end - local function table_getter(key, getter) - return function(data) - local data = getter(data)[key] - if data == nil then error("Not found") end - return data - end - end - local function slow_table_getter(key, getter) - return function(data) - for k,v in pairs(getter(data)) do - if equal(k, key) then return v end - end - error("Not found") - end - end - local function compute_table_getter(grammar, key, getter) - if grammar.string_key then - return table_getter(key[normalize_id(grammar.string_key)], getter) - elseif grammar.key_ctype and grammar.value_ctype then - return ctable_getter(key, getter) - elseif grammar.key_ctype then - return table_getter(key, getter) - else - return slow_table_getter(key, getter) - end - end - local function handle_table_query(grammar, query, getter) - local key = prepare_table_lookup(grammar.keys, grammar.key_ctype, query) - local child_grammar = {type="struct", members=grammar.values, - ctype=grammar.value_ctype} - local child_getter = compute_table_getter(grammar, key, getter) - return child_getter, child_grammar - end - local function handle_array_query(grammar, query, getter) - local idx = prepare_array_lookup(query) - -- Pretend that array elements are scalars. - local child_grammar = {type="scalar", argument_type=grammar.element_type, - ctype=grammar.ctype} - local function child_getter(data) - local array = getter(data) - if idx > #array then error("Index out of bounds") end - return array[idx] - end - return child_getter, child_grammar - end - local function handle_query(grammar, query, getter) - if equal(table_keys(query), {}) then return getter, grammar end - if grammar.type == 'array' then - return handle_array_query(grammar, query, getter) - elseif grammar.type == 'table' then - return handle_table_query(grammar, query, getter) - else - error("Path query parameters only supported for structs and tables.") - end - end - local function compute_getter(grammar, name, query, getter) - local child_grammar - child_grammar = grammar.members[name] - if not child_grammar then - for member_name, member in pairs(grammar.members) do - if child_grammar then break end - if member.type == 'choice' then - for case_name, case in pairs(member.choices) do - if child_grammar then break end - if case[name] then child_grammar = case[name] end - end - end - end - end - if not child_grammar then - error("Struct has no field named '"..name.."'.") - end - local id = normalize_id(name) - local function child_getter(data) - local struct = getter(data) - local child = struct[id] - if child == nil then - error("Struct instance has no field named '"..name.."'.") - end - return child - end - return handle_query(child_grammar, query, child_getter) - end - local getter, grammar = function(data) return data end, grammar - for _, elt in ipairs(parse_path(path_string)) do - -- All non-leaves of the path tree must be structs. - if grammar.type ~= 'struct' then error("Invalid path.") end - getter, grammar = compute_getter(grammar, elt.name, elt.query, getter) - end - return getter, grammar -end -resolver = util.memoize(resolver) - function selftest() print("selftest: lib.yang.path") + local schemalib = require("lib.yang.schema") local schema_src = [[module snabb-simple-router { namespace snabb:simple-router; prefix simple-router; @@ -319,77 +187,6 @@ function selftest() assert(path[1].name == "blocked-ips") assert(path[1].key == 4) - -- Test resolving a key to a path. - local data_src = [[ - active true; - - blocked-ips 8.8.8.8; - blocked-ips 8.8.4.4; - - routes { - route { addr 1.2.3.4; port 2; } - route { addr 2.3.4.5; port 2; } - route { addr 255.255.255.255; port 7; } - } - ]] - - local data = datalib.load_config_for_schema(scm, data_src) - - -- Try resolving a path in a list (ctable). - local getter = resolver(grammar, "/routes/route[addr=1.2.3.4]/port") - assert(getter(data) == 2) - - local getter = resolver(grammar, "/routes/route[addr=255.255.255.255]/port") - assert(getter(data) == 7) - - -- Try resolving a leaf-list - local getter = resolver(grammar, "/blocked-ips[position()=1]") - assert(getter(data) == util.ipv4_pton("8.8.8.8")) - - -- Try resolving a path for a list (non-ctable) - local fruit_schema_src = [[module fruit-bowl { - namespace snabb:fruit-bowl; - prefix simple-router; - - import ietf-inet-types {prefix inet;} - - container bowl { - list fruit { - key name; - leaf name { type string; mandatory true; } - leaf rating { type uint8 { range 0..10; } mandatory true; } - choice C { - case A { leaf AA { type string; } } - case B { leaf BB { type string; } } - } - } - }}]] - local fruit_data_src = [[ - bowl { - fruit { name "banana"; rating 10; } - fruit { name "pear"; rating 2; } - fruit { name "apple"; rating 6; } - fruit { name "kumquat"; rating 6; AA aa; } - fruit { name "tangerine"; rating 6; BB bb; } - } - ]] - - local fruit_scm = schemalib.load_schema(fruit_schema_src, "xpath-fruit-test") - local fruit_prod = datalib.config_grammar_from_schema(fruit_scm) - local fruit_data = datalib.load_config_for_schema(fruit_scm, fruit_data_src) - - local getter = resolver(fruit_prod, "/bowl/fruit[name=banana]/rating") - assert(getter(fruit_data) == 10) - - local getter = resolver(fruit_prod, "/bowl/fruit[name=apple]/rating") - assert(getter(fruit_data) == 6) - - local getter = resolver(fruit_prod, "/bowl/fruit[name=kumquat]/AA") - assert(getter(fruit_data) == 'aa') - - local getter = resolver(fruit_prod, "/bowl/fruit[name=tangerine]/BB") - assert(getter(fruit_data) == 'bb') - assert(normalize_path('') == '/') assert(normalize_path('//') == '/') assert(normalize_path('/') == '/') diff --git a/src/lib/yang/path_data.lua b/src/lib/yang/path_data.lua new file mode 100644 index 0000000000..b16df5ee0e --- /dev/null +++ b/src/lib/yang/path_data.lua @@ -0,0 +1,243 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(..., package.seeall) + +local equal = require("core.lib").equal +local datalib = require("lib.yang.data") +local valuelib = require("lib.yang.value") +local pathlib = require("lib.yang.path") +local util = require("lib.yang.util") +local normalize_id = datalib.normalize_id + +local function table_keys(t) + local ret = {} + for k, v in pairs(t) do table.insert(ret, k) end + return ret +end + +function prepare_array_lookup(query) + if not equal(table_keys(query), {"position()"}) then + error("Arrays can only be indexed by position.") + end + local idx = tonumber(query["position()"]) + if idx < 1 or idx ~= math.floor(idx) then + error("Arrays can only be indexed by positive integers.") + end + return idx +end + +function prepare_table_lookup(keys, ctype, query) + local static_key = ctype and datalib.typeof(ctype)() or {} + for k,_ in pairs(query) do + if not keys[k] then error("'"..k.."' is not a table key") end + end + for k,grammar in pairs(keys) do + local v = query[k] or grammar.default + if v == nil then + error("Table query missing required key '"..k.."'") + end + local key_primitive_type = grammar.argument_type.primitive_type + local parser = valuelib.types[key_primitive_type].parse + static_key[normalize_id(k)] = parser(v, 'path query value') + end + return static_key +end + +-- Returns a resolver for a particular schema and *lua* path. +function resolver(grammar, path_string) + local function ctable_getter(key, getter) + return function(data) + local data = getter(data):lookup_ptr(key) + if data == nil then error("Not found") end + return data.value + end + end + local function table_getter(key, getter) + return function(data) + local data = getter(data)[key] + if data == nil then error("Not found") end + return data + end + end + local function slow_table_getter(key, getter) + return function(data) + for k,v in pairs(getter(data)) do + if equal(k, key) then return v end + end + error("Not found") + end + end + local function compute_table_getter(grammar, key, getter) + if grammar.string_key then + return table_getter(key[normalize_id(grammar.string_key)], getter) + elseif grammar.key_ctype and grammar.value_ctype then + return ctable_getter(key, getter) + elseif grammar.key_ctype then + return table_getter(key, getter) + else + return slow_table_getter(key, getter) + end + end + local function handle_table_query(grammar, query, getter) + local key = prepare_table_lookup(grammar.keys, grammar.key_ctype, query) + local child_grammar = {type="struct", members=grammar.values, + ctype=grammar.value_ctype} + local child_getter = compute_table_getter(grammar, key, getter) + return child_getter, child_grammar + end + local function handle_array_query(grammar, query, getter) + local idx = prepare_array_lookup(query) + -- Pretend that array elements are scalars. + local child_grammar = {type="scalar", argument_type=grammar.element_type, + ctype=grammar.ctype} + local function child_getter(data) + local array = getter(data) + if idx > #array then error("Index out of bounds") end + return array[idx] + end + return child_getter, child_grammar + end + local function handle_query(grammar, query, getter) + if equal(table_keys(query), {}) then return getter, grammar end + if grammar.type == 'array' then + return handle_array_query(grammar, query, getter) + elseif grammar.type == 'table' then + return handle_table_query(grammar, query, getter) + else + error("Path query parameters only supported for structs and tables.") + end + end + local function compute_getter(grammar, name, query, getter) + local child_grammar + child_grammar = grammar.members[name] + if not child_grammar then + for member_name, member in pairs(grammar.members) do + if child_grammar then break end + if member.type == 'choice' then + for case_name, case in pairs(member.choices) do + if child_grammar then break end + if case[name] then child_grammar = case[name] end + end + end + end + end + if not child_grammar then + error("Struct has no field named '"..name.."'.") + end + local id = normalize_id(name) + local function child_getter(data) + local struct = getter(data) + local child = struct[id] + if child == nil then + error("Struct instance has no field named '"..name.."'.") + end + return child + end + return handle_query(child_grammar, query, child_getter) + end + local getter, grammar = function(data) return data end, grammar + for _, elt in ipairs(pathlib.parse_path(path_string)) do + -- All non-leaves of the path tree must be structs. + if grammar.type ~= 'struct' then error("Invalid path.") end + getter, grammar = compute_getter(grammar, elt.name, elt.query, getter) + end + return getter, grammar +end +resolver = util.memoize(resolver) + +function selftest() + print("selftest: lib.yang.path_data") + local schemalib = require("lib.yang.schema") + local schema_src = [[module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + leaf-list blocked-ips { type inet:ipv4-address; } + + container routes { + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; } mandatory true; } + } + }}]] + + local scm = schemalib.load_schema(schema_src, "xpath-test") + local grammar = datalib.config_grammar_from_schema(scm) + + -- Test resolving a key to a path. + local data_src = [[ + active true; + + blocked-ips 8.8.8.8; + blocked-ips 8.8.4.4; + + routes { + route { addr 1.2.3.4; port 2; } + route { addr 2.3.4.5; port 2; } + route { addr 255.255.255.255; port 7; } + } + ]] + + local data = datalib.load_config_for_schema(scm, data_src) + + -- Try resolving a path in a list (ctable). + local getter = resolver(grammar, "/routes/route[addr=1.2.3.4]/port") + assert(getter(data) == 2) + + local getter = resolver(grammar, "/routes/route[addr=255.255.255.255]/port") + assert(getter(data) == 7) + + -- Try resolving a leaf-list + local getter = resolver(grammar, "/blocked-ips[position()=1]") + assert(getter(data) == util.ipv4_pton("8.8.8.8")) + + -- Try resolving a path for a list (non-ctable) + local fruit_schema_src = [[module fruit-bowl { + namespace snabb:fruit-bowl; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + container bowl { + list fruit { + key name; + leaf name { type string; mandatory true; } + leaf rating { type uint8 { range 0..10; } mandatory true; } + choice C { + case A { leaf AA { type string; } } + case B { leaf BB { type string; } } + } + } + }}]] + local fruit_data_src = [[ + bowl { + fruit { name "banana"; rating 10; } + fruit { name "pear"; rating 2; } + fruit { name "apple"; rating 6; } + fruit { name "kumquat"; rating 6; AA aa; } + fruit { name "tangerine"; rating 6; BB bb; } + } + ]] + + local fruit_scm = schemalib.load_schema(fruit_schema_src, "xpath-fruit-test") + local fruit_prod = datalib.config_grammar_from_schema(fruit_scm) + local fruit_data = datalib.load_config_for_schema(fruit_scm, fruit_data_src) + + local getter = resolver(fruit_prod, "/bowl/fruit[name=banana]/rating") + assert(getter(fruit_data) == 10) + + local getter = resolver(fruit_prod, "/bowl/fruit[name=apple]/rating") + assert(getter(fruit_data) == 6) + + local getter = resolver(fruit_prod, "/bowl/fruit[name=kumquat]/AA") + assert(getter(fruit_data) == 'aa') + + local getter = resolver(fruit_prod, "/bowl/fruit[name=tangerine]/BB") + assert(getter(fruit_data) == 'bb') + + print("selftest: ok") +end diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua index 24e2f389ad..45096a5336 100644 --- a/src/lib/yang/state.lua +++ b/src/lib/yang/state.lua @@ -3,7 +3,6 @@ module(..., package.seeall) local lib = require("core.lib") local shm = require("core.shm") -local xpath = require("lib.yang.path") local yang = require("lib.yang.yang") local data = require("lib.yang.data") local util = require("lib.yang.util") diff --git a/src/program/config/common.lua b/src/program/config/common.lua index bd4838b4d7..2faed6ba98 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -8,7 +8,7 @@ local shm = require("core.shm") local rpc = require("lib.yang.rpc") local yang = require("lib.yang.yang") local data = require("lib.yang.data") -local path_resolver = require("lib.yang.path").resolver +local path_resolver = require("lib.yang.path_data").resolver function show_usage(command, status, err_msg) if err_msg then print('error: '..err_msg) end diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 143c7ec0f1..e16005bdd2 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -12,7 +12,6 @@ local stream = require('lib.yang.stream') local binding_table = require("apps.lwaftr.binding_table") local Parser = require("program.lwaftr.migrate_configuration.conf_parser").Parser local data = require('lib.yang.data') -local path = require('lib.yang.path') local br_address_t = ffi.typeof('uint8_t[16]') local SOFTWIRE_TABLE_LOAD_FACTOR = 0.4 From 667b8432fa84b1c5bf061dfe84ad7b301d645632 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 10:57:43 +0100 Subject: [PATCH 005/100] Move generic data-at-path utilities to lib.yang.path_data --- src/lib/ptree/ptree.lua | 309 ++--------------------------------- src/lib/yang/path_data.lua | 325 ++++++++++++++++++++++++++++++++++--- 2 files changed, 318 insertions(+), 316 deletions(-) diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index 16ed507aee..d4c3c71df7 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -14,7 +14,6 @@ local cltable = require("lib.cltable") local cpuset = require("lib.cpuset") local scheduling = require("lib.scheduling") local yang = require("lib.yang.yang") -local data = require("lib.yang.data") local util = require("lib.yang.util") local schema = require("lib.yang.schema") local rpc = require("lib.yang.rpc") @@ -259,39 +258,13 @@ function Manager:rpc_get_schema (args) if success then return response else return {status=1, error=response} end end -local function path_printer_for_grammar(grammar, path, format, print_default) - local getter, subgrammar = path_data.resolver(grammar, path) - local printer - if format == "xpath" then - printer = data.xpath_printer_from_grammar(subgrammar, print_default, path) - else - printer = data.data_printer_from_grammar(subgrammar, print_default) - end - return function(data, file) - return printer(getter(data), file) - end -end - -local function path_printer_for_schema(schema, path, is_config, - format, print_default) - local grammar = data.data_grammar_from_schema(schema, is_config) - return path_printer_for_grammar(grammar, path, format, print_default) -end - -local function path_printer_for_schema_by_name(schema_name, path, is_config, - format, print_default) - local schema = yang.load_schema_by_name(schema_name) - return path_printer_for_schema(schema, path, is_config, format, - print_default) -end - function Manager:rpc_get_config (args) local function getter() if args.schema ~= self.schema_name then return self:foreign_rpc_get_config( args.schema, args.path, args.format, args.print_default) end - local printer = path_printer_for_schema_by_name( + local printer = path_data.printer_for_schema_by_name( args.schema, args.path, true, args.format, args.print_default) local config = printer(self.current_configuration, yang.string_output_file()) return { config = config } @@ -339,262 +312,6 @@ function Manager:rpc_compress_alarms (args) if success then return response else return {status=1, error=response} end end - -local function path_parser_for_grammar(grammar, path) - local getter, subgrammar = path_data.resolver(grammar, path) - return data.data_parser_from_grammar(subgrammar) -end - -local function path_parser_for_schema(schema, path) - local grammar = data.config_grammar_from_schema(schema) - return path_parser_for_grammar(grammar, path) -end - -local function path_parser_for_schema_by_name(schema_name, path) - return path_parser_for_schema(yang.load_schema_by_name(schema_name), path) -end - -local function path_setter_for_grammar(grammar, path) - if path == "/" then - return function(config, subconfig) return subconfig end - end - local head, tail = lib.dirname(path), lib.basename(path) - local tail_path = path_mod.parse_path(tail) - local tail_name, query = tail_path[1].name, tail_path[1].query - if lib.equal(query, {}) then - -- No query; the simple case. - local getter, grammar = path_data.resolver(grammar, head) - assert(grammar.type == 'struct') - local tail_id = data.normalize_id(tail_name) - return function(config, subconfig) - getter(config)[tail_id] = subconfig - return config - end - end - - -- Otherwise the path ends in a query; it must denote an array or - -- table item. - local getter, grammar = path_data.resolver(grammar, head..'/'..tail_name) - if grammar.type == 'array' then - local idx = path_data.prepare_array_lookup(query) - return function(config, subconfig) - local array = getter(config) - assert(idx <= #array) - array[idx] = subconfig - return config - end - elseif grammar.type == 'table' then - local key = path_data.prepare_table_lookup(grammar.keys, - grammar.key_ctype, query) - if grammar.string_key then - key = key[data.normalize_id(grammar.string_key)] - return function(config, subconfig) - local tab = getter(config) - assert(tab[key] ~= nil) - tab[key] = subconfig - return config - end - elseif grammar.key_ctype and grammar.value_ctype then - return function(config, subconfig) - getter(config):update(key, subconfig) - return config - end - elseif grammar.key_ctype then - return function(config, subconfig) - local tab = getter(config) - assert(tab[key] ~= nil) - tab[key] = subconfig - return config - end - else - return function(config, subconfig) - local tab = getter(config) - for k,v in pairs(tab) do - if lib.equal(k, key) then - tab[k] = subconfig - return config - end - end - error("Not found") - end - end - else - error('Query parameters only allowed on arrays and tables') - end -end - -local function path_setter_for_schema(schema, path) - local grammar = data.config_grammar_from_schema(schema) - return path_setter_for_grammar(grammar, path) -end - -function compute_set_config_fn (schema_name, path) - return path_setter_for_schema(yang.load_schema_by_name(schema_name), path) -end - -local function path_adder_for_grammar(grammar, path) - local top_grammar = grammar - local getter, grammar = path_data.resolver(grammar, path) - if grammar.type == 'array' then - if grammar.ctype then - -- It's an FFI array; have to create a fresh one, sadly. - local setter = path_setter_for_grammar(top_grammar, path) - local elt_t = data.typeof(grammar.ctype) - local array_t = ffi.typeof('$[?]', elt_t) - return function(config, subconfig) - local cur = getter(config) - local new = array_t(#cur + #subconfig) - local i = 1 - for _,elt in ipairs(cur) do new[i-1] = elt; i = i + 1 end - for _,elt in ipairs(subconfig) do new[i-1] = elt; i = i + 1 end - return setter(config, util.ffi_array(new, elt_t)) - end - end - -- Otherwise we can add entries in place. - return function(config, subconfig) - local cur = getter(config) - for _,elt in ipairs(subconfig) do table.insert(cur, elt) end - return config - end - elseif grammar.type == 'table' then - -- Invariant: either all entries in the new subconfig are added, - -- or none are. - if grammar.key_ctype and grammar.value_ctype then - -- ctable. - return function(config, subconfig) - local ctab = getter(config) - for entry in subconfig:iterate() do - if ctab:lookup_ptr(entry.key) ~= nil then - error('already-existing entry') - end - end - for entry in subconfig:iterate() do - ctab:add(entry.key, entry.value) - end - return config - end - elseif grammar.string_key or grammar.key_ctype then - -- cltable or string-keyed table. - local pairs = grammar.key_ctype and cltable.pairs or pairs - return function(config, subconfig) - local tab = getter(config) - for k,_ in pairs(subconfig) do - if tab[k] ~= nil then error('already-existing entry') end - end - for k,v in pairs(subconfig) do tab[k] = v end - return config - end - else - -- Sad quadratic loop. - return function(config, subconfig) - local tab = getter(config) - for key,val in pairs(tab) do - for k,_ in pairs(subconfig) do - if lib.equal(key, k) then - error('already-existing entry', key) - end - end - end - for k,v in pairs(subconfig) do tab[k] = v end - return config - end - end - else - error('Add only allowed on arrays and tables') - end -end - -local function path_adder_for_schema(schema, path) - local grammar = data.config_grammar_from_schema(schema) - return path_adder_for_grammar(grammar, path) -end - -function compute_add_config_fn (schema_name, path) - return path_adder_for_schema(yang.load_schema_by_name(schema_name), path) -end -compute_add_config_fn = util.memoize(compute_add_config_fn) - -local function path_remover_for_grammar(grammar, path) - local top_grammar = grammar - local head, tail = lib.dirname(path), lib.basename(path) - local tail_path = path_mod.parse_path(tail) - local tail_name, query = tail_path[1].name, tail_path[1].query - local head_and_tail_name = head..'/'..tail_name - local getter, grammar = path_data.resolver(grammar, head_and_tail_name) - if grammar.type == 'array' then - if grammar.ctype then - -- It's an FFI array; have to create a fresh one, sadly. - local idx = path_data.prepare_array_lookup(query) - local setter = path_setter_for_grammar(top_grammar, head_and_tail_name) - local elt_t = data.typeof(grammar.ctype) - local array_t = ffi.typeof('$[?]', elt_t) - return function(config) - local cur = getter(config) - assert(idx <= #cur) - local new = array_t(#cur - 1) - for i,elt in ipairs(cur) do - if i < idx then new[i-1] = elt end - if i > idx then new[i-2] = elt end - end - return setter(config, util.ffi_array(new, elt_t)) - end - end - -- Otherwise we can remove the entry in place. - return function(config) - local cur = getter(config) - assert(i <= #cur) - table.remove(cur, i) - return config - end - elseif grammar.type == 'table' then - local key = path_data.prepare_table_lookup(grammar.keys, - grammar.key_ctype, query) - if grammar.string_key then - key = key[data.normalize_id(grammar.string_key)] - return function(config) - local tab = getter(config) - assert(tab[key] ~= nil) - tab[key] = nil - return config - end - elseif grammar.key_ctype and grammar.value_ctype then - return function(config) - getter(config):remove(key) - return config - end - elseif grammar.key_ctype then - return function(config) - local tab = getter(config) - assert(tab[key] ~= nil) - tab[key] = nil - return config - end - else - return function(config) - local tab = getter(config) - for k,v in pairs(tab) do - if lib.equal(k, key) then - tab[k] = nil - return config - end - end - error("Not found") - end - end - else - error('Remove only allowed on arrays and tables') - end -end - -local function path_remover_for_schema(schema, path) - local grammar = data.config_grammar_from_schema(schema) - return path_remover_for_grammar(grammar, path) -end - -function compute_remove_config_fn (schema_name, path) - return path_remover_for_schema(yang.load_schema_by_name(schema_name), path) -end - function Manager:notify_pre_update (config, verb, path, ...) for _,translator in pairs(self.support.translators) do translator.pre_update(config, verb, path, ...) @@ -633,7 +350,7 @@ end function Manager:handle_rpc_update_config (args, verb, compute_update_fn) local path = path_mod.normalize_path(args.path) - local parser = path_parser_for_schema_by_name(args.schema, path) + local parser = path_data.parser_for_schema_by_name(args.schema, path) self:update_configuration(compute_update_fn(args.schema, path), verb, path, parser(args.config)) return {} @@ -668,7 +385,7 @@ function Manager:foreign_rpc_get_config (schema_name, path, format, path = path_mod.normalize_path(path) local translate = self:get_translator(schema_name) local foreign_config = translate.get_config(self.current_configuration) - local printer = path_printer_for_schema_by_name( + local printer = path_data.printer_for_schema_by_name( schema_name, path, true, format, print_default) local config = printer(foreign_config, yang.string_output_file()) return { config = config } @@ -678,7 +395,7 @@ function Manager:foreign_rpc_get_state (schema_name, path, format, path = path_mod.normalize_path(path) local translate = self:get_translator(schema_name) local foreign_state = translate.get_state(self:get_native_state()) - local printer = path_printer_for_schema_by_name( + local printer = path_data.printer_for_schema_by_name( schema_name, path, false, format, print_default) local state = printer(foreign_state, yang.string_output_file()) return { state = state } @@ -686,7 +403,7 @@ end function Manager:foreign_rpc_set_config (schema_name, path, config_str) path = path_mod.normalize_path(path) local translate = self:get_translator(schema_name) - local parser = path_parser_for_schema_by_name(schema_name, path) + local parser = path_data.parser_for_schema_by_name(schema_name, path) local updates = translate.set_config(self.current_configuration, path, parser(config_str)) return self:apply_translated_rpc_updates(updates) @@ -694,7 +411,7 @@ end function Manager:foreign_rpc_add_config (schema_name, path, config_str) path = path_mod.normalize_path(path) local translate = self:get_translator(schema_name) - local parser = path_parser_for_schema_by_name(schema_name, path) + local parser = path_data.parser_for_schema_by_name(schema_name, path) local updates = translate.add_config(self.current_configuration, path, parser(config_str)) return self:apply_translated_rpc_updates(updates) @@ -714,7 +431,8 @@ function Manager:rpc_set_config (args) if args.schema ~= self.schema_name then return self:foreign_rpc_set_config(args.schema, args.path, args.config) end - return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) + return self:handle_rpc_update_config( + args, 'set', path_data.setter_for_schema_by_name) end local success, response = pcall(setter) if success then return response else return {status=1, error=response} end @@ -728,7 +446,8 @@ function Manager:rpc_add_config (args) if args.schema ~= self.schema_name then return self:foreign_rpc_add_config(args.schema, args.path, args.config) end - return self:handle_rpc_update_config(args, 'add', compute_add_config_fn) + return self:handle_rpc_update_config( + args, 'add', path_data.adder_for_schema_by_name) end local success, response = pcall(adder) if success then return response else return {status=1, error=response} end @@ -743,8 +462,8 @@ function Manager:rpc_remove_config (args) return self:foreign_rpc_remove_config(args.schema, args.path) end local path = path_mod.normalize_path(args.path) - self:update_configuration(compute_remove_config_fn(args.schema, path), - 'remove', path) + self:update_configuration( + path_data.remover_for_schema_by_name(args.schema, path), 'remove', path) return {} end local success, response = pcall(remover) @@ -768,7 +487,7 @@ function Manager:rpc_get_state (args) args.format, args.print_default) end local state = self:get_native_state() - local printer = path_printer_for_schema_by_name( + local printer = path_data.printer_for_schema_by_name( self.schema_name, args.path, false, args.format, args.print_default) return { state = printer(state, yang.string_output_file()) } end @@ -779,7 +498,7 @@ end function Manager:rpc_get_alarms_state (args) local function getter() assert(args.schema == "ietf-alarms") - local printer = path_printer_for_schema_by_name( + local printer = path_data.printer_for_schema_by_name( args.schema, args.path, false, args.format, args.print_default) local state = { alarms = alarms.get_state() diff --git a/src/lib/yang/path_data.lua b/src/lib/yang/path_data.lua index b16df5ee0e..28753477fd 100644 --- a/src/lib/yang/path_data.lua +++ b/src/lib/yang/path_data.lua @@ -2,12 +2,14 @@ module(..., package.seeall) -local equal = require("core.lib").equal -local datalib = require("lib.yang.data") -local valuelib = require("lib.yang.value") -local pathlib = require("lib.yang.path") +local ffi = require("ffi") +local lib = require("core.lib") +local data = require("lib.yang.data") +local value = require("lib.yang.value") +local schema = require("lib.yang.schema") +local parse_path = require("lib.yang.path").parse_path local util = require("lib.yang.util") -local normalize_id = datalib.normalize_id +local normalize_id = data.normalize_id local function table_keys(t) local ret = {} @@ -16,7 +18,7 @@ local function table_keys(t) end function prepare_array_lookup(query) - if not equal(table_keys(query), {"position()"}) then + if not lib.equal(table_keys(query), {"position()"}) then error("Arrays can only be indexed by position.") end local idx = tonumber(query["position()"]) @@ -27,7 +29,7 @@ function prepare_array_lookup(query) end function prepare_table_lookup(keys, ctype, query) - local static_key = ctype and datalib.typeof(ctype)() or {} + local static_key = ctype and data.typeof(ctype)() or {} for k,_ in pairs(query) do if not keys[k] then error("'"..k.."' is not a table key") end end @@ -37,7 +39,7 @@ function prepare_table_lookup(keys, ctype, query) error("Table query missing required key '"..k.."'") end local key_primitive_type = grammar.argument_type.primitive_type - local parser = valuelib.types[key_primitive_type].parse + local parser = value.types[key_primitive_type].parse static_key[normalize_id(k)] = parser(v, 'path query value') end return static_key @@ -62,7 +64,7 @@ function resolver(grammar, path_string) local function slow_table_getter(key, getter) return function(data) for k,v in pairs(getter(data)) do - if equal(k, key) then return v end + if lib.equal(k, key) then return v end end error("Not found") end @@ -98,7 +100,7 @@ function resolver(grammar, path_string) return child_getter, child_grammar end local function handle_query(grammar, query, getter) - if equal(table_keys(query), {}) then return getter, grammar end + if lib.equal(table_keys(query), {}) then return getter, grammar end if grammar.type == 'array' then return handle_array_query(grammar, query, getter) elseif grammar.type == 'table' then @@ -136,7 +138,7 @@ function resolver(grammar, path_string) return handle_query(child_grammar, query, child_getter) end local getter, grammar = function(data) return data end, grammar - for _, elt in ipairs(pathlib.parse_path(path_string)) do + for _, elt in ipairs(parse_path(path_string)) do -- All non-leaves of the path tree must be structs. if grammar.type ~= 'struct' then error("Invalid path.") end getter, grammar = compute_getter(grammar, elt.name, elt.query, getter) @@ -145,9 +147,290 @@ function resolver(grammar, path_string) end resolver = util.memoize(resolver) +local function printer_for_grammar(grammar, path, format, print_default) + local getter, subgrammar = resolver(grammar, path) + local printer + if format == "xpath" then + printer = data.xpath_printer_from_grammar(subgrammar, print_default, path) + else + printer = data.data_printer_from_grammar(subgrammar, print_default) + end + return function(data, file) + return printer(getter(data), file) + end +end + +local function printer_for_schema(schema, path, is_config, format, + print_default) + local grammar = data.data_grammar_from_schema(schema, is_config) + return printer_for_grammar(grammar, path, format, print_default) +end + +function printer_for_schema_by_name(schema_name, path, is_config, format, + print_default) + local schema = schema.load_schema_by_name(schema_name) + return printer_for_schema(schema, path, is_config, format, print_default) +end +printer_for_schema_by_name = util.memoize(printer_for_schema_by_name) + +local function parser_for_grammar(grammar, path) + local getter, subgrammar = resolver(grammar, path) + return data.data_parser_from_grammar(subgrammar) +end + +local function parser_for_schema(schema, path) + local grammar = data.config_grammar_from_schema(schema) + return parser_for_grammar(grammar, path) +end + +function parser_for_schema_by_name(schema_name, path) + return parser_for_schema(schema.load_schema_by_name(schema_name), path) +end +parser_for_schema_by_name = util.memoize(parser_for_schema_by_name) + +local function setter_for_grammar(grammar, path) + if path == "/" then + return function(config, subconfig) return subconfig end + end + local head, tail = lib.dirname(path), lib.basename(path) + local tail_path = parse_path(tail) + local tail_name, query = tail_path[1].name, tail_path[1].query + if lib.equal(query, {}) then + -- No query; the simple case. + local getter, grammar = resolver(grammar, head) + assert(grammar.type == 'struct') + local tail_id = data.normalize_id(tail_name) + return function(config, subconfig) + getter(config)[tail_id] = subconfig + return config + end + end + + -- Otherwise the path ends in a query; it must denote an array or + -- table item. + local getter, grammar = resolver(grammar, head..'/'..tail_name) + if grammar.type == 'array' then + local idx = prepare_array_lookup(query) + return function(config, subconfig) + local array = getter(config) + assert(idx <= #array) + array[idx] = subconfig + return config + end + elseif grammar.type == 'table' then + local key = prepare_table_lookup(grammar.keys, grammar.key_ctype, query) + if grammar.string_key then + key = key[data.normalize_id(grammar.string_key)] + return function(config, subconfig) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = subconfig + return config + end + elseif grammar.key_ctype and grammar.value_ctype then + return function(config, subconfig) + getter(config):update(key, subconfig) + return config + end + elseif grammar.key_ctype then + return function(config, subconfig) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = subconfig + return config + end + else + return function(config, subconfig) + local tab = getter(config) + for k,v in pairs(tab) do + if lib.equal(k, key) then + tab[k] = subconfig + return config + end + end + error("Not found") + end + end + else + error('Query parameters only allowed on arrays and tables') + end +end + +local function setter_for_schema(schema, path) + local grammar = data.config_grammar_from_schema(schema) + return setter_for_grammar(grammar, path) +end + +function setter_for_schema_by_name(schema_name, path) + return setter_for_schema(schema.load_schema_by_name(schema_name), path) +end +setter_for_schema_by_name = util.memoize(setter_for_schema_by_name) + +local function adder_for_grammar(grammar, path) + local top_grammar = grammar + local getter, grammar = resolver(grammar, path) + if grammar.type == 'array' then + if grammar.ctype then + -- It's an FFI array; have to create a fresh one, sadly. + local setter = setter_for_grammar(top_grammar, path) + local elt_t = data.typeof(grammar.ctype) + local array_t = ffi.typeof('$[?]', elt_t) + return function(config, subconfig) + local cur = getter(config) + local new = array_t(#cur + #subconfig) + local i = 1 + for _,elt in ipairs(cur) do new[i-1] = elt; i = i + 1 end + for _,elt in ipairs(subconfig) do new[i-1] = elt; i = i + 1 end + return setter(config, util.ffi_array(new, elt_t)) + end + end + -- Otherwise we can add entries in place. + return function(config, subconfig) + local cur = getter(config) + for _,elt in ipairs(subconfig) do table.insert(cur, elt) end + return config + end + elseif grammar.type == 'table' then + -- Invariant: either all entries in the new subconfig are added, + -- or none are. + if grammar.key_ctype and grammar.value_ctype then + -- ctable. + return function(config, subconfig) + local ctab = getter(config) + for entry in subconfig:iterate() do + if ctab:lookup_ptr(entry.key) ~= nil then + error('already-existing entry') + end + end + for entry in subconfig:iterate() do + ctab:add(entry.key, entry.value) + end + return config + end + elseif grammar.string_key or grammar.key_ctype then + -- cltable or string-keyed table. + local pairs = grammar.key_ctype and cltable.pairs or pairs + return function(config, subconfig) + local tab = getter(config) + for k,_ in pairs(subconfig) do + if tab[k] ~= nil then error('already-existing entry') end + end + for k,v in pairs(subconfig) do tab[k] = v end + return config + end + else + -- Sad quadratic loop. + return function(config, subconfig) + local tab = getter(config) + for key,val in pairs(tab) do + for k,_ in pairs(subconfig) do + if lib.equal(key, k) then + error('already-existing entry', key) + end + end + end + for k,v in pairs(subconfig) do tab[k] = v end + return config + end + end + else + error('Add only allowed on arrays and tables') + end +end + +local function adder_for_schema(schema, path) + local grammar = data.config_grammar_from_schema(schema) + return adder_for_grammar(grammar, path) +end + +function adder_for_schema_by_name (schema_name, path) + return adder_for_schema(schema.load_schema_by_name(schema_name), path) +end +adder_for_schema_by_name = util.memoize(adder_for_schema_by_name) + +local function remover_for_grammar(grammar, path) + local top_grammar = grammar + local head, tail = lib.dirname(path), lib.basename(path) + local tail_path = parse_path(tail) + local tail_name, query = tail_path[1].name, tail_path[1].query + local head_and_tail_name = head..'/'..tail_name + local getter, grammar = resolver(grammar, head_and_tail_name) + if grammar.type == 'array' then + if grammar.ctype then + -- It's an FFI array; have to create a fresh one, sadly. + local idx = prepare_array_lookup(query) + local setter = setter_for_grammar(top_grammar, head_and_tail_name) + local elt_t = data.typeof(grammar.ctype) + local array_t = ffi.typeof('$[?]', elt_t) + return function(config) + local cur = getter(config) + assert(idx <= #cur) + local new = array_t(#cur - 1) + for i,elt in ipairs(cur) do + if i < idx then new[i-1] = elt end + if i > idx then new[i-2] = elt end + end + return setter(config, util.ffi_array(new, elt_t)) + end + end + -- Otherwise we can remove the entry in place. + return function(config) + local cur = getter(config) + assert(i <= #cur) + table.remove(cur, i) + return config + end + elseif grammar.type == 'table' then + local key = prepare_table_lookup(grammar.keys, grammar.key_ctype, query) + if grammar.string_key then + key = key[data.normalize_id(grammar.string_key)] + return function(config) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = nil + return config + end + elseif grammar.key_ctype and grammar.value_ctype then + return function(config) + getter(config):remove(key) + return config + end + elseif grammar.key_ctype then + return function(config) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = nil + return config + end + else + return function(config) + local tab = getter(config) + for k,v in pairs(tab) do + if lib.equal(k, key) then + tab[k] = nil + return config + end + end + error("Not found") + end + end + else + error('Remove only allowed on arrays and tables') + end +end + +local function remover_for_schema(schema, path) + local grammar = data.config_grammar_from_schema(schema) + return remover_for_grammar(grammar, path) +end + +function remover_for_schema_by_name (schema_name, path) + return remover_for_schema(schema.load_schema_by_name(schema_name), path) +end +remover_for_schema_by_name = util.memoize(remover_for_schema_by_name) + function selftest() print("selftest: lib.yang.path_data") - local schemalib = require("lib.yang.schema") local schema_src = [[module snabb-simple-router { namespace snabb:simple-router; prefix simple-router; @@ -165,8 +448,8 @@ function selftest() } }}]] - local scm = schemalib.load_schema(schema_src, "xpath-test") - local grammar = datalib.config_grammar_from_schema(scm) + local scm = schema.load_schema(schema_src, "xpath-test") + local grammar = data.config_grammar_from_schema(scm) -- Test resolving a key to a path. local data_src = [[ @@ -182,18 +465,18 @@ function selftest() } ]] - local data = datalib.load_config_for_schema(scm, data_src) + local d = data.load_config_for_schema(scm, data_src) -- Try resolving a path in a list (ctable). local getter = resolver(grammar, "/routes/route[addr=1.2.3.4]/port") - assert(getter(data) == 2) + assert(getter(d) == 2) local getter = resolver(grammar, "/routes/route[addr=255.255.255.255]/port") - assert(getter(data) == 7) + assert(getter(d) == 7) -- Try resolving a leaf-list local getter = resolver(grammar, "/blocked-ips[position()=1]") - assert(getter(data) == util.ipv4_pton("8.8.8.8")) + assert(getter(d) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -223,9 +506,9 @@ function selftest() } ]] - local fruit_scm = schemalib.load_schema(fruit_schema_src, "xpath-fruit-test") - local fruit_prod = datalib.config_grammar_from_schema(fruit_scm) - local fruit_data = datalib.load_config_for_schema(fruit_scm, fruit_data_src) + local fruit_scm = schema.load_schema(fruit_schema_src, "xpath-fruit-test") + local fruit_prod = data.config_grammar_from_schema(fruit_scm) + local fruit_data = data.load_config_for_schema(fruit_scm, fruit_data_src) local getter = resolver(fruit_prod, "/bowl/fruit[name=banana]/rating") assert(getter(fruit_data) == 10) From feab0129dcd76ba93744bfdaf09c1d740139cd07 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 11:29:11 +0100 Subject: [PATCH 006/100] Rename program.config.json to lib.ptree.json --- src/{program/config => lib/ptree}/json.lua | 2 +- src/program/config/bench/bench.lua | 2 +- src/program/config/listen/listen.lua | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/{program/config => lib/ptree}/json.lua (99%) diff --git a/src/program/config/json.lua b/src/lib/ptree/json.lua similarity index 99% rename from src/program/config/json.lua rename to src/lib/ptree/json.lua index 53aa40a11d..46c5e41d2d 100644 --- a/src/program/config/json.lua +++ b/src/lib/ptree/json.lua @@ -198,7 +198,7 @@ function write_json_object(output, obj) end function selftest () - print('selftest: program.config.json') + print('selftest: lib.ptree.json') local equal = require('core.lib').equal local function test_json(str, obj) local tmp = os.tmpname() diff --git a/src/program/config/bench/bench.lua b/src/program/config/bench/bench.lua index 6ddef1d319..d27ba4aad0 100644 --- a/src/program/config/bench/bench.lua +++ b/src/program/config/bench/bench.lua @@ -4,7 +4,7 @@ module(..., package.seeall) local S = require("syscall") local lib = require("core.lib") local ffi = require("ffi") -local json_lib = require("program.config.json") +local json_lib = require("lib.ptree.json") function show_usage(command, status, err_msg) if err_msg then print('error: '..err_msg) end diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 747e7248fe..860b9e90db 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -6,8 +6,8 @@ local ffi = require("ffi") local rpc = require("lib.yang.rpc") local data = require("lib.yang.data") local path_lib = require("lib.yang.path") +local json_lib = require("lib.ptree.json") local common = require("program.config.common") -local json_lib = require("program.config.json") local function open_socket(file) S.signal('pipe', 'ign') From a83a82786eab37406735cc15d8d7c5c10ce221b7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 12:07:31 +0100 Subject: [PATCH 007/100] "snabb config listen" more lenient about missing paths * src/program/config/listen/listen.lua: Default path to '/'. --- src/program/config/listen/listen.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 860b9e90db..6c03476fba 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -55,7 +55,7 @@ end local function read_request(client, schema_name, revision_date) local json = json_lib.read_json_object(client) - local id, verb, path = assert(json.id), assert(json.verb), assert(json.path) + local id, verb, path = assert(json.id), assert(json.verb), json.path or '/' path = path_lib.normalize_path(path) local handler = assert(request_handlers[data.normalize_id(verb)]) local req = handler(schema_name, revision_date, path, json.value) From bd689ffdf9e80b1c15bda7750a2da7a777fc3d2a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 13:01:48 +0100 Subject: [PATCH 008/100] "snabb config listen" can take schema / revision from directives --- src/program/config/listen/listen.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 6c03476fba..c81d3ade08 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -57,6 +57,8 @@ local function read_request(client, schema_name, revision_date) local json = json_lib.read_json_object(client) local id, verb, path = assert(json.id), assert(json.verb), json.path or '/' path = path_lib.normalize_path(path) + if json.schema then schema_name = json.schema end + if json.revision then revision_date = json.revision end local handler = assert(request_handlers[data.normalize_id(verb)]) local req = handler(schema_name, revision_date, path, json.value) local function print_reply(reply, fd) From 879830747736f7529689305843893cd17fb19201 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 12:10:16 +0100 Subject: [PATCH 009/100] Add ability for ptree manager to record incoming RPCs The trace log is in a JSON format that "snabb config listen" can play back. --- src/lib/ptree/ptree.lua | 16 +++++++- src/lib/ptree/trace.lua | 84 +++++++++++++++++++++++++++++++++++++++++ src/lib/yang/rpc.lua | 3 +- 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 src/lib/ptree/trace.lua diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index d4c3c71df7..744b1d592d 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -24,6 +24,7 @@ local action_codec = require("lib.ptree.action_codec") local alarm_codec = require("lib.ptree.alarm_codec") local support = require("lib.ptree.support") local channel = require("lib.ptree.channel") +local trace = require("lib.ptree.trace") local alarms = require("lib.yang.alarms") local Manager = {} @@ -42,6 +43,7 @@ local manager_config_spec = { worker_default_scheduling = {default={busywait=true}}, default_schema = {}, log_level = {default=default_log_level}, + rpc_trace_file = {}, cpuset = {default=cpuset.global_cpuset()}, Hz = {default=100}, } @@ -77,8 +79,20 @@ function new_manager (conf) ret.worker_default_scheduling = conf.worker_default_scheduling ret.workers = {} ret.state_change_listeners = {} + + if conf.rpc_trace_file then + ret:info("Logging RPCs to %s", conf.rpc_trace_file) + ret.trace = trace.new({file=conf.rpc_trace_file}) + + -- Start trace with initial configuration. + local p = path_data.printer_for_schema_by_name( + ret.schema_name, "/", true, "yang", false) + local conf_str = p(conf.initial_configuration, yang.string_output_file()) + ret.trace:record('set-config', {schema=ret.schema_name, config=conf_str}) + end + ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') - ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') + ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_', ret.trace) ret:set_initial_configuration(conf.initial_configuration) diff --git a/src/lib/ptree/trace.lua b/src/lib/ptree/trace.lua new file mode 100644 index 0000000000..e12cd02b8f --- /dev/null +++ b/src/lib/ptree/trace.lua @@ -0,0 +1,84 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local lib = require('core.lib') +local json = require("lib.ptree.json") + +local Trace = {} +local trace_config_spec = { + file = {required=true}, + file_mode = {default="w"}, +} + +function new (conf) + local conf = lib.parse(conf, trace_config_spec) + local ret = setmetatable({}, {__index=Trace}) + ret.id = 0 + ret.output = io.open(conf.file, conf.file_mode) + return ret +end + +local function listen_directive_for_rpc(rpc_id, args) + local ret = { path=args.path, schema=args.schema, revision=args.revision } + if rpc_id == 'get-config' then + ret.verb = 'get' + return ret + elseif rpc_id == 'set-config' then + ret.verb, ret.value = 'set', args.config + return ret + elseif rpc_id == 'add-config' then + ret.verb, ret.value = 'add', args.config + return ret + elseif rpc_id == 'remove-config' then + ret.verb = 'remove' + return ret + elseif rpc_id == 'get-state' then + ret.verb = 'get-state' + return ret + else + return nil + end +end + +function Trace:record(id, args) + assert(self.output, "trace closed") + local obj = listen_directive_for_rpc(id, args) + if not obj then return end + obj.id = tostring(self.id) + self.id = self.id + 1 + json.write_json_object(self.output, obj) + self.output:write('\n') + self.output:flush() +end + +function Trace:close() + self.output:close() + self.output = nil +end + +function selftest () + print('selftest: lib.ptree.trace') + local S = require('syscall') + + local tmp = os.tmpname() + local trace = new({file=tmp}) + trace:record("foo", {bar="baz"}) + trace:record("qux", {bar="baz", zog="100"}) + trace:close() + + local fd = S.open(tmp, 'rdonly') + local input = json.buffered_input(fd) + json.skip_whitespace(input) + local parsed = json.read_json_object(input) + assert(lib.equal(parsed, {id="0", verb="foo", bar="baz"})) + json.skip_whitespace(input) + parsed = json.read_json_object(input) + assert(lib.equal(parsed, {id="1", verb="qux", bar="baz", zog="100"})) + json.skip_whitespace(input) + assert(input:eof()) + fd:close() + os.remove(tmp) + + print('selftest: ok') +end diff --git a/src/lib/yang/rpc.lua b/src/lib/yang/rpc.lua index 0d2fde5972..59184ba7b9 100644 --- a/src/lib/yang/rpc.lua +++ b/src/lib/yang/rpc.lua @@ -52,10 +52,11 @@ function handle_calls(callee, str, handle) return callee.print_output(responses, util.string_output_file()) end -function dispatch_handler(obj, prefix) +function dispatch_handler(obj, prefix, trace) prefix = prefix or 'rpc_' local normalize_id = data.normalize_id return function(id, data) + if trace then trace:record(id, data) end local id = prefix..normalize_id(id) local f = assert(obj[id], 'handler not found: '..id) return f(obj, data) From 9572bde3d620408dd98c5e6bf401672ad3e4ad42 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 12:15:47 +0100 Subject: [PATCH 010/100] Wire up --trace for "snabb ptree" and document --- src/lib/ptree/README.md | 3 +++ src/program/ptree/README | 1 + src/program/ptree/ptree.lua | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/ptree/README.md b/src/lib/ptree/README.md index a64a54a2e1..20207d3916 100644 --- a/src/lib/ptree/README.md +++ b/src/lib/ptree/README.md @@ -103,6 +103,9 @@ Optional entries that may be present in the *parameters* table include: address being used by the worker. * `Hz`: Frequency at which to poll the config socket. Default is 1000. + * `rpc_trace_file`: File to which to write a trace of incoming RPCs + from "snabb config". The trace is written in a format that can later + be piped to "snabb config listen" to replay the trace. The return value is a ptree manager object, whose public methods are as follows: diff --git a/src/program/ptree/README b/src/program/ptree/README index 38e165374d..43da46ce50 100644 --- a/src/program/ptree/README +++ b/src/program/ptree/README @@ -32,6 +32,7 @@ Optional arguments: Default is "flush". -D Duration in seconds. -v Verbose (repeat for more verbosity). + -t FILE, --trace FILE File to which to write a trace of incoming RPCs. Optional arguments for debugging and profiling: -jv, -jv=FILE Print out when traces are recorded diff --git a/src/program/ptree/ptree.lua b/src/program/ptree/ptree.lua index b1da7bd735..f62b09c613 100644 --- a/src/program/ptree/ptree.lua +++ b/src/program/ptree/ptree.lua @@ -25,6 +25,7 @@ function parse_args (args) local handlers = {} function handlers.n (arg) opts.name = assert(arg) end function handlers.v () opts.verbosity = opts.verbosity + 1 end + function handlers.t (arg) opts.trace = assert(arg) end function handlers.D (arg) opts.duration = assert(tonumber(arg), "duration must be a number") assert(opts.duration >= 0, "duration can't be negative") @@ -48,8 +49,8 @@ function parse_args (args) function handlers.j (arg) scheduling.j = arg end function handlers.h () show_usage(0) end - args = lib.dogetopt(args, handlers, "vD:hn:j:", - { verbose = "v", duration = "D", help = "h", cpu = 1, + args = lib.dogetopt(args, handlers, "vD:hn:j:t:", + { verbose = "v", duration = "D", help = "h", cpu = 1, trace = "t", ["real-time"] = 0, ["on-ingress-drop"] = 1, name="n" }) @@ -78,6 +79,7 @@ function run (args) schema_name = schema_name, worker_default_scheduling = scheduling, log_level = ({"WARN","INFO","DEBUG"})[opts.verbosity or 1] or "DEBUG", + rpc_trace_file = opts.trace, } manager:main(opts.duration) From fe328da94bc1a7e33992959e2b5fddf20b960ce0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 12:23:59 +0100 Subject: [PATCH 011/100] Wire up a --trace argument for the lwaftr --- src/program/lwaftr/bench/bench.lua | 10 +++++++--- src/program/lwaftr/run/run.lua | 9 ++++++--- src/program/lwaftr/setup.lua | 11 ++++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index f9cea5dc62..10e7f2adaa 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -27,12 +27,14 @@ function parse_args(args) cpuset.global_cpuset():add_from_string(arg) end function handlers.n(arg) opts.name = assert(arg) end + function handlers.t (arg) opts.trace = assert(arg) end function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.j(arg) scheduling.j = arg end function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "j:n:hyb:D:", { - help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1}) + args = lib.dogetopt(args, handlers, "j:n:hyb:D:t:", { + help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1, + trace="t" }) if #args ~= 3 then show_usage(1) end return opts, scheduling, unpack(args) end @@ -52,7 +54,9 @@ function run(args) 'sinkv6') end - local manager = setup.ptree_manager(scheduling, setup_fn, conf) + local manager_opts = { worker_default_scheduling=scheduling, + rpc_trace_file=opts.trace } + local manager = setup.ptree_manager(setup_fn, conf, manager_opts) local stats = {csv={}} function stats:worker_starting(id) end diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 3f09c45505..740e9e87f2 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -53,6 +53,7 @@ function parse_args(args) local handlers = {} function handlers.n (arg) opts.name = assert(arg) end function handlers.v () opts.verbosity = opts.verbosity + 1 end + function handlers.t (arg) opts.trace = assert(arg) end function handlers.i () opts.virtio_net = true end function handlers.D (arg) opts.duration = assert(tonumber(arg), "duration must be a number") @@ -103,12 +104,12 @@ function parse_args(args) end function handlers.j(arg) scheduling.j = arg end function handlers.h() show_usage(0) end - lib.dogetopt(args, handlers, "b:c:vD:yhir:n:j:", + lib.dogetopt(args, handlers, "b:c:vD:yhir:n:j:t:", { conf = "c", v4 = 1, v6 = 1, ["v4-pci"] = 1, ["v6-pci"] = 1, verbose = "v", duration = "D", help = "h", virtio = "i", cpu = 1, ["ring-buffer-size"] = "r", ["real-time"] = 0, ["bench-file"] = "b", ["ingress-drop-monitor"] = 1, ["on-a-stick"] = 1, mirror = 1, - hydra = "y", reconfigurable = 0, name="n" }) + hydra = "y", reconfigurable = 0, name = "n", trace = "t" }) if ring_buffer_size ~= nil then if opts.virtio_net then fatal("setting --ring-buffer-size does not work with --virtio") @@ -172,7 +173,9 @@ function run(args) end end - local manager = setup.ptree_manager(scheduling, setup_fn, conf) + local manager_opts = { worker_default_scheduling=scheduling, + rpc_trace_file=opts.trace } + local manager = setup.ptree_manager(setup_fn, conf, manager_opts) -- FIXME: Doesn't work in multi-process environment. if false and opts.verbosity >= 2 then diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 001ed6a6b5..d665b3b329 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -589,7 +589,7 @@ local function compute_worker_configs(conf) return ret end -function ptree_manager(scheduling, f, conf) +function ptree_manager(f, conf, manager_opts) -- Always enabled in reconfigurable mode. alarm_notification = true @@ -603,12 +603,17 @@ function ptree_manager(scheduling, f, conf) return worker_app_graphs end - return manager.new_manager { + local initargs = { setup_fn = setup_fn, initial_configuration = conf, schema_name = 'snabb-softwire-v2', default_schema = 'ietf-softwire-br', - worker_default_scheduling = scheduling, -- log_level="DEBUG" } + for k, v in pairs(manager_opts or {}) do + assert(not initargs[k]) + initargs[k] = v + end + + return manager.new_manager(initargs) end From 966228015925014d081b8a73ebfcc4307e75582c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 15:21:08 +0000 Subject: [PATCH 012/100] Reduce memory use for get-config RPC The get-config RPC would want to create an array of Lua strings for the returned config. For large configs (e.g. the 1M-entry binding table that's already 122 MB raw), this is too much memory overhead. Instead change to incrementally write bytes to an output device held off-heap. --- src/lib/ptree/ptree.lua | 12 +-- src/lib/ptree/support/snabb-softwire-v2.lua | 2 +- src/lib/yang/data.lua | 87 ++++++++----------- src/lib/yang/rpc.lua | 4 +- src/lib/yang/util.lua | 45 ++++++++-- src/lib/yang/yang.lua | 2 +- src/program/config/common.lua | 2 +- .../migrate_configuration.lua | 2 +- 8 files changed, 88 insertions(+), 68 deletions(-) diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index 744b1d592d..b8c100f1e7 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -87,7 +87,7 @@ function new_manager (conf) -- Start trace with initial configuration. local p = path_data.printer_for_schema_by_name( ret.schema_name, "/", true, "yang", false) - local conf_str = p(conf.initial_configuration, yang.string_output_file()) + local conf_str = p(conf.initial_configuration, yang.string_io_file()) ret.trace:record('set-config', {schema=ret.schema_name, config=conf_str}) end @@ -280,7 +280,7 @@ function Manager:rpc_get_config (args) end local printer = path_data.printer_for_schema_by_name( args.schema, args.path, true, args.format, args.print_default) - local config = printer(self.current_configuration, yang.string_output_file()) + local config = printer(self.current_configuration, yang.string_io_file()) return { config = config } end local success, response = pcall(getter) @@ -401,7 +401,7 @@ function Manager:foreign_rpc_get_config (schema_name, path, format, local foreign_config = translate.get_config(self.current_configuration) local printer = path_data.printer_for_schema_by_name( schema_name, path, true, format, print_default) - local config = printer(foreign_config, yang.string_output_file()) + local config = printer(foreign_config, yang.string_io_file()) return { config = config } end function Manager:foreign_rpc_get_state (schema_name, path, format, @@ -411,7 +411,7 @@ function Manager:foreign_rpc_get_state (schema_name, path, format, local foreign_state = translate.get_state(self:get_native_state()) local printer = path_data.printer_for_schema_by_name( schema_name, path, false, format, print_default) - local state = printer(foreign_state, yang.string_output_file()) + local state = printer(foreign_state, yang.string_io_file()) return { state = state } end function Manager:foreign_rpc_set_config (schema_name, path, config_str) @@ -503,7 +503,7 @@ function Manager:rpc_get_state (args) local state = self:get_native_state() local printer = path_data.printer_for_schema_by_name( self.schema_name, args.path, false, args.format, args.print_default) - return { state = printer(state, yang.string_output_file()) } + return { state = printer(state, yang.string_io_file()) } end local success, response = pcall(getter) if success then return response else return {status=1, error=response} end @@ -517,7 +517,7 @@ function Manager:rpc_get_alarms_state (args) local state = { alarms = alarms.get_state() } - state = printer(state, yang.string_output_file()) + state = printer(state, yang.string_io_file()) return { state = state } end local success, response = pcall(getter) diff --git a/src/lib/ptree/support/snabb-softwire-v2.lua b/src/lib/ptree/support/snabb-softwire-v2.lua index 999c2da85b..9722423dfc 100644 --- a/src/lib/ptree/support/snabb-softwire-v2.lua +++ b/src/lib/ptree/support/snabb-softwire-v2.lua @@ -252,7 +252,7 @@ end local function serialize_binding_table(bt) local _, grammar = snabb_softwire_getter('/softwire-config/binding-table') local printer = data.data_printer_from_grammar(grammar) - return printer(bt, yang.string_output_file()) + return printer(bt, yang.string_io_file()) end local uint64_ptr_t = ffi.typeof('uint64_t*') diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index b791bfd5cc..05851f9117 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -757,38 +757,37 @@ function rpc_output_parser_from_schema(schema) return data_parser_from_grammar(rpc_output_grammar_from_schema(schema)) end -local function encode_yang_string(str) - if #str == 0 then return "''" end - if str:match("^[^%s;{}\"'/]*$") then return str end - local out = {} - table.insert(out, '"') - for i=1,#str do - local chr = str:sub(i,i) - if chr == '\n' then - table.insert(out, '\\n') - elseif chr == '\t' then - table.insert(out, '\\t') - elseif chr == '"' or chr == '\\' then - table.insert(out, '\\') - table.insert(out, chr) - else - table.insert(out, chr) - end - end - table.insert(out, '"') - return table.concat(out) -end - local value_serializers = {} local function value_serializer(typ) local prim = typ.primitive_type if value_serializers[prim] then return value_serializers[prim] end local tostring = assert(value.types[prim], prim).tostring - local function serializer(val) - return encode_yang_string(tostring(val)) + value_serializers[prim] = tostring + return tostring +end + +local function print_yang_string(str, file) + if #str == 0 then + file:write("''") + elseif str:match("^[^%s;{}\"'/]*$") then + file:write(str) + else + file:write('"') + for i=1,#str do + local chr = str:sub(i,i) + if chr == '\n' then + file:write('\\n') + elseif chr == '\t' then + file:write('\\t') + elseif chr == '"' or chr == '\\' then + file:write('\\') + file:write(chr) + else + file:write(chr) + end + end + file:write('"') end - value_serializers[prim] = serializer - return serializer end function xpath_printer_from_grammar(production, print_default, root) @@ -800,13 +799,10 @@ function xpath_printer_from_grammar(production, print_default, root) local function printer(keyword, production, printers) return assert(handlers[production.type])(keyword, production, printers) end - local function print_string(str, file) - file:write(encode_yang_string(str)) - end local function print_keyword(k, file, path) path = path:sub(1, 1) ~= '[' and root..'/'..path or root..path file:write(path) - print_string(k, file) + print_yang_string(k, file) file:write(' ') end local function body_printer(productions, order) @@ -888,7 +884,7 @@ function xpath_printer_from_grammar(production, print_default, root) local count = 1 for _,v in ipairs(data) do print_keyword(keyword.."[position()="..count.."]", file, '') - file:write(serialize(v)) + print_yang_string(serialize(v), file) file:write('\n') count = count + 1 end @@ -945,7 +941,7 @@ function xpath_printer_from_grammar(production, print_default, root) local str = serialize(data) if print_default or str ~= production.default then print_keyword(keyword, file, path) - file:write(str) + print_yang_string(str, file) file:write('\n') end end @@ -986,7 +982,7 @@ function xpath_printer_from_grammar(production, print_default, root) for _,v in ipairs(data) do file:write(root.."[position()="..count.."]") file:write(' ') - file:write(serialize(v)) + print_yang_string(serialize(v), file) file:write('\n') count = count + 1 end @@ -1000,7 +996,7 @@ function xpath_printer_from_grammar(production, print_default, root) if print_default or str ~= production.default then file:write(root) file:write(' ') - file:write(str) + print_yang_string(str, file) file:write('\n') return file:flush() end @@ -1017,12 +1013,9 @@ function data_printer_from_grammar(production, print_default) local function printer(keyword, production, printers) return assert(handlers[production.type])(keyword, production, printers) end - local function print_string(str, file) - file:write(encode_yang_string(str)) - end local function print_keyword(k, file, indent) file:write(indent) - print_string(k, file) + print_yang_string(k, file) file:write(' ') end local function body_printer(productions, order) @@ -1082,7 +1075,7 @@ function data_printer_from_grammar(production, print_default) return function(data, file, indent) for _,v in ipairs(data) do print_keyword(keyword, file, indent) - file:write(serialize(v)) + print_yang_string(serialize(v), file) file:write(';\n') end end @@ -1146,7 +1139,7 @@ function data_printer_from_grammar(production, print_default) local str = serialize(data) if print_default or str ~= production.default then print_keyword(keyword, file, indent) - file:write(str) + print_yang_string(str, file) file:write(';\n') end end @@ -1184,7 +1177,7 @@ function data_printer_from_grammar(production, print_default) local serialize = value_serializer(production.element_type) return function(data, file, indent) for _,v in ipairs(data) do - file:write(serialize(v)) + print_yang_string(serialize(v), file) file:write('\n') end return file:flush() @@ -1193,7 +1186,7 @@ function data_printer_from_grammar(production, print_default) function top_printers.scalar(production) local serialize = value_serializer(production.argument_type) return function(data, file) - file:write(serialize(data)) + print_yang_string(serialize(data), file) return file:flush() end end @@ -1201,14 +1194,6 @@ function data_printer_from_grammar(production, print_default) end data_printer_from_grammar = util.memoize(data_printer_from_grammar) -local function string_output_file() - local file = {} - local out = {} - function file:write(str) table.insert(out, str) end - function file:flush(str) return table.concat(out) end - return file -end - function data_printer_from_schema(schema, is_config) local grammar = data_grammar_from_schema(schema, is_config) return data_printer_from_grammar(grammar) @@ -1319,7 +1304,7 @@ function selftest() assert(parse_uint32('1') == 1) assert(parse_uint32('"1"') == 1) assert(parse_uint32(' "1" \n ') == 1) - assert(print_uint32(1, string_output_file()) == '1') + assert(print_uint32(1, util.string_io_file()) == '1') -- Verify that lists can lack keys when "config false;" is set. local list_wo_key_config_false = [[module config-false-schema { diff --git a/src/lib/yang/rpc.lua b/src/lib/yang/rpc.lua index 59184ba7b9..2e2ccd2833 100644 --- a/src/lib/yang/rpc.lua +++ b/src/lib/yang/rpc.lua @@ -23,7 +23,7 @@ function prepare_caller(schema_name) end function prepare_calls(caller, calls) - local str = caller.print_input(calls, util.string_output_file()) + local str = caller.print_input(calls, util.string_io_file()) local function parse_responses(str) local responses = caller.parse_output(str) assert(#responses == #calls) @@ -49,7 +49,7 @@ function handle_calls(callee, str, handle) table.insert(responses, { id=call.id, data=handle(call.id, call.data) }) end - return callee.print_output(responses, util.string_output_file()) + return callee.print_output(responses, util.string_io_file()) end function dispatch_handler(obj, prefix, trace) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 7314b8f255..a3df4710ea 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -87,12 +87,47 @@ function ipv4_ntop(addr) return ipv4:ntop(ffi.new('uint32_t[1]', lib.htonl(addr))) end -function string_output_file() +ffi.cdef [[ +void* malloc (size_t); +void free (void*); +]] + +function string_io_file() + local function alloc(n) + return ffi.gc(ffi.cast('char*', ffi.C.malloc(n)), ffi.C.free) + end + local file = {} - local out = {} - function file:write(str) table.insert(out, str) end - function file:flush(str) return table.concat(out) end - function file:clear(str) out = {} end + local size = 1024 + local buf = alloc(size) + local written = 0 + local read = 0 + function file:write(str) + while size - written < #str do + if 0 < read then + ffi.copy(buf, buf + read, written - read) + read, written = 0, written - read + else + local old_buf, old_written = buf, written + size = size * 2 + buf = alloc(size) + ffi.copy(buf, old_buf, written) + end + end + ffi.copy(buf + written, str, #str) + written = written + #str + end + function file:peek() + return buf + read, written - read + end + function file:flush() + local ptr, len = buf + read, written - read + return ffi.string(ptr, len) + end + function file:clear(str) + size, written, read = 1024, 0, 0 + buf = alloc(size) + end return file end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index fe92e6fea4..fac86acac3 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -20,7 +20,7 @@ load_config_for_schema_by_name = data.load_config_for_schema_by_name print_config_for_schema = data.print_config_for_schema print_config_for_schema_by_name = data.print_config_for_schema_by_name -string_output_file = util.string_output_file +string_io_file = util.string_io_file compile_config_for_schema = binary.compile_config_for_schema compile_config_for_schema_by_name = binary.compile_config_for_schema_by_name diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 2faed6ba98..bb3f03ccd9 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -146,7 +146,7 @@ end function serialize_data(data, schema_name, path, is_config) local printer = data_serializer(schema_name, path, is_config) - return printer(data, yang.string_output_file()) + return printer(data, yang.string_io_file()) end function serialize_config(config, schema_name, path) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index e16005bdd2..ced46d9840 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -275,7 +275,7 @@ local function config_to_string(schema, conf) schema = yang.load_schema_by_name(schema) end -- To keep memory usage as low as possible write it out to a temp file. - local memfile = util.string_output_file() + local memfile = util.string_io_file() yang.print_config_for_schema(schema, conf, memfile) conf = memfile:flush() From 008a4b865298d5da523f747861566af10a5bbf8d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 15:30:52 +0000 Subject: [PATCH 013/100] Increase response length limit from leader from 100M to 1G chars --- src/program/config/common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index bb3f03ccd9..6680e08f01 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -169,7 +169,7 @@ local function read_length(socket) if ch == '\n' then return len end assert(tonumber(ch), 'not a number: '..ch) len = len * 10 + tonumber(ch) - assert(len < 1e8, 'length too long: '..len) + assert(len < 1e9, 'length too long: '..len) end end From bd9a8da34d5d5e9cc58e067dac661970712f1ce0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 15 Dec 2017 15:38:15 +0000 Subject: [PATCH 014/100] Improve speed of yang qstring parser. --- src/lib/yang/parser.lua | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index a05ccf4782..bbd331d587 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -163,28 +163,27 @@ function Parser:parse_qstring(quote) local terminators = "\n"..quote if quote == '"' then terminators = terminators.."\\" end - local result = "" + local result = {} while true do - result = result..self:take_while("[^"..terminators.."]") + table.insert(result, self:take_while("[^"..terminators.."]")) if self:check(quote) then break end if self:check("\n") then while self.column < start_column do if not self:check(" ") and not self:check("\t") then break end end - result = result.."\n" + table.insert(result, "\n") if self.column > start_column then - result = result..string.rep(" ", self.column-start_column) + table.insert(result, string.rep(" ", self.column-start_column)) end elseif self:check("\\") then - if self:check("n") then result = result.."\n" - elseif self:check("t") then result = result.."\t" - elseif self:check('"') then result = result..'"' - elseif self:check("\\") then result = result.."\\" - else - result = result.."\\" - end + if self:check("n") then table.insert(result, "\n") + elseif self:check("t") then table.insert(result, "\t") + elseif self:check('"') then table.insert(result, '"') + elseif self:check("\\") then table.insert(result, "\\") + else table.insert(result, "\\") end end end + result = table.concat(result) self:check(quote) self:skip_whitespace() From 3e2fecb8e707389feb7069d82b031873eac424d8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Dec 2017 17:17:25 +0100 Subject: [PATCH 015/100] Fix ptree trace selftest --- src/lib/ptree/trace.lua | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/ptree/trace.lua b/src/lib/ptree/trace.lua index e12cd02b8f..b6d48f3b13 100644 --- a/src/lib/ptree/trace.lua +++ b/src/lib/ptree/trace.lua @@ -63,18 +63,24 @@ function selftest () local tmp = os.tmpname() local trace = new({file=tmp}) - trace:record("foo", {bar="baz"}) - trace:record("qux", {bar="baz", zog="100"}) + trace:record("get-config", + {path="/", schema="foo", revision="bar"}) + trace:record("set-config", + {path="/", schema="foo", revision="bar", config="baz"}) + trace:record("unsupported-rpc", + {path="/", schema="foo", revision="bar", config="baz"}) trace:close() local fd = S.open(tmp, 'rdonly') local input = json.buffered_input(fd) json.skip_whitespace(input) local parsed = json.read_json_object(input) - assert(lib.equal(parsed, {id="0", verb="foo", bar="baz"})) + assert(lib.equal(parsed, {id="0", verb="get", path="/", + schema="foo", revision="bar"})) json.skip_whitespace(input) parsed = json.read_json_object(input) - assert(lib.equal(parsed, {id="1", verb="qux", bar="baz", zog="100"})) + assert(lib.equal(parsed, {id="1", verb="set", path="/", + schema="foo", revision="bar", value="baz"})) json.skip_whitespace(input) assert(input:eof()) fd:close() From 7136c245dcebd3798d1a71e879d0053b1df0d968 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Dec 2017 17:22:08 +0100 Subject: [PATCH 016/100] changelog --- src/program/lwaftr/doc/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 9a4aa71de0..e430533ff7 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log +## [2017.11.01] + +* Add --trace option to "snabb lwaftr run", enabling a trace log of + incoming RPC calls that can be later replayed. + +* Fix excessive CPU and memory use when doing "snabb config get" of a + large configuration. + ## [2017.08.06] * Update IETF yang model from `ietf-softwire` to `ietf-softwire-br`. The From 459a936b1d64edfabfafd1ca7389b8a060a8191f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 15 Jan 2018 12:47:11 +0000 Subject: [PATCH 017/100] Fix property-based test stall --- .../lwaftr/tests/propbased/genyang.lua | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 2cf1dd710d..a2adb27789 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -36,26 +36,11 @@ function generate_any(pid, schema) -- leaf-list cases (for remove, we need a case with a selector too) -- Note: this assumes a list or leaf-list case exists in the schema at all elseif cmd == "add" then - local query, val, schema - local ok = false - while not ok do - query, val, schema = generate_config_xpath_and_val(schema) - if string.match(tostring(val), "^{.*}$") then - ok = true - end - end - --local query, val, schema = generate_config_xpath_and_val(schema) + local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config add -s %s %s \"%s\" \"%s\"", schema, pid, query, val) else - local query, val, schema - local ok = false - while not ok do - query, val, schema = generate_config_xpath_and_val(schema) - if string.match(query, "[]]$") then - ok = true - end - end + local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config remove -s %s %s \"%s\"", schema, pid, query) end @@ -297,6 +282,7 @@ local function generate_xpath_and_node_info(schema, for_state) if handler then handler(node) end end local function visit_body(node) + if not node then return end local ids = {} for id, node in pairs(node.body) do -- only choose nodes that are used in configs unless From ea5228c6d9de864480381ea91c9a26fc01e6df1f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sat, 16 Dec 2017 16:09:49 +0000 Subject: [PATCH 018/100] Fix lwaftr selftest run --- src/program/lwaftr/tests/subcommands/run_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index 5719051e72..bcc39ea9c6 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -22,9 +22,8 @@ class TestRun(BaseTestCase): ) def test_run(self): - output = self.run_cmd(self.cmd_args) - self.assertIn(b'link report', output, - b'\n'.join((b'OUTPUT', output))) + output = self.run_cmd(self.cmd_args).decode(ENC) + self.assertIn("Migrating instance", output) def test_run_on_a_stick_migration(self): # The LwAFTR should be abel to migrate from non-on-a-stick -> on-a-stick From f57c75ebb0f7397bb20b914647b9f4d7837c19e7 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 15 Dec 2017 13:04:24 +0100 Subject: [PATCH 019/100] Rework alarms timestamping logic The conversion of localtime to ISO8601 format didn't include timezone information. For instance, timestamp 0 (UNIX epoch) was converted to in GMT +01:00: 1970-01-01T00:00:00Z While it should be: 1970-01-01T00:00:00Z+01:00 So now all alarms date information works using localtime and converting it to ISO8601 including timezone info. When an ISO8601 date is converted to seconds it takes into account timezone information. That makes possible to compare local timestamps and ISO8601 dates. --- src/lib/yang/alarms.lua | 16 +++++++++++----- src/lib/yang/util.lua | 30 ++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/lib/yang/alarms.lua b/src/lib/yang/alarms.lua index b5fb24836b..7bcb2d018c 100644 --- a/src/lib/yang/alarms.lua +++ b/src/lib/yang/alarms.lua @@ -395,6 +395,12 @@ end local ages = {seconds=1, minutes=60, hours=3600, days=3600*24, weeks=3600*24*7} local function toseconds (date) + local function tz_seconds (t) + if not t.tz_hour then return 0 end + local sign = t.tz_sign or "+" + local seconds = tonumber(t.tz_hour) * 3600 + tonumber(t.tz_min) * 60 + return sign == '+' and seconds or seconds*-1 + end if type(date) == 'table' then assert(date.age_spec and date.value, "Not a valid 'older_than' data type") @@ -402,9 +408,8 @@ local function toseconds (date) "Not a valid 'age_spec' value: "..date.age_spec) return date.value * multiplier elseif type(date) == 'string' then - local t = {} - t.year, t.month, t.day, t.hour, t.min, t.sec = parse_date_as_iso_8601(date) - return os.time(t) + local t = parse_date_as_iso_8601(date) + return os.time(t) + tz_seconds(t) else error('Wrong data type: '..type(date)) end @@ -442,7 +447,7 @@ function purge_alarms (args) assert(type(older_than) == 'table') local alarm_time = toseconds(alarm.time_created) local threshold = toseconds(older_than) - return util.gmtime() - alarm_time >= threshold + return os.time() - alarm_time >= threshold end local function by_severity (alarm, args) local severity = assert(args.severity) @@ -739,7 +744,8 @@ function selftest () -- Test toseconds. assert(toseconds({age_spec='weeks', value=1}) == 3600*24*7) - assert(toseconds('1970-01-01T00:00:00Z') == 0) + local now = os.time() + assert(now == toseconds(format_date_as_iso_8601(now))) -- Purge alarms by status. assert(table_size(state.alarm_list.alarm) == 1) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index a3df4710ea..2043a17037 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -164,26 +164,36 @@ function memoize(f, max_occupancy) end end -function gmtime () +function timezone () local now = os.time() - local utcdate = os.date("!*t", now) - local localdate = os.date("*t", now) - localdate.isdst = false - local timediff = os.difftime(os.time(utcdate), os.time(localdate)) - return now + timediff + local utctime = os.date("!*t", now) + local localtime = os.date("*t", now) + local timediff = os.difftime(os.time(localtime), os.time(utctime)) + if timediff ~= 0 then + local sign = timediff > 0 and "+" or "-" + local time = os.date("!*t", math.abs(timediff)) + return sign..("%.2d:%.2d"):format(time.hour, time.min) + end end function format_date_as_iso_8601 (time) - time = time or gmtime() - return os.date("%Y-%m-%dT%H:%M:%SZ", time) + local ret = {} + time = time or os.time() + local utctime = os.date("!*t", time) + table.insert(ret, ("%.4d-%.2d-%.2dT%.2d:%.2d:%.2dZ"):format( + utctime.year, utctime.month, utctime.day, utctime.hour, utctime.min, utctime.sec)) + table.insert(ret, timezone() or "") + return table.concat(ret, "") end -- XXX: ISO 8601 can be more complex. We asumme date is the format returned -- by 'format_date_as_iso8601'. function parse_date_as_iso_8601 (date) assert(type(date) == 'string') - local pattern = "(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)Z" - return assert(date:match(pattern)) + local gmtdate = "(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)Z" + local year, month, day, hour, min, sec = assert(date:match(gmtdate)) + local tz_sign, tz_hour, tz_min = date:match("Z([+-]?)(%d%d):(%d%d)") + return {year=year, month=month, day=day, hour=hour, min=min, sec=sec, tz_sign=tz_sign, tz_hour=tz_hour, tz_min=tz_min} end function selftest() From ee03a9870c1c3a8b495d2446a70b84811382cd90 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 18 Jan 2018 11:24:15 +0100 Subject: [PATCH 020/100] Fix data_grammar_from_schema for empty cases * src/lib/yang/data.lua (data_grammar_from_schema): Prune cases with no members. Can be the case when we're asking for state data, for example, and a case has no state data. --- src/lib/yang/data.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 05851f9117..5eef38b416 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -181,7 +181,8 @@ function data_grammar_from_schema(schema, is_config) function handlers.choice(node) local choices = {} for choice, n in pairs(node.body) do - choices[choice] = visit_body(n) + local members = visit_body(n) + if not is_empty(members) then choices[choice] = members end end if is_empty(choices) then return end return {type="choice", default=node.default, mandatory=node.mandatory, From 631357ce038f9b84d269644e2da6eddc2f0c1590 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 18 Jan 2018 11:28:56 +0100 Subject: [PATCH 021/100] Rewrite random YANG path selector and value generator The path selector and value generator now use the grammar instead of the schema. Invalid path and value generation are now parameterized by a function argument instead of being a global variable. --- .../lwaftr/tests/propbased/genyang.lua | 401 +++++++++--------- 1 file changed, 208 insertions(+), 193 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index a2adb27789..dfca09cf92 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -3,24 +3,31 @@ module(..., package.seeall) -- This module provides functions for generating snabb config -- commands with random path queries and values -local ffi = require("ffi") -local schema = require("lib.yang.schema") +local ffi = require("ffi") +local schema = require("lib.yang.schema") +local data = require("lib.yang.data") +local path = require("lib.yang.path") +local path_data = require("lib.yang.path_data") +local util = require("lib.yang.util") local capabilities = {['ietf-softwire-br']={feature={'binding'}},} require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire-br", "snabb-softwire-v2" } --- toggles whether functions should intentionally generate invalid --- values for fuzzing purposes -local generate_invalid = true - -- choose an element of an array randomly local function choose(choices) local idx = math.random(#choices) return choices[idx] end +local function maybe(f, default, prob) + return function(...) + if math.random() < (prob or 0.8) then return f(...) end + return default + end +end + -- Generate a get/set/add/remove string given a pid string and optional schema function generate_any(pid, schema) local cmd = choose({ "get", "add", "remove", "set" }) @@ -57,7 +64,7 @@ end -- Like generate_get but for state queries function generate_get_state(pid, schema, query) if not query then - query, schema = generate_config_xpath_state(schema) + query, schema = generate_state_xpath(schema) end return string.format("./snabb config get-state -s %s %s \"%s\"", schema, pid, query) end @@ -120,7 +127,7 @@ end -- return a random number, preferring boundary values and -- sometimes returning results out of range -local function choose_bounded(lo, hi) +local function choose_bounded(lo, hi, generate_invalid) local r = math.random() -- occasionally return values that are invalid for type -- to provoke crashes @@ -138,20 +145,20 @@ end -- Choose a random number from within a range of valid value. RANGES -- is an array of {LO, HI} arrays; each of LO and HI can be numbers. -- LO can additionally be "min" and HI can be "max". -local function choose_value_from_ranges(ranges, type_min, type_max) +local function choose_value_from_ranges(ranges, type_min, type_max, generate_invalid) local r = math.random() if #ranges == 0 or (generate_invalid and r < 0.1) then - return choose_bounded(type_min, type_max) + return choose_bounded(type_min, type_max, generate_invalid) else local lo, hi = unpack(ranges[math.random(1,#ranges)]) if lo == "min" then lo = type_min end if hi == "max" then hi = type_max end - return choose_bounded(lo, hi) + return choose_bounded(lo, hi, generate_invalid) end end -local function value_from_type(a_type) +local function value_from_type(a_type, generate_invalid) local prim = a_type.primitive_type local ranges @@ -162,19 +169,19 @@ local function value_from_type(a_type) end if prim == "int8" then - return choose_value_from_ranges(ranges, -128, 127) + return choose_value_from_ranges(ranges, -128, 127, generate_invalid) elseif prim == "int16" then - return choose_value_from_ranges(ranges, -32768, 32767) + return choose_value_from_ranges(ranges, -32768, 32767, generate_invalid) elseif prim == "int32" then - return choose_value_from_ranges(ranges, -2147483648, 2147483647) + return choose_value_from_ranges(ranges, -2147483648, 2147483647, generate_invalid) elseif prim == "int64" then return ffi.cast("int64_t", random64()) elseif prim == "uint8" then - return choose_value_from_ranges(ranges, 0, 255) + return choose_value_from_ranges(ranges, 0, 255, generate_invalid) elseif prim == "uint16" then - return choose_value_from_ranges(ranges, 0, 65535) + return choose_value_from_ranges(ranges, 0, 65535, generate_invalid) elseif prim == "uint32" then - return choose_value_from_ranges(ranges, 0, 4294967295) + return choose_value_from_ranges(ranges, 0, 4294967295, generate_invalid) elseif prim == "uint64" then return random64() -- TODO: account for fraction-digits and range @@ -213,7 +220,7 @@ local function value_from_type(a_type) end return addr elseif prim == "union" then - return value_from_type(choose(a_type.union)) + return value_from_type(choose(a_type.union), generate_invalid) -- TODO: follow pattern statement elseif prim == "string" then local len = choose_nat() @@ -267,195 +274,201 @@ local function value_from_type(a_type) error("NYI or unknown type: "..prim) end --- from a config schema, generate an xpath query string --- this code is patterned off of the visitor used in lib.yang.data -local function generate_xpath_and_node_info(schema, for_state) - local path = "" - local handlers = {} - - -- data describing how to generate a value for the chosen path - -- it's a table with `node` and possibly-nil `selector` keys - local gen_info +local function value_generator(typ, generate_invalid) + -- FIXME: memoize dispatch. + return function() return tostring(value_from_type(typ), generate_invalid) end +end - local function visit(node) - local handler = handlers[node.kind] - if handler then handler(node) end +local function data_generator_from_grammar(production, generate_invalid) + local handlers = {} + local function visit1(keyword, production) + return assert(handlers[production.type])(keyword, production) end - local function visit_body(node) - if not node then return end - local ids = {} - for id, node in pairs(node.body) do - -- only choose nodes that are used in configs unless - -- for_state is passed - if for_state or node.config ~= false then - table.insert(ids, id) - end + local function body_generator(productions) + local order = {} + local gens = {} + for k,v in pairs(productions) do + table.insert(order, k) + gens[k] = visit1(k, v) + if not v.mandatory then gens[k] = maybe(gens[k]) end end - - local id = choose(ids) - if id then - visit(node.body[id]) - else - gen_info = { node = node } + table.sort(order) + return function() + local ret = {} + for _,k in ipairs(order) do + local v = gens[k]() + if v ~= nil then table.insert(ret, v) end + end + return table.concat(ret, ' ') end end - function handlers.container(node) - path = path .. "/" .. node.id - - -- don't always go into containers, since we need to test - -- fetching all sub-items too - if math.random() < 0.9 then - visit_body(node) - else - gen_info = { node = node } + function handlers.struct(keyword, production) + local gen = body_generator(production.members) + local prefix, suffix = '', '' + if keyword then prefix, suffix = keyword..' {', '}' end + return function() + return table.concat({prefix, gen(), suffix}, " ") end end - handlers['leaf-list'] = function(node) - if math.random() < 0.7 then - local idx = choose_nat() - local selector = string.format("[position()=%d]", idx) - path = path .. "/" .. node.id .. selector - gen_info = { node = node, selector = idx } - -- sometimes omit the selector, for the benefit of commands - -- like add where a selector is not useful - else - path = path .. "/" .. node.id - gen_info = { node = node } + function handlers.array(keyword, production) + local gen = value_generator(production.element_type, generate_invalid) + local prefix, suffix = '', ';' + if keyword then prefix = keyword..' '..prefix end + return function() + local ret = {} + while math.random() < 0.9 do + table.insert(ret, prefix..gen()..suffix) + end + return table.concat(ret, " ") end end - function handlers.list(node) - local key_types = {} - local r = math.random() - - path = path .. "/" .. node.id - - -- occasionally drop the selectors - if r < 0.7 then - for key in (node.key):split(" +") do - key_types[key] = node.body[key].type - end - - for key, type in pairs(key_types) do - local val = assert(value_from_type(type), type.primitive_type) - path = path .. string.format("[%s=%s]", key, val) - end - - -- continue path for child nodes - if math.random() < 0.5 then - visit_body(node) - else - gen_info = { node = node, selector = key_types } + local function shallow_copy(t) + local ret = {} + for k,v in pairs(t) do ret[k]=v end + return ret + end + function handlers.table(keyword, production) + local keys = {} + for k,v in pairs(production.keys) do + keys[k] = shallow_copy(v) + keys[k].mandatory = true + end + local gen_key = body_generator(production.keys) + local gen_value = body_generator(production.values) + local prefix, suffix = '{', '}' + if keyword then prefix = keyword..' '..prefix end + return function() + local ret = {} + while math.random() < 0.9 do + local x = table.concat({prefix,gen_key(),gen_value(),suffix}, " ") + table.insert(ret, x) end - else - gen_info = { node = node } + return table.concat(ret, " ") end end - function handlers.leaf(node) - path = path .. "/" .. node.id - val = value_from_type(node.type) - gen_info = { node = node } + function handlers.scalar(keyword, production) + local prefix, suffix = '', '' + if keyword then + prefix, suffix = keyword..' '..prefix, ';' + end + local gen = value_generator(production.argument_type, generate_invalid) + return function() + return prefix..gen()..suffix + end end - - -- just produce "/" on rare occasions - if math.random() > 0.01 then - visit_body(schema) + function handlers.choice(keyword, production) + local choices = {} + local cases = {} + for case, choice in pairs(production.choices) do + table.insert(cases, case) + choices[case] = body_generator(choice) + end + table.sort(cases) + return function () + return choices[choose(cases)]() + end end - - return path, gen_info + return visit1(nil, production) end +data_generator_from_grammar = util.memoize(data_generator_from_grammar) --- similar to generating a query path like the function above, but --- generates a compound value for `snabb config set` at some schema --- node -local function generate_value_for_node(gen_info) - -- hack for mutual recursion - local generate_compound - - local function generate(node) - if node.kind == "container" or node.kind == "list" then - return generate_compound(node) - elseif node.kind == "leaf-list" or node.kind == "leaf" then - return value_from_type(node.type) +local function path_generator_from_grammar(production, generate_invalid) + local handlers = {} + local function visit1(keyword, production) + return assert(handlers[production.type])(keyword, production) + end + function handlers.struct(keyword, production) + local members, gen_tail = {}, {} + for k,v in pairs(production.members) do + table.insert(members, k) + gen_tail[k] = assert(visit1(k, v)) + end + table.sort(members) + return function () + local head = keyword or '' + if math.random() < 0.1 then return head end + if head ~= '' then head = head..'/' end + local k = choose(members) + return head..gen_tail[k]() end end - - -- take a node and (optional) keys and generate a compound value - -- the keys are only provided for a list node - generate_compound = function(node, keys) - local ids = {} - for id, node in pairs(node.body) do - -- only choose nodes that are used in configs - if node.config ~= false then - table.insert(ids, id) - end + function handlers.array(keyword, production) + return function () + local head = keyword + if math.random() < 0.3 then return head end + return head..'[position()='..math.random(1,100)..']' end - - local val = "" - - for _, id in ipairs(ids) do - local subnode = node.body[id] - local r = math.random() - if (subnode.mandatory or r > 0.5) and - (not keys or not keys[id]) then - - if subnode.kind == "leaf-list" then - local count = choose_nat() - for i=0, count do - local subval = generate(subnode) - val = val .. string.format("%s %s; ", id, subval) - end - elseif subnode.kind == "container" or subnode.kind == "list" then - local subval = generate(subnode) - val = val .. string.format("%s {%s} ", id, subval) - else - local subval = generate(subnode) - val = val .. string.format("%s %s; ", id, subval) - end + end + function handlers.table(keyword, production) + local keys, values, gen_key, gen_tail = {}, {}, {}, {} + for k,v in pairs(production.keys) do + table.insert(keys, k) + gen_key[k] = data_generator_from_grammar(v, generate_invalid) + end + for k,v in pairs(production.values) do + table.insert(values, k) + gen_tail[k] = visit1(k, v) + end + table.sort(keys) + table.sort(values) + return function () + local head = keyword + if math.random() < 0.1 then return head end + for _,k in ipairs(keys) do + head = head..'['..k..'='..gen_key[k]()..']' end + if math.random() < 0.1 then return head end + return head..'/'..gen_tail[choose(values)]() end - - return val end - - local node = gen_info.node - if node.kind == "list" and gen_info.selector then - generate_compound(node, gen_info.selector) - else - -- a top-level list needs the brackets, e.g., as in - -- snabb config add /routes/route { addr 1.2.3.4; port 1; } - if node.kind == "list" then - return "{" .. generate(node) .. "}" - else - return generate(node) + function handlers.scalar(keyword, production) + assert(keyword) + return function() return keyword end + end + function handlers.choice(keyword, production) + local choices, cases = {}, {} + for case, choice in pairs(production.choices) do + table.insert(cases, case) + choices[case] = visit1(nil, {type='struct',members=choice}) end + table.sort(cases) + return function() return choices[choose(cases)]() end end + local gen = visit1(nil, production) + return function() return '/'..gen() end end +path_generator_from_grammar = util.memoize(path_generator_from_grammar) -local function generate_xpath(schema, for_state) - local path = generate_xpath_and_node_info(schema, for_state) - return path +local function choose_path_for_grammar(grammar, generate_invalid) + return path_generator_from_grammar(grammar, generate_invalid)() end -local function generate_xpath_and_val(schema) - local val, path, gen_info +local function choose_path_and_value_generator_for_grammar(grammar, generate_invalid) + local path = choose_path_for_grammar(grammar, generate_invalid) + local getter, subgrammar = path_data.resolver(grammar, path) + return path, data_generator_from_grammar(subgrammar, generate_invalid) +end - while not val do - path, gen_info = generate_xpath_and_node_info(schema) +local function choose_path_and_value_generator(schema, is_config, generate_invalid) + local grammar = data.data_grammar_from_schema(schema, is_config) + return choose_path_and_value_generator_for_grammar(grammar, generate_invalid) +end - if gen_info then - val = generate_value_for_node(gen_info) - end - end +local function generate_xpath(schema, is_config, generate_invalid) + local grammar = data.data_grammar_from_schema(schema, is_config) + return choose_path_for_grammar(grammar, generate_invalid) +end - return path, val +local function generate_xpath_and_val(schema, is_config, generate_invalid) + local path, gen_value = choose_path_and_value_generator( + schema, is_config, generate_invalid) + return path, gen_value() end -function generate_config_xpath(schema_name) - if not schema_name then - schema_name = choose(schemas) - end - local schema = schema.load_schema_by_name(schema_name) - return generate_xpath(schema, false), schema_name +function generate_config_xpath(schema_name, generate_invalid) + schema_name = schema_name or choose(schemas) + local schema = schema.load_schema_by_name(schema_name) + return generate_xpath(schema, true, generate_invalid), schema_name end -- types that may be randomly picked for a fuzzed test case @@ -464,44 +477,45 @@ local types = { "int8", "int16", "int32", "int64", "uint8", "uint16", "ipv6-address", "ipv6-prefix", "mac-address", "string", "binary" } -function generate_config_xpath_and_val(schema_name) - if not schema_name then - schema_name = choose(schemas) - end +function generate_config_xpath_and_val(schema_name, generate_invalid) + schema_name = schema_name or choose(schemas) local schema = schema.load_schema_by_name(schema_name) local r = math.random() local path, val -- once in a while, generate a nonsense value if generate_invalid and r < 0.05 then - path = generate_xpath(schema, false) - val = value_from_type({ primitive_type=choose(types) }) + path = generate_xpath(schema, true) + val = value_from_type({ primitive_type=choose(types) }, generate_invalid) else - path, val = generate_xpath_and_val(schema) + path, val = generate_xpath_and_val(schema, true, generate_invalid) end return path, val, schema_name end -function generate_config_xpath_state(schema_name) - if not schema_name then - schema_name = choose(schemas) - end - local schema = schema.load_schema_by_name(schema_name) - local path = generate_xpath(schema.body["softwire-state"], true) - return "/softwire-state" .. path, schema_name +function generate_state_xpath(schema_name, generate_invalid) + schema_name = schema_name or choose(schemas) + local schema = schema.load_schema_by_name(schema_name) + return generate_xpath(schema, false, generate_invalid), schema_name end function selftest() + print('selftest: program.lwaftr.tests.propbased.genyang') local data = require("lib.yang.data") local path = require("lib.yang.path") - local schema = schema.load_schema_by_name("snabb-softwire-v1") + local schema = schema.load_schema_by_name("snabb-softwire-v2") local grammar = data.config_grammar_from_schema(schema) - path.convert_path(grammar, generate_xpath(schema)) + for i=1,1000 do + local subpath, value = generate_xpath_and_val(schema, true) + -- print(i, subpath, value) + end - -- set flag to false to make tests predictable - generate_invalid = false + for i=1,1000 do + local subpath, value = generate_xpath_and_val(schema, false) + -- print(i, subpath, value) + end -- check some int types with range statements for i=1, 100 do @@ -532,4 +546,5 @@ function selftest() local cmd = string.format("echo \"%s\" | base64 -d > /dev/null", val) assert(os.execute(cmd) == 0, string.format("test value: %s", val)) end + print('selftest: ok') end From c752d1d5b661ab98f877e4d10a960243edd04bde Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 18 Jan 2018 12:36:56 +0100 Subject: [PATCH 022/100] Remove unused imports --- src/program/lwaftr/tests/propbased/genyang.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index dfca09cf92..46d2ce515b 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -6,7 +6,6 @@ module(..., package.seeall) local ffi = require("ffi") local schema = require("lib.yang.schema") local data = require("lib.yang.data") -local path = require("lib.yang.path") local path_data = require("lib.yang.path_data") local util = require("lib.yang.util") @@ -502,8 +501,6 @@ end function selftest() print('selftest: program.lwaftr.tests.propbased.genyang') - local data = require("lib.yang.data") - local path = require("lib.yang.path") local schema = schema.load_schema_by_name("snabb-softwire-v2") local grammar = data.config_grammar_from_schema(schema) From cf61265784e0204b9906f97ea02632a776fe4638 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 18 Jan 2018 12:39:59 +0100 Subject: [PATCH 023/100] Simplify test --- src/program/lwaftr/tests/propbased/genyang.lua | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 46d2ce515b..4f2614fe1e 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -504,15 +504,8 @@ function selftest() local schema = schema.load_schema_by_name("snabb-softwire-v2") local grammar = data.config_grammar_from_schema(schema) - for i=1,1000 do - local subpath, value = generate_xpath_and_val(schema, true) - -- print(i, subpath, value) - end - - for i=1,1000 do - local subpath, value = generate_xpath_and_val(schema, false) - -- print(i, subpath, value) - end + for i=1,1000 do generate_xpath_and_val(schema, true) end + for i=1,1000 do generate_xpath_and_val(schema, false) end -- check some int types with range statements for i=1, 100 do From 73caaaf8b75a62026bfde351f6fc0a6631cfdc60 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sun, 24 Dec 2017 17:06:30 +0100 Subject: [PATCH 024/100] Add DNS-SD program DNS-SD (DNS Service Discovery) is the protocol used by Apple's Bonjour to discover services and devices in a local network. It's widely used as well for discovering IoT devices such as Chromecast, Amazon FireTV or services such as Spotify in a local area network. Snabb's dnssd program is similar to avahi-browser. Given an OS network interface sends a multicast query to discover the services and devices available in the network. The program reads incoming traffic and prints out responses. Currently supported DNS records are PTR, A, SRV and TXT. --- src/lib/protocol/dns/dns.lua | 671 ++++++++++++++++++++++++++++ src/lib/protocol/dns/mdns.lua | 186 ++++++++ src/lib/protocol/dns/mdns_query.lua | 108 +++++ src/program/dnssd/README | 17 + src/program/dnssd/README.inc | 1 + src/program/dnssd/README.md | 36 ++ src/program/dnssd/dnssd.lua | 168 +++++++ 7 files changed, 1187 insertions(+) create mode 100644 src/lib/protocol/dns/dns.lua create mode 100644 src/lib/protocol/dns/mdns.lua create mode 100644 src/lib/protocol/dns/mdns_query.lua create mode 100644 src/program/dnssd/README create mode 120000 src/program/dnssd/README.inc create mode 100644 src/program/dnssd/README.md create mode 100644 src/program/dnssd/dnssd.lua diff --git a/src/lib/protocol/dns/dns.lua b/src/lib/protocol/dns/dns.lua new file mode 100644 index 0000000000..6d4cf2cc72 --- /dev/null +++ b/src/lib/protocol/dns/dns.lua @@ -0,0 +1,671 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local ipv4 = require("lib.protocol.ipv4") +local lib = require("core.lib") +local header = require("lib.protocol.header") + +local ffi = require("ffi") +local C = ffi.C + +ffi.cdef[[ +size_t strlen(const char *s); +]] + +local htons, ntohs = lib.htons, lib.ntohs +local htonl, ntohl = lib.htonl, lib.ntohl + +A = 0x01 +PTR = 0x0c +SRV = 0x21 +TXT = 0x10 + +CLASS_IN = 0x1 + +local function r16 (ptr) + return ffi.cast("uint16_t*", ptr)[0] +end +local function contains (t, e) + for _, each in ipairs(t) do + if each == e then return true end + end + return false +end + +DNS = {} + +local function encode_name_string (str) + local function repetitions (str, char) + local ret = 0 + for each in str:gmatch(char) do + ret = ret + 1 + end + return ret + end + local extra = repetitions(str, '%.') + 1 + local ret = ffi.new("char[?]", #str + extra + 1) + local buffer = ret + local function write_len (num) + buffer[0] = num + buffer = buffer + 1 + end + local function write_str (arg) + ffi.copy(buffer, arg, #arg) + buffer = buffer + #arg + end + local total_length = 0 + for each in str:gmatch('([^%.]+)') do + write_len(#each) + write_str(each) + total_length = #each + 1 + end + return ret, total_length +end + +local function decode_name_string (cdata) + local t = {} + local buffer, i = cdata, 0 + local function read_len () + local len = tonumber(buffer[0]) + buffer = buffer + 1 + return len + end + local function read_str (len) + table.insert(t, ffi.string(buffer, len)) + buffer = buffer + len + end + local function eol () + return buffer[0] == 0 + end + local function flush () + return table.concat(t, ".") + end + while not eol() do + local len = read_len() + if len < 0 then break end + read_str(len) + end + return flush() +end + +local function encode_string (str) + assert(type(str) == "string") + local ret = ffi.new("char[?]", #str+1) + ret[0] = #str + ffi.copy(ret + 1, str) + return ret +end + +local function encode_strings (t) + assert(type(t) == "table") + local ret = ffi.new("char*[?]", #t) + for i, each in ipairs(arg) do + ret[i-1] = encode_single(each) + end + return ret +end + +local function decode_string (cstr, cstr_len) + local t = {} + local pos = 0 + while pos < cstr_len do + local len = tonumber(cstr[pos]) + pos = pos + 1 + table.insert(t, ffi.string(cstr + pos, len)) + pos = pos + len + end + return t +end + +-- DNS Query Record. + +query_record = subClass(header) +query_record._name = "query_record" +query_record:init({ + [1] = ffi.typeof[[ + struct { + char* name; + uint16_t type; + uint16_t class; + } __attribute__((packed)) + ]] +}) + +function query_record:new_from_mem (data, length) + local o = query_record:superClass().new(self) + local name, len = parse_name(data, length) + local h = o:header() + h.name = name + h.type = r16(data + len) + h.class = r16(data + len + 2) + return o, len + 4 +end + +function parse_name (data, size) + local len = 2 + local maybe_type = r16(data + len) + if maybe_type ~= htons(TXT) then + len = name_length(data, size) + end + if len then + local name = ffi.new("uint8_t[?]", len) + ffi.copy(name, data, len) + return name, len + end +end + +-- Returns dns_record.name's length. +function name_length (data, size) + local ptr = data + local i = 0 + while i < size do + -- PTR records's name end with an end-of-string character. Next byte + -- belongs to type. + if ptr[i] == 0 and ptr[i + 1] == 0 then i = i + 1 break end + -- This zero belongs to type so break. + if ptr[i] == 0 then break end + i = i + 1 + end + return i < size and i or nil +end + +function query_record:new (config) + local o = query_record:superClass().new(self) + o:name(config.name) + o:type(config.type) + o:klass(config.class) + return o +end + +function query_record:name (name) + local h = self:header() + if name then + h.name, len = encode_name_string(name) + end + return h.name ~= nil and decode_name_string(h.name) or "" +end + +function query_record:type (type) + if type then + self:header().type = htons(type) + end + return ntohs(self:header().type) +end + +function query_record:klass (class) + if class then + self:header().class = htons(class) + end + return ntohs(self:header().class) +end + +-- Size of record depends of length of name. +function query_record:sizeof () + local success, h = pcall(self.header, self) + if not success then + return self:superClass().sizeof(self) + else + return tonumber(C.strlen(h.name) + 1) + 4 + end +end + +-- DNS Response Record common fields. +-- Abstract class. Used by all other types of records: A, PTR, SRV and TXT. + +local dns_record_header_typedef = [[ + struct { + char *name; + uint16_t type; + uint16_t class; + uint32_t ttl; + uint16_t data_length; + } __attribute__((packed)) +]] + +local dns_record_header = subClass(header) +dns_record_header._name = "dns_record_header" + +function dns_record_header:initialize(o, config) + o:name(config.name) + o:klass(config.class) + o:ttl(config.ttl) + o:data_length(config.data_length) +end + +function dns_record_header:new_from_mem(header, data, size) + -- Copy name. + local name, len = parse_name(data, size) + header.name = name + + -- Cast a temporary pointer for the rest of dns_record_header fields. + local dns_record_subheader_t = ffi.typeof[[ + struct { + uint16_t type; + uint16_t class; + uint32_t ttl; + uint16_t data_length; + } __attribute__((packed)) + ]] + local dns_record_subheader_ptr_t = ffi.typeof("$*", dns_record_subheader_t) + local ptr = ffi.cast(dns_record_subheader_ptr_t, data + len) + + header.type = ptr.type + header.class = ptr.class + header.ttl = ptr.ttl + header.data_length = ptr.data_length + + return len + ffi.sizeof(dns_record_subheader_t) +end + +function dns_record_header:name (name) + local h = self:header() + if name then + h.name = ffi.new("char[?]", #name) + ffi.copy(h.name, name) + end + return h.name ~= nil and ffi.string(h.name) or "" +end + +function dns_record_header:type (type) + if type then + self:header().type = htons(type) + end + return ntohs(self:header().type) +end + +-- TODO: Cannot call method 'class' because it is already defined probably in +-- the parent class). +function dns_record_header:klass (class) + if class then + self:header().class = htons(class) + end + return ntohs(self:header().class) +end + +function dns_record_header:ttl(ttl) + if ttl then + self:header().ttl = htonl(ttl) + end + return ntohl(self:header().ttl) +end + +function dns_record_header:data_length(data_length) + if data_length then + self:header().data_length = htons(data_length) + end + return ntohs(self:header().data_length) +end + +-- TXT record. + +txt_record = subClass(dns_record_header) +txt_record._name = "txt_record" +txt_record:init({ + [1] = ffi.typeof(([[ + struct { + %s; + char* chunks; + } __attribute__((packed)) + ]]):format(dns_record_header_typedef)) +}) + +function txt_record:new_from_mem(data, size) + local o = txt_record:superClass().new(self) + local offset = dns_record_header:new_from_mem(o:header(), data, size) + o:header().chunks = ffi.new("char[?]", o:data_length()) + ffi.copy(o:header().chunks, data + offset, o:data_length()) + local total_length = offset + o:data_length() + return o, total_length +end + +function txt_record:new (config) + local o = txt_record:superClass().new(self) + dns_record_header:initialize(o, config) + o:type(TXT) + if config.chunks then + o:chunks(config.chunks) + end +end + +function txt_record:chunks (chunks) + if chunks then + self:header().chunks = encode_string(chunks) + end + return decode_string(self:header().chunks, self:data_length()) +end + +function txt_record:tostring () + local t = decode_string(self:header().chunks, self:data_length()) + return ("{%s}"):format(table.concat(t, ";")) +end + +-- SRV record. + +srv_record = subClass(dns_record_header) +srv_record._name = "srv_record" +srv_record:init({ + [1] = ffi.typeof(([[ + struct { + %s; + uint16_t priority; + uint16_t weight; + uint16_t port; + char* target; + } __attribute__((packed)) + ]]):format(dns_record_header_typedef)) +}) + +function srv_record:new_from_mem(data, size) + local o = srv_record:superClass().new(self) + local offset = dns_record_header:new_from_mem(o:header(), data, size) + o:header().priority = r16(data + offset) + o:header().weight = r16(data + offset + 2) + o:header().port = r16(data + offset + 4) + o:header().target = ffi.new("char[?]", o:data_length() - 6) + ffi.copy(o:header().target, data + offset + 6, o:data_length() - 6) + local total_length = offset + o:data_length() + return o, total_length +end + +function srv_record:new (config) + local o = srv_record:superClass().new(self) + o:type(SRV) + o:priority(config.priority or 0) + o:weight(config.weight) + o:port(config.port) + o:target(config.target) + return o +end + +function srv_record:priority (priority) + if priority then + self:header().priority = htons(priority) + end + return ntohs(self:header().priority) +end + +function srv_record:weight (weight) + if weight then + self:header().weight = htons(weight) + end + return ntohs(self:header().weight) +end + +function srv_record:port (port) + if port then + self:header().port = htons(port) + end + return ntohs(self:header().port) +end + +function srv_record:target (target) + local h = self:header() + if target then + h.target = ffi.new("char[?]", #target) + ffi.copy(h.target, target) + end + return h.target ~= nil and ffi.string(h.target) or "" +end + +function srv_record:tostring () + local target = decode_name_string(self:header().target) + return ("{target: %s; port: %d}"):format(target, self:port()) +end + +-- PTR record. + +ptr_record = subClass(dns_record_header) +ptr_record._name = "ptr_record" +ptr_record:init({ + [1] = ffi.typeof(([[ + struct { + %s; /* DNS record header */ + char* domain_name; /* PTR record own fields */ + } __attribute__((packed)) + ]]):format(dns_record_header_typedef)) +}) + +function ptr_record:new_from_mem(data, size) + local o = ptr_record:superClass().new(self) + local offset = dns_record_header:new_from_mem(o:header(), data, size) + o:header().domain_name = ffi.new("char[?]", o:data_length()) + ffi.copy(o:header().domain_name, data + offset, o:data_length()) + local total_length = offset + o:data_length() + return o, total_length +end + +function ptr_record:new (config) + local o = ptr_record:superClass().new(self) + dns_record_header:initialize(o, config) + o:type(PTR) + o:domain_name(config.domain_name) + return o +end + +function ptr_record:domain_name (domain_name) + local h = self:header() + if domain_name then + h.domain_name = ffi.new("char[?]", #domain_name) + ffi.copy(h.domain_name, domain_name) + end + return h.domain_name ~= nil and ffi.string(h.domain_name) or "" +end + +function ptr_record:tostring () + local name = decode_name_string(self:header().name) + local domain_name = decode_name_string(self:header().domain_name) + if #name > 0 then + return ("{name: %s; domain_name: %s}"):format(name, domain_name) + else + return ("{domain_name: %s}"):format(domain_name) + end +end + +-- A record. + +local a_record = subClass(dns_record_header) +a_record._name = "address_record" +a_record:init({ + [1] = ffi.typeof(([[ + struct { + %s; /* DNS record header */ + uint8_t address[4]; /* A record own fields */ + } __attribute__((packed)) + ]]):format(dns_record_header_typedef)) +}) + +function a_record:new_from_mem(data, size) + local o = a_record:superClass().new(self) + local offset = dns_record_header:new_from_mem(o:header(), data, size) + ffi.copy(o:header().address, data + offset, o:data_length()) + local total_length = offset + o:data_length() + return o, total_length +end + +function a_record:new (config) + local o = a_record:superClass().new(self) + dns_record_header:initialize(o, config) + o:type(A) + o:address(config.address) + return o +end + +function a_record:address (address) + if address then + ffi.copy(self:header().address, ipv4:pton(address), 4) + end + return ipv4:ntop(self:header().address) +end + +function a_record:tostring () + local name = decode_name_string(self:header().name) + if #name > 0 then + return ("{name: %s; address: %s}"):format(name, self:address()) + else + return ("{address: %s}"):format(self:address()) + end +end + +function DNS.parse_records (data, size, n) + n = n or 1 + assert(n >= 0) + local rrs, total_len = {}, 0 + local ptr = data + for _=1,n do + local rr, len = DNS.parse_record(ptr, size) + if len == 0 then break end + ptr = ptr + len + total_len = total_len + len + table.insert(rrs, rr) + end + return rrs, total_len +end + +function DNS.parse_record (data, size) + local function is_supported (type) + local supported_types = {A, PTR, SRV, TXT} + return type and contains(supported_types, type) + end + local type = parse_type(data, size) + type = ntohs(assert(type)) + if not is_supported(type) then return nil, 0 end + return DNS.create_record_by_type(type, data, size) +end + +function parse_type (data, size) + local maybe_type = r16(data + 2) + if maybe_type == htons(TXT) then + return maybe_type + else + local len = name_length(data, size) + if len then + return r16(data + len) + end + end +end + +function DNS.create_record_by_type (type, data, size) + if type == A then + return a_record:new_from_mem(data, size) + elseif type == PTR then + return ptr_record:new_from_mem(data, size) + elseif type == SRV then + return srv_record:new_from_mem(data, size) + elseif type == TXT then + return txt_record:new_from_mem(data, size) + end +end + +function selftest () + -- Test PTR record. + local pkt = packet.from_string(lib.hexundump([[ + 09 5f 73 65 72 76 69 63 65 73 07 5f 64 6e 73 2d + 73 64 04 5f 75 64 70 05 6c 6f 63 61 6c 00 00 0c + 00 01 00 00 0e 0f 00 18 10 5f 73 70 6f 74 69 66 + 79 2d 63 6f 6e 6e 65 63 74 04 5f 74 63 70 c0 23 + ]], 64)) + local ptr_rr, len = ptr_record:new_from_mem(pkt.data, 64) + assert(ptr_rr:type() == PTR) + assert(ptr_rr:ttl() == 3599) + assert(ptr_rr:klass() == 0x1) + assert(ptr_rr:data_length() == 24) + assert(len == 64) + + -- Test A record. + pkt = packet.from_string(lib.hexundump([[ + 14 61 6d 61 7a 6f 6e 2d 32 39 64 36 39 35 38 31 + 65 2d 6c 61 6e c0 23 00 01 80 01 00 00 0e 0f 00 + 04 c0 a8 56 37 + ]], 37)) + local address_rr, len = a_record:new_from_mem(pkt.data, 37) + assert(address_rr:type() == A) + assert(address_rr:ttl() == 3599) + assert(address_rr:klass() == 0x8001) + assert(address_rr:data_length() == 4) + assert(address_rr:address() == "192.168.86.55") + assert(len == 37) + + -- Test SRV record. + pkt = packet.from_string(lib.hexundump([[ + 3c 61 6d 7a 6e 2e 64 6d 67 72 3a 31 32 31 31 34 + 43 39 35 32 43 36 36 39 31 46 39 30 35 43 45 30 + 45 35 39 43 45 36 34 31 45 39 38 3a 72 50 50 4b + 75 54 44 79 49 45 3a 36 38 31 32 37 37 0b 5f 61 + 6d 7a 6e 2d 77 70 6c 61 79 c0 45 00 21 80 01 00 + 00 0e 0f 00 08 00 00 00 00 b9 46 c0 4c + ]], 93)) + local srv_rr, len = srv_record:new_from_mem(pkt.data, 93) + assert(srv_rr:type() == SRV) + assert(srv_rr:ttl() == 3599) + assert(srv_rr:klass() == 0x8001) + assert(srv_rr:data_length() == 8) + assert(srv_rr:priority() == 0) + assert(srv_rr:weight() == 0) + assert(srv_rr:port() == 47430) + assert(len == 93) + + -- Test TXT record. + pkt = packet.from_string(lib.hexundump([[ + c0 71 00 10 80 01 00 00 0e 0f 00 91 03 73 3d 30 + 0f 61 74 3d 6b 37 59 79 41 70 53 54 68 43 48 4a + 17 6e 3d 61 65 69 6f 75 61 65 69 6f 75 61 65 69 + 6f 75 61 65 69 6f 75 61 06 74 72 3d 74 63 70 08 + 73 70 3d 34 32 31 37 38 04 70 76 3d 31 04 6d 76 + 3d 32 03 76 3d 32 03 61 3d 30 22 75 3d 31 32 31 + 31 34 43 39 35 32 43 36 36 39 31 46 39 30 35 43 + 45 30 45 35 39 43 45 36 34 31 45 39 38 11 61 64 + 3d 41 32 4c 57 41 52 55 47 4a 4c 42 59 45 57 05 + 64 70 76 3d 31 03 74 3d 38 03 66 3d 30 + ]], 157)) + local txt_rr, len = txt_record:new_from_mem(pkt.data, 157) + assert(txt_rr:type() == TXT) + assert(txt_rr:ttl() == 3599) + assert(txt_rr:klass() == 0x8001) + assert(txt_rr:data_length() == 145) + assert(#txt_rr:chunks() == 14) + assert(len == 157) + + -- MDNS response body containing many records. + local answers = packet.from_string(lib.hexundump([[ + 09 5f 73 65 72 76 69 63 65 73 07 5f 64 6e 73 2d + 73 64 04 5f 75 64 70 05 6c 6f 63 61 6c 00 00 0c + 00 01 00 00 0e 0f 00 18 10 5f 73 70 6f 74 69 66 + 79 2d 63 6f 6e 6e 65 63 74 04 5f 74 63 70 c0 23 + 14 61 6d 61 7a 6f 6e 2d 32 39 64 36 39 35 38 31 + 65 2d 6c 61 6e c0 23 00 01 80 01 00 00 0e 0f 00 + 04 c0 a8 56 37 3c 61 6d 7a 6e 2e 64 6d 67 72 3a + 31 32 31 31 34 43 39 35 32 43 36 36 39 31 46 39 + 30 35 43 45 30 45 35 39 43 45 36 34 31 45 39 38 + 3a 72 50 50 4b 75 54 44 79 49 45 3a 36 38 31 32 + 37 37 0b 5f 61 6d 7a 6e 2d 77 70 6c 61 79 c0 45 + 00 21 80 01 00 00 0e 0f 00 08 00 00 00 00 b9 46 + c0 4c c0 71 00 10 80 01 00 00 0e 0f 00 91 03 73 + 3d 30 0f 61 74 3d 6b 37 59 79 41 70 53 54 68 43 + 48 4a 17 61 65 69 6f 75 61 65 69 6f 75 61 65 69 + 6f 75 61 65 69 6f 75 61 65 69 06 74 72 3d 74 63 + 70 08 73 70 3d 34 32 31 37 38 04 70 76 3d 31 04 + 6d 76 3d 32 03 76 3d 32 03 61 3d 30 22 75 3d 31 + 32 31 31 34 43 39 35 32 43 36 36 39 31 46 39 30 + 35 43 45 30 45 35 39 43 45 36 34 31 45 39 38 11 + 61 64 3d 41 32 4c 57 41 52 55 47 4a 4c 42 59 45 + 57 05 64 70 76 3d 31 03 74 3d 38 03 66 3d 30 + ]], 351)) + local rrs, total_length = DNS.parse_records(answers.data, 351, 4) + assert(#rrs == 4) + assert(total_length == 351) + + -- DNS query record. + local pkt = packet.from_string(lib.hexundump([[ + 0b 5f 67 6f 6f 67 6c 65 7a 6f 6e 65 04 5f 74 63 + 70 05 6c 6f 63 61 6c 00 00 0c 00 01 + ]], 28)) + local query_rr, len = query_record:new_from_mem(pkt.data, 28) + assert(query_rr:name() == "_googlezone._tcp.local") + assert(query_rr:type() == PTR) + assert(query_rr:klass() == 0x1) + assert(query_rr:sizeof() == len) + assert(query_record:sizeof() == 12) + + local query = "_services._dns-sd._udp.local" + assert(decode_name_string((encode_name_string(query))) == query) +end diff --git a/src/lib/protocol/dns/mdns.lua b/src/lib/protocol/dns/mdns.lua new file mode 100644 index 0000000000..ef7249ec78 --- /dev/null +++ b/src/lib/protocol/dns/mdns.lua @@ -0,0 +1,186 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local DNS = require("lib.protocol.dns.dns").DNS +local ethernet = require("lib.protocol.ethernet") +local ffi = require("ffi") +local header = require("lib.protocol.header") +local ipv4 = require("lib.protocol.ipv4") +local lib = require("core.lib") +local udp = require("lib.protocol.udp") + +local htons, ntohs = lib.htons, lib.ntohs + +DST_ETHER = "01:00:5e:00:00:fb" +DST_IPV4 = "224.0.0.251" +DST_PORT = 5353 + +local STANDARD_QUERY_RESPONSE = 0x8400 + +local ethernet_header_size = 14 +local ipv4_header_size = 20 +local udp_header_size = 8 + +MDNS = subClass(header) +MDNS._name = "mdns" +MDNS:init({ + [1] = ffi.typeof[[ + struct { + uint16_t id; + uint16_t flags; + uint16_t questions; + uint16_t answer_rrs; + uint16_t authority_rrs; + uint16_t additional_rrs; + } __attribute__((packed)) + ]] +}) + +function MDNS:new (config) + local o = MDNS:superClass().new(self) + o:id(config.id) + o:flags(config.flags) + o:questions(config.questions) + o:answer_rrs(config.answer_rrs) + o:authority_rrs(config.authority_rrs) + o:additional_rrs(config.additional_rrs) + return o +end + +function MDNS:id (id) + if id then + self:header().id = htons(id) + end + return ntohs(self:header().id) +end + +function MDNS:flags (flags) + if flags then + self:header().flags = htons(flags) + end + return ntohs(self:header().flags) +end + +function MDNS:questions (questions) + if questions then + self:header().questions = htons(questions) + end + return ntohs(self:header().questions) +end + +function MDNS:answer_rrs (answer_rrs) + if answer_rrs then + self:header().answer_rrs = htons(answer_rrs) + end + return ntohs(self:header().answer_rrs) +end + +function MDNS:authority_rrs (authority_rrs) + if authority_rrs then + self:header().authority_rrs = htons(authority_rrs) + end + return ntohs(self:header().authority_rrs) +end + +function MDNS:additional_rrs (additional_rrs) + if additional_rrs then + self:header().additional_rrs = htons(additional_rrs) + end + return ntohs(self:header().additional_rrs) +end + +function MDNS.is_mdns (pkt) + local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) + local ipv4_hdr = ipv4:new_from_mem(pkt.data + ethernet_header_size, ipv4_header_size) + local udp_hdr = udp:new_from_mem(pkt.data + ethernet_header_size + ipv4_header_size, udp_header_size) + + return ethernet:ntop(ether_hdr:dst()) == DST_ETHER and + ipv4:ntop(ipv4_hdr:dst()) == DST_IPV4 and + udp_hdr:dst_port() == DST_PORT +end + +local function mdns_payload (pkt) + local payload_offset = ethernet_header_size + ipv4_header_size + udp_header_size + return pkt.data + payload_offset, pkt.length - payload_offset +end + +function MDNS.is_response (pkt) + local payload = mdns_payload(pkt) + local mdns = MDNS:new_from_mem(payload, MDNS:sizeof()) + return mdns:flags() == STANDARD_QUERY_RESPONSE +end + +function MDNS.parse_packet (pkt) + assert(MDNS.is_mdns(pkt)) + local ret = { + questions = {}, + answer_rrs = {}, + authority_rrs = {}, + additional_rrs = {}, + } + local payload, length = mdns_payload(pkt) + local mdns_hdr = MDNS:new_from_mem(payload, MDNS:sizeof()) + -- Skip header. + payload, length = payload + MDNS:sizeof(), length - MDNS:sizeof() + local function collect_records (n) + local t = {} + local rrs, rrs_len = DNS.parse_records(payload, length, n) + for _, each in ipairs(rrs) do table.insert(t, each) end + payload = payload + rrs_len + length = length - rrs_len + return t + end + ret.questions = collect_records(mdns_hdr:questions()) + ret.answer_rrs = collect_records(mdns_hdr:answer_rrs()) + ret.authority_rrs = collect_records(mdns_hdr:authority_rrs()) + ret.additional_rrs = collect_records(mdns_hdr:additional_rrs()) + return ret +end + +function selftest() + local function parse_response () + -- MDNS response. + local pkt = packet.from_string(lib.hexundump ([[ + 01:00:5e:00:00:fb ce:6c:59:f2:f3:c1 08 00 45 00 + 01 80 00 00 40 00 ff 11 82 88 c0 a8 56 40 e0 00 + 00 fb 14 e9 14 e9 01 6c d2 12 00 00 84 00 00 00 + 00 01 00 00 00 03 0b 5f 67 6f 6f 67 6c 65 63 61 + 73 74 04 5f 74 63 70 05 6c 6f 63 61 6c 00 00 0c + 00 01 00 00 00 78 00 2e 2b 43 68 72 6f 6d 65 63 + 61 73 74 2d 38 34 38 64 61 35 39 64 38 63 62 36 + 34 35 39 61 39 39 37 31 34 33 34 62 31 64 35 38 + 38 62 61 65 c0 0c c0 2e 00 10 80 01 00 00 11 94 + 00 b3 23 69 64 3d 38 34 38 64 61 35 39 64 38 63 + 62 36 34 35 39 61 39 39 37 31 34 33 34 62 31 64 + 35 38 38 62 61 65 23 63 64 3d 37 32 39 32 37 38 + 45 30 32 35 46 43 46 44 34 44 43 44 43 37 46 42 + 39 45 38 42 43 39 39 35 42 37 13 72 6d 3d 39 45 + 41 37 31 43 38 33 43 43 45 46 37 39 32 37 05 76 + 65 3d 30 35 0d 6d 64 3d 43 68 72 6f 6d 65 63 61 + 73 74 12 69 63 3d 2f 73 65 74 75 70 2f 69 63 6f + 6e 2e 70 6e 67 09 66 6e 3d 4b 69 62 62 6c 65 07 + 63 61 3d 34 31 30 31 04 73 74 3d 30 0f 62 73 3d + 46 41 38 46 43 41 39 33 42 35 43 34 04 6e 66 3d + 31 03 72 73 3d c0 2e 00 21 80 01 00 00 00 78 00 + 2d 00 00 00 00 1f 49 24 38 34 38 64 61 35 39 64 + 2d 38 63 62 36 2d 34 35 39 61 2d 39 39 37 31 2d + 34 33 34 62 31 64 35 38 38 62 61 65 c0 1d c1 2d + 00 01 80 01 00 00 00 78 00 04 c0 a8 56 40 + ]], 398)) + local response = MDNS.parse_packet(pkt) + assert(#response.answer_rrs == 1) + assert(#response.additional_rrs == 3) + end + local function parse_request () + local mDNSQuery = require("lib.protocol.dns.mdns_query").mDNSQuery + local requester = mDNSQuery.new({ + src_eth = "ce:6c:59:f2:f3:c1", + src_ipv4 = "192.168.0.1", + }) + local query = "_services._dns-sd._udp.local" + local request = MDNS.parse_packet(requester:build(query)) + assert(#request.questions == 1) + end + parse_response() + parse_request() +end diff --git a/src/lib/protocol/dns/mdns_query.lua b/src/lib/protocol/dns/mdns_query.lua new file mode 100644 index 0000000000..f7420de688 --- /dev/null +++ b/src/lib/protocol/dns/mdns_query.lua @@ -0,0 +1,108 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local datagram = require("lib.protocol.datagram") +local dns = require("lib.protocol.dns.dns") +local ethernet = require("lib.protocol.ethernet") +local ffi = require("ffi") +local ipv4 = require("lib.protocol.ipv4") +local lib = require("core.lib") +local mdns = require("lib.protocol.dns.mdns") +local udp = require("lib.protocol.udp") + +local MDNS = mdns.MDNS + +local query_record = dns.query_record + +local htons, ntohs = lib.htons, lib.ntohs + +local ETHER_PROTO_IPV4 = 0x0800 +local STANDARD_QUERY = 0x0 +local UDP_PROTOCOL = 0x11 + +mDNSQuery = {} + +function mDNSQuery.new (args) + local o = { + src_eth = assert(args.src_eth), + src_ipv4 = assert(args.src_ipv4), + } + return setmetatable(o, {__index=mDNSQuery}) +end + +function mDNSQuery:build (...) + local queries = assert({...}) + local dgram = datagram:new() + local ether_h = ethernet:new({dst = ethernet:pton(mdns.DST_ETHER), + src = ethernet:pton(self.src_eth), + type = ETHER_PROTO_IPV4}) + local ipv4_h = ipv4:new({dst = ipv4:pton(mdns.DST_IPV4), + src = ipv4:pton(self.src_ipv4), + protocol = UDP_PROTOCOL, + ttl = 255, + flags = 0x02}) + local udp_h = udp:new({src_port = 5353, + dst_port = mdns.DST_PORT}) + -- Add payload. + local payload, len = mDNSQuery:payload(queries) + -- Calculate checksums. + udp_h:length(udp_h:sizeof() + len) + udp_h:checksum(payload, len, ipv4_h) + ipv4_h:total_length(ipv4_h:sizeof() + udp_h:sizeof() + len) + ipv4_h:checksum() + -- Generate packet. + dgram:payload(payload, len) + dgram:push(udp_h) + dgram:push(ipv4_h) + dgram:push(ether_h) + return dgram:packet() +end + +function mDNSQuery:payload (queries) + local function w16 (buffer, val) + ffi.cast("uint16_t*", buffer)[0] = val + end + local function serialize (rr) + local ret = ffi.new("uint8_t[?]", rr:sizeof()) + local length = rr:sizeof() - 4 + local h = rr:header() + ffi.copy(ret, h.name, length) + w16(ret + length, h.type) + w16(ret + length + 2, h.class) + return ret, rr:sizeof() + end + local dgram = datagram:new() + local mdns_header = MDNS:new({ + id = 0, + flags = STANDARD_QUERY, + questions = #queries, + answer_rrs = 0, + authority_rrs = 0, + additional_rrs = 0, + }) + local t = {} + for _, each in ipairs(queries) do + local rr = query_record:new({ + name = each, + type = dns.PTR, + class = dns.CLASS_IN, + }) + -- TODO: dgram:push doesn't work. I think is due to the variable-length + -- nature of the header. + local data, len = serialize(rr) + dgram:push_raw(data, len) + end + dgram:push(mdns_header) + local pkt = dgram:packet() + return pkt.data, pkt.length +end + +function selftest() + local mdns_query = mDNSQuery.new({ + src_eth = "ce:6c:59:f2:f3:c1", + src_ipv4 = "192.168.0.1", + }) + local query = "_services._dns-sd._udp.local" + local pkt = assert(mdns_query:build(query)) + assert(pkt.length == 88) +end diff --git a/src/program/dnssd/README b/src/program/dnssd/README new file mode 100644 index 0000000000..e3a0495268 --- /dev/null +++ b/src/program/dnssd/README @@ -0,0 +1,17 @@ +Usage: snabb dnssd [OPTS] [arg] + +Options: + + -p|--pcap Read incoming packets from pcap file. + -i|--interface Read incoming packets from network interface. + +If no option is set, arg might be taken as pcap file or interface, depending on +whether a pcap file exists on current directory or not. + +Lists all devices and services available in a local network. Press Ctrl+C to stop program. + +Example: + +$ sudo ./snabb dnssd wlp3s0 +$ sudo ./snabb dnssd --interface wlp3s0 +$ sudo ./snabb dnssd --pcap packets.pcap diff --git a/src/program/dnssd/README.inc b/src/program/dnssd/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/dnssd/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/dnssd/README.md b/src/program/dnssd/README.md new file mode 100644 index 0000000000..16ecd757f0 --- /dev/null +++ b/src/program/dnssd/README.md @@ -0,0 +1,36 @@ +DNS-SD +------ + +Implement DNS Service Discovery. + +DNS-SD sends a local-link request to the broadcast address 224.0.0.251 port 5353. The default query is "_service.dns-sd._tcp.local", unless a different query is set in the argument list. If there are any devices or services listening to Multicast DNS request in the network, they will announce their domain-name to 224.0.0.251. The app listens to mDNS response and prints out A, PTR, SRV and TXT records. + +Example: + +``` +$ sudo ./snabb dnssd --interface wlan0 +Capturing packets from interface 'wlan0' +PTR: (name: _services._dns-sd._udp.local; domain-name: _googlecast._tcp) +PTR: (name: _services._dns-sd._udp.local; domain-name: _googlezone._tcp) +PTR: (name: _services._dns-sd._udp.local; domain-name: _spotify-connect._tcp) +``` + +Further information of _googlecast._tcp.local: + +``` +$ sudo ./snabb dnssd --interface wlan0 _googlecast._tcp.local +Capturing packets from interface 'wlan0' +Capturing packets from interface 'wlp3s0' +{name: _googlecast._tcp.local; domain_name: Google-Home-65b8d37105107f33691010baf74b84102f103363} +{id=65b8d37105107f33691010baf74b84102f103363;cd=b51a5e1cd4953a7b9f2f49622fdaf97b;rm=104e6110afdaf491f5;ve=05;md=Google Home;ic=/setup/icon.png;fn=Home;ca=2052;st=0;bs=5d3101063a3cdb;nf=1;rs=} +{target: eb9910bed-52310-94a3-b371-c6f3bf10b19e2; port: 8009} +{address: 192.168.86.61} +{name: _googlecast._tcp.local; domain_name: Chromecast-Audio-7cc91fd53d3c64e425b3b86a5107c11074} +{id=7cc91fd53d3c64e425b3b86a5107c11074;cd=224708C2E61AED24676383796588FF7E;rm=8F2EE2757C6626CC;ve=05;md=Chromecast Audio;ic=/setup/icon.png;fn=Jukebox;ca=2052;st=0;bs=4f4105104dcf5a;nf=1;rs=} +{target: 4742e2a5-a6bd-e137-2fa3-1215425bf2f6; port: 8009} +{address: 192.168.86.57} +{name: _googlecast._tcp.local; domain_name: Google-Cast-Group-63419dcd2372412882ac2762a2c58706} +{id=3410642cb-aad1-210210-f148-910fbdf3cdfa2;cd=3410642cb-aad1-210210-f148-910fbdf3cdfa2;rm=8F2EE2757C6626CC;ve=05;md=Google Cast Group;ic=/setup/icon.png;fn=Batcave;ca=2084;st=0;bs=4f4105104dcf5a;nf=1;rs=} +{target: 4742e2a5-a6bd-e137-2fa3-1215425bf2f6; port: 42238} +{address: 192.168.86.57} +``` diff --git a/src/program/dnssd/dnssd.lua b/src/program/dnssd/dnssd.lua new file mode 100644 index 0000000000..3876dccd6e --- /dev/null +++ b/src/program/dnssd/dnssd.lua @@ -0,0 +1,168 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local DNS = require("lib.protocol.dns.dns").DNS +local MDNS = require("lib.protocol.dns.mdns").MDNS +local RawSocket = require("apps.socket.raw").RawSocket +local basic_apps = require("apps.basic.basic_apps") +local ffi = require("ffi") +local lib = require("core.lib") +local mDNSQuery = require("lib.protocol.dns.mdns_query").mDNSQuery +local pcap = require("apps.pcap.pcap") + +local long_opts = { + help = "h", + pcap = "p", + interface = "i", +} + +local function usage(exit_code) + print(require("program.dnssd.README_inc")) + main.exit(exit_code) +end + +function parse_args (args) + local function fexists (filename) + local fd = io.open(filename, "r") + if fd then + fd:close() + return true + end + return false + end + local opts = {} + local handlers = {} + function handlers.h (arg) + usage(0) + end + function handlers.p (arg) + opts.pcap = arg + end + function handlers.i (arg) + opts.interface = arg + end + args = lib.dogetopt(args, handlers, "hp:i:", long_opts) + if not (opts.pcap or opts.interface) then + local filename = args[1] + if fexists(filename) then + opts.pcap = filename + else + opts.interface = filename + end + table.remove(args, 1) + end + return opts, args +end + +DNSSD = {} + +function DNSSD:new (args) + local o = { + interval = 1, -- Delay between broadcast messages. + threshold = 0, + } + if args then + o.requester = mDNSQuery.new({ + src_eth = assert(args.src_eth), + src_ipv4 = assert(args.src_ipv4), + }) + o.query = args.query or "_services._dns-sd._udp.local" + end + return setmetatable(o, {__index = DNSSD}) +end + +-- Generate a new broadcast mDNS packet every interval seconds. +function DNSSD:pull () + local output = self.output.output + if not output then return end + + local now = os.time() + if now > self.threshold then + self.threshold = now + self.interval + local pkt = self.requester:build(self.query) + link.transmit(output, pkt) + end +end + +function DNSSD:push () + local input = assert(self.input.input) + + while not link.empty(input) do + local pkt = link.receive(input) + if MDNS.is_mdns(pkt) then + self:log(pkt) + end + packet.free(pkt) + end +end + +function DNSSD:log (pkt) + if not MDNS.is_response(pkt) then return end + local response = MDNS.parse_packet(pkt) + local answer_rrs = response.answer_rrs + if #answer_rrs > 0 then + for _, rr in ipairs(answer_rrs) do + print(rr:tostring()) + end + end + local additional_rrs = response.additional_rrs + if #additional_rrs > 0 then + for _, rr in ipairs(additional_rrs) do + print(rr:tostring()) + end + end +end + +local function execute (cmd) + local fd = assert(io.popen(cmd, 'r')) + local ret = fd:read("*all") + fd:close() + return ret +end + +local function chomp (str) + return str:gsub("\n", "") +end + +local function ethernet_address_of (iface) + local cmd = ("ip li sh %s | grep 'link/ether' | awk '{print $2}'"):format(iface) + return chomp(execute(cmd)) +end + +local function ipv4_address_of (iface) + local cmd = ("ip addr sh %s | grep 'inet ' | awk '{print $2}'"):format(iface) + local output = chomp(execute(cmd)) + local pos = output:find("/") + return pos and output:sub(0, pos-1) or output +end + +function run(args) + local opts, args = parse_args(args) + + local duration + local c = config.new() + if opts.pcap then + print("Reading from file: "..opts.pcap) + config.app(c, "dnssd", DNSSD) + config.app(c, "pcap", pcap.PcapReader, opts.pcap) + config.link(c, "pcap.output-> dnssd.input") + duration = 3 + elseif opts.interface then + local iface = opts.interface + local query = args[1] + print(("Capturing packets from interface '%s'"):format(iface)) + config.app(c, "dnssd", DNSSD, { + src_eth = ethernet_address_of(iface), + src_ipv4 = ipv4_address_of(iface), + query = query, + }) + config.app(c, "iface", RawSocket, iface) + config.link(c, "iface.tx -> dnssd.input") + config.link(c, "dnssd.output -> iface.rx") + else + error("Unreachable") + end + engine.busy = false + engine.configure(c) + engine.main({duration = duration, report = {showapps = true, showlinks = true}}) +end From 468e72c50b86e60e458225a27ae2f0b34b80738f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 7 Feb 2018 16:36:18 +0100 Subject: [PATCH 025/100] Fix small mistake in arp documentation. --- src/apps/ipv4/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/ipv4/README.md b/src/apps/ipv4/README.md index 97b4914267..9978ce436f 100644 --- a/src/apps/ipv4/README.md +++ b/src/apps/ipv4/README.md @@ -53,7 +53,7 @@ traffic will go to a single MAC address. If this address is provided as part of the configuration, no ARP request will be made; otherwise it will be determined from the *next_ip* via ARP. -— Key **self_ip** +— Key **next_ip** *Optional*. The IPv4 address of the next-hop host. Required only if *next_mac* is not specified as part of the configuration. From 8ce88727426ef9cc230b46e2184a4ab7b5ca3041 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Feb 2018 17:36:04 +0100 Subject: [PATCH 026/100] Add "snabb unhexdump" tool See README for usage. --- src/lib/pcap/pcap.lua | 15 +++-- src/program/unhexdump/README | 35 +++++++++++ src/program/unhexdump/README.inc | 1 + src/program/unhexdump/unhexdump.lua | 91 +++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 src/program/unhexdump/README create mode 120000 src/program/unhexdump/README.inc create mode 100644 src/program/unhexdump/unhexdump.lua diff --git a/src/lib/pcap/pcap.lua b/src/lib/pcap/pcap.lua index f0bcb5c37c..c0f890def7 100644 --- a/src/lib/pcap/pcap.lua +++ b/src/lib/pcap/pcap.lua @@ -28,7 +28,7 @@ struct { } ]] -function write_file_header(file) +function write_file_header (file) local pcap_file = ffi.new(pcap_file_t) pcap_file.magic_number = 0xa1b2c3d4 pcap_file.version_major = 2 @@ -52,16 +52,21 @@ function write_record_header (file, length) file:write(ffi.string(pcap_record, ffi.sizeof(pcap_record))) end --- Return an iterator for pcap records in FILENAME. -function records (filename) - local file = io.open(filename, "r") - if file == nil then error("Unable to open file: " .. filename) end +function read_file_header (file) local pcap_file = readc(file, pcap_file_t) if pcap_file.magic_number == 0xD4C3B2A1 then error("Endian mismatch in " .. filename) elseif pcap_file.magic_number ~= 0xA1B2C3D4 then error("Bad PCAP magic number in " .. filename) end + return pcap_file +end + +-- Return an iterator for pcap records in FILENAME. +function records (filename) + local file = io.open(filename, "r") + if file == nil then error("Unable to open file: " .. filename) end + read_file_header(file) local function pcap_records_it (t, i) local record = readc(file, pcap_record_t) if record == nil then return nil end diff --git a/src/program/unhexdump/README b/src/program/unhexdump/README new file mode 100644 index 0000000000..4d9bc8d541 --- /dev/null +++ b/src/program/unhexdump/README @@ -0,0 +1,35 @@ +Usage: + unhexdump [-hta] FILE.PCAP + + -h, --help + Print usage information + -t, --truncate + Truncate the output file if it already exists + -a, --append + Append packets to the output file if it already exists + +Read "hexdump" packet representations from stdin and write out binary +packets to a pcap savefile. Useful when you need to get some data to +wireshark but you only have it in text format for some reason. + +Blank lines delimit packets. Packet data is a non-empty sequence of hex +pairs. Hex pairs may be separated by sequences of whitespace, +punctuation, 'x' or 'X' characters. + +Example: + + $ snabb unhexdump foo.pcap < Date: Fri, 9 Feb 2018 19:57:05 +0100 Subject: [PATCH 027/100] intel_app: run selftest even though its not the default driver anymore --- src/apps/intel/intel_app.lua | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index 950dc514ac..de835e8ed6 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -252,13 +252,9 @@ end function selftest () print("selftest: intel_app") - local pcideva = lib.getenv("SNABB_PCI_INTEL0") or lib.getenv("SNABB_PCI0") - local pcidevb = lib.getenv("SNABB_PCI_INTEL1") or lib.getenv("SNABB_PCI1") - if not pcideva - or pci.device_info(pcideva).driver ~= 'apps.intel.intel_app' - or not pcidevb - or pci.device_info(pcidevb).driver ~= 'apps.intel.intel_app' - then + local pcideva = lib.getenv("SNABB_PCI_INTEL0") + local pcidevb = lib.getenv("SNABB_PCI_INTEL1") + if not pcideva or not pcidevb then print("SNABB_PCI_INTEL[0|1]/SNABB_PCI[0|1] not set or not suitable.") os.exit(engine.test_skipped_code) end From 383fd83aa2f54506047e291749d7e703294ccb44 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 12 Feb 2018 15:24:38 +0100 Subject: [PATCH 028/100] Process tree runs data-plane processes with busywait=true by default --- src/lib/ptree/ptree.lua | 2 +- src/lib/scheduling.lua | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index b8c100f1e7..41ea1119a7 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -40,7 +40,7 @@ local manager_config_spec = { -- Could relax this requirement. initial_configuration = {required=true}, schema_name = {required=true}, - worker_default_scheduling = {default={busywait=true}}, + worker_default_scheduling = {default={}}, default_schema = {}, log_level = {default=default_log_level}, rpc_trace_file = {}, diff --git a/src/lib/scheduling.lua b/src/lib/scheduling.lua index 66b43638b8..b53bfb9f99 100644 --- a/src/lib/scheduling.lua +++ b/src/lib/scheduling.lua @@ -16,7 +16,7 @@ local scheduling_opts = { cpu = {}, -- CPU index (integer). real_time = {}, -- Boolean. ingress_drop_monitor = {}, -- Action string: one of 'flush' or 'warn'. - busywait = {}, -- Boolean. + busywait = {default=true}, -- Boolean. j = {}, -- Profiling argument string, e.g. "p" or "v". eval = {} -- String. } @@ -99,7 +99,7 @@ end function selftest () print('selftest: lib.scheduling') loadstring(stage({}))() - loadstring(stage({busywait=true}))() + loadstring(stage({busywait=false}))() loadstring(stage({eval='print("lib.scheduling: eval test")'}))() print('selftest: ok') end From c6d0e2296f1e90438b42be052aaee14ab18c62ff Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 07:22:29 +0000 Subject: [PATCH 029/100] Pcap savefile writer records frame sizes Since the pcap savefile writer marks its frames as being ethernet frames, we should record the frame size in the written records, including the 4-byte CRC that we don't see. That way we can tell in wireshark when we have 64-byte frames. --- src/lib/pcap/pcap.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/pcap/pcap.lua b/src/lib/pcap/pcap.lua index f0bcb5c37c..68f3efe931 100644 --- a/src/lib/pcap/pcap.lua +++ b/src/lib/pcap/pcap.lua @@ -28,13 +28,15 @@ struct { } ]] +local EN10MB = 1 + function write_file_header(file) local pcap_file = ffi.new(pcap_file_t) pcap_file.magic_number = 0xa1b2c3d4 pcap_file.version_major = 2 pcap_file.version_minor = 4 pcap_file.snaplen = 65535 - pcap_file.network = 1 + pcap_file.network = EN10MB file:write(ffi.string(pcap_file, ffi.sizeof(pcap_file))) file:flush() end @@ -45,10 +47,14 @@ function write_record (file, ffi_buffer, length) file:flush() end +local en10mb_crc_size = 4 + function write_record_header (file, length) local pcap_record = ffi.new(pcap_record_t) pcap_record.incl_len = length - pcap_record.orig_len = length + -- Since we mark our saved packets as having EN10MB encapsulation, + -- add the CRC size on to the frame size. + pcap_record.orig_len = length + en10mb_crc_size file:write(ffi.string(pcap_record, ffi.sizeof(pcap_record))) end From 3cf4cdaf4266b909f32e3e04a8e9326233af97de Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 07:24:12 +0000 Subject: [PATCH 030/100] Sizes for "packetblaster lwaftr" are frame sizes This change makes it so that the sizes passed to "packetblaster lwaftr" are frame sizes instead of packet sizes without the ethernet headers. This seems to better correspond to the intent of including 64 as a default size. Often you want to generate test traffic of the minimum frame size, and this makes that goal more attainable. Additionally this commit fixes a bug whereby the frame size was longer than the actual packet data, because the length in the UDP headers was wrong. --- src/program/packetblaster/lwaftr/README | 79 ++++++++++++--------- src/program/packetblaster/lwaftr/lib.lua | 53 ++++++++------ src/program/packetblaster/lwaftr/lwaftr.lua | 16 ++--- 3 files changed, 85 insertions(+), 63 deletions(-) diff --git a/src/program/packetblaster/lwaftr/README b/src/program/packetblaster/lwaftr/README index 3ae59a2058..4c340c98c7 100644 --- a/src/program/packetblaster/lwaftr/README +++ b/src/program/packetblaster/lwaftr/README @@ -16,54 +16,69 @@ Usage: packetblaster lwaftr [OPTIONS] --vlan VLANID VLAN tag traffic with VLANID if set - --src_mac SOURCE - Source MAC-Address + --src_mac SOURCE Source MAC-Address Default: 00:00:00:00:00:00 - --dst_mac DESTINATION - Destination MAC-Address + + --dst_mac DESTINATION Destination MAC-Address Default: 00:00:00:00:00:00 - --size SIZES - A comma separated list of numbers. Send packets of - SIZES bytes. The size specifies the lenght of the IPv4 - packet. The actual packet size on the wire is 14 Bytes - longer (Ethernet header). Smallest allowed IPv4 packet - size is 28 (20 Bytes for IPv4 header, 8 Bytes payload - for packet loss detection) - Default: 64,64,64,64,64,64,64,594,594,594,1500 (IMIX) - --b4 IPV6,IPV4,PORT - First B4 Client IPv6 mapped to IPv4 and UDP Port. + + --size SIZES A comma separated list of numbers. Send packets whose + frames are SIZES bytes long. The frame size includes + the size of the packet, including its ethernet + headers, and additionally a 4-byte CRC that is written + and read by the NIC. + + Note that the minimum ethernet frame size is 64 bytes. + While it's technically possible to make smaller frames + and we do allow it, the NIC will pad it up to the + minimum before sending, so it's a bit pointless. + Since Snabb does not see the CRC in the packet, that + means that from Snabb's perspective the minimum useful + packet size is 60 bytes. + + The smallest allowed frame size is 46 bytes, + comprising 14 bytes for the ethernet header, 20 for + the IPv4 header, 8 for the UDP header, and 4 + additional bytes for the ethernet checksum. If the + packet has at least 8 bytes of payload, the generated + packets will include a unique identifier in the + payload as well. + + Default: 64,64,64,64,64,64,64,594,594,594,1500 + + --b4 IPV6,IPV4,PORT First B4 Client IPv6 mapped to IPv4 and UDP Port. B4 IPv6,IPv4 and Port are incremented for every count, then rolled over. Port is incremented by the port number: e.g. 1024 -> 1024, 2048, 3096 .. 64512 (63 in total) Default: 2001:db8::,10.0.0.0,1024 - --aftr IPV6 - IPv6 address of lwaftr server (softwire tunnel endpoint) + + --aftr IPV6 IPv6 address of lwaftr server (softwire tunnel endpoint) Default: 2001:db8:ffff::100 - --ipv4 IPV4 - Public IPv4. Used as source for IPv4 traffic and + + --ipv4 IPV4 Public IPv4. Used as source for IPv4 traffic and as destination in IPv6 packets from B4 Default: 8.8.8.8 - --count COUNT - Number of B4 clients to simulate. + + --count COUNT Number of B4 clients to simulate. Default: 1 - --rate RATE - Rate in MPPS for the generated traffic. Fractions are + + --rate RATE Rate in MPPS for the generated traffic. Fractions are allowed (e.g. 3.148 for IMIX line rate). If set too high, the actual transmitted rate depends on the interfaces capacity. Setting rate to 0 turns it to listening only mode while reporting on incoming packets Default: 1 MPPS - --v4only, -4 - Generate only IPv4 packets from the Internet towards lwaftr - --v6only, -6 - Generate only IPv6 packets from B4 to lwaftr - --duration DURATION - Run for DURATION seconds. + + --v4only, -4 Generate only IPv4 packets from the Internet towards lwaftr + + --v6only, -6 Generate only IPv6 packets from B4 to lwaftr + + --duration DURATION Run for DURATION seconds. Default: unlimited - -V, --verbose - Display verbose link information every second - -h, --help - Print usage information. + + -V, --verbose Display verbose link information every second + + -h, --help Print usage information. This tool generates two types of traffic according to RFC7596: diff --git a/src/program/packetblaster/lwaftr/lib.lua b/src/program/packetblaster/lwaftr/lib.lua index d52039119d..1b617f80bf 100644 --- a/src/program/packetblaster/lwaftr/lib.lua +++ b/src/program/packetblaster/lwaftr/lib.lua @@ -56,6 +56,7 @@ struct { uint8_t dst_ip[4]; } __attribute__((packed)) ]] +local ipv4_header_size = ffi.sizeof(ipv4hdr_t) local ipv4_header_ptr_type = ffi.typeof("$*", ipv4hdr_t) local ipv6_ptr_type = ffi.typeof([[ @@ -367,37 +368,45 @@ function Lwaftrgen:pull () ipv4_udp_hdr.dst_port = C.htons(self.current_port) ipv6_ipv4_udp_hdr.src_port = C.htons(self.current_port) - for _,size in ipairs(self.sizes) do + -- The sizes are frame sizes, including the 4-byte ethernet CRC + -- that we don't see in Snabb. + + local vlan_size = self.vlan and ether_vlan_header_size or 0 + local ethernet_crc_size = 4 + local ethernet_total_size = ethernet_header_size + vlan_size + local minimum_size = ethernet_total_size + ipv4_header_size + + udp_header_size + ethernet_crc_size + for _,size in ipairs(self.sizes) do + assert(size >= minimum_size) + local packet_len = size - ethernet_crc_size + local ipv4_len = packet_len - ethernet_total_size + local udp_len = ipv4_len - ipv4_header_size if not self.ipv6_only then - ipv4_hdr.total_length = C.htons(size) - if self.vlan then - ipv4_udp_hdr.len = C.htons(size - 28 + 4) - self.ipv4_pkt.length = size + ethernet_header_size + 4 - else - ipv4_udp_hdr.len = C.htons(size - 28) - self.ipv4_pkt.length = size + ethernet_header_size - end + ipv4_hdr.total_length = C.htons(ipv4_len) + ipv4_udp_hdr.len = C.htons(udp_len) + self.ipv4_pkt.length = packet_len ipv4_hdr.checksum = 0 - ipv4_hdr.checksum = C.htons(ipsum(self.ipv4_pkt.data + ethernet_header_size, 20, 0)) - self.ipv4_payload.number = self.ipv4_packet_number; - self.ipv4_packet_number = self.ipv4_packet_number + 1 + ipv4_hdr.checksum = C.htons(ipsum(self.ipv4_pkt.data + ethernet_total_size, 20, 0)) + if size >= minimum_size + payload_size then + self.ipv4_payload.number = self.ipv4_packet_number; + self.ipv4_packet_number = self.ipv4_packet_number + 1 + end local ipv4_pkt = packet.clone(self.ipv4_pkt) transmit(output, ipv4_pkt) end if not self.ipv4_only then - ipv6_hdr.payload_length = C.htons(size) - ipv6_ipv4_hdr.total_length = C.htons(size) - if self.vlan then - ipv6_ipv4_udp_hdr.len = C.htons(size - 28 + 4) - self.ipv6_pkt.length = size + 54 + 4 - else - ipv6_ipv4_udp_hdr.len = C.htons(size - 28) - self.ipv6_pkt.length = size + 54 + -- Expectation from callers is to make packets that are SIZE + -- bytes big, *plus* the IPv6 header. + ipv6_hdr.payload_length = C.htons(ipv4_len) + ipv6_ipv4_hdr.total_length = C.htons(ipv4_len) + ipv6_ipv4_udp_hdr.len = C.htons(udp_len) + self.ipv6_pkt.length = packet_len + ipv6_header_size + if size >= minimum_size + payload_size then + self.ipv6_payload.number = self.ipv6_packet_number; + self.ipv6_packet_number = self.ipv6_packet_number + 1 end - self.ipv6_payload.number = self.ipv6_packet_number; - self.ipv6_packet_number = self.ipv6_packet_number + 1 local ipv6_pkt = packet.clone(self.ipv6_pkt) transmit(output, ipv6_pkt) end diff --git a/src/program/packetblaster/lwaftr/lwaftr.lua b/src/program/packetblaster/lwaftr/lwaftr.lua index 2d85469be3..b65b21848e 100644 --- a/src/program/packetblaster/lwaftr/lwaftr.lua +++ b/src/program/packetblaster/lwaftr/lwaftr.lua @@ -25,7 +25,7 @@ local long_opts = { duration = "D", -- terminate after n seconds verbose = "V", -- verbose, display stats help = "h", -- display help text - size = "S", -- packet size list (defaults to IMIX) + size = "S", -- frame size list (defaults to IMIX) src_mac = "s", -- source ethernet address dst_mac = "d", -- destination ethernet address vlan = "v", -- VLAN id @@ -64,18 +64,14 @@ function run (args) end local sizes = { 64, 64, 64, 64, 64, 64, 64, 594, 594, 594, 1500 } - local sizes_ipv6 = { 104, 104, 104, 104, 104, 104, 104, 634, 634, 634, 1540 } function opt.S (arg) sizes = {} - sizes_ipv6 = {} for size in string.gmatch(arg, "%d+") do local s = tonumber(size) - if s < 28 then - s = 28 - print("Warning: Increasing IPv4 packet size to 28") + if s < 18 + 20 + 8 then + error("Minimum frame size is 46 bytes (18 ethernet+CRC, 20 IPv4, and 8 UDP)") end sizes[#sizes+1] = s - sizes_ipv6[#sizes_ipv6+1] = s + 40 end end @@ -172,14 +168,16 @@ function run (args) if not ipv4_only then print(string.format("IPv6: %s > %s: %s:%d > %s:12345", b4_ipv6, aftr_ipv6, b4_ipv4, b4_port, public_ipv4)) print(" source IPv6 and source IPv4/Port adjusted per client") - print("IPv6 packet sizes: " .. table.concat(sizes_ipv6,",")) + local sizes_ipv6 = {} + for i,size in ipairs(sizes) do sizes_ipv6[i] = size + 40 end + print("IPv6 frame sizes: " .. table.concat(sizes_ipv6,",")) end if not ipv6_only then print() print(string.format("IPv4: %s:12345 > %s:%d", public_ipv4, b4_ipv4, b4_port)) print(" destination IPv4 and Port adjusted per client") - print("IPv4 packet sizes: " .. table.concat(sizes,",")) + print("IPv4 frame sizes: " .. table.concat(sizes,",")) end if ipv4_only and ipv6_only then From c55199b2f5fa996c416fe6d053bfded96d277adb Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 07:52:51 +0000 Subject: [PATCH 031/100] Fix up packetblaster lwaftr selfchecks --- src/program/packetblaster/lwaftr/lwaftr.lua | 12 +++++++----- .../packetblaster/lwaftr/test_lwaftr_1.pcap | Bin 17144 -> 8188 bytes .../packetblaster/lwaftr/test_lwaftr_2.pcap | Bin 352 -> 352 bytes src/program/packetblaster/selftest.sh | 3 ++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/program/packetblaster/lwaftr/lwaftr.lua b/src/program/packetblaster/lwaftr/lwaftr.lua index b65b21848e..e9c5deeb97 100644 --- a/src/program/packetblaster/lwaftr/lwaftr.lua +++ b/src/program/packetblaster/lwaftr/lwaftr.lua @@ -67,11 +67,7 @@ function run (args) function opt.S (arg) sizes = {} for size in string.gmatch(arg, "%d+") do - local s = tonumber(size) - if s < 18 + 20 + 8 then - error("Minimum frame size is 46 bytes (18 ethernet+CRC, 20 IPv4, and 8 UDP)") - end - sizes[#sizes+1] = s + sizes[#sizes + 1] = assert(tonumber(size), "size not a number: "..size) end end @@ -157,6 +153,12 @@ function run (args) args = lib.dogetopt(args, opt, "VD:hS:s:a:d:b:iI:c:r:46p:v:o:t:i:k:", long_opts) + for _,s in ipairs(sizes) do + if s < 18 + (vlan and 4 or 0) + 20 + 8 then + error("Minimum frame size is 46 bytes (18 ethernet+CRC, 20 IPv4, and 8 UDP)") + end + end + if not target then print("either --pci, --tap, --sock, --int or --pcap are required parameters") main.exit(1) diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap index 90b0622b618d0393d82fbb810b80e142a04bb200..584586d084a8963b0004bc8c47583fd837653441 100644 GIT binary patch literal 8188 zcmca|c+)~A1{MYw`2U}Qff2?5(l$WsfW!weIT&0S81z6ae!+XetDS$|xW<1jsAnIEXcVRNaUU z0os;vqp5I2CoZBrPhJ_vNwoQ+3P*AX(6)>lO@$*lZ4vMH8^A939kgyZsISU;3SI2kKqtW_c TNW}f9BS%AEGz3ON02l%Q)WX?L literal 17144 zcmeHNF;2rU6m>%bg;GKa>jkoORgvC;R@*DU@v?i+yMow0&(LY{1Q&4Q$QC7cXfl|uGT9MHx9xt zTkl{>0X}V#fC@KyFHYywlI636zZa)3<$Ez@IAKr#3P1rUu$c-BZR(T?w=zSO&LtTt zH$&7jRG1;So~T0sC;$aEQ-P50g2n%Si@uO-(09SUWWPnh_FG6%=s4~-f}<3Lj^m&a z9Hl6991j}7QHnyxF>3@zDGD7&+N341xG_hHLdP+e5VU>ba+IP7$*&j2eb*ieIiedB zs4EbZU-zsgmiJrGY65;)R~cua3h3hCQRpz}4+VY|h#LptmvA$k0=hVOtQ!n>wO)a^ zaS(pldIwVq5VGH*Fz##GiE=dD1_hu16o3NjS0HG=g(pL`yx)Q{RG1;w|IY#e6o3Ly HK&!wH$o9QH diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap index cefd6f43925aac049652fd72435360fd025c03d5..9c303f834c565032af264eda16a0bf3f232d227d 100644 GIT binary patch literal 352 zcmca|c+)~A1{MYw`2U}Qff2?5(t1E_gv19i8yOf^e!+={IDmi)$TqM9 zn#2JTg@7<16$Rr!X^`%=y9ok7t_;h61xDT-$O=GwkQ*R?5y=(?29OOPn^=JMAlpK! R8-Wg_#Ep!^xRC?uMgYc8A6)vbh;L5-t0}|yIoE*mi1YAJ2fhEu+ zgkcO}Krs+T1|Z#ScM}AFVlpiM6&QJUAj^XIAU8k&Ba$r)3?LgoHn9NhLAHfdH*!$s MMn+=X$N_aD02e?Wc>n+a diff --git a/src/program/packetblaster/selftest.sh b/src/program/packetblaster/selftest.sh index 1a2ca09f32..aa6a1eaf78 100755 --- a/src/program/packetblaster/selftest.sh +++ b/src/program/packetblaster/selftest.sh @@ -17,6 +17,7 @@ function test_lwaftr_pcap { exit 1 fi cmp $TEMP_PCAP $PCAP + status=$? rm $TEMP_PCAP if [ $status != 0 ]; then echo "Error: lwaftr generated pcap differs from ${PCAP}" @@ -25,7 +26,7 @@ function test_lwaftr_pcap { } test_lwaftr_pcap program/packetblaster/lwaftr/test_lwaftr_1.pcap --count 1 -test_lwaftr_pcap program/packetblaster/lwaftr/test_lwaftr_2.pcap --count 2 --vlan 100 --size 0 +test_lwaftr_pcap program/packetblaster/lwaftr/test_lwaftr_2.pcap --count 2 --vlan 100 --size 50 # lwaftr tap test sudo ip netns add snabbtest || exit $TEST_SKIPPED From fe08efb5d0c58e8932ad091f4d5a9bda966e352e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 07:54:48 +0000 Subject: [PATCH 032/100] Address review nit --- src/program/packetblaster/lwaftr/lib.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/program/packetblaster/lwaftr/lib.lua b/src/program/packetblaster/lwaftr/lib.lua index 1b617f80bf..9960fa9783 100644 --- a/src/program/packetblaster/lwaftr/lib.lua +++ b/src/program/packetblaster/lwaftr/lib.lua @@ -33,6 +33,11 @@ local ether_header_ptr_type = ffi.typeof("$*", ether_header_t) local ethernet_header_size = ffi.sizeof(ether_header_t) local OFFSET_ETHERTYPE = 12 +-- The ethernet CRC field is not included in the packet as seen by +-- Snabb, but it is part of the frame and therefore a contributor to the +-- frame size. +local ethernet_crc_size = 4 + local ether_vlan_header_type = ffi.typeof([[ struct { uint16_t tag; @@ -372,7 +377,6 @@ function Lwaftrgen:pull () -- that we don't see in Snabb. local vlan_size = self.vlan and ether_vlan_header_size or 0 - local ethernet_crc_size = 4 local ethernet_total_size = ethernet_header_size + vlan_size local minimum_size = ethernet_total_size + ipv4_header_size + udp_header_size + ethernet_crc_size From 2876c6e7b061a7f7518d072300c54c880d8c6cb1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 08:07:22 +0000 Subject: [PATCH 033/100] Revert "Pcap savefile writer records frame sizes" This reverts commit c6d0e2296f1e90438b42be052aaee14ab18c62ff. It's a good change but we would need to update all savefiles in tree to update test expectations. --- src/lib/pcap/pcap.lua | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/lib/pcap/pcap.lua b/src/lib/pcap/pcap.lua index 68f3efe931..f0bcb5c37c 100644 --- a/src/lib/pcap/pcap.lua +++ b/src/lib/pcap/pcap.lua @@ -28,15 +28,13 @@ struct { } ]] -local EN10MB = 1 - function write_file_header(file) local pcap_file = ffi.new(pcap_file_t) pcap_file.magic_number = 0xa1b2c3d4 pcap_file.version_major = 2 pcap_file.version_minor = 4 pcap_file.snaplen = 65535 - pcap_file.network = EN10MB + pcap_file.network = 1 file:write(ffi.string(pcap_file, ffi.sizeof(pcap_file))) file:flush() end @@ -47,14 +45,10 @@ function write_record (file, ffi_buffer, length) file:flush() end -local en10mb_crc_size = 4 - function write_record_header (file, length) local pcap_record = ffi.new(pcap_record_t) pcap_record.incl_len = length - -- Since we mark our saved packets as having EN10MB encapsulation, - -- add the CRC size on to the frame size. - pcap_record.orig_len = length + en10mb_crc_size + pcap_record.orig_len = length file:write(ffi.string(pcap_record, ffi.sizeof(pcap_record))) end From 75ee99030940347c8064b41fb79ab30590dbf302 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 08:08:50 +0000 Subject: [PATCH 034/100] Regenerate packetblaster lwaftr test files --- .../packetblaster/lwaftr/test_lwaftr_1.pcap | Bin 8188 -> 8584 bytes .../packetblaster/lwaftr/test_lwaftr_2.pcap | Bin 352 -> 440 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap index 584586d084a8963b0004bc8c47583fd837653441..10ac316bf68206013546e1db0401380734b9e531 100644 GIT binary patch literal 8584 zcmeHLJ4ysW5Upuv+)Pp)k6_?f z#n?ShF2- z`{Iha`CDOFZpwGqCTYoN=3MUK-KmjNK1$I+`GcKKqYwIgi)!I^_iYvrsg$gT# P))Q?g00p4HZYl5wMyK9( literal 8188 zcmca|c+)~A1{MYw`2U}Qff2?5(l$WsfW!weIT&0S81z6ae!+XetDS$|xW<1jsAnIEXcVRNaUU z0os;vqp5I2CoZBrPhJ_vNwoQ+3P*AX(6)>lO@$*lZ4vMH8^A939kgyZsISU;3SI2kKqtW_c TNW}f9BS%AEGz3ON02l%Q)WX?L diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap index 9c303f834c565032af264eda16a0bf3f232d227d..545c5e109d1742bada82cbe535eefc6ed54efe73 100644 GIT binary patch literal 440 zcmca|c+)~A1{MYw`2U}Qff2?5(k@VpB*f6jz>vbh;L5;Y1QO*Job15?1YAJ2fhEu+ zDF%jfzy7U9)`7~-0dheY8Gy`eyPF^Y6f^e!+={IDmi)$TqM9 zn#2JTg@7<16$Rr!X^`%=y9ok7t_;h61xDT-$O=GwkQ*R?5y=(?29OOPn^=JMAlpK! R8-Wg_#Ep!^xRC?uMgYc8A6) Date: Tue, 13 Feb 2018 08:21:07 +0000 Subject: [PATCH 035/100] Regenerate packetblaster lwaftr test files --- .../packetblaster/lwaftr/test_lwaftr_1.pcap | Bin 8584 -> 8188 bytes .../packetblaster/lwaftr/test_lwaftr_2.pcap | Bin 440 -> 352 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_1.pcap index 10ac316bf68206013546e1db0401380734b9e531..45f85921c676b8caebd4fa6d5495851a9de0ee98 100644 GIT binary patch literal 8188 zcmeHLu}TCn5S@#=x(c!`SXuc2a@YzM+t~UI7Jh@DU?qq@AovA}h1lC&VP#?Ck65_Q zBQBYfpdpar*f%iTB%8~7`)+PR=I!P2&PgSs4TH#R9O&oqkDDoJDHSEu}pY0Qj!N%LnV2S__xRs**jhV;aDljte z+{^KgW9IR9fwYOr>L16<oL=I*SEHQx&vQ7H}H@~1^!XsLNg0)&7Y{QPdQOtk8if!@`=hU zgetUD1@dZKle#_EOEud&`MespnY=$2PFaDnHI7+0<=jet_jkR5!>!@2 zS}!ouc2~Rhu3A6&yDEBdG%7#^r~nmM2?fxq{MD}XU88!D(lr{-gXOLf&x4it?Madf KPys5S3VZ`EBG~o- literal 8584 zcmeHLJ4ysW5Upuv+)Pp)k6_?f z#n?ShF2- z`{Iha`CDOFZpwGqCTYoN=3MUK-KmjNK1$I+`GcKKqYwIgi)!I^_iYvrsg$gT# P))Q?g00p4HZYl5wMyK9( diff --git a/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap b/src/program/packetblaster/lwaftr/test_lwaftr_2.pcap index 545c5e109d1742bada82cbe535eefc6ed54efe73..287dd1b8fb6149e1ab98e75e4f84df980ec3e851 100644 GIT binary patch literal 352 zcmca|c+)~A1{MYw`2U}Qff2?5(t1#gB*f6jz>vbh;L5-t0}|yIoOp->2)KZ3152Pu z90;{xKqd$y1CZ{vy9ok7F&UQs3XHrvkYz!9kQ*R?5y=(?29OOPn^=JMAlpK!8-Wg_ N#Ep!^xRC?uMgX~>A58!N literal 440 zcmca|c+)~A1{MYw`2U}Qff2?5(k@VpB*f6jz>vbh;L5;Y1QO*Job15?1YAJ2fhEu+ zDF%jfzy7U9)`7~-0dheY8Gy`eyPF^Y6f Date: Tue, 13 Feb 2018 09:50:38 +0000 Subject: [PATCH 036/100] Fix next-hop discovery with multiple devices Fixes https://github.com/Igalia/snabb/issues/1014. --- src/program/lwaftr/setup.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index d665b3b329..7b804e89c5 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -116,14 +116,14 @@ function lwaftr_app(c, conf) { self_ip = iinternal_interface.ip, self_mac = iinternal_interface.mac, next_mac = iinternal_interface.next_hop.mac, - shared_next_mac_key = "group/ipv6-next-mac", + shared_next_mac_key = "group/"..device.."-ipv6-next-mac", next_ip = iinternal_interface.next_hop.ip, alarm_notification = conf.alarm_notification }) config.app(c, "arp", arp.ARP, { self_ip = convert_ipv4(iexternal_interface.ip), self_mac = iexternal_interface.mac, next_mac = iexternal_interface.next_hop.mac, - shared_next_mac_key = "group/ipv4-next-mac", + shared_next_mac_key = "group/"..device.."-ipv4-next-mac", next_ip = convert_ipv4(iexternal_interface.next_hop.ip), alarm_notification = conf.alarm_notification }) From b38ae3a86a4f707103071328a8e7da50eacd7883 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 10:04:04 +0000 Subject: [PATCH 037/100] Remove early lwAFTR NUMA affinity check Now that the ptree manager handles NUMA affinity and appropriate CPU selection, the check that we had was both too early (as the manager hadn't had time to bind the NUMA affinity) and unnecessary (as the manager handled the whole issue for us, including issuing warnings). --- src/program/lwaftr/setup.lua | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index d665b3b329..af678a7462 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -19,7 +19,6 @@ local ipv6_reassemble = require("apps.ipv6.reassemble") local ndp = require("apps.lwaftr.ndp") local vlan = require("apps.vlan.vlan") local pci = require("lib.hardware.pci") -local numa = require("lib.numa") local cltable = require("lib.cltable") local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") @@ -49,10 +48,8 @@ local function convert_ipv4(addr) if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end end --- Checks the existance and NUMA affinity of PCI devices --- NB: "nil" can be passed in and will be siliently ignored. +-- Checks the existence of PCI devices. local function validate_pci_devices(devices) - numa.check_affinity_for_pci_addresses(devices) for _, address in pairs(devices) do assert(lwutil.nic_exists(address), ("Could not locate PCI device '%s'"):format(address)) From 05bffbc90cbf0e7ededfa2a78aa72c16d2063993 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 13 Feb 2018 12:02:57 +0000 Subject: [PATCH 038/100] Move "loadtest" command out of lwaftr --- src/program/loadtest/README | 7 ++ src/program/{lwaftr => }/loadtest/README.inc | 0 src/program/loadtest/loadtest.lua | 26 +++++++ src/program/{lwaftr => }/loadtest/promise.lua | 10 ++- .../loadtest => loadtest/transient}/README | 10 ++- src/program/loadtest/transient/README.inc | 1 + .../transient/transient.lua} | 73 ++++++++++++------- src/program/lwaftr/README | 1 - src/program/lwaftr/doc/benchmarking.md | 2 +- .../lwaftr/doc/continuous-integration.md | 2 +- src/program/lwaftr/doc/performance.md | 2 +- src/program/lwaftr/doc/running.md | 4 +- .../lwaftr/tests/subcommands/loadtest_test.py | 4 +- 13 files changed, 99 insertions(+), 43 deletions(-) create mode 100644 src/program/loadtest/README rename src/program/{lwaftr => }/loadtest/README.inc (100%) create mode 100644 src/program/loadtest/loadtest.lua rename src/program/{lwaftr => }/loadtest/promise.lua (88%) rename src/program/{lwaftr/loadtest => loadtest/transient}/README (88%) create mode 120000 src/program/loadtest/transient/README.inc rename src/program/{lwaftr/loadtest/loadtest.lua => loadtest/transient/transient.lua} (88%) diff --git a/src/program/loadtest/README b/src/program/loadtest/README new file mode 100644 index 0000000000..4a9bbacdb5 --- /dev/null +++ b/src/program/loadtest/README @@ -0,0 +1,7 @@ +Usage: + snabb loadtest transient + snabb loadtest find-limit + +Use --help for per-command usage. +Example: + snabb loadtest find-limit --help diff --git a/src/program/lwaftr/loadtest/README.inc b/src/program/loadtest/README.inc similarity index 100% rename from src/program/lwaftr/loadtest/README.inc rename to src/program/loadtest/README.inc diff --git a/src/program/loadtest/loadtest.lua b/src/program/loadtest/loadtest.lua new file mode 100644 index 0000000000..bbdf4d01e4 --- /dev/null +++ b/src/program/loadtest/loadtest.lua @@ -0,0 +1,26 @@ +module(..., package.seeall) + +local lib = require("core.lib") + +local function latest_version() + local v = require('core.version') + return v.version, v.extra_version +end + +local function show_usage(exit_code) + local content = require("program.loadtest.README_inc") + require('core.main').version() + print('') + print(content) + main.exit(exit_code) +end + +function run(args) + if #args == 0 then show_usage(1) end + local command = string.gsub(table.remove(args, 1), "-", "_") + local modname = ("program.loadtest.%s.%s"):format(command, command) + if not lib.have_module(modname) then + show_usage(1) + end + require(modname).run(args) +end diff --git a/src/program/lwaftr/loadtest/promise.lua b/src/program/loadtest/promise.lua similarity index 88% rename from src/program/lwaftr/loadtest/promise.lua rename to src/program/loadtest/promise.lua index 9b8af97663..97f4f9e132 100644 --- a/src/program/lwaftr/loadtest/promise.lua +++ b/src/program/loadtest/promise.lua @@ -47,15 +47,17 @@ function Promise:resolve(...) self.resolved = true self.vals = { self.transform(...) } if #self.vals == 1 and is_promise(self.vals[1]) then - if self.next then self.vals[1]:chain(self.next) end - self.next = self.vals[1] - else - if self.next then self:dispatch_next() end + local new_next, old_next = self.vals[1], self.next + self.next = nil + if old_next then new_next:chain(old_next) end + elseif self.next then + self:dispatch_next() end end function Promise:chain(next) assert(next) + assert(not next.resolved) assert(not self.next) self.next = next if self.resolved then self:dispatch_next() end diff --git a/src/program/lwaftr/loadtest/README b/src/program/loadtest/transient/README similarity index 88% rename from src/program/lwaftr/loadtest/README rename to src/program/loadtest/transient/README index fdc7d1fb01..c8211180ed 100644 --- a/src/program/lwaftr/loadtest/README +++ b/src/program/loadtest/transient/README @@ -1,4 +1,4 @@ -Usage: loadtest [OPTIONS] [ ]... +Usage: transient [OPTIONS] [ ]... -b BITRATE, --bitrate BITRATE Peak at BITRATE bits/second. @@ -48,6 +48,10 @@ The available workload programs are: A ramp_up followed by a ramp_down. + --program constant + + A constant BITRATE bits per second load. + The default workload program is ramp_up_down. Packets received on the network interfaces are counted and recorded, @@ -60,5 +64,5 @@ be looking for a response. Packets sent but not received will be counted as loss. Examples: - loadtest cap1.pcap tx tx 01:00.0 - loadtest -D 1 -b 5e9 -s 0.2e9 cap1.pcap "NIC 0" "NIC 1" 01:00.0 cap2.pcap "NIC 1" "NIC 0" 01:00.1 + transient cap1.pcap tx tx 01:00.0 + transient -D 1 -b 5e9 -s 0.2e9 cap1.pcap "NIC 0" "NIC 1" 01:00.0 cap2.pcap "NIC 1" "NIC 0" 01:00.1 diff --git a/src/program/loadtest/transient/README.inc b/src/program/loadtest/transient/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/loadtest/transient/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/lwaftr/loadtest/loadtest.lua b/src/program/loadtest/transient/transient.lua similarity index 88% rename from src/program/lwaftr/loadtest/loadtest.lua rename to src/program/loadtest/transient/transient.lua index 668ab8174e..8a9aca14f0 100644 --- a/src/program/lwaftr/loadtest/loadtest.lua +++ b/src/program/loadtest/transient/transient.lua @@ -10,14 +10,17 @@ local main = require("core.main") local PcapReader = require("apps.pcap.pcap").PcapReader local lib = require("core.lib") local numa = require("lib.numa") -local promise = require("program.lwaftr.loadtest.promise") +local promise = require("program.loadtest.promise") local lwutil = require("apps.lwaftr.lwutil") -local fatal = lwutil.fatal - local WARM_UP_BIT_RATE = 5e9 local WARM_UP_TIME = 2 +local function fatal (msg) + print(msg) + main.exit(1) +end + local function show_usage(code) print(require("program.lwaftr.loadtest.README_inc")) main.exit(code) @@ -50,31 +53,29 @@ end local programs = {} function programs.ramp_up(tester, opts) - local head = promise.new() - local tail = head - for step = 1, math.ceil(opts.bitrate / opts.step) do - tail = tail:and_then(tester.measure, - math.min(opts.bitrate, opts.step * step), - opts.duration, - opts.bench_file, - opts.hydra) + local function next(step) + if step <= math.ceil(opts.bitrate / opts.step) then + return tester.measure(math.min(opts.bitrate, opts.step * step), + opts.duration, + opts.bench_file, + opts.hydra): + and_then(next, step + 1) + end end - head:resolve() - return tail + return next(1) end function programs.ramp_down(tester, opts) - local head = promise.new() - local tail = head - for step = math.ceil(opts.bitrate / opts.step), 1, -1 do - tail = tail:and_then(tester.measure, - math.min(opts.bitrate, opts.step * step), - opts.duration, - opts.bench_file, - opts.hydra) + local function next(step) + if step >= 1 then + return tester.measure(math.min(opts.bitrate, opts.step * step), + opts.duration, + opts.bench_file, + opts.hydra): + and_then(next, step - 1) + end end - head:resolve() - return tail + return next(math.ceil(opts.bitrate / opts.step)) end function programs.ramp_up_down(tester, opts) @@ -82,6 +83,25 @@ function programs.ramp_up_down(tester, opts) :and_then(programs.ramp_down, tester, opts) end +function programs.constant(tester, opts) + local function step() + local gbps_bitrate = opts.bitrate/1e9 + local start_counters = tester.record_counters() + local function report() + local end_counters = tester.record_counters() + tester.print_counter_diff(start_counters, end_counters, + opts.duration, gbps_bitrate, + opts.bench_file, opts.hydra_mode) + end + -- No quiet period; keep the packets going! + return tester.generate_load(opts.bitrate, opts.duration): + and_then(report): + and_then(step) + end + print(string.format('Applying %f Gbps of load.', opts.bitrate/1e9)) + return step() +end + function parse_args(args) local handlers = {} local opts = { bitrate = 10e9, duration = 5, program=programs.ramp_up_down } @@ -324,13 +344,11 @@ function run(args) return bench_file end - local function run_engine(head, tail) + local function run_engine(tail) local is_done = false local function mark_done() is_done = true end tail:and_then(mark_done) - local function done() return is_done end - head:resolve() engine.main({done=done}) end @@ -339,7 +357,6 @@ function run(args) end engine.busywait = true local head = promise.new() - run_engine(head, - head:and_then(tester.warm_up) + run_engine(tester.warm_up() :and_then(opts.program, tester, opts)) end diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 6995334f28..5c4f45be5c 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -2,7 +2,6 @@ Usage: snabb lwaftr bench snabb lwaftr check snabb lwaftr generate-binding-table - snabb lwaftr loadtest snabb lwaftr migrate-configuration snabb lwaftr monitor snabb lwaftr query diff --git a/src/program/lwaftr/doc/benchmarking.md b/src/program/lwaftr/doc/benchmarking.md index b2fe0aaf00..0557eaa146 100644 --- a/src/program/lwaftr/doc/benchmarking.md +++ b/src/program/lwaftr/doc/benchmarking.md @@ -26,7 +26,7 @@ NIC. In the other server, run the `loadtest` command: ``` -$ sudo numactl -m 0 taskset -c 1 ./snabb lwaftr loadtest -D 1 -b 10e9 -s 0.2e9 \ +$ sudo numactl -m 0 taskset -c 1 ./snabb loadtest transient -D 1 -b 10e9 -s 0.2e9 \ program/lwaftr/tests/benchdata/ipv4-0550.pcap "NIC 0" "NIC 1" 02:00.0 \ program/lwaftr/tests/benchdata/ipv6-0550.pcap "NIC 1" "NIC 0" 02:00.1 ``` diff --git a/src/program/lwaftr/doc/continuous-integration.md b/src/program/lwaftr/doc/continuous-integration.md index 6a5e82f3a1..ae3f93e445 100644 --- a/src/program/lwaftr/doc/continuous-integration.md +++ b/src/program/lwaftr/doc/continuous-integration.md @@ -23,7 +23,7 @@ instance hosts the lwAftr CI benchmarks. Three jobsets are currently defined: `snabb lwaftr bench` command, which executes lwAftr benchmarks with no interaction with physical NICs; - [lwaftr-nic](https://hydra.snabb.co/jobset/igalia/lwaftr-nic) runs the - `snabb lwaftr loadtest` and `snabb lwaftr run` commands: the first command + `snabb loadtest transient` and `snabb lwaftr run` commands: the first command generates network traffic to a physical NIC, which is then received by another NIC and processed by the lwAftr instance launched by the second command; diff --git a/src/program/lwaftr/doc/performance.md b/src/program/lwaftr/doc/performance.md index 932165d586..15090581ae 100644 --- a/src/program/lwaftr/doc/performance.md +++ b/src/program/lwaftr/doc/performance.md @@ -24,7 +24,7 @@ MTUs are set such that fragmentation is rare. ## CPU affinity -The `snabb lwaftr run` and `snabb lwaftr loadtest` commands take a +The `snabb lwaftr run` and `snabb loadtest transient` commands take a `--cpu` argument, which will arrange for the Snabb process to run on a particular CPU. It will also arrange to make sure that all memory used by that Snabb process is on the same NUMA node as that CPU, and it will diff --git a/src/program/lwaftr/doc/running.md b/src/program/lwaftr/doc/running.md index 8ee9878b00..554bda48ea 100644 --- a/src/program/lwaftr/doc/running.md +++ b/src/program/lwaftr/doc/running.md @@ -65,7 +65,7 @@ Then run a load generator: ```bash $ cd src -$ sudo ./snabb lwaftr loadtest \ +$ sudo ./snabb loadtest transient \ program/lwaftr/tests/benchdata/ipv4-0550.pcap IPv4 IPv6 0000:01:00.0 \ program/lwaftr/tests/benchdata/ipv6-0550.pcap IPv6 IPv4 0000:02:00.0 ``` @@ -89,7 +89,7 @@ $ sudo ./snabb lwaftr run --conf /tmp/icmp_on_fail.conf \ You can run a load generator in on-a-stick mode: ``` -$ sudo ./snabb lwaftr loadtest \ +$ sudo ./snabb loadtest transient \ program/lwaftr/tests/benchdata/ipv4_and_ipv6_stick_imix.pcap ALL ALL \ 0000:82:00.1 ``` diff --git a/src/program/lwaftr/tests/subcommands/loadtest_test.py b/src/program/lwaftr/tests/subcommands/loadtest_test.py index 242e6af808..e5656278d3 100644 --- a/src/program/lwaftr/tests/subcommands/loadtest_test.py +++ b/src/program/lwaftr/tests/subcommands/loadtest_test.py @@ -1,5 +1,5 @@ """ -Test the "snabb lwaftr loadtest" subcommand. Needs NIC names. +Test the "snabb loadtest transient" subcommand. Needs NIC names. Since there are only two NIC names available in snabb-bot, and we need to execute two programs networked to each other ("run" and "loadtest"), they @@ -23,7 +23,7 @@ class TestLoadtest(BaseTestCase): '--on-a-stick', SNABB_PCI0, ) loadtest_args = ( - str(SNABB_CMD), 'lwaftr', 'loadtest', + str(SNABB_CMD), 'loadtest', 'transient', '--bench-file', '/dev/null', # Something quick and easy. '--program', 'ramp_up', From 09ce69e87cf06ec1ef80f00115a98e81daaf5fb1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 14 Feb 2018 10:21:50 +0000 Subject: [PATCH 039/100] Fix ctable resizing logic Ctable sizes should be integers. Also, allow zero-sized tables by having calloc return nil when allocating zero elements. --- src/lib/ctable.lua | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 96df39bf8f..621ddeb7f4 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -7,7 +7,7 @@ local lib = require("core.lib") local binary_search = require("lib.binary_search") local multi_copy = require("lib.multi_copy") local siphash = require("lib.hash.siphash") -local max, floor, ceil = math.max, math.floor, math.ceil +local min, max, floor, ceil = math.min, math.max, math.floor, math.ceil CTable = {} LookupStreamer = {} @@ -154,6 +154,7 @@ end local try_huge_pages = true local huge_page_threshold = 1e6 local function calloc(t, count) + if count == 0 then return 0, 0 end local byte_size = ffi.sizeof(t) * count local mem, err if try_huge_pages and byte_size > huge_page_threshold then @@ -204,6 +205,7 @@ end function CTable:resize(size) assert(size >= (self.occupancy / self.max_occupancy_rate)) + assert(size == floor(size)) local old_entries = self.entries local old_size = self.size local old_max_displacement = self.max_displacement @@ -280,7 +282,7 @@ function CTable:add(key, value, updates_allowed) if self.occupancy + 1 > self.occupancy_hi then -- Note that resizing will invalidate all hash keys, so we need -- to hash the key after resizing. - self:resize(self.size * 2) + self:resize(min(self.size * 2, 10)) -- Could be current size is 0. end local hash = self.hash_fn(key) @@ -402,7 +404,7 @@ function CTable:remove_ptr(entry) end if self.occupancy < self.occupancy_lo then - self:resize(self.size / 2) + self:resize(max(ceil(self.size / 2), 1)) end end @@ -589,7 +591,7 @@ function CTable:next_entry(offset, limit) elseif limit == nil then limit = self.size + self.max_displacement else - limit = math.min(limit, self.size + self.max_displacement) + limit = min(limit, self.size + self.max_displacement) end for offset=offset, limit-1 do if self.entries[offset].hash ~= HASH_MAX then @@ -612,7 +614,6 @@ function selftest() initial_size = ceil(occupancy / 0.4) } local ctab = new(params) - ctab:resize(occupancy / 0.4 + 1) -- Fill with {i} -> { bnot(i), ... }. local k = ffi.new('uint32_t[1]'); @@ -706,7 +707,7 @@ function selftest() repeat local streamer = ctab:make_lookup_streamer(width) for i = 1, occupancy, width do - local n = math.min(width, occupancy-i+1) + local n = min(width, occupancy-i+1) for j = 0, n-1 do streamer.entries[j].key[0] = i + j end From 3142fad8c9236b4a8540a38ceec921e2255ff2df Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 14 Feb 2018 10:43:10 +0000 Subject: [PATCH 040/100] Fix sense of ctable resize limit --- src/lib/ctable.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 621ddeb7f4..ac3e8b8958 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -282,7 +282,7 @@ function CTable:add(key, value, updates_allowed) if self.occupancy + 1 > self.occupancy_hi then -- Note that resizing will invalidate all hash keys, so we need -- to hash the key after resizing. - self:resize(min(self.size * 2, 10)) -- Could be current size is 0. + self:resize(max(self.size * 2, 10)) -- Could be current size is 0. end local hash = self.hash_fn(key) From db8b2c4dcb2bf4337de5e48d2ded95663958d8ff Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 14 Feb 2018 10:48:36 +0000 Subject: [PATCH 041/100] Make minimum ctable size uniform --- src/lib/ctable.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index ac3e8b8958..a59f0fc4d2 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -282,7 +282,7 @@ function CTable:add(key, value, updates_allowed) if self.occupancy + 1 > self.occupancy_hi then -- Note that resizing will invalidate all hash keys, so we need -- to hash the key after resizing. - self:resize(max(self.size * 2, 10)) -- Could be current size is 0. + self:resize(max(self.size * 2, 1)) -- Could be current size is 0. end local hash = self.hash_fn(key) From 279aa3845f89b258cccf1d6be3e40ec2ee79517f Mon Sep 17 00:00:00 2001 From: Alexander Gall Date: Wed, 14 Feb 2018 11:52:00 +0100 Subject: [PATCH 042/100] core.app: remove garbage when using pace_breathing The accumulators lastfrees, lastfreebytes, lastfreebits for the counters frees, freebytes and freebits are initialized as Lua numbers but implicitly converted to cdata objects in the assignments in pace_breathing(). This causes allocations that cannot be removed by the sink optimizer. Conversion to Lua numbers avoids this and reduces GC noise. --- src/core/app.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 54fd5c98d7..13243c4f97 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -505,9 +505,9 @@ function pace_breathing () else sleep = math.floor(sleep/2) end - lastfrees = counter.read(frees) - lastfreebytes = counter.read(freebytes) - lastfreebits = counter.read(freebits) + lastfrees = tonumber(counter.read(frees)) + lastfreebytes = tonumber(counter.read(freebytes)) + lastfreebits = tonumber(counter.read(freebits)) end end From f31e840fa06de309281e4a85189ad3b4832f8355 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 14 Feb 2018 16:55:00 +0000 Subject: [PATCH 043/100] Add limit-finding loadtester. --- src/program/loadtest/find-limit/README | 24 +++ src/program/loadtest/find-limit/README.inc | 1 + .../loadtest/find-limit/find-limit.lua | 203 ++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 src/program/loadtest/find-limit/README create mode 120000 src/program/loadtest/find-limit/README.inc create mode 100644 src/program/loadtest/find-limit/find-limit.lua diff --git a/src/program/loadtest/find-limit/README b/src/program/loadtest/find-limit/README new file mode 100644 index 0000000000..cd9359ed59 --- /dev/null +++ b/src/program/loadtest/find-limit/README @@ -0,0 +1,24 @@ +Usage: find-limit [OPTIONS] DEVICE PCAP-FILE + + -b BITRATE, --bitrate BITRATE + Test bitrates up to BITRATE bits/second. + Default: 10e9. + -D DURATION, --duration DURATION + Linger on each step for DURATION seconds. + Default: 1. + -p PRECISION, --precision PRECISION + Measure to PRECISION bits/second. + Default: 0.001e9. + -r RETRY-COUNT, --retry-count RETRY-COUNT + If a step fails, retry it RETRY-COUNT times. + Default: 3. + --cpu CPU + Bind to the given CPU. + -h, --help + Print usage information. + +Apply load on DEVICE by replaying packets from PCAP-FILE, and attempt to +determine the highest bitrate at which a test passes. + +Examples: + find-limit 01:00.0 cap1.pcap diff --git a/src/program/loadtest/find-limit/README.inc b/src/program/loadtest/find-limit/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/loadtest/find-limit/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua new file mode 100644 index 0000000000..55fe7e1ee5 --- /dev/null +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -0,0 +1,203 @@ +module(..., package.seeall) + +local engine = require("core.app") +local counter = require("core.counter") +local config = require("core.config") +local pci = require("lib.hardware.pci") +local basic_apps = require("apps.basic.basic_apps") +local loadgen = require("apps.lwaftr.loadgen") +local main = require("core.main") +local PcapReader = require("apps.pcap.pcap").PcapReader +local lib = require("core.lib") +local numa = require("lib.numa") +local promise = require("program.loadtest.promise") + +local WARM_UP_BIT_RATE = 1e9 +local WARM_UP_TIME = 1 + +local function fatal (msg) + print(msg) + main.exit(1) +end + +local function show_usage(code) + print(require("program.loadtest.find_limit.README_inc")) + main.exit(code) +end + +local function find_limit(tester, max_bitrate, precision, duration, retry_count) + local function round(x) + return math.floor((x + precision/2) / precision) * precision + end + + -- lo and hi are bitrates, in bits per second. + local function bisect(lo, hi, iter) + local function continue(cur, result) + if result then + print("Success.") + return bisect(cur, hi, 1) + elseif iter <= retry_count then + print("Failed; "..(retry_count - iter).. " retries remaining.") + return bisect(lo, hi, iter + 1) + else + print("Failed.") + return bisect(lo, cur, 1) + end + end + local cur = round((lo + hi) / 2) + if cur == lo or cur == hi then return lo end + return tester.measure(cur, duration): + and_then(continue, cur) + end + return bisect(0, round(max_bitrate), 1) +end + +function parse_args(args) + local opts = { max_bitrate = 10e9, duration = 1, precision = 0.001e9, + retry_count = 3 } + local function parse_positive_number(prop) + return function(arg) + local val = assert(tonumber(arg), prop.." must be a number") + assert(val > 0, prop.." must be positive") + opts[prop] = val + end + end + local function parse_nonnegative_integer(prop) + return function(arg) + local val = assert(tonumber(arg), prop.." must be a number") + assert(val >= 0, prop.." must be non-negative") + assert(val == math.floor(val), prop.." must be an integer") + opts[prop] = val + end + end + local function parse_string(prop) + return function(arg) opts[prop] = assert(arg) end + end + local handlers = { b = parse_positive_number("max_bitrate"), + e = parse_string("exec"), + D = parse_positive_number("duration"), + p = parse_positive_number("precision"), + r = parse_nonnegative_integer("retry_count"), + cpu = parse_nonnegative_integer("cpu") } + function handlers.h() show_usage(0) end + args = lib.dogetopt(args, handlers, "hb:D:p:r:e:", + { bitrate="b", duration="D", precision="p", + ["retry-count"]="r", help="h", cpu=1, + exec="e"}) + if #args ~= 2 then show_usage(1) end + local device, capture_file = unpack(args) + if opts.cpu then numa.bind_to_cpu(opts.cpu) end + numa.check_affinity_for_pci_addresses({device}) + return opts, device, capture_file +end + +function run(args) + local opts, device, capture_file = parse_args(args) + local device_info = pci.device_info(device) + local driver = require(device_info.driver).driver + local c = config.new() + + -- Links are named directionally with respect to NIC apps, but we + -- want to name tx and rx with respect to the whole network + -- function. + local tx_link_name = device_info.rx + local rx_link_name = device_info.tx + + config.app(c, "replay", PcapReader, capture_file) + config.app(c, "repeater", loadgen.RateLimitedRepeater, {}) + config.app(c, "nic", driver, { pciaddr = device_info.pciaddress }) + config.app(c, "blackhole", basic_apps.Sink) + + config.link(c, "replay.output -> repeater.input") + config.link(c, "repeater.output -> nic."..tx_link_name) + config.link(c, "nic."..rx_link_name.." -> blackhole.input") + + engine.configure(c) + + local nic_app = assert(engine.app_table.nic) + local repeater_app = assert(engine.app_table.repeater) + + local function read_counters() + local tx, rx = nic_app.input[tx_link_name], nic_app.output[rx_link_name] + return { txpackets = counter.read(tx.stats.txpackets), + txbytes = counter.read(tx.stats.txbytes), + rxpackets = counter.read(rx.stats.txpackets), + rxbytes = counter.read(rx.stats.txbytes), + rxdrop = nic_app:rxdrop() } + end + + local function print_diff(diff, duration) + local function bitrate(packets, bytes) + -- 7 bytes preamble, 1 start-of-frame, 4 CRC, 12 interframe gap. + local overhead = 7 + 1 + 4 + 12 + return (bytes + packets * overhead) * 8 / duration + end + local tx_mpps = diff.txpackets / duration / 1e6 + local tx_gbps = bitrate(diff.txpackets, diff.txbytes) / 1e9 + local rx_mpps = diff.rxpackets / duration / 1e6 + local rx_gbps = bitrate(diff.rxpackets, diff.rxbytes) / 1e9 + local lost_packets = diff.txpackets - diff.rxpackets - diff.rxdrop + local lost_percent = lost_packets / diff.txpackets * 100 + print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', + diff.txpackets, tx_mpps, diff.txbytes, tx_gbps)) + print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', + diff.rxpackets, rx_mpps, diff.rxbytes, rx_gbps)) + print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', + diff.rxdrop, lost_packets, lost_percent)) + end + + local function check_results(diff) + if opts.exec then + -- Could pass on some arguments to this string. + return os.execute(opts.exec) == 0 + else + return diff.rxpackets == diff.txpackets and diff.rxdrop == 0 + end + end + + local tester = {} + + function tester.adjust_rates(bit_rate) + repeater_app:set_rate(bit_rate) + end + + function tester.generate_load(bitrate, duration) + tester.adjust_rates(bitrate) + return promise.Wait(duration):and_then(tester.adjust_rates, 0) + end + + function tester.warm_up() + print(string.format("Warming up at %f Gb/s for %s seconds.", + WARM_UP_BIT_RATE / 1e9, WARM_UP_TIME)) + return tester.generate_load(WARM_UP_BIT_RATE, WARM_UP_TIME) + end + + function tester.measure(bitrate, duration) + local gbps_bitrate = bitrate/1e9 + local start_counters = read_counters() + local function report() + local end_counters = read_counters() + local diff = {} + for k,v in pairs(start_counters) do + diff[k] = tonumber(end_counters[k] - start_counters[k]) + end + print_diff(diff, duration) + return diff + end + print(string.format('Applying %f Gbps of load.', gbps_bitrate)) + return tester.generate_load(bitrate, duration): + -- Wait 2ms for packets in flight to arrive + and_then(promise.Wait, 0.002): + and_then(report): + and_then(check_results) + end + + engine.busywait = true + local is_done = false + local function mark_done() is_done = true end + tester.warm_up(): + and_then(find_limit, tester, opts.max_bitrate, opts.precision, + opts.duration, opts.retry_count): + and_then(mark_done) + engine.main({done=function() return is_done end}) +end From 047018a96598d9d8befb719d534522260fefb7e5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 15 Feb 2018 09:38:12 +0000 Subject: [PATCH 044/100] Add find-limit --exec to README --- src/program/loadtest/find-limit/README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/loadtest/find-limit/README b/src/program/loadtest/find-limit/README index cd9359ed59..941cb2d481 100644 --- a/src/program/loadtest/find-limit/README +++ b/src/program/loadtest/find-limit/README @@ -14,6 +14,11 @@ Usage: find-limit [OPTIONS] DEVICE PCAP-FILE Default: 3. --cpu CPU Bind to the given CPU. + --exec EXEC + Run EXEC in a shell after every step to + determine if the step succeeded. Otherwise + a successful step is one that receives as + many packets on its interface as it sends. -h, --help Print usage information. From 35539ece7ffc25444aafce5fbaada7a6797a82fb Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 6 Feb 2018 20:38:54 +0100 Subject: [PATCH 045/100] Correct/extend doc/snabblab.md --- src/doc/snabblab.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/doc/snabblab.md b/src/doc/snabblab.md index d328073909..b30de09a4f 100644 --- a/src/doc/snabblab.md +++ b/src/doc/snabblab.md @@ -11,15 +11,16 @@ Want to be a known developer? Sure! Just edit the [user account list](https://gi ## Servers -Name | Purpose | SSH | Xeon model | NICs +Name | Purpose | SSH | Intel CPU | NICs ------------|---------------------------------------------------|-------------------------| -------- | ------------------------------------------------ -lugano-1 | General use | lugano-1.snabb.co | E3 1650v3 | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710) -lugano-2 | General use | lugano-2.snabb.co | E3 1650v3 | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710) -lugano-3 | General use | lugano-3.snabb.co | E3 1650v3 | 2 x 10G (82599), 2 x 100G (ConnectX-4) -lugano-4 | General use | lugano-4.snabb.co | E3 1650v3 | 2 x 10G (82599), 2 x 100G (ConnectX-4) +lugano-1 | General use | lugano-1.snabb.co | E5 1650v3 | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710) +lugano-2 | General use | lugano-2.snabb.co | E5 1650v3 | 2 x 10G (82599), 4 x 10G (X710), 2 x 40G (XL710) +lugano-3 | General use | lugano-3.snabb.co | E5 1650v3 | 2 x 10G (82599), 2 x 100G (ConnectX-4) +lugano-4 | General use | lugano-4.snabb.co | E5 1650v3 | 2 x 10G (82599), 2 x 100G (ConnectX-4) davos | Continuous Integration tests & driver development | lab1.snabb.co port 2000 | 2x E5 2603 | Diverse 10G/40G: Intel, SolarFlare, Mellanox, Chelsio, Broadcom. Installed upon request. grindelwald | Snabb NFV testing | lab1.snabb.co port 2010 | 2x E5 2697v2 | 12 x 10G (Intel 82599) interlaken | Haswell/AVX2 testing | lab1.snabb.co port 2030 | 2x E5 2620v3 | 12 x 10G (Intel 82599) +murren-* | Hydra fleet for tests without NICs | (none) | i7-6700 | (none) ## Get started From 3c9edf08ba86957eeea4dbdc496b4a0415eb7ef5 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Feb 2018 16:02:37 +0100 Subject: [PATCH 046/100] Show error if running on loop-back interface --- src/program/dnssd/dnssd.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/program/dnssd/dnssd.lua b/src/program/dnssd/dnssd.lua index 3876dccd6e..40b305ca39 100644 --- a/src/program/dnssd/dnssd.lua +++ b/src/program/dnssd/dnssd.lua @@ -126,7 +126,12 @@ end local function ethernet_address_of (iface) local cmd = ("ip li sh %s | grep 'link/ether' | awk '{print $2}'"):format(iface) - return chomp(execute(cmd)) + local ret = chomp(execute(cmd)) + if #ret == 0 then + print(("Unsupported interface: '%s' (missing MAC address)"):format(iface)) + os.exit() + end + return ret end local function ipv4_address_of (iface) @@ -150,9 +155,10 @@ function run(args) elseif opts.interface then local iface = opts.interface local query = args[1] + local src_eth = ethernet_address_of(iface) print(("Capturing packets from interface '%s'"):format(iface)) config.app(c, "dnssd", DNSSD, { - src_eth = ethernet_address_of(iface), + src_eth = src_eth, src_ipv4 = ipv4_address_of(iface), query = query, }) From 373b42b5500497e32955ea35ed8b7febbe046643 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Feb 2018 15:41:56 +0000 Subject: [PATCH 047/100] Enable alarm notification --- src/program/lwaftr/setup.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index a366360ef8..23e7c196bb 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -587,13 +587,11 @@ local function compute_worker_configs(conf) end function ptree_manager(f, conf, manager_opts) - -- Always enabled in reconfigurable mode. - alarm_notification = true - local function setup_fn(conf) local worker_app_graphs = {} for worker_id, worker_config in pairs(compute_worker_configs(conf)) do local app_graph = config.new() + worker_config.alarm_notification = true f(app_graph, worker_config) worker_app_graphs[worker_id] = app_graph end From dca505ca67208419ebae5ebc78bbd50b8fdd320c Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Feb 2018 16:59:15 +0000 Subject: [PATCH 048/100] Fix variable name --- src/lib/yang/alarms.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/alarms.lua b/src/lib/yang/alarms.lua index 7bcb2d018c..58a6c6e6f9 100644 --- a/src/lib/yang/alarms.lua +++ b/src/lib/yang/alarms.lua @@ -202,7 +202,7 @@ function alarm_list:set_defaults_if_any (key) k = alarm_type_keys:normalize(key) local default = self.defaults[k] if default then - for k,v in pairs(defaults) do + for k,v in pairs(default) do self.list[key][k] = v end end From 191e655cb8cfd702f200374798c5570943bc41b7 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Feb 2018 16:48:17 +0000 Subject: [PATCH 049/100] Fix error message --- src/lib/ptree/ptree.lua | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lib/ptree/ptree.lua b/src/lib/ptree/ptree.lua index 41ea1119a7..710cf6518b 100644 --- a/src/lib/ptree/ptree.lua +++ b/src/lib/ptree/ptree.lua @@ -290,8 +290,7 @@ end function Manager:rpc_set_alarm_operator_state (args) local function getter() if args.schema ~= self.schema_name then - return false, ("Set-operator-state operation not supported in".. - "'%s' schema"):format(args.schema) + error(("Set-operator-state operation not supported in '%s' schema"):format(args.schema)) end local key = {resource=args.resource, alarm_type_id=args.alarm_type_id, alarm_type_qualifier=args.alarm_type_qualifier} @@ -305,8 +304,7 @@ end function Manager:rpc_purge_alarms (args) local function purge() if args.schema ~= self.schema_name then - return false, ("Purge-alarms operation not supported in".. - "'%s' schema"):format(args.schema) + error(("Purge-alarms operation not supported in '%s' schema"):format(args.schema)) end return { purged_alarms = alarms.purge_alarms(args) } end @@ -317,8 +315,7 @@ end function Manager:rpc_compress_alarms (args) local function compress() if args.schema ~= self.schema_name then - return false, ("Compress-alarms operation not supported in".. - "'%s' schema"):format(args.schema) + error(("Compress-alarms operation not supported in '%s' schema"):format(args.schema)) end return { compressed_alarms = alarms.compress_alarms(args) } end From c81176e688c54d0574b1c6be71569d60543b8197 Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Sun, 18 Feb 2018 10:13:51 +0100 Subject: [PATCH 050/100] build snabb docker container --- Dockerfile | 10 ++++++++++ Makefile | 3 +++ 2 files changed, 13 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..0f99c3b41d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.7 AS build +RUN apk add --no-cache libgcc alpine-sdk gcc libpcap-dev linux-headers findutils +COPY . /snabb +RUN cd /snabb && make -j + +FROM alpine:3.7 +RUN apk add --no-cache libgcc +COPY --from=build /snabb/src/snabb /usr/local/bin/ + +ENTRYPOINT ["/usr/local/bin/snabb"] diff --git a/Makefile b/Makefile index 27a63966c0..797e387485 100644 --- a/Makefile +++ b/Makefile @@ -47,4 +47,7 @@ dist: all cd "$(DISTDIR)/.." && tar cJvf "`basename '$(DISTDIR)'`.tar.xz" "`basename '$(DISTDIR)'`" rm -rf "$(DISTDIR)" +docker: + docker build -t snabb . + @echo "Usage: docker run -ti --rm snabb ..." .SERIAL: all From 3337ae80a399f6ebe5b59dbed0bdf3f0e350320a Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Sun, 18 Feb 2018 10:28:29 +0100 Subject: [PATCH 051/100] make clean before build to avoid using binaries built outside of docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0f99c3b41d..e9175e1bd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.7 AS build RUN apk add --no-cache libgcc alpine-sdk gcc libpcap-dev linux-headers findutils COPY . /snabb -RUN cd /snabb && make -j +RUN cd /snabb && make clean && make -j FROM alpine:3.7 RUN apk add --no-cache libgcc From dff887754e12bbbafcbbae73372128581ad0be1e Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Sun, 18 Feb 2018 10:52:30 +0100 Subject: [PATCH 052/100] adding container build and run instructions --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 1cfa9dab73..2d7d3d9314 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,20 @@ $ cp src/snabb /usr/local/bin/ $ sudo snabb packetblaster replay capture.pcap 01:00.0 ``` +### snabb container + +Basic support for building and running snabb in a Docker container is available via + +``` +$ make docker +``` + +This will build a tiny snabb container (8MB), ready to be used: + +``` +$ docker run -ti --rm snabb --help +``` + ## How do I get involved? Here are the ways you can get involved: From 77f7abc7b3db4c69859f3ee03e9d7453b268d455 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 19 Feb 2018 11:18:52 +0000 Subject: [PATCH 053/100] Fix path to transient README --- src/program/loadtest/transient/transient.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/loadtest/transient/transient.lua b/src/program/loadtest/transient/transient.lua index 8a9aca14f0..0bb378374e 100644 --- a/src/program/loadtest/transient/transient.lua +++ b/src/program/loadtest/transient/transient.lua @@ -22,7 +22,7 @@ local function fatal (msg) end local function show_usage(code) - print(require("program.lwaftr.loadtest.README_inc")) + print(require("program.loadtest.transient.README_inc")) main.exit(code) end From d9ac3666f766231cac19192af8929a1b9a8fef5e Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 19 Feb 2018 12:00:57 +0000 Subject: [PATCH 054/100] Fix find-limit by checking actual throughput rather than requested. This also includes some changes to increase the default warmup time and perform a brief warm-up before each load we apply. --- .../loadtest/find-limit/find-limit.lua | 94 ++++++++++++------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua index 55fe7e1ee5..7a8dd5bfa0 100644 --- a/src/program/loadtest/find-limit/find-limit.lua +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -13,7 +13,7 @@ local numa = require("lib.numa") local promise = require("program.loadtest.promise") local WARM_UP_BIT_RATE = 1e9 -local WARM_UP_TIME = 1 +local WARM_UP_TIME = 5 local function fatal (msg) print(msg) @@ -31,22 +31,28 @@ local function find_limit(tester, max_bitrate, precision, duration, retry_count) end -- lo and hi are bitrates, in bits per second. - local function bisect(lo, hi, iter) - local function continue(cur, result) + local function bisect(lo, hi, iter, actual) + local function continue(cur, result, actual) if result then print("Success.") - return bisect(cur, hi, 1) + return bisect(cur, hi, 1, actual) elseif iter <= retry_count then print("Failed; "..(retry_count - iter).. " retries remaining.") - return bisect(lo, hi, iter + 1) + return bisect(lo, hi, iter + 1, actual) else print("Failed.") - return bisect(lo, cur, 1) + return bisect(lo, cur, 1, actual) end end local cur = round((lo + hi) / 2) - if cur == lo or cur == hi then return lo end - return tester.measure(cur, duration): + if cur == lo or cur == hi then + print(round(actual or lo) * 1e-9) + return lo + end + + -- We need to + + return tester.start_load(cur, duration): and_then(continue, cur) end return bisect(0, round(max_bitrate), 1) @@ -126,32 +132,16 @@ function run(args) rxdrop = nic_app:rxdrop() } end - local function print_diff(diff, duration) - local function bitrate(packets, bytes) - -- 7 bytes preamble, 1 start-of-frame, 4 CRC, 12 interframe gap. - local overhead = 7 + 1 + 4 + 12 - return (bytes + packets * overhead) * 8 / duration - end - local tx_mpps = diff.txpackets / duration / 1e6 - local tx_gbps = bitrate(diff.txpackets, diff.txbytes) / 1e9 - local rx_mpps = diff.rxpackets / duration / 1e6 - local rx_gbps = bitrate(diff.rxpackets, diff.rxbytes) / 1e9 - local lost_packets = diff.txpackets - diff.rxpackets - diff.rxdrop - local lost_percent = lost_packets / diff.txpackets * 100 - print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', - diff.txpackets, tx_mpps, diff.txbytes, tx_gbps)) - print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', - diff.rxpackets, rx_mpps, diff.rxbytes, rx_gbps)) - print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', - diff.rxdrop, lost_packets, lost_percent)) + local function print_stats(s) end local function check_results(diff) + local tx_bitrate = diff.tx_gbps * 1e9 if opts.exec then -- Could pass on some arguments to this string. - return os.execute(opts.exec) == 0 + return os.execute(opts.exec) == 0, tx_bitrate else - return diff.rxpackets == diff.txpackets and diff.rxdrop == 0 + return diff.rxpackets == diff.txpackets and diff.rxdrop == 0, tx_bitrate end end @@ -172,26 +162,60 @@ function run(args) return tester.generate_load(WARM_UP_BIT_RATE, WARM_UP_TIME) end + local function compute_bitrate(packets, bytes, duration) + -- 7 bytes preamble, 1 start-of-frame, 4 CRC, 12 interframe gap. + local overhead = 7 + 1 + 4 + 12 + return (bytes + packets * overhead) * 8 / duration + end + + function tester.start_load(bitrate, duration) + return tester.generate_load(WARM_UP_BIT_RATE, 1): + and_then(promise.Wait, 0.002): + and_then(tester.measure, bitrate, duration) + end + function tester.measure(bitrate, duration) local gbps_bitrate = bitrate/1e9 local start_counters = read_counters() - local function report() + local function compute_stats() local end_counters = read_counters() - local diff = {} + local s = {} for k,v in pairs(start_counters) do - diff[k] = tonumber(end_counters[k] - start_counters[k]) + s[k] = tonumber(end_counters[k] - start_counters[k]) end - print_diff(diff, duration) - return diff + s.applied_gbps = gbps_bitrate + s.tx_mpps = s.txpackets / duration / 1e6 + s.tx_gbps = compute_bitrate(s.txpackets, s.txbytes, duration) / 1e9 + s.rx_mpps = s.rxpackets / duration / 1e6 + s.rx_gbps = compute_bitrate(s.rxpackets, s.rxbytes, duration) / 1e9 + s.lost_packets = s.txpackets - s.rxpackets - s.rxdrop + s.lost_percent = s.lost_packets / s.txpackets * 100 + print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', + s.txpackets, s.tx_mpps, s.txbytes, s.tx_gbps)) + print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', + s.rxpackets, s.rx_mpps, s.rxbytes, s.rx_gbps)) + print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', + s.rxdrop, s.lost_packets, s.lost_percent)) + return s + end + local function verify_load(s) + if s.tx_gbps < 0.5 * s.applied_gbps then + print("Invalid result.") + return tester.start_load(bitrate, duration) + else + return check_results(s) + end end print(string.format('Applying %f Gbps of load.', gbps_bitrate)) return tester.generate_load(bitrate, duration): -- Wait 2ms for packets in flight to arrive and_then(promise.Wait, 0.002): - and_then(report): - and_then(check_results) + and_then(compute_stats): + and_then(verify_load) end + io.stdout:setvbuf("line") + engine.busywait = true local is_done = false local function mark_done() is_done = true end From 58883b1188da861a3e279ee9bdbb174b7bac48d2 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 20 Feb 2018 14:24:13 +0100 Subject: [PATCH 055/100] snabb top: minor formatting fixes 1. Widen the value column in list_shm a bit. 2. Do not emit trailing whitespace for trailing columns. --- src/program/top/top.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 61e98896fe..ef4f6de88e 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -69,7 +69,7 @@ function list_shm (pid, object) table.sort(sorted) for _, name in ipairs(sorted) do if name ~= 'path' and name ~= 'specs' and name ~= 'readonly' then - print_row({30, 30}, {name, tostring(frame[name])}) + print_row({30, 47}, {name, tostring(frame[name])}) end end shm.delete_frame(frame) @@ -195,14 +195,14 @@ function print_link_metrics (new_stats, last_stats) end end -function pad_str (s, n) +function pad_str (s, n, no_pad) local padding = math.max(n - s:len(), 0) - return ("%s%s"):format(s:sub(1, n), (" "):rep(padding)) + return ("%s%s"):format(s:sub(1, n), (no_pad and "") or (" "):rep(padding)) end function print_row (spec, args) for i, s in ipairs(args) do - io.write((" %s"):format(pad_str(s, spec[i]))) + io.write((" %s"):format(pad_str(s, spec[i], i == #args))) end io.write("\n") end From ba390ecc3bf689426cbf16c85c99d8b93790c17e Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Thu, 22 Feb 2018 21:12:21 +0100 Subject: [PATCH 056/100] src/snabb wrapper when building container snabb --- Dockerfile | 3 +++ Makefile | 3 +++ launch-docker-container.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100755 launch-docker-container.sh diff --git a/Dockerfile b/Dockerfile index e9175e1bd3..b6cfc14cf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,7 @@ FROM alpine:3.7 RUN apk add --no-cache libgcc COPY --from=build /snabb/src/snabb /usr/local/bin/ +VOLUME /u +WORKDIR /u + ENTRYPOINT ["/usr/local/bin/snabb"] diff --git a/Makefile b/Makefile index 797e387485..1bfb7a1295 100644 --- a/Makefile +++ b/Makefile @@ -49,5 +49,8 @@ dist: all docker: docker build -t snabb . + @chmod a+rx launch-docker-container.sh + @ln -sf ../launch-docker-container.sh src/snabb @echo "Usage: docker run -ti --rm snabb ..." + @echo "or simply call 'src/snabb ...'" .SERIAL: all diff --git a/launch-docker-container.sh b/launch-docker-container.sh new file mode 100755 index 0000000000..a2ee784bce --- /dev/null +++ b/launch-docker-container.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +name=$(basename $0) +if [ ".$name" == ".launch-docker-container.sh" ]; then + cat < Date: Thu, 22 Feb 2018 21:13:36 +0100 Subject: [PATCH 057/100] document snabb wrapper script --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 2d7d3d9314..31c1cc46a1 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,13 @@ This will build a tiny snabb container (8MB), ready to be used: $ docker run -ti --rm snabb --help ``` +Or simply run snabb, as you would under linux. This is made possible by using a wrapper shell script that +gets linked to as part of 'make docker': + +``` +$ src/snabb --help +``` + ## How do I get involved? Here are the ways you can get involved: From e9ee18a2a45df1117b4eaa2bb0b8aa7c65574b21 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Wed, 28 Feb 2018 09:36:31 +0000 Subject: [PATCH 058/100] README.md: Add link to Slack auto-invite page --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1cfa9dab73..cab1ff7c88 100644 --- a/README.md +++ b/README.md @@ -121,5 +121,5 @@ Here are the ways you can get involved: - Use the Snabb applications in your network. - Create your very own application: [Getting Started](src/doc/getting-started.md). - Create Github Issues with your ideas and questions and problems. -- Hang out on the [Snabb Slack chat](https://snabb.slack.com/). You can get a no-questions-asked invitation by mailing `luke@snabb.co` with the addresses/domains you want invited. +- [Join](https://join.slack.com/t/snabb/shared_invite/enQtMzIyOTIwMTg5ODYyLTMzY2FjMGEzM2QzNDlhMDYxNzU0M2UyNjQ1MDc4MDRjY2Q3MWMwY2Q4YWQ1NDllY2E3NTZkZGUyZTQxNzgyNjc) the [Snabb Slack chat](https://snabb.slack.com/) to hang out and shoot the breeze. From 664f3786a0a7ba310246dde3f14b34b20e148150 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 1 Mar 2018 20:59:20 +0100 Subject: [PATCH 059/100] intel_app: fix skipped message with regard to 2468f7c13 --- src/apps/intel/intel_app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/intel/intel_app.lua b/src/apps/intel/intel_app.lua index de835e8ed6..82defdeaa4 100644 --- a/src/apps/intel/intel_app.lua +++ b/src/apps/intel/intel_app.lua @@ -255,7 +255,7 @@ function selftest () local pcideva = lib.getenv("SNABB_PCI_INTEL0") local pcidevb = lib.getenv("SNABB_PCI_INTEL1") if not pcideva or not pcidevb then - print("SNABB_PCI_INTEL[0|1]/SNABB_PCI[0|1] not set or not suitable.") + print("SNABB_PCI_INTEL[0|1] not set or not suitable.") os.exit(engine.test_skipped_code) end From b9ae0d7a7710a0f15be007a40527e2200d3f69c3 Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Fri, 2 Mar 2018 08:46:54 -0500 Subject: [PATCH 060/100] adjust if statement to work in dash too --- launch-docker-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch-docker-container.sh b/launch-docker-container.sh index a2ee784bce..f81fcb7800 100755 --- a/launch-docker-container.sh +++ b/launch-docker-container.sh @@ -1,7 +1,7 @@ #!/bin/sh name=$(basename $0) -if [ ".$name" == ".launch-docker-container.sh" ]; then +if [ ".$name" = ".launch-docker-container.sh" ]; then cat < Date: Wed, 14 Mar 2018 12:58:50 +0000 Subject: [PATCH 061/100] Check status code in monitor test --- src/program/lwaftr/tests/subcommands/monitor_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py index 4a298e64bd..05fc629f8c 100644 --- a/src/program/lwaftr/tests/subcommands/monitor_test.py +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -6,7 +6,7 @@ """ from random import randint -from subprocess import call, check_call +from subprocess import call, check_call, PIPE, Popen import unittest from test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names @@ -46,11 +46,11 @@ def setUpClass(cls): raise def test_monitor(self): - output = self.run_cmd(self.monitor_args) - self.assertIn(b'Mirror address set', output, - b'\n'.join((b'OUTPUT', output))) - self.assertIn(b'255.255.255.255', output, - b'\n'.join((b'OUTPUT', output))) + proc = Popen(self.monitor_args, stdout=PIPE, stderr=PIPE) + proc.wait() + proc.stdout.close() + proc.stderr.close() + assert(proc.returncode == 0) @classmethod def tearDownClass(cls): From da3a2f825719d3b3568da6eab14bbc45e89b5807 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 19 Mar 2018 10:41:02 +0000 Subject: [PATCH 062/100] Make supervisor setup more reliable There was a race condition when setting up the supervisor such that in some cases it was possible to miss a signal when the parent process died. We could reproduce this with by running a "snabb lwaftr bench", but only on our test machine with two NUMA nodes and only when setting --cpu on the lwaftr. In that case the problem would appear when running "snabb lwaftr monitor" on the lwaftr; the monitor process would hang reading from the signalfd. Because the monitor process still had stdout open, then when piping its output to "grep", the grep process would hang because the write side of its stdin pipe would still be open as well. This patch fixes this error by making cleanup reliable. It does so by taking a POSIX lock on an unnamed file in the parent, then taking another lock from the supervisor child process. In this way we avoid some of the more arcane parts of Linux. --- src/core/main.lua | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 39aed20cfb..5d3f552c07 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -225,23 +225,23 @@ end -- Fork a child process that monitors us and performs cleanup actions -- when we terminate. local snabbpid = S.getpid() +local lockfile = os.tmpname() +local lock = S.open(lockfile, "wronly") +S.unlink(lockfile) +S.sigprocmask("block", "hup, int, quit, term") +lock:lockf("lock", 0) if assert(S.fork()) ~= 0 then - -- parent process: run snabb + -- Parent process; run Snabb. + S.sigprocmask("unblock", "hup, int, quit, term") xpcall(main, handler) + -- Lock will be released however the process exits. else - -- child process: supervise parent & perform cleanup - -- Subscribe to SIGHUP on parent death + -- Child process: Supervise parent & perform cleanup. Lock not + -- inherited from parent. S.prctl("set_name", "[snabb sup]") - S.prctl("set_pdeathsig", "hup") - -- Trap relevant signals to a file descriptor - local exit_signals = "hup, int, quit, term" - local signalfd = S.signalfd(exit_signals) - S.sigprocmask("block", exit_signals) - -- wait until we receive a signal - local signals - repeat signals = assert(S.util.signalfd_read(signalfd)) until #signals > 0 - -- cleanup after parent process + -- Wait for parent to release lock. + lock:lockf("lock", 0) + -- Finally, clean up after parent process. shutdown(snabbpid) - -- exit with signal-appropriate status - os.exit(128 + signals[1].signo) + os.exit(128) end From 459ee99839fd39a07761cc0fcdeff0a27812082e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 19 Mar 2018 10:41:02 +0000 Subject: [PATCH 063/100] Make supervisor setup more reliable There was a race condition when setting up the supervisor such that in some cases it was possible to miss a signal when the parent process died. We could reproduce this with by running a "snabb lwaftr bench", but only on our test machine with two NUMA nodes and only when setting --cpu on the lwaftr. In that case the problem would appear when running "snabb lwaftr monitor" on the lwaftr; the monitor process would hang reading from the signalfd. Because the monitor process still had stdout open, then when piping its output to "grep", the grep process would hang because the write side of its stdin pipe would still be open as well. This patch fixes this error by making cleanup reliable. It does so by taking a POSIX lock on an unnamed file in the parent, then taking another lock from the supervisor child process. In this way we avoid some of the more arcane parts of Linux. --- src/core/main.lua | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 39aed20cfb..5d3f552c07 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -225,23 +225,23 @@ end -- Fork a child process that monitors us and performs cleanup actions -- when we terminate. local snabbpid = S.getpid() +local lockfile = os.tmpname() +local lock = S.open(lockfile, "wronly") +S.unlink(lockfile) +S.sigprocmask("block", "hup, int, quit, term") +lock:lockf("lock", 0) if assert(S.fork()) ~= 0 then - -- parent process: run snabb + -- Parent process; run Snabb. + S.sigprocmask("unblock", "hup, int, quit, term") xpcall(main, handler) + -- Lock will be released however the process exits. else - -- child process: supervise parent & perform cleanup - -- Subscribe to SIGHUP on parent death + -- Child process: Supervise parent & perform cleanup. Lock not + -- inherited from parent. S.prctl("set_name", "[snabb sup]") - S.prctl("set_pdeathsig", "hup") - -- Trap relevant signals to a file descriptor - local exit_signals = "hup, int, quit, term" - local signalfd = S.signalfd(exit_signals) - S.sigprocmask("block", exit_signals) - -- wait until we receive a signal - local signals - repeat signals = assert(S.util.signalfd_read(signalfd)) until #signals > 0 - -- cleanup after parent process + -- Wait for parent to release lock. + lock:lockf("lock", 0) + -- Finally, clean up after parent process. shutdown(snabbpid) - -- exit with signal-appropriate status - os.exit(128 + signals[1].signo) + os.exit(128) end From 9e2f5da31e0206884c0314484429c499b9be07be Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Mar 2018 14:31:40 +0000 Subject: [PATCH 064/100] Temporarily disable ipsec selftest --- src/apps/ipsec/selftest.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/apps/ipsec/selftest.sh b/src/apps/ipsec/selftest.sh index f2c8f5b882..fc4b8f549e 100755 --- a/src/apps/ipsec/selftest.sh +++ b/src/apps/ipsec/selftest.sh @@ -3,6 +3,9 @@ SKIPPED_CODE=43 +# Temporary disabled test. +exit $SKIPPED_CODE + if [ "$SNABB_IPSEC_SKIP_E2E_TEST" = yes ]; then exit $SKIPPED_CODE fi From b4629a759bc37541ab0772850c498d1ca67c350d Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 19 Mar 2018 15:29:24 +0100 Subject: [PATCH 065/100] Snabb lwAFTR v2018.01.2.1 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d68499be55..81e6d4fd6c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2018.03.01 +2018.01.2.1 From 1ad64e40fb749ff3d75d2f3f49a77afb5432f83c Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 16 Mar 2018 15:35:46 +0100 Subject: [PATCH 066/100] Changelog 2018.01.2.1 --- src/program/lwaftr/doc/CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index e430533ff7..a079dc0b44 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,31 @@ # Change Log +## [2018.01.2.1] + +* Features: + + - Added limit-finding loadtester. See: + + https://github.com/Igalia/snabb/blob/lwaftr/src/program/loadtest/find-limit/README.inc + + - Move "loadtest" command out of lwaftr. Now the "loadtest" command consists of + two subcommands: "transient" and "find-limit". Example: + + $ sudo ./snabb loadtest transient -D 1 -b 5e9 -s 0.2e9 \ + cap1.pcap "NIC 0" "NIC 1" 01:00.0 \ + cap2.pcap "NIC 1" "NIC 0" 01:00.1 + + $ sudo ./snabb loadtest find-limit 01:00.0 cap1.pcap + +* Bug fixes: + + - Fix next-hop discovery with multiple devices (Issue: https://github.com/Igalia/snabb/issues/1014). + - Improve effectiveness of property-based tests. + - Process tree runs data-plane processes with busywait=true by default + - Remove early lwAFTR NUMA affinity check. The check was unnecessary since + now ptree manager handles NUMA affinity and appropriate CPU selection. + - Sizes for "packetblaster lwaftr" are frame sizes. + ## [2017.11.01] * Add --trace option to "snabb lwaftr run", enabling a trace log of From 0736ffd7d2c6149938c9e8f671dbb9a6343c6135 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 21 Mar 2018 17:13:09 +0100 Subject: [PATCH 067/100] Fix out-of-bounds write in ctable test suite This is another instance of the bug from commit 93ef6bdbbd92eab4b96790f825cefec3988dc65a. We didn't see any issue on upstream Snabb's test suites, but with RaptorJIT's new LJ_GC64 usage did manifest itself as intermittent heap corruption. Fixes https://github.com/snabbco/snabb/issues/1307. --- src/lib/ctable.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 96df39bf8f..03fb8d3035 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -682,7 +682,8 @@ function selftest() -- keep references to avoid GCing too early local handle = {} local function read(size) - local buf = ffi.new('uint8_t[?]', size, file:read(size)) + local buf = ffi.new('uint8_t[?]', size) + ffi.copy(buf, file:read(size), size) table.insert(handle, buf) return buf end From 80109514b2fe6af89db718781308fe52620d77c7 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 22 Mar 2018 17:37:43 +0000 Subject: [PATCH 068/100] XPath formater fixes - If root is '/' set to empty. - Remember carried 'path'. --- src/lib/yang/data.lua | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 5eef38b416..045b52de26 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -792,8 +792,8 @@ local function print_yang_string(str, file) end function xpath_printer_from_grammar(production, print_default, root) - if #root > 1 and root:sub(#root, #root) == '/' then - root = root:sub(1, #root-1) + if #root == 1 and root:sub(1, 1) == '/' then + root = '' end local handlers = {} local translators = {} @@ -903,34 +903,38 @@ function xpath_printer_from_grammar(production, print_default, root) local print_value = body_printer(production.values, value_order) if production.key_ctype and production.value_ctype then return function(data, file, path) + path = path or '' for entry in data:iterate() do local key = compose_key(entry.key) - local path = keyword and keyword..key..'/' or key..'/' + local path = path..(keyword or '')..key..'/' print_value(entry.value, file, path) end end elseif production.string_key then local id = normalize_id(production.string_key) return function(data, file, path) + path = path or '' for key, value in pairs(data) do local key = compose_key({[id]=key}) - local path = keyword and keyword..key..'/' or key..'/' + local path = path..(keyword or '')..key..'/' print_value(value, file, path) end end elseif production.key_ctype then return function(data, file, path) + path = path or '' for key, value in cltable.pairs(data) do local key = compose_key(key) - local path = keyword and keyword..key..'/' or key..'/' + local path = path..(keyword or '')..key..'/' print_value(value, file, path) end end else return function(data, file, path) + path = path or '' for key, value in pairs(data) do local key = compose_key(key) - local path = keyword and keyword..key..'/' or key..'/' + local path = path..(keyword or '')..key..'/' print_value(value, file, path) end end From a0959685fe1e313da72cdc7539a6cc58b4d06cf0 Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Fri, 23 Mar 2018 17:52:02 +0100 Subject: [PATCH 069/100] merging launch-docker-container.sh with dock.sh --- Makefile | 3 +-- launch-docker-container.sh | 33 --------------------------------- src/scripts/dock.sh | 3 +++ 3 files changed, 4 insertions(+), 35 deletions(-) delete mode 100755 launch-docker-container.sh diff --git a/Makefile b/Makefile index 1bfb7a1295..1348eb7fab 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,7 @@ dist: all docker: docker build -t snabb . - @chmod a+rx launch-docker-container.sh - @ln -sf ../launch-docker-container.sh src/snabb + @ln -sf ../src/scripts/dock.sh src/snabb @echo "Usage: docker run -ti --rm snabb ..." @echo "or simply call 'src/snabb ...'" .SERIAL: all diff --git a/launch-docker-container.sh b/launch-docker-container.sh deleted file mode 100755 index f81fcb7800..0000000000 --- a/launch-docker-container.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -name=$(basename $0) -if [ ".$name" = ".launch-docker-container.sh" ]; then - cat < Date: Sat, 24 Mar 2018 09:04:06 +0100 Subject: [PATCH 070/100] use separate docker run commands --- src/scripts/dock.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/scripts/dock.sh b/src/scripts/dock.sh index 8b69ac0370..377af3292f 100755 --- a/src/scripts/dock.sh +++ b/src/scripts/dock.sh @@ -1,12 +1,21 @@ #!/usr/bin/env bash name=$(basename $0) -if [ "$name" != "dock.sh" ]; then export SNABB_TEST_IMAGE=$name; fi -export SNABB_TEST_IMAGE=${SNABB_TEST_IMAGE:=eugeneia/snabb-nfv-test-vanilla} +if [ "$name" != "dock.sh" ]; then -# Snabb Docker environment + img=$(docker images -q $name) + if [ -z "$img" ]; then + echo "docker image $name doesn't exist" + fi + exec docker run -ti --rm -v ${PWD}:/u --workdir /u $name $@ -docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ +else + + export SNABB_TEST_IMAGE=${SNABB_TEST_IMAGE:=eugeneia/snabb-nfv-test-vanilla} + + # Snabb Docker environment + + docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ --workdir /snabb \ -e SNABB_PCI0=$SNABB_PCI0 \ -e SNABB_PCI1=$SNABB_PCI1 \ @@ -27,3 +36,4 @@ docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ -e SNABB_IPSEC_SKIP_E2E_TEST=$SNABB_IPSEC_SKIP_E2E_TEST \ $SNABB_TEST_IMAGE \ bash -c "mount -t hugetlbfs none /hugetlbfs && (cd snabb/src; $*)" +fi From b7f6ab26880673fd109b30179e024e531b12078f Mon Sep 17 00:00:00 2001 From: Marcel Wiget Date: Sat, 24 Mar 2018 12:08:54 +0100 Subject: [PATCH 071/100] run container privileged and build src --- Dockerfile | 2 +- src/scripts/dock.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6cfc14cf6..6ad95a42cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.7 AS build RUN apk add --no-cache libgcc alpine-sdk gcc libpcap-dev linux-headers findutils COPY . /snabb -RUN cd /snabb && make clean && make -j +RUN cd /snabb && make clean && make -j && cd src && make -j FROM alpine:3.7 RUN apk add --no-cache libgcc diff --git a/src/scripts/dock.sh b/src/scripts/dock.sh index 377af3292f..f67a3d1440 100755 --- a/src/scripts/dock.sh +++ b/src/scripts/dock.sh @@ -7,7 +7,7 @@ if [ "$name" != "dock.sh" ]; then if [ -z "$img" ]; then echo "docker image $name doesn't exist" fi - exec docker run -ti --rm -v ${PWD}:/u --workdir /u $name $@ + exec docker run -ti --rm --privileged -v ${PWD}:/u --workdir /u $name $@ else @@ -16,7 +16,6 @@ else # Snabb Docker environment docker run --rm --privileged -i -v $(dirname $PWD):/snabb $DOCKERFLAGS \ - --workdir /snabb \ -e SNABB_PCI0=$SNABB_PCI0 \ -e SNABB_PCI1=$SNABB_PCI1 \ -e SNABB_PCI_INTEL0=$SNABB_PCI_INTEL0 \ From c55b21bdbeea0c857ecf2e9a56a2e21c65ff7497 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 27 Mar 2018 18:10:51 +0000 Subject: [PATCH 072/100] Fix tests --- .../lwaftr/tests/subcommands/config_test.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 99db0366ac..30ba43e8a1 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -10,6 +10,7 @@ from subprocess import PIPE, Popen import time import unittest +import re from test_env import BENCHDATA_DIR, DATA_DIR, ENC, SNABB_CMD, \ DAEMON_STARTUP_WAIT, BaseTestCase, nic_names @@ -239,14 +240,13 @@ def test_snabb_get_state_summation(self): for line in state: if "softwire-state" not in line: continue + [cname, cvalue] = line.split(" ") + cname = os.path.basename(cname) + cvalue = int(cvalue) - path = [elem for elem in line.split("/") if elem] - cname = path[-1].split()[0] - cvalue = int(path[-1].split()[1]) - - if path[0].startswith("instance"): + if line.startswith("/softwire-config"): instance[cname] = instance.get(cname, 0) + cvalue - elif len(path) < 3: + elif line.startswith("/softwire-state"): summed[cname] = cvalue # Now assert they're the same :) @@ -268,12 +268,13 @@ def test_snabb_get_state_lists_instances(self): instances = set() for line in state: - path = [elem for elem in line.split("/") if elem] - if not path[0].startswith("instance"): + [key, value] = line.split(" ") + if key.startswith("/softwire-config") and "instance" not in key: continue - - device_name = path[0][path[0].find("=")+1:-1] - instances.add(device_name) + m = re.search(r"\[device=(.*)\]", key) + if m: + device_name = m.group(1) + instances.add(device_name) self.assertTrue(len(instances) == 2) self.assertTrue("test" in instances) From 25c8ad6a579fb97097d69a93e10735d71aff805d Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 14 Dec 2017 08:06:13 +0100 Subject: [PATCH 073/100] Implement alarm shelving --- src/lib/yang/alarms.lua | 95 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/src/lib/yang/alarms.lua b/src/lib/yang/alarms.lua index 58a6c6e6f9..a94a98eccc 100644 --- a/src/lib/yang/alarms.lua +++ b/src/lib/yang/alarms.lua @@ -9,6 +9,12 @@ local counter = require("core.counter") local format_date_as_iso_8601 = util.format_date_as_iso_8601 local parse_date_as_iso_8601 = util.parse_date_as_iso_8601 +local control = { + alarm_shelving = { + shelf = {} + } +} + local state = { alarm_inventory = { alarm_type = {}, @@ -17,8 +23,17 @@ local state = { alarm = {}, number_of_alarms = 0, }, + shelved_alarms = { + shelved_alarms = {} + } } +local function table_size (t) + local size = 0 + for _ in pairs(t) do size = size + 1 end + return size +end + function get_state () -- status-change is stored as an array while according to ietf-alarms schema -- it should be a hashmap indexed by time. @@ -102,6 +117,8 @@ function build_summary (alarms) end ret[severity] = entry end + local shelved_alarms = table_size(state.shelved_alarms.shelved_alarms) + if shelved_alarms > 0 then ret['shelved_alarms'] = shelved_alarms end return ret end @@ -182,13 +199,6 @@ function alarm_keys:normalize (key) return self:fetch(resource, alarm_type_id, alarm_type_qualifier) end -local function table_size (t) - local size = 0 - for _ in pairs(t) do size = size + 1 end - return size -end - - -- Contains a table with all the declared alarms. local alarm_list = { list = {}, @@ -335,9 +345,17 @@ local function update_alarm (alarm, args) end end +local function is_shelved(key) + return control.alarm_shelving.shelf[key] +end + -- Check up if the alarm already exists in state.alarm_list. local function lookup_alarm (key) - return state.alarm_list.alarm[key] + if is_shelved(key) then + return state.shelved_alarms.shelved_alarms[key] + else + return state.alarm_list.alarm[key] + end end function raise_alarm (key, args) @@ -365,6 +383,22 @@ function clear_alarm (key) end end +-- Alarm shelving. + +function shelve_alarm (key, alarm) + alarm = alarm or state.alarm_list.alarm[key] + state.shelved_alarms.shelved_alarms[key] = alarm + state.alarm_list.alarm[key] = nil + control.alarm_shelving.shelf[key] = true +end + +function unshelve_alarm (key, alarm) + alarm = alarm or state.shelved_alarms.shelved_alarms[key] + state.alarm_list.alarm[key] = alarm + state.shelved_alarms.shelved_alarms[key] = nil + control.alarm_shelving.shelf[key] = nil +end + -- Set operator state. local operator_states = lib.set('none', 'ack', 'closed', 'shelved', 'un-shelved') @@ -373,9 +407,12 @@ function set_operator_state (key, args) assert(args.state and operator_states[args.state], 'Not a valid operator state: '..args.state) key = alarm_keys:normalize(key) - local alarm = state.alarm_list.alarm[key] - if not alarm then - error('Set operate state operation failed. Could not locate alarm.') + local alarm + if args.state == 'un-shelved' then + alarm = assert(state.shelved_alarms.shelved_alarms[key], 'Could not locate alarm in shelved-alarms') + control.alarm_shelving.shelf[key] = nil + else + alarm = assert(state.alarm_list.alarm[key], 'Could not locate alarm in alarm-list') end if not alarm.operator_state_change then alarm.operator_state_change = {} @@ -387,6 +424,11 @@ function set_operator_state (key, args) state = args.state, text = args.text, }) + if args.state == 'shelved' then + shelve_alarm(key, alarm) + elseif args.state == 'un-shelved' then + unshelve_alarm(key, alarm) + end return true end @@ -785,5 +827,36 @@ function selftest () assert(table_size(alarm.operator_state_change) == 1) assert(purge_alarms({operator_state_filter={state='ack'}}) == 1) + -- Shelving and alarm should: + -- - Add shelving criteria to alarms/control. + -- - Move alarm from alarms/alarm-list to alarms/shelved-alarms. + -- - Do not generate notifications if the alarm changes its status. + -- - Increase the number of shelved alarms in summary. + local key = alarm_keys:fetch('nic-v4', 'arp-resolution') + raise_alarm(key, {perceived_severity='minor'}) + local success = set_operator_state(key, {state='shelved'}) + assert(success) + assert(table_size(control.alarm_shelving.shelf) == 1) + assert(table_size(state.shelved_alarms.shelved_alarms) == 1) + + -- Changing alarm status should create a new status in shelved alarm. + alarm = state.shelved_alarms.shelved_alarms[key] + assert(table_size(alarm.status_change) == 1) + raise_alarm(key, {perceived_severity='critical'}) + assert(table_size(state.alarm_list.alarm) == 0) + assert(table_size(alarm.status_change) == 2) + + -- Un-shelving and alarm should: + -- - Remove shelving criteria from alarms/control. + -- - Move alarm from alarms/shelved-alarms to alarms/alarm-list. + -- - The alarm now generates notifications if it changes its status. + -- - Decrease the number of shelved alarms in summary. + local success = set_operator_state(key, {state='un-shelved'}) + assert(success) + assert(table_size(control.alarm_shelving.shelf) == 0) + raise_alarm(key, {perceived_severity='critical'}) + assert(not state.shelved_alarms.shelved_alarms[key]) + assert(state.alarm_list.alarm[key]) + print("ok") end From 7f6791de7a944775999a12b9a14752c1da063040 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 29 Mar 2018 21:23:57 +0000 Subject: [PATCH 074/100] Add failsafe parameter to common.parse_args In the case a command that relies on the common.parse_args function needs to do additional parsing, the failsafe mode prevents the common.parse_args function from failing if there are leftover arguments after parsing --- src/program/config/common.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 6680e08f01..0bf01f1ad7 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -24,6 +24,7 @@ local parse_command_line_opts = { require_schema = { default=false }, is_config = { default=true }, usage = { default=show_usage }, + failsafe = { default=false }, } local function path_grammar(schema_name, path, is_config) @@ -120,8 +121,8 @@ function parse_command_line(args, opts) end ret.value = parser(ret.value_str) end - if #args ~= 0 then err("too many arguments") end - return ret + if not opts.failsafe and #args ~= 0 then err("too many arguments") end + return ret, args end function open_socket_or_die(instance_id) From 76d0a89fe331e47dcdc6671ce23aad3c41091a7e Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 29 Mar 2018 21:25:55 +0000 Subject: [PATCH 075/100] Refactor argument parsing --- .../set_operator_state/set_operator_state.lua | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/program/alarms/set_operator_state/set_operator_state.lua b/src/program/alarms/set_operator_state/set_operator_state.lua index 48fcfe0aa9..9156c3704a 100644 --- a/src/program/alarms/set_operator_state/set_operator_state.lua +++ b/src/program/alarms/set_operator_state/set_operator_state.lua @@ -3,43 +3,39 @@ module(..., package.seeall) local common = require("program.config.common") -function show_usage(command, status, err_msg) +function show_usage(status, err_msg) if err_msg then print('error: '..err_msg) end print(require("program.alarms.set_operator_state.README_inc")) main.exit(status) end local function fatal() - show_usage(nil, 1) + show_usage(1) end local function parse_args (args) - if #args < 4 or #args > 5 then fatal() end - local alarm_type_id, alarm_type_qualifier = (args[3]):match("([%w]+)/([%w]+)") + if #args < 3 or #args > 4 then fatal() end + local alarm_type_id, alarm_type_qualifier = (args[2]):match("([%w]+)/([%w]+)") if not alarm_type_id then - alarm_type_id, alarm_type_qualifier = args[3], '' + alarm_type_id, alarm_type_qualifier = args[2], '' end local ret = { key = { - resource = args[2], + resource = args[1], alarm_type_id = alarm_type_id, alarm_type_qualifier = alarm_type_qualifier, }, - state = args[4], - text = args[5] or '', + state = args[3], + text = args[4] or '', } - -- Remove all arguments except first one. - for i=2,#args do - table.remove(args, #args) - end return ret end function run(args) - local l_args = parse_args(args) local opts = { command='set-alarm-operator-state', with_path=false, is_config=false, - usage = show_usage } - args = common.parse_command_line(args, opts) + usage = show_usage, failsafe = true } + local args, cdr = common.parse_command_line(args, opts) + local l_args = parse_args(cdr) local response = common.call_leader( args.instance_id, 'set-alarm-operator-state', { schema = args.schema_name, revision = args.revision_date, From 72b9b3c40c545324de6f26d8add0082ac0cedab6 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 13 Apr 2018 11:46:02 +0000 Subject: [PATCH 076/100] .version: bump to 2018.04 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 29275cd626..7b63fdf2ae 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2018.01 +2018.04 From 419c943b8e72a11a21631c0b63ac60ad5de70f13 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 16 Apr 2018 15:39:05 +0000 Subject: [PATCH 077/100] Fix error message --- .../alarms/set_operator_state/set_operator_state.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/alarms/set_operator_state/set_operator_state.lua b/src/program/alarms/set_operator_state/set_operator_state.lua index 9156c3704a..cb7567b777 100644 --- a/src/program/alarms/set_operator_state/set_operator_state.lua +++ b/src/program/alarms/set_operator_state/set_operator_state.lua @@ -3,14 +3,14 @@ module(..., package.seeall) local common = require("program.config.common") -function show_usage(status, err_msg) +function show_usage(program, status, err_msg) if err_msg then print('error: '..err_msg) end print(require("program.alarms.set_operator_state.README_inc")) main.exit(status) end local function fatal() - show_usage(1) + show_usage('set-operator-state', 1) end local function parse_args (args) @@ -33,7 +33,7 @@ end function run(args) local opts = { command='set-alarm-operator-state', with_path=false, is_config=false, - usage = show_usage, failsafe = true } + usage=show_usage, failsafe=true } local args, cdr = common.parse_command_line(args, opts) local l_args = parse_args(cdr) local response = common.call_leader( From 6e073324352642060f8c6743b720369858d1c4eb Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Apr 2018 15:39:02 +0200 Subject: [PATCH 078/100] Extend find-limit to support multiple NICs. This extends loadtest's find-limit command so that multiple sets of traffic and NICs can be configured. The support is similar to how the loadtest transient command accepts multiple NICs. --- src/program/loadtest/find-limit/README | 11 +- .../loadtest/find-limit/find-limit.lua | 187 +++++++++++------- 2 files changed, 122 insertions(+), 76 deletions(-) diff --git a/src/program/loadtest/find-limit/README b/src/program/loadtest/find-limit/README index 941cb2d481..282dbef994 100644 --- a/src/program/loadtest/find-limit/README +++ b/src/program/loadtest/find-limit/README @@ -1,4 +1,4 @@ -Usage: find-limit [OPTIONS] DEVICE PCAP-FILE +Usage: find-limit [OPTIONS] [ ]... -b BITRATE, --bitrate BITRATE Test bitrates up to BITRATE bits/second. @@ -22,8 +22,11 @@ Usage: find-limit [OPTIONS] DEVICE PCAP-FILE -h, --help Print usage information. -Apply load on DEVICE by replaying packets from PCAP-FILE, and attempt to -determine the highest bitrate at which a test passes. +Apply load by replaying packets from PCAP-FILE to the corresponding PCI network +adaptors. It will attempt to determin the highest bitrate at which a test passes. +If no script is supplied then find-limit will attempt to see if packets are being +dropped by checking the traffic sent vs traffic recieved. Examples: - find-limit 01:00.0 cap1.pcap + find-limit cap1.pcap tx tx cap1.pcap + find-limit cap1.pcap "NIC 0" "NIC 1" 01:00.0 cap2.pcap "NIC 1" "NIC 0" 01:00.0 diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua index 7a8dd5bfa0..3ad38e0185 100644 --- a/src/program/loadtest/find-limit/find-limit.lua +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -31,34 +31,32 @@ local function find_limit(tester, max_bitrate, precision, duration, retry_count) end -- lo and hi are bitrates, in bits per second. - local function bisect(lo, hi, iter, actual) - local function continue(cur, result, actual) + local function bisect(lo, hi, iter) + local function continue(cur, result) if result then print("Success.") - return bisect(cur, hi, 1, actual) + return bisect(cur, hi, 1) elseif iter <= retry_count then print("Failed; "..(retry_count - iter).. " retries remaining.") - return bisect(lo, hi, iter + 1, actual) + return bisect(lo, hi, iter + 1) else print("Failed.") - return bisect(lo, cur, 1, actual) + return bisect(lo, cur, 1) end end local cur = round((lo + hi) / 2) if cur == lo or cur == hi then - print(round(actual or lo) * 1e-9) - return lo + print(round(lo) * 1e-9) + return lo end - - -- We need to - + -- We need to return tester.start_load(cur, duration): and_then(continue, cur) end return bisect(0, round(max_bitrate), 1) end -function parse_args(args) +local function parse_args(args) local opts = { max_bitrate = 10e9, duration = 1, precision = 0.001e9, retry_count = 3 } local function parse_positive_number(prop) @@ -90,65 +88,100 @@ function parse_args(args) { bitrate="b", duration="D", precision="p", ["retry-count"]="r", help="h", cpu=1, exec="e"}) - if #args ~= 2 then show_usage(1) end - local device, capture_file = unpack(args) + + if #args == 0 or #args % 4 ~= 0 then show_usage(1) end + local streams, streams_by_tx_id, pci_devices = {}, {}, {} + for i=1,#args,4 do + local stream = {} + stream.pcap_file = args[i] + stream.tx_name = args[i+1] + stream.rx_name = args[i+2] + stream.tx_id = stream.tx_name:gsub('[^%w]', '_') + stream.rx_id = stream.rx_name:gsub('[^%w]', '_') + stream.tx_device = pci.device_info(args[i+3]) + stream.tx_driver = require(stream.tx_device.driver).driver + table.insert(streams, stream) + table.insert(pci_devices, stream.tx_device.pciaddress) + assert(streams_by_tx_id[streams.tx_id] == nil, 'Duplicate: '..stream.tx_name) + streams_by_tx_id[stream.tx_id] = stream + end + for _, stream in ipairs(streams) do + assert(streams_by_tx_id[stream.rx_id], 'Missing stream: '..stream.rx_id) + stream.rx_device = streams_by_tx_id[stream.rx_id].tx_device + end if opts.cpu then numa.bind_to_cpu(opts.cpu) end - numa.check_affinity_for_pci_addresses({device}) - return opts, device, capture_file + numa.check_affinity_for_pci_addresses(pci_devices) + return opts, streams end function run(args) - local opts, device, capture_file = parse_args(args) - local device_info = pci.device_info(device) - local driver = require(device_info.driver).driver - local c = config.new() + local opts, streams = parse_args(args) - -- Links are named directionally with respect to NIC apps, but we - -- want to name tx and rx with respect to the whole network - -- function. - local tx_link_name = device_info.rx - local rx_link_name = device_info.tx + local c = config.new() + for _, stream in ipairs(streams) do + stream.pcap_id = 'pcap_'..stream.tx_id + stream.repeater_id = 'repeater'..stream.tx_id + stream.nic_tx_id = 'nic_'..stream.tx_id + stream.nic_rx_id = 'nic_'..stream.rx_id + -- Links are named directionally with respect to NIC apps, but we + -- want to name tx and rx with respect to the whole network + -- function. + stream.nic_tx_link = stream.tx_device.rx + stream.nic_rx_link = stream.rx_device.tx + stream.rx_sink_id = 'rx_sink_'..stream.rx_id - config.app(c, "replay", PcapReader, capture_file) - config.app(c, "repeater", loadgen.RateLimitedRepeater, {}) - config.app(c, "nic", driver, { pciaddr = device_info.pciaddress }) - config.app(c, "blackhole", basic_apps.Sink) + config.app(c, stream.pcap_id, PcapReader, stream.pcap_file) + config.app(c, stream.repeater_id, loadgen.RateLimitedRepeater) + config.app(c, stream.nic_tx_id, stream.tx_driver, { pciaddr = stream.tx_device.pciaddress}) + config.app(c, stream.rx_sink_id, basic_apps.Sink) - config.link(c, "replay.output -> repeater.input") - config.link(c, "repeater.output -> nic."..tx_link_name) - config.link(c, "nic."..rx_link_name.." -> blackhole.input") + config.link(c, stream.pcap_id..".output -> "..stream.repeater_id..".input") + config.link(c, stream.repeater_id..".output -> "..stream.nic_tx_id.."."..stream.nic_tx_link) + config.link(c, stream.nic_rx_id.."."..stream.nic_rx_link.." -> "..stream.rx_sink_id..".input") + end engine.configure(c) - local nic_app = assert(engine.app_table.nic) - local repeater_app = assert(engine.app_table.repeater) - local function read_counters() - local tx, rx = nic_app.input[tx_link_name], nic_app.output[rx_link_name] - return { txpackets = counter.read(tx.stats.txpackets), - txbytes = counter.read(tx.stats.txbytes), - rxpackets = counter.read(rx.stats.txpackets), - rxbytes = counter.read(rx.stats.txbytes), - rxdrop = nic_app:rxdrop() } + local counters = {} + for _, stream in ipairs(streams) do + local tx_app = assert(engine.app_table[stream.nic_tx_id]) + local rx_app = assert(engine.app_table[stream.nic_rx_id]) + local tx, rx = tx_app.input[stream.nic_tx_link], rx_app.output[stream.nic_rx_link] + counters[stream.nic_tx_id] = { + txpackets = counter.read(tx.stats.txpackets), + txbytes = counter.read(tx.stats.txbytes), + rxpackets = counter.read(rx.stats.txpackets), + rxbytes = counter.read(rx.stats.txbytes), + rxdrop = rx_app:rxdrop() + } + end + return counters end local function print_stats(s) end - local function check_results(diff) - local tx_bitrate = diff.tx_gbps * 1e9 + local function check_results(stats) if opts.exec then - -- Could pass on some arguments to this string. - return os.execute(opts.exec) == 0, tx_bitrate - else - return diff.rxpackets == diff.txpackets and diff.rxdrop == 0, tx_bitrate + return os.execute(opts.exec) == 0 + end + + local success = true + for _, stream in ipairs(streams) do + local diff = stats[stream.nic_tx_id] + success = (diff.rxpackets == diff.txpackets and diff.rxdrop == 0) and success end + return success end local tester = {} function tester.adjust_rates(bit_rate) - repeater_app:set_rate(bit_rate) + for _, stream in ipairs(streams) do + local app = assert(engine.app_table[stream.repeater_id]) + app:set_rate(bit_rate) + end end function tester.generate_load(bitrate, duration) @@ -173,38 +206,48 @@ function run(args) and_then(promise.Wait, 0.002): and_then(tester.measure, bitrate, duration) end - + function tester.measure(bitrate, duration) local gbps_bitrate = bitrate/1e9 local start_counters = read_counters() local function compute_stats() local end_counters = read_counters() - local s = {} - for k,v in pairs(start_counters) do - s[k] = tonumber(end_counters[k] - start_counters[k]) + local stats = {} + for _, stream in ipairs(streams) do + local s = {} + for k,_ in pairs(start_counters[stream.nic_tx_id]) do + local end_value = end_counters[stream.nic_tx_id][k] + local start_value = start_counters[stream.nic_tx_id][k] + s[k] = tonumber(end_value - start_value) + end + s.applied_gbps = gbps_bitrate + s.tx_mpps = s.txpackets / duration / 1e6 + s.tx_gbps = compute_bitrate(s.txpackets, s.txbytes, duration) / 1e9 + s.rx_mpps = s.rxpackets / duration / 1e6 + s.rx_gbps = compute_bitrate(s.rxpackets, s.rxbytes, duration) / 1e9 + s.lost_packets = s.txpackets - s.rxpackets - s.rxdrop + s.lost_percent = s.lost_packets / s.txpackets * 100 + print(string.format(' %s:', stream.tx_name)) + print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', + s.txpackets, s.tx_mpps, s.txbytes, s.tx_gbps)) + print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', + s.rxpackets, s.rx_mpps, s.rxbytes, s.rx_gbps)) + print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', + s.rxdrop, s.lost_packets, s.lost_percent)) + + stats[stream.nic_tx_id] = s end - s.applied_gbps = gbps_bitrate - s.tx_mpps = s.txpackets / duration / 1e6 - s.tx_gbps = compute_bitrate(s.txpackets, s.txbytes, duration) / 1e9 - s.rx_mpps = s.rxpackets / duration / 1e6 - s.rx_gbps = compute_bitrate(s.rxpackets, s.rxbytes, duration) / 1e9 - s.lost_packets = s.txpackets - s.rxpackets - s.rxdrop - s.lost_percent = s.lost_packets / s.txpackets * 100 - print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', - s.txpackets, s.tx_mpps, s.txbytes, s.tx_gbps)) - print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', - s.rxpackets, s.rx_mpps, s.rxbytes, s.rx_gbps)) - print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', - s.rxdrop, s.lost_packets, s.lost_percent)) - return s + return stats end - local function verify_load(s) - if s.tx_gbps < 0.5 * s.applied_gbps then - print("Invalid result.") - return tester.start_load(bitrate, duration) - else - return check_results(s) - end + local function verify_load(stats) + for _, stream in ipairs(streams) do + local s = stats[stream.nic_tx_id] + if s.tx_gbps < 0.5 * s.applied_gbps then + print("Invalid result.") + return tester.start_load(bitrate, duration) + end + end + return check_results(stats) end print(string.format('Applying %f Gbps of load.', gbps_bitrate)) return tester.generate_load(bitrate, duration): From aa435ba1029e34c0514a7dad7c30d7aac8380805 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Apr 2018 16:33:43 +0200 Subject: [PATCH 079/100] Support the old argument format and fix nits --- src/program/loadtest/find-limit/README | 7 ++++++- src/program/loadtest/find-limit/find-limit.lua | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/program/loadtest/find-limit/README b/src/program/loadtest/find-limit/README index 282dbef994..1fefd2f8a0 100644 --- a/src/program/loadtest/find-limit/README +++ b/src/program/loadtest/find-limit/README @@ -1,4 +1,5 @@ Usage: find-limit [OPTIONS] [ ]... + find-limit [OPTIONS] -b BITRATE, --bitrate BITRATE Test bitrates up to BITRATE bits/second. @@ -27,6 +28,10 @@ adaptors. It will attempt to determin the highest bitrate at which a test passes If no script is supplied then find-limit will attempt to see if packets are being dropped by checking the traffic sent vs traffic recieved. +If you specify only a PCI address and PCAP file it will configure itself to work +the same as specify the a single device as both the recieve and transmit interface. + Examples: - find-limit cap1.pcap tx tx cap1.pcap + find-limit 01:00.0 cap1.pcap + find-limit cap1.pcap tx tx 01:00.0 find-limit cap1.pcap "NIC 0" "NIC 1" 01:00.0 cap2.pcap "NIC 1" "NIC 0" 01:00.0 diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua index 3ad38e0185..4456754f48 100644 --- a/src/program/loadtest/find-limit/find-limit.lua +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -49,7 +49,6 @@ local function find_limit(tester, max_bitrate, precision, duration, retry_count) print(round(lo) * 1e-9) return lo end - -- We need to return tester.start_load(cur, duration): and_then(continue, cur) end @@ -89,6 +88,14 @@ local function parse_args(args) ["retry-count"]="r", help="h", cpu=1, exec="e"}) + if #args == 2 then + -- Assume legacy mode + args = { + args[2], -- PCAP file + 'NIC 0', 'NIC 0', + args[1] -- PCI device + } + end if #args == 0 or #args % 4 ~= 0 then show_usage(1) end local streams, streams_by_tx_id, pci_devices = {}, {}, {} for i=1,#args,4 do From d10a84010611e235cd66ab36f3e4719dc49bec33 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Apr 2018 16:34:23 +0200 Subject: [PATCH 080/100] Remove unused fatal function --- src/program/loadtest/find-limit/find-limit.lua | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua index 4456754f48..8614feefc6 100644 --- a/src/program/loadtest/find-limit/find-limit.lua +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -15,11 +15,6 @@ local promise = require("program.loadtest.promise") local WARM_UP_BIT_RATE = 1e9 local WARM_UP_TIME = 5 -local function fatal (msg) - print(msg) - main.exit(1) -end - local function show_usage(code) print(require("program.loadtest.find_limit.README_inc")) main.exit(code) From bcda559b7ad3ed7ee997e7535eda4e4de66d4d7c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Apr 2018 16:39:35 +0200 Subject: [PATCH 081/100] Fix a few more minor nits :) --- src/program/loadtest/find-limit/README | 4 ++-- src/program/loadtest/find-limit/find-limit.lua | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/program/loadtest/find-limit/README b/src/program/loadtest/find-limit/README index 1fefd2f8a0..517ec78763 100644 --- a/src/program/loadtest/find-limit/README +++ b/src/program/loadtest/find-limit/README @@ -1,5 +1,5 @@ Usage: find-limit [OPTIONS] [ ]... - find-limit [OPTIONS] + find-limit [OPTIONS] -b BITRATE, --bitrate BITRATE Test bitrates up to BITRATE bits/second. @@ -29,7 +29,7 @@ If no script is supplied then find-limit will attempt to see if packets are bein dropped by checking the traffic sent vs traffic recieved. If you specify only a PCI address and PCAP file it will configure itself to work -the same as specify the a single device as both the recieve and transmit interface. +the same as specifying a single device as both the recieve and transmit interface. Examples: find-limit 01:00.0 cap1.pcap diff --git a/src/program/loadtest/find-limit/find-limit.lua b/src/program/loadtest/find-limit/find-limit.lua index 8614feefc6..192839387b 100644 --- a/src/program/loadtest/find-limit/find-limit.lua +++ b/src/program/loadtest/find-limit/find-limit.lua @@ -84,11 +84,10 @@ local function parse_args(args) exec="e"}) if #args == 2 then - -- Assume legacy mode args = { - args[2], -- PCAP file - 'NIC 0', 'NIC 0', - args[1] -- PCI device + args[1], + 'NIC', 'NIC', + args[2] } end if #args == 0 or #args % 4 ~= 0 then show_usage(1) end From 55bf2e38a104a6d8fe01b3750a5840061bd66407 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 18 Apr 2018 12:15:00 +0200 Subject: [PATCH 082/100] Promptly close util.ls() dir fd; fix bug with deleted entries This commit fixes two bugs: * It was impossible to distinguish between a nonempty batch of dirents that contains all deleted entries (d.ino == 0) and a terminal batch of dirents (size == 0). To fix this, we modify the getdents / getdirentries interfaces to return nil, nil in the terminal case. This is an incompatible change, but most users are probably using the util.ls or util.dirtable methods. * The util.ls method left the directory FD open, to be later closed by GC. This can lead to running out of FDs if GC doesn't happen soon enough. The fix is to close the directory fd after the iterator terminates normally, in addition to closing the fd in error cases. --- lib/ljsyscall/syscall/bsd/syscalls.lua | 1 + lib/ljsyscall/syscall/syscalls.lua | 1 + lib/ljsyscall/syscall/types.lua | 9 ++------- lib/ljsyscall/syscall/util.lua | 25 +++++++++++-------------- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/lib/ljsyscall/syscall/bsd/syscalls.lua b/lib/ljsyscall/syscall/bsd/syscalls.lua index eb98ed8065..f4fad72288 100644 --- a/lib/ljsyscall/syscall/bsd/syscalls.lua +++ b/lib/ljsyscall/syscall/bsd/syscalls.lua @@ -31,6 +31,7 @@ if C.getdirentries then basep = basep or t.long1() local ret, err = C.getdirentries(getfd(fd), buf, size, basep) if ret == -1 then return nil, t.error(err or errno()) end + if ret == 0 then return nil, nil end return t.dirents(buf, ret) end end diff --git a/lib/ljsyscall/syscall/syscalls.lua b/lib/ljsyscall/syscall/syscalls.lua index e76fe725d1..33051be18a 100644 --- a/lib/ljsyscall/syscall/syscalls.lua +++ b/lib/ljsyscall/syscall/syscalls.lua @@ -740,6 +740,7 @@ if C.getdents then buf = buf or t.buffer(size) local ret, err = C.getdents(getfd(fd), buf, size) if ret == -1 then return nil, t.error(err or errno()) end + if ret == 0 then return nil, nil end return t.dirents(buf, ret) end end diff --git a/lib/ljsyscall/syscall/types.lua b/lib/ljsyscall/syscall/types.lua index 17c74b1fec..539ab3c715 100644 --- a/lib/ljsyscall/syscall/types.lua +++ b/lib/ljsyscall/syscall/types.lua @@ -601,15 +601,10 @@ if bsdtypes then types = bsdtypes.init(c, types) end -- define dents type if dirent is defined if t.dirent then t.dirents = function(buf, size) -- buf should be char* - local d, i = nil, 0 + local i = 0 return function() -- TODO work out if possible to make stateless - if size > 0 and not d then - d = pt.dirent(buf) - i = i + d.d_reclen - return d - end while i < size do - d = pt.dirent(pt.char(d) + d.d_reclen) + local d = pt.dirent(buf + i) i = i + d.d_reclen if d.ino ~= 0 then return d end -- some systems use ino = 0 for deleted files before removed eg OSX; it is never valid end diff --git a/lib/ljsyscall/syscall/util.lua b/lib/ljsyscall/syscall/util.lua index 8fb543edf4..3f3515853f 100644 --- a/lib/ljsyscall/syscall/util.lua +++ b/lib/ljsyscall/syscall/util.lua @@ -56,22 +56,19 @@ function util.ls(name, buf, size) if err then return nil, err end local di return function() - local d, first - repeat + while true do + if di then + local d = di() + if d then return d.name, d end + end + -- Fetch more entries. + local err + di, err = fd:getdents(buf, size) if not di then - local err - di, err = fd:getdents(buf, size) - if not di then - fd:close() - error(err) - end - first = true + fd:close() + if err then error(err) else return nil end end - d = di() - if not d then di = nil end - if not d and first then return nil end - until d - return d.name, d + end end end From 828641fe21372603636050aee6f9d22869c2cff6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 18 Apr 2018 12:54:17 +0200 Subject: [PATCH 083/100] Remove lib.files_in_directory There's no need to use this when ljsyscall's "ls" and "dirtable" exist. --- src/README.md | 4 ---- src/core/lib.lua | 8 -------- src/lib/hardware/pci.lua | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/README.md b/src/README.md index a6295e76e5..02876789c5 100644 --- a/src/README.md +++ b/src/README.md @@ -725,10 +725,6 @@ Returns the filename of the first file in *directory*. Returns the first line of file at *filename* as a string. -— Function **lib.files_in_directory** *directory* - -Returns an array of filenames in *directory*. - — Function **lib.load_string** *string* Evaluates and returns the value of the Lua expression in *string*. diff --git a/src/core/lib.lua b/src/core/lib.lua index 4bface458e..c3b8f4c701 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -107,14 +107,6 @@ end function firstline (filename) return readfile(filename, "*l") end -function files_in_directory (dir) - local files = {} - for line in assert(io.popen('ls -1 "'..dir..'" 2>/dev/null')):lines() do - table.insert(files, line) - end - return files -end - -- Load Lua value from string. function load_string (string) return loadstring("return "..string)() diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index 7d9b6c5209..29d147b240 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -27,7 +27,7 @@ devices = {} --- Initialize (or re-initialize) the `devices` table. function scan_devices () - for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do + for _,device in ipairs(S.util.dirtable("/sys/bus/pci/devices")) do local info = device_info(device) if info.driver then table.insert(devices, info) end end From f133f7f90bde3c201bd4b34790f0e2ac89e3b3f5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 18 Apr 2018 12:55:49 +0200 Subject: [PATCH 084/100] Promptly close fd in random_bytes_from_dev_urandom --- src/core/lib.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib.lua b/src/core/lib.lua index c3b8f4c701..f6018194ae 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -718,6 +718,7 @@ function random_bytes_from_dev_urandom (count) while written < count do written = written + assert(f:read(bytes, count-written)) end + f:close() return bytes end From 1e6f85237e54ac9bbeeb7aa7c90f6b52cdd21169 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 18 Apr 2018 13:06:51 +0200 Subject: [PATCH 085/100] Fix modification to lib.hardware.pci.scan_devices --- src/lib/hardware/pci.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index 29d147b240..b6a25a5e02 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -27,9 +27,11 @@ devices = {} --- Initialize (or re-initialize) the `devices` table. function scan_devices () - for _,device in ipairs(S.util.dirtable("/sys/bus/pci/devices")) do - local info = device_info(device) - if info.driver then table.insert(devices, info) end + for device in assert(S.util.ls("/sys/bus/pci/devices")) do + if device ~= '.' and device ~= '..' then + local info = device_info(device) + if info.driver then table.insert(devices, info) end + end end end From 65cbc01a31360c9ef0612b2b5d4f8fa85855cbe3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 18 Apr 2018 14:48:37 +0200 Subject: [PATCH 086/100] Set SNABB_RANDOM_SEED for intel_mp tests This should fix some flakiness in these tests, as they use a random number as the RSS key. --- src/apps/intel_mp/selftest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/intel_mp/selftest.sh b/src/apps/intel_mp/selftest.sh index dc54fe2c33..efa8732ec1 100755 --- a/src/apps/intel_mp/selftest.sh +++ b/src/apps/intel_mp/selftest.sh @@ -14,6 +14,7 @@ TESTS=$(echo "$TESTS1G" "$TESTS10G" | grep -e "$FILTER" | sort) ESTATUS=0 export SNABB_RECV_DEBUG=true export SNABB_RECV_MASTER_STATS=true +export SNABB_RANDOM_SEED=0xacabba9e for i in $TESTS; do pkill -P $$ -f snabb sleep 1 From 5cd8b1b2b8ab09ffdff98ab35d937e81bbb8bbe9 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Apr 2018 09:43:55 +0000 Subject: [PATCH 087/100] Update README --- src/program/alarms/set_operator_state/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/alarms/set_operator_state/README b/src/program/alarms/set_operator_state/README index 8d2b674c36..f88e83eb13 100644 --- a/src/program/alarms/set_operator_state/README +++ b/src/program/alarms/set_operator_state/README @@ -21,7 +21,7 @@ An OPERATOR-STATE can take the following values: 'none', 'ack', 'closed', Typical usage: -$ snabb alarms set-operator-state lwaftr nic-v4 arp-resolution ack +$ snabb alarms set-operator-state --schema snabb-softwire-v2 lwaftr resource arp-resolution ack See https://github.com/Igalia/snabb/blob/lwaftr/src/program/alarms/README.md for full documentation. From aa31d24e97265a95d3b48cfaca99cf73b665a0b7 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Apr 2018 09:48:52 +0000 Subject: [PATCH 088/100] Implement table_is_empty --- src/lib/yang/alarms.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/alarms.lua b/src/lib/yang/alarms.lua index a94a98eccc..0c084a1a86 100644 --- a/src/lib/yang/alarms.lua +++ b/src/lib/yang/alarms.lua @@ -34,6 +34,11 @@ local function table_size (t) return size end +local function table_is_empty(t) + for k,v in pairs(t) do return false end + return true +end + function get_state () -- status-change is stored as an array while according to ietf-alarms schema -- it should be a hashmap indexed by time. @@ -117,8 +122,9 @@ function build_summary (alarms) end ret[severity] = entry end - local shelved_alarms = table_size(state.shelved_alarms.shelved_alarms) - if shelved_alarms > 0 then ret['shelved_alarms'] = shelved_alarms end + if not table_is_empty(state.shelved_alarms.shelved_alarms) then + ret['shelved_alarms'] = table_size(state.shelved_alarms.shelved_alarms) + end return ret end From d35cdd5e55cc9eec2bcf1287994b732cf927c2c4 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Apr 2018 10:18:10 +0000 Subject: [PATCH 089/100] Rename 'failsafe' to 'allow_extra_args' --- src/program/alarms/set_operator_state/set_operator_state.lua | 2 +- src/program/config/common.lua | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/alarms/set_operator_state/set_operator_state.lua b/src/program/alarms/set_operator_state/set_operator_state.lua index cb7567b777..6fbc1d00e0 100644 --- a/src/program/alarms/set_operator_state/set_operator_state.lua +++ b/src/program/alarms/set_operator_state/set_operator_state.lua @@ -33,7 +33,7 @@ end function run(args) local opts = { command='set-alarm-operator-state', with_path=false, is_config=false, - usage=show_usage, failsafe=true } + usage=show_usage, allow_extra_args=true } local args, cdr = common.parse_command_line(args, opts) local l_args = parse_args(cdr) local response = common.call_leader( diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 0bf01f1ad7..b11f2d2e0f 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -24,7 +24,7 @@ local parse_command_line_opts = { require_schema = { default=false }, is_config = { default=true }, usage = { default=show_usage }, - failsafe = { default=false }, + allow_extra_args = { default=false }, } local function path_grammar(schema_name, path, is_config) @@ -121,7 +121,7 @@ function parse_command_line(args, opts) end ret.value = parser(ret.value_str) end - if not opts.failsafe and #args ~= 0 then err("too many arguments") end + if not opts.allow_extra_args and #args ~= 0 then err("too many arguments") end return ret, args end From 2e8a5f9030d88e7154422277a30813c8d1a318b4 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Apr 2018 16:46:55 +0000 Subject: [PATCH 090/100] Fix undefined variable --- src/program/lwaftr/run/run.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 740e9e87f2..cce78df254 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -196,7 +196,7 @@ function run(args) local ipv4_rx = opts.hydra and 'ipv4tx' or 'IPv4 TX' local ipv6_tx = opts.hydra and 'ipv6rx' or 'IPv6 RX' local ipv6_rx = opts.hydra and 'ipv6tx' or 'IPv6 TX' - if use_splitter then + if requires_splitter(opts, conf) then csv:add_app('v4v6', { 'v4', 'v4' }, { tx=ipv4_tx, rx=ipv4_rx }) csv:add_app('v4v6', { 'v6', 'v6' }, { tx=ipv6_tx, rx=ipv6_rx }) else From a5bed82201e5dfd2d847a5c809dda1895b3e95ff Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 23 Apr 2018 15:50:09 +0200 Subject: [PATCH 091/100] Claim name in central lwaftr ptree manager function Otherwise, we would never claim the name when starting with an empty config. --- src/program/lwaftr/setup.lua | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 23e7c196bb..b89c5405db 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -62,24 +62,6 @@ function lwaftr_app(c, conf) local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end - -- Claim the name if one is defined. - local function switch_names(config) - local currentname = engine.program_name - local name = config.softwire_config.name - -- Don't do anything if the name isn't set. - if name == nil then - return - end - - local success, err = pcall(engine.claim_name, name) - if success == false then - -- Restore the previous name. - config.softwire_config.name = currentname - assert(success, err) - end - end - switch_names(conf) - local device, id, queue = lwutil.parse_instance(conf) -- Global interfaces @@ -587,7 +569,24 @@ local function compute_worker_configs(conf) end function ptree_manager(f, conf, manager_opts) + -- Claim the name if one is defined. + local function switch_names(config) + local currentname = engine.program_name + local name = config.softwire_config.name + -- Don't do anything if the name isn't set. + if name == nil then + return + end + local success, err = pcall(engine.claim_name, name) + if success == false then + -- Restore the previous name. + config.softwire_config.name = currentname + assert(success, err) + end + end + local function setup_fn(conf) + switch_names(conf) local worker_app_graphs = {} for worker_id, worker_config in pairs(compute_worker_configs(conf)) do local app_graph = config.new() From 703f1a957ce0d5269ebb0b8c6f85e26bb48fda83 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 23 Apr 2018 15:50:50 +0200 Subject: [PATCH 092/100] "snabb config" tests wait until ptree manager socket is available This replaces "sleep" calls with deterministic waits. --- .../lwaftr/tests/subcommands/config_test.py | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 30ba43e8a1..23d6ba9fc1 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -24,7 +24,20 @@ str(BENCHDATA_DIR / 'ipv4-0550.pcap'), str(BENCHDATA_DIR / 'ipv6-0550.pcap'), ] -SOCKET_PATH = '/tmp/snabb-lwaftr-listen-sock-%s' % DAEMON_PROC_NAME +LISTEN_SOCKET_PATH = '/tmp/snabb-lwaftr-listen-sock-%s' % DAEMON_PROC_NAME +MANAGER_SOCKET_PATH = '/var/run/snabb/by-name/%s/config-leader-socket' % DAEMON_PROC_NAME + +def wait_for_socket(socket_path, timeout=5, step=0.1): + for i in range(0, int(timeout/step)): + if os.access(socket_path, os.F_OK): + return True + time.sleep(step) + return False + +def wait_for_listen_socket(**kwargs): + return wait_for_socket(LISTEN_SOCKET_PATH, **kwargs) +def wait_for_manager_socket(**kwargs): + return wait_for_socket(MANAGER_SOCKET_PATH, **kwargs) class TestConfigGet(BaseTestCase): """ @@ -35,6 +48,13 @@ class TestConfigGet(BaseTestCase): daemon_args = DAEMON_ARGS config_args = (str(SNABB_CMD), 'config', 'get', '--schema=snabb-softwire-v2', DAEMON_PROC_NAME) + @classmethod + def setUpClass(cls): + super().setUpClass() + if not wait_for_manager_socket(): + cls.daemon.terminate() + cls.reportAndFail('Config manager socket not present', None) + def test_get_internal_iface(self): cmd_args = list(self.config_args) cmd_args.append('/softwire-config/instance[device=test]/queue[id=0]' @@ -110,15 +130,13 @@ def start_daemon(self, config, additional=None): # Start the daemon itself self.daemon = Popen(daemon_args, stdout=PIPE, stderr=PIPE) - time.sleep(DAEMON_STARTUP_WAIT) - return_code = self.daemon.poll() - if return_code is not None: + if not wait_for_manager_socket(): + self.daemon.terminate() stdout = self.daemon.stdout.read().decode(ENC) stderr = self.daemon.stderr.read().decode(ENC) self.fail("\n".join(( "Failed starting daemon", "Command:", " ".join(daemon_args), - "Exit code: {0}".format(return_code), "STDOUT", stdout, "STDOUT", stderr, ))) @@ -290,20 +308,36 @@ class TestConfigListen(BaseTestCase): daemon_args = DAEMON_ARGS listen_args = (str(SNABB_CMD), 'config', 'listen', - '--socket', SOCKET_PATH, DAEMON_PROC_NAME) + '--socket', LISTEN_SOCKET_PATH, DAEMON_PROC_NAME) + + @classmethod + def setUpClass(cls): + super().setUpClass() + if not wait_for_manager_socket(): + cls.daemon.terminate() + cls.reportAndFail('Config manager socket not present', None) def test_listen(self): # Start the listen command with a socket. listen_daemon = Popen(self.listen_args, stdout=PIPE, stderr=PIPE) - # Wait a short while for the socket to be created. - time.sleep(1) + if not wait_for_listen_socket(): + listen_daemon.terminate() + listen_daemon.wait() + stdout = listen_daemon.stdout.read().decode(ENC) + stderr = listen_daemon.stderr.read().decode(ENC) + self.fail("\n".join(( + "Failed to run 'snabb listen'", + "Command:", " ".join(daemon_args), + "STDOUT", stdout, + "STDOUT", stderr, + ))) # Send command to and receive response from the listen command. # (Implicit string concatenation, no summing needed.) get_cmd = (b'{ "id": "0", "verb": "get",' b' "path": "/routes/route[addr=1.2.3.4]/port" }\n') sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: - sock.connect(SOCKET_PATH) + sock.connect(LISTEN_SOCKET_PATH) sock.sendall(get_cmd) resp = str(sock.recv(200), ENC) finally: @@ -320,13 +354,20 @@ def test_listen(self): print('STDERR\n', str(listen_daemon.stderr.read(), ENC)) listen_daemon.stdout.close() listen_daemon.stderr.close() - os.unlink(SOCKET_PATH) + os.unlink(LISTEN_SOCKET_PATH) class TestConfigMisc(BaseTestCase): daemon_args = DAEMON_ARGS + @classmethod + def setUpClass(cls): + super().setUpClass() + if not wait_for_manager_socket(): + cls.daemon.terminate() + cls.reportAndFail('Config manager socket not present', None) + def get_cmd_args(self, action): cmd_args = list((str(SNABB_CMD), 'config', 'XXX', '--schema=snabb-softwire-v2', DAEMON_PROC_NAME)) cmd_args[2] = action From 69f545b5715ddf09d0e6da30a97ead1838b9e9e8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 09:46:03 +0200 Subject: [PATCH 093/100] Revert lwAFTR-specific version change for merge to upstream --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 1a0764e0c1..7b63fdf2ae 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2018.04.01 +2018.04 From a1efb46e8650c6fa4ee3954c003d47fbbddfcbb3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 10:16:04 +0200 Subject: [PATCH 094/100] Harmonize style in changelog --- src/program/lwaftr/doc/CHANGELOG.md | 38 +++++++++++++++++------------ 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index a079dc0b44..9fd4478f61 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -2,29 +2,35 @@ ## [2018.01.2.1] -* Features: +### Features - - Added limit-finding loadtester. See: +* Added limit-finding loadtester. See: - https://github.com/Igalia/snabb/blob/lwaftr/src/program/loadtest/find-limit/README.inc + https://github.com/Igalia/snabb/blob/lwaftr/src/program/loadtest/find-limit/README.inc - - Move "loadtest" command out of lwaftr. Now the "loadtest" command consists of - two subcommands: "transient" and "find-limit". Example: +* Move "loadtest" command out of lwaftr. Now the "loadtest" command consists of + two subcommands: "transient" and "find-limit". Example: - $ sudo ./snabb loadtest transient -D 1 -b 5e9 -s 0.2e9 \ - cap1.pcap "NIC 0" "NIC 1" 01:00.0 \ - cap2.pcap "NIC 1" "NIC 0" 01:00.1 + $ sudo ./snabb loadtest transient -D 1 -b 5e9 -s 0.2e9 \ + cap1.pcap "NIC 0" "NIC 1" 01:00.0 \ + cap2.pcap "NIC 1" "NIC 0" 01:00.1 - $ sudo ./snabb loadtest find-limit 01:00.0 cap1.pcap + $ sudo ./snabb loadtest find-limit 01:00.0 cap1.pcap -* Bug fixes: +### Bug fixes - - Fix next-hop discovery with multiple devices (Issue: https://github.com/Igalia/snabb/issues/1014). - - Improve effectiveness of property-based tests. - - Process tree runs data-plane processes with busywait=true by default - - Remove early lwAFTR NUMA affinity check. The check was unnecessary since - now ptree manager handles NUMA affinity and appropriate CPU selection. - - Sizes for "packetblaster lwaftr" are frame sizes. +* Fix next-hop discovery with multiple devices. See: + + https://github.com/Igalia/snabb/issues/1014 + +* Improve effectiveness of property-based tests. + +* Process tree runs data-plane processes with busywait=true by default + +* Remove early lwAFTR NUMA affinity check. The check was unnecessary since + now ptree manager handles NUMA affinity and appropriate CPU selection. + +* Sizes for "packetblaster lwaftr" are frame sizes. ## [2017.11.01] From d51d0d53abd63eb6cc0a7f910052292b31c0420a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 10:55:43 +0200 Subject: [PATCH 095/100] Changelog for lwAFTR v2018.04.01 --- src/program/lwaftr/doc/CHANGELOG.md | 68 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 9fd4478f61..a378385bac 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,6 +1,72 @@ # Change Log -## [2018.01.2.1] +## [2018.04.01] (unreleased) + +### Features + +* Implement alarm shelving. For documentation, see: + + https://github.com/Igalia/snabb/blob/lwaftr/src/program/alarms/README.md + +* Extend `snabb loadtest find-limit` to handle multiple NICs, as in a + lwAFTR bump-in-the-wire configuration. For documentation, see: + + https://github.com/Igalia/snabb/blob/lwaftr/src/program/loadtest/find-limit/README + +### Bug fixes + +* Fix the `--format xpath` output for `snabb config get`; broken in + the switch to the `snabb-softwire-v2` model, which exercises + different kinds of paths. + +* Fix alarm notification (accidentally disabled during refactor). + Update documentation for `snabb alarms set-operator-state. + +* Fix lwAFTR to claim name when run with empty configuration (no + workers). + +* Fix `--verbose` lwAFTR mode when run on-a-stick. + +* Improve reliability of unit and integration tests. + +* Fix bug that can cause the lwAFTR to run out of file descriptors in + some circumstances (https://github.com/snabbco/snabb/pull/1325). + +* Fix documentation for ARP snabb component. + +* Improve self-tests for unified Intel NIC driver. + +* Improve reliability when piping `snabb config` output to files in the + shell. (https://github.com/snabbco/snabb/pull/1300). + +* Make necessary modifications to support 64-bit Lua allocations, as + will be the case with RaptorJIT. + +### Other enhancements from upstream + +From the +[2018.04](https://github.com/snabbco/snabb/releases/tag/v2018.04) +release: + +* Add `snabb dnnsd` tool for browsing local DNS-SD records. See: + + https://github.com/snabbco/snabb/blob/next/src/program/dnssd/README.md + +* Add `snabb unhexdump` tool for converting packet dumps to pcap files. + See: + + https://github.com/snabbco/snabb/blob/next/src/program/unhexdump/README + +* Improve performance when using non-busy-wait mode. + +* Add `Makefile` target to build Docker image. See: + + https://github.com/snabbco/snabb/blob/next/README.md#snabb-container + +* Improve output from `snabb top`. + +Thanks to Marcel Wiget, Alexander Gall, Max Rottenkolber, and Luke +Gorrie for upstream work in this period. ### Features From 99ffe315879a017a1ea6846a36492a18766e2a5d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 10:58:23 +0200 Subject: [PATCH 096/100] Fix removal of v2018.01.2.1 header --- src/program/lwaftr/doc/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index a378385bac..70703aa611 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -68,6 +68,8 @@ release: Thanks to Marcel Wiget, Alexander Gall, Max Rottenkolber, and Luke Gorrie for upstream work in this period. +## [2018.01.2.1] + ### Features * Added limit-finding loadtester. See: From bb63b39a44ff72674edd7b2aa1cfdd6ddfa6e717 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 11:10:44 +0200 Subject: [PATCH 097/100] Fix missing closing backtick --- src/program/lwaftr/doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 70703aa611..9da15f0ca1 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -20,7 +20,7 @@ different kinds of paths. * Fix alarm notification (accidentally disabled during refactor). - Update documentation for `snabb alarms set-operator-state. + Update documentation for `snabb alarms set-operator-state`. * Fix lwAFTR to claim name when run with empty configuration (no workers). From 54a5402425edb522b0f646fa83665671ba54523a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 12:02:25 +0200 Subject: [PATCH 098/100] Flip sense of conditional-skip of IPsec end-to-end test --- src/apps/ipsec/selftest.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/apps/ipsec/selftest.sh b/src/apps/ipsec/selftest.sh index fc4b8f549e..9ac165bd2d 100755 --- a/src/apps/ipsec/selftest.sh +++ b/src/apps/ipsec/selftest.sh @@ -3,10 +3,7 @@ SKIPPED_CODE=43 -# Temporary disabled test. -exit $SKIPPED_CODE - -if [ "$SNABB_IPSEC_SKIP_E2E_TEST" = yes ]; then +if [ "$SNABB_IPSEC_ENABLE_E2E_TEST" != yes ]; then exit $SKIPPED_CODE fi From f03ba1b9e016f7c147a1102d015d199040cc5e2f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 12:08:41 +0200 Subject: [PATCH 099/100] Test on presence of SNABB_IPSEC_ENABLE_E2E_TEST, not value --- src/apps/ipsec/selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/ipsec/selftest.sh b/src/apps/ipsec/selftest.sh index 9ac165bd2d..5e6e7228fa 100755 --- a/src/apps/ipsec/selftest.sh +++ b/src/apps/ipsec/selftest.sh @@ -3,7 +3,7 @@ SKIPPED_CODE=43 -if [ "$SNABB_IPSEC_ENABLE_E2E_TEST" != yes ]; then +if [ -n "$SNABB_IPSEC_ENABLE_E2E_TEST" ]; then exit $SKIPPED_CODE fi From 3f94ce5bdba1ba8cee7dcca8153165530ffb53c6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 24 Apr 2018 13:45:11 +0200 Subject: [PATCH 100/100] Bash programming is known to the State of California to cause confusion --- src/apps/ipsec/selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/ipsec/selftest.sh b/src/apps/ipsec/selftest.sh index 5e6e7228fa..c4bf3e70ff 100755 --- a/src/apps/ipsec/selftest.sh +++ b/src/apps/ipsec/selftest.sh @@ -3,7 +3,7 @@ SKIPPED_CODE=43 -if [ -n "$SNABB_IPSEC_ENABLE_E2E_TEST" ]; then +if [ -z "$SNABB_IPSEC_ENABLE_E2E_TEST" ]; then exit $SKIPPED_CODE fi