From 485614dd94b05559cd2e88e8e218a413ac368cca Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Tue, 22 Sep 2020 17:38:00 +0300 Subject: [PATCH 1/6] chore(deps) bump resty.healthcheck from 1.3.0 to 2.0.0 ### Summary * BREAKING: fallback for deprecated top-level field `type` is now removed (deprecated since `0.5.0`) [#56](https://github.com/Kong/lua-resty-healthcheck/pull/56) * BREAKING: Bump `lua-resty-worker-events` dependency to `2.0.0`. This makes a lot of the APIs in this library asynchronous as the worker events `post` and `post_local` won't anymore call `poll` on a running worker automatically, for more information, see: https://github.com/Kong/lua-resty-worker-events#200-16-september-2020 * BREAKING: tcp_failures can no longer be 0 on http(s) checks (unless http(s)_failures are also set to 0) [#55](https://github.com/Kong/lua-resty-healthcheck/pull/55) * feature: Added support for https_sni [#49](https://github.com/Kong/lua-resty-healthcheck/pull/49) * fix: properly log line numbers by using tail calls [#29](https://github.com/Kong/lua-resty-healthcheck/pull/29) * fix: when not providing a hostname, use IP [#48](https://github.com/Kong/lua-resty-healthcheck/pull/48) * fix: makefile; make install * feature: added a status version field [#54](https://github.com/Kong/lua-resty-healthcheck/pull/54) * feature: add headers for probe request [#54](https://github.com/Kong/lua-resty-healthcheck/pull/54) * fix: exit early when reloading during a probe [#47](https://github.com/Kong/lua-resty-healthcheck/pull/47) * fix: prevent target-list from being nil, due to async behaviour [#44](https://github.com/Kong/lua-resty-healthcheck/pull/44) * fix: replace timer and node-wide locks with resty-timer, to prevent interval skips [#59](https://github.com/Kong/lua-resty-healthcheck/pull/59) * change: added additional logging on posting events [#25](https://github.com/Kong/lua-resty-healthcheck/issues/25) * fix: do not run out of timers during init/init_worker when adding a vast amount of targets [#57](https://github.com/Kong/lua-resty-healthcheck/pull/57) * fix: do not call on the module table, but use a method for locks. Also in [#57](https://github.com/Kong/lua-resty-healthcheck/pull/57) --- kong-2.4.1-0.rockspec | 2 +- spec/01-unit/09-balancer_spec.lua | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/kong-2.4.1-0.rockspec b/kong-2.4.1-0.rockspec index 5e1379934df..7570dd5784f 100644 --- a/kong-2.4.1-0.rockspec +++ b/kong-2.4.1-0.rockspec @@ -32,7 +32,7 @@ dependencies = { "lua-resty-dns-client == 6.0.1", "lua-protobuf == 0.3.2", "lua-resty-worker-events == 1.0.0", - "lua-resty-healthcheck == 1.4.1", + "lua-resty-healthcheck == 2.0.0", "lua-resty-cookie == 0.1.0", "lua-resty-mlcache == 2.5.0", "lua-messagepack == 0.5.2", diff --git a/spec/01-unit/09-balancer_spec.lua b/spec/01-unit/09-balancer_spec.lua index 26f0690e934..9032ed8a82d 100644 --- a/spec/01-unit/09-balancer_spec.lua +++ b/spec/01-unit/09-balancer_spec.lua @@ -112,6 +112,7 @@ for _, consistency in ipairs({"strict", "eventual"}) do local passive_hc = utils.deep_copy(hc_defaults) passive_hc.passive.healthy.successes = 1 + passive_hc.passive.unhealthy.tcp_failures = 1 -- 1 = required because http failures is 1 as well passive_hc.passive.unhealthy.http_failures = 1 UPSTREAMS_FIXTURES = { From c8646c343df0331066e00a7cfc114efc99606d26 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Thu, 17 Sep 2020 10:54:14 +0300 Subject: [PATCH 2/6] chore(deps) bump resty.worker.events from 1.0.0 to 2.0.0 ### Summary #### 2.0.0, 16-September-2020 - BREAKING: the `post` function does not call `poll` anymore, making all events asynchronous. When an immediate treatment to an event is needed an explicit call to `poll` must be done. - BREAKING: the `post_local` function does not immediately execute the event anymore, making all local events asynchronous. When an immediate treatment to an event is needed an explicit call to `poll` must be done. - fix: prevent spinning at 100% CPU when during a reload the event-shm is cleared - fix: improved logging in case of failure to write to shm (add payload size for troubleshooting purposes) - fix: do not log the payload anymore, since it might expose sensitive data through the logs - change: updated `shm_retries` default to 999 - change: changed timer loop to a sleep-loop (performance) - fix: when re-configuring make sure callbacks table is initialized --- kong-2.4.1-0.rockspec | 2 +- kong/db/declarative/init.lua | 2 +- spec/01-unit/09-balancer_spec.lua | 5 +++ .../10-balancer/01-healthchecks_spec.lua | 43 ++++++++++++++----- spec/fixtures/balancer_utils.lua | 6 +-- .../plugins/worker-events-poll/handler.lua | 20 +++++++++ .../plugins/worker-events-poll/schema.lua | 17 ++++++++ 7 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/handler.lua create mode 100644 spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/schema.lua diff --git a/kong-2.4.1-0.rockspec b/kong-2.4.1-0.rockspec index 7570dd5784f..92e23975d66 100644 --- a/kong-2.4.1-0.rockspec +++ b/kong-2.4.1-0.rockspec @@ -31,7 +31,7 @@ dependencies = { "lua_pack == 1.0.5", "lua-resty-dns-client == 6.0.1", "lua-protobuf == 0.3.2", - "lua-resty-worker-events == 1.0.0", + "lua-resty-worker-events == 2.0.1", "lua-resty-healthcheck == 2.0.0", "lua-resty-cookie == 0.1.0", "lua-resty-mlcache == 2.5.0", diff --git a/kong/db/declarative/init.lua b/kong/db/declarative/init.lua index 6afa181cb9b..8b6ae815974 100644 --- a/kong/db/declarative/init.lua +++ b/kong/db/declarative/init.lua @@ -839,7 +839,7 @@ do ok, err, default_ws = declarative.load_into_cache(entities, meta, hash, SHADOW) if ok then ok, err = kong.worker_events.post("declarative", "flip_config", default_ws) - if ok ~= "done" then + if not ok then ngx.shared.kong:delete(DECLARATIVE_LOCK_KEY) return nil, "failed to flip declarative config cache pages: " .. (err or ok) end diff --git a/spec/01-unit/09-balancer_spec.lua b/spec/01-unit/09-balancer_spec.lua index 9032ed8a82d..855ffdbf01d 100644 --- a/spec/01-unit/09-balancer_spec.lua +++ b/spec/01-unit/09-balancer_spec.lua @@ -1,5 +1,7 @@ local utils = require "kong.tools.utils" local mocker = require "spec.fixtures.mocker" +local we = require "resty.worker.events" + local ws_id = utils.uuid() @@ -470,6 +472,7 @@ for _, consistency in ipairs({"strict", "eventual"}) do } for _, t in ipairs(tests) do assert(balancer.post_health(upstream_ph, t.host, nil, t.port, t.health)) + we.poll() local health_info = assert(balancer.get_upstream_health("ph")) local response = t.health and "HEALTHY" or "UNHEALTHY" assert.same(response, @@ -507,12 +510,14 @@ for _, consistency in ipairs({"strict", "eventual"}) do port = 1111, host = {hostname = "localhost"}, }}, 429) + we.poll() my_balancer.report_http_status({ address = { ip = "127.0.0.1", port = 1111, host = {hostname = "localhost"}, }}, 200) + we.poll() balancer.unsubscribe_from_healthcheck_events(cb) my_balancer.report_http_status({ address = { diff --git a/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua b/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua index 799d9a9b3fe..9774bea0307 100644 --- a/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua +++ b/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua @@ -8,7 +8,7 @@ local https_server = helpers.https_server for _, strategy in helpers.each_strategy() do - local bp + local bp, db local DB_UPDATE_PROPAGATION = strategy == "cassandra" and 0.1 or 0 local DB_UPDATE_FREQUENCY = strategy == "cassandra" and 0.1 or 0.1 @@ -23,12 +23,15 @@ for _, strategy in helpers.each_strategy() do describe("Healthcheck #" .. strategy, function() lazy_setup(function() - bp = bu.get_db_utils_for_dc_and_admin_api(strategy, { + bp, db = bu.get_db_utils_for_dc_and_admin_api(strategy, { "routes", "services", "plugins", "upstreams", "targets", + }, { + "worker-events-poll", + "fail-once-auth", }) local fixtures = { @@ -73,6 +76,10 @@ for _, strategy in helpers.each_strategy() do address = "127.0.0.2", } + db.plugins:insert({ + name = "worker-events-poll", + }) + assert(helpers.start_kong({ database = strategy, dns_resolver = "127.0.0.1", @@ -81,6 +88,7 @@ for _, strategy in helpers.each_strategy() do nginx_conf = "spec/fixtures/custom_nginx.template", db_update_frequency = DB_UPDATE_FREQUENCY, db_update_propagation = DB_UPDATE_PROPAGATION, + plugins = "bundled,fail-once-auth,worker-events-poll" }, nil, nil, fixtures)) end) @@ -361,11 +369,14 @@ for _, strategy in helpers.each_strategy() do lazy_setup(function() - bp = bu.get_db_utils_for_dc_and_admin_api(strategy, { + bp, db = bu.get_db_utils_for_dc_and_admin_api(strategy, { "services", "routes", "upstreams", "targets", + }, { + "worker-events-poll", + "fail-once-auth", }) local fixtures = { @@ -377,6 +388,10 @@ for _, strategy in helpers.each_strategy() do address = "127.0.0.1", } + db.plugins:insert({ + name = "worker-events-poll", + }) + assert(helpers.start_kong({ database = strategy, admin_listen = default_admin_listen, @@ -387,7 +402,7 @@ for _, strategy in helpers.each_strategy() do client_ssl_cert_key = "spec/fixtures/kong_spec.key", db_update_frequency = 0.1, stream_listen = "off", - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", }, nil, nil, fixtures)) end) @@ -469,11 +484,18 @@ for _, strategy in helpers.each_strategy() do describe("Ring-balancer #" .. strategy, function() lazy_setup(function() - bp = bu.get_db_utils_for_dc_and_admin_api(strategy, { + bp, db = bu.get_db_utils_for_dc_and_admin_api(strategy, { "services", "routes", "upstreams", "targets", + }, { + "fail-once-auth", + "worker-events-poll", + }) + + db.plugins:insert({ + name = "worker-events-poll", }) assert(helpers.start_kong({ @@ -486,7 +508,7 @@ for _, strategy in helpers.each_strategy() do stream_listen = "off", db_update_frequency = DB_UPDATE_FREQUENCY, db_update_propagation = DB_UPDATE_PROPAGATION, - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", })) end) @@ -512,6 +534,7 @@ for _, strategy in helpers.each_strategy() do log_level = "debug", db_update_frequency = DB_UPDATE_FREQUENCY, db_update_propagation = DB_UPDATE_PROPAGATION, + plugins = "bundled,fail-once-auth,worker-events-poll", }) end) @@ -1053,7 +1076,7 @@ for _, strategy in helpers.each_strategy() do lua_ssl_trusted_certificate = "spec/fixtures/kong_spec.crt", db_update_frequency = 0.1, stream_listen = "off", - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", }, nil, fixtures) bu.end_testcase_setup(strategy, bp) @@ -1481,7 +1504,7 @@ for _, strategy in helpers.each_strategy() do lua_ssl_trusted_certificate = "spec/fixtures/kong_spec.crt", db_update_frequency = 0.1, stream_listen = "off", - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", }, nil, fixtures) bu.end_testcase_setup(strategy, bp) @@ -1590,7 +1613,7 @@ for _, strategy in helpers.each_strategy() do lua_ssl_trusted_certificate = "spec/fixtures/kong_spec.crt", db_update_frequency = 0.1, stream_listen = "off", - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", }, nil, fixtures) bu.end_testcase_setup(strategy, bp) @@ -1813,7 +1836,7 @@ for _, strategy in helpers.each_strategy() do lua_ssl_trusted_certificate = "spec/fixtures/kong_spec.crt", db_update_frequency = 0.1, stream_listen = "off", - plugins = "bundled,fail-once-auth", + plugins = "bundled,fail-once-auth,worker-events-poll", }) bu.end_testcase_setup(strategy, bp) diff --git a/spec/fixtures/balancer_utils.lua b/spec/fixtures/balancer_utils.lua index 9dd33b6a187..1edecf90b1a 100644 --- a/spec/fixtures/balancer_utils.lua +++ b/spec/fixtures/balancer_utils.lua @@ -479,12 +479,12 @@ local function end_testcase_setup(strategy, bp, consistency) end -local function get_db_utils_for_dc_and_admin_api(strategy, tables) - local bp = assert(helpers.get_db_utils(strategy, tables)) +local function get_db_utils_for_dc_and_admin_api(strategy, tables, plugins) + local bp, db = assert(helpers.get_db_utils(strategy, tables, plugins)) if strategy ~= "off" then bp = require("spec.fixtures.admin_api") end - return bp + return bp, db end diff --git a/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/handler.lua new file mode 100644 index 00000000000..a71b0232520 --- /dev/null +++ b/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/handler.lua @@ -0,0 +1,20 @@ +local kong = kong +local math = math + + +local WorkerEventsPoll = { + PRIORITY = math.huge +} + + +function WorkerEventsPoll:preread() + kong.worker_events.poll() +end + + +function WorkerEventsPoll:rewrite() + kong.worker_events.poll() +end + + +return WorkerEventsPoll diff --git a/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/schema.lua b/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/schema.lua new file mode 100644 index 00000000000..b79b7cd89a3 --- /dev/null +++ b/spec/fixtures/custom_plugins/kong/plugins/worker-events-poll/schema.lua @@ -0,0 +1,17 @@ +local typedefs = require "kong.db.schema.typedefs" + +return { + name = "worker-events-poll", + fields = { + { + protocols = typedefs.protocols { default = { "http", "https", "tcp", "tls", "grpc", "grpcs" } }, + }, + { + config = { + type = "record", + fields = { + }, + }, + }, + }, +} From dfae8ce8814035a795ef1be46a1c510e1f860c7b Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Thu, 3 Dec 2020 20:03:21 +0200 Subject: [PATCH 3/6] feat(db) poll events before calling status api --- kong/init.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/kong/init.lua b/kong/init.lua index b8c202f0134..e2b4243b855 100644 --- a/kong/init.lua +++ b/kong/init.lua @@ -1464,6 +1464,7 @@ end function Kong.status_content() + kong.worker_events.poll() return serve_content("kong.status") end From 3f9af15ce47f4ff088cae851e72392fab218a1fa Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Thu, 3 Dec 2020 22:33:31 +0200 Subject: [PATCH 4/6] tests(oauth2) fix tests with worker events 2.0 --- .../25-oauth2/04-invalidations_spec.lua | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/spec/03-plugins/25-oauth2/04-invalidations_spec.lua b/spec/03-plugins/25-oauth2/04-invalidations_spec.lua index 0a1aded14c8..774c66a01b5 100644 --- a/spec/03-plugins/25-oauth2/04-invalidations_spec.lua +++ b/spec/03-plugins/25-oauth2/04-invalidations_spec.lua @@ -316,6 +316,15 @@ for _, strategy in helpers.each_strategy() do local token = cjson.decode(assert.res_status(200, res)) assert.is_table(token) + -- Check that cache is not populated + local cache_key = db.oauth2_tokens:cache_key(token.access_token) + local res = assert(admin_client:send { + method = "GET", + path = "/cache/" .. cache_key, + headers = {} + }) + assert.res_status(404, res) + -- The token should work local res = assert(proxy_ssl_client:send { method = "GET", @@ -327,7 +336,6 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, res) -- Check that cache is populated - local cache_key = db.oauth2_tokens:cache_key(token.access_token) local res = assert(admin_client:send { method = "GET", path = "/cache/" .. cache_key, @@ -376,6 +384,15 @@ for _, strategy in helpers.each_strategy() do local token = cjson.decode(assert.res_status(200, res)) assert.is_table(token) + -- Check that cache is not populated + local cache_key = db.oauth2_tokens:cache_key(token.access_token) + local res = assert(admin_client:send { + method = "GET", + path = "/cache/" .. cache_key, + headers = {} + }) + assert.res_status(404, res) + -- The token should work local res = assert(proxy_ssl_client:send { method = "GET", @@ -387,8 +404,6 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, res) -- Check that cache is populated - local cache_key = db.oauth2_tokens:cache_key(token.access_token) - local res = assert(admin_client:send { method = "GET", path = "/cache/" .. cache_key, @@ -454,6 +469,15 @@ for _, strategy in helpers.each_strategy() do local token = cjson.decode(assert.res_status(200, res)) assert.is_table(token) + -- Check that cache is not populated + local cache_key = db.oauth2_tokens:cache_key(token.access_token) + local res = assert(admin_client:send { + method = "GET", + path = "/cache/" .. cache_key, + headers = {} + }) + assert.res_status(404, res) + -- The token should work local res = assert(proxy_ssl_client:send { method = "GET", @@ -465,8 +489,6 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, res) -- Check that cache is populated - local cache_key = db.oauth2_tokens:cache_key(token.access_token) - local res = assert(admin_client:send { method = "GET", path = "/cache/" .. cache_key, From b5d9329c42eec19bec115d42b86cc0e3ff051fd1 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Thu, 3 Dec 2020 23:52:28 +0200 Subject: [PATCH 5/6] poll events --- kong/db/declarative/init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kong/db/declarative/init.lua b/kong/db/declarative/init.lua index 8b6ae815974..d4e08b52556 100644 --- a/kong/db/declarative/init.lua +++ b/kong/db/declarative/init.lua @@ -841,7 +841,7 @@ do ok, err = kong.worker_events.post("declarative", "flip_config", default_ws) if not ok then ngx.shared.kong:delete(DECLARATIVE_LOCK_KEY) - return nil, "failed to flip declarative config cache pages: " .. (err or ok) + return nil, "failed to flip declarative config cache pages: " .. (err or "unknown error") end else From 1a1cd5fe415d62d90cc715ed5ec66b71f7ac6ab3 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Wed, 13 Jan 2021 23:55:09 +0200 Subject: [PATCH 6/6] mark flaky --- .../05-proxy/10-balancer/01-healthchecks_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua b/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua index 9774bea0307..ab4357a4ef3 100644 --- a/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua +++ b/spec/02-integration/05-proxy/10-balancer/01-healthchecks_spec.lua @@ -2022,7 +2022,7 @@ for _, strategy in helpers.each_strategy() do end) - it("perform passive health checks -- connection #timeouts", function() + it("#flaky perform passive health checks -- connection #timeouts", function() -- configure healthchecks bu.begin_testcase_setup(strategy, bp)