fix: the problem of high cpu and memory usage (apache#9015)

cocurrent requests after reload or update the upstream nodes will
cause high cpu and memory usage, the checker created by healthcheck.new
in create_checker won't be released if the program crashes after
cancel_clean_handler failed

apisix/core/config_util.lua

@@ -88,9 +88,15 @@ function _M.cancel_clean_handler(item, idx, fire)
     end
 
     core_tab.remove(item.clean_handlers, pos)
-    if fire then
+    if not fire then
+        return
+    end
+
+    if f then
         f(item)
     end
+
+    log.error("call cancel_clean_handler error, f is nil")
 end
 
 

apisix/upstream.lua

@@ -23,6 +23,7 @@ local error = error
 local tostring = tostring
 local ipairs = ipairs
 local pairs = pairs
+local pcall = pcall
 local ngx_var = ngx.var
 local is_http = ngx.config.subsystem == "http"
 local upstreams
@@ -103,6 +104,12 @@ local function create_checker(upstream)
         return healthcheck_parent.checker
     end
 
+    if upstream.is_creating_checker then
+        core.log.info("another request is creating new checker")
+        return nil
+    end
+    upstream.is_creating_checker = true
+
     local checker, err = healthcheck.new({
         name = get_healthchecker_name(healthcheck_parent),
         shm_name = "upstream-healthcheck",
@@ -115,8 +122,11 @@ local function create_checker(upstream)
     end
 
     if healthcheck_parent.checker then
-        core.config_util.cancel_clean_handler(healthcheck_parent,
+        local ok, err = pcall(core.config_util.cancel_clean_handler, healthcheck_parent,
                                               healthcheck_parent.checker_idx, true)
+        if not ok then
+            core.log.error("cancel clean handler error: ", err)
+        end
     end
 
     core.log.info("create new checker: ", tostring(checker))
@@ -140,6 +150,8 @@ local function create_checker(upstream)
     healthcheck_parent.checker_idx =
         core.config_util.add_clean_handler(healthcheck_parent, release_checker)
 
+    upstream.is_creating_checker = nil
+
     return checker
 end