diff --git a/Makefile b/Makefile index 5a82f5e93ebe..f2adafadba56 100644 --- a/Makefile +++ b/Makefile @@ -341,9 +341,6 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/serverless $(ENV_INSTALL) apisix/plugins/serverless/*.lua $(ENV_INST_LUADIR)/apisix/plugins/serverless/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/slslog - $(ENV_INSTALL) apisix/plugins/slslog/*.lua $(ENV_INST_LUADIR)/apisix/plugins/slslog/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/syslog $(ENV_INSTALL) apisix/plugins/syslog/*.lua $(ENV_INST_LUADIR)/apisix/plugins/syslog/ diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 6c2415ac6d29..de2fbae67ffa 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -21,7 +21,7 @@ local bp_manager_mod = require("apisix.utils.batch-processor-manager") local plugin_name = "sls-logger" local ngx = ngx -local rf5424 = require("apisix.plugins.slslog.rfc5424") +local rf5424 = require("apisix.utils.rfc5424") local tcp = ngx.socket.tcp local tostring = tostring local ipairs = ipairs @@ -138,9 +138,14 @@ function _M.log(conf, ctx) return end - local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host,"apisix", - ctx.var.pid, conf.project, conf.logstore, - conf.access_key_id, conf.access_key_secret, json_str) + local structured_data = { + {name = "project", value = conf.project}, + {name = "logstore", value = conf.logstore}, + {name = "access-key-id", value = conf.access_key_id}, + {name = "access-key-secret", value = conf.access_key_secret}, + } + local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", + ctx.var.pid, json_str, structured_data) local process_context = { data = rf5424_data, diff --git a/apisix/plugins/syslog/init.lua b/apisix/plugins/syslog/init.lua index 24f2f62ab812..0ab34f8054e9 100644 --- a/apisix/plugins/syslog/init.lua +++ b/apisix/plugins/syslog/init.lua @@ -18,6 +18,10 @@ local core = require("apisix.core") local bp_manager_mod = require("apisix.utils.batch-processor-manager") local logger_socket = require("resty.logger.socket") +local rfc5424 = require("apisix.utils.rfc5424") +local ipairs = ipairs +local table_insert = core.table.insert +local table_concat = core.table.concat local batch_processor_manager = bp_manager_mod.new("sys logger") @@ -63,7 +67,8 @@ local function send_syslog_data(conf, log_message, api_ctx) end -- reuse the logger object - local ok, err = logger:log(core.json.encode(log_message)) + local ok, err = logger:log(log_message) + if not ok then res = false err_msg = "failed to log message" .. err @@ -75,28 +80,32 @@ end -- called in log phase of APISIX function _M.push_entry(conf, ctx, entry) - if batch_processor_manager:add_entry(conf, entry) then + local json_str, err = core.json.encode(entry) + if not json_str then + core.log.error('error occurred while encoding the data: ', err) + return + end + + local rfc5424_data = rfc5424.encode("SYSLOG", "INFO", ctx.var.host, + "apisix", ctx.var.pid, json_str) + + if batch_processor_manager:add_entry(conf, rfc5424_data) then return end -- Generate a function to be executed by the batch processor local cp_ctx = core.table.clone(ctx) - local func = function(entries, batch_max_size) - local data, err - if batch_max_size == 1 then - data, err = core.json.encode(entries[1]) -- encode as single {} - else - data, err = core.json.encode(entries) -- encode as array [{}] - end - - if not data then - return false, 'error occurred while encoding the data: ' .. err + local func = function(entries) + local items = {} + for _, e in ipairs(entries) do + table_insert(items, e) + core.log.debug("buffered logs:", e) end - return send_syslog_data(conf, data, cp_ctx) + return send_syslog_data(conf, table_concat(items), cp_ctx) end - batch_processor_manager:add_entry_to_new_processor(conf, entry, ctx, func) + batch_processor_manager:add_entry_to_new_processor(conf, rfc5424_data, ctx, func) end diff --git a/apisix/plugins/slslog/rfc5424.lua b/apisix/utils/rfc5424.lua similarity index 83% rename from apisix/plugins/slslog/rfc5424.lua rename to apisix/utils/rfc5424.lua index 5d09a58a5165..e046194bf349 100644 --- a/apisix/plugins/slslog/rfc5424.lua +++ b/apisix/utils/rfc5424.lua @@ -79,12 +79,13 @@ local Severity = { } local log_util = require("apisix.utils.log-util") - +local ipairs = ipairs +local str_format = string.format local _M = { version = 0.1 } -function _M.encode(facility, severity, hostname, appname, pid, project, - logstore, access_key_id, access_key_secret, msg) + +function _M.encode(facility, severity, hostname, appname, pid, msg, structured_data) local pri = (Facility[facility] * 8 + Severity[severity]) local t = log_util.get_rfc3339_zulu_timestamp() if not hostname then @@ -95,10 +96,19 @@ function _M.encode(facility, severity, hostname, appname, pid, project, appname = "-" end - return "<" .. pri .. ">1 " .. t .. " " .. hostname .. " " .. appname .. " " .. pid - .. " - [logservice project=\"" .. project .. "\" logstore=\"" .. logstore - .. "\" access-key-id=\"" .. access_key_id .. "\" access-key-secret=\"" - .. access_key_secret .. "\"] " .. msg .. "\n" + local structured_data_str = "-" + + if structured_data then + structured_data_str = "[logservice" + for _, sd_param in ipairs(structured_data) do + structured_data_str = structured_data_str .. " " .. sd_param.name + .. "=\"" .. sd_param.value .. "\"" + end + structured_data_str = structured_data_str .. "]" + end + + return str_format("<%d>1 %s %s %s %d - %s %s\n", pri, t, hostname, + appname, pid, structured_data_str, msg) end return _M diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml index f4e3916987ed..0979e1e617b5 100644 --- a/ci/pod/docker-compose.plugin.yml +++ b/ci/pod/docker-compose.plugin.yml @@ -321,6 +321,8 @@ services: ports: - '3000:3000' - '43000:43000' + - '5140:5140' + - '5150:5150/udp' networks: vector_net: diff --git a/ci/pod/vector/vector.toml b/ci/pod/vector/vector.toml index 0e02e0fd29a1..953f30746c05 100644 --- a/ci/pod/vector/vector.toml +++ b/ci/pod/vector/vector.toml @@ -35,8 +35,18 @@ tls.ca_file = "/certs/vector_logs_ca.crt" tls.crt_file = "/certs/vector_logs_server.crt" tls.key_file = "/certs/vector_logs_server.key" +[sources.log-from-syslog-tcp] +type = "syslog" +address = "0.0.0.0:5140" +mode = "tcp" + +[sources.log-from-syslog-udp] +type = "syslog" +address = "0.0.0.0:5150" +mode = "udp" + [sinks.log-2-console] -inputs = [ "log-from-tcp", "log-from-tls" ] +inputs = [ "log-from-tcp", "log-from-tls", "log-from-syslog-tcp", "log-from-syslog-udp" ] type = "console" encoding.codec = "json" @@ -51,3 +61,15 @@ inputs = [ "log-from-tls" ] type = "file" encoding.codec = "json" path = "/etc/vector/tls-datas.log" + +[sinks.log-2-syslog-tcp-file] +inputs = [ "log-from-syslog-tcp" ] +type = "file" +encoding.codec = "text" +path = "/etc/vector/syslog-tcp.log" + +[sinks.log-2-syslog-udp-file] +inputs = [ "log-from-syslog-udp" ] +type = "file" +encoding.codec = "text" +path = "/etc/vector/syslog-udp.log" diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index a56d6121f876..9e668e1bf3b0 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -173,16 +173,20 @@ hello world end math.randomseed(os.time()) - local rfc5424 = require("apisix.plugins.slslog.rfc5424") + local rfc5424 = require("apisix.utils.rfc5424") local m = 0 -- because the millisecond value obtained by `ngx.now` may be `0` -- it is executed multiple times to ensure the accuracy of the test for i = 1, 5 do ngx.sleep(string.format("%0.3f", math.random())) + local structured_data = { + {name = "project", value = "apisix.apache.org"}, + {name = "logstore", value = "apisix.apache.org"}, + {name = "access-key-id", value = "apisix.sls.logger"}, + {name = "access-key-secret", value = "BD274822-96AA-4DA6-90EC-15940FB24444"} + } local log_entry = rfc5424.encode("SYSLOG", "INFO", "localhost", "apisix", - 123456, "apisix.apache.org", "apisix.apache.log", - "apisix.sls.logger", "BD274822-96AA-4DA6-90EC-15940FB24444", - "hello world") + 123456, "hello world", structured_data) m = get_syslog_timestamp_millisecond(log_entry) + m end @@ -226,15 +230,13 @@ passed === TEST 9: access --- extra_init_by_lua local json = require("toolkit.json") - local rfc5424 = require("apisix.plugins.slslog.rfc5424") + local rfc5424 = require("apisix.utils.rfc5424") local old_f = rfc5424.encode - rfc5424.encode = function(facility, severity, hostname, appname, pid, project, - logstore, access_key_id, access_key_secret, msg) + rfc5424.encode = function(facility, severity, hostname, appname, pid, msg, structured_data) local r = json.decode(msg) assert(r.client_ip == "127.0.0.1", r.client_ip) assert(r.host == "localhost", r.host) - return old_f(facility, severity, hostname, appname, pid, project, - logstore, access_key_id, access_key_secret, msg) + return old_f(facility, severity, hostname, appname, pid, msg, structured_data) end --- request GET /hello @@ -372,14 +374,12 @@ passed === TEST 13: access --- extra_init_by_lua local json = require("toolkit.json") - local rfc5424 = require("apisix.plugins.slslog.rfc5424") + local rfc5424 = require("apisix.utils.rfc5424") local old_f = rfc5424.encode - rfc5424.encode = function(facility, severity, hostname, appname, pid, project, - logstore, access_key_id, access_key_secret, msg) + rfc5424.encode = function(facility, severity, hostname, appname, pid, msg, structured_data) local r = json.decode(msg) assert(r.vip == "127.0.0.1", r.vip) - return old_f(facility, severity, hostname, appname, pid, project, - logstore, access_key_id, access_key_secret, msg) + return old_f(facility, severity, hostname, appname, pid, msg, structured_data) end --- request GET /hello diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index a8795bc24be6..5e13aa301c56 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -31,7 +31,7 @@ __DATA__ local plugin = require("apisix.plugins.syslog") local ok, err = plugin.check_schema({ host = "127.0.0.1", - port = 3000, + port = 5140, }) if not ok then ngx.say(err) @@ -73,7 +73,7 @@ done local plugin = require("apisix.plugins.syslog") local ok, err = plugin.check_schema({ host = "127.0.0.1", - port = "3000", + port = "5140", }) if not ok then ngx.say(err) @@ -100,7 +100,7 @@ done "plugins": { "syslog": { "host" : "127.0.0.1", - "port" : 5044 + "port" : 5140 } }, "upstream": { @@ -142,7 +142,7 @@ hello world local logger_socket = require("resty.logger.socket") local logger, err = logger_socket:new({ host = "127.0.0.1", - port = 5044, + port = 5140, flush_limit = 100, }) @@ -183,7 +183,7 @@ done "plugins": { "syslog": { "host" : "127.0.0.1", - "port" : 5044, + "port" : 5140, "flush_limit" : 1, "inactive_timeout": 1 } @@ -236,7 +236,15 @@ unlock with key route#1 -=== TEST 8: check plugin configuration updating +=== TEST 8: check log +--- exec +tail -n 1 ci/pod/vector/syslog-tcp.log +--- response_body eval +qr/.*apisix_latency.*/ + + + +=== TEST 9: check plugin configuration updating --- config location /t { content_by_lua_block { @@ -327,7 +335,7 @@ sending a batch logs to 127.0.0.1:5045 -=== TEST 9: add log format +=== TEST 10: add log format --- config location /t { content_by_lua_block { @@ -355,7 +363,7 @@ passed -=== TEST 10: Add route and Enable Syslog Plugin, batch_max_size=1 +=== TEST 11: Add route and Enable Syslog Plugin, batch_max_size=1 --- config location /t { content_by_lua_block { @@ -369,7 +377,7 @@ passed "disable": false, "flush_limit": 1, "host" : "127.0.0.1", - "port" : 5050 + "port" : 5140 } }, "upstream": { @@ -394,7 +402,7 @@ passed -=== TEST 11: hit route and report sys logger +=== TEST 12: hit route and report sys logger --- extra_init_by_lua local syslog = require("apisix.plugins.syslog.init") local json = require("apisix.core.json") @@ -416,7 +424,15 @@ qr/syslog-log-format.*\{.*"upstream":"127.0.0.1:\d+"/ -=== TEST 12: log format in plugin +=== TEST 13: check log +--- exec +tail -n 1 ci/pod/vector/syslog-tcp.log +--- response_body eval +qr/.*\"host\":\"localhost\".*/ + + + +=== TEST 14: log format in plugin --- config location /t { content_by_lua_block { @@ -432,7 +448,7 @@ qr/syslog-log-format.*\{.*"upstream":"127.0.0.1:\d+"/ "vip": "$remote_addr" }, "host" : "127.0.0.1", - "port" : 5050 + "port" : 5140 } }, "upstream": { @@ -461,7 +477,7 @@ passed -=== TEST 13: access +=== TEST 15: access --- extra_init_by_lua local syslog = require("apisix.plugins.syslog.init") local json = require("apisix.core.json") @@ -481,3 +497,65 @@ hello world [error] --- error_log push_entry is called with data + + + +=== TEST 16: check log +--- exec +tail -n 1 ci/pod/vector/syslog-tcp.log +--- response_body eval +qr/.*vip.*/ + + + +=== TEST 17: test udp mode +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "disable": false, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5150, + "sock_type": "udp" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 18: hit +--- request +GET /hello + + + +=== TEST 19: check log +--- exec +tail -n 1 ci/pod/vector/syslog-udp.log +--- response_body eval +qr/.*upstream.*/ diff --git a/t/utils/rfc5424.t b/t/utils/rfc5424.t new file mode 100644 index 000000000000..06051e627cc4 --- /dev/null +++ b/t/utils/rfc5424.t @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: Compatibility testing +--- config + location /t { + content_by_lua_block { + local rfc5424 = require("apisix.utils.rfc5424") + local structured_data = { + {name = "project", value = "apisix.apache.org"}, + {name = "logstore", value = "apisix.apache.org"}, + {name = "access-key-id", value = "apisix.sls.logger"}, + {name = "access-key-secret", value = "BD274822-96AA-4DA6-90EC-15940FB24444"} + } + local data = rfc5424.encode("SYSLOG", "INFO", "localhost", "apisix", + 123456, "hello world", structured_data) + ngx.say(data) + } + } +--- response_body eval +qr/<46>1.*localhost apisix 123456 - \[logservice project=\"apisix\.apache\.org\" logstore=\"apisix\.apache\.org\" access-key-id=\"apisix\.sls\.logger\" access-key-secret=\"BD274822-96AA-4DA6-90EC-15940FB24444\"\] hello world/ + + + +=== TEST 2: No structured data test +--- config + location /t { + content_by_lua_block { + local rfc5424 = require("apisix.utils.rfc5424") + local data = rfc5424.encode("SYSLOG", "INFO", "localhost", "apisix", + 123456, "hello world") + ngx.say(data) + } + } +--- response_body eval +qr/<46>1.*localhost apisix 123456 - - hello world/ + + + +=== TEST 3: No host and appname test +--- config + location /t { + content_by_lua_block { + local rfc5424 = require("apisix.utils.rfc5424") + local data = rfc5424.encode("SYSLOG", "INFO", nil, nil, + 123456, "hello world") + ngx.say(data) + } + } +--- response_body eval +qr/<46>1.*- - 123456 - - hello world/ diff --git a/t/xrpc/pingpong2.t b/t/xrpc/pingpong2.t index fc77fa1482df..7365929ac326 100644 --- a/t/xrpc/pingpong2.t +++ b/t/xrpc/pingpong2.t @@ -497,7 +497,7 @@ passed --- stream_conf_enable --- wait: 0.5 --- error_log eval -qr/message received:.*\"client_ip\\"\:\\"127.0.0.1\\"/ +qr/message received:.*\"client_ip\"\:\"127.0.0.1\"/ @@ -556,7 +556,7 @@ passed --- stream_conf_enable --- wait: 0.5 --- error_log eval -qr/message received:.*\"client_ip\\"\:\\"127.0.0.1\\"/ +qr/message received:.*\"client_ip\"\:\"127.0.0.1\"/ diff --git a/t/xrpc/pingpong3.t b/t/xrpc/pingpong3.t index c6d98810d656..da16e626b9e4 100644 --- a/t/xrpc/pingpong3.t +++ b/t/xrpc/pingpong3.t @@ -190,4 +190,4 @@ passed --- stream_conf_enable --- wait: 0.5 --- error_log eval -qr/message received:.*\"rpc_time\\"\:(0.\d+|0)\}\"/ +qr/message received:.*\"rpc_time\"\:(0.\d+|0)\}/ diff --git a/t/xrpc/redis2.t b/t/xrpc/redis2.t index 65ca9829c616..7e378f836e49 100644 --- a/t/xrpc/redis2.t +++ b/t/xrpc/redis2.t @@ -193,8 +193,8 @@ ping: pong --- stream_conf_enable --- wait: 1 --- grep_error_log eval -qr/message received:.*\"redis_cmd_line\\"\:[^,]+/ +qr/message received:.*\"redis_cmd_line\":[^,]+/ --- grep_error_log_out eval -[qr/message received:.*\"redis_cmd_line\\"\:\\\"hmset animals dog bark cat meow\\\"/, -qr/message received:.*\"redis_cmd_line\\"\:\\\"hmget animals dog cat\\\"/, -qr/message received:.*\"redis_cmd_line\\"\:\\\"ping\\\"/] +[qr/message received:.*\"redis_cmd_line\":\"hmset animals dog bark cat meow\"/, +qr/message received:.*\"redis_cmd_line\":\"hmget animals dog cat\"/, +qr/message received:.*\"redis_cmd_line\":\"ping\"/]