[prometheus] export captured log lines by log level

gateway/conf/nginx.conf.liquid

@@ -40,6 +40,7 @@ http {
 
   lua_package_path "{{ lua_path | default: package.path }}";
   lua_package_cpath "{{ lua_cpath | default: package.cpath }}";
+  lua_capture_error_log 4k;
 
   ignore_invalid_headers off;
 
@@ -108,6 +109,7 @@ http {
 
   {% if port.metrics %}
     lua_shared_dict prometheus_metrics 16M;
+
     server {
       access_log off;
       listen {{ port.metrics }} default_server;

gateway/src/apicast/policy/apicast/apicast.lua

@@ -3,6 +3,8 @@ local math = math
 local setmetatable = setmetatable
 
 local user_agent = require('apicast.user_agent')
+local errlog = require('ngx.errlog')
+local prometheus = require('apicast.prometheus')
 
 local _M = require('apicast.policy').new('APIcast', require('apicast.version'))
 
@@ -17,6 +19,8 @@ function _M.new()
 end
 
 function _M.init()
+  errlog.set_filter_level(ngx.ERR)
+
   user_agent.cache()
 
   math.randomseed(ngx.now())
@@ -84,4 +88,30 @@ end
 
 _M.balancer = balancer.call
 
+local logs_metric = prometheus('counter', 'nginx_error_log', "Items in nginx error log", {'level'})
+
+local log_map = {
+  'emerg',
+  'alert',
+  'crit',
+  'error',
+  'warn',
+  'notice',
+  'info',
+  'debug',
+}
+
+_M.metrics = function()
+  local logs = errlog.get_logs()
+
+  if not logs then return nil, 'could not get logs' end
+
+  local labels = {}
+
+  for i = 1, #logs, 3 do
+    labels[1] = log_map[logs[i]]
+    logs_metric:inc(1, labels)
+  end
+end
+
 return _M