refactor(userspace/engine): remove Lua from Falco and re-implement the rule loader

.gitignore

@@ -12,6 +12,4 @@ test/build
 
 .vscode/*
 
-.luacheckcache
-
 *.idea*

CMakeLists.txt

@@ -115,9 +115,6 @@ include(ExternalProject)
 # libs
 include(falcosecurity-libs)
 
-# LuaJit provided by libs
-include(luajit)
-
 # jq
 include(jq)
 
@@ -152,12 +149,6 @@ endif()
 
 include(cxxopts)
 
-# libyaml
-include(libyaml)
-
-# lyaml
-include(lyaml)
-
 # One TBB
 include(tbb)
 

cmake/modules/falcosecurity-libs.cmake

@@ -62,14 +62,9 @@ set(LIBSINSP_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
 set(CREATE_TEST_TARGETS OFF CACHE BOOL "")
 set(BUILD_LIBSCAP_EXAMPLES OFF CACHE BOOL "")
 
-# todo(leogr): although Falco does not actually depend on chisels, we need this for the lua_parser. 
-# Hopefully, we can switch off this in the future
-set(WITH_CHISEL ON CACHE BOOL "")
-
 set(USE_BUNDLED_TBB ON CACHE BOOL "")
 set(USE_BUNDLED_B64 ON CACHE BOOL "")
 set(USE_BUNDLED_JSONCPP ON CACHE BOOL "")
-set(USE_BUNDLED_LUAJIT ON CACHE BOOL "")
 
 list(APPEND CMAKE_MODULE_PATH "${FALCOSECURITY_LIBS_SOURCE_DIR}/cmake/modules")
 

test/.gitignore

@@ -1 +1,2 @@
-falco_traces.yaml
+falco_traces.yaml
+venv/*

test/falco_tests.yaml

@@ -254,9 +254,6 @@ trace_files: !mux
     stdout_is: |+
       1 errors:
       Rules content is not yaml
-      ---
-      This is not yaml
-      ---
     validate_rules_file:
       - rules/invalid_not_yaml.yaml
     trace_file: trace_files/cat_write.scap
@@ -266,9 +263,6 @@ trace_files: !mux
     stdout_is: |+
       1 errors:
       Rules content is not yaml array of objects
-      ---
-      foo: bar
-      ---
     validate_rules_file:
       - rules/invalid_not_array.yaml
     trace_file: trace_files/cat_write.scap
@@ -277,7 +271,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Unexpected element of type string. Each element should be a yaml associative array.
+      Unexpected element type. Each element should be a yaml associative array.
       ---
       - foo
       ---
@@ -299,12 +293,6 @@ trace_files: !mux
 
   invalid_yaml_parse_error:
     exit_status: 1
-    stdout_is: |+
-      1 errors:
-      mapping values are not allowed in this context
-      ---
-      this : is : not : yaml
-      ---
     validate_rules_file:
       - rules/invalid_yaml_parse_error.yaml
     trace_file: trace_files/cat_write.scap
@@ -339,7 +327,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Rule must have property output
+      Rule must have properties 'condition', 'output', 'desc', and 'priority'
       ---
       - rule: no output rule
         desc: some desc

userspace/engine/CMakeLists.txt

@@ -10,51 +10,47 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-add_subdirectory(lua)
-
 set(FALCO_ENGINE_SOURCE_FILES
-    rules.cpp
     falco_common.cpp
     falco_engine.cpp
     falco_utils.cpp
     json_evt.cpp
     ruleset.cpp
     formats.cpp
     filter_macro_resolver.cpp
-    lua_filter_helper.cpp)
+    rule_loader.cpp
+    stats_manager.cpp)
 
 add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})
-add_dependencies(falco_engine njson lyaml string-view-lite)
+add_dependencies(falco_engine njson string-view-lite)
 
 if(USE_BUNDLED_DEPS)
-  add_dependencies(falco_engine libyaml)
+  add_dependencies(falco_engine yamlcpp)
 endif()
 
 if(MINIMAL_BUILD)
   target_include_directories(
     falco_engine
     PUBLIC
-      "${LUAJIT_INCLUDE}"
       "${NJSON_INCLUDE}"
       "${TBB_INCLUDE_DIR}"
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine"
-      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
+      "${YAMLCPP_INCLUDE_DIR}"
+      "${PROJECT_BINARY_DIR}/userspace/engine")
 else()
   target_include_directories(
     falco_engine
     PUBLIC
-      "${LUAJIT_INCLUDE}"
       "${NJSON_INCLUDE}"
       "${CURL_INCLUDE_DIR}"
       "${TBB_INCLUDE_DIR}"
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine"
-      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
+      "${YAMLCPP_INCLUDE_DIR}"
+      "${PROJECT_BINARY_DIR}/userspace/engine")
 endif()
 
-target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LYAML_LIB}" "${LIBYAML_LIB}" luafiles)
+target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${YAMLCPP_LIB}")

userspace/engine/falco_common.cpp

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,67 +14,44 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <fstream>
-
 #include "falco_common.h"
-#include "banned.h" // This raises a compilation error when certain functions are used
-#include "falco_engine_lua_files.hh"
 
-std::vector<std::string> falco_common::priority_names = {
+vector<string> falco_common::priority_names = {
 	"Emergency",
 	"Alert",
 	"Critical",
 	"Error",
 	"Warning",
 	"Notice",
-	"Informational",
-	"Debug"};
+	"Info",
+	"Debug"
+};
 
-falco_common::falco_common()
+bool falco_common::parse_priority(string v, priority_type& out)
 {
-	m_ls = lua_open();
-	if(!m_ls)
+	transform(v.begin(), v.end(), v.begin(), [](int c){return tolower(c);});
+	for (size_t i = 0; i < priority_names.size(); i++)
 	{
-		throw falco_exception("Cannot open lua");
-	}
-	luaL_openlibs(m_ls);
-}
-
-falco_common::~falco_common()
-{
-	if(m_ls)
-	{
-		lua_close(m_ls);
-	}
-}
-
-void falco_common::init()
-{
-	// Strings in the list lua_module_strings need to be loaded as
-	// lua modules, which also involves adding them to the
-	// package.module table.
-	for(const auto &pair : lua_module_strings)
-	{
-		lua_getglobal(m_ls, "package");
-		lua_getfield(m_ls, -1, "preload");
-
-		if(luaL_loadstring(m_ls, pair.first))
+		auto p = priority_names[i];
+		transform(p.begin(), p.end(), p.begin(), [](int c){return tolower(c);});
+		// note: for legacy reasons, "Info" and "Informational" has been used
+		// interchangeably and ambiguously, so this is the only edge case for
+		// which we can't apply strict equality check
+		if (p == v || (v == "informational" && p == "info"))
 		{
-			throw falco_exception("Failed to load embedded lua code " +
-					      string(pair.second) + ": " + lua_tostring(m_ls, -1));
+			out = (priority_type) i;
+			return true;
 		}
-
-		lua_setfield(m_ls, -2, pair.second);
 	}
+	return false;
+}
 
-	// Strings in the list lua_code_strings need to be loaded and
-	// evaluated so any public functions can be directly called.
-	for(const auto &str : lua_code_strings)
+bool falco_common::format_priority(priority_type v, string& out)
+{
+	if ((size_t) v < priority_names.size())
 	{
-		if(luaL_loadstring(m_ls, str) || lua_pcall(m_ls, 0, 0, 0))
-		{
-			throw falco_exception("Failed to load + evaluate embedded lua code " +
-					      string(str) + ": " + lua_tostring(m_ls, -1));
-		}
+		out = priority_names[(size_t) v];
+		return true;
 	}
+	return false;
 }

userspace/engine/falco_common.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,13 +19,6 @@ limitations under the License.
 #include <string>
 #include <exception>
 #include <mutex>
-
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-}
-
 #include <sinsp.h>
 
 //
@@ -57,22 +50,12 @@ struct falco_exception : std::exception
 	std::string m_error_str;
 };
 
-//
-// This is the base class of falco_engine/falco_output. It is
-// responsible for managing a lua state and associated inspector and
-// loading a single "main" lua file into that state.
-//
-
-class falco_common
+namespace falco_common
 {
-public:
-	falco_common();
-	virtual ~falco_common();
-
-	void init();
+	const string syscall_source = "syscall";
 
-    // Priority levels, as a vector of strings
-	static std::vector<std::string> priority_names;
+	// Priority levels, as a vector of strings
+	extern std::vector<std::string> priority_names;
 
 	// Same as numbers/indices into the above vector
 	enum priority_type
@@ -86,9 +69,7 @@ class falco_common
 		PRIORITY_INFORMATIONAL = 6,
 		PRIORITY_DEBUG = 7
 	};
-
-protected:
-	lua_State *m_ls;
-
-	std::mutex m_ls_semaphore;
+
+	bool parse_priority(std::string v, priority_type& out);
+	bool format_priority(priority_type v, std::string& out);
 };