From a33cda131aaf3b00fb90f20feb01e8c16d36faa7 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 12:46:07 -0800 Subject: [PATCH 1/6] Add ability to provide external rules reader/collector/compiler In some cases, a user of the falco engine may want to extend the falco rules format to provide additional objects to the rules file. To support that, add a new method set_rule_loader() that allows a user to provide classes that derive from rule_loader::{reader,collector,compiler} and read those additional objects from the rules file. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine.cpp | 69 ++++++++++++++++++++++--------- userspace/engine/falco_engine.h | 19 +++++++-- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index b2b043a3f03..d4fd6178108 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -27,6 +27,7 @@ limitations under the License. #include #include #include +#include #include #include @@ -38,8 +39,6 @@ limitations under the License. #include "falco_engine.h" #include "falco_utils.h" #include "falco_engine_version.h" -#include "rule_loader_reader.h" -#include "rule_loader_compiler.h" #include "formats.h" @@ -53,6 +52,9 @@ using namespace falco; falco_engine::falco_engine(bool seed_rng) : m_syscall_source(NULL), m_syscall_source_idx(SIZE_MAX), + m_rule_reader(std::make_shared()), + m_rule_collector(std::make_shared()), + m_rule_compiler(std::make_shared()), m_next_ruleset_id(0), m_min_priority(falco_common::PRIORITY_DEBUG), m_sampling_ratio(1), m_sampling_multiplier(0), @@ -69,7 +71,7 @@ falco_engine::falco_engine(bool seed_rng) falco_engine::~falco_engine() { m_rules.clear(); - m_rule_collector.clear(); + m_rule_collector->clear(); m_rule_stats_manager.clear(); m_sources.clear(); } @@ -79,6 +81,36 @@ sinsp_version falco_engine::engine_version() return sinsp_version(FALCO_ENGINE_VERSION); } +void falco_engine::set_rule_reader(std::shared_ptr reader) +{ + m_rule_reader = reader; +} + +std::shared_ptr falco_engine::get_rule_reader() +{ + return m_rule_reader; +} + +void falco_engine::set_rule_collector(std::shared_ptr collector) +{ + m_rule_collector = collector; +} + +std::shared_ptr falco_engine::get_rule_collector() +{ + return m_rule_collector; +} + +void falco_engine::set_rule_compiler(std::shared_ptr compiler) +{ + m_rule_compiler = compiler; +} + +std::shared_ptr falco_engine::get_rule_compiler() +{ + return m_rule_compiler; +} + // Return a key that uniquely represents a field class. // For now, we assume name + shortdesc is unique. static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info) @@ -164,12 +196,11 @@ std::unique_ptr falco_engine::load_rules(const std::string &rules_c cfg.replace_output_container_info = m_replace_container_info; // read rules YAML file and collect its definitions - rule_loader::reader reader; - if (reader.read(cfg, m_rule_collector)) + if(m_rule_reader->read(cfg, *(m_rule_collector.get()))) { // compile the definitions (resolve macro/list refs, exceptions, ...) - m_last_compile_output = std::make_unique(); - rule_loader::compiler().compile(cfg, m_rule_collector, *m_last_compile_output.get()); + m_last_compile_output = m_rule_compiler->new_compile_output(); + m_rule_compiler->compile(cfg, *(m_rule_collector.get()), *m_last_compile_output.get()); // clear the rules known by the engine and each ruleset m_rules.clear(); @@ -187,7 +218,7 @@ std::unique_ptr falco_engine::load_rules(const std::string &rules_c continue; } - auto info = m_rule_collector.rules().at(rule.name); + auto info = m_rule_collector->rules().at(rule.name); if (!info) { // this is just defensive, it should never happen @@ -465,12 +496,12 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector< if(!rule) { // Store required engine version - auto required_engine_version = m_rule_collector.required_engine_version(); + auto required_engine_version = m_rule_collector->required_engine_version(); output["required_engine_version"] = required_engine_version.version.as_string(); // Store required plugin versions nlohmann::json plugin_versions = nlohmann::json::array(); - auto required_plugin_versions = m_rule_collector.required_plugin_versions(); + auto required_plugin_versions = m_rule_collector->required_plugin_versions(); for(const auto& req : required_plugin_versions) { nlohmann::json r; @@ -495,7 +526,7 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector< nlohmann::json rules_array = nlohmann::json::array(); for(const auto& r : m_last_compile_output->rules) { - auto info = m_rule_collector.rules().at(r.name); + auto info = m_rule_collector->rules().at(r.name); nlohmann::json rule; get_json_details(rule, r, *info, plugins); rules_array.push_back(std::move(rule)); @@ -506,7 +537,7 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector< nlohmann::json macros_array = nlohmann::json::array(); for(const auto &m : m_last_compile_output->macros) { - auto info = m_rule_collector.macros().at(m.name); + auto info = m_rule_collector->macros().at(m.name); nlohmann::json macro; get_json_details(macro, m, *info, plugins); macros_array.push_back(std::move(macro)); @@ -517,7 +548,7 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector< nlohmann::json lists_array = nlohmann::json::array(); for(const auto &l : m_last_compile_output->lists) { - auto info = m_rule_collector.lists().at(l.name); + auto info = m_rule_collector->lists().at(l.name); nlohmann::json list; get_json_details(list, l, *info, plugins); lists_array.push_back(std::move(list)); @@ -527,7 +558,7 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector< else { // build json information for just the specified rule - auto ri = m_rule_collector.rules().at(*rule); + auto ri = m_rule_collector->rules().at(*rule); if(ri == nullptr || ri->unknown_source) { throw falco_exception("Rule \"" + *rule + "\" is not loaded"); @@ -571,12 +602,12 @@ void falco_engine::get_json_details( filter_details details; filter_details compiled_details; nlohmann::json json_details; - for(const auto &m : m_rule_collector.macros()) + for(const auto &m : m_rule_collector->macros()) { details.known_macros.insert(m.name); compiled_details.known_macros.insert(m.name); } - for(const auto &l : m_rule_collector.lists()) + for(const auto &l : m_rule_collector->lists()) { details.known_lists.insert(l.name); compiled_details.known_lists.insert(l.name); @@ -672,12 +703,12 @@ void falco_engine::get_json_details( filter_details details; filter_details compiled_details; nlohmann::json json_details; - for(const auto &m : m_rule_collector.macros()) + for(const auto &m : m_rule_collector->macros()) { details.known_macros.insert(m.name); compiled_details.known_macros.insert(m.name); } - for(const auto &l : m_rule_collector.lists()) + for(const auto &l : m_rule_collector->lists()) { details.known_lists.insert(l.name); compiled_details.known_lists.insert(l.name); @@ -958,7 +989,7 @@ bool falco_engine::check_plugin_requirements( std::string& err) const { err = ""; - for (const auto &alternatives : m_rule_collector.required_plugin_versions()) + for(const auto &alternatives : m_rule_collector->required_plugin_versions()) { if (!check_plugin_requirement_alternatives(plugins, alternatives, err)) { diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 6d88be9afe8..12a09f514fc 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -33,14 +33,14 @@ limitations under the License. #include "gen_filter.h" #include "filter_ruleset.h" #include "rule_loader.h" +#include "rule_loader_reader.h" #include "rule_loader_collector.h" +#include "rule_loader_compiler.h" #include "stats_manager.h" #include "falco_common.h" #include "falco_source.h" #include "falco_load_result.h" #include "filter_details_resolver.h" -#include "rule_loader_reader.h" -#include "rule_loader_compiler.h" // // This class acts as the primary interface between a program and the @@ -73,6 +73,17 @@ class falco_engine // If source is non-empty, only fields for the provided source are printed. void list_fields(std::string &source, bool verbose, bool names_only, bool markdown) const; + // Provide an alternate rule reader, collector, and compiler + // to compile any rules provided via load_rules* + void set_rule_reader(std::shared_ptr reader); + std::shared_ptr get_rule_reader(); + + void set_rule_collector(std::shared_ptr collector); + std::shared_ptr get_rule_collector(); + + void set_rule_compiler(std::shared_ptr compiler); + std::shared_ptr get_rule_compiler(); + // // Load rules and returns a result object. // @@ -395,8 +406,10 @@ class falco_engine const std::unordered_set& fields, const std::vector>& plugins) const; - rule_loader::collector m_rule_collector; indexed_vector m_rules; + std::shared_ptr m_rule_reader; + std::shared_ptr m_rule_collector; + std::shared_ptr m_rule_compiler; stats_manager m_rule_stats_manager; uint16_t m_next_ruleset_id; From 929e5266786fc7a2277fd55ef538fb12ad601e9a Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 12:52:13 -0800 Subject: [PATCH 2/6] Provide the entire compile output to ruleset vs individual add()s In order to support external rules loaders that may extend the falco rules format with new top level objects, move away from providing individual filter objects to the filter_ruleset via calls to add(). Instead, pass the entire compile output returned by the compiler to the ruleset using a new method add_compile_output(). Custom users can then cast back the compile output to the appropriate derived class for use in the ruleset. Move the declaration of the compile output to a standalone class so it can be used by rulesets without including the entire rules loader header files, and add a new factory method new_compile_output() to the compiler so it can create a derived class if necessary. This change is backwards-compatible with existing rulesets, as the default implementation of add_compile_output() simply iterates over rules and calls add() for each rule. This change also speeds up rule loading. Previously, each rule condition was compiled twice: 1. First, in the compiler, to see if it was valid. 2. Second, in the falco engine before providing each rule to the ruleset. Add the compiled filter to the falco_rule object instead of throwing it away in the compiler. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine.cpp | 11 +++--- userspace/engine/falco_engine.h | 2 +- userspace/engine/falco_rule.h | 1 + userspace/engine/filter_ruleset.h | 30 ++++++++++++++ userspace/engine/rule_loader_compile_output.h | 39 +++++++++++++++++++ userspace/engine/rule_loader_compiler.cpp | 5 +++ userspace/engine/rule_loader_compiler.h | 22 +++-------- 7 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 userspace/engine/rule_loader_compile_output.h diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index d4fd6178108..e02711b4c54 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -205,8 +205,13 @@ std::unique_ptr falco_engine::load_rules(const std::string &rules_c // clear the rules known by the engine and each ruleset m_rules.clear(); for (auto &src : m_sources) + + // add rules to each ruleset { src.ruleset = src.ruleset_factory->new_ruleset(); + src.ruleset->add_compile_output(*(m_last_compile_output.get()), + m_min_priority, + src.name); } // add rules to the engine and the rulesets @@ -225,15 +230,9 @@ std::unique_ptr falco_engine::load_rules(const std::string &rules_c throw falco_exception("can't find internal rule info at name: " + name); } - // the rule is ok, we can add it to the engine and the rulesets - // note: the compiler should guarantee that the rule's condition - // is a valid sinsp filter auto source = find_source(rule.source); - std::shared_ptr filter( - sinsp_filter_compiler(source->filter_factory, rule.condition.get()).compile()); auto rule_id = m_rules.insert(rule, rule.name); m_rules.at(rule_id)->id = rule_id; - source->ruleset->add(rule, filter, rule.condition); // By default rules are enabled/disabled for the default ruleset if(info->enabled) diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 12a09f514fc..cc9487aa287 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -416,7 +416,7 @@ class falco_engine std::map m_known_rulesets; falco_common::priority_type m_min_priority; - std::unique_ptr m_last_compile_output; + std::unique_ptr m_last_compile_output; // // Here's how the sampling ratio and multiplier influence diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index d2028bfe9d3..1e2bbdcb41d 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -83,4 +83,5 @@ struct falco_rule std::set exception_fields; falco_common::priority_type priority; std::shared_ptr condition; + std::shared_ptr filter; }; diff --git a/userspace/engine/filter_ruleset.h b/userspace/engine/filter_ruleset.h index e84af57f285..34f9be7a065 100644 --- a/userspace/engine/filter_ruleset.h +++ b/userspace/engine/filter_ruleset.h @@ -18,6 +18,7 @@ limitations under the License. #pragma once #include "falco_rule.h" +#include "rule_loader_compile_output.h" #include #include #include @@ -50,6 +51,35 @@ class filter_ruleset std::shared_ptr filter, std::shared_ptr condition) = 0; + /*! + \brief Adds all rules contained in the provided + rule_loader::compile_output struct. Only + those rules with the provided source and those rules + with priority >= min_priority should be added. The + intent is that this replaces add(). However, we retain + add() for backwards compatibility. Any rules added via + add() are also added to this ruleset. The default + implementation iterates over rules and calls add(), + but can be overridden. + \param rule The compile output. + \param min_priority Only add rules with priority above this priority. + \param source Only add rules with source equal to this source. + */ + virtual void add_compile_output( + const rule_loader::compile_output& compile_output, + falco_common::priority_type min_priority, + const std::string& source) + { + for (const auto& rule : compile_output.rules) + { + if(rule.priority <= min_priority && + rule.source == source) + { + add(rule, rule.filter, rule.condition); + } + } + }; + /*! \brief Erases the internal state. All rules are disabled in each ruleset, and all the rules defined with add() are removed. diff --git a/userspace/engine/rule_loader_compile_output.h b/userspace/engine/rule_loader_compile_output.h new file mode 100644 index 00000000000..9a2873d2c0c --- /dev/null +++ b/userspace/engine/rule_loader_compile_output.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2023 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include "indexed_vector.h" +#include "falco_rule.h" + +namespace rule_loader +{ + struct compile_output + { + compile_output() = default; + virtual ~compile_output() = default; + compile_output(compile_output&&) = default; + compile_output& operator = (compile_output&&) = default; + compile_output(const compile_output&) = default; + compile_output& operator = (const compile_output&) = default; + + indexed_vector lists; + indexed_vector macros; + indexed_vector rules; + }; +}; + diff --git a/userspace/engine/rule_loader_compiler.cpp b/userspace/engine/rule_loader_compiler.cpp index b45c14d861a..27ab227152a 100644 --- a/userspace/engine/rule_loader_compiler.cpp +++ b/userspace/engine/rule_loader_compiler.cpp @@ -529,6 +529,11 @@ void rule_loader::compiler::compile_rule_infos( } } +std::unique_ptr rule_loader::compiler::new_compile_output() +{ + return std::make_unique(); +} + void rule_loader::compiler::compile( configuration& cfg, const collector& col, diff --git a/userspace/engine/rule_loader_compiler.h b/userspace/engine/rule_loader_compiler.h index bc7b53d9e70..533a4013765 100644 --- a/userspace/engine/rule_loader_compiler.h +++ b/userspace/engine/rule_loader_compiler.h @@ -18,6 +18,7 @@ limitations under the License. #pragma once #include "rule_loader.h" +#include "rule_loader_compile_output.h" #include "rule_loader_collector.h" #include "indexed_vector.h" #include "falco_rule.h" @@ -31,23 +32,6 @@ namespace rule_loader class compiler { public: - /*! - \brief The output of a compilation. - */ - struct compile_output - { - compile_output() = default; - virtual ~compile_output() = default; - compile_output(compile_output&&) = default; - compile_output& operator = (compile_output&&) = default; - compile_output(const compile_output&) = default; - compile_output& operator = (const compile_output&) = default; - - indexed_vector lists; - indexed_vector macros; - indexed_vector rules; - }; - compiler() = default; virtual ~compiler() = default; compiler(compiler&&) = default; @@ -55,6 +39,10 @@ class compiler compiler(const compiler&) = default; compiler& operator = (const compiler&) = default; + // Return a new result object, suitable for passing to + // compile(). + virtual std::unique_ptr new_compile_output(); + /*! \brief Compiles a list of falco rules */ From e559bad45810e88705fb46b376c641e9aaf6f400 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 13:02:10 -0800 Subject: [PATCH 3/6] Add addl support for rules reader/compiler subclasses To support subclasses that may extend the falco rules format, add additional error/warning/item types for an extension item. When subclasses report errors and warnings, they can use these codes/item types in context objects and still provide an exact line/column context. Also make some previously static functions in rules reader protected methods so they can be used in sub-classes. Signed-off-by: Mark Stemm --- userspace/engine/falco_load_result.h | 6 ++++-- userspace/engine/rule_loader.cpp | 3 ++- userspace/engine/rule_loader.h | 3 ++- userspace/engine/rule_loader_reader.cpp | 14 ++++++++++---- userspace/engine/rule_loader_reader.h | 13 +++++++++++++ 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/userspace/engine/falco_load_result.h b/userspace/engine/falco_load_result.h index 2b4b43c8343..a3641e59f38 100644 --- a/userspace/engine/falco_load_result.h +++ b/userspace/engine/falco_load_result.h @@ -34,7 +34,8 @@ class load_result { LOAD_ERR_YAML_VALIDATE, LOAD_ERR_COMPILE_CONDITION, LOAD_ERR_COMPILE_OUTPUT, - LOAD_ERR_VALIDATE + LOAD_ERR_VALIDATE, + LOAD_ERR_EXTENSION }; // The error code as a string @@ -55,7 +56,8 @@ class load_result { LOAD_UNUSED_MACRO, LOAD_UNUSED_LIST, LOAD_UNKNOWN_ITEM, - LOAD_DEPRECATED_ITEM + LOAD_DEPRECATED_ITEM, + LOAD_WARNING_EXTENSION }; virtual ~load_result() = default; diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index 9ea32a90540..a801ff2b0c7 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -42,7 +42,8 @@ static const std::string item_type_strings[] = { "rule output", "rule output expression", "rule priority", - "overrides" + "overrides", + "extension item" }; const std::string& rule_loader::context::item_type_as_string(enum item_type it) diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 2401b1492d5..53b36611db2 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -58,7 +58,8 @@ namespace rule_loader RULE_OUTPUT, RULE_OUTPUT_EXPRESSION, RULE_PRIORITY, - OVERRIDE + OVERRIDE, + EXTENSION_ITEM }; static const std::string& item_type_as_string(enum item_type it); diff --git a/userspace/engine/rule_loader_reader.cpp b/userspace/engine/rule_loader_reader.cpp index abf23143707..168d219315e 100644 --- a/userspace/engine/rule_loader_reader.cpp +++ b/userspace/engine/rule_loader_reader.cpp @@ -56,21 +56,27 @@ static void decode_val_generic(const YAML::Node& item, const char *key, std::opt } template -static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx) +void rule_loader::reader::decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx) { bool optional = false; decode_val_generic(item, key, out, ctx, optional); } +template void rule_loader::reader::decode_val(const YAML::Node& item, const char *key, std::string& out, const rule_loader::context& ctx); + template -static void decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx) +void rule_loader::reader::decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx) { bool optional = true; decode_val_generic(item, key, out, ctx, optional); } +template void rule_loader::reader::decode_optional_val(const YAML::Node& item, const char *key, std::string& out, const rule_loader::context& ctx); + +template void rule_loader::reader::decode_optional_val(const YAML::Node& item, const char *key, bool& out, const rule_loader::context& ctx); + // Don't call this directly, call decode_items/decode_tags instead. template static void decode_seq(const YAML::Node& item, const char *key, @@ -289,7 +295,7 @@ static void read_rule_exceptions( rule_loader::context tmp(ex, rule_loader::context::EXCEPTION, "", exes_ctx); THROW(!ex.IsMap(), "Rule exception must be a mapping", tmp); - decode_val(ex, "name", name, tmp); + rule_loader::reader::decode_val(ex, "name", name, tmp); // Now use a real context including the exception name. rule_loader::context ex_ctx(ex, rule_loader::context::EXCEPTION, name, parent); @@ -346,7 +352,7 @@ inline static bool check_update_expected(std::set& expected_keys, c return true; } -static void read_item( +void rule_loader::reader::read_item( rule_loader::configuration& cfg, rule_loader::collector& collector, const YAML::Node& item, diff --git a/userspace/engine/rule_loader_reader.h b/userspace/engine/rule_loader_reader.h index 378556c1dbf..484f267c29d 100644 --- a/userspace/engine/rule_loader_reader.h +++ b/userspace/engine/rule_loader_reader.h @@ -57,6 +57,19 @@ class reader + std::to_string(minor) + "." + std::to_string(FALCO_ENGINE_VERSION_PATCH)); } + + template + static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx); + + template + static void decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx); + +protected: + + virtual void read_item(rule_loader::configuration& cfg, + rule_loader::collector& collector, + const YAML::Node& item, + const rule_loader::context& parent); }; }; // namespace rule_loader From 6d03bbd439f0c71943d174dc2a62d6cba94d1988 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 13:03:35 -0800 Subject: [PATCH 4/6] Add ability for rulesets to access falco engine state Some rulesets may need information which is held by the falco_engine that created this ruleset. So define a set of functions in a struct and have setters/getters for those functions in the base class. Derived classes can use the struct's functions to obtain the falco engine information. The only function so far is to obtain the filter_ruleset for a given event source. Signed-off-by: Mark Stemm --- userspace/engine/CMakeLists.txt | 1 + userspace/engine/falco_engine.cpp | 31 +++++++++++++++++++++++++++-- userspace/engine/falco_engine.h | 8 ++++++++ userspace/engine/filter_ruleset.cpp | 28 ++++++++++++++++++++++++++ userspace/engine/filter_ruleset.h | 17 ++++++++++++++++ 5 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 userspace/engine/filter_ruleset.cpp diff --git a/userspace/engine/CMakeLists.txt b/userspace/engine/CMakeLists.txt index 78f681cef38..b3d6f086836 100644 --- a/userspace/engine/CMakeLists.txt +++ b/userspace/engine/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(falco_engine STATIC falco_engine.cpp falco_load_result.cpp falco_utils.cpp + filter_ruleset.cpp evttype_index_ruleset.cpp formats.cpp filter_details_resolver.cpp diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index e02711b4c54..cc13461ffe6 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -66,6 +66,8 @@ falco_engine::falco_engine(bool seed_rng) } m_default_ruleset_id = find_ruleset_id(s_default_ruleset); + + fill_engine_state_funcs(m_engine_state); } falco_engine::~falco_engine() @@ -208,7 +210,7 @@ std::unique_ptr falco_engine::load_rules(const std::string &rules_c // add rules to each ruleset { - src.ruleset = src.ruleset_factory->new_ruleset(); + src.ruleset = create_ruleset(src.ruleset_factory); src.ruleset->add_compile_output(*(m_last_compile_output.get()), m_min_priority, src.name); @@ -467,7 +469,7 @@ std::size_t falco_engine::add_source(const std::string &source, src.filter_factory = filter_factory; src.formatter_factory = formatter_factory; src.ruleset_factory = ruleset_factory; - src.ruleset = ruleset_factory->new_ruleset(); + src.ruleset = create_ruleset(src.ruleset_factory); return m_sources.insert(src, source); } @@ -1007,6 +1009,31 @@ bool falco_engine::check_plugin_requirements( return true; } +std::shared_ptr falco_engine::create_ruleset(std::shared_ptr &ruleset_factory) +{ + auto ret = ruleset_factory->new_ruleset(); + + ret->set_engine_state(m_engine_state); + + return ret; +} + +void falco_engine::fill_engine_state_funcs(filter_ruleset::engine_state_funcs &engine_state) +{ + engine_state.get_ruleset = [this](const std::string &source_name, std::shared_ptr &ruleset) -> bool + { + falco_source *src = m_sources.at(source_name); + if(src == nullptr) + { + return false; + } + + ruleset = src->ruleset; + + return true; + }; +}; + void falco_engine::complete_rule_loading() const { for (const auto &src : m_sources) diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index cc9487aa287..07b0f4ba43c 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -321,6 +321,14 @@ class falco_engine std::string& err) const; private: + // Create a ruleset using the provided factory and set the + // engine state funcs for it. + std::shared_ptr create_ruleset(std::shared_ptr& ruleset_factory); + + // Functions to retrieve state from this engine + void fill_engine_state_funcs(filter_ruleset::engine_state_funcs& engine_state); + + filter_ruleset::engine_state_funcs m_engine_state; // Throws falco_exception if the file can not be read void read_file(const std::string& filename, std::string& contents); diff --git a/userspace/engine/filter_ruleset.cpp b/userspace/engine/filter_ruleset.cpp new file mode 100644 index 00000000000..f99c2434063 --- /dev/null +++ b/userspace/engine/filter_ruleset.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2024 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "filter_ruleset.h" + +void filter_ruleset::set_engine_state(filter_ruleset::engine_state_funcs& engine_state) +{ + m_engine_state = engine_state; +} + +filter_ruleset::engine_state_funcs& filter_ruleset::get_engine_state() +{ + return m_engine_state; +} diff --git a/userspace/engine/filter_ruleset.h b/userspace/engine/filter_ruleset.h index 34f9be7a065..1214b983f45 100644 --- a/userspace/engine/filter_ruleset.h +++ b/userspace/engine/filter_ruleset.h @@ -21,6 +21,7 @@ limitations under the License. #include "rule_loader_compile_output.h" #include #include +#include #include #include #include @@ -29,11 +30,24 @@ limitations under the License. \brief Manages a set of rulesets. A ruleset is a set of enabled rules that is able to process events and find matches for those rules. */ + class filter_ruleset { public: + // A set of functions that can be used to retrieve state from + // the falco engine that created this ruleset. + struct engine_state_funcs + { + using ruleset_retriever_func_t = std::function &ruleset)>; + + ruleset_retriever_func_t get_ruleset; + }; + virtual ~filter_ruleset() = default; + void set_engine_state(engine_state_funcs &engine_state); + engine_state_funcs &get_engine_state(); + /*! \brief Adds a rule and its filtering filter + condition inside the manager. This method only adds the rule inside the internal collection, @@ -211,6 +225,9 @@ class filter_ruleset virtual void disable_tags( const std::set &tags, uint16_t ruleset_id) = 0; + +private: + engine_state_funcs m_engine_state; }; /*! From 02b6d599ae2b102686d054a15e7a47b3edafa8a3 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 13:06:03 -0800 Subject: [PATCH 5/6] Make compile_condition() a protected method for use in subclasses Move the part of compile_rule_infos that actually compiled a condition string into a sinsp_filter into a standalone method compile_condition(). That way it can be used by classes that derive from rule_loader::compiler() and want to compile condition strings. This implementation also saves the compiled filter as a part of the falco_rule object so it does not need to be compiled again wihin the falco engine after rules loading. Signed-off-by: Mark Stemm --- userspace/engine/rule_loader_compiler.cpp | 119 ++++++++++++++-------- userspace/engine/rule_loader_compiler.h | 21 ++++ 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/userspace/engine/rule_loader_compiler.cpp b/userspace/engine/rule_loader_compiler.cpp index 27ab227152a..67ef7dea0aa 100644 --- a/userspace/engine/rule_loader_compiler.cpp +++ b/userspace/engine/rule_loader_compiler.cpp @@ -405,17 +405,72 @@ static bool err_is_unknown_type_or_field(const std::string& err) || err.find("unknown event type") != std::string::npos; } -void rule_loader::compiler::compile_rule_infos( - configuration& cfg, - const collector& col, - indexed_vector& lists, - indexed_vector& macros, - indexed_vector& out) const +bool rule_loader::compiler::compile_condition( + configuration& cfg, + indexed_vector& lists, + const indexed_vector& macros, + const std::string& condition, + std::shared_ptr filter_factory, + rule_loader::context cond_ctx, + rule_loader::context parent_ctx, + bool allow_unknown_fields, + indexed_vector& macros_out, + std::shared_ptr& ast_out, + std::shared_ptr& filter_out) const { - std::string err, condition; std::set warn_codes; filter_warning_resolver warn_resolver; - for (const auto &r : col.rules()) + ast_out = parse_condition(condition, lists, cond_ctx); + resolve_macros(macros, macros_out, ast_out, condition, MAX_VISIBILITY, parent_ctx); + + // check for warnings in the filtering condition + if(warn_resolver.run(ast_out.get(), warn_codes)) + { + for(const auto& w : warn_codes) + { + cfg.res->add_warning(w, "", parent_ctx); + } + } + + // validate the rule's condition: we compile it into a sinsp filter + // on-the-fly and we throw an exception with details on failure + sinsp_filter_compiler compiler(filter_factory, ast_out.get()); + try + { + filter_out.reset(compiler.compile()); + } + catch(const sinsp_exception& e) + { + // skip the rule silently if skip_if_unknown_filter is true and + // we encountered some specific kind of errors + std::string err = e.what(); + if(err_is_unknown_type_or_field(err) && allow_unknown_fields) + { + cfg.res->add_warning( + falco::load_result::load_result::LOAD_UNKNOWN_FILTER, + err, + cond_ctx); + return false; + } + rule_loader::context ctx(compiler.get_pos(), condition, cond_ctx); + throw rule_loader::rule_load_exception( + falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION, + err, + ctx); + } + + return true; +} + +void rule_loader::compiler::compile_rule_infos( + configuration& cfg, + const collector& col, + indexed_vector& lists, + indexed_vector& macros, + indexed_vector& out) const +{ + std::string err, condition; + for(const auto& r : col.rules()) { // skip the rule if it has an unknown source if (r.unknown_source) @@ -439,18 +494,6 @@ void rule_loader::compiler::compile_rule_infos( build_rule_exception_infos( r.exceptions, rule.exception_fields, condition); } - rule.condition = parse_condition(condition, lists, r.cond_ctx); - resolve_macros(col.macros(), macros, rule.condition, condition, MAX_VISIBILITY, r.ctx); - - // check for warnings in the filtering condition - warn_codes.clear(); - if (warn_resolver.run(rule.condition.get(), warn_codes)) - { - for (const auto &w : warn_codes) - { - cfg.res->add_warning(w, "", r.ctx); - } - } // build rule output message rule.output = r.output; @@ -478,31 +521,19 @@ void rule_loader::compiler::compile_rule_infos( r.output_ctx); } - // validate the rule's condition: we compile it into a sinsp filter - // on-the-fly and we throw an exception with details on failure - sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, rule.condition.get()); - try - { - std::shared_ptr sfPtr(compiler.compile()); - } - catch (const sinsp_exception& e) + if (!compile_condition(cfg, + lists, + col.macros(), + condition, + cfg.sources.at(r.source)->filter_factory, + r.cond_ctx, + r.ctx, + r.skip_if_unknown_filter, + macros, + rule.condition, + rule.filter)) { - // skip the rule silently if skip_if_unknown_filter is true and - // we encountered some specific kind of errors - std::string err = e.what(); - if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter) - { - cfg.res->add_warning( - falco::load_result::load_result::LOAD_UNKNOWN_FILTER, - err, - r.cond_ctx); - continue; - } - rule_loader::context ctx(compiler.get_pos(), condition, r.cond_ctx); - throw rule_loader::rule_load_exception( - falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION, - err, - ctx); + continue; } // populate set of event types and emit an special warning diff --git a/userspace/engine/rule_loader_compiler.h b/userspace/engine/rule_loader_compiler.h index 533a4013765..47e8ad141f4 100644 --- a/userspace/engine/rule_loader_compiler.h +++ b/userspace/engine/rule_loader_compiler.h @@ -50,6 +50,27 @@ class compiler configuration& cfg, const collector& col, compile_output& out) const; +protected: + /*! + \brief Compile a single condition expression, + including expanding macro and list references. + + returns true if the condition could be compiled, and sets + ast_out/filter_out with the compiled filter + ast. Returns false if + the condition could not be compiled and should be skipped. + */ + bool compile_condition( + configuration& cfg, + indexed_vector& lists, + const indexed_vector& macros, + const std::string& condition, + std::shared_ptr filter_factory, + rule_loader::context cond_ctx, + rule_loader::context parent_ctx, + bool allow_unknown_fields, + indexed_vector& macros_out, + std::shared_ptr& ast_out, + std::shared_ptr& filter_out) const; private: void compile_list_infos( From e7e68b5f77d4c6ffce7203a7139e1fd9df62f45f Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 11 Jan 2024 13:06:31 -0800 Subject: [PATCH 6/6] Add unit test for alternate rules loader Add a unit test for providing an alternate rules loader that also demonstrates how users can define sub-classes that may want to extend the falco rules syntax. This test creates a test rules reader/collector/compiler that supports top-level objects "test_object". The reader reads them and saves them in the collector. The compiler iterates over all test_objects and puts the property values into a single set. Signed-off-by: Mark Stemm --- unit_tests/engine/test_alt_rule_loader.cpp | 368 +++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 unit_tests/engine/test_alt_rule_loader.cpp diff --git a/unit_tests/engine/test_alt_rule_loader.cpp b/unit_tests/engine/test_alt_rule_loader.cpp new file mode 100644 index 00000000000..196079a7de4 --- /dev/null +++ b/unit_tests/engine/test_alt_rule_loader.cpp @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2023 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include + +#include + +#include +#include +#include +#include + +#include +#include "indexed_vector.h" +#include "evttype_index_ruleset.h" + +#include "rule_loader_reader.h" +#include "rule_loader_collector.h" +#include "rule_loader_compiler.h" + +namespace +{ + +struct test_object_info +{ + std::string name; + std::string property; +}; + +struct test_compile_output : public rule_loader::compile_output +{ + test_compile_output() = default; + ~test_compile_output() = default; + + std::set defined_test_properties; +}; + +class test_compiler : public rule_loader::compiler +{ +public: + test_compiler() = default; + virtual ~test_compiler() = default; + + virtual std::unique_ptr new_compile_output() + { + return std::unique_ptr(new test_compile_output()); + }; + + void compile( + rule_loader::configuration& cfg, + const rule_loader::collector& col, + rule_loader::compile_output& out) const override; +}; + +class test_collector : public rule_loader::collector +{ +public: + test_collector() = default; + virtual ~test_collector() = default; + + indexed_vector test_object_infos; +}; + +class test_reader : public rule_loader::reader +{ +public: + test_reader() = default; + virtual ~test_reader() = default; + +protected: + rule_loader::context create_context(const YAML::Node& item, + const rule_loader::context& parent) + { + return rule_loader::context(item, + rule_loader::context::EXTENSION_ITEM, + "test object", + parent); + }; + + void read_item(rule_loader::configuration& cfg, + rule_loader::collector& collector, + const YAML::Node& item, + const rule_loader::context& parent) override + { + test_collector& test_col = + dynamic_cast(collector); + + if(item["test_object"].IsDefined()) + { + rule_loader::context tmp = create_context(item, parent); + test_object_info obj; + std::string name; + std::string property; + + decode_val(item, "test_object", name, tmp); + decode_val(item, "property", property, tmp); + + obj.name = name; + obj.property = property; + + test_col.test_object_infos.insert(obj, obj.name); + } + else + { + rule_loader::reader::read_item(cfg, collector, item, parent); + } + }; +}; + +class test_ruleset : public evttype_index_ruleset +{ +public: + test_ruleset(std::shared_ptr factory): + evttype_index_ruleset(factory){}; + virtual ~test_ruleset() = default; + + void add_compile_output( + const rule_loader::compile_output& compile_output, + falco_common::priority_type min_priority, + const std::string& source) + { + + evttype_index_ruleset::add_compile_output(compile_output, + min_priority, + source); + + std::shared_ptr ruleset; + get_engine_state().get_ruleset(source, ruleset); + EXPECT_EQ(this, ruleset.get()); + + const test_compile_output& test_output = + dynamic_cast(compile_output); + + defined_properties = test_output.defined_test_properties; + }; + + std::set defined_properties; +}; + +class test_ruleset_factory : public filter_ruleset_factory +{ +public: + test_ruleset_factory(std::shared_ptr factory): + m_filter_factory(factory) + { + } + + virtual ~test_ruleset_factory() = default; + + inline std::shared_ptr new_ruleset() override + { + std::shared_ptr ret(new test_ruleset(m_filter_factory)); + return ret; + } + + std::shared_ptr m_filter_factory; +}; +}; // namespace + +void test_compiler::compile( + rule_loader::configuration& cfg, + const rule_loader::collector& col, + rule_loader::compile_output& out) const +{ + rule_loader::compiler::compile(cfg, col, out); + + const test_collector& test_col = + dynamic_cast(col); + + test_compile_output& test_output = + dynamic_cast(out); + + for(auto& test_obj : test_col.test_object_infos) + { + test_output.defined_test_properties.insert(test_obj.property); + } +} + +static std::string content = R"END( + +- test_object: test + property: my-value + +- test_object: test2 + property: other-value + +- list: shell_binaries + items: [sh, bash] + +- macro: spawned_process + condition: evt.type=execve and proc.name in (shell_binaries) + +- rule: test info rule + desc: A test info rule + condition: spawned_process + output: A test info rule matched (evt.type=%evt.type proc.name=%proc.name) + priority: INFO + source: syscall + tags: [process] + +- rule: test k8s_audit rule + desc: A k8s audit test rule + condition: ka.target.resource=deployments + output: A k8s audit rule matched (ka.verb=%ka.verb resource=%ka.target.resource) + priority: INFO + source: k8s_audit + tags: [process] + +- rule: test debug rule + desc: A test debug rule + condition: spawned_process and proc.name="bash" + output: A test debug rule matched (evt.type=%evt.type proc.name=%proc.name) + priority: DEBUG + source: syscall + tags: [process] +)END"; + +static std::string syscall_source_name = "syscall"; + +static std::shared_ptr create_configuration(sinsp& inspector, + sinsp_filter_check_list& filterchecks, + indexed_vector& sources) +{ + auto filter_factory = std::shared_ptr( + new sinsp_filter_factory(&inspector, filterchecks)); + auto formatter_factory = std::shared_ptr( + new sinsp_evt_formatter_factory(&inspector, filterchecks)); + auto ruleset_factory = std::shared_ptr( + new evttype_index_ruleset_factory(filter_factory)); + + falco_source syscall_source; + syscall_source.name = syscall_source_name; + syscall_source.ruleset = ruleset_factory->new_ruleset(); + syscall_source.ruleset_factory = ruleset_factory; + syscall_source.filter_factory = filter_factory; + syscall_source.formatter_factory = formatter_factory; + + sources.insert(syscall_source, syscall_source_name); + + std::shared_ptr configuration = + std::make_shared(content, + sources, + "test configuration"); + + return configuration; +} + +static void load_rules(sinsp& inspector, + sinsp_filter_check_list& filterchecks, + std::unique_ptr& compile_output, + indexed_vector& sources) +{ + std::shared_ptr cfg = create_configuration(inspector, filterchecks, sources); + + rule_loader::reader reader; + rule_loader::collector collector; + rule_loader::compiler compiler; + + EXPECT_TRUE(reader.read(*(cfg.get()), collector)); + + compile_output = compiler.new_compile_output(); + + compiler.compile(*(cfg.get()), collector, *(compile_output.get())); +} + +TEST(engine_loader_alt_loader, load_rules) +{ + sinsp inspector; + sinsp_filter_check_list filterchecks; + std::unique_ptr compile_output; + indexed_vector sources; + + load_rules(inspector, filterchecks, compile_output, sources); + + // Note that the k8s_audit rule will be skipped as load_rules + // only adds a syscall source. + EXPECT_EQ(compile_output->lists.size(), 1); + EXPECT_TRUE(compile_output->lists.at("shell_binaries") != nullptr); + + EXPECT_EQ(compile_output->macros.size(), 1); + EXPECT_TRUE(compile_output->macros.at("spawned_process") != nullptr); + + EXPECT_EQ(compile_output->rules.size(), 2); + EXPECT_TRUE(compile_output->rules.at("test info rule") != nullptr); + EXPECT_TRUE(compile_output->rules.at("test debug rule") != nullptr); +} + +TEST(engine_loader_alt_loader, pass_compile_output_to_ruleset) +{ + sinsp inspector; + sinsp_filter_check_list filterchecks; + std::unique_ptr compile_output; + indexed_vector sources; + + load_rules(inspector, filterchecks, compile_output, sources); + + std::shared_ptr ruleset = sources.at(syscall_source_name)->ruleset; + + ruleset->add_compile_output(*(compile_output.get()), + falco_common::PRIORITY_INFORMATIONAL, + syscall_source_name); + + // Enable all rules for a ruleset id. Because the compile + // output contained one rule with priority >= INFO, that rule + // should be enabled. + bool match_exact = true; + uint16_t ruleset_id = 0; + ruleset->enable("", match_exact, ruleset_id); + + EXPECT_EQ(ruleset->enabled_count(ruleset_id), 1); +} + +TEST(engine_loader_alt_loader, falco_engine_alternate_loader) +{ + falco_engine engine; + sinsp inspector; + sinsp_filter_check_list filterchecks; + + auto filter_factory = std::shared_ptr( + new sinsp_filter_factory(&inspector, filterchecks)); + auto formatter_factory = std::shared_ptr( + new sinsp_evt_formatter_factory(&inspector, filterchecks)); + auto ruleset_factory = std::shared_ptr( + new test_ruleset_factory(filter_factory)); + + engine.add_source(syscall_source_name, + filter_factory, + formatter_factory, + ruleset_factory); + + std::shared_ptr reader(new test_reader()); + test_collector* test_col = new test_collector(); + std::shared_ptr collector(test_col); + std::shared_ptr compiler(new test_compiler()); + + engine.set_rule_reader(reader); + engine.set_rule_collector(collector); + engine.set_rule_compiler(compiler); + + EXPECT_EQ(reader, engine.get_rule_reader()); + EXPECT_EQ(collector, engine.get_rule_collector()); + EXPECT_EQ(compiler, engine.get_rule_compiler()); + + engine.load_rules(content, "test_rules.yaml"); + + EXPECT_EQ(test_col->test_object_infos.size(), 2); + + std::shared_ptr ruleset = engine.ruleset_for_source(syscall_source_name); + std::set& defined_properties = std::dynamic_pointer_cast(ruleset)->defined_properties; + + EXPECT_TRUE(defined_properties.find("my-value") != defined_properties.end()); + EXPECT_TRUE(defined_properties.find("other-value") != defined_properties.end()); + EXPECT_TRUE(defined_properties.find("not-exists-value") == defined_properties.end()); +};