diff --git a/crates/bpf-builder/include/loop.bpf.h b/crates/bpf-builder/include/loop.bpf.h index 71220bd5..539dd02a 100644 --- a/crates/bpf-builder/include/loop.bpf.h +++ b/crates/bpf-builder/include/loop.bpf.h @@ -13,6 +13,15 @@ // Note: callback_fn must be declared as `static __always_inline` to satisfy the // verifier. For some reason, having this double call to the same non-inline // function seems to cause issues. +#ifdef NOLOOP +// On kernel <= 5.13 taking the address of a function results in a verifier +// error, even if inside a dead-code elimination branch. +#define LOOP(max_iterations, callback_fn, ctx) \ + _Pragma("unroll") for (int i = 0; i < max_iterations; i++) { \ + if (callback_fn(i, ctx) == 1) \ + break; \ + } +#else #define LOOP(max_iterations, callback_fn, ctx) \ if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(5, 17, 0)) { \ bpf_loop(max_iterations, callback_fn, ctx, 0); \ @@ -22,3 +31,4 @@ break; \ } \ } +#endif diff --git a/crates/bpf-builder/src/lib.rs b/crates/bpf-builder/src/lib.rs index 15d4be4b..5a45a163 100644 --- a/crates/bpf-builder/src/lib.rs +++ b/crates/bpf-builder/src/lib.rs @@ -1,11 +1,20 @@ use std::{env, path::PathBuf, process::Command, string::String}; -use anyhow::Context; +use anyhow::{bail, Context}; static CLANG_DEFAULT: &str = "clang"; static LLVM_STRIP: &str = "llvm-strip"; static INCLUDE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/include"); +// Given the filename of an eBPF program source code, compile it to OUT_DIR. +// We'll build two versions: +// - `probe_full.bpf.o`: will contain the full version +// - `probe_noloop.bpf.o`: will contain a version with the NOLOOP constant +// defined. This version should be loaded on kernel < 5.13, where taking +// the address of a static function would result in a verifier error. +// See +// - https://github.com/Exein-io/pulsar/issues/158 +// - https://github.com/torvalds/linux/commit/69c087ba6225b574afb6e505b72cb75242a3d844 pub fn build(probe: &str) -> Result<(), Box> { println!("cargo:rerun-if-changed={probe}"); println!("cargo:rerun-if-changed={INCLUDE_PATH}/common.bpf.h"); @@ -14,14 +23,17 @@ pub fn build(probe: &str) -> Result<(), Box> { println!("cargo:rerun-if-changed={INCLUDE_PATH}/loop.bpf.h"); let out_path = PathBuf::from(env::var("OUT_DIR")?); - let out_object = out_path.join("probe.bpf.o"); - let clang = match env::var("CLANG") { - Ok(val) => val, - Err(_) => String::from(CLANG_DEFAULT), - }; + compile(probe, out_path.join("probe_full.bpf.o"), &[]) + .context("Error compiling full version")?; + compile(probe, out_path.join("probe_noloop.bpf.o"), &["-DNOLOOP"]) + .context("Error compiling no-loop version")?; - // Compile + Ok(()) +} + +fn compile(probe: &str, out_object: PathBuf, extra_args: &[&str]) -> anyhow::Result<()> { + let clang = env::var("CLANG").unwrap_or_else(|_| String::from(CLANG_DEFAULT)); let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); let include_path = PathBuf::from(INCLUDE_PATH); let status = Command::new(clang) @@ -41,6 +53,7 @@ pub fn build(probe: &str) -> Result<(), Box> { _ => arch.clone(), } )) + .args(extra_args) .arg(probe) .arg("-o") .arg(&out_object) @@ -48,7 +61,7 @@ pub fn build(probe: &str) -> Result<(), Box> { .context("Failed to execute clang")?; if !status.success() { - Err("Failed to compile eBPF program")?; + bail!("Failed to compile eBPF program"); } // Strip debug symbols @@ -59,7 +72,7 @@ pub fn build(probe: &str) -> Result<(), Box> { .context("Failed to execute llvm-strip")?; if !status.success() { - Err("Failed strip eBPF program")?; + bail!("Failed strip eBPF program"); } Ok(()) diff --git a/crates/bpf-common/ProbeTutorial.md b/crates/bpf-common/ProbeTutorial.md index 473a9e8e..65600ea3 100644 --- a/crates/bpf-common/ProbeTutorial.md +++ b/crates/bpf-common/ProbeTutorial.md @@ -78,7 +78,7 @@ The module implementation in Rust is also relatively short. use std::fmt; use bpf_common::{ - aya::include_bytes_aligned, program::BpfContext, BpfSender, Program, + aya::program::BpfContext, BpfSender, Program, ProgramBuilder, ProgramError, }; @@ -86,10 +86,11 @@ pub async fn program( ctx: BpfContext, sender: impl BpfSender, ) -> Result { + let binary = ebpf_program!(&ctx); let program = ProgramBuilder::new( ctx, "file_created", - include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(), + binary, ) .kprobe("security_inode_create") .start() diff --git a/crates/bpf-common/src/feature_autodetect/lsm.rs b/crates/bpf-common/src/feature_autodetect/lsm.rs index f61cfe71..0d6cbc7f 100644 --- a/crates/bpf-common/src/feature_autodetect/lsm.rs +++ b/crates/bpf-common/src/feature_autodetect/lsm.rs @@ -22,7 +22,8 @@ pub fn lsm_supported() -> bool { } const PATH: &str = "/sys/kernel/security/lsm"; -static TEST_LSM_PROBE: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")); +static TEST_LSM_PROBE: &[u8] = + include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_full.bpf.o")); fn try_load() -> Result<()> { // Check if LSM enabled diff --git a/crates/bpf-common/src/program.rs b/crates/bpf-common/src/program.rs index cb196f24..80cddfa8 100644 --- a/crates/bpf-common/src/program.rs +++ b/crates/bpf-common/src/program.rs @@ -110,6 +110,39 @@ impl BpfContext { pub fn lsm_supported(&self) -> bool { self.lsm_supported } + + pub fn kernel_version(&self) -> &KernelVersion { + &self.kernel_version + } +} + +/// Return the correct version of the eBPF binary to load. +/// On kernel >= 5.13.0 we'll load 'probe_full.bpf.o' +/// On kernel < 5.13.0 we'll load 'probe_noloop.bpf.o' +/// Note: Both programs are embedded in the pulsar binary. The choice is made +/// at runtime. +#[macro_export] +macro_rules! ebpf_program { + ( $ctx: expr ) => {{ + use bpf_common::aya::include_bytes_aligned; + use bpf_common::feature_autodetect::kernel_version::KernelVersion; + + let full = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_full.bpf.o")).into(); + let no_loop = + include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_noloop.bpf.o")).into(); + if $ctx.kernel_version().as_i32() + >= (KernelVersion { + major: 5, + minor: 13, + patch: 0, + }) + .as_i32() + { + full + } else { + no_loop + } + }}; } #[derive(Error, Debug)] diff --git a/crates/modules/file-system-monitor/src/lib.rs b/crates/modules/file-system-monitor/src/lib.rs index d1db5dcc..63dabd8d 100644 --- a/crates/modules/file-system-monitor/src/lib.rs +++ b/crates/modules/file-system-monitor/src/lib.rs @@ -1,6 +1,6 @@ use bpf_common::{ - aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Program, - ProgramBuilder, ProgramError, + ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Program, ProgramBuilder, + ProgramError, }; const MODULE_NAME: &str = "file-system-monitor"; @@ -10,11 +10,8 @@ pub async fn program( sender: impl BpfSender, ) -> Result { let attach_to_lsm = ctx.lsm_supported(); - let mut builder = ProgramBuilder::new( - ctx, - MODULE_NAME, - include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(), - ); + let binary = ebpf_program!(&ctx); + let mut builder = ProgramBuilder::new(ctx, MODULE_NAME, binary); // LSM hooks provide the perfet intercept point for file system operations. // If LSM eBPF programs is not supported, we'll attach to the same kernel // functions, but using kprobes. diff --git a/crates/modules/network-monitor/src/lib.rs b/crates/modules/network-monitor/src/lib.rs index b48e73f1..d352876c 100644 --- a/crates/modules/network-monitor/src/lib.rs +++ b/crates/modules/network-monitor/src/lib.rs @@ -4,7 +4,7 @@ use std::{ }; use bpf_common::{ - aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program, + ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program, ProgramBuilder, ProgramError, }; use nix::sys::socket::{SockaddrIn, SockaddrIn6}; @@ -50,20 +50,17 @@ pub async fn program( sender: impl BpfSender, ) -> Result { let attach_to_lsm = ctx.lsm_supported(); - let mut builder = ProgramBuilder::new( - ctx, - MODULE_NAME, - include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(), - ) - .tracepoint("syscalls", "sys_exit_accept4") - .tracepoint("syscalls", "sys_exit_accept") - .tracepoint("syscalls", "sys_exit_recvmsg") - .tracepoint("syscalls", "sys_exit_recvmmsg") - .tracepoint("syscalls", "sys_enter_recvfrom") - .tracepoint("syscalls", "sys_exit_recvfrom") - .tracepoint("syscalls", "sys_exit_read") - .tracepoint("syscalls", "sys_exit_readv") - .kprobe("tcp_set_state"); + let binary = ebpf_program!(&ctx); + let mut builder = ProgramBuilder::new(ctx, MODULE_NAME, binary) + .tracepoint("syscalls", "sys_exit_accept4") + .tracepoint("syscalls", "sys_exit_accept") + .tracepoint("syscalls", "sys_exit_recvmsg") + .tracepoint("syscalls", "sys_exit_recvmmsg") + .tracepoint("syscalls", "sys_enter_recvfrom") + .tracepoint("syscalls", "sys_exit_recvfrom") + .tracepoint("syscalls", "sys_exit_read") + .tracepoint("syscalls", "sys_exit_readv") + .kprobe("tcp_set_state"); if attach_to_lsm { builder = builder .lsm("socket_bind") diff --git a/crates/modules/process-monitor/src/lib.rs b/crates/modules/process-monitor/src/lib.rs index 92bb40df..5e4043ae 100644 --- a/crates/modules/process-monitor/src/lib.rs +++ b/crates/modules/process-monitor/src/lib.rs @@ -1,6 +1,6 @@ use anyhow::Context; use bpf_common::{ - aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program, + ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program, ProgramBuilder, ProgramError, }; mod filtering; @@ -11,7 +11,8 @@ pub async fn program( ctx: BpfContext, sender: impl BpfSender, ) -> Result { - let mut program = ProgramBuilder::new(ctx, MODULE_NAME, PROCESS_MONITOR_PROBE.into()) + let binary = ebpf_program!(&ctx); + let mut program = ProgramBuilder::new(ctx, MODULE_NAME, binary) .raw_tracepoint("sched_process_exec") .raw_tracepoint("sched_process_exit") .raw_tracepoint("sched_process_fork") @@ -22,9 +23,6 @@ pub async fn program( Ok(program) } -static PROCESS_MONITOR_PROBE: &[u8] = - include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")); - // The events sent from eBPF to userspace must be byte by byte // re-interpretable as Rust types. So pointers to the heap are // not allowed. @@ -359,7 +357,7 @@ pub mod test_suite { (Some((false, false)), false), ] { // load ebpf and clear interest map - let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap(); + let mut bpf = load_ebpf(); attach_raw_tracepoint(&mut bpf, "sched_process_fork"); let mut interest_map = InterestMap::load(&mut bpf).unwrap(); interest_map.clear().unwrap(); @@ -413,7 +411,7 @@ pub mod test_suite { [(true, true), (true, false), (false, true), (false, false)] { // load ebpf and clear interest map - let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap(); + let mut bpf = load_ebpf(); attach_raw_tracepoint(&mut bpf, "sched_process_exec"); let mut interest_map = InterestMap::load(&mut bpf).unwrap(); interest_map.clear().unwrap(); @@ -501,7 +499,7 @@ pub mod test_suite { fn threads_are_ignored() -> TestCase { TestCase::new("threads_are_ignored", async { // load ebpf and clear interest map - let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap(); + let mut bpf = load_ebpf(); attach_raw_tracepoint(&mut bpf, "sched_process_fork"); let mut interest_map = InterestMap::load(&mut bpf).unwrap(); interest_map.clear().unwrap(); @@ -541,7 +539,7 @@ pub mod test_suite { fn exit_cleans_up_resources() -> TestCase { TestCase::new("exit_cleans_up_resources", async { // setup - let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap(); + let mut bpf = load_ebpf(); attach_raw_tracepoint(&mut bpf, "sched_process_exit"); let mut interest_map = InterestMap::load(&mut bpf).unwrap(); interest_map.clear().unwrap(); @@ -609,4 +607,15 @@ pub mod test_suite { .report() }) } + + fn load_ebpf() -> Bpf { + let ctx = BpfContext::new( + bpf_common::program::Pinning::Disabled, + bpf_common::program::PERF_PAGES_DEFAULT, + bpf_common::program::BpfLogLevel::Debug, + false, + ) + .unwrap(); + load_test_program(ebpf_program!(&ctx)).unwrap() + } } diff --git a/src/pulsard/mod.rs b/src/pulsard/mod.rs index 113a480d..84f8ef12 100644 --- a/src/pulsard/mod.rs +++ b/src/pulsard/mod.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, ensure, Result}; +use anyhow::{ensure, Result}; use bpf_common::bpf_fs; use engine_api::server::{self, EngineAPIContext}; use pulsar_core::{bus::Bus, pdk::TaskLauncher};