fix(bpf-builder): embed two versions of eBPF programs

Compile and embed two eBPF programs for each *.bpf.c source:
- On kernel < 5.13 NOLOOP is defined and we won't take the address
  of functions.
- On kernel >= 5.13, the regular LOOP macro can be used.

Fix #158

crates/bpf-builder/include/loop.bpf.h

@@ -13,6 +13,15 @@
 // Note: callback_fn must be declared as `static __always_inline` to satisfy the
 // verifier. For some reason, having this double call to the same non-inline
 // function seems to cause issues.
+#ifdef NOLOOP
+// On kernel <= 5.13 taking the address of a function results in a verifier
+// error, even if inside a dead-code elimination branch.
+#define LOOP(max_iterations, callback_fn, ctx)                                 \
+  _Pragma("unroll") for (int i = 0; i < max_iterations; i++) {                 \
+    if (callback_fn(i, ctx) == 1)                                              \
+      break;                                                                   \
+  }
+#else
 #define LOOP(max_iterations, callback_fn, ctx)                                 \
   if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(5, 17, 0)) {                      \
     bpf_loop(max_iterations, callback_fn, ctx, 0);                             \
@@ -22,3 +31,4 @@
         break;                                                                 \
     }                                                                          \
   }
+#endif

crates/bpf-builder/src/lib.rs

@@ -1,11 +1,20 @@
 use std::{env, path::PathBuf, process::Command, string::String};
 
-use anyhow::Context;
+use anyhow::{bail, Context};
 
 static CLANG_DEFAULT: &str = "clang";
 static LLVM_STRIP: &str = "llvm-strip";
 static INCLUDE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/include");
 
+// Given the filename of an eBPF program source code, compile it to OUT_DIR.
+// We'll build two versions:
+// - `probe_full.bpf.o`: will contain the full version
+// - `probe_noloop.bpf.o`: will contain a version with the NOLOOP constant
+//   defined. This version should be loaded on kernel < 5.13, where taking
+//   the address of a static function would result in a verifier error.
+//   See
+//   - https://github.com/Exein-io/pulsar/issues/158
+//   - https://github.com/torvalds/linux/commit/69c087ba6225b574afb6e505b72cb75242a3d844
 pub fn build(probe: &str) -> Result<(), Box<dyn std::error::Error>> {
     println!("cargo:rerun-if-changed={probe}");
     println!("cargo:rerun-if-changed={INCLUDE_PATH}/common.bpf.h");
@@ -14,14 +23,17 @@ pub fn build(probe: &str) -> Result<(), Box<dyn std::error::Error>> {
     println!("cargo:rerun-if-changed={INCLUDE_PATH}/loop.bpf.h");
 
     let out_path = PathBuf::from(env::var("OUT_DIR")?);
-    let out_object = out_path.join("probe.bpf.o");
 
-    let clang = match env::var("CLANG") {
-        Ok(val) => val,
-        Err(_) => String::from(CLANG_DEFAULT),
-    };
+    compile(probe, out_path.join("probe_full.bpf.o"), &[])
+        .context("Error compiling full version")?;
+    compile(probe, out_path.join("probe_noloop.bpf.o"), &["-DNOLOOP"])
+        .context("Error compiling no-loop version")?;
 
-    // Compile
+    Ok(())
+}
+
+fn compile(probe: &str, out_object: PathBuf, extra_args: &[&str]) -> anyhow::Result<()> {
+    let clang = env::var("CLANG").unwrap_or_else(|_| String::from(CLANG_DEFAULT));
     let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
     let include_path = PathBuf::from(INCLUDE_PATH);
     let status = Command::new(clang)
@@ -41,14 +53,15 @@ pub fn build(probe: &str) -> Result<(), Box<dyn std::error::Error>> {
                 _ => arch.clone(),
             }
         ))
+        .args(extra_args)
         .arg(probe)
         .arg("-o")
         .arg(&out_object)
         .status()
         .context("Failed to execute clang")?;
 
     if !status.success() {
-        Err("Failed to compile eBPF program")?;
+        bail!("Failed to compile eBPF program");
     }
 
     // Strip debug symbols
@@ -59,7 +72,7 @@ pub fn build(probe: &str) -> Result<(), Box<dyn std::error::Error>> {
         .context("Failed to execute llvm-strip")?;
 
     if !status.success() {
-        Err("Failed strip eBPF program")?;
+        bail!("Failed strip eBPF program");
     }
 
     Ok(())

crates/bpf-common/ProbeTutorial.md

@@ -78,18 +78,19 @@ The module implementation in Rust is also relatively short.
 use std::fmt;
 
 use bpf_common::{
-    aya::include_bytes_aligned, program::BpfContext, BpfSender, Program,
+    aya::program::BpfContext, BpfSender, Program,
     ProgramBuilder, ProgramError,
 };
 
 pub async fn program(
     ctx: BpfContext,
     sender: impl BpfSender<EventT>,
 ) -> Result<Program, ProgramError> {
+    let binary = ebpf_program!(&ctx);
     let program = ProgramBuilder::new(
         ctx,
         "file_created",
-        include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(),
+        binary,
     )
     .kprobe("security_inode_create")
     .start()

crates/bpf-common/src/feature_autodetect/lsm.rs

@@ -22,7 +22,8 @@ pub fn lsm_supported() -> bool {
 }
 
 const PATH: &str = "/sys/kernel/security/lsm";
-static TEST_LSM_PROBE: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o"));
+static TEST_LSM_PROBE: &[u8] =
+    include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_full.bpf.o"));
 
 fn try_load() -> Result<()> {
     // Check if LSM enabled

crates/bpf-common/src/program.rs

@@ -110,6 +110,39 @@ impl BpfContext {
     pub fn lsm_supported(&self) -> bool {
         self.lsm_supported
     }
+
+    pub fn kernel_version(&self) -> &KernelVersion {
+        &self.kernel_version
+    }
+}
+
+/// Return the correct version of the eBPF binary to load.
+/// On kernel >= 5.13.0 we'll load 'probe_full.bpf.o'
+/// On kernel < 5.13.0 we'll load 'probe_noloop.bpf.o'
+/// Note: Both programs are embedded in the pulsar binary. The choice is made
+/// at runtime.
+#[macro_export]
+macro_rules! ebpf_program {
+    ( $ctx: expr ) => {{
+        use bpf_common::aya::include_bytes_aligned;
+        use bpf_common::feature_autodetect::kernel_version::KernelVersion;
+
+        let full = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_full.bpf.o")).into();
+        let no_loop =
+            include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe_noloop.bpf.o")).into();
+        if $ctx.kernel_version().as_i32()
+            >= (KernelVersion {
+                major: 5,
+                minor: 13,
+                patch: 0,
+            })
+            .as_i32()
+        {
+            full
+        } else {
+            no_loop
+        }
+    }};
 }
 
 #[derive(Error, Debug)]

crates/modules/file-system-monitor/src/lib.rs

@@ -1,6 +1,6 @@
 use bpf_common::{
-    aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Program,
-    ProgramBuilder, ProgramError,
+    ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Program, ProgramBuilder,
+    ProgramError,
 };
 
 const MODULE_NAME: &str = "file-system-monitor";
@@ -10,11 +10,8 @@ pub async fn program(
     sender: impl BpfSender<FsEvent>,
 ) -> Result<Program, ProgramError> {
     let attach_to_lsm = ctx.lsm_supported();
-    let mut builder = ProgramBuilder::new(
-        ctx,
-        MODULE_NAME,
-        include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(),
-    );
+    let binary = ebpf_program!(&ctx);
+    let mut builder = ProgramBuilder::new(ctx, MODULE_NAME, binary);
     // LSM hooks provide the perfet intercept point for file system operations.
     // If LSM eBPF programs is not supported, we'll attach to the same kernel
     // functions, but using kprobes.

crates/modules/network-monitor/src/lib.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 use bpf_common::{
-    aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program,
+    ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program,
     ProgramBuilder, ProgramError,
 };
 use nix::sys::socket::{SockaddrIn, SockaddrIn6};
@@ -50,20 +50,17 @@ pub async fn program(
     sender: impl BpfSender<NetworkEvent>,
 ) -> Result<Program, ProgramError> {
     let attach_to_lsm = ctx.lsm_supported();
-    let mut builder = ProgramBuilder::new(
-        ctx,
-        MODULE_NAME,
-        include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o")).into(),
-    )
-    .tracepoint("syscalls", "sys_exit_accept4")
-    .tracepoint("syscalls", "sys_exit_accept")
-    .tracepoint("syscalls", "sys_exit_recvmsg")
-    .tracepoint("syscalls", "sys_exit_recvmmsg")
-    .tracepoint("syscalls", "sys_enter_recvfrom")
-    .tracepoint("syscalls", "sys_exit_recvfrom")
-    .tracepoint("syscalls", "sys_exit_read")
-    .tracepoint("syscalls", "sys_exit_readv")
-    .kprobe("tcp_set_state");
+    let binary = ebpf_program!(&ctx);
+    let mut builder = ProgramBuilder::new(ctx, MODULE_NAME, binary)
+        .tracepoint("syscalls", "sys_exit_accept4")
+        .tracepoint("syscalls", "sys_exit_accept")
+        .tracepoint("syscalls", "sys_exit_recvmsg")
+        .tracepoint("syscalls", "sys_exit_recvmmsg")
+        .tracepoint("syscalls", "sys_enter_recvfrom")
+        .tracepoint("syscalls", "sys_exit_recvfrom")
+        .tracepoint("syscalls", "sys_exit_read")
+        .tracepoint("syscalls", "sys_exit_readv")
+        .kprobe("tcp_set_state");
     if attach_to_lsm {
         builder = builder
             .lsm("socket_bind")

crates/modules/process-monitor/src/lib.rs

@@ -1,6 +1,6 @@
 use anyhow::Context;
 use bpf_common::{
-    aya::include_bytes_aligned, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program,
+    ebpf_program, parsing::BufferIndex, program::BpfContext, BpfSender, Pid, Program,
     ProgramBuilder, ProgramError,
 };
 mod filtering;
@@ -11,7 +11,8 @@ pub async fn program(
     ctx: BpfContext,
     sender: impl BpfSender<ProcessEvent>,
 ) -> Result<Program, ProgramError> {
-    let mut program = ProgramBuilder::new(ctx, MODULE_NAME, PROCESS_MONITOR_PROBE.into())
+    let binary = ebpf_program!(&ctx);
+    let mut program = ProgramBuilder::new(ctx, MODULE_NAME, binary)
         .raw_tracepoint("sched_process_exec")
         .raw_tracepoint("sched_process_exit")
         .raw_tracepoint("sched_process_fork")
@@ -22,9 +23,6 @@ pub async fn program(
     Ok(program)
 }
 
-static PROCESS_MONITOR_PROBE: &[u8] =
-    include_bytes_aligned!(concat!(env!("OUT_DIR"), "/probe.bpf.o"));
-
 // The events sent from eBPF to userspace must be byte by byte
 // re-interpretable as Rust types. So pointers to the heap are
 // not allowed.
@@ -359,7 +357,7 @@ pub mod test_suite {
                 (Some((false, false)), false),
             ] {
                 // load ebpf and clear interest map
-                let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap();
+                let mut bpf = load_ebpf();
                 attach_raw_tracepoint(&mut bpf, "sched_process_fork");
                 let mut interest_map = InterestMap::load(&mut bpf).unwrap();
                 interest_map.clear().unwrap();
@@ -413,7 +411,7 @@ pub mod test_suite {
                 [(true, true), (true, false), (false, true), (false, false)]
             {
                 // load ebpf and clear interest map
-                let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap();
+                let mut bpf = load_ebpf();
                 attach_raw_tracepoint(&mut bpf, "sched_process_exec");
                 let mut interest_map = InterestMap::load(&mut bpf).unwrap();
                 interest_map.clear().unwrap();
@@ -501,7 +499,7 @@ pub mod test_suite {
     fn threads_are_ignored() -> TestCase {
         TestCase::new("threads_are_ignored", async {
             // load ebpf and clear interest map
-            let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap();
+            let mut bpf = load_ebpf();
             attach_raw_tracepoint(&mut bpf, "sched_process_fork");
             let mut interest_map = InterestMap::load(&mut bpf).unwrap();
             interest_map.clear().unwrap();
@@ -541,7 +539,7 @@ pub mod test_suite {
     fn exit_cleans_up_resources() -> TestCase {
         TestCase::new("exit_cleans_up_resources", async {
             // setup
-            let mut bpf = load_test_program(PROCESS_MONITOR_PROBE).unwrap();
+            let mut bpf = load_ebpf();
             attach_raw_tracepoint(&mut bpf, "sched_process_exit");
             let mut interest_map = InterestMap::load(&mut bpf).unwrap();
             interest_map.clear().unwrap();
@@ -609,4 +607,15 @@ pub mod test_suite {
                 .report()
         })
     }
+
+    fn load_ebpf() -> Bpf {
+        let ctx = BpfContext::new(
+            bpf_common::program::Pinning::Disabled,
+            bpf_common::program::PERF_PAGES_DEFAULT,
+            bpf_common::program::BpfLogLevel::Debug,
+            false,
+        )
+        .unwrap();
+        load_test_program(ebpf_program!(&ctx)).unwrap()
+    }
 }

src/pulsard/mod.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, bail, ensure, Result};
+use anyhow::{ensure, Result};
 use bpf_common::bpf_fs;
 use engine_api::server::{self, EngineAPIContext};
 use pulsar_core::{bus::Bus, pdk::TaskLauncher};