Testing: Add fuzz testing build target

Adds fuzz testing via AFL++

Code and instructions adapted from https://github.com/squeek502/zig-fuzzing-example

.gitignore

@@ -1,2 +1,3 @@
 zig-cache/
-zig-out/
+zig-out/
+test/fuzz-output/

build.zig

@@ -1,7 +1,40 @@
 const std = @import("std");
 const Builder = std.build.Builder;
 
-pub fn build(b: *Builder) void {
+fn addFuzzStep(b: *Builder) !void {
+    const fuzz_lib = b.addStaticLibrary("fuzz-lib", "test/fuzz.zig");
+    fuzz_lib.addPackagePath("aro", "src/lib.zig");
+    fuzz_lib.setBuildMode(.Debug);
+    fuzz_lib.want_lto = true;
+    fuzz_lib.bundle_compiler_rt = true;
+
+    const fuzz_executable_name = "fuzz";
+    const fuzz_exe_path = try std.fs.path.join(b.allocator, &.{ b.cache_root, fuzz_executable_name });
+
+    // We want `afl-clang-lto -o path/to/output path/to/library`
+    const fuzz_compile = b.addSystemCommand(&.{ "afl-clang-lto", "-o", fuzz_exe_path });
+    // Add the path to the library file to afl-clang-lto's args
+    fuzz_compile.addArtifactArg(fuzz_lib);
+
+    // Install the cached output to the install 'bin' path
+    const fuzz_install = b.addInstallBinFile(.{ .path = fuzz_exe_path }, fuzz_executable_name);
+
+    // Add a top-level step that compiles and installs the fuzz executable
+    const fuzz_compile_run = b.step("fuzz", "Build executable for fuzz testing using afl-clang-lto");
+    fuzz_compile_run.dependOn(&fuzz_compile.step);
+    fuzz_compile_run.dependOn(&fuzz_install.step);
+
+    // Compile a companion exe for debugging crashes
+    const fuzz_debug_exe = b.addExecutable("fuzz-debug", "test/fuzz.zig");
+    fuzz_debug_exe.addPackagePath("aro", "src/lib.zig");
+    fuzz_debug_exe.setBuildMode(.Debug);
+
+    // Only install fuzz-debug when the fuzz step is run
+    const install_fuzz_debug_exe = b.addInstallArtifact(fuzz_debug_exe);
+    fuzz_compile_run.dependOn(&install_fuzz_debug_exe.step);
+}
+
+pub fn build(b: *Builder) !void {
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
     // means any target is allowed, and the default is native. Other options
@@ -50,4 +83,6 @@ pub fn build(b: *Builder) void {
     integration_test_runner.addArg(b.pathFromRoot("test/cases"));
     integration_test_runner.addArg(b.zig_exe);
     tests_step.dependOn(&integration_test_runner.step);
+
+    try addFuzzStep(b);
 }

test/README.MD

@@ -63,3 +63,16 @@ If a test case is currently broken, it should be commented and a `TESTS_SKIPPED`
 
 // <something broken here>
 ```
+---
+## Running fuzz tests
+Fuzz testing requires [AFLplusplus](https://github.com/AFLplusplus/AFLplusplus). Run `zig build fuzz` to build the fuzz target,
+then `afl-fuzz -i test/cases -o test/fuzz-output -- ./zig-out/bin/fuzz`
+
+A Dockerfile is provided to make it easier to get started with fuzzing:
+
+```sh-session
+docker build -t aro-fuzz test/docker/fuzz
+docker run --rm -it -v $PWD:/arocc -w /arocc aro-fuzz
+zig build fuzz
+afl-fuzz -i test/cases -o test/fuzz-output -- ./zig-out/bin/fuzz
+```

test/docker/fuzz/Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+env NO_ARCH_OPT 1
+
+RUN apt-get update && \
+    apt-get -y install --no-install-suggests --no-install-recommends \
+    automake \
+    ninja-build \
+    bison flex \
+    build-essential \
+    git \
+    python3 python3-dev python3-setuptools python-is-python3 \
+    libtool libtool-bin \
+    libglib2.0-dev \
+    wget jupp nano bash-completion less \
+    apt-utils apt-transport-https ca-certificates gnupg dialog \
+    libpixman-1-dev \
+    gnuplot-nox \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" >> /etc/apt/sources.list && \
+    wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+
+RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal main" >> /etc/apt/sources.list && \
+    apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F
+
+RUN apt-get update && apt-get full-upgrade -y && \
+    apt-get -y install --no-install-suggests --no-install-recommends \
+    gcc-10 g++-10 gcc-10-plugin-dev gcc-10-multilib gcc-multilib gdb lcov \
+    clang-13 clang-tools-13 libc++1-13 libc++-13-dev \
+    libc++abi1-13 libc++abi-13-dev libclang1-13 libclang-13-dev \
+    libclang-common-13-dev libclang-cpp13 libclang-cpp13-dev liblld-13 \
+    liblld-13-dev liblldb-13 liblldb-13-dev libllvm13 libomp-13-dev \
+    libomp5-13 lld-13 lldb-13 llvm-13 llvm-13-dev llvm-13-runtime llvm-13-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 0
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 0
+
+ENV LLVM_CONFIG=llvm-config-13
+ENV AFL_SKIP_CPUFREQ=1
+ENV AFL_TRY_AFFINITY=1
+ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
+
+RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov /afl-cov
+RUN cd /afl-cov && make install && cd ..
+
+RUN git clone https://github.com/AFLplusplus/AFLplusplus.git
+
+WORKDIR /AFLplusplus
+
+RUN export CC=gcc-10 && export CXX=g++-10 && make clean && \
+    make distrib && make install && make clean
+
+WORKDIR /
+
+RUN wget https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.1561+5ebdc8c46.tar.xz && \
+    echo 044ce7cc2d405c10c22ece61880a863f50ee080cfb0547999a5ce3b097093916 zig-linux-x86_64-0.9.0-dev.1561+5ebdc8c46.tar.xz | sha256sum --check --strict && \
+    tar -xvf zig-linux-x86_64-0.9.0-dev.1561+5ebdc8c46.tar.xz && \
+    rm zig-linux-x86_64-0.9.0-dev.1561+5ebdc8c46.tar.xz
+
+ENV PATH="/zig-linux-x86_64-0.9.0-dev.1561+5ebdc8c46:$PATH"

test/fuzz.zig

@@ -0,0 +1,80 @@
+const std = @import("std");
+const aro = @import("aro");
+const mem = std.mem;
+const Allocator = mem.Allocator;
+const process = std.process;
+const Codegen = aro.Codegen;
+const Compilation = aro.Compilation;
+const Source = aro.Source;
+const Preprocessor = aro.Preprocessor;
+const Parser = aro.Parser;
+
+fn cMain() callconv(.C) void {
+    main() catch unreachable;
+}
+
+comptime {
+    @export(cMain, .{ .name = "main", .linkage = .Strong });
+}
+
+var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
+
+pub fn main() !void {
+    const gpa = &general_purpose_allocator.allocator;
+    defer std.debug.assert(general_purpose_allocator.deinit() == false);
+
+    var comp = Compilation.init(gpa);
+    defer comp.deinit();
+
+    try compileInput(&comp);
+}
+
+fn compileInput(comp: *Compilation) !void {
+    try comp.addDefaultPragmaHandlers();
+    try comp.defineSystemIncludes();
+
+    const builtin = try comp.generateBuiltinMacros();
+    const user_source = blk: {
+        const path = try comp.gpa.dupe(u8, "<STDIN>");
+        errdefer comp.gpa.free(path);
+
+        const stdin = std.io.getStdIn();
+        const buf = try stdin.readToEndAlloc(comp.gpa, std.math.maxInt(usize));
+        errdefer comp.gpa.free(buf);
+
+        const source = Source{
+            .id = @intToEnum(Source.Id, comp.sources.count() + 2),
+            .path = path,
+            .buf = buf,
+        };
+        try comp.sources.put(path, source);
+        break :blk source;
+    };
+
+    processSource(comp, builtin, user_source) catch |e| switch (e) {
+        error.FatalError => {},
+        else => |err| return err,
+    };
+}
+
+fn processSource(comp: *Compilation, builtin: Source, user_source: Source) !void {
+    var pp = Preprocessor.init(comp);
+    defer pp.deinit();
+    try pp.addBuiltinMacros();
+
+    try pp.preprocess(builtin);
+    try pp.preprocess(user_source);
+    try pp.tokens.append(pp.comp.gpa, .{
+        .id = .eof,
+        .loc = .{ .id = user_source.id, .byte_offset = @intCast(u32, user_source.buf.len) },
+    });
+
+    var tree = try Parser.parse(&pp);
+    defer tree.deinit();
+
+    var buf_writer = std.io.bufferedWriter(std.io.getStdOut().writer());
+    try tree.dump(buf_writer.writer());
+    try buf_writer.flush();
+
+    comp.renderErrors();
+}