Cleanup build scripts (#561)

* aws-lc-sys: log clang version used w/ bindgen

* aws-lc-fips-sys: log clang version used w/ bindgen

* Add testing

* bindgen-cli not needed on FreeBSD

.github/workflows/cross.yml

@@ -264,7 +264,7 @@ jobs:
           usesh: true
           copyback: false
           prepare: |
-            pkg install -y git gmake bash sudo cmake-core llvm-devel-lite curl rust-bindgen-cli go
+            pkg install -y git gmake bash sudo cmake-core llvm-devel-lite curl go
           run: |
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
              . "$HOME/.cargo/env"

.github/workflows/tests.yml

@@ -602,3 +602,33 @@ jobs:
         working-directory: ./aws-lc-rs
         # Rust doc tests dislike dynamic linking
         run: cargo careful test --tests --features fips,bindgen
+
+  clang-19-bindgen:
+    if: github.repository_owner == 'aws'
+    name: Clang 19.1 + bindgen tests
+    runs-on: macos-14-xlarge
+    env:
+      LIBCLANG_PATH: /opt/homebrew/opt/llvm/lib
+      LLVM_CONFIG_PATH: /opt/homebrew/opt/llvm/llvm-config
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '>=1.18'
+      - uses: dtolnay/rust-toolchain@master
+        id: toolchain
+        with:
+          toolchain: stable
+      - run: |
+          brew update
+          brew uninstall --force llvm
+          brew install llvm@19
+          echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> "$GITHUB_ENV"
+      - name: aws-lc-sys bindgen build
+        working-directory: ./aws-lc-sys
+        run: cargo test --features bindgen
+      - name: aws-lc-fips-sys bindgen build
+        working-directory: ./aws-lc-fips-sys
+        run: cargo test --features bindgen

aws-lc-fips-sys/Cargo.toml

@@ -66,10 +66,10 @@ fs_extra = "1.3"
 cc = "1.0.100"
 
 [target.'cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = "")))'.build-dependencies]
-bindgen = { version = "0.69.2", optional = true }
+bindgen = { version = "0.69.5", optional = true }
 
 [target.'cfg(not(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos"), any(target_env = "gnu", target_env = "musl", target_env = ""))))'.build-dependencies]
-bindgen = { version = "0.69.2" }
+bindgen = { version = "0.69.5" }
 
 [dependencies]
 libc = "0.2.121"

aws-lc-fips-sys/builder/cmake_builder.rs

@@ -8,7 +8,7 @@ use crate::{
 };
 use std::collections::HashMap;
 use std::env;
-use std::ffi::OsStr;
+use std::ffi::OsString;
 use std::path::PathBuf;
 
 pub(crate) struct CmakeBuilder {
@@ -39,11 +39,21 @@ fn test_nasm_command() -> bool {
     execute_command("nasm".as_ref(), &["-version".as_ref()]).status
 }
 
-fn find_cmake_command() -> Option<&'static OsStr> {
-    if execute_command("cmake3".as_ref(), &["--version".as_ref()]).status {
-        Some("cmake3".as_ref())
+fn find_cmake_command() -> Option<OsString> {
+    if let Some(cmake) = option_env("CMAKE") {
+        emit_warning(&format!(
+            "CMAKE environment variable set: {}",
+            cmake.clone()
+        ));
+        if execute_command(cmake.as_ref(), &["--version".as_ref()]).status {
+            Some(cmake.into())
+        } else {
+            None
+        }
+    } else if execute_command("cmake3".as_ref(), &["--version".as_ref()]).status {
+        Some("cmake3".into())
     } else if execute_command("cmake".as_ref(), &["--version".as_ref()]).status {
-        Some("cmake".as_ref())
+        Some("cmake".into())
     } else {
         None
     }
@@ -87,18 +97,10 @@ impl CmakeBuilder {
         } else {
             cmake_cfg.define("BUILD_SHARED_LIBS", "0");
         }
-        let opt_level = cargo_env("OPT_LEVEL");
-
-        if is_no_asm() {
-            if opt_level == "0" {
-                cmake_cfg.define("OPENSSL_NO_ASM", "1");
-            } else {
-                panic!("AWS_LC_FIPS_SYS_NO_ASM only allowed for debug builds!")
-            }
-        }
 
-        if opt_level != "0" {
-            if opt_level == "1" || opt_level == "2" {
+        let opt_level = cargo_env("OPT_LEVEL");
+        if opt_level.ne("0") {
+            if opt_level.eq("1") || opt_level.eq("2") {
                 cmake_cfg.define("CMAKE_BUILD_TYPE", "relwithdebinfo");
             } else {
                 cmake_cfg.define("CMAKE_BUILD_TYPE", "release");
@@ -153,6 +155,15 @@ impl CmakeBuilder {
         }
         cmake_cfg.define("FIPS", "1");
 
+        if is_no_asm() {
+            let opt_level = cargo_env("OPT_LEVEL");
+            if opt_level == "0" {
+                cmake_cfg.define("OPENSSL_NO_ASM", "1");
+            } else {
+                panic!("AWS_LC_SYS_NO_ASM only allowed for debug builds!")
+            }
+        }
+
         if cfg!(feature = "asan") {
             env::set_var("CC", "clang");
             env::set_var("CXX", "clang++");
@@ -162,9 +173,13 @@ impl CmakeBuilder {
         }
 
         // Allow environment to specify CMake toolchain.
-        if option_env("CMAKE_TOOLCHAIN_FILE").is_some()
-            || option_env(format!("CMAKE_TOOLCHAIN_FILE_{}", target_underscored())).is_some()
-        {
+        if let Some(toolchain) = option_env("CMAKE_TOOLCHAIN_FILE").or(option_env(format!(
+            "CMAKE_TOOLCHAIN_FILE_{}",
+            target_underscored()
+        ))) {
+            emit_warning(&format!(
+                "CMAKE_TOOLCHAIN_FILE environment variable set: {toolchain}"
+            ));
             return cmake_cfg;
         }
 

aws-lc-fips-sys/builder/main.rs

@@ -16,15 +16,27 @@ use std::process::Command;
 
 use cmake_builder::CmakeBuilder;
 
-#[cfg(any(
-    feature = "bindgen",
-    not(all(
-        any(target_arch = "x86_64", target_arch = "aarch64"),
-        any(target_os = "linux", target_os = "macos"),
-        any(target_env = "gnu", target_env = "musl", target_env = "")
-    ))
-))]
-mod bindgen;
+macro_rules! bindgen_available {
+    ($top:ident, $item:item) => {
+        #[allow(clippy::non_minimal_cfg)]
+        #[cfg($top(any(
+            feature = "bindgen",
+            not(all(
+                any(target_arch = "x86_64", target_arch = "aarch64"),
+                any(target_os = "linux", target_os = "macos"),
+                any(target_env = "gnu", target_env = "musl", target_env = "")
+            ))
+        )))]
+        $item
+    };
+    ($item:item) => {
+        bindgen_available!(any, $item);
+    };
+}
+
+bindgen_available!(
+    mod sys_bindgen;
+);
 mod cmake_builder;
 
 pub(crate) fn get_aws_lc_include_path(manifest_dir: &Path) -> PathBuf {
@@ -184,31 +196,25 @@ fn execute_command(executable: &OsStr, args: &[&OsStr]) -> TestCommandResult {
     }
 }
 
-#[cfg(any(
-    feature = "bindgen",
-    not(all(
-        any(target_arch = "x86_64", target_arch = "aarch64"),
-        any(target_os = "linux", target_os = "macos"),
-        any(target_env = "gnu", target_env = "musl", target_env = "")
-    ))
-))]
-fn generate_bindings(manifest_dir: &Path, prefix: &Option<String>, bindings_path: &PathBuf) {
-    let options = BindingOptions {
-        build_prefix: prefix.clone(),
-        include_ssl: cfg!(feature = "ssl"),
-        disable_prelude: true,
-    };
+bindgen_available!(
+    fn generate_bindings(manifest_dir: &Path, prefix: &Option<String>, bindings_path: &PathBuf) {
+        let options = BindingOptions {
+            build_prefix: prefix.clone(),
+            include_ssl: cfg!(feature = "ssl"),
+            disable_prelude: true,
+        };
 
-    let bindings = bindgen::generate_bindings(manifest_dir, &options);
+        let bindings = sys_bindgen::generate_bindings(manifest_dir, &options);
 
-    bindings
-        .write(Box::new(std::fs::File::create(bindings_path).unwrap()))
-        .expect("written bindings");
-}
+        bindings
+            .write(Box::new(std::fs::File::create(bindings_path).unwrap()))
+            .expect("written bindings");
+    }
+);
 
 #[cfg(feature = "bindgen")]
 fn generate_src_bindings(manifest_dir: &Path, prefix: &Option<String>, src_bindings_path: &Path) {
-    bindgen::generate_bindings(
+    sys_bindgen::generate_bindings(
         manifest_dir,
         &BindingOptions {
             build_prefix: prefix.clone(),
@@ -327,12 +333,13 @@ fn is_bindgen_required() -> bool {
         || !has_pregenerated()
 }
 
-#[allow(dead_code)]
-fn internal_bindgen_supported() -> bool {
-    // TODO: internal bindgen creates invalid bindings on FreeBSD
-    // See: https://github.com/aws/aws-lc-rs/issues/476
-    target_os() != "freebsd"
-}
+bindgen_available!(
+    fn internal_bindgen_supported() -> bool {
+        let cv = bindgen::clang_version();
+        emit_warning(&format!("Clang version: {}", cv.full));
+        true
+    }
+);
 
 fn is_no_prefix() -> bool {
     unsafe { AWS_LC_FIPS_SYS_NO_PREFIX }
@@ -372,6 +379,30 @@ fn prepare_cargo_cfg() {
     }
 }
 
+bindgen_available!(
+    fn handle_bindgen(manifest_dir: &PathBuf, prefix: &Option<String>) -> bool {
+        if internal_bindgen_supported() && !is_external_bindgen() {
+            emit_warning(&format!(
+                "Generating bindings - internal bindgen. Platform: {}",
+                target()
+            ));
+            let gen_bindings_path = out_dir().join("bindings.rs");
+            generate_bindings(manifest_dir, prefix, &gen_bindings_path);
+            emit_rustc_cfg("use_bindgen_generated");
+            true
+        } else {
+            false
+        }
+    }
+);
+
+bindgen_available!(
+    not,
+    fn handle_bindgen(_manifest_dir: &PathBuf, _prefix: &Option<String>) -> bool {
+        false
+    }
+);
+
 fn main() {
     initialize();
     prepare_cargo_cfg();
@@ -386,6 +417,8 @@ fn main() {
     };
 
     let builder = get_builder(&prefix, &manifest_dir, &out_dir());
+    emit_warning("Building with: CMake");
+    emit_warning(&format!("Symbol Prefix: {:?}", &prefix));
 
     builder.check_dependencies().unwrap();
 
@@ -403,24 +436,7 @@ fn main() {
             bindings_available = true;
         }
     } else if is_bindgen_required() {
-        #[cfg(any(
-            feature = "bindgen",
-            not(all(
-                any(target_arch = "x86_64", target_arch = "aarch64"),
-                any(target_os = "linux", target_os = "macos"),
-                any(target_env = "gnu", target_env = "musl", target_env = "")
-            ))
-        ))]
-        if internal_bindgen_supported() && !is_external_bindgen() {
-            emit_warning(&format!(
-                "Generating bindings - internal bindgen. Platform: {}",
-                target()
-            ));
-            let gen_bindings_path = out_dir().join("bindings.rs");
-            generate_bindings(&manifest_dir, &prefix, &gen_bindings_path);
-            emit_rustc_cfg("use_bindgen_generated");
-            bindings_available = true;
-        }
+        bindings_available = handle_bindgen(&manifest_dir, &prefix);
     } else {
         bindings_available = true;
     }

aws-lc-sys/Cargo.toml

@@ -60,10 +60,10 @@ fs_extra = "1.3"
 cc = { version = "1.0.100", features = ["parallel"] }
 
 [target.'cfg(any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu")))'.build-dependencies]
-bindgen = { version = "0.69.2", optional = true }
+bindgen = { version = "0.69.5", optional = true }
 
 [target.'cfg(not(any(all(any(target_arch="x86_64",target_arch="aarch64"),any(target_os="linux",target_os="macos",target_os="windows"),any(target_env="gnu",target_env="musl",target_env="msvc",target_env="")),all(target_arch="x86",target_os="windows",target_env="msvc"),all(target_arch="x86",target_os="linux",target_env="gnu"))))'.build-dependencies]
-bindgen = { version = "0.69.2" }
+bindgen = { version = "0.69.5" }
 
 [dependencies]
 libc = "0.2.121"