Merge pull request #270 from jmt-lab/ootb/external-kits/download

twoliter: fetch command for downloading and extracting external kits

Cargo.toml

@@ -4,6 +4,7 @@ resolver = "2"
 members = [
     "tools/bottlerocket-variant",
     "tools/buildsys",
+    "tools/buildsys-config",
     "tools/parse-datetime",
     "tools/pubsys",
     "tools/pubsys-config",

tools/buildsys-config/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "buildsys-config"
+version = "0.1.0"
+authors = ["Jarrett Tierney <[email protected]>"]
+license = "MIT OR Apache-2.0"
+edition = "2021"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1", features = ["derive"]}

tools/buildsys-config/src/lib.rs

@@ -0,0 +1,33 @@
+use anyhow::anyhow;
+use serde::Deserialize;
+use std::fmt::{Display, Formatter};
+
+pub const EXTERNAL_KIT_DIRECTORY: &str = "build/external-kits";
+pub const EXTERNAL_KIT_METADATA: &str = "build/external-kits/external-kit-metadata.json";
+
+#[derive(Deserialize, Debug, Clone, PartialEq)]
+pub enum DockerArchitecture {
+    Amd64,
+    Arm64,
+}
+
+impl TryFrom<&str> for DockerArchitecture {
+    type Error = anyhow::Error;
+
+    fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
+        match value {
+            "x86_64" | "amd64" => Ok(DockerArchitecture::Amd64),
+            "aarch64" | "arm64" => Ok(DockerArchitecture::Arm64),
+            _ => Err(anyhow!("invalid architecture '{}'", value)),
+        }
+    }
+}
+
+impl Display for DockerArchitecture {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
+            Self::Amd64 => "amd64",
+            Self::Arm64 => "arm64",
+        })
+    }
+}

tools/buildsys/Cargo.toml

@@ -10,6 +10,7 @@ exclude = ["README.md"]
 
 [dependencies]
 bottlerocket-variant = { version = "0.1", path = "../bottlerocket-variant" }
+buildsys-config = { version = "0.1", path = "../buildsys-config" }
 clap = { version = "4", features = ["derive", "env"] }
 duct = "0.13"
 guppy = "0.17"

tools/buildsys/src/main.rs

@@ -20,6 +20,7 @@ use crate::args::{
 };
 use crate::builder::DockerBuild;
 use buildsys::manifest::{BundleModule, ImageFeature, Manifest, ManifestInfo, SupportedArch};
+use buildsys_config::EXTERNAL_KIT_METADATA;
 use cache::LookasideCache;
 use clap::Parser;
 use gomod::GoMod;
@@ -124,6 +125,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
     let manifest_file = "Cargo.toml";
     let manifest_path = args.common.cargo_manifest_dir.join(manifest_file);
     println!("cargo:rerun-if-changed={}", manifest_file);
+    println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA);
 
     let manifest = Manifest::new(&manifest_path, &args.common.cargo_metadata_path)
         .context(error::ManifestParseSnafu)?;
@@ -205,6 +207,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> {
 fn build_kit(args: BuildKitArgs) -> Result<()> {
     let manifest_file = "Cargo.toml";
     println!("cargo:rerun-if-changed={}", manifest_file);
+    println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA);
 
     let manifest = Manifest::new(
         args.common.cargo_manifest_dir.join(manifest_file),
@@ -221,6 +224,7 @@ fn build_kit(args: BuildKitArgs) -> Result<()> {
 fn build_variant(args: BuildVariantArgs) -> Result<()> {
     let manifest_file = "Cargo.toml";
     println!("cargo:rerun-if-changed={}", manifest_file);
+    println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA);
 
     let manifest = Manifest::new(
         args.common.cargo_manifest_dir.join(manifest_file),

twoliter/Cargo.toml

@@ -14,6 +14,7 @@ anyhow = "1"
 async-recursion = "1"
 async-walkdir = "1"
 base64 = "0.22"
+buildsys-config = { version = "0.1", path = "../tools/buildsys-config" }
 clap = { version = "4", features = ["derive", "env", "std"] }
 env_logger = "0.11"
 filetime = "0.2"
@@ -22,6 +23,7 @@ futures= "0.3"
 hex = "0.4"
 log = "0.4"
 non-empty-string = { version = "0.2", features = [ "serde" ] }
+olpc-cjson = "0.1"
 semver = { version = "1", features = ["serde"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

twoliter/embedded/Makefile.toml

@@ -12,6 +12,7 @@ BUILDSYS_ROOT_DIR = "${CARGO_MAKE_WORKING_DIRECTORY}"
 BUILDSYS_BUILD_DIR = "${BUILDSYS_ROOT_DIR}/build"
 BUILDSYS_PACKAGES_DIR = "${BUILDSYS_BUILD_DIR}/rpms"
 BUILDSYS_KITS_DIR = "${BUILDSYS_BUILD_DIR}/kits"
+BUILDSYS_EXTERNAL_KITS_DIR = "${BUILDSYS_BUILD_DIR}/external-kits"
 BUILDSYS_STATE_DIR = "${BUILDSYS_BUILD_DIR}/state"
 BUILDSYS_IMAGES_DIR = "${BUILDSYS_BUILD_DIR}/images"
 BUILDSYS_LOGS_DIR = "${BUILDSYS_BUILD_DIR}/logs"
@@ -280,6 +281,7 @@ mkdir -p ${BUILDSYS_BUILD_DIR}
 mkdir -p ${BUILDSYS_OUTPUT_DIR}
 mkdir -p ${BUILDSYS_PACKAGES_DIR}
 mkdir -p ${BUILDSYS_KITS_DIR}
+mkdir -p ${BUILDSYS_EXTERNAL_KITS_DIR}
 mkdir -p ${BUILDSYS_STATE_DIR}
 mkdir -p ${BUILDSYS_METADATA_DIR}
 mkdir -p ${GO_MOD_CACHE}
@@ -1642,6 +1644,7 @@ script_runner = "bash"
 script = [
 '''
 rm -rf ${BUILDSYS_KITS_DIR}
+rm -rf ${BUILDSYS_EXTERNAL_KITS_DIR}
 '''
 ]
 

twoliter/src/cmd/fetch.rs

@@ -0,0 +1,24 @@
+use crate::lock::Lock;
+use crate::project;
+use anyhow::Result;
+use clap::Parser;
+use std::path::PathBuf;
+
+#[derive(Debug, Parser)]
+pub(crate) struct Fetch {
+    /// Path to Twoliter.toml. Will search for Twoliter.toml when absent
+    #[clap(long = "project-path")]
+    project_path: Option<PathBuf>,
+
+    #[clap(long = "arch", default_value = "x86_64")]
+    arch: String,
+}
+
+impl Fetch {
+    pub(super) async fn run(&self) -> Result<()> {
+        let project = project::load_or_find_project(self.project_path.clone()).await?;
+        let lock_file = Lock::load(&project).await?;
+        lock_file.fetch(&project, self.arch.as_str()).await?;
+        Ok(())
+    }
+}

twoliter/src/cmd/mod.rs

@@ -1,11 +1,13 @@
 mod build;
 mod build_clean;
 mod debug;
+mod fetch;
 mod make;
 mod update;
 
 use self::build::BuildCommand;
 use crate::cmd::debug::DebugAction;
+use crate::cmd::fetch::Fetch;
 use crate::cmd::make::Make;
 use crate::cmd::update::Update;
 use anyhow::Result;
@@ -35,6 +37,8 @@ pub(crate) enum Subcommand {
     #[clap(subcommand)]
     Build(BuildCommand),
 
+    Fetch(Fetch),
+
     Make(Make),
 
     /// Update Twoliter.lock
@@ -49,6 +53,7 @@ pub(crate) enum Subcommand {
 pub(super) async fn run(args: Args) -> Result<()> {
     match args.subcommand {
         Subcommand::Build(build_command) => build_command.run().await,
+        Subcommand::Fetch(fetch_args) => fetch_args.run().await,
         Subcommand::Make(make_args) => make_args.run().await,
         Subcommand::Update(update_args) => update_args.run().await,
         Subcommand::Debug(debug_action) => debug_action.run().await,