diff --git a/Cargo.lock b/Cargo.lock index 082dcabb3..b58f3500c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -745,6 +745,7 @@ name = "buildsys" version = "0.1.0" dependencies = [ "bottlerocket-variant", + "buildsys-config", "clap", "duct", "guppy", @@ -765,6 +766,14 @@ dependencies = [ "walkdir", ] +[[package]] +name = "buildsys-config" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", +] + [[package]] name = "bumpalo" version = "3.15.4" @@ -3782,6 +3791,7 @@ dependencies = [ "async-walkdir", "base64 0.22.0", "buildsys", + "buildsys-config", "bytes", "clap", "env_logger", @@ -3791,6 +3801,7 @@ dependencies = [ "hex", "log", "non-empty-string", + "olpc-cjson", "pubsys", "pubsys-setup", "semver", diff --git a/Cargo.toml b/Cargo.toml index c935eb3a7..e8d2a18bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ resolver = "2" members = [ "tools/bottlerocket-variant", "tools/buildsys", + "tools/buildsys-config", "tools/parse-datetime", "tools/pubsys", "tools/pubsys-config", diff --git a/tools/buildsys-config/Cargo.toml b/tools/buildsys-config/Cargo.toml new file mode 100644 index 000000000..46af8d05a --- /dev/null +++ b/tools/buildsys-config/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "buildsys-config" +version = "0.1.0" +authors = ["Jarrett Tierney "] +license = "MIT OR Apache-2.0" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"]} diff --git a/tools/buildsys-config/src/lib.rs b/tools/buildsys-config/src/lib.rs new file mode 100644 index 000000000..7b68b3ec5 --- /dev/null +++ b/tools/buildsys-config/src/lib.rs @@ -0,0 +1,33 @@ +use anyhow::anyhow; +use serde::Deserialize; +use std::fmt::{Display, Formatter}; + +pub const EXTERNAL_KIT_DIRECTORY: &str = "build/external-kits"; +pub const EXTERNAL_KIT_METADATA: &str = "build/external-kits/external-kit-metadata.json"; + +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub enum DockerArchitecture { + Amd64, + Arm64, +} + +impl TryFrom<&str> for DockerArchitecture { + type Error = anyhow::Error; + + fn try_from(value: &str) -> std::result::Result { + match value { + "x86_64" | "amd64" => Ok(DockerArchitecture::Amd64), + "aarch64" | "arm64" => Ok(DockerArchitecture::Arm64), + _ => Err(anyhow!("invalid architecture '{}'", value)), + } + } +} + +impl Display for DockerArchitecture { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Amd64 => "amd64", + Self::Arm64 => "arm64", + }) + } +} diff --git a/tools/buildsys/Cargo.toml b/tools/buildsys/Cargo.toml index 5c29ebc3f..acb7d24dc 100644 --- a/tools/buildsys/Cargo.toml +++ b/tools/buildsys/Cargo.toml @@ -10,6 +10,7 @@ exclude = ["README.md"] [dependencies] bottlerocket-variant = { version = "0.1", path = "../bottlerocket-variant" } +buildsys-config = { version = "0.1", path = "../buildsys-config" } clap = { version = "4", features = ["derive", "env"] } duct = "0.13" guppy = "0.17" diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 2098e3770..9a16cb0da 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -20,6 +20,7 @@ use crate::args::{ }; use crate::builder::DockerBuild; use buildsys::manifest::{BundleModule, ImageFeature, Manifest, ManifestInfo, SupportedArch}; +use buildsys_config::EXTERNAL_KIT_METADATA; use cache::LookasideCache; use clap::Parser; use gomod::GoMod; @@ -124,6 +125,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> { let manifest_file = "Cargo.toml"; let manifest_path = args.common.cargo_manifest_dir.join(manifest_file); println!("cargo:rerun-if-changed={}", manifest_file); + println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA); let manifest = Manifest::new(&manifest_path, &args.common.cargo_metadata_path) .context(error::ManifestParseSnafu)?; @@ -205,6 +207,7 @@ fn build_package(args: BuildPackageArgs) -> Result<()> { fn build_kit(args: BuildKitArgs) -> Result<()> { let manifest_file = "Cargo.toml"; println!("cargo:rerun-if-changed={}", manifest_file); + println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA); let manifest = Manifest::new( args.common.cargo_manifest_dir.join(manifest_file), @@ -221,6 +224,7 @@ fn build_kit(args: BuildKitArgs) -> Result<()> { fn build_variant(args: BuildVariantArgs) -> Result<()> { let manifest_file = "Cargo.toml"; println!("cargo:rerun-if-changed={}", manifest_file); + println!("cargo:rerun-if-changed={}", EXTERNAL_KIT_METADATA); let manifest = Manifest::new( args.common.cargo_manifest_dir.join(manifest_file), diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 7f57a8507..ba7085a49 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -14,6 +14,7 @@ anyhow = "1" async-recursion = "1" async-walkdir = "1" base64 = "0.22" +buildsys-config = { version = "0.1", path = "../tools/buildsys-config" } clap = { version = "4", features = ["derive", "env", "std"] } env_logger = "0.11" filetime = "0.2" @@ -22,6 +23,7 @@ futures= "0.3" hex = "0.4" log = "0.4" non-empty-string = { version = "0.2", features = [ "serde" ] } +olpc-cjson = "0.1" semver = { version = "1", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 1ce82093f..90c1a1aa3 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -12,6 +12,7 @@ BUILDSYS_ROOT_DIR = "${CARGO_MAKE_WORKING_DIRECTORY}" BUILDSYS_BUILD_DIR = "${BUILDSYS_ROOT_DIR}/build" BUILDSYS_PACKAGES_DIR = "${BUILDSYS_BUILD_DIR}/rpms" BUILDSYS_KITS_DIR = "${BUILDSYS_BUILD_DIR}/kits" +BUILDSYS_EXTERNAL_KITS_DIR = "${BUILDSYS_BUILD_DIR}/external-kits" BUILDSYS_STATE_DIR = "${BUILDSYS_BUILD_DIR}/state" BUILDSYS_IMAGES_DIR = "${BUILDSYS_BUILD_DIR}/images" BUILDSYS_LOGS_DIR = "${BUILDSYS_BUILD_DIR}/logs" @@ -280,6 +281,7 @@ mkdir -p ${BUILDSYS_BUILD_DIR} mkdir -p ${BUILDSYS_OUTPUT_DIR} mkdir -p ${BUILDSYS_PACKAGES_DIR} mkdir -p ${BUILDSYS_KITS_DIR} +mkdir -p ${BUILDSYS_EXTERNAL_KITS_DIR} mkdir -p ${BUILDSYS_STATE_DIR} mkdir -p ${BUILDSYS_METADATA_DIR} mkdir -p ${GO_MOD_CACHE} @@ -1642,6 +1644,7 @@ script_runner = "bash" script = [ ''' rm -rf ${BUILDSYS_KITS_DIR} +rm -rf ${BUILDSYS_EXTERNAL_KITS_DIR} ''' ] diff --git a/twoliter/src/cmd/fetch.rs b/twoliter/src/cmd/fetch.rs new file mode 100644 index 000000000..1ec5d9e0c --- /dev/null +++ b/twoliter/src/cmd/fetch.rs @@ -0,0 +1,24 @@ +use crate::lock::Lock; +use crate::project; +use anyhow::Result; +use clap::Parser; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +pub(crate) struct Fetch { + /// Path to Twoliter.toml. Will search for Twoliter.toml when absent + #[clap(long = "project-path")] + project_path: Option, + + #[clap(long = "arch", default_value = "x86_64")] + arch: String, +} + +impl Fetch { + pub(super) async fn run(&self) -> Result<()> { + let project = project::load_or_find_project(self.project_path.clone()).await?; + let lock_file = Lock::load(&project).await?; + lock_file.fetch(&project, self.arch.as_str()).await?; + Ok(()) + } +} diff --git a/twoliter/src/cmd/mod.rs b/twoliter/src/cmd/mod.rs index c77a39802..fd56394a9 100644 --- a/twoliter/src/cmd/mod.rs +++ b/twoliter/src/cmd/mod.rs @@ -1,11 +1,13 @@ mod build; mod build_clean; mod debug; +mod fetch; mod make; mod update; use self::build::BuildCommand; use crate::cmd::debug::DebugAction; +use crate::cmd::fetch::Fetch; use crate::cmd::make::Make; use crate::cmd::update::Update; use anyhow::Result; @@ -35,6 +37,8 @@ pub(crate) enum Subcommand { #[clap(subcommand)] Build(BuildCommand), + Fetch(Fetch), + Make(Make), /// Update Twoliter.lock @@ -49,6 +53,7 @@ pub(crate) enum Subcommand { pub(super) async fn run(args: Args) -> Result<()> { match args.subcommand { Subcommand::Build(build_command) => build_command.run().await, + Subcommand::Fetch(fetch_args) => fetch_args.run().await, Subcommand::Make(make_args) => make_args.run().await, Subcommand::Update(update_args) => update_args.run().await, Subcommand::Debug(debug_action) => debug_action.run().await, diff --git a/twoliter/src/lock.rs b/twoliter/src/lock.rs index 4fdac3569..67d91ea5d 100644 --- a/twoliter/src/lock.rs +++ b/twoliter/src/lock.rs @@ -1,33 +1,38 @@ -use crate::common::fs::{remove_file, write}; -use crate::project::{Image, Project, Vendor}; +use crate::common::fs::{create_dir_all, read, remove_dir_all, remove_file, write}; +use crate::project::{Image, Project, ValidIdentifier, Vendor}; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use base64::Engine; +use buildsys_config::DockerArchitecture; +use olpc_cjson::CanonicalFormatter as CanonicalJsonFormatter; use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize}; +use sha2::Digest; +use std::cmp::PartialEq; use std::collections::{HashMap, HashSet}; -use std::fmt::Display; +use std::fmt::{Display, Formatter}; +use std::fs::File; use std::hash::{Hash, Hasher}; use std::mem::take; +use std::path::{Path, PathBuf}; +use tar::Archive as TarArchive; +use tempfile::TempDir; use tokio::fs::read_to_string; use tokio::process::Command; const TWOLITER_LOCK: &str = "Twoliter.lock"; -/// Represents the structure of a `Twoliter.lock` lock file. -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) struct Lock { - /// The version of the Twoliter.toml this was generated from - pub schema_version: SchemaVersion<1>, - /// The workspace release version - pub release_version: String, - /// The resolved bottlerocket sdk - pub sdk: LockedImage, - /// Resolved kit dependencies - pub kit: Vec, - /// sha256 digest of the Project this was generated from - pub digest: String, +macro_rules! docker { + ($arg: expr, $error_msg: expr) => {{ + let output = Command::new("docker") + .args($arg) + .output() + .await + .context($error_msg)?; + ensure!(output.status.success(), $error_msg); + output.stdout + }}; } /// Represents a locked dependency on an image @@ -41,16 +46,38 @@ pub(crate) struct LockedImage { pub vendor: String, /// The resolved image uri of the dependency pub source: String, + /// The digest of the image + pub digest: String, + #[serde(skip)] + manifest: Vec, } impl LockedImage { - pub fn new(vendor: &Vendor, image: &Image) -> Self { - Self { - name: image.name.clone(), + pub async fn new(vendor: &Vendor, image: &Image) -> Result { + let source = format!("{}/{}:v{}", vendor.registry, image.name, image.version); + let manifest_bytes = docker!( + ["manifest", "inspect", source.as_str()], + format!("failed to inspect manifest of resource at {}", source) + ); + + // We calculate a 'digest' of the manifest to use as our unique id + let digest = sha2::Sha256::digest(manifest_bytes.as_slice()); + let digest = base64::engine::general_purpose::STANDARD.encode(digest.as_slice()); + Ok(Self { + name: image.name.to_string(), version: image.version.clone(), - vendor: image.vendor.clone(), - source: format!("{}/{}:v{}", vendor.registry, image.name, image.version), - } + vendor: image.vendor.to_string(), + source, + digest, + manifest: manifest_bytes, + }) + } + + pub fn digest_uri(&self, digest: &str) -> String { + self.source.replace( + format!(":v{}", self.version).as_str(), + format!("@{}", digest).as_str(), + ) } } @@ -92,21 +119,186 @@ struct ManifestListView { manifests: Vec, } -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Debug, Clone)] struct ManifestView { digest: String, + platform: Option, } -macro_rules! docker { - ($arg: expr, $error_msg: expr) => {{ - let output = Command::new("docker") - .args($arg) - .output() +#[derive(Deserialize, Debug, Clone)] +struct Platform { + architecture: DockerArchitecture, +} + +#[derive(Deserialize, Debug)] +struct IndexView { + manifests: Vec, +} + +#[derive(Deserialize, Debug)] +struct ManifestLayoutView { + layers: Vec, +} + +#[derive(Deserialize, Debug)] +struct Layer { + digest: ContainerDigest, +} + +#[derive(Debug)] +struct ContainerDigest(String); + +impl<'de> Deserialize<'de> for ContainerDigest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let digest = String::deserialize(deserializer)?; + if !digest.starts_with("sha256:") { + return Err(D::Error::custom(format!( + "invalid digest detected in layer: {}", + digest + ))); + }; + Ok(Self(digest)) + } +} + +impl Display for ContainerDigest { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0.as_str()) + } +} + +#[derive(Serialize, Debug)] +struct ExternalKitMetadata { + sdk: LockedImage, + #[serde(rename = "kit")] + kits: Vec, +} + +#[derive(Debug)] +struct OCIArchive { + image: LockedImage, + digest: String, + cache_dir: PathBuf, +} + +impl OCIArchive { + fn new

(image: &LockedImage, digest: &str, cache_dir: P) -> Self + where + P: AsRef, + { + Self { + image: image.clone(), + digest: digest.into(), + cache_dir: cache_dir.as_ref().to_path_buf(), + } + } + + fn archive_path(&self) -> PathBuf { + self.cache_dir.join(self.digest.replace(':', "-")) + } + + async fn pull_image(&self) -> Result<()> { + let digest_uri = self.image.digest_uri(self.digest.as_str()); + let oci_archive_path = self.archive_path(); + if !oci_archive_path.exists() { + let oci_archive_str = oci_archive_path.to_string_lossy(); + docker!( + ["save", digest_uri.as_str(), "-o", oci_archive_str.as_ref()], + format!( + "failed to fetch kit and save to disk from {} to {}", + digest_uri, oci_archive_str + ) + ); + } + Ok(()) + } + + async fn unpack_layers

(&self, out_dir: P) -> Result<()> + where + P: AsRef, + { + let path = out_dir.as_ref(); + let digest_file = path.join("digest"); + ensure!(digest_file.exists(), "digest file '{}' does not exist, most likely because the oci archive has not been pulled", digest_file.display()); + let digest = read_to_string(&digest_file).await.context(format!( + "failed to read digest file at {}", + digest_file.display() + ))?; + if digest == self.digest { + return Ok(()); + } + + remove_dir_all(path).await?; + create_dir_all(path).await?; + let oci_archive_path = self.archive_path(); + let mut oci_file = File::open(&oci_archive_path).context(format!( + "failed to open oci archive at {}", + oci_archive_path.display() + ))?; + let mut oci_archive = TarArchive::new(&mut oci_file); + let temp_dir = TempDir::new_in(path).context(format!( + "failed to create temporary directory inside {}", + path.display() + ))?; + oci_archive + .unpack(temp_dir.path()) + .context("failed to unpack oci image")?; + let index_bytes = read(path.join("index.json")).await?; + let index: IndexView = serde_json::from_slice(index_bytes.as_slice()) + .context("failed to deserialize oci image index")?; + + // Read the manifest so we can get the layer digests + let digest = index + .manifests + .first() + .context("empty oci image")? + .digest + .replace(':', "/"); + let manifest_bytes = read(temp_dir.path().join(format!("blobs/{digest}"))) .await - .context($error_msg)?; - ensure!(output.status.success(), $error_msg); - output.stdout - }}; + .context("failed to read manifest blob")?; + let manifest_layout: ManifestLayoutView = serde_json::from_slice(manifest_bytes.as_slice()) + .context("failed to deserialize oci manifest")?; + + // Extract each layer into the target directory + for layer in manifest_layout.layers { + let digest = layer.digest.to_string().replace(':', "/"); + let mut layer_blob = File::open(temp_dir.path().join(format!("blobs/{digest}"))) + .context("failed to read layer of oci image")?; + let decompressed = flate2::read::GzDecoder::new(&mut layer_blob); + let mut layer_archive = TarArchive::new(decompressed); + layer_archive + .unpack(path) + .context("failed to unpack layer to disk")?; + } + write(&digest_file, self.digest.as_str()) + .await + .context(format!( + "failed to record digest to {}", + digest_file.display() + ))?; + + Ok(()) + } +} + +/// Represents the structure of a `Twoliter.lock` lock file. +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Lock { + /// The version of the Twoliter.toml this was generated from + pub schema_version: SchemaVersion<1>, + /// The workspace release version + pub release_version: String, + /// The resolved bottlerocket sdk + pub sdk: LockedImage, + /// Resolved kit dependencies + pub kit: Vec, + /// sha256 digest of the Project this was generated from + pub digest: String, } #[allow(dead_code)] @@ -139,15 +331,93 @@ impl Lock { Ok(lock) } + fn external_kit_metadata(&self) -> ExternalKitMetadata { + ExternalKitMetadata { + sdk: self.sdk.clone(), + kits: self.kit.clone(), + } + } + + /// Fetches all external kits defined in a Twoliter.lock to the build directory + pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { + let target_dir = project.external_kits_dir(); + create_dir_all(&target_dir).await.context(format!( + "failed to create external-kits directory at {}", + target_dir.display() + ))?; + for image in self.kit.iter() { + self.extract_kit(&project.external_kits_dir(), image, arch) + .await?; + } + let mut kit_list = Vec::new(); + let mut ser = + serde_json::Serializer::with_formatter(&mut kit_list, CanonicalJsonFormatter::new()); + self.external_kit_metadata() + .serialize(&mut ser) + .context("failed to serialize external kit metadata")?; + write(project.external_kits_metadata(), kit_list.as_slice()) + .await + .context(format!( + "failed to write external kit metadata: {}", + project.external_kits_metadata().display() + ))?; + + Ok(()) + } + + async fn get_manifest(&self, image: &LockedImage, arch: &str) -> Result { + let manifest_bytes = docker!( + ["manifest", "inspect", image.source.as_str()], + format!("failed to find a kit {}", image.to_string()) + ); + let manifest_list: ManifestListView = serde_json::from_slice(manifest_bytes.as_slice()) + .context("failed to deserialize manifest list")?; + let docker_arch = DockerArchitecture::try_from(arch)?; + manifest_list + .manifests + .iter() + .find(|x| x.platform.as_ref().unwrap().architecture == docker_arch) + .cloned() + .context(format!( + "could not find kit image for architecture '{}' at {}", + docker_arch, image.source + )) + } + + async fn extract_kit

(&self, path: P, image: &LockedImage, arch: &str) -> Result<()> + where + P: AsRef, + { + let vendor = image.vendor.clone(); + let name = image.name.clone(); + let target_path = path.as_ref().join(format!("{vendor}/{name}/{arch}")); + let cache_path = path.as_ref().join("cache"); + create_dir_all(&target_path).await?; + create_dir_all(&cache_path).await?; + + // First get the manifest for the specific requested architecture + let manifest = self.get_manifest(image, arch).await?; + let oci_archive = OCIArchive::new(image, manifest.digest.as_str(), &cache_path); + + // Checks for the saved image locally, or else pulls and saves it + oci_archive.pull_image().await?; + + // Checks if this archive has already been extracted by checking a digest file + // otherwise cleans up the path and unpacks the archive + oci_archive.unpack_layers(&target_path).await?; + + Ok(()) + } + async fn resolve(project: &Project) -> Result { let vendor_table = project.vendor(); - let mut known: HashMap<(String, String), Version> = HashMap::new(); + let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); let mut remaining: Vec = project.kits(); let mut sdk_set: HashSet = HashSet::new(); if let Some(sdk) = project.sdk_image() { - remaining.push(sdk.clone()); + // We don't scan over the sdk images as they are not kit images and there is no kit metadata to fetch sdk_set.insert(sdk.clone()); } while !remaining.is_empty() { @@ -171,7 +441,7 @@ impl Lock { (image.name.clone(), image.vendor.clone()), image.version.clone(), ); - let locked_image = LockedImage::new(vendor, image); + let locked_image = LockedImage::new(vendor, image).await?; let kit = Self::find_kit(vendor, &locked_image).await?; locked.push(locked_image); sdk_set.insert(kit.sdk); @@ -197,42 +467,18 @@ impl Lock { "vendor '{}' is not specified in Twoliter.toml", sdk.vendor ))?; - Ok(Self { schema_version: project.schema_version(), release_version: project.release_version().to_string(), digest: project.digest()?, - sdk: LockedImage::new(vendor, sdk), + sdk: LockedImage::new(vendor, sdk).await?, kit: locked, }) } - async fn resolve_kit( - vendors: &HashMap, - image: &Image, - ) -> Result { - let vendor = vendors.get(&image.vendor).context(format!( - "no vendor '{}' specified in Twoliter.toml", - image.vendor - ))?; - let locked_image = LockedImage { - name: image.name.clone(), - version: image.version.clone(), - vendor: image.vendor.clone(), - source: format!("{}/{}:v{}", vendor.registry, image.name, image.version), - }; - Self::find_kit(vendor, &locked_image).await - } - async fn find_kit(vendor: &Vendor, image: &LockedImage) -> Result { - // Now inspect the manifest list - let manifest_bytes = docker!( - ["manifest", "inspect", image.source.as_str()], - format!("failed to find a kit {}", image.to_string()) - ); - let manifest_list: ManifestListView = serde_json::from_slice(manifest_bytes.as_slice()) + let manifest_list: ManifestListView = serde_json::from_slice(image.manifest.as_slice()) .context("failed to deserialize manifest list")?; - let mut encoded_metadata: Option = None; for manifest in manifest_list.manifests.iter() { let image_uri = format!("{}/{}@{}", vendor.registry, image.name, manifest.digest); @@ -243,7 +489,8 @@ impl Lock { "image", "inspect", image_uri.as_str(), - "--format \"{{ json .Config.Labels }}\"", + "--format", + "\"{{ json .Config.Labels }}\"", ], format!( "failed to fetch kit metadata for {} with digest {}", @@ -251,12 +498,11 @@ impl Lock { manifest.digest ) ); - // Otherwise we should have a list of json blobs we can fetch the metadata from the label - let labels: HashMap = serde_json::from_slice(label_bytes.as_slice()) - .context(format!( - "could not deserialize labels on the image for {}", - image - ))?; + let label_str = String::from_utf8_lossy(label_bytes.as_slice()).to_string(); + let label_str = label_str.trim().trim_matches('"'); + let labels: HashMap = serde_json::from_str(label_str).context( + format!("could not deserialize labels on the image for {}", image), + )?; let encoded = labels .get("dev.bottlerocket.kit.v1") .context("no metadata stored on image, this image appears to not be a kit")?; diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index 21868a7df..d90d2a127 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -5,13 +5,16 @@ use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; use async_walkdir::WalkDir; use base64::Engine; +use buildsys_config::{EXTERNAL_KIT_DIRECTORY, EXTERNAL_KIT_METADATA}; use futures::stream::StreamExt; use log::{debug, info, trace, warn}; use semver::Version; -use serde::{Deserialize, Serialize}; +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use sha2::{Digest, Sha256, Sha512}; use std::collections::BTreeMap; use std::ffi::OsStr; +use std::fmt::{Display, Formatter}; use std::hash::Hash; use std::io::Write; use std::path::{Path, PathBuf}; @@ -51,7 +54,7 @@ pub(crate) struct Project { sdk: Option, /// Set of vendors - vendor: BTreeMap, + vendor: BTreeMap, /// Set of kit dependencies kit: Vec, @@ -105,6 +108,14 @@ impl Project { self.project_dir.clone() } + pub(crate) fn external_kits_dir(&self) -> PathBuf { + self.project_dir.join(EXTERNAL_KIT_DIRECTORY) + } + + pub(crate) fn external_kits_metadata(&self) -> PathBuf { + self.project_dir.join(EXTERNAL_KIT_METADATA) + } + pub(crate) fn schema_version(&self) -> SchemaVersion<1> { self.schema_version } @@ -113,7 +124,7 @@ impl Project { self.release_version.as_str() } - pub(crate) fn vendor(&self) -> &BTreeMap { + pub(crate) fn vendor(&self) -> &BTreeMap { &self.vendor } @@ -133,7 +144,7 @@ impl Project { ))?; Ok(Some(ImageUri::new( Some(vendor.registry.clone()), - sdk.name.clone(), + sdk.name.to_string(), format!("v{}", sdk.version), ))) } else { @@ -143,14 +154,14 @@ impl Project { #[allow(unused)] pub(crate) fn kit(&self, name: &str) -> Result> { - if let Some(kit) = self.kit.iter().find(|y| y.name == name) { + if let Some(kit) = self.kit.iter().find(|y| y.name.to_string() == name) { let vendor = self.vendor.get(&kit.vendor).context(format!( "vendor '{}' was not specified in Twoliter.toml", kit.vendor ))?; Ok(Some(ImageUri::new( Some(vendor.registry.clone()), - kit.name.clone(), + kit.name.to_string(), format!("v{}", kit.version), ))) } else { @@ -218,25 +229,25 @@ impl Project { hash.write(self.release_version.as_bytes()) .context("failed to encode release version in hash")?; for (key, value) in self.vendor.iter() { - hash.write(key.as_bytes()) + hash.write(key.to_string().as_bytes()) .context("failed to encode vendor name in hash")?; hash.write(value.registry.as_bytes()) .context("failed to encode vendor registry in hash")?; } if let Some(sdk) = self.sdk.as_ref() { - hash.write(sdk.name.as_bytes()) + hash.write(sdk.name.to_string().as_bytes()) .context("failed to encode sdk name in hash")?; hash.write(sdk.version.to_string().as_bytes()) .context("failed to encode sdk version in hash")?; - hash.write(sdk.vendor.as_bytes()) + hash.write(sdk.vendor.to_string().as_bytes()) .context("failed to encode sdk vendor in hash")?; } for kit in self.kit.iter() { - hash.write(kit.name.as_bytes()) + hash.write(kit.name.to_string().as_bytes()) .context("failed to encode kit name in hash")?; hash.write(kit.version.to_string().as_bytes()) .context("failed to encode kit version in hash")?; - hash.write(kit.vendor.as_bytes()) + hash.write(kit.vendor.to_string().as_bytes()) .context("failed to encode kit vendor in hash")?; } let project_hash = hash.finalize(); @@ -252,13 +263,66 @@ pub(crate) struct Vendor { pub registry: String, } +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct ValidIdentifier(String); + +impl Serialize for ValidIdentifier { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + serializer.serialize_str(self.0.as_str()) + } +} + +impl<'de> Deserialize<'de> for ValidIdentifier { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let input = String::deserialize(deserializer)?; + // Check if the input is empty + if input.is_empty() { + return Err(D::Error::custom( + "cannot define an identifier as an empty string", + )); + } + + // Check if the input contains any invalid characters + for c in input.chars() { + if !is_valid_id_char(c) { + return Err(D::Error::custom(format!( + "invalid character '{}' found in identifier name", + c + ))); + } + } + Ok(Self(input.clone())) + } +} + +impl Display for ValidIdentifier { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0.as_str()) + } +} + +fn is_valid_id_char(c: char) -> bool { + match c { + // Allow alphanumeric characters, underscores, and hyphens + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => true, + // Disallow other characters + _ => false, + } +} + /// This represents a dependency on a container, primarily used for kits #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] #[serde(rename_all = "kebab-case")] pub(crate) struct Image { - pub name: String, + pub name: ValidIdentifier, pub version: Version, - pub vendor: String, + pub vendor: ValidIdentifier, } /// This is used to `Deserialize` a project, then run validation code before returning a valid @@ -271,7 +335,7 @@ struct UnvalidatedProject { schema_version: SchemaVersion<1>, release_version: String, sdk: Option, - vendor: Option>, + vendor: Option>, kit: Option>, } @@ -384,21 +448,27 @@ mod test { // Add checks here as desired to validate deserialization. assert_eq!(SchemaVersion::<1>, deserialized.schema_version); assert_eq!(1, deserialized.vendor.len()); - assert!(deserialized.vendor.contains_key("my-vendor")); + assert!(deserialized + .vendor + .contains_key(&ValidIdentifier("my-vendor".to_string()))); assert_eq!( "a.com/b", - deserialized.vendor.get("my-vendor").unwrap().registry + deserialized + .vendor + .get(&ValidIdentifier("my-vendor".to_string())) + .unwrap() + .registry ); let sdk = deserialized.sdk.unwrap(); - assert_eq!("my-bottlerocket-sdk", sdk.name.as_str()); + assert_eq!("my-bottlerocket-sdk", sdk.name.to_string()); assert_eq!(Version::new(1, 2, 3), sdk.version); - assert_eq!("my-vendor", sdk.vendor.as_str()); + assert_eq!("my-vendor", sdk.vendor.to_string()); assert_eq!(1, deserialized.kit.len()); - assert_eq!("my-core-kit", deserialized.kit[0].name.as_str()); + assert_eq!("my-core-kit", deserialized.kit[0].name.to_string()); assert_eq!(Version::new(1, 2, 3), deserialized.kit[0].version); - assert_eq!("my-vendor", deserialized.kit[0].vendor.as_str()); + assert_eq!("my-vendor", deserialized.kit[0].vendor.to_string()); } /// Ensure that a `Twoliter.toml` cannot be serialized if the `schema_version` is incorrect. @@ -438,12 +508,12 @@ mod test { schema_version: Default::default(), release_version: String::from("1.0.0"), sdk: Some(Image { - name: "foo-abc".into(), + name: ValidIdentifier("foo-abc".to_string()), version: Version::new(1, 2, 3), - vendor: "example".into(), + vendor: ValidIdentifier("example".into()), }), vendor: BTreeMap::from([( - "example".into(), + ValidIdentifier("example".into()), Vendor { registry: "example.com".into(), }, @@ -486,20 +556,20 @@ mod test { schema_version: SchemaVersion::default(), release_version: "1.0.0".into(), sdk: Some(Image { - name: "bottlerocket-sdk".into(), + name: ValidIdentifier("bottlerocket-sdk".into()), version: Version::new(1, 41, 1), - vendor: "bottlerocket".into(), + vendor: ValidIdentifier("bottlerocket".into()), }), vendor: Some(BTreeMap::from([( - "not-bottlerocket".into(), + ValidIdentifier("not-bottlerocket".into()), Vendor { registry: "public.ecr.aws/not-bottlerocket".into(), }, )])), kit: Some(vec![Image { - name: "bottlerocket-core-kit".into(), + name: ValidIdentifier("bottlerocket-core-kit".into()), version: Version::new(1, 20, 0), - vendor: "not-bottlerocket".into(), + vendor: ValidIdentifier("not-bottlerocket".into()), }]), }; assert!(project.check_vendor_availability().await.is_err());