initial metadata store implementation

Cargo.toml

@@ -16,12 +16,16 @@ doc = false
 name = "functional"
 
 [dependencies]
+bincode = "*"
+bitflags = "*"
 rustc-serialize = "*"
 log = "*"
 env_logger = "*"
 ansi_term = "*"
 gpgme = "*"
 hyper = "*"
+lazy_static = "*"
+lmdb-sys = "*"
 tempdir = "*"
 toml = "*"
 regex = "*"

src/bldr/command/configure.rs

@@ -1,4 +1,4 @@
-// Copyright:: Copyright (c) 2015 Chef Software, Inc.
+// Copyright:: Copyright (c) 2015-2016 Chef Software, Inc.
 //
 // The terms of the Evaluation Agreement (Bldr) between Chef Software Inc. and the party accessing
 // this file ("Licensee") apply to Licensee's use of the Software until such time that the Software
@@ -29,11 +29,7 @@ use package::Package;
 /// * If the default.toml does not exist, or cannot be read
 /// * If we can't read the file into a string
 pub fn display(config: &Config) -> BldrResult<()> {
-    let package = try!(Package::load(config.deriv(),
-                                     config.package(),
-                                     config.version().clone(),
-                                     config.release().clone(),
-                                     None));
+    let package = try!(Package::load(config.package(), None));
     let mut file = try!(File::open(package.join_path("default.toml")));
     let mut s = String::new();
     try!(file.read_to_string(&mut s));

src/bldr/command/install.rs

@@ -46,11 +46,12 @@
 //!
 
 use std::fs;
+use std::str::FromStr;
 
 use fs::PACKAGE_CACHE;
 use error::BldrResult;
-use package::Package;
-use repo;
+use package::{Package, PackageIdent};
+use repo::{self, data_object};
 
 static LOGKEY: &'static str = "CI";
 
@@ -61,38 +62,30 @@ static LOGKEY: &'static str = "CI";
 ///
 /// * Fails if it cannot create `/opt/bldr/cache/pkgs`
 /// * Fails if it cannot download the package from the upstream
-pub fn from_url(repo: &str,
-                deriv: &str,
-                name: &str,
-                version: Option<String>,
-                release: Option<String>)
-                -> BldrResult<Package> {
-    let package = try!(repo::client::show_package(repo, deriv, name, version, release));
+pub fn from_url(repo: &str, ident: &PackageIdent) -> BldrResult<data_object::Package> {
+    let package = try!(repo::client::show_package(repo, ident));
     try!(fs::create_dir_all(PACKAGE_CACHE));
-    let mut installed: Vec<Package> = vec![];
-    if let Some(ref pkgs) = package.deps {
-        for pkg in pkgs {
-            try!(install(repo, &pkg, &mut installed));
-        }
+    let mut installed: Vec<PackageIdent> = vec![];
+    for dep in &package.deps {
+        let ident = try!(PackageIdent::from_str(&dep.to_string()));
+        try!(install(repo, &ident, &mut installed));
     }
-    try!(install(repo, &package, &mut installed));
+    try!(install(repo, &ident, &mut installed));
     Ok(package)
 }
 
-fn install(repo: &str, package: &Package, acc: &mut Vec<Package>) -> BldrResult<()> {
+fn install(repo: &str, package: &PackageIdent, acc: &mut Vec<PackageIdent>) -> BldrResult<()> {
     if acc.contains(&package) {
         return Ok(());
     }
-    let archive = try!(repo::client::fetch_package_exact(repo, package, PACKAGE_CACHE));
-    try!(archive.verify());
-    let package = try!(archive.unpack());
+    let archive = try!(repo::client::fetch_package(repo, package, PACKAGE_CACHE));
+    let package = try!(archive.ident());
+    let deps = try!(archive.deps());
+    try!(archive.unpack());
     outputln!("Installed {}", package);
-    let deps = package.deps.clone();
     acc.push(package);
-    if let Some(ref pkgs) = deps {
-        for pkg in pkgs {
-            try!(install(repo, &pkg, acc))
-        }
+    for dep in deps {
+        try!(install(repo, &dep, acc))
     }
     Ok(())
 }

src/bldr/command/key.rs

@@ -6,17 +6,19 @@
 
 use std::fs;
 use std::path::Path;
-use fs::KEY_CACHE;
-use config::Config;
-use error::{BldrResult, ErrorKind};
-use util::gpg;
-use regex::Regex;
+use std::process::{Command, Stdio, Child};
+
 use ansi_term::Colour::{Yellow, Red};
 use time::strptime;
 use rpassword::read_password;
-use package::Package;
-use std::process::{Command, Stdio, Child};
+use regex::Regex;
+
+use config::Config;
+use error::{BldrResult, ErrorKind};
+use fs::KEY_CACHE;
+use package::{Package, PackageIdent};
 use repo;
+use util::gpg;
 
 static LOGKEY: &'static str = "KU"; // "key utils"
 static USER_KEY_COMMENT: &'static str = "bldr user key";
@@ -298,7 +300,6 @@ fn gen_service_key_email(keyname: &str, group: &str) -> String {
 /// generate a user key name in the form of
 /// `bldr_keyname`. If a user key is already in the
 /// form BLDR_KEY_PREFIX.+, then just return it
-
 fn gen_user_key_name(keyname: &str) -> String {
     let re = String::from(BLDR_KEY_PREFIX) + ".+";
     let regex = Regex::new(&re).unwrap();
@@ -325,12 +326,11 @@ impl Drop for DroppableChildProcess {
     }
 }
 
-
 /// run rngd in the background to generate entropy while generating keys.
 /// The process is killed when it goes out of scope via `DroppableChildProcess`.
 fn run_rngd() -> BldrResult<DroppableChildProcess> {
     debug!("Spawning rngd in the background");
-    let res = try!(Package::load("chef", "rngd", None, None, None));
+    let res = try!(Package::load(&PackageIdent::new("chef", "rngd", None, None), None));
     let rngdpath = res.join_path("sbin/rngd");
     debug!("RNGD path = {}", rngdpath);
     let child = Command::new(rngdpath)

src/bldr/command/repo.rs

@@ -25,17 +25,78 @@
 //! Does the same, but the data is stored in `/tmp/whatever`.
 
 use config::Config;
-use error::BldrResult;
-use repo;
+use error::{BldrError, BldrResult, ErrorKind};
+use repo::{self, data_object};
+use repo::data_store::{self, Cursor, Database, Transaction};
 
 static LOGKEY: &'static str = "CR";
 
-/// Starts the repository.
+/// Create a repository with the given name in the database.
 ///
 /// # Failures
 ///
-/// * Fails if the repository fails to start - canot bind to the port, etc.
+/// * The database cannot be read
+/// * A write transaction cannot be acquired.
+pub fn create_repository(name: &str, config: &Config) -> BldrResult<()> {
+    let repo = try!(repo::Repo::new(String::from(config.path())));
+    let txn = try!(repo.datastore.views.txn_rw());
+    let object = data_object::View::new(name);
+    try!(repo.datastore.views.write(&txn, &object));
+    Ok(())
+}
+
+/// List all repositories in the database.
+///
+/// # Failures
+///
+/// * The database cannot be read
+/// * A read transaction cannot be acquired.
+pub fn list_repositories(config: &Config) -> BldrResult<()> {
+    let repo = try!(repo::Repo::new(String::from(config.path())));
+    let mut views: Vec<data_object::View> = vec![];
+    let txn = try!(repo.datastore.views.txn_ro());
+    let mut cursor = try!(txn.cursor_ro());
+    match cursor.first() {
+        Err(BldrError {err: ErrorKind::MdbError(data_store::MdbError::NotFound), ..}) => {
+            outputln!("No repositories. Create one with `bldr repo-create`.");
+            return Ok(());
+        }
+        Err(e) => return Err(e),
+        Ok(value) => views.push(value),
+    }
+    loop {
+        match cursor.next() {
+            Ok((_, value)) => views.push(value),
+            Err(_) => break,
+        }
+    }
+    outputln!("Listing {} repositories", views.len());
+    for view in views.iter() {
+        outputln!("     {}", view);
+    }
+    Ok(())
+}
+
+/// Starts the depot server.
+///
+/// # Failures
+///
+/// * Fails if the depot server fails to start - canot bind to the port, etc.
 pub fn start(config: &Config) -> BldrResult<()> {
     outputln!("Repo listening on {:?}", config.repo_addr());
     repo::run(&config)
 }
+
+/// Analyzes the integrity of the depot's metadata by comparing the metadata with the packages
+/// on disk. If a package is found on disk that is not present in the metadata it is added to the
+/// metadata and if an entry in the metadata doesn't have a matching package archive on disk the
+/// entry is dropped from the database.
+///
+/// # Failures
+///
+/// * The database cannot be read
+/// * A write transaction cannot be acquired
+pub fn repair(config: &Config) -> BldrResult<()> {
+    outputln!("Repairing repo at {:?}", config.path());
+    repo::repair(&config)
+}

src/bldr/command/start.rs

@@ -55,7 +55,7 @@ use std::env;
 use fs::PACKAGE_CACHE;
 use error::{BldrResult, ErrorKind};
 use config::Config;
-use package::Package;
+use package::{Package, PackageIdent};
 use topology::{self, Topology};
 use command::install;
 use repo;
@@ -71,22 +71,26 @@ static LOGKEY: &'static str = "CS";
 /// * Fails if the `run` method for the topology fails
 /// * Fails if an unknown topology was specified on the command line
 pub fn package(config: &Config) -> BldrResult<()> {
-    match Package::load(config.deriv(), config.package(), None, None, None) {
-        Ok(mut package) => {
+    match Package::load(config.package(), None) {
+        Ok(package) => {
             if let Some(ref url) = *config.url() {
                 outputln!("Checking remote for newer versions...");
-                let latest_pkg = try!(repo::client::show_package(&url,
-                                                                 &package.derivation,
-                                                                 &package.name,
-                                                                 None,
-                                                                 None));
+                // It is important to pass `config.package()` to `show_package()` instead of the
+                // package identifier of the loaded package. This will ensure that if the operator
+                // starts a package while specifying a version number, they will only automaticaly
+                // receive release updates for the started package.
+                //
+                // If the operator does not specify a version number they will automatically receive
+                // updates for any releases, regardless of version number, for the started  package.
+                let latest_pkg: Package = try!(repo::client::show_package(&url, config.package()))
+                                              .into();
                 if latest_pkg > package {
                     outputln!("Downloading latest version from remote: {}", &latest_pkg);
-                    let archive = try!(repo::client::fetch_package_exact(&url,
-                                                                         &latest_pkg,
-                                                                         PACKAGE_CACHE));
+                    let archive = try!(repo::client::fetch_package(&url,
+                                                                   &PackageIdent::from(latest_pkg),
+                                                                   PACKAGE_CACHE));
                     try!(archive.verify());
-                    package = try!(archive.unpack());
+                    try!(archive.unpack());
                 } else {
                     outputln!("Already running latest.");
                 };
@@ -95,25 +99,16 @@ pub fn package(config: &Config) -> BldrResult<()> {
         }
         Err(_) => {
             outputln!("{} not found in local cache",
-                      Yellow.bold().paint(config.package()));
+                      Yellow.bold().paint(config.package().to_string()));
             match *config.url() {
                 Some(ref url) => {
                     outputln!("Searching for {} in remote {}",
-                              Yellow.bold().paint(config.package()),
+                              Yellow.bold().paint(config.package().to_string()),
                               url);
-                    let package = try!(install::from_url(url,
-                                                         config.deriv(),
-                                                         config.package(),
-                                                         config.version().clone(),
-                                                         config.release().clone()));
+                    let package: Package = try!(install::from_url(url, config.package())).into();
                     start_package(package, config)
                 }
-                None => {
-                    Err(bldr_error!(ErrorKind::PackageNotFound(config.deriv().to_string(),
-                                                               config.package().to_string(),
-                                                               config.release().clone(),
-                                                               config.release().clone())))
-                }
+                None => Err(bldr_error!(ErrorKind::PackageNotFound(config.package().clone()))),
             }
         }
     }