Add fs_util gc (pantsbuild#6612)

Note that this may break all existing open Environments

src/rust/engine/fs/Cargo.toml

@@ -20,16 +20,17 @@ ignore = "0.3.1"
 indexmap = "1"
 itertools = "0.7.2"
 lazy_static = "1"
-lmdb = "0.8"
+lmdb = { git = "https://github.com/pantsbuild/lmdb-rs.git", rev = "06bdfbfc6348f6804127176e561843f214fc17f8" }
 log = "0.4"
 parking_lot = "0.6"
 protobuf = { version = "2.0.4", features = ["with-bytes"] }
 sha2 = "0.7"
 serde = "1.0"
 serde_derive = "1.0"
+tempfile = "3"
 uuid = { version = "0.7.1", features = ["v4"] }
 
 [dev-dependencies]
 mock = { path = "../testutil/mock" }
-tempfile = "3"
 testutil = { path = "../testutil" }
+walkdir = "2"

src/rust/engine/fs/fs_util/src/main.rs

@@ -182,6 +182,16 @@ to this directory.",
             true,
           )),
       )
+        .subcommand(
+          SubCommand::with_name("gc")
+              .about("Garbage collect the on-disk store. Note that after running this command, any processes with an open store (e.g. a pantsd) may need to re-initialize their store.")
+              .arg(
+                Arg::with_name("target-size-bytes")
+                    .takes_value(true)
+                    .long("target-size-bytes")
+                    .required(true),
+              )
+        )
       .arg(
         Arg::with_name("local-store-path")
           .takes_value(true)
@@ -482,6 +492,12 @@ fn execute(top_match: &clap::ArgMatches) -> Result<(), ExitError> {
         )),
       }
     }
+    ("gc", Some(args)) => {
+      let target_size_bytes = value_t!(args.value_of("target-size-bytes"), usize)
+        .expect("--target-size-bytes must be passed as a non-negative integer");
+      store.garbage_collect(target_size_bytes, fs::ShrinkBehavior::Compact)?;
+      Ok(())
+    }
 
     (_, _) => unimplemented!(),
   }

src/rust/engine/fs/src/lib.rs

@@ -37,7 +37,7 @@ pub use snapshot::{
   OneOffStoreFileByDigest, Snapshot, StoreFileByDigest, EMPTY_DIGEST, EMPTY_FINGERPRINT,
 };
 mod store;
-pub use store::{Store, UploadSummary};
+pub use store::{ShrinkBehavior, Store, UploadSummary, DEFAULT_LOCAL_STORE_GC_TARGET_BYTES};
 mod pool;
 pub use pool::ResettablePool;
 
@@ -67,13 +67,14 @@ extern crate parking_lot;
 extern crate protobuf;
 extern crate serde;
 extern crate sha2;
-#[cfg(test)]
 extern crate tempfile;
 #[cfg(test)]
 extern crate testutil;
 #[macro_use]
 extern crate serde_derive;
 extern crate uuid;
+#[cfg(test)]
+extern crate walkdir;
 
 use std::cmp::min;
 use std::io::{self, Read};