Change course and do pack streaming first

git-odb/src/pack/index/verify.rs

@@ -97,6 +97,19 @@ pub enum Mode {
     Sha1CRC32DecodeEncode,
 }
 
+/// The way of performing the pack verification
+#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
+pub enum Algorithm {
+    /// Lookup each object similarly to what would happen during normal repository use.
+    ///
+    /// Uses more compute resources as it will resolve delta chains from back to front, potentially
+    /// redoing a lot of work across multiple objects.
+    Lookup,
+    /// Read the pack sequentially (without the need for an index) and resolve pack objects from the base towards their deltas,
+    /// doing the necessary work only once.
+    Stream,
+}
+
 /// Verify and validate the content of the index file
 impl index::File {
     pub fn checksum_of_index(&self) -> owned::Id {

gitoxide-core/src/lib.rs

@@ -41,6 +41,40 @@ impl FromStr for OutputFormat {
     }
 }
 
+#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
+pub enum VerifyAlgorithm {
+    Lookup,
+    Stream,
+}
+
+impl VerifyAlgorithm {
+    pub fn variants() -> &'static [&'static str] {
+        &["lookup", "stream"]
+    }
+}
+
+impl FromStr for VerifyAlgorithm {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let s_lc = s.to_ascii_lowercase();
+        Ok(match s_lc.as_str() {
+            "lookup" => VerifyAlgorithm::Lookup,
+            "stream" => VerifyAlgorithm::Stream,
+            _ => return Err(format!("Invalid verification algorithm: '{}'", s)),
+        })
+    }
+}
+
+impl From<VerifyAlgorithm> for index::verify::Algorithm {
+    fn from(v: VerifyAlgorithm) -> Self {
+        match v {
+            VerifyAlgorithm::Lookup => index::verify::Algorithm::Lookup,
+            VerifyAlgorithm::Stream => index::verify::Algorithm::Stream,
+        }
+    }
+}
+
 /// A general purpose context for many operations provided here
 pub struct Context<W1: io::Write, W2: io::Write> {
     /// If set, provide statistics to `out` in the given format
@@ -54,6 +88,7 @@ pub struct Context<W1: io::Write, W2: io::Write> {
     /// A value of 0 is interpreted as no-limit
     pub thread_limit: Option<usize>,
     pub mode: index::verify::Mode,
+    pub algorithm: index::verify::Algorithm,
 }
 
 impl Default for Context<Vec<u8>, Vec<u8>> {
@@ -62,6 +97,7 @@ impl Default for Context<Vec<u8>, Vec<u8>> {
             output_statistics: None,
             thread_limit: None,
             mode: index::verify::Mode::Sha1CRC32,
+            algorithm: index::verify::Algorithm::Lookup,
             out: Vec::new(),
             err: Vec::new(),
         }
@@ -102,6 +138,7 @@ pub fn verify_pack_or_pack_index<P, W1, W2>(
         mode,
         output_statistics,
         thread_limit,
+        algorithm: _,
     }: Context<W1, W2>,
 ) -> Result<(owned::Id, Option<index::verify::Outcome>)>
 where

src/plumbing/lean.rs

@@ -1,5 +1,6 @@
 mod options {
     use argh::FromArgs;
+    use gitoxide_core as core;
     use std::path::PathBuf;
 
     #[derive(FromArgs)]
@@ -46,6 +47,12 @@ mod options {
         /// owned objects, causing plenty of allocation to occour.
         pub re_encode: bool,
 
+        #[argh(option)]
+        /// the algorithm used to verify the pack. They differ in costs.
+        ///
+        /// Possible values are "lookup" and "stream". Default is "stream".
+        pub algorithm: Option<core::VerifyAlgorithm>,
+
         /// output statistical information about the pack
         #[argh(switch, short = 's')]
         pub statistics: bool,
@@ -97,6 +104,7 @@ pub fn main() -> Result<()> {
             path,
             verbose,
             statistics,
+            algorithm,
             decode,
             re_encode,
         }) => {
@@ -110,6 +118,7 @@ pub fn main() -> Result<()> {
                     } else {
                         None
                     },
+                    algorithm: algorithm.unwrap_or(core::VerifyAlgorithm::Lookup).into(),
                     thread_limit,
                     mode: match (decode, re_encode) {
                         (true, false) => core::VerifyMode::Sha1CRC32Decode,

src/plumbing/pretty.rs

@@ -41,6 +41,15 @@ mod options {
             )]
             format: core::OutputFormat,
 
+            /// The algorithm used to verify the pack. They differ in costs.
+            #[structopt(
+                long,
+                short = "a",
+                default_value = "lookup",
+                possible_values(core::VerifyAlgorithm::variants())
+            )]
+            algorithm: core::VerifyAlgorithm,
+
             /// verbose progress messages are printed line by line
             #[structopt(long, short = "v")]
             verbose: bool,
@@ -173,6 +182,7 @@ pub fn main() -> Result<()> {
     match args.cmd {
         Subcommands::VerifyPack {
             path,
+            algorithm,
             verbose,
             progress,
             format,
@@ -198,6 +208,7 @@ pub fn main() -> Result<()> {
                     core::Context {
                         output_statistics,
                         thread_limit,
+                        algorithm: algorithm.into(),
                         mode,
                         out,
                         err,

tasks.md

@@ -18,6 +18,10 @@
   * [x] The Rust repo verifies `--decode` and `--re-encode`
   * [x] add re-encode to all stress tests
   * [x] support easy access to merge-tags and the signature
+* **stream pack for verify-pack**
+  * [ ] stream packs and feed information into a delegate to allow doing anything™️
+  * [ ] support for multi-threading
+  * [ ] use pack streaming in pack-verify
 * **plumbing - explode pack**
   * [ ] write loose object
   * [ ] single threaded