From bee533cf061ba8e5fffd6eedec53aaa24a389c7e Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 29 Jul 2024 11:09:24 +0200 Subject: [PATCH 1/6] Add create_leaf_witness --- src/witness/recursive_aggregation.rs | 64 ++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/witness/recursive_aggregation.rs b/src/witness/recursive_aggregation.rs index ece7ef83..e9d9f49a 100644 --- a/src/witness/recursive_aggregation.rs +++ b/src/witness/recursive_aggregation.rs @@ -67,6 +67,11 @@ pub(crate) fn compute_encodable_item_from_witness< commitment } +pub fn split_reqursion_queue(queue: RecursionQueueSimulator) -> Vec> { + let round_function = ZkSyncDefaultRoundFunction::default(); + queue.split_by(RECURSION_ARITY, &round_function) +} + /// Creates leaf witnesses: each leaf aggregates RECURSION_ARITY (32) basic circuits of a given type. pub fn create_leaf_witnesses( subset: ( @@ -85,33 +90,68 @@ pub fn create_leaf_witnesses( Vec, // proofs for chunks Vec>, ) { - let round_function = ZkSyncDefaultRoundFunction::default(); - let (circuit_type, queue, closed_form_inputs) = subset; assert_eq!(queue.num_items as usize, proofs.len()); assert_eq!(circuit_type, vk.numeric_circuit_type() as u64); - let (t, params) = leaf_params; - assert_eq!(t, circuit_type as u8); + assert_eq!(leaf_params.0, circuit_type as u8); - let queue_splits = queue.split_by(RECURSION_ARITY, &round_function); + let queue_splits = split_reqursion_queue(queue); let mut proofs_iter = proofs.into_iter(); let mut results = Vec::with_capacity(queue_splits.len()); let mut recursive_circuits = Vec::with_capacity(queue_splits.len()); for el in queue_splits.iter().cloned() { - let mut proof_witnesses = VecDeque::new(); + let mut proofs_ = vec![]; for _ in 0..el.num_items { + let t = proofs_iter.next().expect("proof"); + proofs_.push(t); + } + + let (circuit_type, circuit) = create_leaf_witness( + circuit_type, + el.clone(), + proofs_, + &vk, + &leaf_params, + ); + + results.push((circuit_type, el)); + recursive_circuits.push(circuit); + } + + (results, recursive_circuits, closed_form_inputs) +} + +pub fn create_leaf_witness( + circuit_type: u64, // circuit type + queue: RecursionQueueSimulator, + proofs: Vec, // proofs coming from the base layer + vk: &ZkSyncBaseLayerVerificationKey, + leaf_params: &(u8, RecursionLeafParametersWitness), // (cirtuit_type, and ??) +) -> ( + u64, // type of the basic circuit + ZkSyncRecursiveLayerCircuit, // proofs for chunks +) { + assert_eq!(queue.num_items as usize, proofs.len()); + assert_eq!(circuit_type, vk.numeric_circuit_type() as u64); + + let (t, params) = leaf_params; + assert_eq!(*t, circuit_type as u8); + + let mut proofs_iter = proofs.into_iter(); + let mut proof_witnesses = VecDeque::new(); + for _ in 0..queue.num_items { let t = proofs_iter.next().expect("proof"); proof_witnesses.push_back(t.into_inner()); } let leaf_input = RecursionLeafInputWitness:: { params: params.clone(), - queue_state: take_sponge_like_queue_state_from_simulator(&el), + queue_state: take_sponge_like_queue_state_from_simulator(&queue), }; - let elements: VecDeque<_> = el + let elements: VecDeque<_> = queue .witness .iter() .map(|(_, old_tail, element)| (element.reflect(), *old_tail)) @@ -121,7 +161,7 @@ pub fn create_leaf_witnesses( input: leaf_input, vk_witness: vk.clone().into_inner(), queue_witness: FullStateCircuitQueueRawWitness { elements: elements }, - proof_witnesses: proof_witnesses, + proof_witnesses, }; let config = LeafLayerRecursionConfig::< @@ -150,11 +190,7 @@ pub fn create_leaf_witnesses( circuit, ); - results.push((circuit_type, el)); - recursive_circuits.push(circuit); - } - - (results, recursive_circuits, closed_form_inputs) + (circuit_type, circuit) } pub fn compute_leaf_params( From b19b188793e4a806beea6d47720365bb317ee9f0 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 29 Jul 2024 11:09:43 +0200 Subject: [PATCH 2/6] Cargo fmt --- src/witness/recursive_aggregation.rs | 99 +++++++++++++--------------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/src/witness/recursive_aggregation.rs b/src/witness/recursive_aggregation.rs index e9d9f49a..3e7452e3 100644 --- a/src/witness/recursive_aggregation.rs +++ b/src/witness/recursive_aggregation.rs @@ -109,13 +109,8 @@ pub fn create_leaf_witnesses( proofs_.push(t); } - let (circuit_type, circuit) = create_leaf_witness( - circuit_type, - el.clone(), - proofs_, - &vk, - &leaf_params, - ); + let (circuit_type, circuit) = + create_leaf_witness(circuit_type, el.clone(), proofs_, &vk, &leaf_params); results.push((circuit_type, el)); recursive_circuits.push(circuit); @@ -131,7 +126,7 @@ pub fn create_leaf_witness( vk: &ZkSyncBaseLayerVerificationKey, leaf_params: &(u8, RecursionLeafParametersWitness), // (cirtuit_type, and ??) ) -> ( - u64, // type of the basic circuit + u64, // type of the basic circuit ZkSyncRecursiveLayerCircuit, // proofs for chunks ) { assert_eq!(queue.num_items as usize, proofs.len()); @@ -142,55 +137,55 @@ pub fn create_leaf_witness( let mut proofs_iter = proofs.into_iter(); let mut proof_witnesses = VecDeque::new(); - for _ in 0..queue.num_items { - let t = proofs_iter.next().expect("proof"); - proof_witnesses.push_back(t.into_inner()); - } - let leaf_input = RecursionLeafInputWitness:: { - params: params.clone(), - queue_state: take_sponge_like_queue_state_from_simulator(&queue), - }; - - let elements: VecDeque<_> = queue - .witness - .iter() - .map(|(_, old_tail, element)| (element.reflect(), *old_tail)) - .collect(); + for _ in 0..queue.num_items { + let t = proofs_iter.next().expect("proof"); + proof_witnesses.push_back(t.into_inner()); + } + let leaf_input = RecursionLeafInputWitness:: { + params: params.clone(), + queue_state: take_sponge_like_queue_state_from_simulator(&queue), + }; - let witness = RecursionLeafInstanceWitness:: { - input: leaf_input, - vk_witness: vk.clone().into_inner(), - queue_witness: FullStateCircuitQueueRawWitness { elements: elements }, - proof_witnesses, - }; + let elements: VecDeque<_> = queue + .witness + .iter() + .map(|(_, old_tail, element)| (element.reflect(), *old_tail)) + .collect(); + + let witness = RecursionLeafInstanceWitness:: { + input: leaf_input, + vk_witness: vk.clone().into_inner(), + queue_witness: FullStateCircuitQueueRawWitness { elements: elements }, + proof_witnesses, + }; - let config = LeafLayerRecursionConfig::< - F, - >>::NonCircuitSimulator, - EXT, - > { - proof_config: recursion_layer_proof_config(), - vk_fixed_parameters: vk.clone().into_inner().fixed_parameters, - capacity: RECURSION_ARITY, - _marker: std::marker::PhantomData, - }; + let config = LeafLayerRecursionConfig::< + F, + >>::NonCircuitSimulator, + EXT, + > { + proof_config: recursion_layer_proof_config(), + vk_fixed_parameters: vk.clone().into_inner().fixed_parameters, + capacity: RECURSION_ARITY, + _marker: std::marker::PhantomData, + }; - let base_layer_circuit_type = - BaseLayerCircuitType::from_numeric_value(vk.numeric_circuit_type()); - let circuit = ZkSyncLeafLayerRecursiveCircuit { - witness, - config, - transcript_params: (), - base_layer_circuit_type, - _marker: std::marker::PhantomData, - }; + let base_layer_circuit_type = + BaseLayerCircuitType::from_numeric_value(vk.numeric_circuit_type()); + let circuit = ZkSyncLeafLayerRecursiveCircuit { + witness, + config, + transcript_params: (), + base_layer_circuit_type, + _marker: std::marker::PhantomData, + }; - let circuit = ZkSyncRecursiveLayerCircuit::leaf_circuit_from_base_type( - BaseLayerCircuitType::from_numeric_value(vk.numeric_circuit_type()), - circuit, - ); + let circuit = ZkSyncRecursiveLayerCircuit::leaf_circuit_from_base_type( + BaseLayerCircuitType::from_numeric_value(vk.numeric_circuit_type()), + circuit, + ); - (circuit_type, circuit) + (circuit_type, circuit) } pub fn compute_leaf_params( From 12365532b1a6aa23bd6de6807bac19b5caf68af0 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 29 Jul 2024 13:04:13 +0200 Subject: [PATCH 3/6] Cleanup --- src/utils.rs | 97 ++++++++++++++++++++++++++++ src/witness/recursive_aggregation.rs | 12 ++-- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index ecc29038..f37171b7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,6 +4,103 @@ use crate::witness::tree::BinaryHasher; use circuit_definitions::encodings::BytesSerializable; +use std::time::Instant; +use std::{ + sync::{Mutex, OnceLock}, + time::Duration, +}; + +use peak_alloc::PeakAlloc; + +#[global_allocator] +static PEAK_ALLOC: PeakAlloc = PeakAlloc; + +fn mem_array() -> &'static Mutex> { + static MEM_ARRAY: OnceLock>> = OnceLock::new(); + MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn time_instant() -> &'static Mutex { + static TIME_INSTANT: OnceLock> = OnceLock::new(); + TIME_INSTANT.get_or_init(|| Mutex::new(Instant::now())) +} + +fn time_array() -> &'static Mutex> { + static TIME_ARRAY: OnceLock>> = OnceLock::new(); + TIME_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn peak_mem_array() -> &'static Mutex> { + static PEAK_MEM_ARRAY: OnceLock>> = OnceLock::new(); + PEAK_MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) +} + +fn peak_snapshot_mem(label: &str) { + let peak_mem = PEAK_ALLOC.peak_usage_as_mb(); + println!("PEAK MEM: {}: {}", label.to_owned(), peak_mem); + peak_mem_array() + .lock() + .unwrap() + .push((label.to_owned(), peak_mem)); +} + +pub fn print_mem_snapshots() { + let mem = mem_array().lock().unwrap(); + println!("MEMORY SNAPSHOTS"); + for snapshot in mem.clone() { + println!("{}: {}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn print_peak_mem_snapshots() { + let mem = peak_mem_array().lock().unwrap(); + println!("PEAK MEMORY SNAPSHOTS"); + for snapshot in mem.clone() { + println!("{}: {}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn print_time_snapshots() { + let time = time_array().lock().unwrap(); + println!("TIME SNAPSHOTS"); + for snapshot in time.clone() { + println!("{}: {:.2?}", snapshot.0, snapshot.1); + } + println!(); +} + +pub fn snapshot_mem(label: &str) { + let current_mem = PEAK_ALLOC.current_usage_as_mb(); + println!("MEM: {}: {}", label.to_owned(), current_mem); + mem_array() + .lock() + .unwrap() + .push((label.to_owned(), current_mem)); +} + +pub fn snapshot_prof(label: &str) { + snapshot_mem(label); + peak_snapshot_mem(label); + snapshot_time(label); + reset_peak_snapshot_mem(); +} + +pub fn snapshot_time(label: &str) { + let mut instant = time_instant().lock().unwrap(); + println!("TIME: {}: {:.2?}", label.to_owned(), instant.elapsed()); + time_array() + .lock() + .unwrap() + .push((label.to_owned(), instant.elapsed())); + *instant = Instant::now(); +} + +pub fn reset_peak_snapshot_mem() { + PEAK_ALLOC.reset_peak_usage(); +} + pub fn u64_as_u32_le(value: u64) -> [u32; 2] { [value as u32, (value >> 32) as u32] } diff --git a/src/witness/recursive_aggregation.rs b/src/witness/recursive_aggregation.rs index 3e7452e3..d0168f03 100644 --- a/src/witness/recursive_aggregation.rs +++ b/src/witness/recursive_aggregation.rs @@ -102,15 +102,15 @@ pub fn create_leaf_witnesses( let mut results = Vec::with_capacity(queue_splits.len()); let mut recursive_circuits = Vec::with_capacity(queue_splits.len()); - for el in queue_splits.iter().cloned() { - let mut proofs_ = vec![]; + for el in queue_splits.into_iter() { + let mut proofs = vec![]; for _ in 0..el.num_items { let t = proofs_iter.next().expect("proof"); - proofs_.push(t); + proofs.push(t); } let (circuit_type, circuit) = - create_leaf_witness(circuit_type, el.clone(), proofs_, &vk, &leaf_params); + create_leaf_witness(circuit_type, el.clone(), proofs, &vk, &leaf_params); results.push((circuit_type, el)); recursive_circuits.push(circuit); @@ -155,7 +155,7 @@ pub fn create_leaf_witness( let witness = RecursionLeafInstanceWitness:: { input: leaf_input, vk_witness: vk.clone().into_inner(), - queue_witness: FullStateCircuitQueueRawWitness { elements: elements }, + queue_witness: FullStateCircuitQueueRawWitness { elements }, proof_witnesses, }; @@ -429,7 +429,7 @@ pub fn create_node_witnesses( }; let circuit = ZkSyncNodeLayerRecursiveCircuit { - witness: witness, + witness, config: config.clone(), transcript_params: (), _marker: std::marker::PhantomData, From 77c2e419e119def0ac84c80dfc365813f07203c5 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Mon, 29 Jul 2024 13:55:01 +0200 Subject: [PATCH 4/6] Remove profiling --- src/utils.rs | 97 ---------------------------------------------------- 1 file changed, 97 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index f37171b7..ecc29038 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,103 +4,6 @@ use crate::witness::tree::BinaryHasher; use circuit_definitions::encodings::BytesSerializable; -use std::time::Instant; -use std::{ - sync::{Mutex, OnceLock}, - time::Duration, -}; - -use peak_alloc::PeakAlloc; - -#[global_allocator] -static PEAK_ALLOC: PeakAlloc = PeakAlloc; - -fn mem_array() -> &'static Mutex> { - static MEM_ARRAY: OnceLock>> = OnceLock::new(); - MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn time_instant() -> &'static Mutex { - static TIME_INSTANT: OnceLock> = OnceLock::new(); - TIME_INSTANT.get_or_init(|| Mutex::new(Instant::now())) -} - -fn time_array() -> &'static Mutex> { - static TIME_ARRAY: OnceLock>> = OnceLock::new(); - TIME_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn peak_mem_array() -> &'static Mutex> { - static PEAK_MEM_ARRAY: OnceLock>> = OnceLock::new(); - PEAK_MEM_ARRAY.get_or_init(|| Mutex::new(vec![])) -} - -fn peak_snapshot_mem(label: &str) { - let peak_mem = PEAK_ALLOC.peak_usage_as_mb(); - println!("PEAK MEM: {}: {}", label.to_owned(), peak_mem); - peak_mem_array() - .lock() - .unwrap() - .push((label.to_owned(), peak_mem)); -} - -pub fn print_mem_snapshots() { - let mem = mem_array().lock().unwrap(); - println!("MEMORY SNAPSHOTS"); - for snapshot in mem.clone() { - println!("{}: {}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn print_peak_mem_snapshots() { - let mem = peak_mem_array().lock().unwrap(); - println!("PEAK MEMORY SNAPSHOTS"); - for snapshot in mem.clone() { - println!("{}: {}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn print_time_snapshots() { - let time = time_array().lock().unwrap(); - println!("TIME SNAPSHOTS"); - for snapshot in time.clone() { - println!("{}: {:.2?}", snapshot.0, snapshot.1); - } - println!(); -} - -pub fn snapshot_mem(label: &str) { - let current_mem = PEAK_ALLOC.current_usage_as_mb(); - println!("MEM: {}: {}", label.to_owned(), current_mem); - mem_array() - .lock() - .unwrap() - .push((label.to_owned(), current_mem)); -} - -pub fn snapshot_prof(label: &str) { - snapshot_mem(label); - peak_snapshot_mem(label); - snapshot_time(label); - reset_peak_snapshot_mem(); -} - -pub fn snapshot_time(label: &str) { - let mut instant = time_instant().lock().unwrap(); - println!("TIME: {}: {:.2?}", label.to_owned(), instant.elapsed()); - time_array() - .lock() - .unwrap() - .push((label.to_owned(), instant.elapsed())); - *instant = Instant::now(); -} - -pub fn reset_peak_snapshot_mem() { - PEAK_ALLOC.reset_peak_usage(); -} - pub fn u64_as_u32_le(value: u64) -> [u32; 2] { [value as u32, (value >> 32) as u32] } From 02c8d627e4b6381082e1190f395d10108a514617 Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 30 Jul 2024 09:44:08 +0200 Subject: [PATCH 5/6] Update src/witness/recursive_aggregation.rs Co-authored-by: EmilLuta --- src/witness/recursive_aggregation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/witness/recursive_aggregation.rs b/src/witness/recursive_aggregation.rs index d0168f03..442a1683 100644 --- a/src/witness/recursive_aggregation.rs +++ b/src/witness/recursive_aggregation.rs @@ -96,7 +96,7 @@ pub fn create_leaf_witnesses( assert_eq!(leaf_params.0, circuit_type as u8); - let queue_splits = split_reqursion_queue(queue); + let queue_splits = split_recursion_queue(queue); let mut proofs_iter = proofs.into_iter(); let mut results = Vec::with_capacity(queue_splits.len()); From 96f624538357899ead6e408cd1ff11ded8debf8e Mon Sep 17 00:00:00 2001 From: Vladislav Volosnikov Date: Tue, 30 Jul 2024 09:45:53 +0200 Subject: [PATCH 6/6] Fix --- src/witness/recursive_aggregation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/witness/recursive_aggregation.rs b/src/witness/recursive_aggregation.rs index 442a1683..cd7c3e7b 100644 --- a/src/witness/recursive_aggregation.rs +++ b/src/witness/recursive_aggregation.rs @@ -67,7 +67,7 @@ pub(crate) fn compute_encodable_item_from_witness< commitment } -pub fn split_reqursion_queue(queue: RecursionQueueSimulator) -> Vec> { +pub fn split_recursion_queue(queue: RecursionQueueSimulator) -> Vec> { let round_function = ZkSyncDefaultRoundFunction::default(); queue.split_by(RECURSION_ARITY, &round_function) }