From 318a34b723a594ffe7fe854c9dba36e8cf86fcfa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Nov 2024 11:09:18 -0800 Subject: [PATCH 01/44] Add ShredIndex benchmarks --- Cargo.lock | 4 + ledger/Cargo.toml | 6 + ledger/benches/shred_index.rs | 1119 +++++++++++++++++++++++++++++++++ ledger/src/blockstore_meta.rs | 12 +- 4 files changed, 1138 insertions(+), 3 deletions(-) create mode 100644 ledger/benches/shred_index.rs diff --git a/Cargo.lock b/Cargo.lock index f603e6ed027000..88a849f62aa2c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -938,6 +938,7 @@ checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium", + "serde", "tap", "wyz", ] @@ -7332,7 +7333,9 @@ dependencies = [ "assert_matches", "bincode", "bitflags 2.6.0", + "bitvec", "bs58", + "bv", "byteorder", "bzip2", "chrono", @@ -7361,6 +7364,7 @@ dependencies = [ "scopeguard", "serde", "serde_bytes", + "serde_with", "sha2 0.10.8", "solana-account-decoder", "solana-accounts-db", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index f4b1a560a52002..e9c47f003e64e9 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -13,6 +13,8 @@ edition = { workspace = true } assert_matches = { workspace = true } bincode = { workspace = true } bitflags = { workspace = true, features = ["serde"] } +bitvec = { version = "1.0.1", features = ["serde"] } +bv = { workspace = true } byteorder = { workspace = true } bzip2 = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } @@ -40,6 +42,7 @@ reed-solomon-erasure = { workspace = true, features = ["simd-accel"] } scopeguard = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } +serde_with = { workspace = true } sha2 = { workspace = true } solana-account-decoder = { workspace = true } solana-accounts-db = { workspace = true } @@ -115,6 +118,9 @@ name = "sigverify_shreds" [[bench]] name = "blockstore" +[[bench]] +name = "shred_index" + [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/ledger/benches/shred_index.rs b/ledger/benches/shred_index.rs new file mode 100644 index 00000000000000..23ae4c217025f6 --- /dev/null +++ b/ledger/benches/shred_index.rs @@ -0,0 +1,1119 @@ +#![feature(test)] +extern crate test; + +use { + bitvec::{array::BitArray, order::Lsb0}, + rand::Rng, + serde::{Deserialize, Serialize}, + serde_with::serde_as, + solana_ledger::blockstore_meta::ShredIndex as LegacyShredIndex, + std::ops::{Bound, RangeBounds}, + test::Bencher, +}; + +const MAX_DATA_SHREDS_PER_SLOT: usize = 32_768; +const NUM_BYTES: usize = (MAX_DATA_SHREDS_PER_SLOT + 7) / 8; +const NUM_U64S: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; + +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct ByteShredIndex { + #[serde_as(as = "[_; NUM_BYTES]")] + index: [u8; NUM_BYTES], + num_shreds: usize, +} + +impl Default for ByteShredIndex { + fn default() -> Self { + Self { + index: [0; NUM_BYTES], + num_shreds: 0, + } + } +} + +impl ByteShredIndex { + fn new() -> Self { + Self::default() + } + + pub fn contains(&self, idx: u64) -> bool { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return false; + } + let byte_idx = (idx / 8) as usize; + let bit_idx = (idx % 8) as u8; + (self.index[byte_idx] & (1 << bit_idx)) != 0 + } + + pub fn insert(&mut self, idx: u64) { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + let byte_idx = (idx / 8) as usize; + let bit_idx = (idx % 8) as u8; + let old_byte = self.index[byte_idx]; + let new_byte = old_byte | (1 << bit_idx); + if old_byte != new_byte { + self.index[byte_idx] = new_byte; + self.num_shreds += 1; + } + } + + pub fn range(&self, bounds: R) -> impl Iterator + '_ + where + R: RangeBounds, + { + let start = match bounds.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match bounds.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + }; + + let start_byte = (start / 8) as usize; + let end_byte = ((end + 7) / 8) as usize; + + self.index[start_byte..end_byte] + .iter() + .enumerate() + .flat_map(move |(byte_offset, &byte)| { + let base_idx = ((start_byte + byte_offset) * 8) as u64; + + // Determine the range of bits to check in this byte + let lower_bound = if base_idx < start { + (start - base_idx) as u8 + } else { + 0 + }; + + let upper_bound = if base_idx + 8 > end { + (end - base_idx) as u8 + } else { + 8 + }; + + // Create a mask for the relevant bits in this byte + + let lower_mask = !0u8 << lower_bound; + let upper_mask = !0u8 >> (8 - upper_bound); + let mask = byte & lower_mask & upper_mask; + + // Iterate through the bits set in the mask + std::iter::from_fn({ + let mut remaining = mask; + move || { + if remaining == 0 { + None + } else { + let bit_idx = remaining.trailing_zeros() as u8; + remaining &= remaining - 1; + Some(base_idx + bit_idx as u64) + } + } + }) + }) + } +} + +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct U64ShredIndex { + #[serde_as(as = "[_; NUM_U64S]")] + index: [u64; NUM_U64S], + num_shreds: usize, +} + +impl Default for U64ShredIndex { + fn default() -> Self { + Self { + index: [0; NUM_U64S], + num_shreds: 0, + } + } +} + +impl U64ShredIndex { + fn new() -> Self { + Self::default() + } + + pub fn contains(&self, idx: u64) -> bool { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return false; + } + let word_idx = idx / 64; + let bit_idx = idx % 64; + (self.index[word_idx as usize] & (1 << bit_idx)) != 0 + } + + pub fn insert(&mut self, idx: u64) { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + let word_idx = idx / 64; + let bit_idx = idx % 64; + let old_word = self.index[word_idx as usize]; + let new_word = old_word | (1 << bit_idx); + if old_word != new_word { + self.index[word_idx as usize] = new_word; + self.num_shreds += 1; + } + } + + /// Provides an iterator over the set shred indices within a specified range. + /// + /// # Algorithm + /// 1. Divide the specified range into 64-bit words. + /// 2. For each word: + /// - Calculate the base index (position of the word * 64). + /// - Process all set bits in the word. + /// - For words overlapping the range boundaries: + /// - Determine the relevant bit range using boundaries. + /// - Mask out bits outside the range. + /// - Use bit manipulation to iterate over set bits efficiently. + /// + /// ## Explanation + /// > Note we're showing 32 bits per word in examples for brevity, but each word is 64 bits. + /// + /// Given range `[75..205]`: + /// + /// Word layout (each word is 64 bits), where each X represents a bit candidate: + /// ```text + /// Word 1 (0–63): [................................] ← Not included (outside range) + /// Word 2 (64–127): [..........XXXXXXXXXXXXXXXXXXXXXX] ← Partial word (start) + /// Word 3 (128–191): [XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] ← Full word (entirely in range) + /// Word 4 (192–255): [XXXXXXXXXXXXXXXXXXX.............] ← Partial word (end) + /// ``` + /// + /// Partial Word 2 (contains start boundary 75): + /// - Base index = 64 + /// - Lower boundary = 75 - 64 = 11 + /// - Lower mask = `11111111111111110000000000000000` + /// + /// Partial Word 4 (contains end boundary 205): + /// - Base index = 192 + /// - Upper boundary = 205 - 192 = 13 + /// - Upper mask = `00000000000000000000000000001111` + /// + /// Final mask = `word & lower_mask & upper_mask` + /// + /// Bit iteration: + /// 1. Apply masks to restrict the bits to the range. + /// 2. While bits remain in the masked word: + /// a. Find the lowest set bit (`trailing_zeros`). + /// b. Add the bit's position to the base index. + /// c. Clear the lowest set bit (`n & (n - 1)`). + /// ``` + pub fn range(&self, bounds: R) -> impl Iterator + '_ + where + R: RangeBounds, + { + let start = match bounds.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match bounds.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + }; + + let end_word: usize = (((end + 63) / 64) as usize).min(NUM_U64S); + let start_word = ((start / 64) as usize).min(end_word); + + self.index[start_word..end_word] + .iter() + .enumerate() + .flat_map(move |(word_offset, &word)| { + let base_idx = (start_word + word_offset) as u64 * 64; + + let lower_bound = if base_idx < start { + start - base_idx + } else { + 0 + }; + + let upper_bound = if base_idx + 64 > end { + end - base_idx + } else { + 64 + }; + + let lower_mask = !0u64 << lower_bound; + let upper_mask = !0u64 >> (64 - upper_bound); + let mask = word & lower_mask & upper_mask; + + std::iter::from_fn({ + let mut remaining = mask; + move || { + if remaining == 0 { + None + } else { + let bit_idx = remaining.trailing_zeros() as u64; + // Clear the lowest set bit + remaining &= remaining - 1; + Some(base_idx + bit_idx) + } + } + }) + }) + } +} + +type ShredBitArray = BitArray<[u64; NUM_U64S], Lsb0>; + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct BitArrayShredIndex { + index: ShredBitArray, + num_shreds: usize, +} + +impl BitArrayShredIndex { + pub fn new() -> Self { + Self { + index: BitArray::ZERO, + num_shreds: 0, + } + } + + pub fn contains(&self, idx: u64) -> bool { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return false; + } + self.index[idx as usize] + } + + pub fn insert(&mut self, idx: u64) { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + if !self.index[idx as usize] { + self.index.set(idx as usize, true); + self.num_shreds += 1; + } + } + + pub fn range(&self, bounds: R) -> impl Iterator + '_ + where + R: RangeBounds, + { + let start = match bounds.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match bounds.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + }; + + (start..end).filter(|&idx| self.index[idx as usize]) + } +} + +use bv::BitVec; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct BvShredIndex { + index: BitVec, + num_shreds: usize, +} + +impl Default for BvShredIndex { + fn default() -> Self { + Self { + index: BitVec::new_fill(false, MAX_DATA_SHREDS_PER_SLOT as u64), + num_shreds: 0, + } + } +} + +impl BvShredIndex { + fn new() -> Self { + Self::default() + } + + pub fn contains(&self, idx: u64) -> bool { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return false; + } + self.index.get(idx) + } + + pub fn insert(&mut self, idx: u64) { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + if !self.index.get(idx) { + self.index.set(idx, true); + self.num_shreds += 1; + } + } + + pub fn range(&self, bounds: R) -> impl Iterator + '_ + where + R: RangeBounds, + { + let start = match bounds.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match bounds.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + }; + + (start..end).filter(|&i| self.index.get(i)) + } +} + +/// Benchmarks range operations with different window sizes on sparse data. +/// +/// This test: +/// - Creates a new empty index +/// - Randomly sets ~10% of bits across the entire possible range +/// - Tests iteration over windows of different sizes (64, 1024, 4096 bits) +/// - Measures the performance impact of window size on range operations +macro_rules! bench_range_window { + ($name:ident, $index:ty, $window_size:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + + // Set ~10% of bits randomly + for _ in 0..(MAX_DATA_SHREDS_PER_SLOT as f64 * 0.1) as usize { + index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); + } + + let start = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64 - $window_size); + b.iter(|| { + let count = index.range(start..(start + $window_size)).count(); + test::black_box(count) + }); + } + }; +} + +/// This test: +/// - Creates a new empty index +/// - Populates it with specified density (10%, 50%, 90%) +/// - Tests performance impact of data density on range operations +/// - Allows configurable range size +macro_rules! bench_range_density { + ($name:ident, $index:ty, $density:expr, $range_size:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + use rand::seq::SliceRandom; + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + + // Calculate number of entries needed for desired density + let range_size = $range_size; + let num_entries = (range_size as f64 * $density) as u64; + + // Insert entries to achieve target density + let start = 0; + let end = start + range_size; + let mut entries: Vec = (start..end).collect(); + entries.shuffle(&mut rng); + entries.truncate(num_entries as usize); + + for entry in entries { + index.insert(entry); + } + + b.iter(|| { + let count = index.range(start..end).count(); + test::black_box(count) + }); + } + }; +} + +/// Benchmarks range operations with different data distributions. +/// +/// This test: +/// - Creates a new empty index with 50% density in 1000..2000 range +/// - Tests three patterns: +/// - Sequential (every other index) +/// - Random (randomly distributed) +/// - Clustered (groups of consecutive indices) +/// - Measures how data distribution affects range operation performance +macro_rules! bench_range_distribution { + ($name:ident, $index:ty, $pattern:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + use rand::seq::SliceRandom; + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + + match $pattern { + "sequential" => { + for i in (1000..2000).step_by(2) { + index.insert(i); + } + } + "random" => { + let mut indices: Vec = (1000..2000).collect(); + indices.shuffle(&mut rng); + indices.truncate(500); + for i in indices { + index.insert(i); + } + } + "clustered" => { + let mut current = 1000u64; + while current < 2000 { + // Insert clusters of 8 consecutive indices + for i in current..current.saturating_add(8).min(2000) { + index.insert(i); + } + // Skip 8 indices before next cluster + current += 16; + } + } + _ => panic!("Unknown pattern"), + } + + b.iter(|| { + let count = index.range(1000..2000).count(); + test::black_box(count) + }); + } + }; +} + +/// Benchmarks insertion operations with different patterns. +/// +/// This test: +/// - Creates a new empty index +/// - Tests three insertion patterns: +/// - Sequential (consecutive indices) +/// - Random (randomly distributed) +/// - Clustered (groups of consecutive indices) +/// - Measures how insertion pattern affects performance +macro_rules! bench_insert_pattern { + ($name:ident, $index:ty, $pattern:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + let mut next_insert = 0u64; + let mut cluster_start = 0u64; + + b.iter(|| { + match $pattern { + "sequential" => { + index.insert(next_insert); + next_insert = (next_insert + 1) % MAX_DATA_SHREDS_PER_SLOT as u64; + } + "random" => { + index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); + } + "clustered" => { + index.insert(cluster_start + (next_insert % 8)); + next_insert += 1; + if next_insert % 8 == 0 { + cluster_start = (cluster_start + 16) % MAX_DATA_SHREDS_PER_SLOT as u64; + } + } + _ => panic!("Unknown pattern"), + } + test::black_box(&index); + }); + } + }; +} + +// Window size benchmarks (64, 1024, 4096 bits) for each implementation +bench_range_window!(bench_window_64_byte, ByteShredIndex, 64); +bench_range_window!(bench_window_1024_byte, ByteShredIndex, 1024); +bench_range_window!(bench_window_4096_byte, ByteShredIndex, 4096); + +bench_range_window!(bench_window_64_bit_array, BitArrayShredIndex, 64); +bench_range_window!(bench_window_1024_bit_array, BitArrayShredIndex, 1024); +bench_range_window!(bench_window_4096_bit_array, BitArrayShredIndex, 4096); + +bench_range_window!(bench_window_64_legacy, LegacyShredIndex, 64); +bench_range_window!(bench_window_1024_legacy, LegacyShredIndex, 1024); +bench_range_window!(bench_window_4096_legacy, LegacyShredIndex, 4096); + +bench_range_window!(bench_window_64_bv, BvShredIndex, 64); +bench_range_window!(bench_window_1024_bv, BvShredIndex, 1024); +bench_range_window!(bench_window_4096_bv, BvShredIndex, 4096); + +bench_range_window!(bench_window_64_u64, U64ShredIndex, 64); +bench_range_window!(bench_window_1024_u64, U64ShredIndex, 1024); +bench_range_window!(bench_window_4096_u64, U64ShredIndex, 4096); + +// Density benchmarks (10%, 50%, 90%) for each implementation +bench_range_density!(bench_density_10_byte, ByteShredIndex, 0.1, 1000); +bench_range_density!(bench_density_50_byte, ByteShredIndex, 0.5, 1000); +bench_range_density!(bench_density_90_byte, ByteShredIndex, 0.9, 1000); +bench_range_density!(bench_density_10_byte_10k, ByteShredIndex, 0.1, 10000); +bench_range_density!(bench_density_50_byte_10k, ByteShredIndex, 0.5, 10000); +bench_range_density!(bench_density_90_byte_10k, ByteShredIndex, 0.9, 10000); + +bench_range_density!(bench_density_10_bit_array, BitArrayShredIndex, 0.1, 1000); +bench_range_density!(bench_density_50_bit_array, BitArrayShredIndex, 0.5, 1000); +bench_range_density!(bench_density_90_bit_array, BitArrayShredIndex, 0.9, 1000); +bench_range_density!( + bench_density_10_bit_array_10k, + BitArrayShredIndex, + 0.1, + 10000 +); +bench_range_density!( + bench_density_50_bit_array_10k, + BitArrayShredIndex, + 0.5, + 10000 +); +bench_range_density!( + bench_density_90_bit_array_10k, + BitArrayShredIndex, + 0.9, + 10000 +); + +bench_range_density!(bench_density_10_legacy, LegacyShredIndex, 0.1, 1000); +bench_range_density!(bench_density_50_legacy, LegacyShredIndex, 0.5, 1000); +bench_range_density!(bench_density_90_legacy, LegacyShredIndex, 0.9, 1000); +bench_range_density!(bench_density_10_legacy_10k, LegacyShredIndex, 0.1, 10000); +bench_range_density!(bench_density_50_legacy_10k, LegacyShredIndex, 0.5, 10000); +bench_range_density!(bench_density_90_legacy_10k, LegacyShredIndex, 0.9, 10000); + +bench_range_density!(bench_density_10_bv, BvShredIndex, 0.1, 1000); +bench_range_density!(bench_density_50_bv, BvShredIndex, 0.5, 1000); +bench_range_density!(bench_density_90_bv, BvShredIndex, 0.9, 1000); +bench_range_density!(bench_density_10_bv_10k, BvShredIndex, 0.1, 10000); +bench_range_density!(bench_density_50_bv_10k, BvShredIndex, 0.5, 10000); +bench_range_density!(bench_density_90_bv_10k, BvShredIndex, 0.9, 10000); + +bench_range_density!(bench_density_10_u64, U64ShredIndex, 0.1, 1000); +bench_range_density!(bench_density_50_u64, U64ShredIndex, 0.5, 1000); +bench_range_density!(bench_density_90_u64, U64ShredIndex, 0.9, 1000); +bench_range_density!(bench_density_10_u64_10k, U64ShredIndex, 0.1, 10000); +bench_range_density!(bench_density_50_u64_10k, U64ShredIndex, 0.5, 10000); +bench_range_density!(bench_density_90_u64_10k, U64ShredIndex, 0.9, 10000); + +// Distribution pattern benchmarks (sequential, random, clustered) for each implementation +bench_range_distribution!(bench_dist_seq_byte, ByteShredIndex, "sequential"); +bench_range_distribution!(bench_dist_rand_byte, ByteShredIndex, "random"); +bench_range_distribution!(bench_dist_clust_byte, ByteShredIndex, "clustered"); + +bench_range_distribution!(bench_dist_seq_bit_array, BitArrayShredIndex, "sequential"); +bench_range_distribution!(bench_dist_rand_bit_array, BitArrayShredIndex, "random"); +bench_range_distribution!(bench_dist_clust_bit_array, BitArrayShredIndex, "clustered"); + +bench_range_distribution!(bench_dist_seq_legacy, LegacyShredIndex, "sequential"); +bench_range_distribution!(bench_dist_rand_legacy, LegacyShredIndex, "random"); +bench_range_distribution!(bench_dist_clust_legacy, LegacyShredIndex, "clustered"); + +bench_range_distribution!(bench_dist_seq_bv, BvShredIndex, "sequential"); +bench_range_distribution!(bench_dist_rand_bv, BvShredIndex, "random"); +bench_range_distribution!(bench_dist_clust_bv, BvShredIndex, "clustered"); + +bench_range_distribution!(bench_dist_seq_u64, U64ShredIndex, "sequential"); +bench_range_distribution!(bench_dist_rand_u64, U64ShredIndex, "random"); +bench_range_distribution!(bench_dist_clust_u64, U64ShredIndex, "clustered"); + +// Insertion pattern benchmarks for each implementation +bench_insert_pattern!(bench_insert_seq_byte, ByteShredIndex, "sequential"); +bench_insert_pattern!(bench_insert_rand_byte, ByteShredIndex, "random"); +bench_insert_pattern!(bench_insert_clust_byte, ByteShredIndex, "clustered"); + +bench_insert_pattern!(bench_insert_seq_bit_array, BitArrayShredIndex, "sequential"); +bench_insert_pattern!(bench_insert_rand_bit_array, BitArrayShredIndex, "random"); +bench_insert_pattern!( + bench_insert_clust_bit_array, + BitArrayShredIndex, + "clustered" +); + +bench_insert_pattern!(bench_insert_seq_legacy, LegacyShredIndex, "sequential"); +bench_insert_pattern!(bench_insert_rand_legacy, LegacyShredIndex, "random"); +bench_insert_pattern!(bench_insert_clust_legacy, LegacyShredIndex, "clustered"); + +bench_insert_pattern!(bench_insert_seq_bv, BvShredIndex, "sequential"); +bench_insert_pattern!(bench_insert_rand_bv, BvShredIndex, "random"); +bench_insert_pattern!(bench_insert_clust_bv, BvShredIndex, "clustered"); + +bench_insert_pattern!(bench_insert_seq_u64, U64ShredIndex, "sequential"); +bench_insert_pattern!(bench_insert_rand_u64, U64ShredIndex, "random"); +bench_insert_pattern!(bench_insert_clust_u64, U64ShredIndex, "clustered"); + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! test_serde { + ($name:ident, $index:ty) => { + #[test] + fn $name() { + let mut index = <$index>::new(); + for i in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { + if i % 50 == 0 { + index.insert(i); + } + } + let serialized = bincode::serialize(&index).unwrap(); + let deserialized: $index = bincode::deserialize(&serialized).unwrap(); + assert_eq!(index, deserialized); + } + }; + } + + test_serde!(test_serde_byte, ByteShredIndex); + test_serde!(test_serde_bit_array, BitArrayShredIndex); + test_serde!(test_serde_legacy, LegacyShredIndex); + test_serde!(test_serde_bv, BvShredIndex); + test_serde!(test_serde_u64, U64ShredIndex); + + fn create_indices() -> ( + ByteShredIndex, + BitArrayShredIndex, + LegacyShredIndex, + BvShredIndex, + U64ShredIndex, + ) { + let byte_index = ByteShredIndex::new(); + let bit_index = BitArrayShredIndex::new(); + let legacy_index = LegacyShredIndex::default(); + let bv_index = BvShredIndex::new(); + let u64_index = U64ShredIndex::new(); + (byte_index, bit_index, legacy_index, bv_index, u64_index) + } + + #[test] + fn test_basic_insert_contains() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + + let test_indices = [0, 1, 7, 8, 9, 63, 64, 1000, 32767]; + + // Test insertions + for &idx in &test_indices { + byte_index.insert(idx); + bit_index.insert(idx); + legacy_index.insert(idx); + bv_index.insert(idx); + u64_index.insert(idx); + } + + // Test contains + for idx in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { + assert_eq!( + byte_index.contains(idx), + legacy_index.contains(idx), + "ByteShredIndex vs Legacy mismatch at index {}", + idx + ); + assert_eq!( + bit_index.contains(idx), + legacy_index.contains(idx), + "ShredIndex vs Legacy mismatch at index {}", + idx + ); + assert_eq!( + bv_index.contains(idx), + legacy_index.contains(idx), + "BvShredIndex vs Legacy mismatch at index {}", + idx + ); + assert_eq!( + u64_index.contains(idx), + legacy_index.contains(idx), + "U64ShredIndex vs Legacy mismatch at index {}", + idx + ); + } + } + + #[test] + fn test_range_operations() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + + // Insert some patterns + for i in (0..100).step_by(3) { + byte_index.insert(i); + bit_index.insert(i); + legacy_index.insert(i); + bv_index.insert(i); + u64_index.insert(i); + } + + // Test different range sizes + let ranges = [0..10, 90..100, 0..1000, 50..150, 32760..32768]; + + for range in ranges.iter() { + let byte_result: Vec<_> = byte_index.range(range.clone()).collect(); + let bit_result: Vec<_> = bit_index.range(range.clone()).collect(); + let legacy_result: Vec<_> = legacy_index.range(range.clone()).copied().collect(); + let bv_result: Vec<_> = bv_index.range(range.clone()).collect(); + let u64_result: Vec<_> = u64_index.range(range.clone()).collect(); + + assert_eq!( + byte_result, legacy_result, + "ByteShredIndex range results don't match legacy for {:?}", + range + ); + assert_eq!( + bit_result, legacy_result, + "ShredIndex range results don't match legacy for {:?}", + range + ); + assert_eq!( + bv_result, legacy_result, + "BvShredIndex range results don't match legacy for {:?}", + range + ); + assert_eq!( + u64_result, legacy_result, + "U64ShredIndex range results don't match legacy for {:?}", + range + ); + } + } + + #[test] + fn test_boundary_conditions() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + + let boundaries = [ + MAX_DATA_SHREDS_PER_SLOT as u64 - 2, + MAX_DATA_SHREDS_PER_SLOT as u64 - 1, + ]; + + for &idx in &boundaries { + byte_index.insert(idx); + bit_index.insert(idx); + legacy_index.insert(idx); + bv_index.insert(idx); + u64_index.insert(idx); + + assert_eq!( + byte_index.contains(idx), + legacy_index.contains(idx), + "ByteShredIndex boundary mismatch at {}", + idx + ); + assert_eq!( + bit_index.contains(idx), + legacy_index.contains(idx), + "ShredIndex boundary mismatch at {}", + idx + ); + assert_eq!( + bv_index.contains(idx), + legacy_index.contains(idx), + "BvShredIndex boundary mismatch at {}", + idx + ); + assert_eq!( + u64_index.contains(idx), + legacy_index.contains(idx), + "U64ShredIndex boundary mismatch at {}", + idx + ); + } + + // Explicitly test that indices >= MAX_DATA_SHREDS_PER_SLOT are rejected + let invalid_indices = [ + MAX_DATA_SHREDS_PER_SLOT as u64, + MAX_DATA_SHREDS_PER_SLOT as u64 + 1, + ]; + + for &idx in &invalid_indices { + assert!(!byte_index.contains(idx)); + assert!(!bit_index.contains(idx)); + assert!(!bv_index.contains(idx)); + assert!(!u64_index.contains(idx)); + + // Insert should have no effect + byte_index.insert(idx); + bit_index.insert(idx); + bv_index.insert(idx); + u64_index.insert(idx); + assert!(!byte_index.contains(idx)); + assert!(!bit_index.contains(idx)); + assert!(!bv_index.contains(idx)); + assert!(!u64_index.contains(idx)); + } + } + + #[test] + fn test_random_operations() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + let mut rng = rand::thread_rng(); + + // Random insertions + for _ in 0..1000 { + let idx = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64); + byte_index.insert(idx); + bit_index.insert(idx); + legacy_index.insert(idx); + bv_index.insert(idx); + u64_index.insert(idx); + } + + // Test random ranges + for _ in 0..100 { + let start = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64); + let end = rng.gen_range(start..MAX_DATA_SHREDS_PER_SLOT as u64); + + let byte_result: Vec<_> = byte_index.range(start..end).collect(); + let bit_result: Vec<_> = bit_index.range(start..end).collect(); + let legacy_result: Vec<_> = legacy_index.range(start..end).copied().collect(); + let bv_result: Vec<_> = bv_index.range(start..end).collect(); + let u64_result: Vec<_> = u64_index.range(start..end).collect(); + + assert_eq!( + byte_result, legacy_result, + "ByteShredIndex random range mismatch for {}..{}", + start, end + ); + assert_eq!( + bit_result, legacy_result, + "ShredIndex random range mismatch for {}..{}", + start, end + ); + assert_eq!( + bv_result, legacy_result, + "BvShredIndex random range mismatch for {}..{}", + start, end + ); + assert_eq!( + u64_result, legacy_result, + "U64ShredIndex random range mismatch for {}..{}", + start, end + ); + } + } + + #[test] + fn test_num_shreds_tracking() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + + // Test duplicate insertions + for i in 0..100 { + byte_index.insert(i); + byte_index.insert(i); // duplicate + bit_index.insert(i); + bit_index.insert(i); // duplicate + legacy_index.insert(i); + legacy_index.insert(i); // duplicate + bv_index.insert(i); + bv_index.insert(i); // duplicate + u64_index.insert(i); + u64_index.insert(i); // duplicate + } + + assert_eq!( + byte_index.num_shreds, + legacy_index.num_shreds(), + "ByteShredIndex num_shreds mismatch after duplicates" + ); + assert_eq!( + bit_index.num_shreds, + legacy_index.num_shreds(), + "ShredIndex num_shreds mismatch after duplicates" + ); + assert_eq!( + bv_index.num_shreds, + legacy_index.num_shreds(), + "BvShredIndex num_shreds mismatch after duplicates" + ); + assert_eq!( + u64_index.num_shreds, + legacy_index.num_shreds(), + "U64ShredIndex num_shreds mismatch after duplicates" + ); + } + + #[test] + fn test_edge_case_ranges() { + let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = + create_indices(); + + // Insert some values near edges + let edge_cases = [ + 0, + 1, + 7, + 8, // First byte boundary + 63, + 64, + 65, // u64 boundary + MAX_DATA_SHREDS_PER_SLOT as u64 - 2, // End boundary + ]; + + for idx in edge_cases { + byte_index.insert(idx); + bit_index.insert(idx); + legacy_index.insert(idx); + bv_index.insert(idx); + u64_index.insert(idx); + } + + // Test various range patterns + let ranges = [ + // Start of range + 0..2, + // Cross u64 boundary + 63..65, + // Cross byte boundary + 7..9, + // Full range + 0..MAX_DATA_SHREDS_PER_SLOT as u64, + // End of range + (MAX_DATA_SHREDS_PER_SLOT as u64 - 2)..MAX_DATA_SHREDS_PER_SLOT as u64, + ]; + + for range in ranges.iter() { + let byte_result: Vec<_> = byte_index.range(range.clone()).collect(); + let bit_result: Vec<_> = bit_index.range(range.clone()).collect(); + let legacy_result: Vec<_> = legacy_index.range(range.clone()).copied().collect(); + let bv_result: Vec<_> = bv_index.range(range.clone()).collect(); + let u64_result: Vec<_> = u64_index.range(range.clone()).collect(); + + assert_eq!( + byte_result, legacy_result, + "ByteShredIndex edge case range mismatch for {:?}", + range + ); + assert_eq!( + bit_result, legacy_result, + "BitArrayShredIndex edge case range mismatch for {:?}", + range + ); + assert_eq!( + bv_result, legacy_result, + "BvShredIndex edge case range mismatch for {:?}", + range + ); + assert_eq!( + u64_result, legacy_result, + "U64ShredIndex edge case range mismatch for {:?}", + range + ); + } + } +} + +/// Benchmarks serialization operations with different data densities. +/// +/// This test: +/// - Creates a new empty index +/// - Populates it with specified density (1%, 50%, 90%) +/// - Measures the performance impact of data density on serialization +macro_rules! bench_serialize_density { + ($name:ident, $index:ty, $density:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + + // Calculate number of entries needed for desired density + let num_entries = (MAX_DATA_SHREDS_PER_SLOT as f64 * $density) as u64; + + // Insert entries to achieve target density + for _ in 0..num_entries { + index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); + } + + b.iter(|| { + let serialized = bincode::serialize(&index).unwrap(); + test::black_box(serialized) + }); + } + }; +} + +/// Benchmarks deserialization operations with different data densities. +/// +/// This test: +/// - Creates a new empty index +/// - Populates it with specified density (1%, 50%, 90%) +/// - Serializes it once +/// - Measures the performance of repeated deserialization +macro_rules! bench_deserialize_density { + ($name:ident, $index:ty, $density:expr) => { + #[bench] + fn $name(b: &mut Bencher) { + let mut index = <$index>::new(); + let mut rng = rand::thread_rng(); + + // Calculate number of entries needed for desired density + let num_entries = (MAX_DATA_SHREDS_PER_SLOT as f64 * $density) as u64; + + // Insert entries to achieve target density + for _ in 0..num_entries { + index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); + } + + let serialized = bincode::serialize(&index).unwrap(); + + b.iter(|| { + let deserialized: $index = bincode::deserialize(&serialized).unwrap(); + test::black_box(deserialized) + }); + } + }; +} + +// Serialization benchmarks for each implementation and density +bench_serialize_density!(bench_serialize_sparse_bytes, ByteShredIndex, 0.01); +bench_serialize_density!(bench_serialize_medium_bytes, ByteShredIndex, 0.5); +bench_serialize_density!(bench_serialize_dense_bytes, ByteShredIndex, 0.9); + +bench_serialize_density!(bench_serialize_sparse_bit_array, BitArrayShredIndex, 0.01); +bench_serialize_density!(bench_serialize_medium_bit_array, BitArrayShredIndex, 0.5); +bench_serialize_density!(bench_serialize_dense_bit_array, BitArrayShredIndex, 0.9); + +bench_serialize_density!(bench_serialize_sparse_legacy, LegacyShredIndex, 0.01); +bench_serialize_density!(bench_serialize_medium_legacy, LegacyShredIndex, 0.5); +bench_serialize_density!(bench_serialize_dense_legacy, LegacyShredIndex, 0.9); + +bench_serialize_density!(bench_serialize_sparse_bv, BvShredIndex, 0.01); +bench_serialize_density!(bench_serialize_medium_bv, BvShredIndex, 0.5); +bench_serialize_density!(bench_serialize_dense_bv, BvShredIndex, 0.9); + +bench_serialize_density!(bench_serialize_sparse_u64, U64ShredIndex, 0.01); +bench_serialize_density!(bench_serialize_medium_u64, U64ShredIndex, 0.5); +bench_serialize_density!(bench_serialize_dense_u64, U64ShredIndex, 0.9); + +// Deserialization benchmarks for each implementation and density +bench_deserialize_density!(bench_deserialize_sparse_bytes, ByteShredIndex, 0.01); +bench_deserialize_density!(bench_deserialize_medium_bytes, ByteShredIndex, 0.5); +bench_deserialize_density!(bench_deserialize_dense_bytes, ByteShredIndex, 0.9); + +bench_deserialize_density!(bench_deserialize_sparse_bit_array, BitArrayShredIndex, 0.01); +bench_deserialize_density!(bench_deserialize_medium_bit_array, BitArrayShredIndex, 0.5); +bench_deserialize_density!(bench_deserialize_dense_bit_array, BitArrayShredIndex, 0.9); + +bench_deserialize_density!(bench_deserialize_sparse_legacy, LegacyShredIndex, 0.01); +bench_deserialize_density!(bench_deserialize_medium_legacy, LegacyShredIndex, 0.5); +bench_deserialize_density!(bench_deserialize_dense_legacy, LegacyShredIndex, 0.9); + +bench_deserialize_density!(bench_deserialize_sparse_bv, BvShredIndex, 0.01); +bench_deserialize_density!(bench_deserialize_medium_bv, BvShredIndex, 0.5); +bench_deserialize_density!(bench_deserialize_dense_bv, BvShredIndex, 0.9); + +bench_deserialize_density!(bench_deserialize_sparse_u64, U64ShredIndex, 0.01); +bench_deserialize_density!(bench_deserialize_medium_u64, U64ShredIndex, 0.5); +bench_deserialize_density!(bench_deserialize_dense_u64, U64ShredIndex, 0.9); diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index f3940fe618444d..da5cbb5eef9932 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -251,22 +251,28 @@ impl Index { } impl ShredIndex { + pub fn new() -> Self { + ShredIndex { + index: BTreeSet::new(), + } + } + pub fn num_shreds(&self) -> usize { self.index.len() } - pub(crate) fn range(&self, bounds: R) -> impl Iterator + pub fn range(&self, bounds: R) -> impl Iterator where R: RangeBounds, { self.index.range(bounds) } - pub(crate) fn contains(&self, index: u64) -> bool { + pub fn contains(&self, index: u64) -> bool { self.index.contains(&index) } - pub(crate) fn insert(&mut self, index: u64) { + pub fn insert(&mut self, index: u64) { self.index.insert(index); } } From a33db837aa411ac43bd89e81bd40d258249ee556 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 3 Dec 2024 12:58:54 -0800 Subject: [PATCH 02/44] add custom serde for U64ShredIndex --- ledger/benches/shred_index.rs | 52 +++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/ledger/benches/shred_index.rs b/ledger/benches/shred_index.rs index 23ae4c217025f6..fc0a1e3d300f12 100644 --- a/ledger/benches/shred_index.rs +++ b/ledger/benches/shred_index.rs @@ -120,10 +120,8 @@ impl ByteShredIndex { } } -#[serde_as] -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] struct U64ShredIndex { - #[serde_as(as = "[_; NUM_U64S]")] index: [u64; NUM_U64S], num_shreds: usize, } @@ -137,6 +135,54 @@ impl Default for U64ShredIndex { } } +impl Serialize for U64ShredIndex { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // SAFETY: This is safe because: + // 1. Memory initialization & layout: + // - index is a fixed-size array [u64; NUM_U64S] fully initialized + // at construction and never contains uninitialized memory + // - array elements are contiguous with no padding + // + // 2. Size & alignment: + // - size_of::<[u64; NUM_U64S]> is exactly (8 * NUM_U64S) bytes + // - &self.index is u64-aligned, which satisfies u8 alignment + // + // 3. Lifetime: + // - slice lifetime is tied to &self and is read-only + // + // Note: Deserialization will validate the byte length and safely reconstruct the u64 array + serializer.serialize_bytes(unsafe { + std::slice::from_raw_parts( + &self.index as *const _ as *const u8, + std::mem::size_of::<[u64; NUM_U64S]>(), + ) + }) + } +} + +impl<'de> Deserialize<'de> for U64ShredIndex { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes = <&[u8]>::deserialize(deserializer)?; + if bytes.len() != std::mem::size_of::<[u64; NUM_U64S]>() { + return Err(serde::de::Error::custom("invalid length")); + } + let mut index = [0u64; NUM_U64S]; + bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { + index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); + }); + Ok(Self { + index, + num_shreds: index.iter().map(|x| x.count_ones() as usize).sum(), + }) + } +} + impl U64ShredIndex { fn new() -> Self { Self::default() From 46697c14c13e602eafb27a79d7f107f9e56cc815 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Nov 2024 11:09:18 -0800 Subject: [PATCH 03/44] Add ShredIndex benchmarks --- ledger/benches/shred_index.rs | 1165 --------------------------------- 1 file changed, 1165 deletions(-) delete mode 100644 ledger/benches/shred_index.rs diff --git a/ledger/benches/shred_index.rs b/ledger/benches/shred_index.rs deleted file mode 100644 index fc0a1e3d300f12..00000000000000 --- a/ledger/benches/shred_index.rs +++ /dev/null @@ -1,1165 +0,0 @@ -#![feature(test)] -extern crate test; - -use { - bitvec::{array::BitArray, order::Lsb0}, - rand::Rng, - serde::{Deserialize, Serialize}, - serde_with::serde_as, - solana_ledger::blockstore_meta::ShredIndex as LegacyShredIndex, - std::ops::{Bound, RangeBounds}, - test::Bencher, -}; - -const MAX_DATA_SHREDS_PER_SLOT: usize = 32_768; -const NUM_BYTES: usize = (MAX_DATA_SHREDS_PER_SLOT + 7) / 8; -const NUM_U64S: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; - -#[serde_as] -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -struct ByteShredIndex { - #[serde_as(as = "[_; NUM_BYTES]")] - index: [u8; NUM_BYTES], - num_shreds: usize, -} - -impl Default for ByteShredIndex { - fn default() -> Self { - Self { - index: [0; NUM_BYTES], - num_shreds: 0, - } - } -} - -impl ByteShredIndex { - fn new() -> Self { - Self::default() - } - - pub fn contains(&self, idx: u64) -> bool { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return false; - } - let byte_idx = (idx / 8) as usize; - let bit_idx = (idx % 8) as u8; - (self.index[byte_idx] & (1 << bit_idx)) != 0 - } - - pub fn insert(&mut self, idx: u64) { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return; - } - let byte_idx = (idx / 8) as usize; - let bit_idx = (idx % 8) as u8; - let old_byte = self.index[byte_idx]; - let new_byte = old_byte | (1 << bit_idx); - if old_byte != new_byte { - self.index[byte_idx] = new_byte; - self.num_shreds += 1; - } - } - - pub fn range(&self, bounds: R) -> impl Iterator + '_ - where - R: RangeBounds, - { - let start = match bounds.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + 1, - Bound::Unbounded => 0, - }; - let end = match bounds.end_bound() { - Bound::Included(&n) => n + 1, - Bound::Excluded(&n) => n, - Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, - }; - - let start_byte = (start / 8) as usize; - let end_byte = ((end + 7) / 8) as usize; - - self.index[start_byte..end_byte] - .iter() - .enumerate() - .flat_map(move |(byte_offset, &byte)| { - let base_idx = ((start_byte + byte_offset) * 8) as u64; - - // Determine the range of bits to check in this byte - let lower_bound = if base_idx < start { - (start - base_idx) as u8 - } else { - 0 - }; - - let upper_bound = if base_idx + 8 > end { - (end - base_idx) as u8 - } else { - 8 - }; - - // Create a mask for the relevant bits in this byte - - let lower_mask = !0u8 << lower_bound; - let upper_mask = !0u8 >> (8 - upper_bound); - let mask = byte & lower_mask & upper_mask; - - // Iterate through the bits set in the mask - std::iter::from_fn({ - let mut remaining = mask; - move || { - if remaining == 0 { - None - } else { - let bit_idx = remaining.trailing_zeros() as u8; - remaining &= remaining - 1; - Some(base_idx + bit_idx as u64) - } - } - }) - }) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct U64ShredIndex { - index: [u64; NUM_U64S], - num_shreds: usize, -} - -impl Default for U64ShredIndex { - fn default() -> Self { - Self { - index: [0; NUM_U64S], - num_shreds: 0, - } - } -} - -impl Serialize for U64ShredIndex { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - // SAFETY: This is safe because: - // 1. Memory initialization & layout: - // - index is a fixed-size array [u64; NUM_U64S] fully initialized - // at construction and never contains uninitialized memory - // - array elements are contiguous with no padding - // - // 2. Size & alignment: - // - size_of::<[u64; NUM_U64S]> is exactly (8 * NUM_U64S) bytes - // - &self.index is u64-aligned, which satisfies u8 alignment - // - // 3. Lifetime: - // - slice lifetime is tied to &self and is read-only - // - // Note: Deserialization will validate the byte length and safely reconstruct the u64 array - serializer.serialize_bytes(unsafe { - std::slice::from_raw_parts( - &self.index as *const _ as *const u8, - std::mem::size_of::<[u64; NUM_U64S]>(), - ) - }) - } -} - -impl<'de> Deserialize<'de> for U64ShredIndex { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let bytes = <&[u8]>::deserialize(deserializer)?; - if bytes.len() != std::mem::size_of::<[u64; NUM_U64S]>() { - return Err(serde::de::Error::custom("invalid length")); - } - let mut index = [0u64; NUM_U64S]; - bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { - index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); - }); - Ok(Self { - index, - num_shreds: index.iter().map(|x| x.count_ones() as usize).sum(), - }) - } -} - -impl U64ShredIndex { - fn new() -> Self { - Self::default() - } - - pub fn contains(&self, idx: u64) -> bool { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return false; - } - let word_idx = idx / 64; - let bit_idx = idx % 64; - (self.index[word_idx as usize] & (1 << bit_idx)) != 0 - } - - pub fn insert(&mut self, idx: u64) { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return; - } - let word_idx = idx / 64; - let bit_idx = idx % 64; - let old_word = self.index[word_idx as usize]; - let new_word = old_word | (1 << bit_idx); - if old_word != new_word { - self.index[word_idx as usize] = new_word; - self.num_shreds += 1; - } - } - - /// Provides an iterator over the set shred indices within a specified range. - /// - /// # Algorithm - /// 1. Divide the specified range into 64-bit words. - /// 2. For each word: - /// - Calculate the base index (position of the word * 64). - /// - Process all set bits in the word. - /// - For words overlapping the range boundaries: - /// - Determine the relevant bit range using boundaries. - /// - Mask out bits outside the range. - /// - Use bit manipulation to iterate over set bits efficiently. - /// - /// ## Explanation - /// > Note we're showing 32 bits per word in examples for brevity, but each word is 64 bits. - /// - /// Given range `[75..205]`: - /// - /// Word layout (each word is 64 bits), where each X represents a bit candidate: - /// ```text - /// Word 1 (0–63): [................................] ← Not included (outside range) - /// Word 2 (64–127): [..........XXXXXXXXXXXXXXXXXXXXXX] ← Partial word (start) - /// Word 3 (128–191): [XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] ← Full word (entirely in range) - /// Word 4 (192–255): [XXXXXXXXXXXXXXXXXXX.............] ← Partial word (end) - /// ``` - /// - /// Partial Word 2 (contains start boundary 75): - /// - Base index = 64 - /// - Lower boundary = 75 - 64 = 11 - /// - Lower mask = `11111111111111110000000000000000` - /// - /// Partial Word 4 (contains end boundary 205): - /// - Base index = 192 - /// - Upper boundary = 205 - 192 = 13 - /// - Upper mask = `00000000000000000000000000001111` - /// - /// Final mask = `word & lower_mask & upper_mask` - /// - /// Bit iteration: - /// 1. Apply masks to restrict the bits to the range. - /// 2. While bits remain in the masked word: - /// a. Find the lowest set bit (`trailing_zeros`). - /// b. Add the bit's position to the base index. - /// c. Clear the lowest set bit (`n & (n - 1)`). - /// ``` - pub fn range(&self, bounds: R) -> impl Iterator + '_ - where - R: RangeBounds, - { - let start = match bounds.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + 1, - Bound::Unbounded => 0, - }; - let end = match bounds.end_bound() { - Bound::Included(&n) => n + 1, - Bound::Excluded(&n) => n, - Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, - }; - - let end_word: usize = (((end + 63) / 64) as usize).min(NUM_U64S); - let start_word = ((start / 64) as usize).min(end_word); - - self.index[start_word..end_word] - .iter() - .enumerate() - .flat_map(move |(word_offset, &word)| { - let base_idx = (start_word + word_offset) as u64 * 64; - - let lower_bound = if base_idx < start { - start - base_idx - } else { - 0 - }; - - let upper_bound = if base_idx + 64 > end { - end - base_idx - } else { - 64 - }; - - let lower_mask = !0u64 << lower_bound; - let upper_mask = !0u64 >> (64 - upper_bound); - let mask = word & lower_mask & upper_mask; - - std::iter::from_fn({ - let mut remaining = mask; - move || { - if remaining == 0 { - None - } else { - let bit_idx = remaining.trailing_zeros() as u64; - // Clear the lowest set bit - remaining &= remaining - 1; - Some(base_idx + bit_idx) - } - } - }) - }) - } -} - -type ShredBitArray = BitArray<[u64; NUM_U64S], Lsb0>; - -#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] -struct BitArrayShredIndex { - index: ShredBitArray, - num_shreds: usize, -} - -impl BitArrayShredIndex { - pub fn new() -> Self { - Self { - index: BitArray::ZERO, - num_shreds: 0, - } - } - - pub fn contains(&self, idx: u64) -> bool { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return false; - } - self.index[idx as usize] - } - - pub fn insert(&mut self, idx: u64) { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return; - } - if !self.index[idx as usize] { - self.index.set(idx as usize, true); - self.num_shreds += 1; - } - } - - pub fn range(&self, bounds: R) -> impl Iterator + '_ - where - R: RangeBounds, - { - let start = match bounds.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + 1, - Bound::Unbounded => 0, - }; - let end = match bounds.end_bound() { - Bound::Included(&n) => n + 1, - Bound::Excluded(&n) => n, - Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, - }; - - (start..end).filter(|&idx| self.index[idx as usize]) - } -} - -use bv::BitVec; - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -struct BvShredIndex { - index: BitVec, - num_shreds: usize, -} - -impl Default for BvShredIndex { - fn default() -> Self { - Self { - index: BitVec::new_fill(false, MAX_DATA_SHREDS_PER_SLOT as u64), - num_shreds: 0, - } - } -} - -impl BvShredIndex { - fn new() -> Self { - Self::default() - } - - pub fn contains(&self, idx: u64) -> bool { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return false; - } - self.index.get(idx) - } - - pub fn insert(&mut self, idx: u64) { - if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return; - } - if !self.index.get(idx) { - self.index.set(idx, true); - self.num_shreds += 1; - } - } - - pub fn range(&self, bounds: R) -> impl Iterator + '_ - where - R: RangeBounds, - { - let start = match bounds.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + 1, - Bound::Unbounded => 0, - }; - let end = match bounds.end_bound() { - Bound::Included(&n) => n + 1, - Bound::Excluded(&n) => n, - Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, - }; - - (start..end).filter(|&i| self.index.get(i)) - } -} - -/// Benchmarks range operations with different window sizes on sparse data. -/// -/// This test: -/// - Creates a new empty index -/// - Randomly sets ~10% of bits across the entire possible range -/// - Tests iteration over windows of different sizes (64, 1024, 4096 bits) -/// - Measures the performance impact of window size on range operations -macro_rules! bench_range_window { - ($name:ident, $index:ty, $window_size:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - - // Set ~10% of bits randomly - for _ in 0..(MAX_DATA_SHREDS_PER_SLOT as f64 * 0.1) as usize { - index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); - } - - let start = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64 - $window_size); - b.iter(|| { - let count = index.range(start..(start + $window_size)).count(); - test::black_box(count) - }); - } - }; -} - -/// This test: -/// - Creates a new empty index -/// - Populates it with specified density (10%, 50%, 90%) -/// - Tests performance impact of data density on range operations -/// - Allows configurable range size -macro_rules! bench_range_density { - ($name:ident, $index:ty, $density:expr, $range_size:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - use rand::seq::SliceRandom; - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - - // Calculate number of entries needed for desired density - let range_size = $range_size; - let num_entries = (range_size as f64 * $density) as u64; - - // Insert entries to achieve target density - let start = 0; - let end = start + range_size; - let mut entries: Vec = (start..end).collect(); - entries.shuffle(&mut rng); - entries.truncate(num_entries as usize); - - for entry in entries { - index.insert(entry); - } - - b.iter(|| { - let count = index.range(start..end).count(); - test::black_box(count) - }); - } - }; -} - -/// Benchmarks range operations with different data distributions. -/// -/// This test: -/// - Creates a new empty index with 50% density in 1000..2000 range -/// - Tests three patterns: -/// - Sequential (every other index) -/// - Random (randomly distributed) -/// - Clustered (groups of consecutive indices) -/// - Measures how data distribution affects range operation performance -macro_rules! bench_range_distribution { - ($name:ident, $index:ty, $pattern:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - use rand::seq::SliceRandom; - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - - match $pattern { - "sequential" => { - for i in (1000..2000).step_by(2) { - index.insert(i); - } - } - "random" => { - let mut indices: Vec = (1000..2000).collect(); - indices.shuffle(&mut rng); - indices.truncate(500); - for i in indices { - index.insert(i); - } - } - "clustered" => { - let mut current = 1000u64; - while current < 2000 { - // Insert clusters of 8 consecutive indices - for i in current..current.saturating_add(8).min(2000) { - index.insert(i); - } - // Skip 8 indices before next cluster - current += 16; - } - } - _ => panic!("Unknown pattern"), - } - - b.iter(|| { - let count = index.range(1000..2000).count(); - test::black_box(count) - }); - } - }; -} - -/// Benchmarks insertion operations with different patterns. -/// -/// This test: -/// - Creates a new empty index -/// - Tests three insertion patterns: -/// - Sequential (consecutive indices) -/// - Random (randomly distributed) -/// - Clustered (groups of consecutive indices) -/// - Measures how insertion pattern affects performance -macro_rules! bench_insert_pattern { - ($name:ident, $index:ty, $pattern:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - let mut next_insert = 0u64; - let mut cluster_start = 0u64; - - b.iter(|| { - match $pattern { - "sequential" => { - index.insert(next_insert); - next_insert = (next_insert + 1) % MAX_DATA_SHREDS_PER_SLOT as u64; - } - "random" => { - index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); - } - "clustered" => { - index.insert(cluster_start + (next_insert % 8)); - next_insert += 1; - if next_insert % 8 == 0 { - cluster_start = (cluster_start + 16) % MAX_DATA_SHREDS_PER_SLOT as u64; - } - } - _ => panic!("Unknown pattern"), - } - test::black_box(&index); - }); - } - }; -} - -// Window size benchmarks (64, 1024, 4096 bits) for each implementation -bench_range_window!(bench_window_64_byte, ByteShredIndex, 64); -bench_range_window!(bench_window_1024_byte, ByteShredIndex, 1024); -bench_range_window!(bench_window_4096_byte, ByteShredIndex, 4096); - -bench_range_window!(bench_window_64_bit_array, BitArrayShredIndex, 64); -bench_range_window!(bench_window_1024_bit_array, BitArrayShredIndex, 1024); -bench_range_window!(bench_window_4096_bit_array, BitArrayShredIndex, 4096); - -bench_range_window!(bench_window_64_legacy, LegacyShredIndex, 64); -bench_range_window!(bench_window_1024_legacy, LegacyShredIndex, 1024); -bench_range_window!(bench_window_4096_legacy, LegacyShredIndex, 4096); - -bench_range_window!(bench_window_64_bv, BvShredIndex, 64); -bench_range_window!(bench_window_1024_bv, BvShredIndex, 1024); -bench_range_window!(bench_window_4096_bv, BvShredIndex, 4096); - -bench_range_window!(bench_window_64_u64, U64ShredIndex, 64); -bench_range_window!(bench_window_1024_u64, U64ShredIndex, 1024); -bench_range_window!(bench_window_4096_u64, U64ShredIndex, 4096); - -// Density benchmarks (10%, 50%, 90%) for each implementation -bench_range_density!(bench_density_10_byte, ByteShredIndex, 0.1, 1000); -bench_range_density!(bench_density_50_byte, ByteShredIndex, 0.5, 1000); -bench_range_density!(bench_density_90_byte, ByteShredIndex, 0.9, 1000); -bench_range_density!(bench_density_10_byte_10k, ByteShredIndex, 0.1, 10000); -bench_range_density!(bench_density_50_byte_10k, ByteShredIndex, 0.5, 10000); -bench_range_density!(bench_density_90_byte_10k, ByteShredIndex, 0.9, 10000); - -bench_range_density!(bench_density_10_bit_array, BitArrayShredIndex, 0.1, 1000); -bench_range_density!(bench_density_50_bit_array, BitArrayShredIndex, 0.5, 1000); -bench_range_density!(bench_density_90_bit_array, BitArrayShredIndex, 0.9, 1000); -bench_range_density!( - bench_density_10_bit_array_10k, - BitArrayShredIndex, - 0.1, - 10000 -); -bench_range_density!( - bench_density_50_bit_array_10k, - BitArrayShredIndex, - 0.5, - 10000 -); -bench_range_density!( - bench_density_90_bit_array_10k, - BitArrayShredIndex, - 0.9, - 10000 -); - -bench_range_density!(bench_density_10_legacy, LegacyShredIndex, 0.1, 1000); -bench_range_density!(bench_density_50_legacy, LegacyShredIndex, 0.5, 1000); -bench_range_density!(bench_density_90_legacy, LegacyShredIndex, 0.9, 1000); -bench_range_density!(bench_density_10_legacy_10k, LegacyShredIndex, 0.1, 10000); -bench_range_density!(bench_density_50_legacy_10k, LegacyShredIndex, 0.5, 10000); -bench_range_density!(bench_density_90_legacy_10k, LegacyShredIndex, 0.9, 10000); - -bench_range_density!(bench_density_10_bv, BvShredIndex, 0.1, 1000); -bench_range_density!(bench_density_50_bv, BvShredIndex, 0.5, 1000); -bench_range_density!(bench_density_90_bv, BvShredIndex, 0.9, 1000); -bench_range_density!(bench_density_10_bv_10k, BvShredIndex, 0.1, 10000); -bench_range_density!(bench_density_50_bv_10k, BvShredIndex, 0.5, 10000); -bench_range_density!(bench_density_90_bv_10k, BvShredIndex, 0.9, 10000); - -bench_range_density!(bench_density_10_u64, U64ShredIndex, 0.1, 1000); -bench_range_density!(bench_density_50_u64, U64ShredIndex, 0.5, 1000); -bench_range_density!(bench_density_90_u64, U64ShredIndex, 0.9, 1000); -bench_range_density!(bench_density_10_u64_10k, U64ShredIndex, 0.1, 10000); -bench_range_density!(bench_density_50_u64_10k, U64ShredIndex, 0.5, 10000); -bench_range_density!(bench_density_90_u64_10k, U64ShredIndex, 0.9, 10000); - -// Distribution pattern benchmarks (sequential, random, clustered) for each implementation -bench_range_distribution!(bench_dist_seq_byte, ByteShredIndex, "sequential"); -bench_range_distribution!(bench_dist_rand_byte, ByteShredIndex, "random"); -bench_range_distribution!(bench_dist_clust_byte, ByteShredIndex, "clustered"); - -bench_range_distribution!(bench_dist_seq_bit_array, BitArrayShredIndex, "sequential"); -bench_range_distribution!(bench_dist_rand_bit_array, BitArrayShredIndex, "random"); -bench_range_distribution!(bench_dist_clust_bit_array, BitArrayShredIndex, "clustered"); - -bench_range_distribution!(bench_dist_seq_legacy, LegacyShredIndex, "sequential"); -bench_range_distribution!(bench_dist_rand_legacy, LegacyShredIndex, "random"); -bench_range_distribution!(bench_dist_clust_legacy, LegacyShredIndex, "clustered"); - -bench_range_distribution!(bench_dist_seq_bv, BvShredIndex, "sequential"); -bench_range_distribution!(bench_dist_rand_bv, BvShredIndex, "random"); -bench_range_distribution!(bench_dist_clust_bv, BvShredIndex, "clustered"); - -bench_range_distribution!(bench_dist_seq_u64, U64ShredIndex, "sequential"); -bench_range_distribution!(bench_dist_rand_u64, U64ShredIndex, "random"); -bench_range_distribution!(bench_dist_clust_u64, U64ShredIndex, "clustered"); - -// Insertion pattern benchmarks for each implementation -bench_insert_pattern!(bench_insert_seq_byte, ByteShredIndex, "sequential"); -bench_insert_pattern!(bench_insert_rand_byte, ByteShredIndex, "random"); -bench_insert_pattern!(bench_insert_clust_byte, ByteShredIndex, "clustered"); - -bench_insert_pattern!(bench_insert_seq_bit_array, BitArrayShredIndex, "sequential"); -bench_insert_pattern!(bench_insert_rand_bit_array, BitArrayShredIndex, "random"); -bench_insert_pattern!( - bench_insert_clust_bit_array, - BitArrayShredIndex, - "clustered" -); - -bench_insert_pattern!(bench_insert_seq_legacy, LegacyShredIndex, "sequential"); -bench_insert_pattern!(bench_insert_rand_legacy, LegacyShredIndex, "random"); -bench_insert_pattern!(bench_insert_clust_legacy, LegacyShredIndex, "clustered"); - -bench_insert_pattern!(bench_insert_seq_bv, BvShredIndex, "sequential"); -bench_insert_pattern!(bench_insert_rand_bv, BvShredIndex, "random"); -bench_insert_pattern!(bench_insert_clust_bv, BvShredIndex, "clustered"); - -bench_insert_pattern!(bench_insert_seq_u64, U64ShredIndex, "sequential"); -bench_insert_pattern!(bench_insert_rand_u64, U64ShredIndex, "random"); -bench_insert_pattern!(bench_insert_clust_u64, U64ShredIndex, "clustered"); - -#[cfg(test)] -mod tests { - use super::*; - - macro_rules! test_serde { - ($name:ident, $index:ty) => { - #[test] - fn $name() { - let mut index = <$index>::new(); - for i in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { - if i % 50 == 0 { - index.insert(i); - } - } - let serialized = bincode::serialize(&index).unwrap(); - let deserialized: $index = bincode::deserialize(&serialized).unwrap(); - assert_eq!(index, deserialized); - } - }; - } - - test_serde!(test_serde_byte, ByteShredIndex); - test_serde!(test_serde_bit_array, BitArrayShredIndex); - test_serde!(test_serde_legacy, LegacyShredIndex); - test_serde!(test_serde_bv, BvShredIndex); - test_serde!(test_serde_u64, U64ShredIndex); - - fn create_indices() -> ( - ByteShredIndex, - BitArrayShredIndex, - LegacyShredIndex, - BvShredIndex, - U64ShredIndex, - ) { - let byte_index = ByteShredIndex::new(); - let bit_index = BitArrayShredIndex::new(); - let legacy_index = LegacyShredIndex::default(); - let bv_index = BvShredIndex::new(); - let u64_index = U64ShredIndex::new(); - (byte_index, bit_index, legacy_index, bv_index, u64_index) - } - - #[test] - fn test_basic_insert_contains() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - - let test_indices = [0, 1, 7, 8, 9, 63, 64, 1000, 32767]; - - // Test insertions - for &idx in &test_indices { - byte_index.insert(idx); - bit_index.insert(idx); - legacy_index.insert(idx); - bv_index.insert(idx); - u64_index.insert(idx); - } - - // Test contains - for idx in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { - assert_eq!( - byte_index.contains(idx), - legacy_index.contains(idx), - "ByteShredIndex vs Legacy mismatch at index {}", - idx - ); - assert_eq!( - bit_index.contains(idx), - legacy_index.contains(idx), - "ShredIndex vs Legacy mismatch at index {}", - idx - ); - assert_eq!( - bv_index.contains(idx), - legacy_index.contains(idx), - "BvShredIndex vs Legacy mismatch at index {}", - idx - ); - assert_eq!( - u64_index.contains(idx), - legacy_index.contains(idx), - "U64ShredIndex vs Legacy mismatch at index {}", - idx - ); - } - } - - #[test] - fn test_range_operations() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - - // Insert some patterns - for i in (0..100).step_by(3) { - byte_index.insert(i); - bit_index.insert(i); - legacy_index.insert(i); - bv_index.insert(i); - u64_index.insert(i); - } - - // Test different range sizes - let ranges = [0..10, 90..100, 0..1000, 50..150, 32760..32768]; - - for range in ranges.iter() { - let byte_result: Vec<_> = byte_index.range(range.clone()).collect(); - let bit_result: Vec<_> = bit_index.range(range.clone()).collect(); - let legacy_result: Vec<_> = legacy_index.range(range.clone()).copied().collect(); - let bv_result: Vec<_> = bv_index.range(range.clone()).collect(); - let u64_result: Vec<_> = u64_index.range(range.clone()).collect(); - - assert_eq!( - byte_result, legacy_result, - "ByteShredIndex range results don't match legacy for {:?}", - range - ); - assert_eq!( - bit_result, legacy_result, - "ShredIndex range results don't match legacy for {:?}", - range - ); - assert_eq!( - bv_result, legacy_result, - "BvShredIndex range results don't match legacy for {:?}", - range - ); - assert_eq!( - u64_result, legacy_result, - "U64ShredIndex range results don't match legacy for {:?}", - range - ); - } - } - - #[test] - fn test_boundary_conditions() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - - let boundaries = [ - MAX_DATA_SHREDS_PER_SLOT as u64 - 2, - MAX_DATA_SHREDS_PER_SLOT as u64 - 1, - ]; - - for &idx in &boundaries { - byte_index.insert(idx); - bit_index.insert(idx); - legacy_index.insert(idx); - bv_index.insert(idx); - u64_index.insert(idx); - - assert_eq!( - byte_index.contains(idx), - legacy_index.contains(idx), - "ByteShredIndex boundary mismatch at {}", - idx - ); - assert_eq!( - bit_index.contains(idx), - legacy_index.contains(idx), - "ShredIndex boundary mismatch at {}", - idx - ); - assert_eq!( - bv_index.contains(idx), - legacy_index.contains(idx), - "BvShredIndex boundary mismatch at {}", - idx - ); - assert_eq!( - u64_index.contains(idx), - legacy_index.contains(idx), - "U64ShredIndex boundary mismatch at {}", - idx - ); - } - - // Explicitly test that indices >= MAX_DATA_SHREDS_PER_SLOT are rejected - let invalid_indices = [ - MAX_DATA_SHREDS_PER_SLOT as u64, - MAX_DATA_SHREDS_PER_SLOT as u64 + 1, - ]; - - for &idx in &invalid_indices { - assert!(!byte_index.contains(idx)); - assert!(!bit_index.contains(idx)); - assert!(!bv_index.contains(idx)); - assert!(!u64_index.contains(idx)); - - // Insert should have no effect - byte_index.insert(idx); - bit_index.insert(idx); - bv_index.insert(idx); - u64_index.insert(idx); - assert!(!byte_index.contains(idx)); - assert!(!bit_index.contains(idx)); - assert!(!bv_index.contains(idx)); - assert!(!u64_index.contains(idx)); - } - } - - #[test] - fn test_random_operations() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - let mut rng = rand::thread_rng(); - - // Random insertions - for _ in 0..1000 { - let idx = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64); - byte_index.insert(idx); - bit_index.insert(idx); - legacy_index.insert(idx); - bv_index.insert(idx); - u64_index.insert(idx); - } - - // Test random ranges - for _ in 0..100 { - let start = rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64); - let end = rng.gen_range(start..MAX_DATA_SHREDS_PER_SLOT as u64); - - let byte_result: Vec<_> = byte_index.range(start..end).collect(); - let bit_result: Vec<_> = bit_index.range(start..end).collect(); - let legacy_result: Vec<_> = legacy_index.range(start..end).copied().collect(); - let bv_result: Vec<_> = bv_index.range(start..end).collect(); - let u64_result: Vec<_> = u64_index.range(start..end).collect(); - - assert_eq!( - byte_result, legacy_result, - "ByteShredIndex random range mismatch for {}..{}", - start, end - ); - assert_eq!( - bit_result, legacy_result, - "ShredIndex random range mismatch for {}..{}", - start, end - ); - assert_eq!( - bv_result, legacy_result, - "BvShredIndex random range mismatch for {}..{}", - start, end - ); - assert_eq!( - u64_result, legacy_result, - "U64ShredIndex random range mismatch for {}..{}", - start, end - ); - } - } - - #[test] - fn test_num_shreds_tracking() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - - // Test duplicate insertions - for i in 0..100 { - byte_index.insert(i); - byte_index.insert(i); // duplicate - bit_index.insert(i); - bit_index.insert(i); // duplicate - legacy_index.insert(i); - legacy_index.insert(i); // duplicate - bv_index.insert(i); - bv_index.insert(i); // duplicate - u64_index.insert(i); - u64_index.insert(i); // duplicate - } - - assert_eq!( - byte_index.num_shreds, - legacy_index.num_shreds(), - "ByteShredIndex num_shreds mismatch after duplicates" - ); - assert_eq!( - bit_index.num_shreds, - legacy_index.num_shreds(), - "ShredIndex num_shreds mismatch after duplicates" - ); - assert_eq!( - bv_index.num_shreds, - legacy_index.num_shreds(), - "BvShredIndex num_shreds mismatch after duplicates" - ); - assert_eq!( - u64_index.num_shreds, - legacy_index.num_shreds(), - "U64ShredIndex num_shreds mismatch after duplicates" - ); - } - - #[test] - fn test_edge_case_ranges() { - let (mut byte_index, mut bit_index, mut legacy_index, mut bv_index, mut u64_index) = - create_indices(); - - // Insert some values near edges - let edge_cases = [ - 0, - 1, - 7, - 8, // First byte boundary - 63, - 64, - 65, // u64 boundary - MAX_DATA_SHREDS_PER_SLOT as u64 - 2, // End boundary - ]; - - for idx in edge_cases { - byte_index.insert(idx); - bit_index.insert(idx); - legacy_index.insert(idx); - bv_index.insert(idx); - u64_index.insert(idx); - } - - // Test various range patterns - let ranges = [ - // Start of range - 0..2, - // Cross u64 boundary - 63..65, - // Cross byte boundary - 7..9, - // Full range - 0..MAX_DATA_SHREDS_PER_SLOT as u64, - // End of range - (MAX_DATA_SHREDS_PER_SLOT as u64 - 2)..MAX_DATA_SHREDS_PER_SLOT as u64, - ]; - - for range in ranges.iter() { - let byte_result: Vec<_> = byte_index.range(range.clone()).collect(); - let bit_result: Vec<_> = bit_index.range(range.clone()).collect(); - let legacy_result: Vec<_> = legacy_index.range(range.clone()).copied().collect(); - let bv_result: Vec<_> = bv_index.range(range.clone()).collect(); - let u64_result: Vec<_> = u64_index.range(range.clone()).collect(); - - assert_eq!( - byte_result, legacy_result, - "ByteShredIndex edge case range mismatch for {:?}", - range - ); - assert_eq!( - bit_result, legacy_result, - "BitArrayShredIndex edge case range mismatch for {:?}", - range - ); - assert_eq!( - bv_result, legacy_result, - "BvShredIndex edge case range mismatch for {:?}", - range - ); - assert_eq!( - u64_result, legacy_result, - "U64ShredIndex edge case range mismatch for {:?}", - range - ); - } - } -} - -/// Benchmarks serialization operations with different data densities. -/// -/// This test: -/// - Creates a new empty index -/// - Populates it with specified density (1%, 50%, 90%) -/// - Measures the performance impact of data density on serialization -macro_rules! bench_serialize_density { - ($name:ident, $index:ty, $density:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - - // Calculate number of entries needed for desired density - let num_entries = (MAX_DATA_SHREDS_PER_SLOT as f64 * $density) as u64; - - // Insert entries to achieve target density - for _ in 0..num_entries { - index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); - } - - b.iter(|| { - let serialized = bincode::serialize(&index).unwrap(); - test::black_box(serialized) - }); - } - }; -} - -/// Benchmarks deserialization operations with different data densities. -/// -/// This test: -/// - Creates a new empty index -/// - Populates it with specified density (1%, 50%, 90%) -/// - Serializes it once -/// - Measures the performance of repeated deserialization -macro_rules! bench_deserialize_density { - ($name:ident, $index:ty, $density:expr) => { - #[bench] - fn $name(b: &mut Bencher) { - let mut index = <$index>::new(); - let mut rng = rand::thread_rng(); - - // Calculate number of entries needed for desired density - let num_entries = (MAX_DATA_SHREDS_PER_SLOT as f64 * $density) as u64; - - // Insert entries to achieve target density - for _ in 0..num_entries { - index.insert(rng.gen_range(0..MAX_DATA_SHREDS_PER_SLOT as u64)); - } - - let serialized = bincode::serialize(&index).unwrap(); - - b.iter(|| { - let deserialized: $index = bincode::deserialize(&serialized).unwrap(); - test::black_box(deserialized) - }); - } - }; -} - -// Serialization benchmarks for each implementation and density -bench_serialize_density!(bench_serialize_sparse_bytes, ByteShredIndex, 0.01); -bench_serialize_density!(bench_serialize_medium_bytes, ByteShredIndex, 0.5); -bench_serialize_density!(bench_serialize_dense_bytes, ByteShredIndex, 0.9); - -bench_serialize_density!(bench_serialize_sparse_bit_array, BitArrayShredIndex, 0.01); -bench_serialize_density!(bench_serialize_medium_bit_array, BitArrayShredIndex, 0.5); -bench_serialize_density!(bench_serialize_dense_bit_array, BitArrayShredIndex, 0.9); - -bench_serialize_density!(bench_serialize_sparse_legacy, LegacyShredIndex, 0.01); -bench_serialize_density!(bench_serialize_medium_legacy, LegacyShredIndex, 0.5); -bench_serialize_density!(bench_serialize_dense_legacy, LegacyShredIndex, 0.9); - -bench_serialize_density!(bench_serialize_sparse_bv, BvShredIndex, 0.01); -bench_serialize_density!(bench_serialize_medium_bv, BvShredIndex, 0.5); -bench_serialize_density!(bench_serialize_dense_bv, BvShredIndex, 0.9); - -bench_serialize_density!(bench_serialize_sparse_u64, U64ShredIndex, 0.01); -bench_serialize_density!(bench_serialize_medium_u64, U64ShredIndex, 0.5); -bench_serialize_density!(bench_serialize_dense_u64, U64ShredIndex, 0.9); - -// Deserialization benchmarks for each implementation and density -bench_deserialize_density!(bench_deserialize_sparse_bytes, ByteShredIndex, 0.01); -bench_deserialize_density!(bench_deserialize_medium_bytes, ByteShredIndex, 0.5); -bench_deserialize_density!(bench_deserialize_dense_bytes, ByteShredIndex, 0.9); - -bench_deserialize_density!(bench_deserialize_sparse_bit_array, BitArrayShredIndex, 0.01); -bench_deserialize_density!(bench_deserialize_medium_bit_array, BitArrayShredIndex, 0.5); -bench_deserialize_density!(bench_deserialize_dense_bit_array, BitArrayShredIndex, 0.9); - -bench_deserialize_density!(bench_deserialize_sparse_legacy, LegacyShredIndex, 0.01); -bench_deserialize_density!(bench_deserialize_medium_legacy, LegacyShredIndex, 0.5); -bench_deserialize_density!(bench_deserialize_dense_legacy, LegacyShredIndex, 0.9); - -bench_deserialize_density!(bench_deserialize_sparse_bv, BvShredIndex, 0.01); -bench_deserialize_density!(bench_deserialize_medium_bv, BvShredIndex, 0.5); -bench_deserialize_density!(bench_deserialize_dense_bv, BvShredIndex, 0.9); - -bench_deserialize_density!(bench_deserialize_sparse_u64, U64ShredIndex, 0.01); -bench_deserialize_density!(bench_deserialize_medium_u64, U64ShredIndex, 0.5); -bench_deserialize_density!(bench_deserialize_dense_u64, U64ShredIndex, 0.9); From 55f901e521f2b97684616059c21de30f70c4c7c5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Nov 2024 08:10:57 -0800 Subject: [PATCH 04/44] add ShredIndexNext implementation --- Cargo.lock | 3 - ledger/Cargo.toml | 5 - ledger/src/blockstore_meta.rs | 312 ++++++++++++++++++++++++++++++++-- 3 files changed, 299 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88a849f62aa2c2..0d445475d9c4b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -938,7 +938,6 @@ checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium", - "serde", "tap", "wyz", ] @@ -7333,9 +7332,7 @@ dependencies = [ "assert_matches", "bincode", "bitflags 2.6.0", - "bitvec", "bs58", - "bv", "byteorder", "bzip2", "chrono", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index e9c47f003e64e9..c530d97fd2d305 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -13,8 +13,6 @@ edition = { workspace = true } assert_matches = { workspace = true } bincode = { workspace = true } bitflags = { workspace = true, features = ["serde"] } -bitvec = { version = "1.0.1", features = ["serde"] } -bv = { workspace = true } byteorder = { workspace = true } bzip2 = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } @@ -118,9 +116,6 @@ name = "sigverify_shreds" [[bench]] name = "blockstore" -[[bench]] -name = "shred_index" - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index da5cbb5eef9932..a387effeadf6a2 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -1,14 +1,18 @@ use { - crate::shred::{Shred, ShredType}, + crate::{ + blockstore::MAX_DATA_SHREDS_PER_SLOT, + shred::{Shred, ShredType}, + }, bitflags::bitflags, serde::{Deserialize, Deserializer, Serialize, Serializer}, + serde_with::serde_as, solana_sdk::{ clock::{Slot, UnixTimestamp}, hash::Hash, }, std::{ collections::BTreeSet, - ops::{Range, RangeBounds}, + ops::{Bound, Range, RangeBounds}, }, }; @@ -250,31 +254,244 @@ impl Index { } } +/// Superseded by [`ShredIndexNext`]. +/// +/// TODO: Remove this once new [`ShredIndexNext`] is fully rolled out +/// and no longer relies on it for fallback. impl ShredIndex { - pub fn new() -> Self { - ShredIndex { - index: BTreeSet::new(), - } - } - pub fn num_shreds(&self) -> usize { self.index.len() } - pub fn range(&self, bounds: R) -> impl Iterator + pub(crate) fn range(&self, bounds: R) -> impl Iterator where R: RangeBounds, { self.index.range(bounds) } - pub fn contains(&self, index: u64) -> bool { + pub(crate) fn contains(&self, index: u64) -> bool { self.index.contains(&index) } - pub fn insert(&mut self, index: u64) { + pub(crate) fn insert(&mut self, index: u64) { self.index.insert(index); } + + #[allow(unused)] + fn remove(&mut self, index: u64) { + self.index.remove(&index); + } +} + +const MAX_U64S_PER_SLOT: usize = MAX_DATA_SHREDS_PER_SLOT / 64; + +/// A bit array of shred indices, where each u64 represents 64 shred indices. +/// +/// The current implementation of [`ShredIndexLegacy`] utilizes a [`BTreeSet`] to store +/// shred indices. While [`BTreeSet`] remains efficient as operations are amortized +/// over time, the overhead of the B-tree structure becomes significant when frequently +/// serialized and deserialized. In particular: +/// - **Tree Traversal**: Serialization requires walking the non-contiguous tree structure. +/// - **Reconstruction**: Deserialization involves rebuilding the tree in bulk, +/// including dynamic memory allocations and re-balancing nodes. +/// +/// In contrast, our bit array implementation provides: +/// - **Contiguous Memory**: All bits are stored in a contiguous array of u64 words, +/// allowing direct indexing and efficient memory access patterns. +/// - **Direct Range Access**: Can load only the specific words that overlap with a +/// requested range, avoiding unnecessary traversal. +/// - **Simplified Serialization**: The contiguous memory layout allows for efficient +/// serialization/deserialization without tree reconstruction. +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ShredIndexNext { + #[serde_as(as = "[_; MAX_U64S_PER_SLOT]")] + index: [u64; MAX_U64S_PER_SLOT], + num_shreds: usize, +} + +impl Default for ShredIndexNext { + fn default() -> Self { + Self { + index: [0; MAX_U64S_PER_SLOT], + num_shreds: 0, + } + } +} + +impl ShredIndexNext { + pub fn num_shreds(&self) -> usize { + self.num_shreds + } + + fn index_and_mask(index: u64) -> (usize, u64) { + let word_idx = (index / 64) as usize; + let bit_idx = index % 64; + let mask = 1 << bit_idx; + (word_idx, mask) + } + + #[allow(unused)] + fn remove(&mut self, index: u64) { + if index >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + + let (word_idx, mask) = Self::index_and_mask(index); + + if self.index[word_idx] & mask != 0 { + self.index[word_idx] &= !mask; + self.num_shreds -= 1; + } + } + + #[allow(unused)] + pub(crate) fn contains(&self, idx: u64) -> bool { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return false; + } + let (word_idx, mask) = Self::index_and_mask(idx); + (self.index[word_idx] & mask) != 0 + } + + pub(crate) fn insert(&mut self, idx: u64) { + if idx >= MAX_DATA_SHREDS_PER_SLOT as u64 { + return; + } + let (word_idx, mask) = Self::index_and_mask(idx); + if self.index[word_idx] & mask == 0 { + self.index[word_idx] |= mask; + self.num_shreds += 1; + } + } + + /// Provides an iterator over the set shred indices within a specified range. + /// + /// # Algorithm + /// 1. Divide the specified range into 64-bit words. + /// 2. For each word: + /// - Calculate the base index (position of the word * 64). + /// - Process all set bits in the word. + /// - For words overlapping the range boundaries: + /// - Determine the relevant bit range using boundaries. + /// - Mask out bits outside the range. + /// - Use bit manipulation to iterate over set bits efficiently. + /// + /// ## Explanation + /// > Note we're showing 32 bits per word in examples for brevity, but each word is 64 bits. + /// + /// Given range `[75..205]`: + /// + /// Word layout (each word is 64 bits), where each X represents a bit candidate: + /// ```text + /// Word 1 (0–63): [................................] ← Not included (outside range) + /// Word 2 (64–127): [..........XXXXXXXXXXXXXXXXXXXXXX] ← Partial word (start) + /// Word 3 (128–191): [XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] ← Full word (entirely in range) + /// Word 4 (192–255): [XXXXXXXXXXXXXXXXXXX.............] ← Partial word (end) + /// ``` + /// + /// Partial Word 2 (contains start boundary 75): + /// - Base index = 64 + /// - Lower boundary = 75 - 64 = 11 + /// - Lower mask = `11111111111111110000000000000000` + /// + /// Partial Word 4 (contains end boundary 205): + /// - Base index = 192 + /// - Upper boundary = 205 - 192 = 13 + /// - Upper mask = `00000000000000000000000000001111` + /// + /// Final mask = `word & lower_mask & upper_mask` + /// + /// Bit iteration: + /// 1. Apply masks to restrict the bits to the range. + /// 2. While bits remain in the masked word: + /// a. Find the lowest set bit (`trailing_zeros`). + /// b. Add the bit's position to the base index. + /// c. Clear the lowest set bit (`n & (n - 1)`). + /// ``` + pub(crate) fn range(&self, bounds: R) -> impl Iterator + '_ + where + R: RangeBounds, + { + let start = match bounds.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + 1, + Bound::Unbounded => 0, + }; + let end = match bounds.end_bound() { + Bound::Included(&n) => n + 1, + Bound::Excluded(&n) => n, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + }; + + let end_word: usize = ((end + 63) / 64).min(MAX_U64S_PER_SLOT as u64) as usize; + let start_word = ((start / 64) as usize).min(end_word); + + self.index[start_word..end_word] + .iter() + .enumerate() + .flat_map(move |(word_offset, &word)| { + let base_idx = (start_word + word_offset) as u64 * 64; + + let lower_bound = if base_idx < start { + start - base_idx + } else { + 0 + }; + let upper_bound = if base_idx + 64 > end { + end - base_idx + } else { + 64 + }; + + let lower_mask = !0u64 << lower_bound; + let upper_mask = !0u64 >> (64 - upper_bound); + let mask = word & lower_mask & upper_mask; + + std::iter::from_fn({ + let mut remaining = mask; + move || { + if remaining == 0 { + None + } else { + let bit_idx = remaining.trailing_zeros() as u64; + // Clear the lowest set bit + remaining &= remaining - 1; + Some(base_idx + bit_idx) + } + } + }) + }) + } + + fn iter(&self) -> impl Iterator + '_ { + self.range(0..MAX_DATA_SHREDS_PER_SLOT as u64) + } +} + +impl FromIterator for ShredIndexNext { + fn from_iter>(iter: T) -> Self { + let mut next_index = ShredIndexNext::default(); + for idx in iter { + next_index.insert(idx); + } + next_index + } +} + +impl From for ShredIndexNext { + fn from(value: ShredIndex) -> Self { + value.index.into_iter().collect() + } +} + +impl From for ShredIndex { + fn from(value: ShredIndexNext) -> Self { + ShredIndex { + index: value.iter().collect(), + } + } } impl SlotMeta { @@ -643,7 +860,7 @@ mod test { .collect::>() .choose_multiple(&mut rng, erasure_config.num_data) { - index.data_mut().index.remove(&idx); + index.data_mut().remove(idx); assert_eq!(e_meta.status(&index), CanRecover); } @@ -656,12 +873,81 @@ mod test { .collect::>() .choose_multiple(&mut rng, erasure_config.num_coding) { - index.coding_mut().index.remove(&idx); + index.coding_mut().remove(idx); assert_eq!(e_meta.status(&index), DataFull); } } + #[test] + fn shred_index_legacy_compat() { + use rand::Rng; + let mut legacy = ShredIndex::default(); + let mut next_index = ShredIndexNext::default(); + + for i in (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3) { + next_index.insert(i); + legacy.insert(i); + } + + for &i in legacy.index.iter() { + assert!(next_index.contains(i)); + } + + assert_eq!(next_index.num_shreds(), legacy.num_shreds()); + + let rand_range = rand::thread_rng().gen_range(1000..MAX_DATA_SHREDS_PER_SLOT as u64); + assert_eq!( + next_index.range(0..rand_range).sum::(), + legacy.range(0..rand_range).sum::() + ); + + assert_eq!(ShredIndexNext::from(legacy.clone()), next_index); + assert_eq!(ShredIndex::from(next_index), legacy); + } + + #[test] + fn test_shred_index_next_boundary_conditions() { + let mut index = ShredIndexNext::default(); + + // First possible index + index.insert(0); + // Last index in first word + index.insert(63); + // First index in second word + index.insert(64); + // Last index in second word + index.insert(127); + // Last valid index + index.insert(MAX_DATA_SHREDS_PER_SLOT as u64 - 1); + // Should be ignored (too large) + index.insert(MAX_DATA_SHREDS_PER_SLOT as u64); + + // Verify contents + assert!(index.contains(0)); + assert!(index.contains(63)); + assert!(index.contains(64)); + assert!(index.contains(127)); + assert!(index.contains(MAX_DATA_SHREDS_PER_SLOT as u64 - 1)); + assert!(!index.contains(MAX_DATA_SHREDS_PER_SLOT as u64)); + + // Cross-word boundary + assert_eq!(index.range(50..70).collect::>(), vec![63, 64]); + // Full first word + assert_eq!(index.range(0..64).collect::>(), vec![0, 63]); + // Full second word + assert_eq!(index.range(64..128).collect::>(), vec![64, 127]); + + // Empty ranges + assert_eq!(index.range(0..0).count(), 0); + assert_eq!(index.range(1..1).count(), 0); + + // Test range that exceeds max + let oversized_range = index.range(0..MAX_DATA_SHREDS_PER_SLOT as u64 + 1); + assert_eq!(oversized_range.count(), 5); + assert_eq!(index.num_shreds(), 5); + } + #[test] fn test_connected_flags_compatibility() { // Define a couple structs with bool and ConnectedFlags to illustrate From b52dc75ebec0c55c52e6bbbc945c2658a3041980 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 3 Dec 2024 08:33:06 -0800 Subject: [PATCH 05/44] Add ShredIndexNext support to Blockstore::get_index --- ledger/src/blockstore.rs | 11 ++++++++++- ledger/src/blockstore_db.rs | 22 +++++++++++++++++++--- ledger/src/blockstore_meta.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 3669b9fc0581c1..2bff8218000c31 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2407,7 +2407,16 @@ impl Blockstore { } pub fn get_index(&self, slot: Slot) -> Result> { - self.index_cf.get(slot) + self.index_cf.get_with(slot, |slice| { + let index: bincode::Result = bincode::deserialize(slice); + match index { + Ok(index) => Ok(index), + Err(_) => { + let index_next: IndexNext = bincode::deserialize(slice)?; + Ok(index_next.into()) + } + } + }) } /// Manually update the meta for a slot. diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 6289bcab916ddd..dd2e1cb6d6e404 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1,8 +1,7 @@ pub use rocksdb::Direction as IteratorDirection; use { crate::{ - blockstore_meta, - blockstore_meta::MerkleRootMeta, + blockstore_meta::{self, MerkleRootMeta}, blockstore_metrics::{ maybe_enable_rocksdb_perf, report_rocksdb_read_perf, report_rocksdb_write_perf, BlockstoreRocksDbColumnFamilyMetrics, PerfSamplingStatus, PERF_METRIC_OP_NAME_GET, @@ -1709,14 +1708,31 @@ where self.get_raw(&C::key(key)) } + /// Get the value associated with a key, applying a function to the raw bytes. + /// + /// Like [`get`](Self::get), but delegates deserialization to a provided function. + pub fn get_with(&self, key: C::Index, f: F) -> Result> + where + F: FnOnce(&[u8]) -> Result, + { + self.get_raw_with(&C::key(key), f) + } + pub fn get_raw(&self, key: &[u8]) -> Result> { + self.get_raw_with(key, |slice| Ok(deserialize(slice)?)) + } + + pub fn get_raw_with(&self, key: &[u8], f: F) -> Result> + where + F: FnOnce(&[u8]) -> Result, + { let mut result = Ok(None); let is_perf_enabled = maybe_enable_rocksdb_perf( self.column_options.rocks_perf_sample_interval, &self.read_perf_status, ); if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? { - let value = deserialize(pinnable_slice.as_ref())?; + let value = f(pinnable_slice.as_ref())?; result = Ok(Some(value)) } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index a387effeadf6a2..e0bd42d40260ca 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -116,6 +116,33 @@ pub struct Index { coding: ShredIndex, } +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct IndexNext { + pub slot: Slot, + data: ShredIndexNext, + coding: ShredIndexNext, +} + +impl From for Index { + fn from(index: IndexNext) -> Self { + Index { + slot: index.slot, + data: index.data.into(), + coding: index.coding.into(), + } + } +} + +impl From for IndexNext { + fn from(index: Index) -> Self { + IndexNext { + slot: index.slot, + data: index.data.into(), + coding: index.coding.into(), + } + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub struct ShredIndex { /// Map representing presence/absence of shreds From 14507b3a3d04d86cafbc9f613bcb3a6e3c598530 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 3 Dec 2024 12:24:59 -0800 Subject: [PATCH 06/44] Use custom serde for ShredIndexNext to avoid collisions --- Cargo.lock | 1 - ledger/Cargo.toml | 1 - ledger/src/blockstore_meta.rs | 88 +++++++++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d445475d9c4b3..f603e6ed027000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7361,7 +7361,6 @@ dependencies = [ "scopeguard", "serde", "serde_bytes", - "serde_with", "sha2 0.10.8", "solana-account-decoder", "solana-accounts-db", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index c530d97fd2d305..f4b1a560a52002 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -40,7 +40,6 @@ reed-solomon-erasure = { workspace = true, features = ["simd-accel"] } scopeguard = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } -serde_with = { workspace = true } sha2 = { workspace = true } solana-account-decoder = { workspace = true } solana-accounts-db = { workspace = true } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index e0bd42d40260ca..722aa0f7bd312b 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -5,7 +5,6 @@ use { }, bitflags::bitflags, serde::{Deserialize, Deserializer, Serialize, Serializer}, - serde_with::serde_as, solana_sdk::{ clock::{Slot, UnixTimestamp}, hash::Hash, @@ -330,10 +329,8 @@ const MAX_U64S_PER_SLOT: usize = MAX_DATA_SHREDS_PER_SLOT / 64; /// requested range, avoiding unnecessary traversal. /// - **Simplified Serialization**: The contiguous memory layout allows for efficient /// serialization/deserialization without tree reconstruction. -#[serde_as] -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ShredIndexNext { - #[serde_as(as = "[_; MAX_U64S_PER_SLOT]")] index: [u64; MAX_U64S_PER_SLOT], num_shreds: usize, } @@ -347,6 +344,55 @@ impl Default for ShredIndexNext { } } +impl Serialize for ShredIndexNext { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // SAFETY: This is safe because: + // 1. Memory initialization & layout: + // - index is a fixed-size array [u64; MAX_U64S_PER_SLOT] fully initialized + // at construction and never contains uninitialized memory + // - array elements are contiguous with no padding + // + // 2. Size & alignment: + // - size_of::<[u64; MAX_U64S_PER_SLOT]> is exactly (8 * MAX_U64S_PER_SLOT) bytes + // - &self.index is u64-aligned, which satisfies u8 alignment + // + // 3. Lifetime: + // - slice lifetime is tied to &self and is read-only + // + // Note: Deserialization will validate the byte length and safely reconstruct the u64 array + serializer.serialize_bytes(unsafe { + std::slice::from_raw_parts( + &self.index as *const _ as *const u8, + std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>(), + ) + }) + } +} + +impl<'de> Deserialize<'de> for ShredIndexNext { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes = <&[u8]>::deserialize(deserializer)?; + if bytes.len() != std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>() { + return Err(serde::de::Error::custom("invalid length")); + } + let mut index = [0u64; MAX_U64S_PER_SLOT]; + bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { + // Unwrap is safe because `chunks_exact` guarantees the length + index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); + }); + Ok(Self { + index, + num_shreds: index.iter().map(|x| x.count_ones() as usize).sum(), + }) + } +} + impl ShredIndexNext { pub fn num_shreds(&self) -> usize { self.num_shreds @@ -906,6 +952,40 @@ mod test { } } + #[test] + fn shred_index_next_serde() { + let index: ShredIndexNext = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); + let serialized = bincode::serialize(&index).unwrap(); + let deserialized = bincode::deserialize::(&serialized).unwrap(); + assert_eq!(index, deserialized); + } + + #[test] + fn shred_index_collision() { + let mut index = ShredIndex::default(); + // Create a `ShredIndex` that is exactly the expected size of `ShredIndexNext` + for i in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { + index.insert(i); + } + let serialized = bincode::serialize(&index).unwrap(); + // Attempt to deserialize as `ShredIndexNext` + let deserialized = bincode::deserialize::(&serialized); + assert!(deserialized.is_err()); + } + + #[test] + fn shred_index_next_collision() { + let index = ShredIndexNext::default(); + let serialized = bincode::serialize(&index).unwrap(); + let deserialized = bincode::deserialize::(&serialized); + assert!(deserialized.is_err()); + + let index: ShredIndexNext = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); + let serialized = bincode::serialize(&index).unwrap(); + let deserialized = bincode::deserialize::(&serialized); + assert!(deserialized.is_err()); + } + #[test] fn shred_index_legacy_compat() { use rand::Rng; From 090d0c08cc1dfc6f92aa250e7377fcc9956892ed Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 09:31:48 +0400 Subject: [PATCH 07/44] explicitly use bitwise ops for index / mask --- ledger/src/blockstore_meta.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 722aa0f7bd312b..416f3c6f1b87f7 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -399,8 +399,10 @@ impl ShredIndexNext { } fn index_and_mask(index: u64) -> (usize, u64) { - let word_idx = (index / 64) as usize; - let bit_idx = index % 64; + // index / 64 + let word_idx = (index >> 6) as usize; + // index % 64 + let bit_idx = index & 63; let mask = 1 << bit_idx; (word_idx, mask) } From 9fe23536fa0bb3a08b96618197fa6eb99456e064 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 09:35:52 +0400 Subject: [PATCH 08/44] &= !mask -> ^= mask --- ledger/src/blockstore_meta.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 416f3c6f1b87f7..acfb3176c3823d 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -416,7 +416,7 @@ impl ShredIndexNext { let (word_idx, mask) = Self::index_and_mask(index); if self.index[word_idx] & mask != 0 { - self.index[word_idx] &= !mask; + self.index[word_idx] ^= mask; self.num_shreds -= 1; } } From 99b2dd200d677150028671c45710ed320450ccff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 09:39:26 +0400 Subject: [PATCH 09/44] assert index not out of bounds in `remove` --- ledger/src/blockstore_meta.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index acfb3176c3823d..c13957337ae1ab 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -409,9 +409,10 @@ impl ShredIndexNext { #[allow(unused)] fn remove(&mut self, index: u64) { - if index >= MAX_DATA_SHREDS_PER_SLOT as u64 { - return; - } + assert!( + index < MAX_DATA_SHREDS_PER_SLOT as u64, + "index out of bounds. {index} >= {MAX_DATA_SHREDS_PER_SLOT}" + ); let (word_idx, mask) = Self::index_and_mask(index); From 90ca29d8970bed196165b951d2b053065f83aaaf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 09:45:11 +0400 Subject: [PATCH 10/44] Round up `MAX_U64S_PER_SLOT` to accommodate %64!=0 --- ledger/src/blockstore_meta.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index c13957337ae1ab..2d577a3105b11c 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -310,7 +310,7 @@ impl ShredIndex { } } -const MAX_U64S_PER_SLOT: usize = MAX_DATA_SHREDS_PER_SLOT / 64; +const MAX_U64S_PER_SLOT: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; /// A bit array of shred indices, where each u64 represents 64 shred indices. /// From 2c6a43e03cd58deb6c99a2c61b54565c98d43a5e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 10:23:07 +0400 Subject: [PATCH 11/44] Add comment about migration strategy --- ledger/src/blockstore.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 0ad32edd17e822..07b5d37410f861 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2490,6 +2490,12 @@ impl Blockstore { } pub fn get_index(&self, slot: Slot) -> Result> { + // Migration strategy for new column format: + // 1. Release 1: Add ability to read new format as fallback, keep writing old format + // 2. Release 2: Switch to writing new format, keep reading old format as fallback + // 3. Release 3: Remove old format support once stable + // This allows safe downgrade to Release 1 since it can read both formats + // https://github.com/anza-xyz/agave/issues/3570 self.index_cf.get_with(slot, |slice| { let index: bincode::Result = bincode::deserialize(slice); match index { From cac09128932fe6f12e6f20bf44d64427372cb0e8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 12:40:10 +0400 Subject: [PATCH 12/44] IndexNext -> IndexV2 --- ledger/src/blockstore.rs | 2 +- ledger/src/blockstore_meta.rs | 58 +++++++++++++++++------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 07b5d37410f861..591296c215cbb7 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2501,7 +2501,7 @@ impl Blockstore { match index { Ok(index) => Ok(index), Err(_) => { - let index_next: IndexNext = bincode::deserialize(slice)?; + let index_next: IndexV2 = bincode::deserialize(slice)?; Ok(index_next.into()) } } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 0ce131ba119a02..9626301c5e9ba3 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -116,14 +116,14 @@ pub struct Index { } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] -pub struct IndexNext { +pub struct IndexV2 { pub slot: Slot, - data: ShredIndexNext, - coding: ShredIndexNext, + data: ShredIndexV2, + coding: ShredIndexV2, } -impl From for Index { - fn from(index: IndexNext) -> Self { +impl From for Index { + fn from(index: IndexV2) -> Self { Index { slot: index.slot, data: index.data.into(), @@ -132,9 +132,9 @@ impl From for Index { } } -impl From for IndexNext { +impl From for IndexV2 { fn from(index: Index) -> Self { - IndexNext { + IndexV2 { slot: index.slot, data: index.data.into(), coding: index.coding.into(), @@ -280,9 +280,9 @@ impl Index { } } -/// Superseded by [`ShredIndexNext`]. +/// Superseded by [`ShredIndexV2`]. /// -/// TODO: Remove this once new [`ShredIndexNext`] is fully rolled out +/// TODO: Remove this once new [`ShredIndexV2`] is fully rolled out /// and no longer relies on it for fallback. impl ShredIndex { pub fn num_shreds(&self) -> usize { @@ -330,12 +330,12 @@ const MAX_U64S_PER_SLOT: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; /// - **Simplified Serialization**: The contiguous memory layout allows for efficient /// serialization/deserialization without tree reconstruction. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct ShredIndexNext { +pub struct ShredIndexV2 { index: [u64; MAX_U64S_PER_SLOT], num_shreds: usize, } -impl Default for ShredIndexNext { +impl Default for ShredIndexV2 { fn default() -> Self { Self { index: [0; MAX_U64S_PER_SLOT], @@ -344,7 +344,7 @@ impl Default for ShredIndexNext { } } -impl Serialize for ShredIndexNext { +impl Serialize for ShredIndexV2 { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, @@ -372,7 +372,7 @@ impl Serialize for ShredIndexNext { } } -impl<'de> Deserialize<'de> for ShredIndexNext { +impl<'de> Deserialize<'de> for ShredIndexV2 { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, @@ -393,7 +393,7 @@ impl<'de> Deserialize<'de> for ShredIndexNext { } } -impl ShredIndexNext { +impl ShredIndexV2 { pub fn num_shreds(&self) -> usize { self.num_shreds } @@ -546,9 +546,9 @@ impl ShredIndexNext { } } -impl FromIterator for ShredIndexNext { +impl FromIterator for ShredIndexV2 { fn from_iter>(iter: T) -> Self { - let mut next_index = ShredIndexNext::default(); + let mut next_index = ShredIndexV2::default(); for idx in iter { next_index.insert(idx); } @@ -556,14 +556,14 @@ impl FromIterator for ShredIndexNext { } } -impl From for ShredIndexNext { +impl From for ShredIndexV2 { fn from(value: ShredIndex) -> Self { value.index.into_iter().collect() } } -impl From for ShredIndex { - fn from(value: ShredIndexNext) -> Self { +impl From for ShredIndex { + fn from(value: ShredIndexV2) -> Self { ShredIndex { index: value.iter().collect(), } @@ -957,33 +957,33 @@ mod test { #[test] fn shred_index_next_serde() { - let index: ShredIndexNext = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); + let index: ShredIndexV2 = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); let serialized = bincode::serialize(&index).unwrap(); - let deserialized = bincode::deserialize::(&serialized).unwrap(); + let deserialized = bincode::deserialize::(&serialized).unwrap(); assert_eq!(index, deserialized); } #[test] fn shred_index_collision() { let mut index = ShredIndex::default(); - // Create a `ShredIndex` that is exactly the expected size of `ShredIndexNext` + // Create a `ShredIndex` that is exactly the expected size of `ShredIndexV2` for i in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { index.insert(i); } let serialized = bincode::serialize(&index).unwrap(); - // Attempt to deserialize as `ShredIndexNext` - let deserialized = bincode::deserialize::(&serialized); + // Attempt to deserialize as `ShredIndexV2` + let deserialized = bincode::deserialize::(&serialized); assert!(deserialized.is_err()); } #[test] fn shred_index_next_collision() { - let index = ShredIndexNext::default(); + let index = ShredIndexV2::default(); let serialized = bincode::serialize(&index).unwrap(); let deserialized = bincode::deserialize::(&serialized); assert!(deserialized.is_err()); - let index: ShredIndexNext = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); + let index: ShredIndexV2 = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); let serialized = bincode::serialize(&index).unwrap(); let deserialized = bincode::deserialize::(&serialized); assert!(deserialized.is_err()); @@ -993,7 +993,7 @@ mod test { fn shred_index_legacy_compat() { use rand::Rng; let mut legacy = ShredIndex::default(); - let mut next_index = ShredIndexNext::default(); + let mut next_index = ShredIndexV2::default(); for i in (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3) { next_index.insert(i); @@ -1012,13 +1012,13 @@ mod test { legacy.range(0..rand_range).sum::() ); - assert_eq!(ShredIndexNext::from(legacy.clone()), next_index); + assert_eq!(ShredIndexV2::from(legacy.clone()), next_index); assert_eq!(ShredIndex::from(next_index), legacy); } #[test] fn test_shred_index_next_boundary_conditions() { - let mut index = ShredIndexNext::default(); + let mut index = ShredIndexV2::default(); // First possible index index.insert(0); From 2a662d9dd74ebd0ec3654280f39dc64f12ab5b6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Dec 2024 14:28:47 +0400 Subject: [PATCH 13/44] forward compatibility for increases in MAX_DATA_SHREDS_PER_SLOT --- ledger/src/blockstore_meta.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 9626301c5e9ba3..900025c7ad9a02 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -378,8 +378,12 @@ impl<'de> Deserialize<'de> for ShredIndexV2 { D: serde::Deserializer<'de>, { let bytes = <&[u8]>::deserialize(deserializer)?; - if bytes.len() != std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>() { - return Err(serde::de::Error::custom("invalid length")); + // Accept input smaller than our current fixed array size to maintain compatibility with older + // versions that had fewer shreds per slot. This is safe because smaller sets are stored in + // our fixed array with any remaining space automatically zeroed. This approach ensures we can + // read data from any previous version while enforcing a maximum size limit. + if bytes.len() > std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>() { + return Err(serde::de::Error::custom("input too large")); } let mut index = [0u64; MAX_U64S_PER_SLOT]; bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { From 5da104cae5cad44c6a9882e12ad6b8b9a7eff554 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 18:09:01 +0400 Subject: [PATCH 14/44] use serialize_tuple to avoid recounting set bits --- ledger/src/blockstore_meta.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 900025c7ad9a02..38e80864f2100c 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -349,6 +349,9 @@ impl Serialize for ShredIndexV2 { where S: serde::Serializer, { + use serde::ser::SerializeTuple; + + let mut tuple = serializer.serialize_tuple(2)?; // SAFETY: This is safe because: // 1. Memory initialization & layout: // - index is a fixed-size array [u64; MAX_U64S_PER_SLOT] fully initialized @@ -363,12 +366,14 @@ impl Serialize for ShredIndexV2 { // - slice lifetime is tied to &self and is read-only // // Note: Deserialization will validate the byte length and safely reconstruct the u64 array - serializer.serialize_bytes(unsafe { + tuple.serialize_element(&serde_bytes::Bytes::new(unsafe { std::slice::from_raw_parts( &self.index as *const _ as *const u8, std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>(), ) - }) + }))?; + tuple.serialize_element(&self.num_shreds)?; + tuple.end() } } @@ -377,7 +382,7 @@ impl<'de> Deserialize<'de> for ShredIndexV2 { where D: serde::Deserializer<'de>, { - let bytes = <&[u8]>::deserialize(deserializer)?; + let (bytes, num_shreds) = <(&[u8], usize)>::deserialize(deserializer)?; // Accept input smaller than our current fixed array size to maintain compatibility with older // versions that had fewer shreds per slot. This is safe because smaller sets are stored in // our fixed array with any remaining space automatically zeroed. This approach ensures we can @@ -390,10 +395,7 @@ impl<'de> Deserialize<'de> for ShredIndexV2 { // Unwrap is safe because `chunks_exact` guarantees the length index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); }); - Ok(Self { - index, - num_shreds: index.iter().map(|x| x.count_ones() as usize).sum(), - }) + Ok(Self { index, num_shreds }) } } From e416a70c0baf118017ea27d82d98e5f93ba45be4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 18:17:21 +0400 Subject: [PATCH 15/44] remove "next" terminology --- ledger/src/blockstore.rs | 4 ++-- ledger/src/blockstore_meta.rs | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 591296c215cbb7..b118de19579809 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2501,8 +2501,8 @@ impl Blockstore { match index { Ok(index) => Ok(index), Err(_) => { - let index_next: IndexV2 = bincode::deserialize(slice)?; - Ok(index_next.into()) + let index: IndexV2 = bincode::deserialize(slice)?; + Ok(index.into()) } } }) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 38e80864f2100c..e8ab8c3af8e1c6 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -554,11 +554,11 @@ impl ShredIndexV2 { impl FromIterator for ShredIndexV2 { fn from_iter>(iter: T) -> Self { - let mut next_index = ShredIndexV2::default(); + let mut index = ShredIndexV2::default(); for idx in iter { - next_index.insert(idx); + index.insert(idx); } - next_index + index } } @@ -962,7 +962,7 @@ mod test { } #[test] - fn shred_index_next_serde() { + fn shred_index_v2_serde() { let index: ShredIndexV2 = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); let serialized = bincode::serialize(&index).unwrap(); let deserialized = bincode::deserialize::(&serialized).unwrap(); @@ -983,7 +983,7 @@ mod test { } #[test] - fn shred_index_next_collision() { + fn shred_index_v2_collision() { let index = ShredIndexV2::default(); let serialized = bincode::serialize(&index).unwrap(); let deserialized = bincode::deserialize::(&serialized); @@ -999,27 +999,27 @@ mod test { fn shred_index_legacy_compat() { use rand::Rng; let mut legacy = ShredIndex::default(); - let mut next_index = ShredIndexV2::default(); + let mut v2 = ShredIndexV2::default(); for i in (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3) { - next_index.insert(i); + v2.insert(i); legacy.insert(i); } for &i in legacy.index.iter() { - assert!(next_index.contains(i)); + assert!(v2.contains(i)); } - assert_eq!(next_index.num_shreds(), legacy.num_shreds()); + assert_eq!(v2.num_shreds(), legacy.num_shreds()); let rand_range = rand::thread_rng().gen_range(1000..MAX_DATA_SHREDS_PER_SLOT as u64); assert_eq!( - next_index.range(0..rand_range).sum::(), + v2.range(0..rand_range).sum::(), legacy.range(0..rand_range).sum::() ); - assert_eq!(ShredIndexV2::from(legacy.clone()), next_index); - assert_eq!(ShredIndex::from(next_index), legacy); + assert_eq!(ShredIndexV2::from(legacy.clone()), v2); + assert_eq!(ShredIndex::from(v2), legacy); } #[test] From 01bdfbba22cdb44fea4cd791f01d4275cfdb7406 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 18:25:48 +0400 Subject: [PATCH 16/44] address naming and alias nits --- ledger/src/blockstore_meta.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index e8ab8c3af8e1c6..28c5dc9f14b127 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -310,7 +310,8 @@ impl ShredIndex { } } -const MAX_U64S_PER_SLOT: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; +const NUM_U64_PER_SHRED_INDEX_V2: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; +type ShredIndexV2Inner = [u64; NUM_U64_PER_SHRED_INDEX_V2]; /// A bit array of shred indices, where each u64 represents 64 shred indices. /// @@ -331,14 +332,14 @@ const MAX_U64S_PER_SLOT: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; /// serialization/deserialization without tree reconstruction. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ShredIndexV2 { - index: [u64; MAX_U64S_PER_SLOT], + index: ShredIndexV2Inner, num_shreds: usize, } impl Default for ShredIndexV2 { fn default() -> Self { Self { - index: [0; MAX_U64S_PER_SLOT], + index: [0; NUM_U64_PER_SHRED_INDEX_V2], num_shreds: 0, } } @@ -369,7 +370,7 @@ impl Serialize for ShredIndexV2 { tuple.serialize_element(&serde_bytes::Bytes::new(unsafe { std::slice::from_raw_parts( &self.index as *const _ as *const u8, - std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>(), + std::mem::size_of::(), ) }))?; tuple.serialize_element(&self.num_shreds)?; @@ -387,10 +388,10 @@ impl<'de> Deserialize<'de> for ShredIndexV2 { // versions that had fewer shreds per slot. This is safe because smaller sets are stored in // our fixed array with any remaining space automatically zeroed. This approach ensures we can // read data from any previous version while enforcing a maximum size limit. - if bytes.len() > std::mem::size_of::<[u64; MAX_U64S_PER_SLOT]>() { + if bytes.len() > std::mem::size_of::() { return Err(serde::de::Error::custom("input too large")); } - let mut index = [0u64; MAX_U64S_PER_SLOT]; + let mut index = [0u64; NUM_U64_PER_SHRED_INDEX_V2]; bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { // Unwrap is safe because `chunks_exact` guarantees the length index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); @@ -507,7 +508,7 @@ impl ShredIndexV2 { Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, }; - let end_word: usize = ((end + 63) / 64).min(MAX_U64S_PER_SLOT as u64) as usize; + let end_word: usize = ((end + 63) / 64).min(NUM_U64_PER_SHRED_INDEX_V2 as u64) as usize; let start_word = ((start / 64) as usize).min(end_word); self.index[start_word..end_word] From 28a34db353adcd23e502f2180472914c9186e9a0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 18:27:47 +0400 Subject: [PATCH 17/44] add remove tests --- ledger/src/blockstore_meta.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 28c5dc9f14b127..641d743f52b771 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -1063,6 +1063,19 @@ mod test { let oversized_range = index.range(0..MAX_DATA_SHREDS_PER_SLOT as u64 + 1); assert_eq!(oversized_range.count(), 5); assert_eq!(index.num_shreds(), 5); + + index.remove(0); + assert!(!index.contains(0)); + index.remove(63); + assert!(!index.contains(63)); + index.remove(64); + assert!(!index.contains(64)); + index.remove(127); + assert!(!index.contains(127)); + index.remove(MAX_DATA_SHREDS_PER_SLOT as u64 - 1); + assert!(!index.contains(MAX_DATA_SHREDS_PER_SLOT as u64 - 1)); + + assert_eq!(index.num_shreds(), 0); } #[test] From a510eed3990273e08f1bc8285a6a494c9fe28146 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 23:58:55 +0400 Subject: [PATCH 18/44] add `#[cfg(test)]` to `.remove` for clarity --- ledger/src/blockstore_meta.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 641d743f52b771..d372f165c19e3c 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -304,7 +304,7 @@ impl ShredIndex { self.index.insert(index); } - #[allow(unused)] + #[cfg(test)] fn remove(&mut self, index: u64) { self.index.remove(&index); } @@ -414,7 +414,7 @@ impl ShredIndexV2 { (word_idx, mask) } - #[allow(unused)] + #[cfg(test)] fn remove(&mut self, index: u64) { assert!( index < MAX_DATA_SHREDS_PER_SLOT as u64, From c8cc254566c9690439df48d028295ec64d2f4668 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Dec 2024 23:59:43 +0400 Subject: [PATCH 19/44] next -> v2 --- ledger/src/blockstore_meta.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index d372f165c19e3c..b1a021d91cdde3 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -1024,7 +1024,7 @@ mod test { } #[test] - fn test_shred_index_next_boundary_conditions() { + fn test_shred_index_v2_boundary_conditions() { let mut index = ShredIndexV2::default(); // First possible index From 4fd710a28da845f8547660b6fb53a6ea7bc29c77 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 10 Dec 2024 22:12:49 +0400 Subject: [PATCH 20/44] fix dangling reference to ShredIndexLegacy --- ledger/src/blockstore_meta.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index b1a021d91cdde3..2f29c3cd7d6359 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -315,7 +315,7 @@ type ShredIndexV2Inner = [u64; NUM_U64_PER_SHRED_INDEX_V2]; /// A bit array of shred indices, where each u64 represents 64 shred indices. /// -/// The current implementation of [`ShredIndexLegacy`] utilizes a [`BTreeSet`] to store +/// The current implementation of [`ShredIndex`] utilizes a [`BTreeSet`] to store /// shred indices. While [`BTreeSet`] remains efficient as operations are amortized /// over time, the overhead of the B-tree structure becomes significant when frequently /// serialized and deserialized. In particular: From a69f8300520ef20253a61f60b8334ca41e1062ee Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Dec 2024 00:21:31 +0400 Subject: [PATCH 21/44] make ser/de portable with explicit LE conversion --- ledger/src/blockstore_meta.rs | 108 ++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 2f29c3cd7d6359..811b61fc076048 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -11,6 +11,7 @@ use { }, std::{ collections::BTreeSet, + mem::MaybeUninit, ops::{Bound, Range, RangeBounds}, }, }; @@ -353,26 +354,41 @@ impl Serialize for ShredIndexV2 { use serde::ser::SerializeTuple; let mut tuple = serializer.serialize_tuple(2)?; + // Purpose: + // - Serialize `self.index` (a `[u64; NUM_U64_PER_SHRED_INDEX_V2]` array) into a fixed-size buffer as little-endian bytes. + // // SAFETY: This is safe because: - // 1. Memory initialization & layout: - // - index is a fixed-size array [u64; MAX_U64S_PER_SLOT] fully initialized - // at construction and never contains uninitialized memory - // - array elements are contiguous with no padding + // 1. Memory Layout: + // - The buffer is sized to `std::mem::size_of::<[u64; NUM_U64_PER_SHRED_INDEX_V2]>()`, matching the source data size. + // - Each `u64` is serialized into 8 bytes, ensuring no padding or overflow. + // + // 2. Pointer Safety: + // - `dst` is derived from `buffer.as_mut_ptr()` and points to valid, uninitialized memory. + // - The offsets `dst.add(i * 8)` are always within bounds because: + // - `i` ranges from `0` to `NUM_U64_PER_SHRED_INDEX_V2 - 1`. + // - The total size of the buffer is sufficient to accommodate all writes. // - // 2. Size & alignment: - // - size_of::<[u64; MAX_U64S_PER_SLOT]> is exactly (8 * MAX_U64S_PER_SLOT) bytes - // - &self.index is u64-aligned, which satisfies u8 alignment + // 3. Alignment: + // - `dst` is `u8`-aligned from `buffer.as_mut_ptr()`. + // - Writing 8 bytes per iteration (`[u8; 8]` from `to_le_bytes()`) respects alignment requirements. // - // 3. Lifetime: - // - slice lifetime is tied to &self and is read-only + // 4. Initialization: + // - Each 8-byte chunk is written exactly once using `std::ptr::write`, fully initializing the buffer. + // - After the loop, the buffer is guaranteed to be completely initialized, making `assume_init()` safe. // - // Note: Deserialization will validate the byte length and safely reconstruct the u64 array - tuple.serialize_element(&serde_bytes::Bytes::new(unsafe { - std::slice::from_raw_parts( - &self.index as *const _ as *const u8, - std::mem::size_of::(), - ) - }))?; + // Notes: + // - Serialization uses little-endian order for portability. + let buffer = unsafe { + let mut buffer = + MaybeUninit::<[u8; std::mem::size_of::()]>::uninit(); + let dst = buffer.as_mut_ptr() as *mut u8; + for (i, &word) in self.index.iter().enumerate() { + // Explicitly convert to LE bytes + std::ptr::write(dst.add(i * 8) as *mut [u8; 8], word.to_le_bytes()); + } + buffer.assume_init() + }; + tuple.serialize_element(serde_bytes::Bytes::new(&buffer))?; tuple.serialize_element(&self.num_shreds)?; tuple.end() } @@ -384,19 +400,65 @@ impl<'de> Deserialize<'de> for ShredIndexV2 { D: serde::Deserializer<'de>, { let (bytes, num_shreds) = <(&[u8], usize)>::deserialize(deserializer)?; + let total_size = std::mem::size_of::(); // Accept input smaller than our current fixed array size to maintain compatibility with older // versions that had fewer shreds per slot. This is safe because smaller sets are stored in // our fixed array with any remaining space automatically zeroed. This approach ensures we can // read data from any previous version while enforcing a maximum size limit. - if bytes.len() > std::mem::size_of::() { + if bytes.len() > total_size { return Err(serde::de::Error::custom("input too large")); } - let mut index = [0u64; NUM_U64_PER_SHRED_INDEX_V2]; - bytes.chunks_exact(8).enumerate().for_each(|(i, chunk)| { - // Unwrap is safe because `chunks_exact` guarantees the length - index[i] = u64::from_ne_bytes(chunk.try_into().unwrap()); - }); - Ok(Self { index, num_shreds }) + if bytes.is_empty() { + return Err(serde::de::Error::custom("input is empty")); + } + + let mut buffer = MaybeUninit::::uninit(); + let remaining_bytes = total_size - bytes.len(); + + // SAFETY: This operation is safe because: + // + // 1. Memory Layout: + // - The buffer size matches `std::mem::size_of::()`. + // - `bytes.len()` is checked to ensure it does not exceed `total_size`. + // + // 2. Pointer Safety: + // - `buffer` is a fresh `MaybeUninit`, ensuring no aliasing or invalid memory access. + // - `ptr.add(i)` is always within bounds as `i` comes from `bytes.chunks_exact(8).enumerate()`. + // + // 3. Alignment: + // - `buffer.as_mut_ptr()` is aligned for `u64`, matching the type of `ShredIndexV2Inner`. + // - Writing `u64` values respects alignment requirements. + // + // 4. Initialization: + // - Each `u64` in the buffer is initialized from 8-byte chunks of `bytes` using `u64::from_le_bytes`. + // - Remaining bytes are zeroed with `std::ptr::write_bytes`. + // - After the loop, the buffer is fully initialized, making `assume_init()` safe. + // + // 5. Compatibility: + // - Allows smaller inputs (`bytes.len() < total_size`) by zero-filling unused space to maintain backward compatibility. + unsafe { + let ptr = buffer.as_mut_ptr() as *mut u64; + // Write input bytes, converting to LE. + for (i, byte) in bytes.chunks_exact(8).enumerate() { + std::ptr::write( + ptr.add(i), + // SAFETY: chunks_exact(8) guarantees 8 bytes. + u64::from_le_bytes(byte.try_into().unwrap_unchecked()), + ); + } + + // Zero remaining bytes + std::ptr::write_bytes( + (buffer.as_mut_ptr() as *mut u8).add(bytes.len()), + 0, + remaining_bytes, + ); + + Ok(Self { + index: buffer.assume_init(), + num_shreds, + }) + } } } From 345ee42371c906f0fcc08c64c64bef52d935b221 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Dec 2024 17:07:30 +0400 Subject: [PATCH 22/44] add exhaustive ShredIndexV2 range bound tests --- ledger/src/blockstore_meta.rs | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 811b61fc076048..79865cf25ce688 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -1085,6 +1085,44 @@ mod test { assert_eq!(ShredIndex::from(v2), legacy); } + #[test] + fn test_shred_index_v2_range_bounds() { + let mut index = ShredIndexV2::default(); + + index.insert(10); + index.insert(20); + index.insert(30); + index.insert(40); + + use std::ops::Bound::*; + + // Test all combinations of bounds + let test_cases = [ + // (start_bound, end_bound, expected_result) + (Included(10), Included(30), vec![10, 20, 30]), + (Included(10), Excluded(30), vec![10, 20]), + (Excluded(10), Included(30), vec![20, 30]), + (Excluded(10), Excluded(30), vec![20]), + // Unbounded start + (Unbounded, Included(20), vec![10, 20]), + (Unbounded, Excluded(20), vec![10]), + // Unbounded end + (Included(30), Unbounded, vec![30, 40]), + (Excluded(30), Unbounded, vec![40]), + // Both Unbounded + (Unbounded, Unbounded, vec![10, 20, 30, 40]), + ]; + + for (start_bound, end_bound, expected) in test_cases { + let result: Vec<_> = index.range((start_bound, end_bound)).collect(); + assert_eq!( + result, expected, + "Failed for bounds: start={:?}, end={:?}", + start_bound, end_bound + ); + } + } + #[test] fn test_shred_index_v2_boundary_conditions() { let mut index = ShredIndexV2::default(); From 7cf0c89d327994a93584c8bb8cd0019d02fe0de3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Dec 2024 19:19:51 +0400 Subject: [PATCH 23/44] add note about `ShredIndex` & `ShredIndex2` ser/de --- ledger/src/blockstore.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 9406a1e0afb4b3..d64e9a8424f343 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2510,6 +2510,11 @@ impl Blockstore { // This allows safe downgrade to Release 1 since it can read both formats // https://github.com/anza-xyz/agave/issues/3570 self.index_cf.get_with(slot, |slice| { + // Version compatibility note: Index and IndexV2 use different serialization strategies + // that make their formats naturally distinguishable. ShredIndexV2 serializes u64s as + // raw bytes (16 bytes for two u64s), while the original format serializes them as a + // collection (2 bytes for two u64s). This difference in length prevents any risk of + // misinterpreting data between versions without needing explicit version tags. let index: bincode::Result = bincode::deserialize(slice); match index { Ok(index) => Ok(index), From 19a0dd374e25030277010e5dfa620efab6902d03 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Dec 2024 18:38:36 +0400 Subject: [PATCH 24/44] improve readability of index version compatibility note --- ledger/src/blockstore.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index d64e9a8424f343..f961b99e03c64d 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2510,11 +2510,12 @@ impl Blockstore { // This allows safe downgrade to Release 1 since it can read both formats // https://github.com/anza-xyz/agave/issues/3570 self.index_cf.get_with(slot, |slice| { - // Version compatibility note: Index and IndexV2 use different serialization strategies - // that make their formats naturally distinguishable. ShredIndexV2 serializes u64s as - // raw bytes (16 bytes for two u64s), while the original format serializes them as a - // collection (2 bytes for two u64s). This difference in length prevents any risk of - // misinterpreting data between versions without needing explicit version tags. + // Version compatibility note: Index and IndexV2 use different serialization + // strategies in their ShredIndex field that make their formats naturally distinguishable. + // + // For example, serializing two `u64`s: + // - ShredIndexV2 serializes as a collection of bytes, with a length prefix of 16. + // - ShredIndex serializes as a collection of u64s, with a length prefix of 2. let index: bincode::Result = bincode::deserialize(slice); match index { Ok(index) => Ok(index), From 96bd5831320490a3a8c54f89f4d422b8752f0521 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 14 Dec 2024 10:55:51 +0400 Subject: [PATCH 25/44] decouple NUM_U64_PER_SHRED_INDEX_V2 from MAX_DATA_SHREDS_PER_SLOT --- ledger/src/blockstore_meta.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 79865cf25ce688..fc9d600ddb022c 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -9,6 +9,7 @@ use { clock::{Slot, UnixTimestamp}, hash::Hash, }, + static_assertions::const_assert, std::{ collections::BTreeSet, mem::MaybeUninit, @@ -311,7 +312,30 @@ impl ShredIndex { } } -const NUM_U64_PER_SHRED_INDEX_V2: usize = (MAX_DATA_SHREDS_PER_SLOT + 63) / 64; +/// Number of `u64`s required to accommodate each shred in a slot ([`MAX_DATA_SHREDS_PER_SLOT`]). +/// +/// **THIS VALUE MUST NEVER DECREASE ONCE ROLLED OUT.** +/// +/// As it relates to deserializing from the blockstore, using a statically sized structure +/// can trivially accommodate increases in space, but decreases are problematic. For example, +/// +/// Increase case: +/// - Assume `MAX_DATA_SHREDS_PER_SLOT` was bumped from `32_768` to `65_536`. +/// - Assume data being deserialized was written when `MAX_DATA_SHREDS_PER_SLOT` was `32_768`. +/// - ✅ No issue, as `ShredIndexV2Inner` has extra space that can be zeroed to accommodate. +/// +/// Decrease case: +/// - Assume `MAX_DATA_SHREDS_PER_SLOT` was reduced from `65_536` to `32_768`. +/// - Assume data being deserialized was written when `MAX_DATA_SHREDS_PER_SLOT` was `65_536`. +/// - ❌ `ShredIndexV2Inner` only has space to accommodate `32_768` shreds. +/// +/// As such, we: +/// - Decouple the definition of `NUM_U64_PER_SHRED_INDEX_V2` from `MAX_DATA_SHREDS_PER_SLOT`. +/// (i.e., make it not an explicit function of) +/// - Impose a compile time check to ensure that `NUM_U64_PER_SHRED_INDEX_V2 >= (MAX_DATA_SHREDS_PER_SLOT + 63) / 64`. +/// - Impose a convention that `NUM_U64_PER_SHRED_INDEX_V2` must never decrease. +const NUM_U64_PER_SHRED_INDEX_V2: usize = 512; +const_assert!(NUM_U64_PER_SHRED_INDEX_V2 >= (MAX_DATA_SHREDS_PER_SLOT + 63) / 64); type ShredIndexV2Inner = [u64; NUM_U64_PER_SHRED_INDEX_V2]; /// A bit array of shred indices, where each u64 represents 64 shred indices. From b22c49ca72dc2b5fb38ac044bde44a2be057dcfd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Dec 2024 10:26:01 +0400 Subject: [PATCH 26/44] fix missing usage of `get_index` abstraction --- ledger/src/blockstore.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f961b99e03c64d..3c9f438c45ea48 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4789,8 +4789,7 @@ impl Blockstore { let mut total_start = Measure::start("Total elapsed"); let res = index_working_set.entry(slot).or_insert_with(|| { let newly_inserted_meta = self - .index_cf - .get(slot) + .get_index(slot) .unwrap() .unwrap_or_else(|| Index::new(slot)); IndexMetaWorkingSetEntry { From 336dbdb49e07cee08625bf77c1e5f9929b6a4fed Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 20 Dec 2024 08:49:50 -0800 Subject: [PATCH 27/44] add deserialize method to TypedColumn less error prone than adding get_with, as .get .multi_get are the canonical access methods --- ledger/src/blockstore.rs | 23 +---------------- ledger/src/blockstore_db.rs | 49 +++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 3c9f438c45ea48..b746f6e4ada6fc 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2503,28 +2503,7 @@ impl Blockstore { } pub fn get_index(&self, slot: Slot) -> Result> { - // Migration strategy for new column format: - // 1. Release 1: Add ability to read new format as fallback, keep writing old format - // 2. Release 2: Switch to writing new format, keep reading old format as fallback - // 3. Release 3: Remove old format support once stable - // This allows safe downgrade to Release 1 since it can read both formats - // https://github.com/anza-xyz/agave/issues/3570 - self.index_cf.get_with(slot, |slice| { - // Version compatibility note: Index and IndexV2 use different serialization - // strategies in their ShredIndex field that make their formats naturally distinguishable. - // - // For example, serializing two `u64`s: - // - ShredIndexV2 serializes as a collection of bytes, with a length prefix of 16. - // - ShredIndex serializes as a collection of u64s, with a length prefix of 2. - let index: bincode::Result = bincode::deserialize(slice); - match index { - Ok(index) => Ok(index), - Err(_) => { - let index: IndexV2 = bincode::deserialize(slice)?; - Ok(index.into()) - } - } - }) + self.index_cf.get(slot) } /// Manually update the meta for a slot. diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index ed9f263752f600..7e85a8bb49e2ff 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -813,6 +813,10 @@ pub trait ColumnName { pub trait TypedColumn: Column { type Type: Serialize + DeserializeOwned; + + fn deserialize(data: &[u8]) -> bincode::Result { + bincode::deserialize(data) + } } impl TypedColumn for columns::AddressSignatures { @@ -1230,6 +1234,30 @@ impl ColumnName for columns::Index { } impl TypedColumn for columns::Index { type Type = blockstore_meta::Index; + + fn deserialize(data: &[u8]) -> bincode::Result { + // Migration strategy for new column format: + // 1. Release 1: Add ability to read new format as fallback, keep writing old format + // 2. Release 2: Switch to writing new format, keep reading old format as fallback + // 3. Release 3: Remove old format support once stable + // This allows safe downgrade to Release 1 since it can read both formats + // https://github.com/anza-xyz/agave/issues/3570 + + // Version compatibility note: Index and IndexV2 use different serialization + // strategies in their ShredIndex field that make their formats naturally distinguishable. + // + // For example, serializing two `u64`s: + // - ShredIndexV2 serializes as a collection of bytes, with a length prefix of 16. + // - ShredIndex serializes as a collection of u64s, with a length prefix of 2. + let index: bincode::Result = bincode::deserialize(data); + match index { + Ok(index) => Ok(index), + Err(_) => { + let index: blockstore_meta::IndexV2 = bincode::deserialize(data)?; + Ok(index.into()) + } + } + } } impl SlotColumn for columns::DeadSlots {} @@ -1638,7 +1666,7 @@ where let result = self .backend .multi_get_cf(self.handle(), &keys) - .map(|out| Ok(out?.as_deref().map(deserialize).transpose()?)) + .map(|out| Ok(out?.as_deref().map(C::deserialize).transpose()?)) .collect::>>>(); if let Some(op_start_instant) = is_perf_enabled { // use multi-get instead @@ -1658,31 +1686,14 @@ where self.get_raw(&C::key(key)) } - /// Get the value associated with a key, applying a function to the raw bytes. - /// - /// Like [`get`](Self::get), but delegates deserialization to a provided function. - pub fn get_with(&self, key: C::Index, f: F) -> Result> - where - F: FnOnce(&[u8]) -> Result, - { - self.get_raw_with(&C::key(key), f) - } - pub fn get_raw(&self, key: &[u8]) -> Result> { - self.get_raw_with(key, |slice| Ok(deserialize(slice)?)) - } - - pub fn get_raw_with(&self, key: &[u8], f: F) -> Result> - where - F: FnOnce(&[u8]) -> Result, - { let mut result = Ok(None); let is_perf_enabled = maybe_enable_rocksdb_perf( self.column_options.rocks_perf_sample_interval, &self.read_perf_status, ); if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? { - let value = f(pinnable_slice.as_ref())?; + let value = C::deserialize(pinnable_slice.as_ref())?; result = Ok(Some(value)) } From f9e498683c212115ee0e98db1a2a071927566449 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 18:42:43 -0800 Subject: [PATCH 28/44] use simpler Vec implementation --- Cargo.lock | 1 + ledger/Cargo.toml | 1 + ledger/src/blockstore_db.rs | 7 +- ledger/src/blockstore_meta.rs | 361 +++++++++++++--------------------- 4 files changed, 145 insertions(+), 225 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index febc4748f09899..8177c0430d354c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7537,6 +7537,7 @@ dependencies = [ "mockall", "num_cpus", "num_enum", + "proptest", "prost", "qualifier_attr", "rand 0.8.5", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index f4b1a560a52002..e6bfb7d0e699c5 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -31,6 +31,7 @@ lru = { workspace = true } mockall = { workspace = true } num_cpus = { workspace = true } num_enum = { workspace = true } +proptest = { workspace = true } prost = { workspace = true } qualifier_attr = { workspace = true } rand = { workspace = true } diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index f05cefd8d5c526..29f6d058c8fe7a 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -10,7 +10,7 @@ use { }, blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, }, - bincode::{deserialize, serialize}, + bincode::{deserialize, serialize, Options as BincodeOptions}, byteorder::{BigEndian, ByteOrder}, log::*, prost::Message, @@ -1215,6 +1215,7 @@ impl TypedColumn for columns::Index { type Type = blockstore_meta::Index; fn deserialize(data: &[u8]) -> bincode::Result { + let config = bincode::DefaultOptions::new().reject_trailing_bytes(); // Migration strategy for new column format: // 1. Release 1: Add ability to read new format as fallback, keep writing old format // 2. Release 2: Switch to writing new format, keep reading old format as fallback @@ -1228,11 +1229,11 @@ impl TypedColumn for columns::Index { // For example, serializing two `u64`s: // - ShredIndexV2 serializes as a collection of bytes, with a length prefix of 16. // - ShredIndex serializes as a collection of u64s, with a length prefix of 2. - let index: bincode::Result = bincode::deserialize(data); + let index: bincode::Result = config.deserialize(data); match index { Ok(index) => Ok(index), Err(_) => { - let index: blockstore_meta::IndexV2 = bincode::deserialize(data)?; + let index: blockstore_meta::IndexV2 = config.deserialize(data)?; Ok(index.into()) } } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index fc9d600ddb022c..e52ad7379ba512 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -9,10 +9,8 @@ use { clock::{Slot, UnixTimestamp}, hash::Hash, }, - static_assertions::const_assert, std::{ collections::BTreeSet, - mem::MaybeUninit, ops::{Bound, Range, RangeBounds}, }, }; @@ -312,33 +310,7 @@ impl ShredIndex { } } -/// Number of `u64`s required to accommodate each shred in a slot ([`MAX_DATA_SHREDS_PER_SLOT`]). -/// -/// **THIS VALUE MUST NEVER DECREASE ONCE ROLLED OUT.** -/// -/// As it relates to deserializing from the blockstore, using a statically sized structure -/// can trivially accommodate increases in space, but decreases are problematic. For example, -/// -/// Increase case: -/// - Assume `MAX_DATA_SHREDS_PER_SLOT` was bumped from `32_768` to `65_536`. -/// - Assume data being deserialized was written when `MAX_DATA_SHREDS_PER_SLOT` was `32_768`. -/// - ✅ No issue, as `ShredIndexV2Inner` has extra space that can be zeroed to accommodate. -/// -/// Decrease case: -/// - Assume `MAX_DATA_SHREDS_PER_SLOT` was reduced from `65_536` to `32_768`. -/// - Assume data being deserialized was written when `MAX_DATA_SHREDS_PER_SLOT` was `65_536`. -/// - ❌ `ShredIndexV2Inner` only has space to accommodate `32_768` shreds. -/// -/// As such, we: -/// - Decouple the definition of `NUM_U64_PER_SHRED_INDEX_V2` from `MAX_DATA_SHREDS_PER_SLOT`. -/// (i.e., make it not an explicit function of) -/// - Impose a compile time check to ensure that `NUM_U64_PER_SHRED_INDEX_V2 >= (MAX_DATA_SHREDS_PER_SLOT + 63) / 64`. -/// - Impose a convention that `NUM_U64_PER_SHRED_INDEX_V2` must never decrease. -const NUM_U64_PER_SHRED_INDEX_V2: usize = 512; -const_assert!(NUM_U64_PER_SHRED_INDEX_V2 >= (MAX_DATA_SHREDS_PER_SLOT + 63) / 64); -type ShredIndexV2Inner = [u64; NUM_U64_PER_SHRED_INDEX_V2]; - -/// A bit array of shred indices, where each u64 represents 64 shred indices. +/// A bitvec (`Vec`) of shred indices, where each u8 represents 8 shred indices. /// /// The current implementation of [`ShredIndex`] utilizes a [`BTreeSet`] to store /// shred indices. While [`BTreeSet`] remains efficient as operations are amortized @@ -355,149 +327,37 @@ type ShredIndexV2Inner = [u64; NUM_U64_PER_SHRED_INDEX_V2]; /// requested range, avoiding unnecessary traversal. /// - **Simplified Serialization**: The contiguous memory layout allows for efficient /// serialization/deserialization without tree reconstruction. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ShredIndexV2 { - index: ShredIndexV2Inner, + #[serde(with = "serde_bytes")] + index: Vec, num_shreds: usize, } impl Default for ShredIndexV2 { fn default() -> Self { Self { - index: [0; NUM_U64_PER_SHRED_INDEX_V2], + index: vec![0; Self::MAX_WORDS_PER_SHRED], num_shreds: 0, } } } -impl Serialize for ShredIndexV2 { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - use serde::ser::SerializeTuple; - - let mut tuple = serializer.serialize_tuple(2)?; - // Purpose: - // - Serialize `self.index` (a `[u64; NUM_U64_PER_SHRED_INDEX_V2]` array) into a fixed-size buffer as little-endian bytes. - // - // SAFETY: This is safe because: - // 1. Memory Layout: - // - The buffer is sized to `std::mem::size_of::<[u64; NUM_U64_PER_SHRED_INDEX_V2]>()`, matching the source data size. - // - Each `u64` is serialized into 8 bytes, ensuring no padding or overflow. - // - // 2. Pointer Safety: - // - `dst` is derived from `buffer.as_mut_ptr()` and points to valid, uninitialized memory. - // - The offsets `dst.add(i * 8)` are always within bounds because: - // - `i` ranges from `0` to `NUM_U64_PER_SHRED_INDEX_V2 - 1`. - // - The total size of the buffer is sufficient to accommodate all writes. - // - // 3. Alignment: - // - `dst` is `u8`-aligned from `buffer.as_mut_ptr()`. - // - Writing 8 bytes per iteration (`[u8; 8]` from `to_le_bytes()`) respects alignment requirements. - // - // 4. Initialization: - // - Each 8-byte chunk is written exactly once using `std::ptr::write`, fully initializing the buffer. - // - After the loop, the buffer is guaranteed to be completely initialized, making `assume_init()` safe. - // - // Notes: - // - Serialization uses little-endian order for portability. - let buffer = unsafe { - let mut buffer = - MaybeUninit::<[u8; std::mem::size_of::()]>::uninit(); - let dst = buffer.as_mut_ptr() as *mut u8; - for (i, &word) in self.index.iter().enumerate() { - // Explicitly convert to LE bytes - std::ptr::write(dst.add(i * 8) as *mut [u8; 8], word.to_le_bytes()); - } - buffer.assume_init() - }; - tuple.serialize_element(serde_bytes::Bytes::new(&buffer))?; - tuple.serialize_element(&self.num_shreds)?; - tuple.end() - } -} - -impl<'de> Deserialize<'de> for ShredIndexV2 { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let (bytes, num_shreds) = <(&[u8], usize)>::deserialize(deserializer)?; - let total_size = std::mem::size_of::(); - // Accept input smaller than our current fixed array size to maintain compatibility with older - // versions that had fewer shreds per slot. This is safe because smaller sets are stored in - // our fixed array with any remaining space automatically zeroed. This approach ensures we can - // read data from any previous version while enforcing a maximum size limit. - if bytes.len() > total_size { - return Err(serde::de::Error::custom("input too large")); - } - if bytes.is_empty() { - return Err(serde::de::Error::custom("input is empty")); - } - - let mut buffer = MaybeUninit::::uninit(); - let remaining_bytes = total_size - bytes.len(); - - // SAFETY: This operation is safe because: - // - // 1. Memory Layout: - // - The buffer size matches `std::mem::size_of::()`. - // - `bytes.len()` is checked to ensure it does not exceed `total_size`. - // - // 2. Pointer Safety: - // - `buffer` is a fresh `MaybeUninit`, ensuring no aliasing or invalid memory access. - // - `ptr.add(i)` is always within bounds as `i` comes from `bytes.chunks_exact(8).enumerate()`. - // - // 3. Alignment: - // - `buffer.as_mut_ptr()` is aligned for `u64`, matching the type of `ShredIndexV2Inner`. - // - Writing `u64` values respects alignment requirements. - // - // 4. Initialization: - // - Each `u64` in the buffer is initialized from 8-byte chunks of `bytes` using `u64::from_le_bytes`. - // - Remaining bytes are zeroed with `std::ptr::write_bytes`. - // - After the loop, the buffer is fully initialized, making `assume_init()` safe. - // - // 5. Compatibility: - // - Allows smaller inputs (`bytes.len() < total_size`) by zero-filling unused space to maintain backward compatibility. - unsafe { - let ptr = buffer.as_mut_ptr() as *mut u64; - // Write input bytes, converting to LE. - for (i, byte) in bytes.chunks_exact(8).enumerate() { - std::ptr::write( - ptr.add(i), - // SAFETY: chunks_exact(8) guarantees 8 bytes. - u64::from_le_bytes(byte.try_into().unwrap_unchecked()), - ); - } - - // Zero remaining bytes - std::ptr::write_bytes( - (buffer.as_mut_ptr() as *mut u8).add(bytes.len()), - 0, - remaining_bytes, - ); - - Ok(Self { - index: buffer.assume_init(), - num_shreds, - }) - } - } -} - +type ShredIndexV2Word = u8; impl ShredIndexV2 { + const SIZE_OF_WORD: usize = std::mem::size_of::(); + const WORD_BITS: usize = Self::SIZE_OF_WORD * 8; + const MAX_WORDS_PER_SHRED: usize = MAX_DATA_SHREDS_PER_SLOT.div_ceil(Self::WORD_BITS); + pub fn num_shreds(&self) -> usize { self.num_shreds } - fn index_and_mask(index: u64) -> (usize, u64) { - // index / 64 - let word_idx = (index >> 6) as usize; - // index % 64 - let bit_idx = index & 63; + fn index_and_mask(index: u64) -> (usize, ShredIndexV2Word) { + let word_idx = index as usize / Self::WORD_BITS; + let bit_idx = index as usize % Self::WORD_BITS; let mask = 1 << bit_idx; - (word_idx, mask) + (word_idx, mask as ShredIndexV2Word) } #[cfg(test)] @@ -584,38 +444,34 @@ impl ShredIndexV2 { R: RangeBounds, { let start = match bounds.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + 1, + Bound::Included(&n) => n as usize, + Bound::Excluded(&n) => n as usize + 1, Bound::Unbounded => 0, }; let end = match bounds.end_bound() { - Bound::Included(&n) => n + 1, - Bound::Excluded(&n) => n, - Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT as u64, + Bound::Included(&n) => n as usize + 1, + Bound::Excluded(&n) => n as usize, + Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT, }; - let end_word: usize = ((end + 63) / 64).min(NUM_U64_PER_SHRED_INDEX_V2 as u64) as usize; - let start_word = ((start / 64) as usize).min(end_word); + let end_word = end.div_ceil(Self::WORD_BITS).min(Self::MAX_WORDS_PER_SHRED); + let start_word = (start / Self::WORD_BITS).min(end_word); self.index[start_word..end_word] .iter() .enumerate() .flat_map(move |(word_offset, &word)| { - let base_idx = (start_word + word_offset) as u64 * 64; + let base_idx = (start_word + word_offset) * Self::WORD_BITS; - let lower_bound = if base_idx < start { - start - base_idx - } else { - 0 - }; - let upper_bound = if base_idx + 64 > end { + let lower_bound = start.saturating_sub(base_idx); + let upper_bound = if base_idx + Self::WORD_BITS > end { end - base_idx } else { - 64 + Self::WORD_BITS }; - let lower_mask = !0u64 << lower_bound; - let upper_mask = !0u64 >> (64 - upper_bound); + let lower_mask = !0 << lower_bound; + let upper_mask = !0 >> (Self::WORD_BITS - upper_bound); let mask = word & lower_mask & upper_mask; std::iter::from_fn({ @@ -624,10 +480,10 @@ impl ShredIndexV2 { if remaining == 0 { None } else { - let bit_idx = remaining.trailing_zeros() as u64; + let bit_idx = remaining.trailing_zeros(); // Clear the lowest set bit remaining &= remaining - 1; - Some(base_idx + bit_idx) + Some(base_idx as u64 + bit_idx as u64) } } }) @@ -649,6 +505,14 @@ impl FromIterator for ShredIndexV2 { } } +impl FromIterator for ShredIndex { + fn from_iter>(iter: T) -> Self { + ShredIndex { + index: iter.into_iter().collect(), + } + } +} + impl From for ShredIndexV2 { fn from(value: ShredIndex) -> Self { value.index.into_iter().collect() @@ -981,6 +845,8 @@ impl OptimisticSlotMetaVersioned { mod test { use { super::*, + bincode::Options, + proptest::prelude::*, rand::{seq::SliceRandom, thread_rng}, }; @@ -1048,65 +914,116 @@ mod test { } } - #[test] - fn shred_index_v2_serde() { - let index: ShredIndexV2 = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); - let serialized = bincode::serialize(&index).unwrap(); - let deserialized = bincode::deserialize::(&serialized).unwrap(); - assert_eq!(index, deserialized); - } + proptest! { + // Property: ShredIndexV2 can be converted to ShredIndex and vice versa + #[test] + fn shred_index_legacy_compat( + shreds in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + start in 0..MAX_DATA_SHREDS_PER_SLOT, + end in 0..MAX_DATA_SHREDS_PER_SLOT, + ) { + let mut legacy = ShredIndex::default(); + let mut v2 = ShredIndexV2::default(); + + for i in shreds { + v2.insert(i as u64); + legacy.insert(i as u64); + } - #[test] - fn shred_index_collision() { - let mut index = ShredIndex::default(); - // Create a `ShredIndex` that is exactly the expected size of `ShredIndexV2` - for i in 0..MAX_DATA_SHREDS_PER_SLOT as u64 { - index.insert(i); - } - let serialized = bincode::serialize(&index).unwrap(); - // Attempt to deserialize as `ShredIndexV2` - let deserialized = bincode::deserialize::(&serialized); - assert!(deserialized.is_err()); - } + for &i in legacy.index.iter() { + assert!(v2.contains(i)); + } - #[test] - fn shred_index_v2_collision() { - let index = ShredIndexV2::default(); - let serialized = bincode::serialize(&index).unwrap(); - let deserialized = bincode::deserialize::(&serialized); - assert!(deserialized.is_err()); + assert_eq!(v2.num_shreds(), legacy.num_shreds()); + let range = std::cmp::min(start, end) as u64..std::cmp::max(start, end) as u64; - let index: ShredIndexV2 = (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3).collect(); - let serialized = bincode::serialize(&index).unwrap(); - let deserialized = bincode::deserialize::(&serialized); - assert!(deserialized.is_err()); - } + assert_eq!( + v2.range(range.clone()).sum::(), + legacy.range(range).sum::() + ); - #[test] - fn shred_index_legacy_compat() { - use rand::Rng; - let mut legacy = ShredIndex::default(); - let mut v2 = ShredIndexV2::default(); - - for i in (0..MAX_DATA_SHREDS_PER_SLOT as u64).skip(3) { - v2.insert(i); - legacy.insert(i); + assert_eq!(ShredIndexV2::from(legacy.clone()), v2.clone()); + assert_eq!(ShredIndex::from(v2), legacy); } - for &i in legacy.index.iter() { - assert!(v2.contains(i)); + // Property: Index cannot be deserialized from IndexV2 + #[test] + fn test_legacy_collision( + coding_indices in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + data_indices in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + slot in 0..u64::MAX + ) { + let index = IndexV2 { + coding: coding_indices.into_iter().map(|i| i as u64).collect(), + data: data_indices.into_iter().map(|i| i as u64).collect(), + slot, + }; + let legacy = bincode::deserialize::(&bincode::serialize(&index).unwrap()); + prop_assert!(legacy.is_err()); } - assert_eq!(v2.num_shreds(), legacy.num_shreds()); + // Property: IndexV2 cannot be deserialized from Index + #[test] + fn test_legacy_collision_inverse( + coding_indices in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + data_indices in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + slot in 0..u64::MAX + ) { + let index = Index { + coding: coding_indices.into_iter().map(|i| i as u64).collect(), + data: data_indices.into_iter().map(|i| i as u64).collect(), + slot, + }; + let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + let v2 = config.deserialize::(&bincode::serialize(&index).unwrap()); + prop_assert!(v2.is_err()); + } - let rand_range = rand::thread_rng().gen_range(1000..MAX_DATA_SHREDS_PER_SLOT as u64); - assert_eq!( - v2.range(0..rand_range).sum::(), - legacy.range(0..rand_range).sum::() - ); + // Property: range queries should return correct indices + #[test] + fn range_query_correctness( + indices in prop::collection::btree_set( + 0..MAX_DATA_SHREDS_PER_SLOT, + 0..MAX_DATA_SHREDS_PER_SLOT + ), + ranges in prop::collection::vec( + (0..MAX_DATA_SHREDS_PER_SLOT) + .prop_flat_map(|start| { + (start + 1..=MAX_DATA_SHREDS_PER_SLOT) + .prop_map(move |end| (start, end)) + }), + 1..1000 + ) + ) { + let mut index = ShredIndexV2::default(); + + for &idx in &indices { + index.insert(idx as u64); + } - assert_eq!(ShredIndexV2::from(legacy.clone()), v2); - assert_eq!(ShredIndex::from(v2), legacy); + for (start, end) in ranges { + // Test indices match + assert_eq!( + index.range(start as u64..end as u64).collect::>(), + indices.range(start..end).map(|&i| i as u64).collect::>() + ); + } + } } #[test] From b545d79c07f727ce476bcc8c949f8833b9559850 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 18:49:11 -0800 Subject: [PATCH 29/44] update lock files --- programs/sbf/Cargo.lock | 78 +++++++++++++++++++++++++++++++++++++++++ svm/examples/Cargo.lock | 78 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index d683dc6c22039b..88db107601497f 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -675,6 +675,21 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -3876,6 +3891,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.6.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.11.9" @@ -3974,6 +4009,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quinn" version = "0.11.6" @@ -4106,6 +4147,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -4523,6 +4573,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.4" @@ -6029,6 +6091,7 @@ dependencies = [ "mockall", "num_cpus", "num_enum", + "proptest", "prost", "qualifier_attr", "rand 0.8.5", @@ -9586,6 +9649,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.6.0" @@ -9757,6 +9826,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.1" diff --git a/svm/examples/Cargo.lock b/svm/examples/Cargo.lock index 6c51ee2b4a3a0a..3256099d997642 100644 --- a/svm/examples/Cargo.lock +++ b/svm/examples/Cargo.lock @@ -595,6 +595,21 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -3767,6 +3782,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.6.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.11.9" @@ -3865,6 +3900,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quinn" version = "0.11.6" @@ -3998,6 +4039,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -4395,6 +4445,18 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.18" @@ -5849,6 +5911,7 @@ dependencies = [ "mockall", "num_cpus", "num_enum", + "proptest", "prost", "qualifier_attr", "rand 0.8.5", @@ -8892,6 +8955,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.8.0" @@ -9054,6 +9123,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" From d1a816c80380432924b4ca4eb085c06e5a04ea28 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 18:59:15 -0800 Subject: [PATCH 30/44] reject trailing bytes on serialize --- ledger/src/blockstore_db.rs | 19 ++++++++++++++----- ledger/src/blockstore_meta.rs | 5 +++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index d23e0a59a4169e..37386fa18393e7 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -796,8 +796,12 @@ pub trait ColumnName { pub trait TypedColumn: Column { type Type: Serialize + DeserializeOwned; - fn deserialize(data: &[u8]) -> bincode::Result { - bincode::deserialize(data) + fn deserialize(data: &[u8]) -> Result { + Ok(bincode::deserialize(data)?) + } + + fn serialize(data: &Self::Type) -> Result> { + Ok(bincode::serialize(data)?) } } @@ -1214,7 +1218,7 @@ impl ColumnName for columns::Index { impl TypedColumn for columns::Index { type Type = blockstore_meta::Index; - fn deserialize(data: &[u8]) -> bincode::Result { + fn deserialize(data: &[u8]) -> Result { let config = bincode::DefaultOptions::new().reject_trailing_bytes(); // Migration strategy for new column format: // 1. Release 1: Add ability to read new format as fallback, keep writing old format @@ -1238,6 +1242,11 @@ impl TypedColumn for columns::Index { } } } + + fn serialize(data: &Self::Type) -> Result> { + let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + Ok(config.serialize(data)?) + } } impl SlotColumn for columns::DeadSlots {} @@ -1723,7 +1732,7 @@ where self.column_options.rocks_perf_sample_interval, &self.write_perf_status, ); - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; let key = Self::key_from_index(index); let result = self.backend.put_cf(self.handle(), &key, &serialized_value); @@ -1746,7 +1755,7 @@ where value: &C::Type, ) -> Result<()> { let key = Self::key_from_index(index); - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; batch.put_cf(self.handle(), &key, &serialized_value) } } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index e52ad7379ba512..62fabbaa00591b 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -967,7 +967,8 @@ mod test { data: data_indices.into_iter().map(|i| i as u64).collect(), slot, }; - let legacy = bincode::deserialize::(&bincode::serialize(&index).unwrap()); + let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + let legacy = config.deserialize::(&config.serialize(&index).unwrap()); prop_assert!(legacy.is_err()); } @@ -990,7 +991,7 @@ mod test { slot, }; let config = bincode::DefaultOptions::new().reject_trailing_bytes(); - let v2 = config.deserialize::(&bincode::serialize(&index).unwrap()); + let v2 = config.deserialize::(&config.serialize(&index).unwrap()); prop_assert!(v2.is_err()); } From 069fb5db01bbf24ed5cae98f099bc3585d124d57 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 19:22:03 -0800 Subject: [PATCH 31/44] remove unused import --- ledger/src/blockstore_db.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 37386fa18393e7..3407c7e228e3cb 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -10,7 +10,7 @@ use { }, blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, }, - bincode::{deserialize, serialize, Options as BincodeOptions}, + bincode::{deserialize, Options as BincodeOptions}, byteorder::{BigEndian, ByteOrder}, log::*, prost::Message, @@ -2277,7 +2277,7 @@ pub mod tests { C: ColumnIndexDeprecation + TypedColumn + ColumnName, { pub fn put_deprecated(&self, index: C::DeprecatedIndex, value: &C::Type) -> Result<()> { - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; self.backend .put_cf(self.handle(), &C::deprecated_key(index), &serialized_value) } From 32a5457a3198bcb05903f9d5aba392c7aae347a2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 19:25:35 -0800 Subject: [PATCH 32/44] remove old comment --- ledger/src/blockstore_db.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 3407c7e228e3cb..b4d338eb9b556f 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1226,13 +1226,6 @@ impl TypedColumn for columns::Index { // 3. Release 3: Remove old format support once stable // This allows safe downgrade to Release 1 since it can read both formats // https://github.com/anza-xyz/agave/issues/3570 - - // Version compatibility note: Index and IndexV2 use different serialization - // strategies in their ShredIndex field that make their formats naturally distinguishable. - // - // For example, serializing two `u64`s: - // - ShredIndexV2 serializes as a collection of bytes, with a length prefix of 16. - // - ShredIndex serializes as a collection of u64s, with a length prefix of 2. let index: bincode::Result = config.deserialize(data); match index { Ok(index) => Ok(index), From 214d1b3998d6941266374b119dd6dfd28fc5024d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 19:39:38 -0800 Subject: [PATCH 33/44] clippy --- ledger/src/blockstore_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index b4d338eb9b556f..5bfdedf6391af3 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1677,7 +1677,7 @@ where let result = self .backend .multi_get_cf(self.handle(), &keys) - .map(|out| Ok(out?.as_deref().map(C::deserialize).transpose()?)) + .map(|out| out?.as_deref().map(C::deserialize).transpose()) .collect::>>>(); if let Some(op_start_instant) = is_perf_enabled { // use multi-get instead From 9d4cf1b534a70707c79f64007ba576199e5cfa42 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Jan 2025 19:54:50 -0800 Subject: [PATCH 34/44] update outdated comment --- ledger/src/blockstore_meta.rs | 36 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 62fabbaa00591b..b90358c5be3ed6 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -320,7 +320,7 @@ impl ShredIndex { /// - **Reconstruction**: Deserialization involves rebuilding the tree in bulk, /// including dynamic memory allocations and re-balancing nodes. /// -/// In contrast, our bit array implementation provides: +/// In contrast, our bit vec implementation provides: /// - **Contiguous Memory**: All bits are stored in a contiguous array of u64 words, /// allowing direct indexing and efficient memory access patterns. /// - **Direct Range Access**: Can load only the specific words that overlap with a @@ -398,9 +398,9 @@ impl ShredIndexV2 { /// Provides an iterator over the set shred indices within a specified range. /// /// # Algorithm - /// 1. Divide the specified range into 64-bit words. - /// 2. For each word: - /// - Calculate the base index (position of the word * 64). + /// 1. Divide the specified range into 8-bit words (u8). + /// 2. For each word:HH + /// - Calculate the base index (position of the word * 8). /// - Process all set bits in the word. /// - For words overlapping the range boundaries: /// - Determine the relevant bit range using boundaries. @@ -408,27 +408,25 @@ impl ShredIndexV2 { /// - Use bit manipulation to iterate over set bits efficiently. /// /// ## Explanation - /// > Note we're showing 32 bits per word in examples for brevity, but each word is 64 bits. - /// /// Given range `[75..205]`: /// - /// Word layout (each word is 64 bits), where each X represents a bit candidate: + /// Word layout (each word is 8 bits), where each X represents a bit candidate: /// ```text - /// Word 1 (0–63): [................................] ← Not included (outside range) - /// Word 2 (64–127): [..........XXXXXXXXXXXXXXXXXXXXXX] ← Partial word (start) - /// Word 3 (128–191): [XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] ← Full word (entirely in range) - /// Word 4 (192–255): [XXXXXXXXXXXXXXXXXXX.............] ← Partial word (end) + /// Word 9 (72-79): [..XXXXXX] ← Partial word (start) + /// Word 10 (80-87): [XXXXXXXX] ← Full word (entirely in range) + /// ... + /// Word 25 (200-207): [XXXXXX..] ← Partial word (end) /// ``` /// - /// Partial Word 2 (contains start boundary 75): - /// - Base index = 64 - /// - Lower boundary = 75 - 64 = 11 - /// - Lower mask = `11111111111111110000000000000000` + /// Partial Word 9 (contains start boundary 75): + /// - Base index = 72 + /// - Lower boundary = 75 - 72 = 3 + /// - Lower mask = `11111000` (right-shift) /// - /// Partial Word 4 (contains end boundary 205): - /// - Base index = 192 - /// - Upper boundary = 205 - 192 = 13 - /// - Upper mask = `00000000000000000000000000001111` + /// Partial Word 25 (contains end boundary 205): + /// - Base index = 200 + /// - Upper boundary = 205 - 200 = 5 + /// - Upper mask = `00111111` (left-shift) /// /// Final mask = `word & lower_mask & upper_mask` /// From bc9da4864d4ee2970a8a46ec80b1c4d77099e454 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 7 Jan 2025 04:14:57 +0000 Subject: [PATCH 35/44] Add padding to IndexV2 to prevent collisions --- ledger/src/blockstore_meta.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index b90358c5be3ed6..0235f63b811d68 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -117,6 +117,8 @@ pub struct Index { #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub struct IndexV2 { + /// Prevent de(serialization) collisions with [`Index`]. + _pad: u8, pub slot: Slot, data: ShredIndexV2, coding: ShredIndexV2, @@ -135,6 +137,7 @@ impl From for Index { impl From for IndexV2 { fn from(index: Index) -> Self { IndexV2 { + _pad: 0, slot: index.slot, data: index.data.into(), coding: index.coding.into(), @@ -964,6 +967,7 @@ mod test { coding: coding_indices.into_iter().map(|i| i as u64).collect(), data: data_indices.into_iter().map(|i| i as u64).collect(), slot, + _pad: 0, }; let config = bincode::DefaultOptions::new().reject_trailing_bytes(); let legacy = config.deserialize::(&config.serialize(&index).unwrap()); From 28115a317e8e998e4517587db06b680130e25c5f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Jan 2025 22:13:08 +0000 Subject: [PATCH 36/44] use bincode fixint encoding --- ledger/src/blockstore_db.rs | 23 +++++++++-------------- ledger/src/blockstore_meta.rs | 10 ++++------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 7501b288fd27af..1f3129aed85582 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -10,7 +10,7 @@ use { }, blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, }, - bincode::{deserialize, Options as BincodeOptions}, + bincode::{deserialize, serialize, Options as BincodeOptions}, byteorder::{BigEndian, ByteOrder}, log::*, prost::Message, @@ -799,10 +799,6 @@ pub trait TypedColumn: Column { fn deserialize(data: &[u8]) -> Result { Ok(bincode::deserialize(data)?) } - - fn serialize(data: &Self::Type) -> Result> { - Ok(bincode::serialize(data)?) - } } impl TypedColumn for columns::AddressSignatures { @@ -1219,7 +1215,11 @@ impl TypedColumn for columns::Index { type Type = blockstore_meta::Index; fn deserialize(data: &[u8]) -> Result { - let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + let config = bincode::DefaultOptions::new() + // `bincode::serialize` uses fixint encoding by default, so we need to use the same here + .with_fixint_encoding() + .reject_trailing_bytes(); + // Migration strategy for new column format: // 1. Release 1: Add ability to read new format as fallback, keep writing old format // 2. Release 2: Switch to writing new format, keep reading old format as fallback @@ -1235,11 +1235,6 @@ impl TypedColumn for columns::Index { } } } - - fn serialize(data: &Self::Type) -> Result> { - let config = bincode::DefaultOptions::new().reject_trailing_bytes(); - Ok(config.serialize(data)?) - } } impl SlotColumn for columns::DeadSlots {} @@ -1739,7 +1734,7 @@ where self.column_options.rocks_perf_sample_interval, &self.write_perf_status, ); - let serialized_value = C::serialize(value)?; + let serialized_value = serialize(value)?; let key = Self::key_from_index(index); let result = self.backend.put_cf(self.handle(), &key, &serialized_value); @@ -1762,7 +1757,7 @@ where value: &C::Type, ) -> Result<()> { let key = Self::key_from_index(index); - let serialized_value = C::serialize(value)?; + let serialized_value = serialize(value)?; batch.put_cf(self.handle(), &key, &serialized_value) } } @@ -2284,7 +2279,7 @@ pub mod tests { C: ColumnIndexDeprecation + TypedColumn + ColumnName, { pub fn put_deprecated(&self, index: C::DeprecatedIndex, value: &C::Type) -> Result<()> { - let serialized_value = C::serialize(value)?; + let serialized_value = serialize(value)?; self.backend .put_cf(self.handle(), &C::deprecated_key(index), &serialized_value) } diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 967c84c573b71f..51b84bb7836cf7 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -117,8 +117,6 @@ pub struct Index { #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub struct IndexV2 { - /// Prevent de(serialization) collisions with [`Index`]. - _pad: u8, pub slot: Slot, data: ShredIndexV2, coding: ShredIndexV2, @@ -137,7 +135,6 @@ impl From for Index { impl From for IndexV2 { fn from(index: Index) -> Self { IndexV2 { - _pad: 0, slot: index.slot, data: index.data.into(), coding: index.coding.into(), @@ -954,9 +951,8 @@ mod test { coding: coding_indices.into_iter().map(|i| i as u64).collect(), data: data_indices.into_iter().map(|i| i as u64).collect(), slot, - _pad: 0, }; - let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + let config = bincode::DefaultOptions::new().with_fixint_encoding().reject_trailing_bytes(); let legacy = config.deserialize::(&config.serialize(&index).unwrap()); prop_assert!(legacy.is_err()); } @@ -979,7 +975,9 @@ mod test { data: data_indices.into_iter().map(|i| i as u64).collect(), slot, }; - let config = bincode::DefaultOptions::new().reject_trailing_bytes(); + let config = bincode::DefaultOptions::new() + .with_fixint_encoding() + .reject_trailing_bytes(); let v2 = config.deserialize::(&config.serialize(&index).unwrap()); prop_assert!(v2.is_err()); } From 5edeb17cd18fd2bbb65321fd27fc38380de6fff8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Jan 2025 23:26:20 +0000 Subject: [PATCH 37/44] re-introduce TypedColumn::serialize --- ledger/src/blockstore_db.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 1f3129aed85582..f6bf40ba3b7abf 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -10,7 +10,7 @@ use { }, blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, }, - bincode::{deserialize, serialize, Options as BincodeOptions}, + bincode::{deserialize, Options as BincodeOptions}, byteorder::{BigEndian, ByteOrder}, log::*, prost::Message, @@ -799,6 +799,10 @@ pub trait TypedColumn: Column { fn deserialize(data: &[u8]) -> Result { Ok(bincode::deserialize(data)?) } + + fn serialize(data: &Self::Type) -> Result> { + Ok(bincode::serialize(data)?) + } } impl TypedColumn for columns::AddressSignatures { @@ -1734,7 +1738,7 @@ where self.column_options.rocks_perf_sample_interval, &self.write_perf_status, ); - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; let key = Self::key_from_index(index); let result = self.backend.put_cf(self.handle(), &key, &serialized_value); @@ -1757,7 +1761,7 @@ where value: &C::Type, ) -> Result<()> { let key = Self::key_from_index(index); - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; batch.put_cf(self.handle(), &key, &serialized_value) } } @@ -2279,7 +2283,7 @@ pub mod tests { C: ColumnIndexDeprecation + TypedColumn + ColumnName, { pub fn put_deprecated(&self, index: C::DeprecatedIndex, value: &C::Type) -> Result<()> { - let serialized_value = serialize(value)?; + let serialized_value = C::serialize(value)?; self.backend .put_cf(self.handle(), &C::deprecated_key(index), &serialized_value) } From c276d5bd4d563b6e0621c888cf1800c80fcfdb68 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Jan 2025 23:26:37 +0000 Subject: [PATCH 38/44] add proptest regression file --- ledger/proptest-regressions/blockstore_meta.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ledger/proptest-regressions/blockstore_meta.txt diff --git a/ledger/proptest-regressions/blockstore_meta.txt b/ledger/proptest-regressions/blockstore_meta.txt new file mode 100644 index 00000000000000..2d7f3ba59ebe77 --- /dev/null +++ b/ledger/proptest-regressions/blockstore_meta.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc de06086cdefde07319a41f7e78761901b2ffaabd0ef8b03e8a321f682fd3ab47 # shrinks to coding_indices = {}, data_indices = {0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 166, 167, 168, 169, 170, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 192, 193, 194, 195, 196, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 247, 248, 249, 250, 251, 252, 253, 254, 255, 257, 258, 259, 260, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 309, 310, 311, 312, 313, 314, 315, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 334, 335, 336, 337, 340, 341, 342, 343, 344, 345, 346, 347, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 449, 450, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 638, 639, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 693, 694, 695, 696, 697, 699, 700, 701, 703, 704, 706, 707, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 733, 734, 735, 736, 737, 738, 739, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 763, 764, 765, 766, 767, 768, 769, 771, 772, 773, 774, 775, 776, 777, 778, 779, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 883, 884, 885, 886, 887, 888, 890, 891, 892, 893, 894, 895, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 957, 958, 959, 960, 961, 962, 963, 964, 965, 967, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 992, 993, 994, 995, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1067, 1068, 1070, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1138, 1139, 1140, 1142, 1144, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1214, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1245, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1259, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1322, 1324, 1325, 1326, 1327, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1344, 1345, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1375, 1377, 1380, 1382, 1383, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1436, 1437, 1438, 1439, 1440, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1492, 1493, 1495, 1496, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1667, 1668, 1669, 1670, 1671, 1672, 1674, 1675, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1700, 1701, 1702, 1703, 1704, 1705, 1708, 1709, 1710, 1711, 1713, 1714, 1715, 1716, 1717, 1718, 1720, 1721, 1722, 1723, 1726, 1727, 1728, 1730, 1731, 1732, 1733, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1745, 1746, 1748, 1749, 1750, 1751, 1752, 1754, 1755, 1756, 1757, 1759, 1760, 1761, 1762, 1763, 1764, 1766, 1767, 1768, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1828, 1829, 1830, 1831, 1832, 1835, 1836, 1837, 1838, 1839, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, 1857, 1858, 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1987, 1988, 1989, 1990, 1991, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2037, 2039, 2040, 2042, 2043, 2044, 2045, 2047, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, 2103, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2114, 2115, 2116, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2140, 2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, 2179, 2180, 2181, 2183, 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2200, 2201, 2202, 2203, 2204, 2205, 2207, 2209, 2210, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2426, 2427, 2428, 2429, 2430, 2431, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2453, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2477, 2478, 2479, 2480, 2481, 2482, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2639, 2640, 2641, 2642, 2643, 2644, 2646, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, 2701, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2741, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2864, 2865, 2866, 2867, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2886, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2898, 2899, 2900, 2901, 2903, 2904, 2905, 2906, 2907, 2908, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2928, 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2978, 2979, 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3011, 3012, 3013, 3014, 3015, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, 3034, 3035, 3036, 3038, 3039, 3040, 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3059, 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, 3069, 3070, 3071, 3072, 3073, 3074, 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3107, 3108, 3109, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3138, 3139, 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3199, 3200, 3201, 3202, 3203, 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, 3242, 3243, 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, 3252, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3277, 3278, 3279, 3280, 3281, 3283, 3284, 3285, 3286, 3287, 3289, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3310, 3311, 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364, 3365, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3404, 3406, 3407, 3408, 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, 3449, 3451, 3452, 3454, 3455, 3456, 3458, 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3497, 3498, 3499, 3502, 3503, 3504, 3505, 3506, 3507, 3509, 3510, 3511, 3512, 3513, 3514, 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, 3531, 3532, 3533, 3534, 3535, 3536, 3538, 3539, 3541, 3542, 3543, 3546, 3547, 3548, 3549, 3550, 3552, 3554, 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, 3564, 3565, 3566, 3567, 3568, 3569, 3570, 3571, 3572, 3573, 3574, 3575, 3576, 3577, 3578, 3579, 3580, 3581, 3582, 3583, 3584, 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3596, 3597, 3598, 3600, 3601, 3602, 3604, 3605, 3606, 3607, 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, 3624, 3625, 3626, 3628, 3629, 3630, 3631, 3632, 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 3643, 3644, 3645, 3646, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656, 3657, 3658, 3659, 3661, 3662, 3663, 3664, 3666, 3667, 3668, 3669, 3670, 3672, 3673, 3674, 3676, 3677, 3678, 3679, 3680, 3681, 3682, 3683, 3684, 3685, 3686, 3687, 3688, 3689, 3690, 3691, 3692, 3693, 3694, 3695, 3696, 3697, 3698, 3699, 3700, 3701, 3702, 3703, 3704, 3705, 3706, 3707, 3709, 3710, 3711, 3712, 3713, 3714, 3715, 3716, 3717, 3718, 3719, 3720, 3721, 3722, 3723, 3724, 3725, 3726, 3727, 3728, 3729, 3730, 3731, 3732, 3733, 3734, 3735, 3736, 3737, 3738, 3739, 3740, 3741, 3742, 3743, 3744, 3745, 3746, 3747, 3748, 3749, 3750, 3752, 3753, 3754, 3755, 3756, 3758, 3759, 3760, 3761, 3762, 3763, 3764, 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780, 3781, 3783, 3784, 3786, 3787, 3788, 3789, 3790, 3791, 3792, 3793, 3794, 3796, 3798, 3799, 3800, 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3811, 3812, 3813, 3814, 3815, 3816, 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3825, 3826, 3827, 3828, 3829, 3830, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3860, 3861, 3862, 3863, 3864, 3865, 3867, 3868, 3869, 3870, 3871, 3872, 3873, 3874, 3875, 3876, 3877, 3879, 3880, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890, 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905, 3906, 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938, 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3954, 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3965, 3966, 3967, 3968, 3969, 3970, 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3982, 3984, 3985, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032, 4033, 4034, 4035, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047, 4048, 4049, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058, 4059, 4060, 4061, 4063, 4065, 4066, 4067, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4077, 4078, 4079, 4080, 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, 4096, 4097, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4121, 4122, 4123, 4124, 4126, 4127, 4128, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176, 4177, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191, 4192, 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4214, 4215, 4217, 4218, 4219, 4220, 4221, 4222, 4223, 4224, 4225, 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, 4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4369, 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4384, 4385, 4387, 4388, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4402, 4403, 4404, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414, 4415, 4416, 4417, 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4443, 4444, 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4460, 4461, 4462, 4463, 4464, 4465, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481, 4482, 4485, 4486, 4487, 4488, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497, 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4513, 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4528, 4529, 4530, 4531, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545, 4546, 4547, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561, 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4597, 4598, 4599, 4600, 4603, 4604, 4605, 4606, 4607, 4608, 4610, 4611, 4612, 4613, 4615, 4616, 4617, 4618, 4619, 4620, 4623, 4624, 4625, 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657, 4658, 4659, 4660, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4701, 4702, 4703, 4704, 4705, 4706, 4707, 4708, 4709, 4710, 4711, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4727, 4728, 4729, 4730, 4731, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761, 4762, 4764, 4765, 4766, 4767, 4768, 4769, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783, 4784, 4785, 4786, 4787, 4788, 4789, 4790, 4793, 4795, 4796, 4797, 4798, 4799, 4801, 4802, 4803, 4804, 4805, 4807, 4808, 4809, 4811, 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4830, 4831, 4832, 4833, 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4852, 4854, 4855, 4856, 4857, 4858, 4859, 4862, 4863, 4864, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4875, 4876, 4877, 4878, 4879, 4880, 4881, 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897, 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4914, 4916, 4917, 4918, 4919, 4920, 4922, 4923, 4924, 4925, 4927, 4928, 4929, 4930, 4931, 4932, 4933, 4934, 4935, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945, 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4954, 4955, 4957, 4958, 4959, 4960, 4961, 4962, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977, 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4987, 4988, 4990, 4991, 4992, 4993, 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009, 5010, 5011, 5012, 5014, 5015, 5016, 5017, 5018, 5019, 5020, 5021, 5023, 5024, 5025, 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5055, 5056, 5057, 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5101, 5102, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 5111, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121, 5122, 5123, 5124, 5125, 5126, 5127, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137, 5138, 5139, 5140, 5141, 5142, 5143, 5144, 5146, 5147, 5148, 5149, 5150, 5152, 5153, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169, 5170, 5171, 5172, 5173, 5174, 5175, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185, 5186, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201, 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214, 5215, 5216, 5217, 5218, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230, 5231, 5232, 5233, 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249, 5250, 5251, 5252, 5253, 5254, 5255, 5256, 5257, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265, 5266, 5267, 5268, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281, 5282, 5284, 5285, 5286, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5309, 5310, 5311, 5312, 5313, 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5328, 5329, 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345, 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5356, 5357, 5358, 5359, 5361, 5362, 5363, 5365, 5366, 5367, 5368, 5369, 5370, 5371, 5372, 5373, 5374, 5375, 5376, 5377, 5378, 5379, 5381, 5383, 5384, 5385, 5386, 5387, 5388, 5389, 5391, 5392, 5393, 5394, 5395, 5396, 5397, 5398, 5399, 5400, 5401, 5402, 5403, 5404, 5405, 5406, 5407, 5408, 5409, 5410, 5411, 5412, 5413, 5414, 5415, 5416, 5417, 5418, 5419, 5420, 5421, 5423, 5424, 5426, 5427, 5428, 5429, 5430, 5431, 5432, 5433, 5434, 5435, 5436, 5437, 5438, 5439, 5440, 5441, 5442, 5444, 5445, 5446, 5447, 5448, 5449, 5450, 5451, 5452, 5453, 5454, 5455, 5457, 5458, 5459, 5460, 5461, 5462, 5463, 5464, 5465, 5466, 5467, 5468, 5469, 5470, 5471, 5472, 5473, 5474, 5475, 5476, 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489, 5490, 5491, 5492, 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5502, 5503, 5504, 5505, 5506, 5507, 5508, 5509, 5510, 5511, 5512, 5513, 5515, 5516, 5518, 5519, 5520, 5521, 5522, 5523, 5524, 5525, 5526, 5528, 5529, 5530, 5531, 5532, 5533, 5534, 5535, 5539, 5540, 5541, 5542, 5543, 5545, 5546, 5547, 5548, 5549, 5550, 5551, 5553, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567, 5568, 5569, 5570, 5571, 5572, 5573, 5574, 5575, 5576, 5577, 5578, 5579, 5580, 5581, 5583, 5584, 5585, 5586, 5587, 5588, 5589, 5590, 5591, 5592, 5593, 5594, 5595, 5598, 5599, 5600, 5601, 5602, 5603, 5604, 5605, 5606, 5607, 5608, 5609, 5610, 5611, 5612, 5613, 5614, 5615, 5616, 5617, 5618, 5619, 5620, 5621, 5622, 5623, 5624, 5625, 5626, 5627, 5628, 5629, 5630, 5631, 5633, 5634, 5636, 5637, 5638, 5639, 5640, 5641, 5642, 5643, 5644, 5645, 5646, 5647, 5648, 5649, 5650, 5651, 5652, 5653, 5654, 5656, 5657, 5658, 5659, 5660, 5661, 5662, 5663, 5664, 5665, 5666, 5667, 5668, 5669, 5671, 5672, 5674, 5675, 5676, 5677, 5678, 5680, 5681, 5682, 5683, 5684, 5685, 5686, 5687, 5688, 5689, 5690, 5691, 5693, 5694, 5695, 5696, 5697, 5698, 5700, 5701, 5702, 5703, 5705, 5706, 5707, 5708, 5709, 5710, 5711, 5712, 5713, 5714, 5715, 5716, 5717, 5718, 5719, 5720, 5721, 5722, 5723, 5724, 5725, 5726, 5729, 5730, 5731, 5732, 5733, 5734, 5735, 5736, 5737, 5738, 5739, 5740, 5741, 5742, 5744, 5745, 5746, 5747, 5748, 5749, 5750, 5751, 5752, 5753, 5754, 5755, 5756, 5757, 5759, 5760, 5761, 5762, 5763, 5764, 5765, 5766, 5767, 5768, 5769, 5770, 5771, 5772, 5773, 5774, 5775, 5776, 5777, 5778, 5779, 5780, 5781, 5782, 5783, 5784, 5785, 5786, 5787, 5788, 5789, 5790, 5791, 5792, 5793, 5794, 5795, 5796, 5797, 5798, 5799, 5800, 5801, 5802, 5803, 5804, 5805, 5806, 5807, 5808, 5809, 5810, 5811, 5813, 5814, 5815, 5816, 5817, 5818, 5819, 5820, 5821, 5823, 5824, 5825, 5826, 5827, 5828, 5829, 5830, 5831, 5832, 5833, 5834, 5836, 5837, 5838, 5839, 5840, 5841, 5842, 5843, 5844, 5846, 5847, 5848, 5849, 5850, 5851, 5853, 5854, 5855, 5856, 5857, 5858, 5859, 5860, 5861, 5862, 5863, 5864, 5865, 5866, 5867, 5868, 5869, 5870, 5871, 5872, 5873, 5874, 5875, 5876, 5877, 5878, 5879, 5880, 5881, 5883, 5884, 5885, 5886, 5887, 5889, 5890, 5891, 5892, 5893, 5894, 5895, 5896, 5897, 5898, 5899, 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907, 5908, 5909, 5910, 5911, 5912, 5913, 5914, 5915, 5916, 5917, 5918, 5919, 5920, 5921, 5922, 5923, 5924, 5925, 5926, 5927, 5928, 5929, 5930, 5931, 5932, 5933, 5934, 5937, 5938, 5939, 5940, 5941, 5942, 5943, 5944, 5945, 5946, 5947, 5948, 5950, 5951, 5952, 5953, 5954, 5955, 5956, 5957, 5958, 5959, 5960, 5961, 5962, 5963, 5964, 5965, 5966, 5967, 5968, 5969, 5970, 5971, 5973, 5974, 5975, 5976, 5977, 5978, 5979, 5980, 5981, 5982, 5983, 5984, 5985, 5986, 5987, 5989, 5990, 5991, 5992, 5993, 5994, 5995, 5996, 5998, 5999, 6000, 6001, 6002, 6003, 6004, 6005, 6006, 6007, 6009, 6010, 6011, 6012, 6013, 6014, 6015, 6016, 6017, 6018, 6019, 6020, 6021, 6022, 6023, 6024, 6025, 6026, 6027, 6028, 6029, 6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6046, 6047, 6048, 6049, 6050, 6051, 6053, 6054, 6055, 6056, 6057, 6058, 6059, 6060, 6061, 6062, 6063, 6064, 6065, 6066, 6067, 6068, 6069, 6070, 6071, 6072, 6073, 6075, 6076, 6077, 6079, 6080, 6081, 6083, 6084, 6085, 6086, 6087, 6088, 6089, 6090, 6091, 6092, 6093, 6094, 6095, 6096, 6097, 6098, 6099, 6100, 6101, 6102, 6103, 6104, 6105, 6106, 6107, 6108, 6109, 6110, 6111, 6113, 6114, 6115, 6116, 6117, 6118, 6120, 6121, 6122, 6123, 6124, 6125, 6126, 6127, 6128, 6130, 6131, 6132, 6133, 6134, 6135, 6136, 6137, 6139, 6140, 6141, 6142, 6143, 6144, 6145, 6146, 6148, 6149, 6150, 6151, 6152, 6154, 6155, 6156, 6157, 6158, 6159, 6160, 6161, 6163, 6164, 6165, 6166, 6167, 6168, 6169, 6170, 6171, 6172, 6173, 6174, 6175, 6176, 6177, 6178, 6179, 6181, 6182, 6183, 6184, 6185, 6186, 6187, 6188, 6189, 6190, 6191, 6192, 6193, 6194, 6195, 6196, 6197, 6198, 6199, 6200, 6201, 6202, 6203, 6204, 6205, 6206, 6207, 6208, 6209, 6210, 6211, 6212, 6213, 6214, 6215, 6216, 6217, 6218, 6220, 6221, 6222, 6223, 6224, 6225, 6226, 6227, 6228, 6230, 6232, 6233, 6234, 6235, 6236, 6237, 6238, 6239, 6240, 6241, 6242, 6243, 6244, 6245, 6246, 6247, 6248, 6249, 6250, 6251, 6252, 6253, 6254, 6255, 6258, 6259, 6260, 6261, 6262, 6263, 6264, 6265, 6266, 6267, 6268, 6269, 6270, 6272, 6273, 6274, 6275, 6276, 6277, 6278, 6279, 6280, 6281, 6282, 6283, 6284, 6285, 6286, 6290, 6291, 6292, 6293, 6294, 6295, 6296, 6297, 6298, 6300, 6301, 6302, 6303, 6304, 6305, 6306, 6308, 6309, 6310, 6312, 6313, 6314, 6316, 6317, 6318, 6319, 6320, 6321, 6322, 6323, 6324, 6325, 6326, 6327, 6328, 6329, 6330, 6331, 6332, 6333, 6334, 6335, 6336, 6337, 6339, 6340, 6341, 6342, 6343, 6344, 6345, 6346, 6347, 6348, 6349, 6350, 6351, 6352, 6353, 6354, 6355, 6357, 6358, 6359, 6360, 6361, 6362, 6363, 6364, 6365, 6366, 6367, 6368, 6369, 6370, 6371, 6372, 6373, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6381, 6382, 6384, 6385, 6386, 6387, 6388, 6389, 6390, 6391, 6392, 6393, 6394, 6395, 6396, 6397, 6400, 6401, 6402, 6403, 6404, 6405, 6406, 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6429, 6430, 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6448, 6449, 6450, 6451, 6452, 6453, 6454, 6456, 6457, 6458, 6459, 6460, 6461, 6462, 6463, 6464, 6465, 6466, 6467, 6468, 6469, 6470, 6471, 6472, 6473, 6474, 6475, 6476, 6477, 6480, 6481, 6482, 6483, 6484, 6485, 6486, 6488, 6489, 6490, 6491, 6492, 6493, 6494, 6496, 6497, 6498, 6499, 6500, 6501, 6502, 6503, 6504, 6505, 6506, 6507, 6508, 6509, 6510, 6511, 6512, 6513, 6514, 6515, 6516, 6517, 6518, 6521, 6522, 6523, 6524, 6525, 6526, 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 6537, 6538, 6539, 6540, 6541, 6542, 6543, 6544, 6545, 6548, 6549, 6550, 6551, 6552, 6553, 6554, 6555, 6556, 6557, 6558, 6561, 6562, 6563, 6565, 6566, 6567, 6568, 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6578, 6579, 6580, 6581, 6582, 6583, 6584, 6585, 6586, 6587, 6588, 6589, 6590, 6591, 6592, 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6603, 6604, 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6617, 6618, 6619, 6621, 6624, 6625, 6627, 6628, 6629, 6630, 6631, 6632, 6633, 6634, 6635, 6636, 6638, 6639, 6640, 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6649, 6650, 6651, 6652, 6653, 6654, 6655, 6658, 6659, 6660, 6661, 6662, 6663, 6664, 6665, 6666, 6668, 6669, 6670, 6671, 6672, 6673, 6675, 6676, 6678, 6679, 6681, 6682, 6683, 6684, 6686, 6687, 6688, 6690, 6691, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6725, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6736, 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6773, 6774, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 6784, 6785, 6786, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, 6797, 6798, 6799, 6800, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813, 6814, 6816, 6817, 6818, 6819, 6820, 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6830, 6831, 6832, 6833, 6834, 6835, 6836, 6837, 6838, 6841, 6842, 6843, 6844, 6845, 6846, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, 6917, 6918, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, 6941, 6943, 6944, 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6971, 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7004, 7005, 7007, 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, 7029, 7030, 7031, 7032, 7033, 7036, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7060, 7061, 7062, 7064, 7065, 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7077, 7078, 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, 7092, 7093, 7094, 7095, 7096, 7098, 7099, 7100, 7102, 7103, 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7114, 7115, 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, 7129, 7130, 7131, 7132, 7134, 7135, 7136, 7137, 7138, 7139, 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7148, 7149, 7150, 7151, 7152, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175, 7176, 7177, 7178, 7180, 7181, 7182, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7193, 7194, 7195, 7196, 7197, 7198, 7199, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, 7236, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7254, 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, 7285, 7286, 7287, 7288, 7289, 7291, 7292, 7293, 7294, 7295, 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7345, 7346, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7364, 7365, 7366, 7367, 7368, 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, 7382, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7398, 7399, 7400, 7402, 7403, 7405, 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, 7423, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7475, 7476, 7478, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7501, 7502, 7503, 7504, 7505, 7506, 7508, 7510, 7511, 7512, 7513, 7514, 7515, 7518, 7519, 7520, 7521, 7522, 7523, 7525, 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, 7551, 7552, 7553, 7554, 7556, 7557, 7558, 7559, 7560, 7561, 7562, 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, 7575, 7576, 7577, 7578, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7587, 7588, 7589, 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605, 7606, 7607, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, 7617, 7618, 7619, 7620, 7622, 7623, 7624, 7625, 7626, 7628, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7640, 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, 7677, 7678, 7679, 7680, 7681, 7682, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7699, 7700, 7701, 7702, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, 7773, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7809, 7810, 7811, 7812, 7813, 7814, 7816, 7817, 7818, 7819, 7820, 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833, 7834, 7835, 7836, 7837, 7839, 7840, 7841, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861, 7862, 7863, 7864, 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873, 7874, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 7936, 7937, 7938, 7939, 7940, 7941, 7942, 7944, 7946, 7947, 7948, 7949, 7951, 7952, 7953, 7954, 7956, 7957, 7958, 7959, 7960, 7961, 7962, 7963, 7965, 7966, 7967, 7968, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979, 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7993, 7994, 7995, 7996, 7997, 7998, 8000, 8001, 8002, 8003, 8004, 8005, 8006, 8007, 8008, 8009, 8010, 8011, 8012, 8014, 8015, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8028, 8029, 8030, 8031, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8040, 8042, 8043, 8044, 8046, 8047, 8048, 8049, 8050, 8051, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8061, 8062, 8063, 8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8083, 8084, 8085, 8086, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8115, 8116, 8117, 8118, 8119, 8120, 8123, 8124, 8125, 8126, 8127, 8128, 8129, 8130, 8131, 8132, 8133, 8134, 8135, 8136, 8137, 8138, 8139, 8140, 8142, 8143, 8144, 8145, 8146, 8147, 8148, 8149, 8150, 8152, 8153, 8154, 8155, 8157, 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8168, 8170, 8171, 8172, 8173, 8174, 8175, 8176, 8177, 8178, 8179, 8180, 8181, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8190, 8191, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8208, 8209, 8210, 8211, 8212, 8213, 8214, 8215, 8216, 8217, 8218, 8219, 8220, 8221, 8222, 8223, 8224, 8225, 8226, 8227, 8228, 8229, 8230, 8231, 8232, 8233, 8234, 8235, 8236, 8237, 8238, 8239, 8240, 8241, 8242, 8243, 8244, 8245, 8246, 8248, 8249, 8250, 8251, 8252, 8253, 8255, 8256, 8257, 8258, 8259, 8260, 8262, 8265, 8266, 8267, 8268, 8269, 8270, 8271, 8272, 8273, 8274, 8275, 8276, 8277, 8278, 8279, 8280, 8281, 8282, 8283, 8284, 8285, 8286, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 8304, 8305, 8306, 8307, 8308, 8309, 8311, 8312, 8313, 8314, 8315, 8316, 8317, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331, 8332, 8333, 8334, 8335, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8348, 8349, 8350, 8351, 8352, 8353, 8354, 8355, 8356, 8357, 8358, 8359, 8360, 8361, 8362, 8363, 8364, 8365, 8366, 8367, 8368, 8369, 8370, 8371, 8372, 8373, 8374, 8375, 8376, 8377, 8378, 8379, 8380, 8381, 8382, 8383, 8384, 8385, 8386, 8387, 8388, 8389, 8390, 8391, 8392, 8393, 8394, 8395, 8396, 8397, 8398, 8399, 8400, 8401, 8402, 8404, 8405, 8407, 8408, 8409, 8410, 8411, 8412, 8413, 8414, 8415, 8416, 8417, 8418, 8419, 8420, 8421, 8422, 8423, 8424, 8425, 8427, 8428, 8429, 8430, 8431, 8433, 8434, 8435, 8436, 8437, 8439, 8440, 8441, 8442, 8443, 8444, 8446, 8448, 8449, 8450, 8451, 8452, 8453, 8454, 8455, 8456, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464, 8465, 8466, 8467, 8468, 8469, 8470, 8471, 8472, 8473, 8474, 8475, 8476, 8477, 8478, 8480, 8481, 8482, 8483, 8484, 8485, 8487, 8488, 8489, 8490, 8491, 8492, 8493, 8494, 8495, 8496, 8497, 8498, 8499, 8501, 8502, 8503, 8504, 8505, 8506, 8507, 8508, 8509, 8510, 8512, 8514, 8515, 8516, 8517, 8518, 8519, 8520, 8521, 8523, 8524, 8525, 8526, 8527, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537, 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8546, 8547, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8568, 8569, 8570, 8571, 8572, 8574, 8575, 8576, 8577, 8578, 8579, 8580, 8581, 8582, 8583, 8584, 8585, 8586, 8587, 8588, 8589, 8591, 8592, 8594, 8595, 8596, 8597, 8598, 8599, 8600, 8601, 8602, 8603, 8604, 8605, 8606, 8607, 8608, 8609, 8610, 8611, 8612, 8613, 8614, 8615, 8616, 8617, 8618, 8619, 8620, 8621, 8622, 8623, 8624, 8625, 8626, 8628, 8629, 8630, 8632, 8633, 8634, 8635, 8636, 8637, 8638, 8639, 8640, 8641, 8642, 8643, 8644, 8645, 8646, 8647, 8648, 8649, 8650, 8651, 8654, 8655, 8656, 8657, 8658, 8659, 8660, 8661, 8662, 8663, 8664, 8665, 8666, 8667, 8668, 8669, 8670, 8672, 8673, 8674, 8675, 8676, 8677, 8678, 8679, 8680, 8681, 8682, 8684, 8685, 8686, 8687, 8688, 8689, 8690, 8691, 8693, 8694, 8695, 8696, 8697, 8698, 8699, 8700, 8701, 8702, 8704, 8705, 8706, 8707, 8709, 8710, 8711, 8712, 8714, 8715, 8716, 8717, 8718, 8719, 8720, 8721, 8722, 8723, 8724, 8725, 8726, 8727, 8728, 8729, 8730, 8731, 8733, 8734, 8735, 8736, 8737, 8739, 8740, 8741, 8742, 8743, 8744, 8745, 8747, 8748, 8749, 8750, 8751, 8752, 8753, 8755, 8756, 8757, 8758, 8759, 8760, 8761, 8764, 8765, 8766, 8767, 8768, 8769, 8770, 8771, 8772, 8773, 8774, 8775, 8776, 8777, 8778, 8780, 8781, 8782, 8783, 8784, 8785, 8786, 8787, 8788, 8789, 8790, 8791, 8792, 8793, 8794, 8795, 8796, 8797, 8798, 8800, 8801, 8802, 8803, 8804, 8805, 8807, 8808, 8809, 8810, 8811, 8812, 8813, 8814, 8815, 8816, 8817, 8818, 8819, 8820, 8821, 8822, 8823, 8824, 8825, 8827, 8828, 8829, 8830, 8832, 8833, 8834, 8835, 8836, 8837, 8838, 8839, 8840, 8841, 8842, 8843, 8844, 8845, 8846, 8847, 8848, 8849, 8850, 8851, 8852, 8853, 8854, 8855, 8856, 8857, 8858, 8859, 8861, 8862, 8863, 8864, 8865, 8866, 8867, 8869, 8870, 8871, 8872, 8873, 8874, 8876, 8877, 8878, 8879, 8880, 8881, 8882, 8884, 8885, 8886, 8887, 8888, 8889, 8890, 8891, 8893, 8895, 8896, 8897, 8898, 8900, 8901, 8902, 8903, 8904, 8905, 8906, 8907, 8908, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8918, 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8927, 8928, 8929, 8930, 8931, 8932, 8933, 8934, 8935, 8936, 8937, 8938, 8939, 8940, 8941, 8942, 8943, 8944, 8945, 8946, 8947, 8948, 8949, 8950, 8951, 8952, 8953, 8954, 8955, 8956, 8957, 8958, 8959, 8960, 8961, 8962, 8963, 8964, 8965, 8966, 8967, 8968, 8969, 8970, 8971, 8972, 8974, 8975, 8976, 8977, 8978, 8979, 8982, 8984, 8985, 8986, 8987, 8988, 8989, 8990, 8991, 8993, 8994, 8995, 8996, 8997, 8998, 8999, 9000, 9001, 9002, 9003, 9004, 9005, 9006, 9007, 9008, 9009, 9010, 9011, 9012, 9013, 9014, 9015, 9016, 9017, 9018, 9019, 9020, 9021, 9022, 9023, 9025, 9026, 9028, 9029, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9038, 9039, 9040, 9042, 9043, 9044, 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9054, 9055, 9056, 9057, 9058, 9059, 9060, 9061, 9062, 9063, 9064, 9065, 9067, 9068, 9069, 9070, 9072, 9073, 9074, 9075, 9076, 9077, 9078, 9079, 9080, 9081, 9082, 9083, 9085, 9086, 9087, 9088, 9089, 9091, 9092, 9093, 9094, 9096, 9097, 9098, 9099, 9100, 9101, 9102, 9103, 9104, 9105, 9106, 9107, 9109, 9110, 9112, 9113, 9114, 9115, 9117, 9118, 9119, 9120, 9121, 9122, 9123, 9124, 9125, 9126, 9127, 9128, 9129, 9130, 9131, 9132, 9133, 9134, 9136, 9137, 9138, 9139, 9140, 9141, 9143, 9144, 9145, 9146, 9147, 9149, 9150, 9151, 9153, 9154, 9155, 9156, 9157, 9158, 9159, 9161, 9162, 9163, 9165, 9166, 9167, 9168, 9169, 9170, 9171, 9172, 9173, 9174, 9175, 9176, 9177, 9179, 9180, 9181, 9182, 9183, 9184, 9185, 9186, 9187, 9188, 9189, 9190, 9191, 9192, 9193, 9194, 9195, 9196, 9197, 9198, 9199, 9200, 9202, 9203, 9204, 9205, 9206, 9207, 9208, 9209, 9210, 9211, 9212, 9213, 9214, 9215, 9216, 9217, 9218, 9219, 9220, 9221, 9222, 9223, 9224, 9225, 9226, 9227, 9228, 9229, 9230, 9231, 9232, 9233, 9234, 9235, 9236, 9237, 9238, 9239, 9241, 9242, 9243, 9244, 9245, 9248, 9249, 9250, 9253, 9254, 9255, 9256, 9257, 9258, 9259, 9261, 9262, 9263, 9264, 9265, 9266, 9268, 9269, 9270, 9271, 9272, 9273, 9274, 9276, 9277, 9278, 9279, 9280, 9281, 9282, 9283, 9285, 9286, 9287, 9288, 9290, 9291, 9292, 9293, 9294, 9295, 9297, 9298, 9299, 9300, 9301, 9302, 9303, 9304, 9305, 9306, 9307, 9308, 9310, 9311, 9312, 9313, 9314, 9315, 9316, 9317, 9318, 9319, 9320, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9329, 9330, 9331, 9332, 9333, 9334, 9336, 9337, 9338, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348, 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9361, 9362, 9363, 9364, 9365, 9366, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9381, 9382, 9383, 9384, 9385, 9387, 9389, 9390, 9391, 9393, 9395, 9396, 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9449, 9450, 9451, 9452, 9453, 9454, 9455, 9456, 9457, 9458, 9459, 9460, 9461, 9462, 9463, 9464, 9465, 9466, 9467, 9468, 9469, 9470, 9471, 9472, 9473, 9474, 9475, 9476, 9477, 9478, 9479, 9480, 9481, 9482, 9483, 9484, 9485, 9486, 9490, 9491, 9492, 9493, 9494, 9495, 9496, 9497, 9498, 9499, 9500, 9501, 9502, 9503, 9505, 9506, 9507, 9508, 9509, 9510, 9511, 9512, 9514, 9515, 9516, 9517, 9518, 9519, 9520, 9521, 9522, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9532, 9533, 9534, 9535, 9536, 9537, 9538, 9539, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550, 9552, 9554, 9555, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9564, 9565, 9566, 9567, 9568, 9569, 9570, 9571, 9572, 9573, 9575, 9577, 9578, 9579, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9591, 9592, 9593, 9594, 9595, 9596, 9597, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606, 9607, 9608, 9609, 9610, 9611, 9612, 9613, 9614, 9615, 9616, 9617, 9618, 9619, 9620, 9621, 9623, 9625, 9626, 9627, 9628, 9629, 9630, 9631, 9632, 9633, 9634, 9635, 9637, 9638, 9639, 9640, 9641, 9642, 9644, 9645, 9646, 9647, 9648, 9649, 9651, 9652, 9653, 9655, 9656, 9657, 9658, 9659, 9660, 9661, 9662, 9663, 9664, 9666, 9667, 9668, 9669, 9670, 9671, 9672, 9673, 9675, 9676, 9677, 9679, 9680, 9681, 9682, 9683, 9684, 9685, 9686, 9687, 9688, 9689, 9691, 9692, 9693, 9694, 9695, 9696, 9697, 9699, 9700, 9701, 9702, 9703, 9704, 9705, 9706, 9707, 9708, 9709, 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9717, 9718, 9719, 9720, 9721, 9722, 9723, 9724, 9726, 9727, 9728, 9729, 9730, 9731, 9732, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9756, 9757, 9758, 9759, 9760, 9761, 9762, 9763, 9764, 9765, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, 9775, 9776, 9777, 9778, 9779, 9780, 9781, 9783, 9784, 9785, 9787, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, 9806, 9808, 9809, 9810, 9811, 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820, 9821, 9822, 9823, 9824, 9825, 9826, 9827, 9828, 9829, 9830, 9831, 9832, 9833, 9834, 9835, 9836, 9837, 9838, 9839, 9840, 9841, 9842, 9843, 9844, 9845, 9848, 9849, 9850, 9851, 9852, 9853, 9855, 9856, 9857, 9858, 9860, 9861, 9862, 9863, 9864, 9865, 9866, 9867, 9868, 9869, 9870, 9871, 9872, 9873, 9874, 9875, 9876, 9878, 9879, 9880, 9881, 9882, 9883, 9884, 9885, 9886, 9887, 9888, 9889, 9890, 9892, 9893, 9894, 9896, 9897, 9898, 9899, 9900, 9901, 9902, 9903, 9904, 9905, 9906, 9907, 9908, 9910, 9911, 9912, 9913, 9914, 9915, 9916, 9917, 9918, 9919, 9920, 9921, 9922, 9923, 9924, 9925, 9928, 9929, 9930, 9933, 9936, 9938, 9939, 9940, 9941, 9942, 9943, 9944, 9947, 9948, 9949, 9950, 9951, 9952, 9953, 9954, 9955, 9957, 9959, 9960, 9961, 9962, 9964, 9966, 9967, 9968, 9969, 9970, 9971, 9972, 9973, 9974, 9976, 9977, 9978, 9980, 9981, 9982, 9983, 9984, 9985, 9986, 9987, 9988, 9990, 9991, 9992, 9993, 9994, 9995, 9996, 9997, 9998, 9999, 10000, 10001, 10002, 10003, 10004, 10006, 10007, 10008, 10009, 10010, 10011, 10012, 10013, 10014, 10015, 10018, 10019, 10020, 10021, 10022, 10023, 10025, 10026, 10028, 10029, 10030, 10031, 10033, 10034, 10035, 10037, 10039, 10040, 10041, 10042, 10043, 10044, 10045, 10046, 10047, 10048, 10049, 10050, 10051, 10053, 10054, 10055, 10056, 10057, 10058, 10059, 10060, 10061, 10062, 10063, 10064, 10065, 10066, 10067, 10068, 10069, 10072, 10073, 10074, 10075, 10076, 10077, 10078, 10079, 10080, 10081, 10082, 10084, 10085, 10086, 10087, 10089, 10092, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10110, 10111, 10112, 10113, 10114, 10115, 10116, 10117, 10118, 10119, 10120, 10121, 10122, 10123, 10124, 10125, 10126, 10127, 10128, 10129, 10130, 10131, 10133, 10134, 10135, 10136, 10137, 10138, 10139, 10140, 10141, 10143, 10144, 10145, 10146, 10148, 10151, 10152, 10153, 10154, 10156, 10157, 10158, 10159, 10160, 10161, 10162, 10163, 10164, 10166, 10168, 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185, 10186, 10187, 10188, 10189, 10190, 10191, 10192, 10193, 10194, 10195, 10196, 10197, 10199, 10200, 10201, 10202, 10203, 10204, 10207, 10208, 10209, 10210, 10212, 10213, 10214, 10215, 10216, 10217, 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10225, 10226, 10227, 10228, 10229, 10230, 10231, 10232, 10233, 10234, 10235, 10237, 10238, 10239, 10240, 10241, 10242, 10243, 10244, 10245, 10247, 10248, 10249, 10250, 10251, 10252, 10253, 10254, 10255, 10257, 10258, 10259, 10260, 10262, 10263, 10264, 10265, 10266, 10268, 10269, 10270, 10271, 10272, 10273, 10274, 10275, 10276, 10277, 10278, 10279, 10280, 10281, 10282, 10283, 10284, 10285, 10286, 10287, 10288, 10289, 10290, 10291, 10292, 10293, 10294, 10295, 10296, 10297, 10298, 10299, 10300, 10301, 10302, 10303, 10304, 10305, 10307, 10308, 10309, 10310, 10311, 10312, 10313, 10314, 10315, 10317, 10318, 10319, 10320, 10323, 10324, 10325, 10327, 10328, 10329, 10330, 10331, 10332, 10333, 10334, 10335, 10336, 10337, 10339, 10340, 10341, 10342, 10343, 10344, 10345, 10346, 10347, 10348, 10349, 10350, 10351, 10353, 10354, 10355, 10356, 10357, 10358, 10359, 10361, 10362, 10363, 10364, 10365, 10366, 10367, 10368, 10369, 10370, 10371, 10372, 10373, 10374, 10375, 10376, 10378, 10380, 10381, 10382, 10383, 10384, 10385, 10386, 10387, 10388, 10389, 10390, 10391, 10392, 10393, 10394, 10395, 10396, 10397, 10398, 10400, 10403, 10404, 10405, 10406, 10407, 10408, 10410, 10411, 10412, 10413, 10414, 10415, 10416, 10417, 10418, 10419, 10420, 10421, 10422, 10423, 10424, 10425, 10426, 10427, 10428, 10430, 10431, 10432, 10434, 10435, 10436, 10437, 10439, 10440, 10441, 10442, 10443, 10444, 10445, 10446, 10447, 10448, 10449, 10450, 10451, 10452, 10454, 10455, 10456, 10458, 10459, 10460, 10461, 10462, 10463, 10464, 10466, 10468, 10469, 10470, 10471, 10472, 10473, 10474, 10475, 10476, 10477, 10478, 10479, 10480, 10482, 10483, 10484, 10486, 10487, 10488, 10489, 10490, 10491, 10492, 10493, 10494, 10495, 10496, 10497, 10499, 10500, 10501, 10502, 10503, 10504, 10505, 10506, 10507, 10508, 10509, 10510, 10511, 10512, 10513, 10514, 10515, 10516, 10518, 10519, 10520, 10522, 10524, 10525, 10526, 10527, 10528, 10529, 10530, 10531, 10532, 10533, 10534, 10535, 10537, 10538, 10539, 10540, 10541, 10542, 10543, 10544, 10545, 10546, 10547, 10548, 10549, 10551, 10552, 10553, 10554, 10555, 10556, 10557, 10558, 10559, 10560, 10561, 10562, 10563, 10565, 10566, 10567, 10568, 10569, 10570, 10571, 10572, 10573, 10574, 10575, 10576, 10577, 10579, 10580, 10581, 10582, 10583, 10584, 10585, 10586, 10587, 10589, 10590, 10591, 10595, 10596, 10597, 10598, 10599, 10600, 10601, 10602, 10603, 10605, 10606, 10607, 10608, 10609, 10610, 10611, 10614, 10615, 10616, 10617, 10618, 10619, 10620, 10621, 10622, 10623, 10624, 10625, 10626, 10627, 10628, 10629, 10631, 10633, 10634, 10635, 10636, 10637, 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 10648, 10649, 10650, 10651, 10652, 10653, 10654, 10655, 10657, 10658, 10660, 10661, 10662, 10663, 10664, 10665, 10666, 10667, 10668, 10669, 10670, 10671, 10672, 10673, 10674, 10675, 10676, 10677, 10678, 10679, 10680, 10681, 10682, 10683, 10684, 10685, 10686, 10687, 10688, 10689, 10690, 10691, 10692, 10693, 10694, 10696, 10697, 10698, 10699, 10700, 10701, 10702, 10703, 10704, 10705, 10706, 10707, 10708, 10709, 10711, 10712, 10713, 10714, 10715, 10716, 10717, 10718, 10719, 10720, 10721, 10722, 10723, 10724, 10725, 10726, 10727, 10728, 10729, 10730, 10731, 10732, 10733, 10734, 10735, 10736, 10737, 10738, 10739, 10740, 10741, 10742, 10743, 10744, 10746, 10747, 10748, 10750, 10751, 10752, 10753, 10754, 10755, 10756, 10757, 10758, 10759, 10760, 10761, 10762, 10763, 10764, 10765, 10766, 10767, 10768, 10769, 10770, 10771, 10773, 10774, 10775, 10776, 10777, 10778, 10779, 10780, 10781, 10782, 10783, 10784, 10785, 10786, 10787, 10788, 10789, 10790, 10791, 10792, 10793, 10794, 10795, 10796, 10797, 10798, 10799, 10800, 10801, 10802, 10803, 10804, 10805, 10806, 10807, 10808, 10809, 10811, 10812, 10813, 10814, 10815, 10816, 10817, 10818, 10819, 10820, 10821, 10822, 10823, 10825, 10826, 10827, 10828, 10829, 10830, 10831, 10832, 10833, 10834, 10836, 10837, 10838, 10839, 10840, 10841, 10842, 10843, 10844, 10846, 10847, 10848, 10849, 10850, 10851, 10853, 10854, 10855, 10856, 10857, 10858, 10859, 10861, 10862, 10863, 10864, 10865, 10866, 10867, 10868, 10869, 10870, 10871, 10872, 10873, 10874, 10875, 10878, 10879, 10880, 10883, 10885, 10886, 10887, 10889, 10890, 10891, 10892, 10893, 10894, 10895, 10896, 10897, 10898, 10899, 10900, 10901, 10902, 10903, 10904, 10905, 10906, 10907, 10908, 10909, 10910, 10911, 10912, 10913, 10914, 10915, 10916, 10917, 10918, 10919, 10920, 10921, 10922, 10923, 10924, 10925, 10926, 10927, 10928, 10929, 10931, 10933, 10934, 10935, 10936, 10937, 10938, 10939, 10941, 10942, 10943, 10945, 10946, 10947, 10949, 10950, 10951, 10952, 10953, 10954, 10955, 10957, 10958, 10959, 10960, 10962, 10963, 10964, 10966, 10967, 10968, 10969, 10970, 10971, 10973, 10974, 10975, 10976, 10977, 10978, 10979, 10980, 10981, 10982, 10983, 10984, 10986, 10990, 10991, 10992, 10993, 10994, 10995, 10996, 10999, 11000, 11001, 11003, 11006, 11008, 11009, 11010, 11011, 11012, 11013, 11015, 11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025, 11026, 11027, 11028, 11029, 11030, 11031, 11032, 11033, 11034, 11035, 11036, 11037, 11038, 11039, 11040, 11041, 11042, 11043, 11044, 11046, 11048, 11049, 11050, 11051, 11052, 11053, 11054, 11055, 11056, 11057, 11058, 11059, 11060, 11061, 11062, 11063, 11064, 11065, 11066, 11067, 11068, 11069, 11070, 11071, 11072, 11073, 11074, 11075, 11077, 11079, 11080, 11081, 11082, 11083, 11084, 11085, 11086, 11087, 11088, 11089, 11090, 11091, 11092, 11093, 11094, 11095, 11096, 11097, 11098, 11099, 11100, 11101, 11102, 11103, 11104, 11105, 11106, 11107, 11109, 11110, 11111, 11112, 11113, 11115, 11116, 11117, 11118, 11119, 11120, 11121, 11122, 11123, 11124, 11125, 11126, 11129, 11130, 11131, 11132, 11133, 11134, 11135, 11136, 11137, 11138, 11139, 11140, 11141, 11142, 11145, 11146, 11147, 11148, 11149, 11150, 11151, 11152, 11153, 11154, 11155, 11156, 11157, 11158, 11159, 11160, 11161, 11162, 11163, 11164, 11165, 11166, 11167, 11168, 11169, 11170, 11171, 11172, 11173, 11175, 11176, 11177, 11178, 11179, 11180, 11181, 11182, 11183, 11184, 11185, 11186, 11187, 11188, 11189, 11190, 11191, 11192, 11193, 11194, 11195, 11196, 11200, 11202, 11203, 11204, 11205, 11209, 11210, 11211, 11213, 11214, 11215, 11216, 11217, 11218, 11219, 11220, 11221, 11222, 11223, 11224, 11225, 11226, 11227, 11228, 11229, 11230, 11231, 11232, 11233, 11234, 11235, 11236, 11237, 11238, 11240, 11241, 11242, 11244, 11245, 11246, 11247, 11248, 11249, 11250, 11251, 11252, 11255, 11256, 11257, 11258, 11259, 11260, 11261, 11262, 11263, 11264, 11265, 11266, 11267, 11268, 11270, 11271, 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11283, 11284, 11286, 11288, 11289, 11290, 11292, 11293, 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, 11304, 11306, 11307, 11308, 11309, 11310, 11311, 11314, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, 11338, 11339, 11340, 11341, 11342, 11343, 11345, 11346, 11347, 11348, 11349, 11350, 11351, 11352, 11354, 11355, 11356, 11358, 11361, 11362, 11363, 11364, 11365, 11366, 11367, 11373, 11374, 11375, 11376, 11378, 11379, 11380, 11381, 11382, 11383, 11384, 11385, 11386, 11387, 11388, 11389, 11390, 11391, 11392, 11394, 11396, 11397, 11398, 11399, 11401, 11402, 11403, 11404, 11405, 11406, 11407, 11408, 11409, 11410, 11411, 11412, 11413, 11414, 11415, 11416, 11417, 11418, 11419, 11420, 11421, 11422, 11423, 11424, 11425, 11426, 11427, 11428, 11429, 11430, 11431, 11432, 11433, 11434, 11435, 11436, 11437, 11438, 11439, 11440, 11441, 11442, 11444, 11446, 11447, 11448, 11449, 11450, 11451, 11452, 11453, 11454, 11455, 11456, 11457, 11458, 11459, 11460, 11461, 11462, 11463, 11465, 11466, 11468, 11469, 11470, 11471, 11472, 11473, 11474, 11475, 11476, 11477, 11478, 11479, 11480, 11481, 11482, 11483, 11484, 11485, 11486, 11488, 11489, 11490, 11491, 11493, 11496, 11497, 11498, 11499, 11500, 11501, 11502, 11503, 11504, 11505, 11506, 11507, 11508, 11509, 11510, 11511, 11516, 11517, 11518, 11519, 11521, 11522, 11523, 11524, 11526, 11527, 11528, 11529, 11530, 11531, 11532, 11533, 11534, 11535, 11536, 11537, 11538, 11539, 11540, 11541, 11542, 11543, 11548, 11549, 11550, 11551, 11552, 11553, 11554, 11555, 11556, 11557, 11558, 11559, 11560, 11561, 11562, 11563, 11564, 11565, 11566, 11567, 11568, 11569, 11570, 11571, 11572, 11573, 11574, 11575, 11576, 11577, 11578, 11579, 11581, 11582, 11583, 11584, 11585, 11586, 11587, 11588, 11591, 11592, 11593, 11594, 11595, 11596, 11597, 11598, 11601, 11604, 11606, 11607, 11608, 11609, 11610, 11613, 11614, 11615, 11618, 11619, 11620, 11621, 11622, 11623, 11624, 11625, 11626, 11627, 11629, 11630, 11631, 11632, 11633, 11634, 11635, 11636, 11637, 11638, 11639, 11640, 11641, 11642, 11643, 11644, 11645, 11646, 11647, 11648, 11649, 11650, 11652, 11653, 11654, 11655, 11656, 11657, 11659, 11660, 11661, 11662, 11664, 11665, 11666, 11667, 11668, 11669, 11670, 11671, 11672, 11673, 11675, 11676, 11679, 11681, 11682, 11683, 11684, 11686, 11687, 11688, 11689, 11690, 11691, 11693, 11694, 11695, 11696, 11697, 11698, 11699, 11701, 11702, 11703, 11704, 11705, 11706, 11707, 11708, 11709, 11710, 11713, 11714, 11715, 11716, 11718, 11719, 11720, 11721, 11722, 11723, 11724, 11725, 11726, 11727, 11728, 11729, 11730, 11731, 11733, 11734, 11735, 11736, 11737, 11738, 11739, 11740, 11741, 11742, 11743, 11744, 11745, 11746, 11747, 11748, 11749, 11750, 11751, 11752, 11754, 11755, 11756, 11757, 11758, 11759, 11761, 11762, 11763, 11764, 11765, 11766, 11767, 11768, 11769, 11770, 11771, 11772, 11773, 11774, 11775, 11776, 11777, 11778, 11779, 11780, 11781, 11782, 11783, 11784, 11785, 11788, 11789, 11790, 11791, 11792, 11793, 11794, 11795, 11796, 11797, 11798, 11799, 11800, 11801, 11802, 11803, 11804, 11805, 11806, 11807, 11808, 11809, 11810, 11811, 11812, 11813, 11814, 11815, 11816, 11817, 11818, 11819, 11820, 11821, 11822, 11823, 11824, 11825, 11826, 11827, 11828, 11829, 11830, 11831, 11833, 11834, 11835, 11836, 11837, 11838, 11839, 11840, 11841, 11842, 11843, 11844, 11845, 11846, 11847, 11848, 11849, 11850, 11851, 11852, 11853, 11854, 11856, 11857, 11858, 11859, 11860, 11862, 11863, 11864, 11865, 11866, 11867, 11868, 11869, 11870, 11871, 11872, 11873, 11874, 11875, 11877, 11878, 11879, 11880, 11881, 11882, 11883, 11884, 11886, 11887, 11888, 11889, 11890, 11892, 11893, 11894, 11895, 11896, 11897, 11899, 11900, 11901, 11902, 11903, 11904, 11905, 11906, 11907, 11909, 11910, 11911, 11912, 11913, 11914, 11915, 11916, 11917, 11918, 11919, 11920, 11921, 11922, 11923, 11924, 11925, 11926, 11927, 11928, 11930, 11931, 11932, 11933, 11934, 11936, 11937, 11938, 11939, 11941, 11942, 11943, 11944, 11946, 11947, 11948, 11950, 11951, 11952, 11953, 11954, 11955, 11956, 11957, 11958, 11960, 11962, 11964, 11965, 11966, 11967, 11968, 11969, 11970, 11971, 11972, 11974, 11975, 11976, 11977, 11978, 11979, 11980, 11981, 11982, 11983, 11984, 11985, 11988, 11989, 11990, 11991, 11992, 11993, 11995, 11996, 11998, 12000, 12001, 12003, 12004, 12005, 12006, 12007, 12008, 12009, 12010, 12011, 12012, 12013, 12014, 12015, 12016, 12017, 12018, 12019, 12020, 12021, 12022, 12024, 12025, 12027, 12028, 12029, 12030, 12031, 12032, 12035, 12036, 12037, 12038, 12039, 12040, 12041, 12042, 12043, 12044, 12045, 12047, 12048, 12049, 12050, 12052, 12053, 12054, 12055, 12056, 12057, 12058, 12059, 12060, 12061, 12062, 12063, 12064, 12065, 12066, 12067, 12068, 12069, 12071, 12072, 12073, 12074, 12075, 12077, 12078, 12079, 12080, 12081, 12082, 12083, 12084, 12085, 12086, 12087, 12089, 12090, 12091, 12092, 12093, 12094, 12095, 12096, 12097, 12098, 12099, 12100, 12101, 12102, 12103, 12104, 12105, 12106, 12107, 12108, 12109, 12110, 12111, 12112, 12113, 12114, 12115, 12116, 12117, 12118, 12119, 12120, 12121, 12122, 12123, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, 12133, 12134, 12135, 12136, 12137, 12138, 12139, 12140, 12141, 12142, 12143, 12144, 12145, 12147, 12148, 12149, 12150, 12151, 12152, 12155, 12156, 12157, 12158, 12159, 12160, 12163, 12164, 12165, 12166, 12168, 12169, 12170, 12171, 12172, 12173, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185, 12186, 12187, 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12195, 12196, 12198, 12199, 12200, 12202, 12203, 12204, 12205, 12207, 12209, 12210, 12212, 12214, 12215, 12216, 12217, 12218, 12219, 12220, 12221, 12222, 12223, 12224, 12225, 12226, 12227, 12228, 12230, 12231, 12232, 12233, 12234, 12235, 12236, 12237, 12239, 12240, 12242, 12243, 12244, 12245, 12246, 12247, 12248, 12249, 12250, 12251, 12252, 12253, 12254, 12255, 12256, 12257, 12258, 12259, 12260, 12261, 12262, 12264, 12266, 12267, 12268, 12269, 12270, 12271, 12272, 12273, 12274, 12275, 12276, 12277, 12279, 12280, 12281, 12282, 12283, 12284, 12285, 12286, 12288, 12289, 12291, 12292, 12293, 12294, 12295, 12296, 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12306, 12307, 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12316, 12317, 12318, 12319, 12320, 12321, 12322, 12324, 12325, 12326, 12327, 12328, 12329, 12331, 12332, 12334, 12335, 12336, 12337, 12338, 12339, 12340, 12343, 12344, 12345, 12346, 12347, 12349, 12350, 12351, 12352, 12353, 12354, 12356, 12357, 12358, 12359, 12360, 12361, 12362, 12363, 12365, 12366, 12367, 12368, 12369, 12370, 12371, 12373, 12374, 12375, 12376, 12377, 12378, 12379, 12381, 12382, 12383, 12384, 12385, 12387, 12388, 12389, 12390, 12391, 12393, 12395, 12396, 12397, 12398, 12399, 12400, 12401, 12402, 12403, 12406, 12407, 12408, 12409, 12410, 12411, 12412, 12413, 12415, 12416, 12417, 12418, 12419, 12420, 12421, 12422, 12423, 12424, 12425, 12426, 12427, 12429, 12430, 12431, 12432, 12433, 12434, 12435, 12437, 12438, 12439, 12440, 12441, 12442, 12444, 12445, 12446, 12447, 12448, 12449, 12450, 12451, 12452, 12453, 12454, 12455, 12456, 12457, 12458, 12459, 12460, 12461, 12462, 12463, 12464, 12465, 12466, 12467, 12468, 12470, 12471, 12472, 12473, 12474, 12475, 12476, 12477, 12479, 12480, 12481, 12482, 12483, 12484, 12485, 12486, 12487, 12488, 12489, 12490, 12491, 12492, 12493, 12494, 12495, 12496, 12497, 12498, 12499, 12500, 12501, 12502, 12503, 12504, 12505, 12506, 12509, 12510, 12511, 12512, 12513, 12514, 12515, 12516, 12517, 12520, 12521, 12522, 12525, 12526, 12527, 12528, 12529, 12530, 12531, 12532, 12534, 12535, 12536, 12537, 12538, 12539, 12540, 12541, 12542, 12543, 12544, 12545, 12546, 12547, 12548, 12549, 12550, 12551, 12552, 12554, 12556, 12557, 12558, 12559, 12560, 12561, 12562, 12563, 12564, 12565, 12566, 12567, 12568, 12569, 12570, 12571, 12572, 12573, 12574, 12575, 12576, 12577, 12578, 12579, 12580, 12581, 12582, 12583, 12584, 12585, 12586, 12587, 12588, 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12597, 12598, 12599, 12600, 12601, 12603, 12604, 12606, 12607, 12608, 12609, 12610, 12611, 12612, 12613, 12614, 12615, 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12624, 12625, 12626, 12627, 12628, 12629, 12630, 12631, 12632, 12633, 12634, 12635, 12636, 12637, 12638, 12639, 12640, 12641, 12642, 12643, 12644, 12645, 12647, 12648, 12649, 12650, 12651, 12652, 12654, 12655, 12656, 12657, 12658, 12659, 12660, 12661, 12662, 12664, 12665, 12666, 12667, 12668, 12669, 12670, 12671, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680, 12684, 12686, 12687, 12688, 12689, 12690, 12691, 12692, 12693, 12694, 12695, 12696, 12697, 12698, 12699, 12700, 12701, 12702, 12703, 12705, 12706, 12707, 12708, 12709, 12710, 12711, 12712, 12713, 12714, 12715, 12718, 12719, 12720, 12721, 12722, 12723, 12724, 12725, 12726, 12727, 12728, 12729, 12730, 12732, 12733, 12735, 12736, 12738, 12739, 12740, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749, 12750, 12751, 12752, 12753, 12754, 12755, 12756, 12758, 12759, 12760, 12761, 12762, 12764, 12765, 12766, 12767, 12768, 12769, 12770, 12771, 12772, 12773, 12775, 12776, 12777, 12778, 12779, 12780, 12781, 12782, 12783, 12784, 12785, 12786, 12788, 12789, 12790, 12791, 12793, 12794, 12795, 12796, 12797, 12798, 12799, 12800, 12801, 12802, 12803, 12807, 12808, 12809, 12810, 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818, 12819, 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, 12831, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849, 12851, 12852, 12853, 12855, 12856, 12857, 12858, 12859, 12860, 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, 12872, 12873, 12874, 12875, 12876, 12877, 12878, 12879, 12880, 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12889, 12890, 12891, 12892, 12893, 12894, 12895, 12896, 12897, 12898, 12900, 12901, 12902, 12903, 12904, 12905, 12906, 12907, 12908, 12909, 12910, 12911, 12912, 12913, 12914, 12916, 12917, 12918, 12919, 12920, 12921, 12922, 12923, 12924, 12925, 12926, 12927, 12928, 12929, 12930, 12931, 12932, 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945, 12946, 12947, 12948, 12949, 12950, 12952, 12954, 12956, 12957, 12958, 12959, 12960, 12961, 12962, 12963, 12964, 12965, 12967, 12968, 12969, 12970, 12971, 12973, 12974, 12975, 12977, 12978, 12979, 12980, 12981, 12982, 12983, 12984, 12985, 12986, 12987, 12988, 12989, 12990, 12991, 12992, 12994, 12995, 12997, 12998, 12999, 13000, 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010, 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, 13021, 13022, 13023, 13024, 13025, 13026, 13027, 13028, 13029, 13030, 13031, 13032, 13033, 13034, 13035, 13036, 13037, 13038, 13039, 13040, 13041, 13042, 13043, 13044, 13045, 13046, 13048, 13049, 13050, 13051, 13052, 13054, 13055, 13057, 13058, 13060, 13061, 13063, 13064, 13066, 13068, 13069, 13070, 13071, 13072, 13073, 13077, 13079, 13080, 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089, 13090, 13091, 13092, 13094, 13095, 13097, 13098, 13099, 13100, 13102, 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, 13111, 13112, 13114, 13116, 13117, 13118, 13119, 13120, 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128, 13129, 13130, 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13139, 13140, 13141, 13142, 13144, 13145, 13146, 13147, 13148, 13149, 13150, 13152, 13153, 13155, 13156, 13157, 13158, 13161, 13162, 13163, 13164, 13166, 13168, 13169, 13170, 13171, 13173, 13174, 13177, 13179, 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, 13191, 13192, 13195, 13196, 13197, 13198, 13199, 13200, 13202, 13203, 13204, 13206, 13207, 13208, 13211, 13212, 13213, 13214, 13215, 13218, 13219, 13220, 13221, 13222, 13223, 13224, 13225, 13226, 13228, 13229, 13230, 13231, 13232, 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13243, 13244, 13245, 13246, 13247, 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258, 13259, 13260, 13261, 13262, 13263, 13265, 13266, 13267, 13268, 13269, 13270, 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, 13281, 13282, 13283, 13284, 13285, 13286, 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295, 13297, 13298, 13299, 13300, 13301, 13304, 13305, 13306, 13310, 13311, 13312, 13313, 13314, 13315, 13316, 13318, 13319, 13321, 13322, 13323, 13324, 13325, 13326, 13327, 13328, 13329, 13330, 13333, 13334, 13335, 13336, 13338, 13341, 13342, 13343, 13344, 13345, 13346, 13347, 13348, 13352, 13353, 13354, 13355, 13356, 13357, 13358, 13359, 13360, 13363, 13364, 13365, 13366, 13368, 13369, 13370, 13371, 13372, 13373, 13374, 13375, 13376, 13377, 13378, 13379, 13380, 13381, 13383, 13385, 13386, 13387, 13388, 13389, 13390, 13391, 13393, 13395, 13397, 13398, 13399, 13400, 13401, 13402, 13403, 13404, 13406, 13407, 13408, 13409, 13410, 13411, 13413, 13415, 13416, 13418, 13420, 13422, 13423, 13424, 13425, 13426, 13427, 13428, 13429, 13430, 13431, 13432, 13433, 13434, 13435, 13436, 13437, 13438, 13439, 13442, 13448, 13449, 13451, 13452, 13454, 13456, 13457, 13459, 13460, 13461, 13462, 13464, 13465, 13467, 13468, 13469, 13470, 13471, 13472, 13473, 13474, 13475, 13477, 13478, 13479, 13480, 13481, 13482, 13483, 13484, 13485, 13486, 13487, 13489, 13490, 13491, 13492, 13493, 13494, 13495, 13496, 13497, 13498, 13499, 13500, 13502, 13503, 13505, 13507, 13508, 13509, 13510, 13512, 13513, 13514, 13515, 13516, 13519, 13520, 13521, 13522, 13523, 13524, 13525, 13526, 13527, 13528, 13529, 13530, 13531, 13532, 13533, 13534, 13535, 13536, 13537, 13538, 13539, 13540, 13542, 13543, 13544, 13545, 13547, 13548, 13549, 13550, 13551, 13552, 13553, 13554, 13555, 13556, 13557, 13559, 13560, 13562, 13563, 13564, 13565, 13566, 13567, 13568, 13569, 13570, 13571, 13572, 13573, 13574, 13575, 13576, 13577, 13578, 13580, 13581, 13582, 13583, 13584, 13585, 13586, 13587, 13588, 13589, 13590, 13593, 13594, 13596, 13597, 13598, 13601, 13604, 13605, 13607, 13609, 13610, 13611, 13612, 13613, 13614, 13615, 13616, 13617, 13618, 13619, 13620, 13622, 13623, 13626, 13627, 13628, 13629, 13630, 13631, 13633, 13634, 13635, 13636, 13637, 13638, 13639, 13640, 13641, 13642, 13643, 13644, 13645, 13647, 13648, 13650, 13651, 13652, 13653, 13654, 13655, 13656, 13658, 13659, 13661, 13662, 13663, 13664, 13665, 13666, 13667, 13668, 13669, 13670, 13671, 13672, 13673, 13674, 13676, 13677, 13678, 13679, 13680, 13681, 13682, 13683, 13685, 13686, 13688, 13689, 13690, 13691, 13692, 13693, 13694, 13695, 13696, 13697, 13698, 13699, 13700, 13701, 13702, 13703, 13704, 13705, 13707, 13708, 13709, 13710, 13711, 13712, 13713, 13714, 13715, 13716, 13717, 13718, 13719, 13720, 13722, 13723, 13724, 13725, 13726, 13727, 13728, 13729, 13730, 13731, 13732, 13733, 13734, 13735, 13736, 13737, 13738, 13739, 13740, 13741, 13742, 13743, 13744, 13745, 13746, 13747, 13748, 13749, 13750, 13752, 13753, 13754, 13756, 13757, 13758, 13759, 13761, 13762, 13763, 13764, 13765, 13766, 13768, 13769, 13770, 13771, 13773, 13774, 13776, 13777, 13778, 13779, 13780, 13784, 13785, 13786, 13787, 13788, 13790, 13791, 13792, 13793, 13794, 13795, 13797, 13798, 13799, 13800, 13801, 13802, 13803, 13805, 13808, 13812, 13813, 13816, 13817, 13818, 13819, 13820, 13821, 13822, 13823, 13825, 13826, 13827, 13828, 13829, 13830, 13831, 13832, 13833, 13834, 13835, 13837, 13838, 13840, 13841, 13842, 13843, 13844, 13845, 13846, 13847, 13848, 13849, 13850, 13851, 13852, 13853, 13854, 13855, 13856, 13857, 13858, 13859, 13860, 13861, 13862, 13863, 13864, 13865, 13866, 13867, 13868, 13869, 13870, 13871, 13872, 13873, 13874, 13875, 13876, 13877, 13878, 13879, 13880, 13881, 13882, 13883, 13884, 13885, 13886, 13887, 13888, 13889, 13890, 13891, 13892, 13893, 13894, 13895, 13896, 13897, 13898, 13899, 13900, 13901, 13903, 13904, 13905, 13906, 13907, 13908, 13909, 13910, 13912, 13915, 13916, 13917, 13918, 13919, 13920, 13921, 13922, 13923, 13924, 13926, 13927, 13928, 13929, 13930, 13931, 13932, 13933, 13934, 13935, 13936, 13937, 13938, 13939, 13940, 13941, 13942, 13943, 13944, 13945, 13946, 13947, 13948, 13949, 13950, 13951, 13952, 13953, 13954, 13955, 13960, 13963, 13964, 13965, 13969, 13970, 13971, 13972, 13975, 13976, 13978, 13979, 13980, 13981, 13983, 13987, 13988, 13990, 13991, 13992, 13993, 13994, 13995, 13996, 13997, 13998, 13999, 14000, 14001, 14002, 14003, 14004, 14005, 14006, 14007, 14008, 14009, 14010, 14011, 14013, 14014, 14017, 14018, 14019, 14020, 14021, 14022, 14024, 14025, 14026, 14027, 14028, 14029, 14030, 14031, 14032, 14033, 14034, 14035, 14037, 14038, 14040, 14042, 14043, 14044, 14045, 14046, 14050, 14051, 14052, 14053, 14054, 14055, 14056, 14057, 14059, 14060, 14061, 14062, 14063, 14065, 14066, 14067, 14068, 14069, 14070, 14071, 14073, 14074, 14075, 14077, 14078, 14079, 14080, 14083, 14084, 14085, 14086, 14087, 14093, 14094, 14095, 14096, 14097, 14098, 14099, 14100, 14101, 14102, 14103, 14104, 14105, 14106, 14107, 14108, 14109, 14110, 14111, 14113, 14115, 14117, 14118, 14119, 14120, 14121, 14122, 14123, 14124, 14125, 14126, 14127, 14128, 14129, 14130, 14131, 14132, 14133, 14134, 14135, 14138, 14140, 14141, 14142, 14143, 14147, 14148, 14149, 14151, 14152, 14153, 14154, 14155, 14156, 14158, 14159, 14161, 14162, 14163, 14164, 14166, 14167, 14169, 14170, 14172, 14173, 14174, 14175, 14176, 14177, 14178, 14179, 14180, 14181, 14182, 14183, 14184, 14185, 14187, 14188, 14189, 14190, 14191, 14192, 14193, 14194, 14195, 14196, 14199, 14200, 14201, 14205, 14207, 14208, 14209, 14210, 14212, 14213, 14214, 14215, 14216, 14217, 14218, 14219, 14220, 14221, 14222, 14223, 14224, 14225, 14226, 14227, 14228, 14229, 14231, 14232, 14233, 14236, 14237, 14238, 14239, 14240, 14241, 14242, 14243, 14244, 14245, 14246, 14247, 14250, 14251, 14252, 14256, 14257, 14258, 14259, 14260, 14261, 14262, 14263, 14264, 14265, 14266, 14267, 14268, 14269, 14271, 14272, 14273, 14274, 14275, 14276, 14277, 14278, 14279, 14281, 14282, 14283, 14284, 14285, 14286, 14287, 14288, 14290, 14291, 14292, 14293, 14294, 14295, 14296, 14297, 14298, 14299, 14300, 14301, 14302, 14303, 14304, 14305, 14306, 14307, 14309, 14310, 14311, 14313, 14314, 14315, 14316, 14317, 14318, 14319, 14320, 14321, 14322, 14323, 14324, 14325, 14326, 14327, 14328, 14329, 14330, 14331, 14332, 14333, 14334, 14335, 14336, 14337, 14338, 14339, 14340, 14341, 14342, 14344, 14345, 14346, 14347, 14348, 14349, 14350, 14351, 14352, 14353, 14354, 14355, 14356, 14357, 14358, 14359, 14360, 14361, 14363, 14364, 14365, 14366, 14367, 14368, 14369, 14370, 14371, 14372, 14377, 14378, 14379, 14380, 14381, 14382, 14383, 14384, 14385, 14386, 14387, 14388, 14389, 14390, 14392, 14393, 14396, 14397, 14398, 14400, 14401, 14403, 14404, 14409, 14410, 14411, 14412, 14414, 14415, 14416, 14418, 14419, 14420, 14421, 14422, 14423, 14424, 14425, 14427, 14428, 14429, 14430, 14431, 14432, 14433, 14434, 14435, 14436, 14437, 14438, 14439, 14443, 14444, 14445, 14446, 14447, 14448, 14449, 14450, 14451, 14452, 14453, 14454, 14455, 14456, 14458, 14459, 14460, 14463, 14464, 14465, 14467, 14468, 14469, 14470, 14471, 14472, 14473, 14474, 14475, 14476, 14477, 14478, 14479, 14481, 14482, 14483, 14484, 14485, 14486, 14487, 14488, 14489, 14490, 14492, 14495, 14496, 14497, 14498, 14499, 14500, 14501, 14502, 14503, 14504, 14506, 14507, 14508, 14509, 14510, 14511, 14512, 14513, 14514, 14515, 14516, 14517, 14518, 14519, 14520, 14521, 14522, 14523, 14524, 14525, 14526, 14527, 14528, 14530, 14531, 14532, 14533, 14534, 14535, 14536, 14537, 14538, 14539, 14540, 14541, 14542, 14544, 14545, 14546, 14547, 14548, 14550, 14551, 14552, 14553, 14554, 14556, 14557, 14558, 14559, 14561, 14562, 14563, 14564, 14565, 14566, 14567, 14568, 14569, 14570, 14571, 14572, 14573, 14575, 14576, 14577, 14579, 14580, 14581, 14582, 14583, 14584, 14585, 14586, 14587, 14588, 14589, 14590, 14591, 14592, 14593, 14594, 14595, 14596, 14598, 14599, 14600, 14603, 14604, 14605, 14606, 14607, 14608, 14609, 14610, 14611, 14612, 14613, 14615, 14618, 14619, 14620, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, 14661, 14662, 14663, 14664, 14665, 14666, 14668, 14671, 14672, 14673, 14674, 14675, 14676, 14677, 14678, 14680, 14681, 14682, 14683, 14685, 14686, 14688, 14689, 14690, 14692, 14693, 14694, 14695, 14696, 14697, 14698, 14700, 14701, 14704, 14705, 14706, 14707, 14708, 14709, 14710, 14711, 14712, 14713, 14714, 14716, 14717, 14718, 14719, 14721, 14722, 14723, 14724, 14725, 14726, 14727, 14728, 14729, 14730, 14731, 14732, 14733, 14734, 14735, 14736, 14737, 14738, 14739, 14740, 14742, 14744, 14745, 14746, 14749, 14750, 14751, 14753, 14754, 14755, 14756, 14757, 14758, 14759, 14760, 14761, 14762, 14763, 14764, 14765, 14766, 14767, 14768, 14769, 14770, 14771, 14772, 14773, 14775, 14776, 14777, 14778, 14779, 14780, 14781, 14782, 14783, 14784, 14785, 14786, 14787, 14789, 14790, 14791, 14792, 14793, 14795, 14796, 14797, 14798, 14799, 14800, 14801, 14802, 14803, 14804, 14805, 14807, 14808, 14809, 14810, 14811, 14812, 14813, 14815, 14816, 14817, 14818, 14819, 14820, 14822, 14823, 14825, 14826, 14827, 14828, 14829, 14830, 14831, 14832, 14833, 14834, 14835, 14836, 14837, 14838, 14839, 14840, 14841, 14842, 14843, 14844, 14845, 14846, 14847, 14848, 14849, 14850, 14852, 14853, 14854, 14856, 14857, 14858, 14859, 14861, 14862, 14863, 14865, 14866, 14867, 14868, 14869, 14870, 14871, 14873, 14874, 14875, 14876, 14877, 14878, 14879, 14880, 14881, 14882, 14884, 14885, 14886, 14887, 14888, 14889, 14890, 14891, 14894, 14895, 14896, 14897, 14898, 14899, 14900, 14901, 14902, 14903, 14904, 14905, 14906, 14907, 14908, 14909, 14910, 14911, 14912, 14913, 14914, 14915, 14917, 14919, 14920, 14922, 14923, 14924, 14925, 14926, 14927, 14928, 14930, 14931, 14932, 14934, 14935, 14936, 14937, 14938, 14939, 14940, 14941, 14942, 14943, 14944, 14945, 14946, 14948, 14949, 14950, 14951, 14952, 14953, 14954, 14955, 14956, 14957, 14958, 14959, 14961, 14962, 14968, 14969, 14970, 14974, 14975, 14976, 14977, 14978, 14979, 14980, 14981, 14982, 14983, 14984, 14985, 14986, 14987, 14988, 14989, 14990, 14991, 14992, 14993, 14995, 14996, 14997, 15001, 15002, 15003, 15004, 15006, 15007, 15008, 15009, 15010, 15012, 15013, 15014, 15015, 15016, 15017, 15018, 15019, 15020, 15021, 15022, 15024, 15029, 15030, 15031, 15032, 15033, 15034, 15035, 15036, 15038, 15039, 15040, 15042, 15043, 15046, 15047, 15048, 15049, 15050, 15051, 15052, 15054, 15055, 15056, 15057, 15058, 15060, 15061, 15062, 15063, 15064, 15066, 15067, 15068, 15069, 15070, 15071, 15073, 15074, 15075, 15078, 15079, 15080, 15081, 15083, 15084, 15086, 15088, 15089, 15090, 15091, 15092, 15093, 15094, 15095, 15096, 15097, 15098, 15099, 15100, 15101, 15103, 15104, 15105, 15106, 15107, 15108, 15110, 15111, 15112, 15113, 15114, 15115, 15117, 15118, 15119, 15120, 15121, 15125, 15126, 15127, 15129, 15130, 15132, 15135, 15137, 15138, 15139, 15140, 15141, 15142, 15143, 15144, 15146, 15147, 15148, 15149, 15150, 15151, 15152, 15153, 15154, 15155, 15156, 15157, 15158, 15159, 15160, 15161, 15162, 15164, 15165, 15167, 15168, 15169, 15170, 15171, 15172, 15173, 15175, 15176, 15177, 15178, 15179, 15180, 15184, 15185, 15186, 15187, 15188, 15189, 15190, 15191, 15192, 15194, 15195, 15196, 15198, 15200, 15201, 15202, 15203, 15204, 15205, 15206, 15207, 15209, 15210, 15211, 15212, 15214, 15215, 15216, 15217, 15218, 15219, 15220, 15221, 15222, 15223, 15224, 15225, 15226, 15227, 15228, 15229, 15231, 15232, 15234, 15235, 15236, 15238, 15240, 15241, 15242, 15243, 15244, 15245, 15246, 15247, 15249, 15250, 15251, 15254, 15255, 15259, 15260, 15261, 15262, 15263, 15264, 15265, 15267, 15268, 15269, 15271, 15272, 15273, 15275, 15276, 15278, 15279, 15280, 15281, 15282, 15283, 15284, 15285, 15287, 15289, 15290, 15291, 15293, 15294, 15295, 15296, 15297, 15298, 15299, 15300, 15301, 15302, 15303, 15304, 15305, 15307, 15308, 15309, 15310, 15311, 15312, 15313, 15314, 15317, 15318, 15319, 15320, 15321, 15323, 15325, 15326, 15327, 15328, 15329, 15330, 15331, 15332, 15334, 15335, 15336, 15337, 15340, 15341, 15342, 15343, 15344, 15345, 15346, 15347, 15348, 15349, 15350, 15351, 15352, 15353, 15354, 15355, 15356, 15358, 15360, 15361, 15364, 15365, 15366, 15369, 15370, 15371, 15372, 15373, 15374, 15375, 15376, 15377, 15378, 15379, 15380, 15381, 15382, 15384, 15386, 15388, 15389, 15390, 15391, 15393, 15394, 15395, 15396, 15397, 15398, 15399, 15400, 15401, 15402, 15403, 15404, 15406, 15407, 15408, 15409, 15410, 15411, 15412, 15413, 15414, 15415, 15417, 15418, 15419, 15420, 15421, 15422, 15423, 15425, 15426, 15427, 15428, 15430, 15431, 15432, 15433, 15434, 15435, 15436, 15437, 15438, 15439, 15440, 15441, 15442, 15444, 15445, 15446, 15447, 15448, 15452, 15453, 15454, 15455, 15456, 15457, 15458, 15459, 15461, 15462, 15463, 15464, 15465, 15466, 15467, 15468, 15469, 15470, 15471, 15473, 15474, 15475, 15476, 15477, 15478, 15480, 15481, 15483, 15484, 15485, 15486, 15487, 15488, 15490, 15491, 15492, 15494, 15495, 15497, 15499, 15500, 15501, 15502, 15503, 15504, 15505, 15506, 15507, 15508, 15510, 15511, 15512, 15513, 15514, 15515, 15516, 15517, 15518, 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, 15527, 15528, 15529, 15530, 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15539, 15540, 15541, 15542, 15543, 15544, 15545, 15546, 15547, 15548, 15549, 15550, 15551, 15553, 15554, 15555, 15556, 15557, 15558, 15560, 15561, 15562, 15563, 15564, 15567, 15569, 15570, 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15579, 15580, 15581, 15582, 15583, 15584, 15585, 15586, 15587, 15588, 15590, 15591, 15592, 15593, 15594, 15595, 15596, 15599, 15600, 15601, 15602, 15603, 15605, 15606, 15607, 15608, 15609, 15610, 15611, 15612, 15613, 15614, 15615, 15616, 15618, 15620, 15621, 15623, 15624, 15625, 15626, 15627, 15629, 15630, 15632, 15633, 15634, 15635, 15637, 15638, 15639, 15640, 15642, 15643, 15644, 15645, 15646, 15647, 15648, 15649, 15650, 15653, 15655, 15656, 15657, 15658, 15659, 15660, 15661, 15662, 15663, 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, 15674, 15675, 15676, 15677, 15678, 15679, 15682, 15683, 15684, 15685, 15689, 15690, 15691, 15692, 15693, 15694, 15695, 15696, 15697, 15698, 15699, 15700, 15701, 15703, 15704, 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, 15716, 15719, 15720, 15721, 15722, 15723, 15724, 15725, 15726, 15734, 15736, 15737, 15740, 15741, 15742, 15743, 15744, 15745, 15747, 15748, 15749, 15750, 15751, 15752, 15753, 15754, 15755, 15757, 15758, 15761, 15763, 15764, 15765, 15766, 15767, 15768, 15769, 15770, 15771, 15772, 15773, 15775, 15776, 15777, 15778, 15779, 15781, 15782, 15783, 15785, 15787, 15788, 15789, 15790, 15791, 15792, 15793, 15794, 15795, 15796, 15797, 15800, 15801, 15804, 15805, 15806, 15807, 15808, 15809, 15810, 15811, 15812, 15813, 15814, 15815, 15816, 15817, 15818, 15819, 15820, 15821, 15822, 15824, 15825, 15827, 15828, 15829, 15830, 15831, 15832, 15833, 15834, 15835, 15836, 15837, 15838, 15839, 15840, 15841, 15842, 15843, 15844, 15846, 15847, 15848, 15849, 15850, 15851, 15852, 15853, 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, 15864, 15865, 15866, 15867, 15868, 15869, 15870, 15871, 15872, 15873, 15874, 15875, 15876, 15877, 15878, 15880, 15881, 15882, 15883, 15885, 15886, 15888, 15889, 15890, 15891, 15892, 15893, 15894, 15895, 15897, 15898, 15900, 15901, 15904, 15905, 15906, 15907, 15908, 15909, 15913, 15914, 15915, 15916, 15917, 15918, 15919, 15921, 15922, 15924, 15925, 15926, 15929, 15931, 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, 15940, 15941, 15942, 15943, 15944, 15945, 15948, 15949, 15950, 15951, 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, 15965, 15966, 15967, 15968, 15969, 15970, 15971, 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, 15981, 15984, 15985, 15986, 15987, 15988, 15989, 15993, 15994, 15995, 15996, 15997, 15998, 15999, 16000, 16001, 16002, 16003, 16004, 16005, 16008, 16009, 16010, 16011, 16014, 16015, 16016, 16017, 16018, 16020, 16021, 16022, 16023, 16024, 16025, 16026, 16028, 16029, 16030, 16031, 16032, 16033, 16034, 16035, 16036, 16040, 16041, 16043, 16044, 16045, 16046, 16047, 16049, 16051, 16053, 16054, 16055, 16056, 16057, 16061, 16062, 16065, 16066, 16067, 16068, 16070, 16071, 16072, 16073, 16074, 16075, 16076, 16077, 16078, 16079, 16080, 16081, 16083, 16084, 16085, 16086, 16087, 16088, 16089, 16090, 16091, 16092, 16093, 16094, 16095, 16096, 16097, 16098, 16099, 16101, 16103, 16104, 16105, 16106, 16107, 16108, 16112, 16113, 16114, 16115, 16116, 16117, 16118, 16120, 16121, 16122, 16123, 16124, 16128, 16130, 16131, 16132, 16134, 16135, 16136, 16137, 16138, 16139, 16140, 16141, 16142, 16143, 16144, 16145, 16146, 16148, 16149, 16150, 16151, 16153, 16154, 16155, 16156, 16157, 16158, 16159, 16160, 16161, 16163, 16164, 16166, 16168, 16169, 16171, 16172, 16173, 16174, 16175, 16176, 16177, 16179, 16180, 16182, 16184, 16185, 16186, 16187, 16189, 16190, 16191, 16192, 16193, 16195, 16197, 16198, 16199, 16200, 16201, 16202, 16203, 16204, 16205, 16206, 16208, 16209, 16211, 16213, 16214, 16215, 16216, 16217, 16218, 16219, 16222, 16223, 16224, 16227, 16229, 16230, 16231, 16232, 16233, 16236, 16237, 16238, 16240, 16242, 16243, 16244, 16245, 16246, 16247, 16248, 16249, 16250, 16251, 16252, 16254, 16255, 16256, 16258, 16259, 16261, 16262, 16264, 16265, 16266, 16267, 16268, 16269, 16271, 16273, 16274, 16275, 16276, 16277, 16278, 16279, 16280, 16281, 16282, 16283, 16284, 16285, 16286, 16287, 16288, 16289, 16290, 16291, 16292, 16293, 16294, 16295, 16296, 16297, 16298, 16299, 16300, 16301, 16302, 16303, 16304, 16305, 16308, 16310, 16311, 16312, 16313, 16315, 16316, 16317, 16318, 16319, 16320, 16321, 16323, 16324, 16325, 16327, 16328, 16329, 16330, 16331, 16332, 16334, 16335, 16338, 16339, 16340, 16343, 16344, 16345, 16346, 16347, 16348, 16350, 16351, 16352, 16353, 16354, 16355, 16356, 16357, 16358, 16359, 16363, 16364, 16365, 16366, 16368, 16369, 16370, 16371, 16372, 16374, 16375, 16376, 16377, 16378, 16379, 16382, 16383, 16384, 16385, 16386, 16387, 16389, 16391, 16392, 16393, 16394, 16395, 16397, 16398, 16399, 16400, 16401, 16402, 16403, 16404, 16405, 16407, 16408, 16409, 16410, 16411, 16412, 16413, 16414, 16415, 16417, 16418, 16419, 16420, 16421, 16422, 16423, 16424, 16425, 16426, 16427, 16428, 16431, 16432, 16433, 16434, 16435, 16439, 16440, 16441, 16442, 16445, 16446, 16447, 16448, 16449, 16450, 16451, 16453, 16455, 16456, 16457, 16458, 16459, 16460, 16461, 16462, 16463, 16464, 16465, 16466, 16467, 16469, 16470, 16471, 16473, 16474, 16475, 16476, 16477, 16478, 16479, 16480, 16481, 16482, 16483, 16484, 16485, 16486, 16487, 16488, 16489, 16490, 16491, 16492, 16493, 16494, 16496, 16497, 16498, 16499, 16500, 16503, 16504, 16505, 16506, 16508, 16509, 16510, 16511, 16512, 16513, 16514, 16515, 16516, 16517, 16518, 16519, 16520, 16521, 16522, 16524, 16525, 16526, 16527, 16528, 16529, 16530, 16532, 16533, 16534, 16535, 16536, 16537, 16538, 16539, 16541, 16542, 16544, 16545, 16546, 16547, 16548, 16549, 16551, 16552, 16553, 16555, 16557, 16558, 16559, 16562, 16563, 16565, 16566, 16568, 16569, 16570, 16571, 16572, 16573, 16574, 16577, 16578, 16579, 16580, 16581, 16582, 16583, 16585, 16587, 16588, 16589, 16590, 16591, 16592, 16593, 16595, 16597, 16598, 16599, 16600, 16601, 16602, 16603, 16605, 16606, 16607, 16608, 16610, 16611, 16612, 16613, 16614, 16615, 16616, 16617, 16620, 16621, 16622, 16623, 16624, 16625, 16626, 16627, 16628, 16629, 16630, 16631, 16632, 16633, 16634, 16635, 16636, 16637, 16639, 16640, 16641, 16642, 16643, 16646, 16647, 16648, 16650, 16651, 16652, 16655, 16657, 16658, 16659, 16660, 16662, 16663, 16664, 16665, 16666, 16667, 16668, 16669, 16670, 16671, 16672, 16673, 16674, 16677, 16678, 16680, 16683, 16684, 16685, 16686, 16687, 16688, 16689, 16690, 16691, 16692, 16693, 16694, 16697, 16698, 16699, 16700, 16701, 16705, 16706, 16707, 16712, 16717, 16718, 16719, 16721, 16723, 16724, 16725, 16726, 16727, 16729, 16730, 16731, 16732, 16733, 16734, 16735, 16736, 16737, 16738, 16739, 16743, 16744, 16745, 16746, 16748, 16750, 16751, 16752, 16753, 16754, 16757, 16758, 16759, 16761, 16762, 16764, 16765, 16766, 16767, 16768, 16769, 16770, 16771, 16772, 16773, 16774, 16775, 16776, 16778, 16779, 16782, 16783, 16786, 16787, 16788, 16792, 16793, 16794, 16795, 16796, 16797, 16798, 16802, 16803, 16804, 16805, 16806, 16807, 16808, 16809, 16810, 16811, 16812, 16814, 16815, 16817, 16818, 16820, 16821, 16823, 16825, 16826, 16827, 16828, 16829, 16830, 16831, 16836, 16838, 16839, 16840, 16841, 16842, 16843, 16844, 16845, 16846, 16848, 16849, 16850, 16851, 16852, 16853, 16854, 16855, 16856, 16857, 16858, 16859, 16860, 16861, 16862, 16863, 16866, 16868, 16872, 16873, 16875, 16877, 16878, 16879, 16880, 16881, 16882, 16883, 16885, 16887, 16888, 16889, 16890, 16891, 16892, 16893, 16894, 16896, 16897, 16898, 16899, 16900, 16901, 16902, 16906, 16907, 16910, 16913, 16916, 16917, 16919, 16920, 16921, 16922, 16924, 16925, 16926, 16927, 16928, 16929, 16930, 16931, 16932, 16933, 16936, 16937, 16938, 16940, 16941, 16942, 16943, 16944, 16945, 16946, 16947, 16948, 16950, 16952, 16954, 16955, 16956, 16958, 16959, 16960, 16961, 16962, 16963, 16964, 16967, 16968, 16969, 16971, 16973, 16974, 16979, 16980, 16981, 16982, 16983, 16984, 16985, 16987, 16988, 16989, 16990, 16992, 16993, 16995, 16996, 16997, 16998, 17000, 17001, 17002, 17003, 17004, 17005, 17006, 17007, 17008, 17009, 17011, 17012, 17014, 17015, 17016, 17017, 17018, 17019, 17020, 17021, 17022, 17023, 17024, 17025, 17026, 17027, 17028, 17029, 17030, 17031, 17032, 17033, 17034, 17035, 17036, 17037, 17038, 17039, 17040, 17041, 17044, 17045, 17046, 17048, 17049, 17050, 17051, 17052, 17053, 17055, 17058, 17059, 17060, 17061, 17062, 17065, 17066, 17067, 17068, 17069, 17070, 17071, 17072, 17073, 17075, 17076, 17078, 17079, 17080, 17081, 17082, 17085, 17086, 17091, 17092, 17094, 17095, 17096, 17098, 17099, 17100, 17101, 17102, 17103, 17104, 17105, 17108, 17112, 17113, 17114, 17116, 17117, 17119, 17120, 17121, 17122, 17123, 17124, 17125, 17126, 17127, 17128, 17129, 17130, 17131, 17132, 17133, 17134, 17135, 17136, 17137, 17139, 17140, 17141, 17142, 17144, 17148, 17149, 17150, 17151, 17152, 17154, 17155, 17156, 17158, 17161, 17162, 17164, 17165, 17166, 17167, 17170, 17171, 17172, 17173, 17174, 17175, 17176, 17177, 17178, 17179, 17180, 17181, 17182, 17183, 17184, 17187, 17188, 17189, 17190, 17191, 17192, 17193, 17194, 17195, 17196, 17197, 17202, 17203, 17204, 17205, 17206, 17207, 17208, 17209, 17210, 17211, 17212, 17213, 17214, 17215, 17218, 17219, 17220, 17221, 17222, 17223, 17224, 17225, 17227, 17228, 17230, 17231, 17233, 17234, 17236, 17237, 17238, 17239, 17240, 17241, 17244, 17246, 17247, 17248, 17249, 17250, 17251, 17253, 17254, 17255, 17257, 17258, 17259, 17260, 17261, 17262, 17263, 17264, 17265, 17267, 17268, 17269, 17271, 17272, 17273, 17274, 17275, 17276, 17277, 17278, 17279, 17280, 17281, 17282, 17283, 17284, 17291, 17292, 17296, 17298, 17299, 17300, 17305, 17306, 17307, 17308, 17309, 17310, 17311, 17312, 17313, 17314, 17320, 17321, 17322, 17323, 17324, 17325, 17326, 17329, 17330, 17332, 17334, 17335, 17337, 17338, 17339, 17343, 17344, 17345, 17346, 17348, 17350, 17351, 17352, 17353, 17355, 17356, 17357, 17358, 17359, 17360, 17361, 17365, 17366, 17369, 17370, 17371, 17373, 17374, 17376, 17378, 17381, 17382, 17384, 17385, 17386, 17387, 17388, 17389, 17390, 17391, 17392, 17393, 17394, 17398, 17399, 17400, 17401, 17402, 17403, 17404, 17405, 17406, 17407, 17408, 17409, 17410, 17412, 17413, 17414, 17415, 17416, 17417, 17418, 17419, 17420, 17421, 17422, 17423, 17424, 17425, 17426, 17427, 17430, 17431, 17432, 17433, 17438, 17439, 17441, 17442, 17443, 17444, 17445, 17446, 17447, 17448, 17449, 17450, 17451, 17452, 17453, 17456, 17457, 17458, 17459, 17460, 17461, 17462, 17463, 17464, 17468, 17469, 17474, 17475, 17476, 17477, 17478, 17480, 17481, 17482, 17483, 17484, 17485, 17487, 17491, 17492, 17494, 17495, 17498, 17499, 17500, 17501, 17502, 17503, 17504, 17505, 17506, 17507, 17508, 17511, 17512, 17513, 17514, 17515, 17516, 17517, 17518, 17519, 17520, 17521, 17522, 17523, 17524, 17525, 17526, 17527, 17528, 17530, 17531, 17532, 17533, 17534, 17535, 17536, 17537, 17538, 17539, 17541, 17542, 17543, 17544, 17545, 17546, 17547, 17548, 17549, 17551, 17552, 17553, 17554, 17556, 17557, 17558, 17559, 17560, 17562, 17566, 17567, 17568, 17569, 17570, 17571, 17572, 17573, 17575, 17577, 17578, 17583, 17584, 17585, 17586, 17587, 17589, 17592, 17593, 17594, 17595, 17597, 17598, 17599, 17602, 17604, 17606, 17607, 17608, 17609, 17610, 17611, 17612, 17614, 17615, 17617, 17618, 17619, 17620, 17621, 17622, 17623, 17627, 17629, 17630, 17631, 17632, 17636, 17637, 17638, 17639, 17640, 17641, 17642, 17646, 17648, 17649, 17650, 17651, 17652, 17653, 17654, 17657, 17658, 17662, 17664, 17665, 17666, 17667, 17668, 17669, 17671, 17672, 17674, 17675, 17677, 17678, 17679, 17680, 17681, 17682, 17683, 17684, 17685, 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17697, 17698, 17699, 17700, 17701, 17702, 17703, 17704, 17707, 17708, 17709, 17710, 17713, 17714, 17715, 17716, 17718, 17719, 17720, 17722, 17723, 17724, 17725, 17726, 17727, 17729, 17730, 17731, 17732, 17733, 17736, 17738, 17739, 17740, 17741, 17744, 17745, 17746, 17747, 17748, 17749, 17750, 17751, 17752, 17753, 17754, 17757, 17758, 17760, 17761, 17762, 17763, 17765, 17766, 17767, 17769, 17772, 17773, 17774, 17775, 17779, 17780, 17781, 17782, 17783, 17785, 17786, 17788, 17789, 17790, 17791, 17792, 17793, 17795, 17796, 17797, 17798, 17800, 17801, 17803, 17804, 17805, 17806, 17807, 17808, 17809, 17810, 17811, 17812, 17813, 17815, 17816, 17817, 17818, 17819, 17820, 17824, 17825, 17826, 17827, 17828, 17829, 17830, 17831, 17832, 17836, 17838, 17839, 17841, 17842, 17843, 17844, 17845, 17847, 17848, 17849, 17850, 17851, 17852, 17853, 17854, 17855, 17856, 17857, 17858, 17859, 17860, 17861, 17862, 17864, 17866, 17867, 17868, 17870, 17871, 17872, 17873, 17874, 17875, 17876, 17877, 17878, 17879, 17888, 17889, 17891, 17892, 17895, 17899, 17901, 17902, 17904, 17905, 17906, 17907, 17908, 17912, 17914, 17916, 17917, 17918, 17919, 17920, 17921, 17923, 17924, 17925, 17926, 17927, 17928, 17929, 17930, 17931, 17932, 17933, 17934, 17935, 17936, 17937, 17938, 17939, 17947, 17949, 17950, 17951, 17954, 17955, 17957, 17958, 17959, 17961, 17962, 17963, 17964, 17965, 17966, 17968, 17969, 17970, 17972, 17974, 17975, 17978, 17979, 17983, 17984, 17985, 17986, 17987, 17988, 17990, 17991, 17992, 17996, 17998, 18000, 18002, 18003, 18004, 18008, 18011, 18012, 18017, 18019, 18020, 18021, 18022, 18023, 18024, 18026, 18027, 18029, 18032, 18035, 18036, 18037, 18038, 18039, 18040, 18041, 18042, 18043, 18044, 18047, 18048, 18049, 18051, 18052, 18053, 18054, 18055, 18056, 18057, 18058, 18059, 18060, 18062, 18065, 18066, 18067, 18068, 18069, 18072, 18073, 18076, 18078, 18080, 18081, 18082, 18083, 18084, 18085, 18086, 18087, 18089, 18090, 18091, 18092, 18093, 18094, 18095, 18096, 18097, 18098, 18099, 18100, 18101, 18102, 18103, 18104, 18105, 18106, 18107, 18109, 18110, 18119, 18123, 18124, 18125, 18128, 18129, 18130, 18131, 18134, 18135, 18137, 18138, 18139, 18141, 18142, 18143, 18144, 18145, 18147, 18148, 18149, 18150, 18151, 18154, 18156, 18157, 18158, 18159, 18160, 18161, 18162, 18163, 18164, 18165, 18166, 18170, 18171, 18172, 18173, 18174, 18175, 18176, 18178, 18179, 18180, 18181, 18182, 18183, 18184, 18188, 18189, 18190, 18191, 18192, 18193, 18194, 18195, 18197, 18198, 18201, 18202, 18203, 18205, 18206, 18207, 18208, 18209, 18210, 18211, 18212, 18213, 18215, 18216, 18217, 18218, 18219, 18220, 18221, 18222, 18223, 18224, 18227, 18229, 18230, 18231, 18232, 18233, 18234, 18235, 18236, 18237, 18238, 18239, 18241, 18243, 18244, 18245, 18246, 18247, 18248, 18249, 18250, 18251, 18252, 18253, 18254, 18255, 18256, 18257, 18259, 18260, 18261, 18262, 18263, 18264, 18265, 18266, 18267, 18268, 18269, 18271, 18272, 18273, 18274, 18275, 18276, 18280, 18281, 18282, 18283, 18284, 18285, 18287, 18289, 18290, 18292, 18293, 18294, 18295, 18296, 18297, 18298, 18299, 18300, 18302, 18303, 18304, 18305, 18306, 18315, 18316, 18317, 18318, 18319, 18320, 18321, 18328, 18329, 18331, 18332, 18335, 18338, 18339, 18340, 18341, 18343, 18344, 18345, 18346, 18347, 18348, 18349, 18350, 18351, 18352, 18353, 18354, 18355, 18356, 18357, 18360, 18361, 18362, 18363, 18364, 18365, 18366, 18367, 18368, 18369, 18370, 18371, 18373, 18374, 18378, 18379, 18380, 18381, 18382, 18383, 18384, 18385, 18386, 18387, 18388, 18391, 18395, 18397, 18398, 18399, 18400, 18403, 18408, 18411, 18412, 18414, 18416, 18417, 18419, 18420, 18421, 18422, 18424, 18425, 18426, 18427, 18434, 18435, 18437, 18438, 18439, 18441, 18442, 18443, 18444, 18445, 18446, 18447, 18452, 18453, 18454, 18456, 18457, 18458, 18459, 18460, 18461, 18465, 18466, 18467, 18468, 18469, 18470, 18473, 18474, 18475, 18476, 18477, 18480, 18481, 18482, 18483, 18484, 18485, 18486, 18489, 18490, 18491, 18492, 18493, 18495, 18498, 18500, 18501, 18506, 18507, 18508, 18509, 18510, 18511, 18513, 18516, 18517, 18518, 18519, 18520, 18521, 18525, 18527, 18528, 18529, 18531, 18532, 18534, 18535, 18536, 18538, 18539, 18540, 18541, 18542, 18543, 18544, 18545, 18546, 18547, 18548, 18549, 18550, 18551, 18552, 18553, 18555, 18556, 18557, 18558, 18559, 18560, 18561, 18566, 18568, 18569, 18570, 18571, 18572, 18573, 18574, 18575, 18576, 18577, 18578, 18579, 18582, 18584, 18586, 18587, 18588, 18593, 18594, 18597, 18598, 18600, 18601, 18602, 18603, 18605, 18607, 18608, 18610, 18612, 18613, 18614, 18619, 18620, 18625, 18626, 18630, 18631, 18632, 18635, 18637, 18640, 18641, 18645, 18646, 18648, 18650, 18653, 18654, 18657, 18659, 18660, 18662, 18663, 18664, 18665, 18667, 18668, 18669, 18670, 18671, 18672, 18673, 18674, 18675, 18677, 18678, 18679, 18680, 18681, 18682, 18683, 18684, 18685, 18686, 18687, 18688, 18689, 18691, 18692, 18693, 18695, 18696, 18697, 18698, 18699, 18700, 18701, 18702, 18703, 18704, 18705, 18710, 18711, 18712, 18713, 18718, 18719, 18720, 18721, 18723, 18724, 18725, 18726, 18727, 18731, 18732, 18735, 18736, 18737, 18738, 18739, 18740, 18741, 18742, 18743, 18745, 18746, 18747, 18748, 18749, 18750, 18755, 18756, 18757, 18758, 18759, 18760, 18762, 18764, 18766, 18767, 18768, 18769, 18770, 18771, 18772, 18774, 18775, 18777, 18778, 18782, 18783, 18785, 18789, 18792, 18793, 18795, 18796, 18797, 18798, 18799, 18800, 18801, 18807, 18808, 18809, 18810, 18811, 18812, 18813, 18816, 18817, 18818, 18820, 18821, 18822, 18823, 18824, 18825, 18826, 18827, 18830, 18832, 18833, 18834, 18835, 18840, 18841, 18842, 18843, 18844, 18846, 18847, 18848, 18849, 18850, 18851, 18852, 18853, 18857, 18858, 18859, 18860, 18861, 18862, 18864, 18865, 18866, 18868, 18869, 18870, 18871, 18872, 18877, 18878, 18879, 18880, 18881, 18882, 18883, 18884, 18885, 18886, 18887, 18888, 18889, 18890, 18891, 18892, 18893, 18899, 18900, 18901, 18904, 18905, 18907, 18908, 18910, 18911, 18912, 18913, 18914, 18917, 18918, 18919, 18922, 18924, 18926, 18927, 18928, 18929, 18930, 18931, 18933, 18935, 18936, 18937, 18938, 18939, 18941, 18942, 18945, 18946, 18949, 18950, 18951, 18952, 18953, 18954, 18955, 18957, 18958, 18959, 18960, 18961, 18963, 18964, 18965, 18966, 18967, 18968, 18969, 18970, 18971, 18972, 18973, 18974, 18975, 18976, 18977, 18978, 18979, 18980, 18981, 18982, 18983, 18984, 18985, 18990, 18992, 18993, 18994, 18996, 19000, 19001, 19002, 19003, 19008, 19010, 19012, 19013, 19015, 19016, 19018, 19019, 19021, 19022, 19023, 19025, 19027, 19028, 19029, 19031, 19032, 19033, 19034, 19035, 19036, 19037, 19043, 19044, 19045, 19047, 19048, 19050, 19051, 19052, 19053, 19054, 19055, 19056, 19057, 19058, 19059, 19060, 19061, 19062, 19064, 19065, 19066, 19071, 19073, 19074, 19075, 19076, 19077, 19078, 19082, 19083, 19084, 19085, 19086, 19087, 19088, 19089, 19090, 19092, 19098, 19099, 19100, 19101, 19106, 19110, 19112, 19113, 19114, 19117, 19119, 19120, 19121, 19123, 19124, 19125, 19126, 19129, 19130, 19133, 19134, 19135, 19139, 19141, 19142, 19144, 19145, 19148, 19149, 19151, 19152, 19153, 19154, 19160, 19162, 19165, 19167, 19168, 19170, 19171, 19172, 19173, 19174, 19175, 19176, 19177, 19178, 19179, 19180, 19181, 19182, 19183, 19186, 19187, 19188, 19189, 19190, 19191, 19192, 19193, 19194, 19196, 19197, 19198, 19200, 19201, 19202, 19203, 19204, 19206, 19209, 19210, 19215, 19216, 19217, 19219, 19220, 19221, 19222, 19223, 19224, 19225, 19226, 19227, 19228, 19230, 19231, 19232, 19233, 19234, 19235, 19236, 19238, 19239, 19241, 19242, 19243, 19244, 19246, 19247, 19248, 19249, 19251, 19252, 19253, 19254, 19255, 19256, 19258, 19260, 19261, 19263, 19265, 19267, 19268, 19269, 19270, 19272, 19273, 19274, 19275, 19276, 19277, 19278, 19279, 19280, 19281, 19282, 19285, 19286, 19287, 19288, 19289, 19290, 19291, 19292, 19293, 19294, 19295, 19296, 19298, 19299, 19300, 19301, 19314, 19317, 19318, 19319, 19322, 19324, 19325, 19326, 19327, 19328, 19329, 19330, 19331, 19336, 19337, 19343, 19344, 19345, 19347, 19348, 19349, 19350, 19351, 19352, 19354, 19355, 19356, 19357, 19358, 19359, 19360, 19361, 19362, 19363, 19364, 19365, 19370, 19371, 19373, 19378, 19380, 19381, 19382, 19383, 19384, 19385, 19389, 19390, 19394, 19395, 19396, 19397, 19398, 19405, 19407, 19409, 19411, 19412, 19418, 19419, 19420, 19421, 19424, 19426, 19427, 19428, 19429, 19430, 19433, 19435, 19436, 19437, 19438, 19439, 19440, 19441, 19442, 19443, 19444, 19445, 19446, 19448, 19449, 19451, 19452, 19453, 19454, 19455, 19456, 19457, 19461, 19462, 19463, 19466, 19468, 19469, 19470, 19471, 19478, 19479, 19480, 19482, 19483, 19484, 19485, 19486, 19488, 19489, 19490, 19491, 19492, 19493, 19494, 19495, 19497, 19498, 19499, 19500, 19501, 19502, 19503, 19509, 19510, 19511, 19513, 19514, 19515, 19517, 19518, 19520, 19521, 19522, 19523, 19524, 19525, 19526, 19527, 19528, 19529, 19530, 19531, 19532, 19536, 19537, 19538, 19539, 19540, 19541, 19546, 19547, 19548, 19550, 19552, 19554, 19555, 19556, 19557, 19558, 19559, 19560, 19561, 19565, 19567, 19569, 19570, 19571, 19572, 19573, 19574, 19575, 19576, 19577, 19578, 19579, 19580, 19581, 19582, 19586, 19587, 19588, 19589, 19590, 19591, 19592, 19593, 19594, 19596, 19597, 19598, 19599, 19600, 19601, 19602, 19603, 19604, 19605, 19606, 19608, 19610, 19611, 19612, 19613, 19614, 19615, 19616, 19617, 19618, 19619, 19620, 19621, 19622, 19626, 19627, 19628, 19629, 19631, 19632, 19633, 19634, 19635, 19636, 19637, 19638, 19639, 19640, 19641, 19642, 19644, 19645, 19646, 19647, 19650, 19654, 19659, 19660, 19661, 19663, 19666, 19668, 19669, 19670, 19671, 19672, 19673, 19675, 19676, 19677, 19678, 19679, 19681, 19682, 19683, 19684, 19685, 19686, 19687, 19690, 19693, 19694, 19695, 19696, 19697, 19698, 19699, 19700, 19701, 19704, 19705, 19708, 19709, 19713, 19718, 19719, 19721, 19722, 19723, 19724, 19725, 19726, 19728, 19731, 19733, 19734, 19735, 19736, 19737, 19738, 19740, 19741, 19742, 19743, 19744, 19745, 19747, 19749, 19750, 19751, 19752, 19753, 19754, 19755, 19756, 19757, 19758, 19759, 19760, 19762, 19763, 19764, 19769, 19771, 19772, 19773, 19776, 19777, 19778, 19782, 19785, 19787, 19788, 19789, 19790, 19791, 19792, 19794, 19795, 19796, 19797, 19798, 19799, 19801, 19802, 19803, 19805, 19807, 19809, 19810, 19811, 19812, 19813, 19814, 19815, 19816, 19830, 19832, 19833, 19834, 19838, 19839, 19840, 19841, 19842, 19843, 19845, 19846, 19850, 19854, 19855, 19856, 19857, 19858, 19859, 19860, 19861, 19862, 19863, 19867, 19868, 19869, 19871, 19872, 19873, 19874, 19875, 19878, 19879, 19880, 19881, 19882, 19883, 19885, 19886, 19887, 19888, 19889, 19890, 19891, 19892, 19893, 19894, 19896, 19897, 19900, 19901, 19902, 19903, 19904, 19905, 19906, 19907, 19908, 19909, 19918, 19919, 19921, 19922, 19923, 19924, 19926, 19927, 19930, 19931, 19932, 19934, 19935, 19936, 19937, 19941, 19944, 19945, 19946, 19947, 19948, 19949, 19950, 19952, 19953, 19954, 19955, 19956, 19957, 19958, 19959, 19960, 19962, 19963, 19964, 19965, 19966, 19967, 19968, 19969, 19970, 19971, 19972, 19973, 19974, 19975, 19976, 19977, 19980, 19981, 19985, 19986, 19987, 19992, 19993, 19996, 19997, 19998, 19999, 20000, 20001, 20002, 20003, 20004, 20005, 20006, 20007, 20009, 20010, 20011, 20012, 20013, 20015, 20016, 20017, 20019, 20020, 20021, 20022, 20023, 20024, 20027, 20028, 20029, 20030, 20032, 20033, 20034, 20035, 20036, 20037, 20038, 20039, 20044, 20045, 20046, 20047, 20048, 20049, 20050, 20051, 20052, 20053, 20054, 20055, 20056, 20060, 20061, 20065, 20066, 20067, 20077, 20078, 20079, 20080, 20081, 20082, 20083, 20085, 20088, 20089, 20092, 20093, 20095, 20097, 20098, 20099, 20100, 20101, 20103, 20104, 20105, 20106, 20107, 20109, 20110, 20112, 20113, 20117, 20120, 20121, 20122, 20124, 20125, 20126, 20128, 20129, 20130, 20131, 20132, 20133, 20135, 20136, 20137, 20139, 20140, 20141, 20142, 20143, 20144, 20145, 20146, 20147, 20148, 20149, 20150, 20151, 20152, 20155, 20156, 20160, 20161, 20162, 20165, 20168, 20170, 20171, 20172, 20173, 20174, 20175, 20176, 20177, 20178, 20179, 20181, 20182, 20183, 20184, 20185, 20189, 20190, 20191, 20192, 20193, 20194, 20195, 20196, 20197, 20198, 20199, 20200, 20202, 20210, 20214, 20215, 20216, 20217, 20218, 20219, 20220, 20221, 20226, 20231, 20232, 20233, 20235, 20243, 20244, 20245, 20252, 20253, 20257, 20258, 20261, 20262, 20263, 20264, 20268, 20269, 20273, 20274, 20275, 20276, 20277, 20278, 20279, 20280, 20281, 20282, 20283, 20284, 20285, 20286, 20287, 20288, 20289, 20290, 20291, 20292, 20295, 20297, 20298, 20299, 20300, 20301, 20302, 20303, 20304, 20305, 20306, 20307, 20308, 20309, 20310, 20311, 20312, 20313, 20314, 20315, 20316, 20320, 20321, 20322, 20323, 20324, 20325, 20329, 20330, 20333, 20334, 20335, 20336, 20337, 20338, 20340, 20342, 20343, 20344, 20345, 20347, 20348, 20350, 20351, 20352, 20353, 20356, 20358, 20359, 20360, 20361, 20362, 20363, 20364, 20365, 20366, 20367, 20368, 20369, 20370, 20371, 20372, 20373, 20374, 20375, 20376, 20377, 20381, 20382, 20383, 20384, 20385, 20386, 20387, 20388, 20389, 20390, 20391, 20392, 20393, 20403, 20404, 20408, 20409, 20411, 20412, 20413, 20414, 20417, 20422, 20423, 20424, 20425, 20426, 20427, 20428, 20429, 20430, 20431, 20435, 20436, 20437, 20438, 20439, 20440, 20441, 20442, 20443, 20444, 20445, 20446, 20448, 20449, 20450, 20451, 20452, 20453, 20454, 20455, 20460, 20461, 20462, 20463, 20464, 20465, 20466, 20467, 20468, 20469, 20479, 20481, 20482, 20483, 20484, 20487, 20488, 20490, 20492, 20493, 20494, 20495, 20496, 20497, 20498, 20499, 20500, 20501, 20502, 20503, 20504, 20509, 20510, 20512, 20513, 20514, 20515, 20516, 20518, 20519, 20520, 20521, 20522, 20523, 20524, 20525, 20526, 20527, 20528, 20529, 20535, 20536, 20537, 20538, 20539, 20541, 20543, 20546, 20547, 20550, 20551, 20554, 20555, 20556, 20557, 20558, 20559, 20560, 20561, 20562, 20563, 20564, 20565, 20566, 20571, 20572, 20575, 20579, 20581, 20582, 20583, 20584, 20585, 20591, 20592, 20593, 20594, 20595, 20596, 20597, 20598, 20599, 20600, 20601, 20602, 20603, 20605, 20607, 20608, 20609, 20610, 20611, 20612, 20613, 20614, 20615, 20616, 20624, 20625, 20626, 20629, 20631, 20632, 20634, 20635, 20636, 20637, 20638, 20639, 20640, 20641, 20643, 20644, 20646, 20647, 20648, 20649, 20656, 20658, 20660, 20661, 20662, 20664, 20665, 20668, 20669, 20670, 20671, 20672, 20674, 20675, 20676, 20677, 20678, 20679, 20680, 20681, 20685, 20686, 20687, 20688, 20689, 20696, 20697, 20700, 20701, 20702, 20704, 20705, 20706, 20707, 20708, 20709, 20710, 20711, 20715, 20716, 20717, 20718, 20722, 20723, 20724, 20725, 20726, 20727, 20728, 20729, 20731, 20732, 20733, 20734, 20735, 20736, 20737, 20739, 20740, 20741, 20742, 20743, 20744, 20745, 20746, 20747, 20748, 20749, 20750, 20751, 20752, 20753, 20754, 20755, 20756, 20757, 20759, 20760, 20761, 20764, 20765, 20766, 20767, 20769, 20774, 20776, 20778, 20779, 20780, 20781, 20782, 20783, 20784, 20785, 20786, 20787, 20788, 20789, 20793, 20794, 20795, 20796, 20797, 20798, 20801, 20802, 20803, 20804, 20806, 20807, 20808, 20809, 20810, 20811, 20812, 20813, 20814, 20815, 20816, 20817, 20818, 20819, 20822, 20823, 20824, 20825, 20826, 20827, 20833, 20834, 20835, 20836, 20837, 20838, 20839, 20840, 20841, 20843, 20844, 20845, 20846, 20847, 20848, 20849, 20853, 20855, 20857, 20858, 20861, 20862, 20863, 20864, 20865, 20866, 20867, 20868, 20869, 20870, 20871, 20872, 20873, 20874, 20878, 20881, 20882, 20883, 20886, 20887, 20888, 20893, 20895, 20896, 20897, 20898, 20899, 20900, 20901, 20902, 20903, 20904, 20905, 20906, 20907, 20914, 20915, 20917, 20918, 20919, 20920, 20921, 20922, 20923, 20924, 20925, 20926, 20927, 20928, 20929, 20930, 20931, 20932, 20939, 20943, 20944, 20945, 20947, 20948, 20949, 20950, 20951, 20952, 20954, 20956, 20957, 20958, 20959, 20960, 20961, 20962, 20964, 20965, 20966, 20977, 20979, 20980, 20981, 20982, 20987, 20990, 20992, 20993, 20994, 20999, 21000, 21001, 21002, 21012, 21013, 21017, 21018, 21021, 21023, 21027, 21028, 21031, 21032, 21033, 21034, 21035, 21037, 21038, 21041, 21042, 21043, 21044, 21048, 21051, 21053, 21054, 21055, 21056, 21058, 21059, 21060, 21062, 21063, 21064, 21067, 21069, 21070, 21071, 21072, 21073, 21074, 21075, 21076, 21077, 21078, 21079, 21080, 21081, 21082, 21083, 21085, 21086, 21087, 21091, 21093, 21094, 21097, 21098, 21099, 21100, 21101, 21102, 21103, 21108, 21114, 21115, 21116, 21117, 21118, 21120, 21121, 21122, 21123, 21124, 21125, 21126, 21127, 21128, 21129, 21130, 21131, 21132, 21133, 21134, 21135, 21136, 21137, 21138, 21139, 21142, 21143, 21144, 21145, 21146, 21147, 21158, 21160, 21163, 21168, 21169, 21174, 21177, 21178, 21179, 21180, 21181, 21182, 21183, 21185, 21186, 21188, 21189, 21191, 21192, 21193, 21194, 21197, 21198, 21199, 21202, 21203, 21206, 21207, 21208, 21211, 21212, 21213, 21214, 21215, 21216, 21217, 21218, 21219, 21220, 21221, 21227, 21228, 21229, 21231, 21232, 21233, 21234, 21235, 21236, 21237, 21238, 21239, 21240, 21241, 21242, 21243, 21244, 21253, 21254, 21255, 21256, 21259, 21260, 21261, 21262, 21270, 21271, 21272, 21273, 21274, 21277, 21279, 21281, 21284, 21285, 21292, 21293, 21296, 21297, 21301, 21302, 21303, 21304, 21305, 21306, 21308, 21309, 21310, 21311, 21312, 21313, 21317, 21320, 21323, 21324, 21325, 21326, 21329, 21330, 21331, 21332, 21333, 21337, 21338, 21340, 21341, 21342, 21344, 21345, 21347, 21348, 21349, 21350, 21351, 21352, 21353, 21356, 21357, 21359, 21360, 21361, 21362, 21363, 21364, 21367, 21369, 21370, 21371, 21372, 21374, 21375, 21376, 21377, 21378, 21379, 21380, 21381, 21382, 21383, 21384, 21385, 21389, 21390, 21391, 21392, 21393, 21394, 21395, 21399, 21400, 21401, 21402, 21406, 21411, 21412, 21415, 21416, 21417, 21418, 21419, 21420, 21421, 21422, 21423, 21424, 21426, 21427, 21428, 21429, 21430, 21431, 21433, 21434, 21435, 21437, 21439, 21440, 21441, 21442, 21443, 21444, 21445, 21447, 21448, 21449, 21450, 21451, 21452, 21453, 21454, 21455, 21456, 21457, 21458, 21463, 21469, 21470, 21471, 21472, 21473, 21476, 21478, 21479, 21481, 21482, 21487, 21488, 21489, 21490, 21491, 21492, 21493, 21494, 21495, 21496, 21497, 21498, 21499, 21500, 21501, 21502, 21503, 21504, 21505, 21509, 21510, 21511, 21513, 21516, 21517, 21518, 21519, 21520, 21521, 21522, 21523, 21525, 21526, 21527, 21528, 21529, 21530, 21531, 21532, 21533, 21534, 21535, 21537, 21538, 21540, 21541, 21542, 21543, 21544, 21545, 21546, 21547, 21548, 21550, 21551, 21552, 21553, 21554, 21561, 21562, 21564, 21567, 21570, 21571, 21573, 21574, 21575, 21577, 21578, 21580, 21581, 21582, 21583, 21584, 21585, 21586, 21587, 21588, 21589, 21590, 21591, 21592, 21595, 21597, 21598, 21603, 21604, 21605, 21606, 21607, 21608, 21609, 21610, 21611, 21612, 21613, 21615, 21616, 21617, 21623, 21625, 21627, 21628, 21630, 21631, 21633, 21634, 21635, 21636, 21638, 21639, 21640, 21642, 21643, 21644, 21645, 21646, 21647, 21648, 21649, 21650, 21651, 21652, 21653, 21654, 21655, 21656, 21660, 21661, 21662, 21663, 21664, 21665, 21666, 21667, 21668, 21674, 21677, 21678, 21682, 21687, 21688, 21689, 21691, 21692, 21693, 21694, 21695, 21697, 21699, 21700, 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, 21709, 21710, 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, 21719, 21720, 21721, 21722, 21723, 21724, 21726, 21727, 21728, 21729, 21730, 21731, 21732, 21733, 21748, 21749, 21751, 21753, 21754, 21756, 21757, 21758, 21759, 21760, 21764, 21765, 21766, 21767, 21768, 21769, 21775, 21779, 21780, 21781, 21782, 21783, 21784, 21785, 21786, 21787, 21788, 21789, 21790, 21791, 21796, 21797, 21798, 21799, 21800, 21801, 21802, 21803, 21804, 21805, 21806, 21807, 21808, 21809, 21810, 21812, 21813, 21814, 21815, 21816, 21817, 21818, 21819, 21820, 21821, 21822, 21823, 21826, 21827, 21828, 21829, 21830, 21831, 21832, 21833, 21834, 21835, 21836, 21840, 21845, 21846, 21847, 21851, 21855, 21859, 21860, 21861, 21862, 21865, 21868, 21869, 21870, 21871, 21872, 21873, 21874, 21875, 21876, 21877, 21878, 21879, 21883, 21891, 21892, 21893, 21901, 21907, 21909, 21910, 21911, 21912, 21913, 21916, 21917, 21918, 21919, 21920, 21921, 21922, 21933, 21934, 21935, 21936, 21937, 21938, 21939, 21943, 21944, 21949, 21951, 21952, 21957, 21960, 21961, 21963, 21967, 21968, 21969, 21971, 21972, 21977, 21978, 21981, 21984, 21985, 21986, 21987, 21988, 21989, 21990, 21992, 21993, 21994, 21995, 21998, 22002, 22003, 22004, 22005, 22006, 22034, 22040, 22041, 22042, 22043, 22044, 22045, 22055, 22063, 22070, 22073, 22074, 22075, 22076, 22081, 22082, 22083, 22084, 22085, 22090, 22098, 22100, 22102, 22103, 22104, 22105, 22108, 22109, 22113, 22114, 22115, 22116, 22117, 22118, 22119, 22120, 22123, 22142, 22147, 22148, 22149, 22153, 22155, 22158, 22159, 22162, 22163, 22164, 22165, 22167, 22168, 22169, 22170, 22171, 22172, 22177, 22191, 22194, 22195, 22196, 22199, 22206, 22207, 22208, 22209, 22210, 22211, 22212, 22213, 22214, 22215, 22216, 22217, 22221, 22222, 22230, 22231, 22233, 22244, 22245, 22252, 22253, 22257, 22259, 22260, 22261, 22262, 22266, 22269, 22279, 22282, 22283, 22284, 22285, 22286, 22288, 22291, 22292, 22293, 22294, 22295, 22298, 22299, 22300, 22301, 22302, 22303, 22304, 22305, 22309, 22310, 22312, 22313, 22314, 22315, 22316, 22317, 22319, 22320, 22321, 22322, 22323, 22325, 22326, 22327, 22332, 22333, 22334, 22335, 22336, 22337, 22338, 22347, 22349, 22350, 22360, 22367, 22368, 22369, 22372, 22373, 22374, 22375, 22376, 22377, 22378, 22379, 22389, 22402, 22403, 22404, 22405, 22408, 22411, 22412, 22413, 22414, 22415, 22422, 22423, 22424, 22425, 22427, 22432, 22434, 22435, 22437, 22438, 22439, 22440, 22445, 22446, 22447, 22448, 22449, 22450, 22456, 22457, 22460, 22461, 22462, 22465, 22466, 22469, 22470, 22475, 22478, 22479, 22486, 22488, 22489, 22490, 22494, 22495, 22499, 22506, 22510, 22511, 22512, 22513, 22514, 22515, 22516, 22517, 22518, 22519, 22530, 22531, 22532, 22533, 22538, 22549, 22552, 22553, 22554, 22558, 22559, 22560, 22563, 22564, 22565, 22566, 22570, 22571, 22572, 22573, 22574, 22575, 22576, 22577, 22578, 22579, 22581, 22582, 22584, 22585, 22586, 22587, 22588, 22589, 22590, 22592, 22593, 22594, 22597, 22598, 22599, 22600, 22601, 22602, 22603, 22604, 22605, 22606, 22607, 22608, 22609, 22611, 22612, 22613, 22614, 22619, 22620, 22621, 22622, 22623, 22624, 22625, 22626, 22638, 22639, 22640, 22641, 22642, 22643, 22662, 22665, 22666, 22667, 22668, 22671, 22672, 22678, 22688, 22695, 22696, 22702, 22703, 22708, 22709, 22714, 22718, 22719, 22730, 22733, 22734, 22735, 22737, 22739, 22740, 22741, 22742, 22743, 22744, 22745, 22746, 22747, 22753, 22765, 22767, 22768, 22769, 22773, 22775, 22776, 22777, 22779, 22785, 22788, 22790, 22791, 22793, 22794, 22795, 22797, 22799, 22800, 22801, 22802, 22811, 22812, 22820, 22821, 22822, 22823, 22824, 22825, 22826, 22827, 22828, 22838, 22839, 22840, 22841, 22859, 22865, 22866, 22870, 22875, 22876, 22877, 22878, 22879, 22884, 22885, 22887, 22892, 22893, 22894, 22912, 22913, 22914, 22918, 22921, 22923, 22925, 22928, 22934, 22936, 22937, 22939, 22940, 22941, 22942, 22947, 22948, 22949, 22950, 22951, 22954, 22957, 22961, 22962, 22963, 22964, 22965, 22967, 22968, 22969, 22974, 22975, 22976, 22977, 22978, 22980, 22981, 22982, 22983, 22984, 22985, 22991, 22993, 22996, 22997, 22998, 23008, 23009, 23010, 23011, 23013, 23016, 23017, 23018, 23023, 23025, 23026, 23027, 23034, 23037, 23042, 23044, 23049, 23050, 23051, 23052, 23053, 23058, 23060, 23061, 23062, 23063, 23066, 23069, 23070, 23081, 23082, 23086, 23087, 23089, 23090, 23092, 23093, 23095, 23096, 23097, 23098, 23099, 23100, 23101, 23102, 23103, 23104, 23105, 23107, 23113, 23114, 23115, 23120, 23123, 23124, 23125, 23126, 23133, 23135, 23138, 23139, 23140, 23142, 23143, 23146, 23147, 23150, 23151, 23152, 23153, 23154, 23159, 23161, 23162, 23163, 23164, 23165, 23166, 23167, 23169, 23170, 23172, 23173, 23174, 23177, 23178, 23180, 23181, 23183, 23184, 23185, 23186, 23187, 23190, 23191, 23192, 23193, 23194, 23195, 23196, 23197, 23198, 23199, 23200, 23202, 23205, 23206, 23207, 23208, 23209, 23210, 23211, 23212, 23217, 23218, 23231, 23232, 23233, 23240, 23241, 23245, 23246, 23247, 23248, 23249, 23250, 23251, 23260, 23262, 23264, 23265, 23272, 23273, 23274, 23276, 23277, 23278, 23282, 23283, 23298, 23299, 23302, 23303, 23304, 23308, 23316, 23317, 23320, 23321, 23325, 23326, 23329, 23331, 23332, 23335, 23336, 23337, 23338, 23339, 23340, 23341, 23342, 23343, 23344, 23345, 23349, 23350, 23351, 23354, 23355, 23356, 23357, 23358, 23359, 23360, 23361, 23362, 23363, 23364, 23365, 23366, 23367, 23368, 23369, 23372, 23373, 23374, 23375, 23384, 23391, 23392, 23396, 23397, 23398, 23399, 23401, 23405, 23406, 23410, 23411, 23412, 23413, 23414, 23415, 23416, 23417, 23418, 23419, 23422, 23423, 23426, 23427, 23428, 23429, 23430, 23435, 23436, 23437, 23438, 23439, 23440, 23445, 23446, 23456, 23457, 23458, 23459, 23460, 23461, 23462, 23467, 23470, 23482, 23483, 23489, 23490, 23492, 23493, 23494, 23495, 23503, 23506, 23507, 23508, 23513, 23514, 23515, 23516, 23517, 23518, 23533, 23534, 23535, 23539, 23540, 23547, 23551, 23552, 23553, 23554, 23555, 23556, 23557, 23558, 23559, 23560, 23574, 23576, 23577, 23585, 23586, 23589, 23590, 23591, 23592, 23593, 23594, 23607, 23610, 23611, 23612, 23613, 23614, 23615, 23616, 23617, 23618, 23619, 23620, 23625, 23626, 23627, 23630, 23631, 23632, 23637, 23645, 23675, 23689, 23693, 23696, 23697, 23698, 23699, 23704, 23705, 23706, 23707, 23708, 23709, 23710, 23720, 23721, 23722, 23735, 23738, 23739, 23741, 23742, 23743, 23751, 23753, 23754, 23760, 23762, 23763, 23764, 23766, 23767, 23769, 23770, 23771, 23772, 23773, 23774, 23786, 23789, 23794, 23795, 23797, 23798, 23804, 23805, 23806, 23807, 23817, 23818, 23819, 23820, 23821, 23822, 23823, 23824, 23825, 23832, 23836, 23840, 23841, 23843, 23844, 23845, 23846, 23847, 23848, 23853, 23854, 23855, 23856, 23858, 23860, 23864, 23867, 23868, 23869, 23874, 23875, 23876, 23878, 23880, 23881, 23886, 23887, 23888, 23889, 23890, 23892, 23893, 23900, 23901, 23903, 23904, 23905, 23906, 23907, 23908, 23909, 23920, 23921, 23929, 23933, 23939, 23940, 23941, 23942, 23945, 23949, 23950, 23951, 23952, 23964, 23970, 23971, 23972, 23976, 23981, 23993, 23995, 23998, 23999, 24005, 24008, 24009, 24010, 24014, 24021, 24022, 24025, 24026, 24029, 24040, 24046, 24047, 24051, 24052, 24053, 24054, 24055, 24056, 24057, 24058, 24059, 24064, 24067, 24068, 24069, 24071, 24072, 24073, 24074, 24075, 24076, 24077, 24081, 24088, 24089, 24090, 24104, 24106, 24107, 24108, 24109, 24110, 24122, 24124, 24132, 24133, 24145, 24146, 24147, 24148, 24149, 24150, 24151, 24152, 24156, 24157, 24158, 24159, 24160, 24161, 24175, 24180, 24182, 24183, 24186, 24187, 24189, 24190, 24191, 24192, 24202, 24226, 24227, 24229, 24232, 24233, 24236, 24238, 24241, 24244, 24245, 24246, 24247, 24249, 24250, 24251, 24257, 24259, 24260, 24271, 24276, 24277, 24284, 24285, 24291, 24292, 24299, 24304, 24307, 24308, 24311, 24312, 24316, 24317, 24318, 24320, 24321, 24322, 24323, 24324, 24325, 24326, 24327, 24328, 24329, 24330, 24331, 24332, 24359, 24360, 24361, 24363, 24364, 24369, 24370, 24371, 24372, 24379, 24380, 24381, 24389, 24403, 24405, 24406, 24413, 24424, 24425, 24431, 24435, 24436, 24437, 24439, 24440, 24441, 24442, 24443, 24444, 24445, 24446, 24447, 24452, 24453, 24454, 24455, 24473, 24477, 24479, 24481, 24484, 24487, 24488, 24492, 24493, 24494, 24507, 24508, 24514, 24515, 24517, 24518, 24521, 24524, 24525, 24526, 24527, 24528, 24529, 24534, 24537, 24539, 24540, 24545, 24546, 24547, 24548, 24549, 24550, 24552, 24553, 24554, 24555, 24559, 24575, 24576, 24585, 24586, 24587, 24588, 24590, 24591, 24596, 24597, 24605, 24611, 24612, 24613, 24620, 24621, 24625, 24633, 24634, 24644, 24648, 24649, 24671, 24675, 24680, 24681, 24694, 24695, 24696, 24698, 24705, 24706, 24708, 24709, 24710, 24711, 24713, 24714, 24719, 24722, 24727, 24728, 24729, 24732, 24740, 24748, 24749, 24750, 24754, 24758, 24759, 24760, 24761, 24762, 24763, 24764, 24767, 24768, 24769, 24770, 24772, 24773, 24774, 24775, 24776, 24794, 24795, 24796, 24797, 24798, 24802, 24825, 24843, 24844, 24845, 24852, 24853, 24854, 24864, 24873, 24874, 24875, 24886, 24887, 24891, 24892, 24896, 24897, 24900, 24901, 24912, 24927, 24928, 24929, 24930, 24931, 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24940, 24941, 24942, 24943, 24949, 24950, 24951, 24952, 24953, 24954, 24955, 24961, 24966, 24968, 24969, 24970, 24971, 24972, 25004, 25005, 25006, 25009, 25022, 25024, 25030, 25034, 25035, 25038, 25040, 25057, 25058, 25060, 25061, 25062, 25063, 25064, 25077, 25078, 25079, 25080, 25081, 25086, 25087, 25095, 25096, 25117, 25118, 25119, 25122, 25138, 25149, 25150, 25151, 25152, 25153, 25154, 25155, 25158, 25159, 25161, 25175, 25176, 25177, 25178, 25188, 25201, 25202, 25209, 25212, 25214, 25217, 25218, 25220, 25222, 25223, 25224, 25225, 25226, 25240, 25241, 25242, 25243, 25244, 25259, 25267, 25272, 25273, 25274, 25275, 25276, 25288, 25290, 25291, 25296, 25297, 25321, 25322, 25325, 25334, 25335, 25336, 25338, 25339, 25341, 25342, 25348, 25349, 25350, 25351, 25352, 25365, 25371, 25372, 25377, 25379, 25381, 25382, 25383, 25384, 25385, 25387, 25388, 25392, 25393, 25394, 25395, 25396, 25397, 25398, 25399, 25403, 25406, 25415, 25416, 25417, 25419, 25420, 25431, 25442, 25445, 25450, 25451, 25456, 25457, 25458, 25459, 25462, 25465, 25466, 25467, 25468, 25469, 25470, 25471, 25472, 25473, 25477, 25486, 25487, 25488, 25489, 25504, 25506, 25507, 25508, 25509, 25528, 25529, 25547, 25548, 25550, 25554, 25555, 25556, 25557, 25558, 25559, 25588, 25590, 25591, 25592, 25593, 25594, 25595, 25596, 25624, 25635, 25636, 25643, 25653, 25655, 25663, 25664, 25665, 25666, 25667, 25668, 25669, 25670, 25671, 25672, 25677, 25685, 25686, 25695, 25698, 25699, 25700, 25701, 25702, 25703, 25704, 25705, 25706, 25707, 25708, 25718, 25721, 25722, 25723, 25724, 25725, 25735, 25739, 25740, 25743, 25748, 25749, 25762, 25763, 25764, 25769, 25770, 25784, 25786, 25787, 25788, 25789, 25790, 25796, 25801, 25827, 25828, 25840, 25845, 25846, 25847, 25848, 25849, 25857, 25860, 25861, 25869, 25871, 25872, 25873, 25885, 25891, 25899, 25918, 25930, 25931, 25937, 25940, 25941, 25947, 25960, 25963, 25967, 25968, 25969, 25970, 25971, 25972, 25978, 25979, 25980, 25981, 26006, 26007, 26015, 26017, 26018, 26019, 26021, 26028, 26055, 26066, 26067, 26073, 26084, 26086, 26087, 26096, 26097, 26098, 26099, 26100, 26101, 26102, 26130, 26136, 26137, 26147, 26148, 26160, 26171, 26173, 26174, 26178, 26183, 26184, 26185, 26186, 26188, 26190, 26191, 26193, 26194, 26195, 26205, 26218, 26220, 26222, 26225, 26226, 26227, 26236, 26237, 26238, 26243, 26269, 26279, 26280, 26281, 26285, 26297, 26310, 26318, 26319, 26331, 26341, 26343, 26345, 26346, 26347, 26378, 26379, 26391, 26392, 26408, 26418, 26419, 26420, 26425, 26442, 26443, 26454, 26455, 26456, 26469, 26473, 26486, 26487, 26488, 26490, 26494, 26495, 26496, 26520, 26521, 26522, 26529, 26532, 26533, 26537, 26539, 26541, 26542, 26543, 26544, 26545, 26560, 26568, 26569, 26570, 26571, 26572, 26582, 26589, 26592, 26609, 26610, 26611, 26612, 26613, 26614, 26615, 26625, 26632, 26649, 26662, 26676, 26677, 26678, 26685, 26692, 26696, 26697, 26701, 26702, 26703, 26704, 26717, 26718, 26719, 26724, 26725, 26726, 26737, 26738, 26760, 26761, 26777, 26778, 26786, 26790, 26794, 26803, 26811, 26815, 26833, 26836, 26837, 26841, 26842, 26848, 26849, 26850, 26857, 26884, 26890, 26894, 26895, 26901, 26902, 26906, 26907, 26914, 26916, 26921, 26922, 26923, 26925, 26926, 26940, 26950, 26951, 26953, 26954, 26955, 26956, 27001, 27002, 27003, 27004, 27005, 27009, 27010, 27024, 27026, 27029, 27040, 27041, 27042, 27046, 27047, 27068, 27069, 27070, 27085, 27086, 27103, 27115, 27120, 27123, 27124, 27150, 27155, 27171, 27172, 27179, 27197, 27201, 27206, 27207, 27208, 27224, 27225, 27226, 27227, 27243, 27244, 27245, 27246, 27280, 27284, 27285, 27293, 27299, 27300, 27319, 27321, 27322, 27323, 27324, 27330, 27331, 27343, 27344, 27349, 27351, 27352, 27353, 27354, 27361, 27370, 27381, 27409, 27412, 27413, 27415, 27417, 27418, 27460, 27471, 27472, 27493, 27524, 27539, 27543, 27544, 27551, 27553, 27554, 27555, 27556, 27575, 27578, 27579, 27580, 27596, 27601, 27620, 27624, 27625, 27626, 27627, 27628, 27668, 27669, 27684, 27687, 27690, 27699, 27714, 27724, 27726, 27727, 27729, 27730, 27734, 27738, 27743, 27744, 27746, 27762, 27763, 27764, 27792, 27796, 27797, 27807, 27810, 27822, 27824, 27825, 27838, 27844, 27845, 27853, 27854, 27855, 27864, 27865, 27924, 27926, 27928, 27929, 27940, 27941, 27948, 27949, 27977, 27978, 27991, 27997, 28007, 28010, 28013, 28027, 28028, 28029, 28030, 28031, 28073, 28074, 28075, 28100, 28101, 28108, 28115, 28126, 28127, 28128, 28142, 28153, 28154, 28169, 28179, 28182, 28189, 28210, 28211, 28212, 28216, 28223, 28224, 28225, 28243, 28277, 28311, 28332, 28333, 28336, 28337, 28338, 28352, 28353, 28358, 28364, 28370, 28374, 28398, 28407, 28408, 28419, 28424, 28463, 28482, 28491, 28492, 28496, 28513, 28514, 28534, 28548, 28551, 28552, 28553, 28562, 28572, 28593, 28594, 28595, 28607, 28629, 28640, 28647, 28650, 28651, 28656, 28661, 28741, 28758, 28759, 28786, 28792, 28793, 28795, 28796, 28801, 28802, 28804, 28807, 28814, 28825, 28826, 28844, 28855, 28856, 28872, 28906, 28916, 28926, 28942, 28953, 28954, 28964, 29013, 29021, 29052, 29053, 29082, 29104, 29115, 29131, 29177, 29192, 29243, 29249, 29250, 29269, 29298, 29320, 29321, 29345, 29372, 29377, 29430, 29447, 29450, 29493, 29518, 29519, 29520, 29546, 29560, 29605, 29616, 29620, 29628, 29632, 29633, 29640, 29644, 29645, 29653, 29664, 29680, 29735, 29736, 29748, 29791, 29816, 29831, 29868, 29871, 29876, 29877, 29886, 29897, 29898, 29905, 29908, 29958, 29982, 29984, 30046, 30082, 30086, 30120, 30121, 30133, 30172, 30183, 30184, 30191, 30217, 30236, 30358, 30359, 30366, 30406, 30446, 30451, 30553, 30569, 30597, 30598, 30599, 30695, 30696, 30701, 30796, 30853, 30879, 30880, 30883, 30968, 30984, 31030, 31031, 31043, 31153, 31189, 31196, 31348, 31401, 31440, 31725, 31726, 31768, 31958, 32017, 32358, 32397}, slot = 0 From 77b0963c46670280046d58e894ca0846dad9baeb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Jan 2025 23:29:45 +0000 Subject: [PATCH 39/44] clean up ShredIndexV2 constants --- ledger/src/blockstore_meta.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 51b84bb7836cf7..add04295e4c38d 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -330,7 +330,7 @@ pub struct ShredIndexV2 { impl Default for ShredIndexV2 { fn default() -> Self { Self { - index: vec![0; Self::MAX_WORDS_PER_SHRED], + index: vec![0; Self::MAX_WORDS_PER_SLOT], num_shreds: 0, } } @@ -339,16 +339,16 @@ impl Default for ShredIndexV2 { type ShredIndexV2Word = u8; impl ShredIndexV2 { const SIZE_OF_WORD: usize = std::mem::size_of::(); - const WORD_BITS: usize = Self::SIZE_OF_WORD * 8; - const MAX_WORDS_PER_SHRED: usize = MAX_DATA_SHREDS_PER_SLOT.div_ceil(Self::WORD_BITS); + const BITS_PER_WORD: usize = Self::SIZE_OF_WORD * 8; + const MAX_WORDS_PER_SLOT: usize = MAX_DATA_SHREDS_PER_SLOT.div_ceil(Self::BITS_PER_WORD); pub fn num_shreds(&self) -> usize { self.num_shreds } fn index_and_mask(index: u64) -> (usize, ShredIndexV2Word) { - let word_idx = index as usize / Self::WORD_BITS; - let bit_idx = index as usize % Self::WORD_BITS; + let word_idx = index as usize / Self::BITS_PER_WORD; + let bit_idx = index as usize % Self::BITS_PER_WORD; let mask = 1 << bit_idx; (word_idx, mask as ShredIndexV2Word) } @@ -445,24 +445,26 @@ impl ShredIndexV2 { Bound::Unbounded => MAX_DATA_SHREDS_PER_SLOT, }; - let end_word = end.div_ceil(Self::WORD_BITS).min(Self::MAX_WORDS_PER_SHRED); - let start_word = (start / Self::WORD_BITS).min(end_word); + let end_word = end + .div_ceil(Self::BITS_PER_WORD) + .min(Self::MAX_WORDS_PER_SLOT); + let start_word = (start / Self::BITS_PER_WORD).min(end_word); self.index[start_word..end_word] .iter() .enumerate() .flat_map(move |(word_offset, &word)| { - let base_idx = (start_word + word_offset) * Self::WORD_BITS; + let base_idx = (start_word + word_offset) * Self::BITS_PER_WORD; let lower_bound = start.saturating_sub(base_idx); - let upper_bound = if base_idx + Self::WORD_BITS > end { + let upper_bound = if base_idx + Self::BITS_PER_WORD > end { end - base_idx } else { - Self::WORD_BITS + Self::BITS_PER_WORD }; let lower_mask = !0 << lower_bound; - let upper_mask = !0 >> (Self::WORD_BITS - upper_bound); + let upper_mask = !0 >> (Self::BITS_PER_WORD - upper_bound); let mask = word & lower_mask & upper_mask; std::iter::from_fn({ From 02f8abc608861af47b0b139fecbf50368faf5929 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Jan 2025 16:18:03 +0000 Subject: [PATCH 40/44] use u8 bounds ShredIndexV2 boundary conditions test --- ledger/src/blockstore_meta.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index add04295e4c38d..2998172ff6cdc5 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -1060,12 +1060,12 @@ mod test { // First possible index index.insert(0); - // Last index in first word - index.insert(63); - // First index in second word - index.insert(64); + // Last index in first word (bits 0-7) + index.insert(7); + // First index in second word (bits 8-15) + index.insert(8); // Last index in second word - index.insert(127); + index.insert(15); // Last valid index index.insert(MAX_DATA_SHREDS_PER_SLOT as u64 - 1); // Should be ignored (too large) @@ -1073,18 +1073,18 @@ mod test { // Verify contents assert!(index.contains(0)); - assert!(index.contains(63)); - assert!(index.contains(64)); - assert!(index.contains(127)); + assert!(index.contains(7)); + assert!(index.contains(8)); + assert!(index.contains(15)); assert!(index.contains(MAX_DATA_SHREDS_PER_SLOT as u64 - 1)); assert!(!index.contains(MAX_DATA_SHREDS_PER_SLOT as u64)); // Cross-word boundary - assert_eq!(index.range(50..70).collect::>(), vec![63, 64]); + assert_eq!(index.range(6..10).collect::>(), vec![7, 8]); // Full first word - assert_eq!(index.range(0..64).collect::>(), vec![0, 63]); + assert_eq!(index.range(0..8).collect::>(), vec![0, 7]); // Full second word - assert_eq!(index.range(64..128).collect::>(), vec![64, 127]); + assert_eq!(index.range(8..16).collect::>(), vec![8, 15]); // Empty ranges assert_eq!(index.range(0..0).count(), 0); @@ -1097,12 +1097,12 @@ mod test { index.remove(0); assert!(!index.contains(0)); - index.remove(63); - assert!(!index.contains(63)); - index.remove(64); - assert!(!index.contains(64)); - index.remove(127); - assert!(!index.contains(127)); + index.remove(7); + assert!(!index.contains(7)); + index.remove(8); + assert!(!index.contains(8)); + index.remove(15); + assert!(!index.contains(15)); index.remove(MAX_DATA_SHREDS_PER_SLOT as u64 - 1); assert!(!index.contains(MAX_DATA_SHREDS_PER_SLOT as u64 - 1)); From c6d6b80592cd9a838277cbcbd6c38ba8885f5bef Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Jan 2025 17:49:23 +0000 Subject: [PATCH 41/44] improve runtime of proptests --- .../proptest-regressions/blockstore_meta.txt | 2 +- ledger/src/blockstore_meta.rs | 85 ++++++++----------- 2 files changed, 35 insertions(+), 52 deletions(-) diff --git a/ledger/proptest-regressions/blockstore_meta.txt b/ledger/proptest-regressions/blockstore_meta.txt index 2d7f3ba59ebe77..ba4028c0da445d 100644 --- a/ledger/proptest-regressions/blockstore_meta.txt +++ b/ledger/proptest-regressions/blockstore_meta.txt @@ -4,4 +4,4 @@ # # It is recommended to check this file in to source control so that # everyone who runs the test benefits from these saved cases. -cc de06086cdefde07319a41f7e78761901b2ffaabd0ef8b03e8a321f682fd3ab47 # shrinks to coding_indices = {}, data_indices = {0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 81, 82, 83, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 166, 167, 168, 169, 170, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 192, 193, 194, 195, 196, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 247, 248, 249, 250, 251, 252, 253, 254, 255, 257, 258, 259, 260, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 309, 310, 311, 312, 313, 314, 315, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 334, 335, 336, 337, 340, 341, 342, 343, 344, 345, 346, 347, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 449, 450, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 638, 639, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 693, 694, 695, 696, 697, 699, 700, 701, 703, 704, 706, 707, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 733, 734, 735, 736, 737, 738, 739, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 763, 764, 765, 766, 767, 768, 769, 771, 772, 773, 774, 775, 776, 777, 778, 779, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 883, 884, 885, 886, 887, 888, 890, 891, 892, 893, 894, 895, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 957, 958, 959, 960, 961, 962, 963, 964, 965, 967, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 992, 993, 994, 995, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1067, 1068, 1070, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1138, 1139, 1140, 1142, 1144, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1214, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1245, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1259, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1322, 1324, 1325, 1326, 1327, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1344, 1345, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1375, 1377, 1380, 1382, 1383, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1436, 1437, 1438, 1439, 1440, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1492, 1493, 1495, 1496, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1507, 1508, 1509, 1510, 1511, 1512, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, 1579, 1581, 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1667, 1668, 1669, 1670, 1671, 1672, 1674, 1675, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, 1700, 1701, 1702, 1703, 1704, 1705, 1708, 1709, 1710, 1711, 1713, 1714, 1715, 1716, 1717, 1718, 1720, 1721, 1722, 1723, 1726, 1727, 1728, 1730, 1731, 1732, 1733, 1735, 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, 1745, 1746, 1748, 1749, 1750, 1751, 1752, 1754, 1755, 1756, 1757, 1759, 1760, 1761, 1762, 1763, 1764, 1766, 1767, 1768, 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1828, 1829, 1830, 1831, 1832, 1835, 1836, 1837, 1838, 1839, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, 1857, 1858, 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, 1892, 1893, 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, 1912, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1987, 1988, 1989, 1990, 1991, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2037, 2039, 2040, 2042, 2043, 2044, 2045, 2047, 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, 2103, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2114, 2115, 2116, 2119, 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, 2140, 2141, 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2178, 2179, 2180, 2181, 2183, 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2200, 2201, 2202, 2203, 2204, 2205, 2207, 2209, 2210, 2213, 2214, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, 2400, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, 2424, 2426, 2427, 2428, 2429, 2430, 2431, 2434, 2435, 2436, 2437, 2438, 2439, 2440, 2441, 2442, 2444, 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2453, 2455, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, 2477, 2478, 2479, 2480, 2481, 2482, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2492, 2493, 2494, 2495, 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2639, 2640, 2641, 2642, 2643, 2644, 2646, 2647, 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, 2701, 2703, 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, 2741, 2743, 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, 2764, 2765, 2767, 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, 2802, 2803, 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2864, 2865, 2866, 2867, 2869, 2870, 2871, 2872, 2873, 2874, 2875, 2877, 2878, 2879, 2880, 2881, 2882, 2883, 2884, 2886, 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, 2898, 2899, 2900, 2901, 2903, 2904, 2905, 2906, 2907, 2908, 2910, 2911, 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2928, 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, 2965, 2966, 2967, 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, 2978, 2979, 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3011, 3012, 3013, 3014, 3015, 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, 3034, 3035, 3036, 3038, 3039, 3040, 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, 3057, 3059, 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, 3069, 3070, 3071, 3072, 3073, 3074, 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3107, 3108, 3109, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, 3138, 3139, 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, 3156, 3158, 3159, 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, 3196, 3197, 3199, 3200, 3201, 3202, 3203, 3205, 3206, 3207, 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, 3242, 3243, 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, 3252, 3254, 3255, 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, 3277, 3278, 3279, 3280, 3281, 3283, 3284, 3285, 3286, 3287, 3289, 3291, 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, 3308, 3310, 3311, 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, 3364, 3365, 3367, 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3404, 3406, 3407, 3408, 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, 3449, 3451, 3452, 3454, 3455, 3456, 3458, 3459, 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3497, 3498, 3499, 3502, 3503, 3504, 3505, 3506, 3507, 3509, 3510, 3511, 3512, 3513, 3514, 3515, 3516, 3517, 3518, 3519, 3520, 3521, 3522, 3523, 3524, 3525, 3526, 3527, 3528, 3529, 3530, 3531, 3532, 3533, 3534, 3535, 3536, 3538, 3539, 3541, 3542, 3543, 3546, 3547, 3548, 3549, 3550, 3552, 3554, 3555, 3556, 3557, 3558, 3559, 3560, 3561, 3562, 3564, 3565, 3566, 3567, 3568, 3569, 3570, 3571, 3572, 3573, 3574, 3575, 3576, 3577, 3578, 3579, 3580, 3581, 3582, 3583, 3584, 3585, 3586, 3587, 3588, 3589, 3590, 3591, 3592, 3593, 3594, 3596, 3597, 3598, 3600, 3601, 3602, 3604, 3605, 3606, 3607, 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, 3624, 3625, 3626, 3628, 3629, 3630, 3631, 3632, 3633, 3634, 3635, 3636, 3637, 3638, 3639, 3640, 3641, 3642, 3643, 3644, 3645, 3646, 3647, 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, 3656, 3657, 3658, 3659, 3661, 3662, 3663, 3664, 3666, 3667, 3668, 3669, 3670, 3672, 3673, 3674, 3676, 3677, 3678, 3679, 3680, 3681, 3682, 3683, 3684, 3685, 3686, 3687, 3688, 3689, 3690, 3691, 3692, 3693, 3694, 3695, 3696, 3697, 3698, 3699, 3700, 3701, 3702, 3703, 3704, 3705, 3706, 3707, 3709, 3710, 3711, 3712, 3713, 3714, 3715, 3716, 3717, 3718, 3719, 3720, 3721, 3722, 3723, 3724, 3725, 3726, 3727, 3728, 3729, 3730, 3731, 3732, 3733, 3734, 3735, 3736, 3737, 3738, 3739, 3740, 3741, 3742, 3743, 3744, 3745, 3746, 3747, 3748, 3749, 3750, 3752, 3753, 3754, 3755, 3756, 3758, 3759, 3760, 3761, 3762, 3763, 3764, 3765, 3766, 3767, 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, 3776, 3777, 3778, 3779, 3780, 3781, 3783, 3784, 3786, 3787, 3788, 3789, 3790, 3791, 3792, 3793, 3794, 3796, 3798, 3799, 3800, 3801, 3802, 3803, 3804, 3805, 3806, 3807, 3808, 3809, 3811, 3812, 3813, 3814, 3815, 3816, 3817, 3818, 3819, 3820, 3821, 3822, 3823, 3824, 3825, 3826, 3827, 3828, 3829, 3830, 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, 3856, 3857, 3858, 3860, 3861, 3862, 3863, 3864, 3865, 3867, 3868, 3869, 3870, 3871, 3872, 3873, 3874, 3875, 3876, 3877, 3879, 3880, 3881, 3882, 3883, 3884, 3885, 3886, 3887, 3888, 3889, 3890, 3891, 3892, 3893, 3894, 3895, 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, 3904, 3905, 3906, 3907, 3908, 3909, 3910, 3911, 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, 3936, 3937, 3938, 3939, 3940, 3941, 3942, 3943, 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, 3952, 3954, 3955, 3956, 3957, 3958, 3959, 3960, 3961, 3962, 3963, 3965, 3966, 3967, 3968, 3969, 3970, 3971, 3972, 3973, 3974, 3975, 3976, 3977, 3978, 3979, 3980, 3981, 3982, 3984, 3985, 3986, 3987, 3988, 3989, 3990, 3991, 3992, 3993, 3994, 3995, 3996, 3997, 3999, 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, 4032, 4033, 4034, 4035, 4037, 4038, 4039, 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047, 4048, 4049, 4051, 4052, 4053, 4054, 4055, 4056, 4057, 4058, 4059, 4060, 4061, 4063, 4065, 4066, 4067, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4077, 4078, 4079, 4080, 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, 4096, 4097, 4099, 4100, 4101, 4102, 4103, 4104, 4105, 4106, 4107, 4108, 4109, 4111, 4112, 4113, 4114, 4115, 4116, 4117, 4118, 4119, 4121, 4122, 4123, 4124, 4126, 4127, 4128, 4129, 4130, 4131, 4132, 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4144, 4145, 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, 4159, 4160, 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, 4172, 4173, 4174, 4175, 4176, 4177, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, 4186, 4187, 4188, 4189, 4190, 4191, 4192, 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4214, 4215, 4217, 4218, 4219, 4220, 4221, 4222, 4223, 4224, 4225, 4226, 4227, 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, 4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4265, 4266, 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325, 4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336, 4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4347, 4348, 4349, 4350, 4351, 4352, 4353, 4354, 4355, 4356, 4357, 4358, 4359, 4360, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4369, 4370, 4371, 4372, 4373, 4374, 4375, 4376, 4377, 4378, 4379, 4380, 4381, 4382, 4384, 4385, 4387, 4388, 4389, 4390, 4391, 4392, 4393, 4394, 4395, 4396, 4397, 4398, 4399, 4400, 4402, 4403, 4404, 4405, 4406, 4407, 4408, 4409, 4410, 4411, 4412, 4413, 4414, 4415, 4416, 4417, 4418, 4419, 4420, 4421, 4422, 4423, 4424, 4425, 4426, 4427, 4428, 4429, 4430, 4431, 4432, 4433, 4434, 4435, 4436, 4437, 4438, 4439, 4440, 4441, 4443, 4444, 4445, 4446, 4447, 4448, 4449, 4450, 4451, 4452, 4453, 4454, 4455, 4456, 4457, 4458, 4460, 4461, 4462, 4463, 4464, 4465, 4467, 4468, 4469, 4470, 4471, 4472, 4473, 4474, 4475, 4476, 4477, 4478, 4479, 4480, 4481, 4482, 4485, 4486, 4487, 4488, 4490, 4491, 4492, 4493, 4494, 4495, 4496, 4497, 4498, 4499, 4500, 4501, 4502, 4503, 4504, 4505, 4506, 4507, 4508, 4509, 4510, 4511, 4513, 4514, 4515, 4516, 4517, 4518, 4519, 4520, 4521, 4522, 4523, 4524, 4525, 4526, 4528, 4529, 4530, 4531, 4533, 4534, 4535, 4536, 4537, 4538, 4539, 4540, 4541, 4542, 4543, 4544, 4545, 4546, 4547, 4549, 4550, 4551, 4552, 4553, 4554, 4555, 4556, 4557, 4558, 4559, 4560, 4561, 4562, 4563, 4564, 4565, 4566, 4567, 4568, 4569, 4570, 4571, 4572, 4573, 4574, 4575, 4576, 4577, 4579, 4580, 4581, 4582, 4583, 4584, 4585, 4586, 4587, 4588, 4589, 4590, 4591, 4592, 4593, 4594, 4595, 4597, 4598, 4599, 4600, 4603, 4604, 4605, 4606, 4607, 4608, 4610, 4611, 4612, 4613, 4615, 4616, 4617, 4618, 4619, 4620, 4623, 4624, 4625, 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, 4644, 4645, 4646, 4647, 4648, 4649, 4650, 4651, 4652, 4653, 4654, 4655, 4656, 4657, 4658, 4659, 4660, 4662, 4663, 4664, 4665, 4666, 4667, 4668, 4669, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4701, 4702, 4703, 4704, 4705, 4706, 4707, 4708, 4709, 4710, 4711, 4713, 4714, 4715, 4716, 4717, 4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4727, 4728, 4729, 4730, 4731, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, 4757, 4758, 4759, 4760, 4761, 4762, 4764, 4765, 4766, 4767, 4768, 4769, 4772, 4773, 4774, 4775, 4776, 4777, 4778, 4779, 4780, 4781, 4782, 4783, 4784, 4785, 4786, 4787, 4788, 4789, 4790, 4793, 4795, 4796, 4797, 4798, 4799, 4801, 4802, 4803, 4804, 4805, 4807, 4808, 4809, 4811, 4812, 4813, 4814, 4815, 4816, 4817, 4818, 4819, 4820, 4821, 4822, 4823, 4824, 4825, 4826, 4827, 4828, 4830, 4831, 4832, 4833, 4834, 4835, 4836, 4837, 4838, 4839, 4840, 4841, 4842, 4843, 4844, 4845, 4846, 4847, 4848, 4849, 4850, 4852, 4854, 4855, 4856, 4857, 4858, 4859, 4862, 4863, 4864, 4866, 4867, 4868, 4869, 4870, 4871, 4872, 4873, 4875, 4876, 4877, 4878, 4879, 4880, 4881, 4882, 4883, 4884, 4885, 4886, 4887, 4888, 4889, 4890, 4891, 4892, 4893, 4894, 4895, 4896, 4897, 4898, 4899, 4900, 4901, 4902, 4903, 4904, 4905, 4906, 4907, 4908, 4909, 4910, 4911, 4912, 4914, 4916, 4917, 4918, 4919, 4920, 4922, 4923, 4924, 4925, 4927, 4928, 4929, 4930, 4931, 4932, 4933, 4934, 4935, 4937, 4938, 4939, 4940, 4941, 4942, 4943, 4944, 4945, 4946, 4947, 4948, 4949, 4950, 4951, 4952, 4954, 4955, 4957, 4958, 4959, 4960, 4961, 4962, 4964, 4965, 4966, 4967, 4968, 4969, 4970, 4971, 4972, 4973, 4974, 4975, 4976, 4977, 4978, 4979, 4980, 4981, 4982, 4983, 4984, 4987, 4988, 4990, 4991, 4992, 4993, 4994, 4995, 4996, 4997, 4998, 4999, 5000, 5001, 5002, 5003, 5004, 5005, 5006, 5007, 5008, 5009, 5010, 5011, 5012, 5014, 5015, 5016, 5017, 5018, 5019, 5020, 5021, 5023, 5024, 5025, 5026, 5027, 5028, 5029, 5030, 5031, 5032, 5033, 5034, 5035, 5036, 5037, 5038, 5039, 5040, 5041, 5042, 5043, 5044, 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5055, 5056, 5057, 5058, 5059, 5060, 5061, 5062, 5063, 5064, 5065, 5066, 5067, 5068, 5069, 5070, 5071, 5072, 5073, 5074, 5075, 5076, 5077, 5078, 5079, 5080, 5081, 5082, 5083, 5084, 5085, 5086, 5087, 5088, 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5097, 5098, 5099, 5101, 5102, 5104, 5105, 5106, 5107, 5108, 5109, 5110, 5111, 5112, 5113, 5114, 5115, 5116, 5117, 5118, 5119, 5120, 5121, 5122, 5123, 5124, 5125, 5126, 5127, 5129, 5130, 5131, 5132, 5133, 5134, 5135, 5136, 5137, 5138, 5139, 5140, 5141, 5142, 5143, 5144, 5146, 5147, 5148, 5149, 5150, 5152, 5153, 5155, 5156, 5157, 5158, 5159, 5160, 5161, 5162, 5163, 5164, 5165, 5166, 5167, 5168, 5169, 5170, 5171, 5172, 5173, 5174, 5175, 5176, 5177, 5178, 5179, 5180, 5181, 5182, 5183, 5184, 5185, 5186, 5188, 5189, 5190, 5191, 5192, 5193, 5194, 5195, 5196, 5197, 5198, 5199, 5200, 5201, 5202, 5203, 5204, 5205, 5206, 5207, 5208, 5209, 5210, 5211, 5212, 5213, 5214, 5215, 5216, 5217, 5218, 5219, 5220, 5221, 5222, 5223, 5224, 5225, 5226, 5227, 5228, 5229, 5230, 5231, 5232, 5233, 5234, 5235, 5236, 5237, 5238, 5239, 5240, 5241, 5242, 5243, 5244, 5245, 5246, 5247, 5248, 5249, 5250, 5251, 5252, 5253, 5254, 5255, 5256, 5257, 5258, 5259, 5260, 5261, 5262, 5263, 5264, 5265, 5266, 5267, 5268, 5270, 5271, 5272, 5273, 5274, 5275, 5276, 5277, 5278, 5279, 5280, 5281, 5282, 5284, 5285, 5286, 5287, 5288, 5289, 5290, 5291, 5292, 5293, 5294, 5295, 5298, 5299, 5300, 5301, 5302, 5303, 5304, 5305, 5306, 5307, 5309, 5310, 5311, 5312, 5313, 5314, 5315, 5316, 5317, 5318, 5319, 5320, 5321, 5322, 5323, 5324, 5325, 5326, 5328, 5329, 5330, 5331, 5332, 5333, 5334, 5335, 5336, 5337, 5338, 5339, 5340, 5341, 5342, 5343, 5344, 5345, 5346, 5347, 5348, 5349, 5350, 5351, 5352, 5353, 5354, 5356, 5357, 5358, 5359, 5361, 5362, 5363, 5365, 5366, 5367, 5368, 5369, 5370, 5371, 5372, 5373, 5374, 5375, 5376, 5377, 5378, 5379, 5381, 5383, 5384, 5385, 5386, 5387, 5388, 5389, 5391, 5392, 5393, 5394, 5395, 5396, 5397, 5398, 5399, 5400, 5401, 5402, 5403, 5404, 5405, 5406, 5407, 5408, 5409, 5410, 5411, 5412, 5413, 5414, 5415, 5416, 5417, 5418, 5419, 5420, 5421, 5423, 5424, 5426, 5427, 5428, 5429, 5430, 5431, 5432, 5433, 5434, 5435, 5436, 5437, 5438, 5439, 5440, 5441, 5442, 5444, 5445, 5446, 5447, 5448, 5449, 5450, 5451, 5452, 5453, 5454, 5455, 5457, 5458, 5459, 5460, 5461, 5462, 5463, 5464, 5465, 5466, 5467, 5468, 5469, 5470, 5471, 5472, 5473, 5474, 5475, 5476, 5477, 5478, 5479, 5480, 5481, 5482, 5483, 5484, 5485, 5486, 5487, 5488, 5489, 5490, 5491, 5492, 5493, 5494, 5495, 5496, 5497, 5498, 5499, 5500, 5501, 5502, 5503, 5504, 5505, 5506, 5507, 5508, 5509, 5510, 5511, 5512, 5513, 5515, 5516, 5518, 5519, 5520, 5521, 5522, 5523, 5524, 5525, 5526, 5528, 5529, 5530, 5531, 5532, 5533, 5534, 5535, 5539, 5540, 5541, 5542, 5543, 5545, 5546, 5547, 5548, 5549, 5550, 5551, 5553, 5555, 5556, 5557, 5558, 5559, 5560, 5561, 5562, 5563, 5564, 5565, 5566, 5567, 5568, 5569, 5570, 5571, 5572, 5573, 5574, 5575, 5576, 5577, 5578, 5579, 5580, 5581, 5583, 5584, 5585, 5586, 5587, 5588, 5589, 5590, 5591, 5592, 5593, 5594, 5595, 5598, 5599, 5600, 5601, 5602, 5603, 5604, 5605, 5606, 5607, 5608, 5609, 5610, 5611, 5612, 5613, 5614, 5615, 5616, 5617, 5618, 5619, 5620, 5621, 5622, 5623, 5624, 5625, 5626, 5627, 5628, 5629, 5630, 5631, 5633, 5634, 5636, 5637, 5638, 5639, 5640, 5641, 5642, 5643, 5644, 5645, 5646, 5647, 5648, 5649, 5650, 5651, 5652, 5653, 5654, 5656, 5657, 5658, 5659, 5660, 5661, 5662, 5663, 5664, 5665, 5666, 5667, 5668, 5669, 5671, 5672, 5674, 5675, 5676, 5677, 5678, 5680, 5681, 5682, 5683, 5684, 5685, 5686, 5687, 5688, 5689, 5690, 5691, 5693, 5694, 5695, 5696, 5697, 5698, 5700, 5701, 5702, 5703, 5705, 5706, 5707, 5708, 5709, 5710, 5711, 5712, 5713, 5714, 5715, 5716, 5717, 5718, 5719, 5720, 5721, 5722, 5723, 5724, 5725, 5726, 5729, 5730, 5731, 5732, 5733, 5734, 5735, 5736, 5737, 5738, 5739, 5740, 5741, 5742, 5744, 5745, 5746, 5747, 5748, 5749, 5750, 5751, 5752, 5753, 5754, 5755, 5756, 5757, 5759, 5760, 5761, 5762, 5763, 5764, 5765, 5766, 5767, 5768, 5769, 5770, 5771, 5772, 5773, 5774, 5775, 5776, 5777, 5778, 5779, 5780, 5781, 5782, 5783, 5784, 5785, 5786, 5787, 5788, 5789, 5790, 5791, 5792, 5793, 5794, 5795, 5796, 5797, 5798, 5799, 5800, 5801, 5802, 5803, 5804, 5805, 5806, 5807, 5808, 5809, 5810, 5811, 5813, 5814, 5815, 5816, 5817, 5818, 5819, 5820, 5821, 5823, 5824, 5825, 5826, 5827, 5828, 5829, 5830, 5831, 5832, 5833, 5834, 5836, 5837, 5838, 5839, 5840, 5841, 5842, 5843, 5844, 5846, 5847, 5848, 5849, 5850, 5851, 5853, 5854, 5855, 5856, 5857, 5858, 5859, 5860, 5861, 5862, 5863, 5864, 5865, 5866, 5867, 5868, 5869, 5870, 5871, 5872, 5873, 5874, 5875, 5876, 5877, 5878, 5879, 5880, 5881, 5883, 5884, 5885, 5886, 5887, 5889, 5890, 5891, 5892, 5893, 5894, 5895, 5896, 5897, 5898, 5899, 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907, 5908, 5909, 5910, 5911, 5912, 5913, 5914, 5915, 5916, 5917, 5918, 5919, 5920, 5921, 5922, 5923, 5924, 5925, 5926, 5927, 5928, 5929, 5930, 5931, 5932, 5933, 5934, 5937, 5938, 5939, 5940, 5941, 5942, 5943, 5944, 5945, 5946, 5947, 5948, 5950, 5951, 5952, 5953, 5954, 5955, 5956, 5957, 5958, 5959, 5960, 5961, 5962, 5963, 5964, 5965, 5966, 5967, 5968, 5969, 5970, 5971, 5973, 5974, 5975, 5976, 5977, 5978, 5979, 5980, 5981, 5982, 5983, 5984, 5985, 5986, 5987, 5989, 5990, 5991, 5992, 5993, 5994, 5995, 5996, 5998, 5999, 6000, 6001, 6002, 6003, 6004, 6005, 6006, 6007, 6009, 6010, 6011, 6012, 6013, 6014, 6015, 6016, 6017, 6018, 6019, 6020, 6021, 6022, 6023, 6024, 6025, 6026, 6027, 6028, 6029, 6030, 6031, 6032, 6033, 6034, 6035, 6036, 6037, 6038, 6039, 6040, 6041, 6042, 6043, 6044, 6045, 6046, 6047, 6048, 6049, 6050, 6051, 6053, 6054, 6055, 6056, 6057, 6058, 6059, 6060, 6061, 6062, 6063, 6064, 6065, 6066, 6067, 6068, 6069, 6070, 6071, 6072, 6073, 6075, 6076, 6077, 6079, 6080, 6081, 6083, 6084, 6085, 6086, 6087, 6088, 6089, 6090, 6091, 6092, 6093, 6094, 6095, 6096, 6097, 6098, 6099, 6100, 6101, 6102, 6103, 6104, 6105, 6106, 6107, 6108, 6109, 6110, 6111, 6113, 6114, 6115, 6116, 6117, 6118, 6120, 6121, 6122, 6123, 6124, 6125, 6126, 6127, 6128, 6130, 6131, 6132, 6133, 6134, 6135, 6136, 6137, 6139, 6140, 6141, 6142, 6143, 6144, 6145, 6146, 6148, 6149, 6150, 6151, 6152, 6154, 6155, 6156, 6157, 6158, 6159, 6160, 6161, 6163, 6164, 6165, 6166, 6167, 6168, 6169, 6170, 6171, 6172, 6173, 6174, 6175, 6176, 6177, 6178, 6179, 6181, 6182, 6183, 6184, 6185, 6186, 6187, 6188, 6189, 6190, 6191, 6192, 6193, 6194, 6195, 6196, 6197, 6198, 6199, 6200, 6201, 6202, 6203, 6204, 6205, 6206, 6207, 6208, 6209, 6210, 6211, 6212, 6213, 6214, 6215, 6216, 6217, 6218, 6220, 6221, 6222, 6223, 6224, 6225, 6226, 6227, 6228, 6230, 6232, 6233, 6234, 6235, 6236, 6237, 6238, 6239, 6240, 6241, 6242, 6243, 6244, 6245, 6246, 6247, 6248, 6249, 6250, 6251, 6252, 6253, 6254, 6255, 6258, 6259, 6260, 6261, 6262, 6263, 6264, 6265, 6266, 6267, 6268, 6269, 6270, 6272, 6273, 6274, 6275, 6276, 6277, 6278, 6279, 6280, 6281, 6282, 6283, 6284, 6285, 6286, 6290, 6291, 6292, 6293, 6294, 6295, 6296, 6297, 6298, 6300, 6301, 6302, 6303, 6304, 6305, 6306, 6308, 6309, 6310, 6312, 6313, 6314, 6316, 6317, 6318, 6319, 6320, 6321, 6322, 6323, 6324, 6325, 6326, 6327, 6328, 6329, 6330, 6331, 6332, 6333, 6334, 6335, 6336, 6337, 6339, 6340, 6341, 6342, 6343, 6344, 6345, 6346, 6347, 6348, 6349, 6350, 6351, 6352, 6353, 6354, 6355, 6357, 6358, 6359, 6360, 6361, 6362, 6363, 6364, 6365, 6366, 6367, 6368, 6369, 6370, 6371, 6372, 6373, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6381, 6382, 6384, 6385, 6386, 6387, 6388, 6389, 6390, 6391, 6392, 6393, 6394, 6395, 6396, 6397, 6400, 6401, 6402, 6403, 6404, 6405, 6406, 6407, 6408, 6409, 6410, 6411, 6412, 6413, 6414, 6415, 6416, 6417, 6418, 6419, 6420, 6421, 6422, 6423, 6424, 6425, 6426, 6427, 6429, 6430, 6431, 6432, 6433, 6434, 6435, 6436, 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6448, 6449, 6450, 6451, 6452, 6453, 6454, 6456, 6457, 6458, 6459, 6460, 6461, 6462, 6463, 6464, 6465, 6466, 6467, 6468, 6469, 6470, 6471, 6472, 6473, 6474, 6475, 6476, 6477, 6480, 6481, 6482, 6483, 6484, 6485, 6486, 6488, 6489, 6490, 6491, 6492, 6493, 6494, 6496, 6497, 6498, 6499, 6500, 6501, 6502, 6503, 6504, 6505, 6506, 6507, 6508, 6509, 6510, 6511, 6512, 6513, 6514, 6515, 6516, 6517, 6518, 6521, 6522, 6523, 6524, 6525, 6526, 6527, 6528, 6529, 6530, 6531, 6532, 6533, 6534, 6535, 6536, 6537, 6538, 6539, 6540, 6541, 6542, 6543, 6544, 6545, 6548, 6549, 6550, 6551, 6552, 6553, 6554, 6555, 6556, 6557, 6558, 6561, 6562, 6563, 6565, 6566, 6567, 6568, 6569, 6570, 6571, 6572, 6573, 6574, 6575, 6576, 6578, 6579, 6580, 6581, 6582, 6583, 6584, 6585, 6586, 6587, 6588, 6589, 6590, 6591, 6592, 6593, 6594, 6595, 6596, 6597, 6598, 6599, 6600, 6601, 6603, 6604, 6605, 6606, 6607, 6608, 6609, 6610, 6611, 6612, 6613, 6614, 6615, 6617, 6618, 6619, 6621, 6624, 6625, 6627, 6628, 6629, 6630, 6631, 6632, 6633, 6634, 6635, 6636, 6638, 6639, 6640, 6641, 6642, 6643, 6644, 6645, 6646, 6647, 6649, 6650, 6651, 6652, 6653, 6654, 6655, 6658, 6659, 6660, 6661, 6662, 6663, 6664, 6665, 6666, 6668, 6669, 6670, 6671, 6672, 6673, 6675, 6676, 6678, 6679, 6681, 6682, 6683, 6684, 6686, 6687, 6688, 6690, 6691, 6692, 6693, 6694, 6695, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6725, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6736, 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6773, 6774, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 6784, 6785, 6786, 6788, 6789, 6790, 6791, 6792, 6793, 6794, 6795, 6796, 6797, 6798, 6799, 6800, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813, 6814, 6816, 6817, 6818, 6819, 6820, 6821, 6822, 6823, 6824, 6825, 6826, 6827, 6828, 6830, 6831, 6832, 6833, 6834, 6835, 6836, 6837, 6838, 6841, 6842, 6843, 6844, 6845, 6846, 6849, 6850, 6851, 6852, 6853, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, 6864, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, 6888, 6889, 6890, 6891, 6892, 6894, 6895, 6896, 6897, 6898, 6899, 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, 6912, 6913, 6914, 6915, 6916, 6917, 6918, 6919, 6920, 6921, 6922, 6923, 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, 6939, 6940, 6941, 6943, 6944, 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6971, 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, 6984, 6985, 6986, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7004, 7005, 7007, 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, 7029, 7030, 7031, 7032, 7033, 7036, 7038, 7039, 7040, 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, 7056, 7057, 7058, 7060, 7061, 7062, 7064, 7065, 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7077, 7078, 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, 7092, 7093, 7094, 7095, 7096, 7098, 7099, 7100, 7102, 7103, 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7114, 7115, 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, 7129, 7130, 7131, 7132, 7134, 7135, 7136, 7137, 7138, 7139, 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7148, 7149, 7150, 7151, 7152, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175, 7176, 7177, 7178, 7180, 7181, 7182, 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7193, 7194, 7195, 7196, 7197, 7198, 7199, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, 7236, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7254, 7256, 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, 7285, 7286, 7287, 7288, 7289, 7291, 7292, 7293, 7294, 7295, 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7326, 7327, 7328, 7329, 7330, 7331, 7332, 7333, 7334, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, 7345, 7346, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7364, 7365, 7366, 7367, 7368, 7370, 7371, 7372, 7373, 7374, 7375, 7376, 7377, 7378, 7379, 7380, 7381, 7382, 7384, 7385, 7386, 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7398, 7399, 7400, 7402, 7403, 7405, 7406, 7407, 7408, 7409, 7410, 7411, 7412, 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, 7423, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7433, 7434, 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, 7471, 7472, 7473, 7475, 7476, 7478, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7501, 7502, 7503, 7504, 7505, 7506, 7508, 7510, 7511, 7512, 7513, 7514, 7515, 7518, 7519, 7520, 7521, 7522, 7523, 7525, 7526, 7527, 7528, 7529, 7530, 7531, 7532, 7533, 7534, 7535, 7536, 7537, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, 7551, 7552, 7553, 7554, 7556, 7557, 7558, 7559, 7560, 7561, 7562, 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, 7575, 7576, 7577, 7578, 7579, 7580, 7581, 7582, 7583, 7584, 7585, 7587, 7588, 7589, 7590, 7591, 7592, 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605, 7606, 7607, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, 7617, 7618, 7619, 7620, 7622, 7623, 7624, 7625, 7626, 7628, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7640, 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, 7677, 7678, 7679, 7680, 7681, 7682, 7684, 7685, 7686, 7687, 7688, 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7699, 7700, 7701, 7702, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, 7773, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, 7785, 7786, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7809, 7810, 7811, 7812, 7813, 7814, 7816, 7817, 7818, 7819, 7820, 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833, 7834, 7835, 7836, 7837, 7839, 7840, 7841, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861, 7862, 7863, 7864, 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873, 7874, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 7936, 7937, 7938, 7939, 7940, 7941, 7942, 7944, 7946, 7947, 7948, 7949, 7951, 7952, 7953, 7954, 7956, 7957, 7958, 7959, 7960, 7961, 7962, 7963, 7965, 7966, 7967, 7968, 7969, 7970, 7971, 7972, 7973, 7974, 7975, 7976, 7977, 7978, 7979, 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7993, 7994, 7995, 7996, 7997, 7998, 8000, 8001, 8002, 8003, 8004, 8005, 8006, 8007, 8008, 8009, 8010, 8011, 8012, 8014, 8015, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8028, 8029, 8030, 8031, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8040, 8042, 8043, 8044, 8046, 8047, 8048, 8049, 8050, 8051, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8061, 8062, 8063, 8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081, 8083, 8084, 8085, 8086, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, 8112, 8113, 8115, 8116, 8117, 8118, 8119, 8120, 8123, 8124, 8125, 8126, 8127, 8128, 8129, 8130, 8131, 8132, 8133, 8134, 8135, 8136, 8137, 8138, 8139, 8140, 8142, 8143, 8144, 8145, 8146, 8147, 8148, 8149, 8150, 8152, 8153, 8154, 8155, 8157, 8158, 8159, 8160, 8161, 8162, 8163, 8164, 8165, 8166, 8168, 8170, 8171, 8172, 8173, 8174, 8175, 8176, 8177, 8178, 8179, 8180, 8181, 8182, 8183, 8184, 8185, 8186, 8187, 8188, 8189, 8190, 8191, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200, 8201, 8202, 8203, 8204, 8205, 8206, 8207, 8208, 8209, 8210, 8211, 8212, 8213, 8214, 8215, 8216, 8217, 8218, 8219, 8220, 8221, 8222, 8223, 8224, 8225, 8226, 8227, 8228, 8229, 8230, 8231, 8232, 8233, 8234, 8235, 8236, 8237, 8238, 8239, 8240, 8241, 8242, 8243, 8244, 8245, 8246, 8248, 8249, 8250, 8251, 8252, 8253, 8255, 8256, 8257, 8258, 8259, 8260, 8262, 8265, 8266, 8267, 8268, 8269, 8270, 8271, 8272, 8273, 8274, 8275, 8276, 8277, 8278, 8279, 8280, 8281, 8282, 8283, 8284, 8285, 8286, 8287, 8288, 8289, 8290, 8291, 8292, 8293, 8294, 8295, 8296, 8297, 8298, 8299, 8300, 8301, 8302, 8303, 8304, 8305, 8306, 8307, 8308, 8309, 8311, 8312, 8313, 8314, 8315, 8316, 8317, 8319, 8320, 8321, 8322, 8323, 8324, 8325, 8326, 8327, 8328, 8329, 8330, 8331, 8332, 8333, 8334, 8335, 8336, 8337, 8338, 8339, 8340, 8341, 8342, 8343, 8344, 8345, 8346, 8348, 8349, 8350, 8351, 8352, 8353, 8354, 8355, 8356, 8357, 8358, 8359, 8360, 8361, 8362, 8363, 8364, 8365, 8366, 8367, 8368, 8369, 8370, 8371, 8372, 8373, 8374, 8375, 8376, 8377, 8378, 8379, 8380, 8381, 8382, 8383, 8384, 8385, 8386, 8387, 8388, 8389, 8390, 8391, 8392, 8393, 8394, 8395, 8396, 8397, 8398, 8399, 8400, 8401, 8402, 8404, 8405, 8407, 8408, 8409, 8410, 8411, 8412, 8413, 8414, 8415, 8416, 8417, 8418, 8419, 8420, 8421, 8422, 8423, 8424, 8425, 8427, 8428, 8429, 8430, 8431, 8433, 8434, 8435, 8436, 8437, 8439, 8440, 8441, 8442, 8443, 8444, 8446, 8448, 8449, 8450, 8451, 8452, 8453, 8454, 8455, 8456, 8457, 8458, 8459, 8460, 8461, 8462, 8463, 8464, 8465, 8466, 8467, 8468, 8469, 8470, 8471, 8472, 8473, 8474, 8475, 8476, 8477, 8478, 8480, 8481, 8482, 8483, 8484, 8485, 8487, 8488, 8489, 8490, 8491, 8492, 8493, 8494, 8495, 8496, 8497, 8498, 8499, 8501, 8502, 8503, 8504, 8505, 8506, 8507, 8508, 8509, 8510, 8512, 8514, 8515, 8516, 8517, 8518, 8519, 8520, 8521, 8523, 8524, 8525, 8526, 8527, 8528, 8529, 8530, 8531, 8532, 8533, 8534, 8535, 8536, 8537, 8538, 8539, 8540, 8541, 8542, 8543, 8544, 8546, 8547, 8549, 8550, 8551, 8552, 8553, 8554, 8555, 8557, 8558, 8559, 8560, 8561, 8562, 8563, 8564, 8568, 8569, 8570, 8571, 8572, 8574, 8575, 8576, 8577, 8578, 8579, 8580, 8581, 8582, 8583, 8584, 8585, 8586, 8587, 8588, 8589, 8591, 8592, 8594, 8595, 8596, 8597, 8598, 8599, 8600, 8601, 8602, 8603, 8604, 8605, 8606, 8607, 8608, 8609, 8610, 8611, 8612, 8613, 8614, 8615, 8616, 8617, 8618, 8619, 8620, 8621, 8622, 8623, 8624, 8625, 8626, 8628, 8629, 8630, 8632, 8633, 8634, 8635, 8636, 8637, 8638, 8639, 8640, 8641, 8642, 8643, 8644, 8645, 8646, 8647, 8648, 8649, 8650, 8651, 8654, 8655, 8656, 8657, 8658, 8659, 8660, 8661, 8662, 8663, 8664, 8665, 8666, 8667, 8668, 8669, 8670, 8672, 8673, 8674, 8675, 8676, 8677, 8678, 8679, 8680, 8681, 8682, 8684, 8685, 8686, 8687, 8688, 8689, 8690, 8691, 8693, 8694, 8695, 8696, 8697, 8698, 8699, 8700, 8701, 8702, 8704, 8705, 8706, 8707, 8709, 8710, 8711, 8712, 8714, 8715, 8716, 8717, 8718, 8719, 8720, 8721, 8722, 8723, 8724, 8725, 8726, 8727, 8728, 8729, 8730, 8731, 8733, 8734, 8735, 8736, 8737, 8739, 8740, 8741, 8742, 8743, 8744, 8745, 8747, 8748, 8749, 8750, 8751, 8752, 8753, 8755, 8756, 8757, 8758, 8759, 8760, 8761, 8764, 8765, 8766, 8767, 8768, 8769, 8770, 8771, 8772, 8773, 8774, 8775, 8776, 8777, 8778, 8780, 8781, 8782, 8783, 8784, 8785, 8786, 8787, 8788, 8789, 8790, 8791, 8792, 8793, 8794, 8795, 8796, 8797, 8798, 8800, 8801, 8802, 8803, 8804, 8805, 8807, 8808, 8809, 8810, 8811, 8812, 8813, 8814, 8815, 8816, 8817, 8818, 8819, 8820, 8821, 8822, 8823, 8824, 8825, 8827, 8828, 8829, 8830, 8832, 8833, 8834, 8835, 8836, 8837, 8838, 8839, 8840, 8841, 8842, 8843, 8844, 8845, 8846, 8847, 8848, 8849, 8850, 8851, 8852, 8853, 8854, 8855, 8856, 8857, 8858, 8859, 8861, 8862, 8863, 8864, 8865, 8866, 8867, 8869, 8870, 8871, 8872, 8873, 8874, 8876, 8877, 8878, 8879, 8880, 8881, 8882, 8884, 8885, 8886, 8887, 8888, 8889, 8890, 8891, 8893, 8895, 8896, 8897, 8898, 8900, 8901, 8902, 8903, 8904, 8905, 8906, 8907, 8908, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8918, 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8927, 8928, 8929, 8930, 8931, 8932, 8933, 8934, 8935, 8936, 8937, 8938, 8939, 8940, 8941, 8942, 8943, 8944, 8945, 8946, 8947, 8948, 8949, 8950, 8951, 8952, 8953, 8954, 8955, 8956, 8957, 8958, 8959, 8960, 8961, 8962, 8963, 8964, 8965, 8966, 8967, 8968, 8969, 8970, 8971, 8972, 8974, 8975, 8976, 8977, 8978, 8979, 8982, 8984, 8985, 8986, 8987, 8988, 8989, 8990, 8991, 8993, 8994, 8995, 8996, 8997, 8998, 8999, 9000, 9001, 9002, 9003, 9004, 9005, 9006, 9007, 9008, 9009, 9010, 9011, 9012, 9013, 9014, 9015, 9016, 9017, 9018, 9019, 9020, 9021, 9022, 9023, 9025, 9026, 9028, 9029, 9030, 9031, 9032, 9033, 9034, 9035, 9036, 9038, 9039, 9040, 9042, 9043, 9044, 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9054, 9055, 9056, 9057, 9058, 9059, 9060, 9061, 9062, 9063, 9064, 9065, 9067, 9068, 9069, 9070, 9072, 9073, 9074, 9075, 9076, 9077, 9078, 9079, 9080, 9081, 9082, 9083, 9085, 9086, 9087, 9088, 9089, 9091, 9092, 9093, 9094, 9096, 9097, 9098, 9099, 9100, 9101, 9102, 9103, 9104, 9105, 9106, 9107, 9109, 9110, 9112, 9113, 9114, 9115, 9117, 9118, 9119, 9120, 9121, 9122, 9123, 9124, 9125, 9126, 9127, 9128, 9129, 9130, 9131, 9132, 9133, 9134, 9136, 9137, 9138, 9139, 9140, 9141, 9143, 9144, 9145, 9146, 9147, 9149, 9150, 9151, 9153, 9154, 9155, 9156, 9157, 9158, 9159, 9161, 9162, 9163, 9165, 9166, 9167, 9168, 9169, 9170, 9171, 9172, 9173, 9174, 9175, 9176, 9177, 9179, 9180, 9181, 9182, 9183, 9184, 9185, 9186, 9187, 9188, 9189, 9190, 9191, 9192, 9193, 9194, 9195, 9196, 9197, 9198, 9199, 9200, 9202, 9203, 9204, 9205, 9206, 9207, 9208, 9209, 9210, 9211, 9212, 9213, 9214, 9215, 9216, 9217, 9218, 9219, 9220, 9221, 9222, 9223, 9224, 9225, 9226, 9227, 9228, 9229, 9230, 9231, 9232, 9233, 9234, 9235, 9236, 9237, 9238, 9239, 9241, 9242, 9243, 9244, 9245, 9248, 9249, 9250, 9253, 9254, 9255, 9256, 9257, 9258, 9259, 9261, 9262, 9263, 9264, 9265, 9266, 9268, 9269, 9270, 9271, 9272, 9273, 9274, 9276, 9277, 9278, 9279, 9280, 9281, 9282, 9283, 9285, 9286, 9287, 9288, 9290, 9291, 9292, 9293, 9294, 9295, 9297, 9298, 9299, 9300, 9301, 9302, 9303, 9304, 9305, 9306, 9307, 9308, 9310, 9311, 9312, 9313, 9314, 9315, 9316, 9317, 9318, 9319, 9320, 9321, 9322, 9323, 9324, 9325, 9326, 9327, 9329, 9330, 9331, 9332, 9333, 9334, 9336, 9337, 9338, 9340, 9341, 9342, 9343, 9344, 9345, 9346, 9347, 9348, 9349, 9350, 9351, 9352, 9353, 9354, 9355, 9356, 9357, 9358, 9361, 9362, 9363, 9364, 9365, 9366, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9381, 9382, 9383, 9384, 9385, 9387, 9389, 9390, 9391, 9393, 9395, 9396, 9397, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9449, 9450, 9451, 9452, 9453, 9454, 9455, 9456, 9457, 9458, 9459, 9460, 9461, 9462, 9463, 9464, 9465, 9466, 9467, 9468, 9469, 9470, 9471, 9472, 9473, 9474, 9475, 9476, 9477, 9478, 9479, 9480, 9481, 9482, 9483, 9484, 9485, 9486, 9490, 9491, 9492, 9493, 9494, 9495, 9496, 9497, 9498, 9499, 9500, 9501, 9502, 9503, 9505, 9506, 9507, 9508, 9509, 9510, 9511, 9512, 9514, 9515, 9516, 9517, 9518, 9519, 9520, 9521, 9522, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9532, 9533, 9534, 9535, 9536, 9537, 9538, 9539, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550, 9552, 9554, 9555, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9564, 9565, 9566, 9567, 9568, 9569, 9570, 9571, 9572, 9573, 9575, 9577, 9578, 9579, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9591, 9592, 9593, 9594, 9595, 9596, 9597, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606, 9607, 9608, 9609, 9610, 9611, 9612, 9613, 9614, 9615, 9616, 9617, 9618, 9619, 9620, 9621, 9623, 9625, 9626, 9627, 9628, 9629, 9630, 9631, 9632, 9633, 9634, 9635, 9637, 9638, 9639, 9640, 9641, 9642, 9644, 9645, 9646, 9647, 9648, 9649, 9651, 9652, 9653, 9655, 9656, 9657, 9658, 9659, 9660, 9661, 9662, 9663, 9664, 9666, 9667, 9668, 9669, 9670, 9671, 9672, 9673, 9675, 9676, 9677, 9679, 9680, 9681, 9682, 9683, 9684, 9685, 9686, 9687, 9688, 9689, 9691, 9692, 9693, 9694, 9695, 9696, 9697, 9699, 9700, 9701, 9702, 9703, 9704, 9705, 9706, 9707, 9708, 9709, 9710, 9711, 9712, 9713, 9714, 9715, 9716, 9717, 9718, 9719, 9720, 9721, 9722, 9723, 9724, 9726, 9727, 9728, 9729, 9730, 9731, 9732, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9742, 9743, 9746, 9747, 9748, 9749, 9750, 9751, 9752, 9753, 9754, 9756, 9757, 9758, 9759, 9760, 9761, 9762, 9763, 9764, 9765, 9766, 9767, 9768, 9769, 9770, 9771, 9772, 9773, 9774, 9775, 9776, 9777, 9778, 9779, 9780, 9781, 9783, 9784, 9785, 9787, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, 9800, 9801, 9802, 9803, 9804, 9805, 9806, 9808, 9809, 9810, 9811, 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820, 9821, 9822, 9823, 9824, 9825, 9826, 9827, 9828, 9829, 9830, 9831, 9832, 9833, 9834, 9835, 9836, 9837, 9838, 9839, 9840, 9841, 9842, 9843, 9844, 9845, 9848, 9849, 9850, 9851, 9852, 9853, 9855, 9856, 9857, 9858, 9860, 9861, 9862, 9863, 9864, 9865, 9866, 9867, 9868, 9869, 9870, 9871, 9872, 9873, 9874, 9875, 9876, 9878, 9879, 9880, 9881, 9882, 9883, 9884, 9885, 9886, 9887, 9888, 9889, 9890, 9892, 9893, 9894, 9896, 9897, 9898, 9899, 9900, 9901, 9902, 9903, 9904, 9905, 9906, 9907, 9908, 9910, 9911, 9912, 9913, 9914, 9915, 9916, 9917, 9918, 9919, 9920, 9921, 9922, 9923, 9924, 9925, 9928, 9929, 9930, 9933, 9936, 9938, 9939, 9940, 9941, 9942, 9943, 9944, 9947, 9948, 9949, 9950, 9951, 9952, 9953, 9954, 9955, 9957, 9959, 9960, 9961, 9962, 9964, 9966, 9967, 9968, 9969, 9970, 9971, 9972, 9973, 9974, 9976, 9977, 9978, 9980, 9981, 9982, 9983, 9984, 9985, 9986, 9987, 9988, 9990, 9991, 9992, 9993, 9994, 9995, 9996, 9997, 9998, 9999, 10000, 10001, 10002, 10003, 10004, 10006, 10007, 10008, 10009, 10010, 10011, 10012, 10013, 10014, 10015, 10018, 10019, 10020, 10021, 10022, 10023, 10025, 10026, 10028, 10029, 10030, 10031, 10033, 10034, 10035, 10037, 10039, 10040, 10041, 10042, 10043, 10044, 10045, 10046, 10047, 10048, 10049, 10050, 10051, 10053, 10054, 10055, 10056, 10057, 10058, 10059, 10060, 10061, 10062, 10063, 10064, 10065, 10066, 10067, 10068, 10069, 10072, 10073, 10074, 10075, 10076, 10077, 10078, 10079, 10080, 10081, 10082, 10084, 10085, 10086, 10087, 10089, 10092, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10110, 10111, 10112, 10113, 10114, 10115, 10116, 10117, 10118, 10119, 10120, 10121, 10122, 10123, 10124, 10125, 10126, 10127, 10128, 10129, 10130, 10131, 10133, 10134, 10135, 10136, 10137, 10138, 10139, 10140, 10141, 10143, 10144, 10145, 10146, 10148, 10151, 10152, 10153, 10154, 10156, 10157, 10158, 10159, 10160, 10161, 10162, 10163, 10164, 10166, 10168, 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185, 10186, 10187, 10188, 10189, 10190, 10191, 10192, 10193, 10194, 10195, 10196, 10197, 10199, 10200, 10201, 10202, 10203, 10204, 10207, 10208, 10209, 10210, 10212, 10213, 10214, 10215, 10216, 10217, 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10225, 10226, 10227, 10228, 10229, 10230, 10231, 10232, 10233, 10234, 10235, 10237, 10238, 10239, 10240, 10241, 10242, 10243, 10244, 10245, 10247, 10248, 10249, 10250, 10251, 10252, 10253, 10254, 10255, 10257, 10258, 10259, 10260, 10262, 10263, 10264, 10265, 10266, 10268, 10269, 10270, 10271, 10272, 10273, 10274, 10275, 10276, 10277, 10278, 10279, 10280, 10281, 10282, 10283, 10284, 10285, 10286, 10287, 10288, 10289, 10290, 10291, 10292, 10293, 10294, 10295, 10296, 10297, 10298, 10299, 10300, 10301, 10302, 10303, 10304, 10305, 10307, 10308, 10309, 10310, 10311, 10312, 10313, 10314, 10315, 10317, 10318, 10319, 10320, 10323, 10324, 10325, 10327, 10328, 10329, 10330, 10331, 10332, 10333, 10334, 10335, 10336, 10337, 10339, 10340, 10341, 10342, 10343, 10344, 10345, 10346, 10347, 10348, 10349, 10350, 10351, 10353, 10354, 10355, 10356, 10357, 10358, 10359, 10361, 10362, 10363, 10364, 10365, 10366, 10367, 10368, 10369, 10370, 10371, 10372, 10373, 10374, 10375, 10376, 10378, 10380, 10381, 10382, 10383, 10384, 10385, 10386, 10387, 10388, 10389, 10390, 10391, 10392, 10393, 10394, 10395, 10396, 10397, 10398, 10400, 10403, 10404, 10405, 10406, 10407, 10408, 10410, 10411, 10412, 10413, 10414, 10415, 10416, 10417, 10418, 10419, 10420, 10421, 10422, 10423, 10424, 10425, 10426, 10427, 10428, 10430, 10431, 10432, 10434, 10435, 10436, 10437, 10439, 10440, 10441, 10442, 10443, 10444, 10445, 10446, 10447, 10448, 10449, 10450, 10451, 10452, 10454, 10455, 10456, 10458, 10459, 10460, 10461, 10462, 10463, 10464, 10466, 10468, 10469, 10470, 10471, 10472, 10473, 10474, 10475, 10476, 10477, 10478, 10479, 10480, 10482, 10483, 10484, 10486, 10487, 10488, 10489, 10490, 10491, 10492, 10493, 10494, 10495, 10496, 10497, 10499, 10500, 10501, 10502, 10503, 10504, 10505, 10506, 10507, 10508, 10509, 10510, 10511, 10512, 10513, 10514, 10515, 10516, 10518, 10519, 10520, 10522, 10524, 10525, 10526, 10527, 10528, 10529, 10530, 10531, 10532, 10533, 10534, 10535, 10537, 10538, 10539, 10540, 10541, 10542, 10543, 10544, 10545, 10546, 10547, 10548, 10549, 10551, 10552, 10553, 10554, 10555, 10556, 10557, 10558, 10559, 10560, 10561, 10562, 10563, 10565, 10566, 10567, 10568, 10569, 10570, 10571, 10572, 10573, 10574, 10575, 10576, 10577, 10579, 10580, 10581, 10582, 10583, 10584, 10585, 10586, 10587, 10589, 10590, 10591, 10595, 10596, 10597, 10598, 10599, 10600, 10601, 10602, 10603, 10605, 10606, 10607, 10608, 10609, 10610, 10611, 10614, 10615, 10616, 10617, 10618, 10619, 10620, 10621, 10622, 10623, 10624, 10625, 10626, 10627, 10628, 10629, 10631, 10633, 10634, 10635, 10636, 10637, 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, 10648, 10649, 10650, 10651, 10652, 10653, 10654, 10655, 10657, 10658, 10660, 10661, 10662, 10663, 10664, 10665, 10666, 10667, 10668, 10669, 10670, 10671, 10672, 10673, 10674, 10675, 10676, 10677, 10678, 10679, 10680, 10681, 10682, 10683, 10684, 10685, 10686, 10687, 10688, 10689, 10690, 10691, 10692, 10693, 10694, 10696, 10697, 10698, 10699, 10700, 10701, 10702, 10703, 10704, 10705, 10706, 10707, 10708, 10709, 10711, 10712, 10713, 10714, 10715, 10716, 10717, 10718, 10719, 10720, 10721, 10722, 10723, 10724, 10725, 10726, 10727, 10728, 10729, 10730, 10731, 10732, 10733, 10734, 10735, 10736, 10737, 10738, 10739, 10740, 10741, 10742, 10743, 10744, 10746, 10747, 10748, 10750, 10751, 10752, 10753, 10754, 10755, 10756, 10757, 10758, 10759, 10760, 10761, 10762, 10763, 10764, 10765, 10766, 10767, 10768, 10769, 10770, 10771, 10773, 10774, 10775, 10776, 10777, 10778, 10779, 10780, 10781, 10782, 10783, 10784, 10785, 10786, 10787, 10788, 10789, 10790, 10791, 10792, 10793, 10794, 10795, 10796, 10797, 10798, 10799, 10800, 10801, 10802, 10803, 10804, 10805, 10806, 10807, 10808, 10809, 10811, 10812, 10813, 10814, 10815, 10816, 10817, 10818, 10819, 10820, 10821, 10822, 10823, 10825, 10826, 10827, 10828, 10829, 10830, 10831, 10832, 10833, 10834, 10836, 10837, 10838, 10839, 10840, 10841, 10842, 10843, 10844, 10846, 10847, 10848, 10849, 10850, 10851, 10853, 10854, 10855, 10856, 10857, 10858, 10859, 10861, 10862, 10863, 10864, 10865, 10866, 10867, 10868, 10869, 10870, 10871, 10872, 10873, 10874, 10875, 10878, 10879, 10880, 10883, 10885, 10886, 10887, 10889, 10890, 10891, 10892, 10893, 10894, 10895, 10896, 10897, 10898, 10899, 10900, 10901, 10902, 10903, 10904, 10905, 10906, 10907, 10908, 10909, 10910, 10911, 10912, 10913, 10914, 10915, 10916, 10917, 10918, 10919, 10920, 10921, 10922, 10923, 10924, 10925, 10926, 10927, 10928, 10929, 10931, 10933, 10934, 10935, 10936, 10937, 10938, 10939, 10941, 10942, 10943, 10945, 10946, 10947, 10949, 10950, 10951, 10952, 10953, 10954, 10955, 10957, 10958, 10959, 10960, 10962, 10963, 10964, 10966, 10967, 10968, 10969, 10970, 10971, 10973, 10974, 10975, 10976, 10977, 10978, 10979, 10980, 10981, 10982, 10983, 10984, 10986, 10990, 10991, 10992, 10993, 10994, 10995, 10996, 10999, 11000, 11001, 11003, 11006, 11008, 11009, 11010, 11011, 11012, 11013, 11015, 11016, 11017, 11018, 11019, 11020, 11021, 11022, 11023, 11024, 11025, 11026, 11027, 11028, 11029, 11030, 11031, 11032, 11033, 11034, 11035, 11036, 11037, 11038, 11039, 11040, 11041, 11042, 11043, 11044, 11046, 11048, 11049, 11050, 11051, 11052, 11053, 11054, 11055, 11056, 11057, 11058, 11059, 11060, 11061, 11062, 11063, 11064, 11065, 11066, 11067, 11068, 11069, 11070, 11071, 11072, 11073, 11074, 11075, 11077, 11079, 11080, 11081, 11082, 11083, 11084, 11085, 11086, 11087, 11088, 11089, 11090, 11091, 11092, 11093, 11094, 11095, 11096, 11097, 11098, 11099, 11100, 11101, 11102, 11103, 11104, 11105, 11106, 11107, 11109, 11110, 11111, 11112, 11113, 11115, 11116, 11117, 11118, 11119, 11120, 11121, 11122, 11123, 11124, 11125, 11126, 11129, 11130, 11131, 11132, 11133, 11134, 11135, 11136, 11137, 11138, 11139, 11140, 11141, 11142, 11145, 11146, 11147, 11148, 11149, 11150, 11151, 11152, 11153, 11154, 11155, 11156, 11157, 11158, 11159, 11160, 11161, 11162, 11163, 11164, 11165, 11166, 11167, 11168, 11169, 11170, 11171, 11172, 11173, 11175, 11176, 11177, 11178, 11179, 11180, 11181, 11182, 11183, 11184, 11185, 11186, 11187, 11188, 11189, 11190, 11191, 11192, 11193, 11194, 11195, 11196, 11200, 11202, 11203, 11204, 11205, 11209, 11210, 11211, 11213, 11214, 11215, 11216, 11217, 11218, 11219, 11220, 11221, 11222, 11223, 11224, 11225, 11226, 11227, 11228, 11229, 11230, 11231, 11232, 11233, 11234, 11235, 11236, 11237, 11238, 11240, 11241, 11242, 11244, 11245, 11246, 11247, 11248, 11249, 11250, 11251, 11252, 11255, 11256, 11257, 11258, 11259, 11260, 11261, 11262, 11263, 11264, 11265, 11266, 11267, 11268, 11270, 11271, 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11283, 11284, 11286, 11288, 11289, 11290, 11292, 11293, 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, 11304, 11306, 11307, 11308, 11309, 11310, 11311, 11314, 11316, 11317, 11318, 11319, 11320, 11321, 11322, 11323, 11324, 11325, 11326, 11327, 11328, 11329, 11330, 11331, 11332, 11333, 11334, 11335, 11336, 11337, 11338, 11339, 11340, 11341, 11342, 11343, 11345, 11346, 11347, 11348, 11349, 11350, 11351, 11352, 11354, 11355, 11356, 11358, 11361, 11362, 11363, 11364, 11365, 11366, 11367, 11373, 11374, 11375, 11376, 11378, 11379, 11380, 11381, 11382, 11383, 11384, 11385, 11386, 11387, 11388, 11389, 11390, 11391, 11392, 11394, 11396, 11397, 11398, 11399, 11401, 11402, 11403, 11404, 11405, 11406, 11407, 11408, 11409, 11410, 11411, 11412, 11413, 11414, 11415, 11416, 11417, 11418, 11419, 11420, 11421, 11422, 11423, 11424, 11425, 11426, 11427, 11428, 11429, 11430, 11431, 11432, 11433, 11434, 11435, 11436, 11437, 11438, 11439, 11440, 11441, 11442, 11444, 11446, 11447, 11448, 11449, 11450, 11451, 11452, 11453, 11454, 11455, 11456, 11457, 11458, 11459, 11460, 11461, 11462, 11463, 11465, 11466, 11468, 11469, 11470, 11471, 11472, 11473, 11474, 11475, 11476, 11477, 11478, 11479, 11480, 11481, 11482, 11483, 11484, 11485, 11486, 11488, 11489, 11490, 11491, 11493, 11496, 11497, 11498, 11499, 11500, 11501, 11502, 11503, 11504, 11505, 11506, 11507, 11508, 11509, 11510, 11511, 11516, 11517, 11518, 11519, 11521, 11522, 11523, 11524, 11526, 11527, 11528, 11529, 11530, 11531, 11532, 11533, 11534, 11535, 11536, 11537, 11538, 11539, 11540, 11541, 11542, 11543, 11548, 11549, 11550, 11551, 11552, 11553, 11554, 11555, 11556, 11557, 11558, 11559, 11560, 11561, 11562, 11563, 11564, 11565, 11566, 11567, 11568, 11569, 11570, 11571, 11572, 11573, 11574, 11575, 11576, 11577, 11578, 11579, 11581, 11582, 11583, 11584, 11585, 11586, 11587, 11588, 11591, 11592, 11593, 11594, 11595, 11596, 11597, 11598, 11601, 11604, 11606, 11607, 11608, 11609, 11610, 11613, 11614, 11615, 11618, 11619, 11620, 11621, 11622, 11623, 11624, 11625, 11626, 11627, 11629, 11630, 11631, 11632, 11633, 11634, 11635, 11636, 11637, 11638, 11639, 11640, 11641, 11642, 11643, 11644, 11645, 11646, 11647, 11648, 11649, 11650, 11652, 11653, 11654, 11655, 11656, 11657, 11659, 11660, 11661, 11662, 11664, 11665, 11666, 11667, 11668, 11669, 11670, 11671, 11672, 11673, 11675, 11676, 11679, 11681, 11682, 11683, 11684, 11686, 11687, 11688, 11689, 11690, 11691, 11693, 11694, 11695, 11696, 11697, 11698, 11699, 11701, 11702, 11703, 11704, 11705, 11706, 11707, 11708, 11709, 11710, 11713, 11714, 11715, 11716, 11718, 11719, 11720, 11721, 11722, 11723, 11724, 11725, 11726, 11727, 11728, 11729, 11730, 11731, 11733, 11734, 11735, 11736, 11737, 11738, 11739, 11740, 11741, 11742, 11743, 11744, 11745, 11746, 11747, 11748, 11749, 11750, 11751, 11752, 11754, 11755, 11756, 11757, 11758, 11759, 11761, 11762, 11763, 11764, 11765, 11766, 11767, 11768, 11769, 11770, 11771, 11772, 11773, 11774, 11775, 11776, 11777, 11778, 11779, 11780, 11781, 11782, 11783, 11784, 11785, 11788, 11789, 11790, 11791, 11792, 11793, 11794, 11795, 11796, 11797, 11798, 11799, 11800, 11801, 11802, 11803, 11804, 11805, 11806, 11807, 11808, 11809, 11810, 11811, 11812, 11813, 11814, 11815, 11816, 11817, 11818, 11819, 11820, 11821, 11822, 11823, 11824, 11825, 11826, 11827, 11828, 11829, 11830, 11831, 11833, 11834, 11835, 11836, 11837, 11838, 11839, 11840, 11841, 11842, 11843, 11844, 11845, 11846, 11847, 11848, 11849, 11850, 11851, 11852, 11853, 11854, 11856, 11857, 11858, 11859, 11860, 11862, 11863, 11864, 11865, 11866, 11867, 11868, 11869, 11870, 11871, 11872, 11873, 11874, 11875, 11877, 11878, 11879, 11880, 11881, 11882, 11883, 11884, 11886, 11887, 11888, 11889, 11890, 11892, 11893, 11894, 11895, 11896, 11897, 11899, 11900, 11901, 11902, 11903, 11904, 11905, 11906, 11907, 11909, 11910, 11911, 11912, 11913, 11914, 11915, 11916, 11917, 11918, 11919, 11920, 11921, 11922, 11923, 11924, 11925, 11926, 11927, 11928, 11930, 11931, 11932, 11933, 11934, 11936, 11937, 11938, 11939, 11941, 11942, 11943, 11944, 11946, 11947, 11948, 11950, 11951, 11952, 11953, 11954, 11955, 11956, 11957, 11958, 11960, 11962, 11964, 11965, 11966, 11967, 11968, 11969, 11970, 11971, 11972, 11974, 11975, 11976, 11977, 11978, 11979, 11980, 11981, 11982, 11983, 11984, 11985, 11988, 11989, 11990, 11991, 11992, 11993, 11995, 11996, 11998, 12000, 12001, 12003, 12004, 12005, 12006, 12007, 12008, 12009, 12010, 12011, 12012, 12013, 12014, 12015, 12016, 12017, 12018, 12019, 12020, 12021, 12022, 12024, 12025, 12027, 12028, 12029, 12030, 12031, 12032, 12035, 12036, 12037, 12038, 12039, 12040, 12041, 12042, 12043, 12044, 12045, 12047, 12048, 12049, 12050, 12052, 12053, 12054, 12055, 12056, 12057, 12058, 12059, 12060, 12061, 12062, 12063, 12064, 12065, 12066, 12067, 12068, 12069, 12071, 12072, 12073, 12074, 12075, 12077, 12078, 12079, 12080, 12081, 12082, 12083, 12084, 12085, 12086, 12087, 12089, 12090, 12091, 12092, 12093, 12094, 12095, 12096, 12097, 12098, 12099, 12100, 12101, 12102, 12103, 12104, 12105, 12106, 12107, 12108, 12109, 12110, 12111, 12112, 12113, 12114, 12115, 12116, 12117, 12118, 12119, 12120, 12121, 12122, 12123, 12124, 12125, 12126, 12127, 12128, 12129, 12130, 12131, 12132, 12133, 12134, 12135, 12136, 12137, 12138, 12139, 12140, 12141, 12142, 12143, 12144, 12145, 12147, 12148, 12149, 12150, 12151, 12152, 12155, 12156, 12157, 12158, 12159, 12160, 12163, 12164, 12165, 12166, 12168, 12169, 12170, 12171, 12172, 12173, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185, 12186, 12187, 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12195, 12196, 12198, 12199, 12200, 12202, 12203, 12204, 12205, 12207, 12209, 12210, 12212, 12214, 12215, 12216, 12217, 12218, 12219, 12220, 12221, 12222, 12223, 12224, 12225, 12226, 12227, 12228, 12230, 12231, 12232, 12233, 12234, 12235, 12236, 12237, 12239, 12240, 12242, 12243, 12244, 12245, 12246, 12247, 12248, 12249, 12250, 12251, 12252, 12253, 12254, 12255, 12256, 12257, 12258, 12259, 12260, 12261, 12262, 12264, 12266, 12267, 12268, 12269, 12270, 12271, 12272, 12273, 12274, 12275, 12276, 12277, 12279, 12280, 12281, 12282, 12283, 12284, 12285, 12286, 12288, 12289, 12291, 12292, 12293, 12294, 12295, 12296, 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12306, 12307, 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12316, 12317, 12318, 12319, 12320, 12321, 12322, 12324, 12325, 12326, 12327, 12328, 12329, 12331, 12332, 12334, 12335, 12336, 12337, 12338, 12339, 12340, 12343, 12344, 12345, 12346, 12347, 12349, 12350, 12351, 12352, 12353, 12354, 12356, 12357, 12358, 12359, 12360, 12361, 12362, 12363, 12365, 12366, 12367, 12368, 12369, 12370, 12371, 12373, 12374, 12375, 12376, 12377, 12378, 12379, 12381, 12382, 12383, 12384, 12385, 12387, 12388, 12389, 12390, 12391, 12393, 12395, 12396, 12397, 12398, 12399, 12400, 12401, 12402, 12403, 12406, 12407, 12408, 12409, 12410, 12411, 12412, 12413, 12415, 12416, 12417, 12418, 12419, 12420, 12421, 12422, 12423, 12424, 12425, 12426, 12427, 12429, 12430, 12431, 12432, 12433, 12434, 12435, 12437, 12438, 12439, 12440, 12441, 12442, 12444, 12445, 12446, 12447, 12448, 12449, 12450, 12451, 12452, 12453, 12454, 12455, 12456, 12457, 12458, 12459, 12460, 12461, 12462, 12463, 12464, 12465, 12466, 12467, 12468, 12470, 12471, 12472, 12473, 12474, 12475, 12476, 12477, 12479, 12480, 12481, 12482, 12483, 12484, 12485, 12486, 12487, 12488, 12489, 12490, 12491, 12492, 12493, 12494, 12495, 12496, 12497, 12498, 12499, 12500, 12501, 12502, 12503, 12504, 12505, 12506, 12509, 12510, 12511, 12512, 12513, 12514, 12515, 12516, 12517, 12520, 12521, 12522, 12525, 12526, 12527, 12528, 12529, 12530, 12531, 12532, 12534, 12535, 12536, 12537, 12538, 12539, 12540, 12541, 12542, 12543, 12544, 12545, 12546, 12547, 12548, 12549, 12550, 12551, 12552, 12554, 12556, 12557, 12558, 12559, 12560, 12561, 12562, 12563, 12564, 12565, 12566, 12567, 12568, 12569, 12570, 12571, 12572, 12573, 12574, 12575, 12576, 12577, 12578, 12579, 12580, 12581, 12582, 12583, 12584, 12585, 12586, 12587, 12588, 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12597, 12598, 12599, 12600, 12601, 12603, 12604, 12606, 12607, 12608, 12609, 12610, 12611, 12612, 12613, 12614, 12615, 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12624, 12625, 12626, 12627, 12628, 12629, 12630, 12631, 12632, 12633, 12634, 12635, 12636, 12637, 12638, 12639, 12640, 12641, 12642, 12643, 12644, 12645, 12647, 12648, 12649, 12650, 12651, 12652, 12654, 12655, 12656, 12657, 12658, 12659, 12660, 12661, 12662, 12664, 12665, 12666, 12667, 12668, 12669, 12670, 12671, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680, 12684, 12686, 12687, 12688, 12689, 12690, 12691, 12692, 12693, 12694, 12695, 12696, 12697, 12698, 12699, 12700, 12701, 12702, 12703, 12705, 12706, 12707, 12708, 12709, 12710, 12711, 12712, 12713, 12714, 12715, 12718, 12719, 12720, 12721, 12722, 12723, 12724, 12725, 12726, 12727, 12728, 12729, 12730, 12732, 12733, 12735, 12736, 12738, 12739, 12740, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749, 12750, 12751, 12752, 12753, 12754, 12755, 12756, 12758, 12759, 12760, 12761, 12762, 12764, 12765, 12766, 12767, 12768, 12769, 12770, 12771, 12772, 12773, 12775, 12776, 12777, 12778, 12779, 12780, 12781, 12782, 12783, 12784, 12785, 12786, 12788, 12789, 12790, 12791, 12793, 12794, 12795, 12796, 12797, 12798, 12799, 12800, 12801, 12802, 12803, 12807, 12808, 12809, 12810, 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818, 12819, 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, 12831, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849, 12851, 12852, 12853, 12855, 12856, 12857, 12858, 12859, 12860, 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, 12872, 12873, 12874, 12875, 12876, 12877, 12878, 12879, 12880, 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12889, 12890, 12891, 12892, 12893, 12894, 12895, 12896, 12897, 12898, 12900, 12901, 12902, 12903, 12904, 12905, 12906, 12907, 12908, 12909, 12910, 12911, 12912, 12913, 12914, 12916, 12917, 12918, 12919, 12920, 12921, 12922, 12923, 12924, 12925, 12926, 12927, 12928, 12929, 12930, 12931, 12932, 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945, 12946, 12947, 12948, 12949, 12950, 12952, 12954, 12956, 12957, 12958, 12959, 12960, 12961, 12962, 12963, 12964, 12965, 12967, 12968, 12969, 12970, 12971, 12973, 12974, 12975, 12977, 12978, 12979, 12980, 12981, 12982, 12983, 12984, 12985, 12986, 12987, 12988, 12989, 12990, 12991, 12992, 12994, 12995, 12997, 12998, 12999, 13000, 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010, 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, 13021, 13022, 13023, 13024, 13025, 13026, 13027, 13028, 13029, 13030, 13031, 13032, 13033, 13034, 13035, 13036, 13037, 13038, 13039, 13040, 13041, 13042, 13043, 13044, 13045, 13046, 13048, 13049, 13050, 13051, 13052, 13054, 13055, 13057, 13058, 13060, 13061, 13063, 13064, 13066, 13068, 13069, 13070, 13071, 13072, 13073, 13077, 13079, 13080, 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089, 13090, 13091, 13092, 13094, 13095, 13097, 13098, 13099, 13100, 13102, 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, 13111, 13112, 13114, 13116, 13117, 13118, 13119, 13120, 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128, 13129, 13130, 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13139, 13140, 13141, 13142, 13144, 13145, 13146, 13147, 13148, 13149, 13150, 13152, 13153, 13155, 13156, 13157, 13158, 13161, 13162, 13163, 13164, 13166, 13168, 13169, 13170, 13171, 13173, 13174, 13177, 13179, 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, 13191, 13192, 13195, 13196, 13197, 13198, 13199, 13200, 13202, 13203, 13204, 13206, 13207, 13208, 13211, 13212, 13213, 13214, 13215, 13218, 13219, 13220, 13221, 13222, 13223, 13224, 13225, 13226, 13228, 13229, 13230, 13231, 13232, 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, 13241, 13242, 13243, 13244, 13245, 13246, 13247, 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258, 13259, 13260, 13261, 13262, 13263, 13265, 13266, 13267, 13268, 13269, 13270, 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, 13281, 13282, 13283, 13284, 13285, 13286, 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295, 13297, 13298, 13299, 13300, 13301, 13304, 13305, 13306, 13310, 13311, 13312, 13313, 13314, 13315, 13316, 13318, 13319, 13321, 13322, 13323, 13324, 13325, 13326, 13327, 13328, 13329, 13330, 13333, 13334, 13335, 13336, 13338, 13341, 13342, 13343, 13344, 13345, 13346, 13347, 13348, 13352, 13353, 13354, 13355, 13356, 13357, 13358, 13359, 13360, 13363, 13364, 13365, 13366, 13368, 13369, 13370, 13371, 13372, 13373, 13374, 13375, 13376, 13377, 13378, 13379, 13380, 13381, 13383, 13385, 13386, 13387, 13388, 13389, 13390, 13391, 13393, 13395, 13397, 13398, 13399, 13400, 13401, 13402, 13403, 13404, 13406, 13407, 13408, 13409, 13410, 13411, 13413, 13415, 13416, 13418, 13420, 13422, 13423, 13424, 13425, 13426, 13427, 13428, 13429, 13430, 13431, 13432, 13433, 13434, 13435, 13436, 13437, 13438, 13439, 13442, 13448, 13449, 13451, 13452, 13454, 13456, 13457, 13459, 13460, 13461, 13462, 13464, 13465, 13467, 13468, 13469, 13470, 13471, 13472, 13473, 13474, 13475, 13477, 13478, 13479, 13480, 13481, 13482, 13483, 13484, 13485, 13486, 13487, 13489, 13490, 13491, 13492, 13493, 13494, 13495, 13496, 13497, 13498, 13499, 13500, 13502, 13503, 13505, 13507, 13508, 13509, 13510, 13512, 13513, 13514, 13515, 13516, 13519, 13520, 13521, 13522, 13523, 13524, 13525, 13526, 13527, 13528, 13529, 13530, 13531, 13532, 13533, 13534, 13535, 13536, 13537, 13538, 13539, 13540, 13542, 13543, 13544, 13545, 13547, 13548, 13549, 13550, 13551, 13552, 13553, 13554, 13555, 13556, 13557, 13559, 13560, 13562, 13563, 13564, 13565, 13566, 13567, 13568, 13569, 13570, 13571, 13572, 13573, 13574, 13575, 13576, 13577, 13578, 13580, 13581, 13582, 13583, 13584, 13585, 13586, 13587, 13588, 13589, 13590, 13593, 13594, 13596, 13597, 13598, 13601, 13604, 13605, 13607, 13609, 13610, 13611, 13612, 13613, 13614, 13615, 13616, 13617, 13618, 13619, 13620, 13622, 13623, 13626, 13627, 13628, 13629, 13630, 13631, 13633, 13634, 13635, 13636, 13637, 13638, 13639, 13640, 13641, 13642, 13643, 13644, 13645, 13647, 13648, 13650, 13651, 13652, 13653, 13654, 13655, 13656, 13658, 13659, 13661, 13662, 13663, 13664, 13665, 13666, 13667, 13668, 13669, 13670, 13671, 13672, 13673, 13674, 13676, 13677, 13678, 13679, 13680, 13681, 13682, 13683, 13685, 13686, 13688, 13689, 13690, 13691, 13692, 13693, 13694, 13695, 13696, 13697, 13698, 13699, 13700, 13701, 13702, 13703, 13704, 13705, 13707, 13708, 13709, 13710, 13711, 13712, 13713, 13714, 13715, 13716, 13717, 13718, 13719, 13720, 13722, 13723, 13724, 13725, 13726, 13727, 13728, 13729, 13730, 13731, 13732, 13733, 13734, 13735, 13736, 13737, 13738, 13739, 13740, 13741, 13742, 13743, 13744, 13745, 13746, 13747, 13748, 13749, 13750, 13752, 13753, 13754, 13756, 13757, 13758, 13759, 13761, 13762, 13763, 13764, 13765, 13766, 13768, 13769, 13770, 13771, 13773, 13774, 13776, 13777, 13778, 13779, 13780, 13784, 13785, 13786, 13787, 13788, 13790, 13791, 13792, 13793, 13794, 13795, 13797, 13798, 13799, 13800, 13801, 13802, 13803, 13805, 13808, 13812, 13813, 13816, 13817, 13818, 13819, 13820, 13821, 13822, 13823, 13825, 13826, 13827, 13828, 13829, 13830, 13831, 13832, 13833, 13834, 13835, 13837, 13838, 13840, 13841, 13842, 13843, 13844, 13845, 13846, 13847, 13848, 13849, 13850, 13851, 13852, 13853, 13854, 13855, 13856, 13857, 13858, 13859, 13860, 13861, 13862, 13863, 13864, 13865, 13866, 13867, 13868, 13869, 13870, 13871, 13872, 13873, 13874, 13875, 13876, 13877, 13878, 13879, 13880, 13881, 13882, 13883, 13884, 13885, 13886, 13887, 13888, 13889, 13890, 13891, 13892, 13893, 13894, 13895, 13896, 13897, 13898, 13899, 13900, 13901, 13903, 13904, 13905, 13906, 13907, 13908, 13909, 13910, 13912, 13915, 13916, 13917, 13918, 13919, 13920, 13921, 13922, 13923, 13924, 13926, 13927, 13928, 13929, 13930, 13931, 13932, 13933, 13934, 13935, 13936, 13937, 13938, 13939, 13940, 13941, 13942, 13943, 13944, 13945, 13946, 13947, 13948, 13949, 13950, 13951, 13952, 13953, 13954, 13955, 13960, 13963, 13964, 13965, 13969, 13970, 13971, 13972, 13975, 13976, 13978, 13979, 13980, 13981, 13983, 13987, 13988, 13990, 13991, 13992, 13993, 13994, 13995, 13996, 13997, 13998, 13999, 14000, 14001, 14002, 14003, 14004, 14005, 14006, 14007, 14008, 14009, 14010, 14011, 14013, 14014, 14017, 14018, 14019, 14020, 14021, 14022, 14024, 14025, 14026, 14027, 14028, 14029, 14030, 14031, 14032, 14033, 14034, 14035, 14037, 14038, 14040, 14042, 14043, 14044, 14045, 14046, 14050, 14051, 14052, 14053, 14054, 14055, 14056, 14057, 14059, 14060, 14061, 14062, 14063, 14065, 14066, 14067, 14068, 14069, 14070, 14071, 14073, 14074, 14075, 14077, 14078, 14079, 14080, 14083, 14084, 14085, 14086, 14087, 14093, 14094, 14095, 14096, 14097, 14098, 14099, 14100, 14101, 14102, 14103, 14104, 14105, 14106, 14107, 14108, 14109, 14110, 14111, 14113, 14115, 14117, 14118, 14119, 14120, 14121, 14122, 14123, 14124, 14125, 14126, 14127, 14128, 14129, 14130, 14131, 14132, 14133, 14134, 14135, 14138, 14140, 14141, 14142, 14143, 14147, 14148, 14149, 14151, 14152, 14153, 14154, 14155, 14156, 14158, 14159, 14161, 14162, 14163, 14164, 14166, 14167, 14169, 14170, 14172, 14173, 14174, 14175, 14176, 14177, 14178, 14179, 14180, 14181, 14182, 14183, 14184, 14185, 14187, 14188, 14189, 14190, 14191, 14192, 14193, 14194, 14195, 14196, 14199, 14200, 14201, 14205, 14207, 14208, 14209, 14210, 14212, 14213, 14214, 14215, 14216, 14217, 14218, 14219, 14220, 14221, 14222, 14223, 14224, 14225, 14226, 14227, 14228, 14229, 14231, 14232, 14233, 14236, 14237, 14238, 14239, 14240, 14241, 14242, 14243, 14244, 14245, 14246, 14247, 14250, 14251, 14252, 14256, 14257, 14258, 14259, 14260, 14261, 14262, 14263, 14264, 14265, 14266, 14267, 14268, 14269, 14271, 14272, 14273, 14274, 14275, 14276, 14277, 14278, 14279, 14281, 14282, 14283, 14284, 14285, 14286, 14287, 14288, 14290, 14291, 14292, 14293, 14294, 14295, 14296, 14297, 14298, 14299, 14300, 14301, 14302, 14303, 14304, 14305, 14306, 14307, 14309, 14310, 14311, 14313, 14314, 14315, 14316, 14317, 14318, 14319, 14320, 14321, 14322, 14323, 14324, 14325, 14326, 14327, 14328, 14329, 14330, 14331, 14332, 14333, 14334, 14335, 14336, 14337, 14338, 14339, 14340, 14341, 14342, 14344, 14345, 14346, 14347, 14348, 14349, 14350, 14351, 14352, 14353, 14354, 14355, 14356, 14357, 14358, 14359, 14360, 14361, 14363, 14364, 14365, 14366, 14367, 14368, 14369, 14370, 14371, 14372, 14377, 14378, 14379, 14380, 14381, 14382, 14383, 14384, 14385, 14386, 14387, 14388, 14389, 14390, 14392, 14393, 14396, 14397, 14398, 14400, 14401, 14403, 14404, 14409, 14410, 14411, 14412, 14414, 14415, 14416, 14418, 14419, 14420, 14421, 14422, 14423, 14424, 14425, 14427, 14428, 14429, 14430, 14431, 14432, 14433, 14434, 14435, 14436, 14437, 14438, 14439, 14443, 14444, 14445, 14446, 14447, 14448, 14449, 14450, 14451, 14452, 14453, 14454, 14455, 14456, 14458, 14459, 14460, 14463, 14464, 14465, 14467, 14468, 14469, 14470, 14471, 14472, 14473, 14474, 14475, 14476, 14477, 14478, 14479, 14481, 14482, 14483, 14484, 14485, 14486, 14487, 14488, 14489, 14490, 14492, 14495, 14496, 14497, 14498, 14499, 14500, 14501, 14502, 14503, 14504, 14506, 14507, 14508, 14509, 14510, 14511, 14512, 14513, 14514, 14515, 14516, 14517, 14518, 14519, 14520, 14521, 14522, 14523, 14524, 14525, 14526, 14527, 14528, 14530, 14531, 14532, 14533, 14534, 14535, 14536, 14537, 14538, 14539, 14540, 14541, 14542, 14544, 14545, 14546, 14547, 14548, 14550, 14551, 14552, 14553, 14554, 14556, 14557, 14558, 14559, 14561, 14562, 14563, 14564, 14565, 14566, 14567, 14568, 14569, 14570, 14571, 14572, 14573, 14575, 14576, 14577, 14579, 14580, 14581, 14582, 14583, 14584, 14585, 14586, 14587, 14588, 14589, 14590, 14591, 14592, 14593, 14594, 14595, 14596, 14598, 14599, 14600, 14603, 14604, 14605, 14606, 14607, 14608, 14609, 14610, 14611, 14612, 14613, 14615, 14618, 14619, 14620, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, 14661, 14662, 14663, 14664, 14665, 14666, 14668, 14671, 14672, 14673, 14674, 14675, 14676, 14677, 14678, 14680, 14681, 14682, 14683, 14685, 14686, 14688, 14689, 14690, 14692, 14693, 14694, 14695, 14696, 14697, 14698, 14700, 14701, 14704, 14705, 14706, 14707, 14708, 14709, 14710, 14711, 14712, 14713, 14714, 14716, 14717, 14718, 14719, 14721, 14722, 14723, 14724, 14725, 14726, 14727, 14728, 14729, 14730, 14731, 14732, 14733, 14734, 14735, 14736, 14737, 14738, 14739, 14740, 14742, 14744, 14745, 14746, 14749, 14750, 14751, 14753, 14754, 14755, 14756, 14757, 14758, 14759, 14760, 14761, 14762, 14763, 14764, 14765, 14766, 14767, 14768, 14769, 14770, 14771, 14772, 14773, 14775, 14776, 14777, 14778, 14779, 14780, 14781, 14782, 14783, 14784, 14785, 14786, 14787, 14789, 14790, 14791, 14792, 14793, 14795, 14796, 14797, 14798, 14799, 14800, 14801, 14802, 14803, 14804, 14805, 14807, 14808, 14809, 14810, 14811, 14812, 14813, 14815, 14816, 14817, 14818, 14819, 14820, 14822, 14823, 14825, 14826, 14827, 14828, 14829, 14830, 14831, 14832, 14833, 14834, 14835, 14836, 14837, 14838, 14839, 14840, 14841, 14842, 14843, 14844, 14845, 14846, 14847, 14848, 14849, 14850, 14852, 14853, 14854, 14856, 14857, 14858, 14859, 14861, 14862, 14863, 14865, 14866, 14867, 14868, 14869, 14870, 14871, 14873, 14874, 14875, 14876, 14877, 14878, 14879, 14880, 14881, 14882, 14884, 14885, 14886, 14887, 14888, 14889, 14890, 14891, 14894, 14895, 14896, 14897, 14898, 14899, 14900, 14901, 14902, 14903, 14904, 14905, 14906, 14907, 14908, 14909, 14910, 14911, 14912, 14913, 14914, 14915, 14917, 14919, 14920, 14922, 14923, 14924, 14925, 14926, 14927, 14928, 14930, 14931, 14932, 14934, 14935, 14936, 14937, 14938, 14939, 14940, 14941, 14942, 14943, 14944, 14945, 14946, 14948, 14949, 14950, 14951, 14952, 14953, 14954, 14955, 14956, 14957, 14958, 14959, 14961, 14962, 14968, 14969, 14970, 14974, 14975, 14976, 14977, 14978, 14979, 14980, 14981, 14982, 14983, 14984, 14985, 14986, 14987, 14988, 14989, 14990, 14991, 14992, 14993, 14995, 14996, 14997, 15001, 15002, 15003, 15004, 15006, 15007, 15008, 15009, 15010, 15012, 15013, 15014, 15015, 15016, 15017, 15018, 15019, 15020, 15021, 15022, 15024, 15029, 15030, 15031, 15032, 15033, 15034, 15035, 15036, 15038, 15039, 15040, 15042, 15043, 15046, 15047, 15048, 15049, 15050, 15051, 15052, 15054, 15055, 15056, 15057, 15058, 15060, 15061, 15062, 15063, 15064, 15066, 15067, 15068, 15069, 15070, 15071, 15073, 15074, 15075, 15078, 15079, 15080, 15081, 15083, 15084, 15086, 15088, 15089, 15090, 15091, 15092, 15093, 15094, 15095, 15096, 15097, 15098, 15099, 15100, 15101, 15103, 15104, 15105, 15106, 15107, 15108, 15110, 15111, 15112, 15113, 15114, 15115, 15117, 15118, 15119, 15120, 15121, 15125, 15126, 15127, 15129, 15130, 15132, 15135, 15137, 15138, 15139, 15140, 15141, 15142, 15143, 15144, 15146, 15147, 15148, 15149, 15150, 15151, 15152, 15153, 15154, 15155, 15156, 15157, 15158, 15159, 15160, 15161, 15162, 15164, 15165, 15167, 15168, 15169, 15170, 15171, 15172, 15173, 15175, 15176, 15177, 15178, 15179, 15180, 15184, 15185, 15186, 15187, 15188, 15189, 15190, 15191, 15192, 15194, 15195, 15196, 15198, 15200, 15201, 15202, 15203, 15204, 15205, 15206, 15207, 15209, 15210, 15211, 15212, 15214, 15215, 15216, 15217, 15218, 15219, 15220, 15221, 15222, 15223, 15224, 15225, 15226, 15227, 15228, 15229, 15231, 15232, 15234, 15235, 15236, 15238, 15240, 15241, 15242, 15243, 15244, 15245, 15246, 15247, 15249, 15250, 15251, 15254, 15255, 15259, 15260, 15261, 15262, 15263, 15264, 15265, 15267, 15268, 15269, 15271, 15272, 15273, 15275, 15276, 15278, 15279, 15280, 15281, 15282, 15283, 15284, 15285, 15287, 15289, 15290, 15291, 15293, 15294, 15295, 15296, 15297, 15298, 15299, 15300, 15301, 15302, 15303, 15304, 15305, 15307, 15308, 15309, 15310, 15311, 15312, 15313, 15314, 15317, 15318, 15319, 15320, 15321, 15323, 15325, 15326, 15327, 15328, 15329, 15330, 15331, 15332, 15334, 15335, 15336, 15337, 15340, 15341, 15342, 15343, 15344, 15345, 15346, 15347, 15348, 15349, 15350, 15351, 15352, 15353, 15354, 15355, 15356, 15358, 15360, 15361, 15364, 15365, 15366, 15369, 15370, 15371, 15372, 15373, 15374, 15375, 15376, 15377, 15378, 15379, 15380, 15381, 15382, 15384, 15386, 15388, 15389, 15390, 15391, 15393, 15394, 15395, 15396, 15397, 15398, 15399, 15400, 15401, 15402, 15403, 15404, 15406, 15407, 15408, 15409, 15410, 15411, 15412, 15413, 15414, 15415, 15417, 15418, 15419, 15420, 15421, 15422, 15423, 15425, 15426, 15427, 15428, 15430, 15431, 15432, 15433, 15434, 15435, 15436, 15437, 15438, 15439, 15440, 15441, 15442, 15444, 15445, 15446, 15447, 15448, 15452, 15453, 15454, 15455, 15456, 15457, 15458, 15459, 15461, 15462, 15463, 15464, 15465, 15466, 15467, 15468, 15469, 15470, 15471, 15473, 15474, 15475, 15476, 15477, 15478, 15480, 15481, 15483, 15484, 15485, 15486, 15487, 15488, 15490, 15491, 15492, 15494, 15495, 15497, 15499, 15500, 15501, 15502, 15503, 15504, 15505, 15506, 15507, 15508, 15510, 15511, 15512, 15513, 15514, 15515, 15516, 15517, 15518, 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, 15527, 15528, 15529, 15530, 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15539, 15540, 15541, 15542, 15543, 15544, 15545, 15546, 15547, 15548, 15549, 15550, 15551, 15553, 15554, 15555, 15556, 15557, 15558, 15560, 15561, 15562, 15563, 15564, 15567, 15569, 15570, 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15579, 15580, 15581, 15582, 15583, 15584, 15585, 15586, 15587, 15588, 15590, 15591, 15592, 15593, 15594, 15595, 15596, 15599, 15600, 15601, 15602, 15603, 15605, 15606, 15607, 15608, 15609, 15610, 15611, 15612, 15613, 15614, 15615, 15616, 15618, 15620, 15621, 15623, 15624, 15625, 15626, 15627, 15629, 15630, 15632, 15633, 15634, 15635, 15637, 15638, 15639, 15640, 15642, 15643, 15644, 15645, 15646, 15647, 15648, 15649, 15650, 15653, 15655, 15656, 15657, 15658, 15659, 15660, 15661, 15662, 15663, 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, 15674, 15675, 15676, 15677, 15678, 15679, 15682, 15683, 15684, 15685, 15689, 15690, 15691, 15692, 15693, 15694, 15695, 15696, 15697, 15698, 15699, 15700, 15701, 15703, 15704, 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, 15716, 15719, 15720, 15721, 15722, 15723, 15724, 15725, 15726, 15734, 15736, 15737, 15740, 15741, 15742, 15743, 15744, 15745, 15747, 15748, 15749, 15750, 15751, 15752, 15753, 15754, 15755, 15757, 15758, 15761, 15763, 15764, 15765, 15766, 15767, 15768, 15769, 15770, 15771, 15772, 15773, 15775, 15776, 15777, 15778, 15779, 15781, 15782, 15783, 15785, 15787, 15788, 15789, 15790, 15791, 15792, 15793, 15794, 15795, 15796, 15797, 15800, 15801, 15804, 15805, 15806, 15807, 15808, 15809, 15810, 15811, 15812, 15813, 15814, 15815, 15816, 15817, 15818, 15819, 15820, 15821, 15822, 15824, 15825, 15827, 15828, 15829, 15830, 15831, 15832, 15833, 15834, 15835, 15836, 15837, 15838, 15839, 15840, 15841, 15842, 15843, 15844, 15846, 15847, 15848, 15849, 15850, 15851, 15852, 15853, 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, 15864, 15865, 15866, 15867, 15868, 15869, 15870, 15871, 15872, 15873, 15874, 15875, 15876, 15877, 15878, 15880, 15881, 15882, 15883, 15885, 15886, 15888, 15889, 15890, 15891, 15892, 15893, 15894, 15895, 15897, 15898, 15900, 15901, 15904, 15905, 15906, 15907, 15908, 15909, 15913, 15914, 15915, 15916, 15917, 15918, 15919, 15921, 15922, 15924, 15925, 15926, 15929, 15931, 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, 15940, 15941, 15942, 15943, 15944, 15945, 15948, 15949, 15950, 15951, 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, 15965, 15966, 15967, 15968, 15969, 15970, 15971, 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, 15981, 15984, 15985, 15986, 15987, 15988, 15989, 15993, 15994, 15995, 15996, 15997, 15998, 15999, 16000, 16001, 16002, 16003, 16004, 16005, 16008, 16009, 16010, 16011, 16014, 16015, 16016, 16017, 16018, 16020, 16021, 16022, 16023, 16024, 16025, 16026, 16028, 16029, 16030, 16031, 16032, 16033, 16034, 16035, 16036, 16040, 16041, 16043, 16044, 16045, 16046, 16047, 16049, 16051, 16053, 16054, 16055, 16056, 16057, 16061, 16062, 16065, 16066, 16067, 16068, 16070, 16071, 16072, 16073, 16074, 16075, 16076, 16077, 16078, 16079, 16080, 16081, 16083, 16084, 16085, 16086, 16087, 16088, 16089, 16090, 16091, 16092, 16093, 16094, 16095, 16096, 16097, 16098, 16099, 16101, 16103, 16104, 16105, 16106, 16107, 16108, 16112, 16113, 16114, 16115, 16116, 16117, 16118, 16120, 16121, 16122, 16123, 16124, 16128, 16130, 16131, 16132, 16134, 16135, 16136, 16137, 16138, 16139, 16140, 16141, 16142, 16143, 16144, 16145, 16146, 16148, 16149, 16150, 16151, 16153, 16154, 16155, 16156, 16157, 16158, 16159, 16160, 16161, 16163, 16164, 16166, 16168, 16169, 16171, 16172, 16173, 16174, 16175, 16176, 16177, 16179, 16180, 16182, 16184, 16185, 16186, 16187, 16189, 16190, 16191, 16192, 16193, 16195, 16197, 16198, 16199, 16200, 16201, 16202, 16203, 16204, 16205, 16206, 16208, 16209, 16211, 16213, 16214, 16215, 16216, 16217, 16218, 16219, 16222, 16223, 16224, 16227, 16229, 16230, 16231, 16232, 16233, 16236, 16237, 16238, 16240, 16242, 16243, 16244, 16245, 16246, 16247, 16248, 16249, 16250, 16251, 16252, 16254, 16255, 16256, 16258, 16259, 16261, 16262, 16264, 16265, 16266, 16267, 16268, 16269, 16271, 16273, 16274, 16275, 16276, 16277, 16278, 16279, 16280, 16281, 16282, 16283, 16284, 16285, 16286, 16287, 16288, 16289, 16290, 16291, 16292, 16293, 16294, 16295, 16296, 16297, 16298, 16299, 16300, 16301, 16302, 16303, 16304, 16305, 16308, 16310, 16311, 16312, 16313, 16315, 16316, 16317, 16318, 16319, 16320, 16321, 16323, 16324, 16325, 16327, 16328, 16329, 16330, 16331, 16332, 16334, 16335, 16338, 16339, 16340, 16343, 16344, 16345, 16346, 16347, 16348, 16350, 16351, 16352, 16353, 16354, 16355, 16356, 16357, 16358, 16359, 16363, 16364, 16365, 16366, 16368, 16369, 16370, 16371, 16372, 16374, 16375, 16376, 16377, 16378, 16379, 16382, 16383, 16384, 16385, 16386, 16387, 16389, 16391, 16392, 16393, 16394, 16395, 16397, 16398, 16399, 16400, 16401, 16402, 16403, 16404, 16405, 16407, 16408, 16409, 16410, 16411, 16412, 16413, 16414, 16415, 16417, 16418, 16419, 16420, 16421, 16422, 16423, 16424, 16425, 16426, 16427, 16428, 16431, 16432, 16433, 16434, 16435, 16439, 16440, 16441, 16442, 16445, 16446, 16447, 16448, 16449, 16450, 16451, 16453, 16455, 16456, 16457, 16458, 16459, 16460, 16461, 16462, 16463, 16464, 16465, 16466, 16467, 16469, 16470, 16471, 16473, 16474, 16475, 16476, 16477, 16478, 16479, 16480, 16481, 16482, 16483, 16484, 16485, 16486, 16487, 16488, 16489, 16490, 16491, 16492, 16493, 16494, 16496, 16497, 16498, 16499, 16500, 16503, 16504, 16505, 16506, 16508, 16509, 16510, 16511, 16512, 16513, 16514, 16515, 16516, 16517, 16518, 16519, 16520, 16521, 16522, 16524, 16525, 16526, 16527, 16528, 16529, 16530, 16532, 16533, 16534, 16535, 16536, 16537, 16538, 16539, 16541, 16542, 16544, 16545, 16546, 16547, 16548, 16549, 16551, 16552, 16553, 16555, 16557, 16558, 16559, 16562, 16563, 16565, 16566, 16568, 16569, 16570, 16571, 16572, 16573, 16574, 16577, 16578, 16579, 16580, 16581, 16582, 16583, 16585, 16587, 16588, 16589, 16590, 16591, 16592, 16593, 16595, 16597, 16598, 16599, 16600, 16601, 16602, 16603, 16605, 16606, 16607, 16608, 16610, 16611, 16612, 16613, 16614, 16615, 16616, 16617, 16620, 16621, 16622, 16623, 16624, 16625, 16626, 16627, 16628, 16629, 16630, 16631, 16632, 16633, 16634, 16635, 16636, 16637, 16639, 16640, 16641, 16642, 16643, 16646, 16647, 16648, 16650, 16651, 16652, 16655, 16657, 16658, 16659, 16660, 16662, 16663, 16664, 16665, 16666, 16667, 16668, 16669, 16670, 16671, 16672, 16673, 16674, 16677, 16678, 16680, 16683, 16684, 16685, 16686, 16687, 16688, 16689, 16690, 16691, 16692, 16693, 16694, 16697, 16698, 16699, 16700, 16701, 16705, 16706, 16707, 16712, 16717, 16718, 16719, 16721, 16723, 16724, 16725, 16726, 16727, 16729, 16730, 16731, 16732, 16733, 16734, 16735, 16736, 16737, 16738, 16739, 16743, 16744, 16745, 16746, 16748, 16750, 16751, 16752, 16753, 16754, 16757, 16758, 16759, 16761, 16762, 16764, 16765, 16766, 16767, 16768, 16769, 16770, 16771, 16772, 16773, 16774, 16775, 16776, 16778, 16779, 16782, 16783, 16786, 16787, 16788, 16792, 16793, 16794, 16795, 16796, 16797, 16798, 16802, 16803, 16804, 16805, 16806, 16807, 16808, 16809, 16810, 16811, 16812, 16814, 16815, 16817, 16818, 16820, 16821, 16823, 16825, 16826, 16827, 16828, 16829, 16830, 16831, 16836, 16838, 16839, 16840, 16841, 16842, 16843, 16844, 16845, 16846, 16848, 16849, 16850, 16851, 16852, 16853, 16854, 16855, 16856, 16857, 16858, 16859, 16860, 16861, 16862, 16863, 16866, 16868, 16872, 16873, 16875, 16877, 16878, 16879, 16880, 16881, 16882, 16883, 16885, 16887, 16888, 16889, 16890, 16891, 16892, 16893, 16894, 16896, 16897, 16898, 16899, 16900, 16901, 16902, 16906, 16907, 16910, 16913, 16916, 16917, 16919, 16920, 16921, 16922, 16924, 16925, 16926, 16927, 16928, 16929, 16930, 16931, 16932, 16933, 16936, 16937, 16938, 16940, 16941, 16942, 16943, 16944, 16945, 16946, 16947, 16948, 16950, 16952, 16954, 16955, 16956, 16958, 16959, 16960, 16961, 16962, 16963, 16964, 16967, 16968, 16969, 16971, 16973, 16974, 16979, 16980, 16981, 16982, 16983, 16984, 16985, 16987, 16988, 16989, 16990, 16992, 16993, 16995, 16996, 16997, 16998, 17000, 17001, 17002, 17003, 17004, 17005, 17006, 17007, 17008, 17009, 17011, 17012, 17014, 17015, 17016, 17017, 17018, 17019, 17020, 17021, 17022, 17023, 17024, 17025, 17026, 17027, 17028, 17029, 17030, 17031, 17032, 17033, 17034, 17035, 17036, 17037, 17038, 17039, 17040, 17041, 17044, 17045, 17046, 17048, 17049, 17050, 17051, 17052, 17053, 17055, 17058, 17059, 17060, 17061, 17062, 17065, 17066, 17067, 17068, 17069, 17070, 17071, 17072, 17073, 17075, 17076, 17078, 17079, 17080, 17081, 17082, 17085, 17086, 17091, 17092, 17094, 17095, 17096, 17098, 17099, 17100, 17101, 17102, 17103, 17104, 17105, 17108, 17112, 17113, 17114, 17116, 17117, 17119, 17120, 17121, 17122, 17123, 17124, 17125, 17126, 17127, 17128, 17129, 17130, 17131, 17132, 17133, 17134, 17135, 17136, 17137, 17139, 17140, 17141, 17142, 17144, 17148, 17149, 17150, 17151, 17152, 17154, 17155, 17156, 17158, 17161, 17162, 17164, 17165, 17166, 17167, 17170, 17171, 17172, 17173, 17174, 17175, 17176, 17177, 17178, 17179, 17180, 17181, 17182, 17183, 17184, 17187, 17188, 17189, 17190, 17191, 17192, 17193, 17194, 17195, 17196, 17197, 17202, 17203, 17204, 17205, 17206, 17207, 17208, 17209, 17210, 17211, 17212, 17213, 17214, 17215, 17218, 17219, 17220, 17221, 17222, 17223, 17224, 17225, 17227, 17228, 17230, 17231, 17233, 17234, 17236, 17237, 17238, 17239, 17240, 17241, 17244, 17246, 17247, 17248, 17249, 17250, 17251, 17253, 17254, 17255, 17257, 17258, 17259, 17260, 17261, 17262, 17263, 17264, 17265, 17267, 17268, 17269, 17271, 17272, 17273, 17274, 17275, 17276, 17277, 17278, 17279, 17280, 17281, 17282, 17283, 17284, 17291, 17292, 17296, 17298, 17299, 17300, 17305, 17306, 17307, 17308, 17309, 17310, 17311, 17312, 17313, 17314, 17320, 17321, 17322, 17323, 17324, 17325, 17326, 17329, 17330, 17332, 17334, 17335, 17337, 17338, 17339, 17343, 17344, 17345, 17346, 17348, 17350, 17351, 17352, 17353, 17355, 17356, 17357, 17358, 17359, 17360, 17361, 17365, 17366, 17369, 17370, 17371, 17373, 17374, 17376, 17378, 17381, 17382, 17384, 17385, 17386, 17387, 17388, 17389, 17390, 17391, 17392, 17393, 17394, 17398, 17399, 17400, 17401, 17402, 17403, 17404, 17405, 17406, 17407, 17408, 17409, 17410, 17412, 17413, 17414, 17415, 17416, 17417, 17418, 17419, 17420, 17421, 17422, 17423, 17424, 17425, 17426, 17427, 17430, 17431, 17432, 17433, 17438, 17439, 17441, 17442, 17443, 17444, 17445, 17446, 17447, 17448, 17449, 17450, 17451, 17452, 17453, 17456, 17457, 17458, 17459, 17460, 17461, 17462, 17463, 17464, 17468, 17469, 17474, 17475, 17476, 17477, 17478, 17480, 17481, 17482, 17483, 17484, 17485, 17487, 17491, 17492, 17494, 17495, 17498, 17499, 17500, 17501, 17502, 17503, 17504, 17505, 17506, 17507, 17508, 17511, 17512, 17513, 17514, 17515, 17516, 17517, 17518, 17519, 17520, 17521, 17522, 17523, 17524, 17525, 17526, 17527, 17528, 17530, 17531, 17532, 17533, 17534, 17535, 17536, 17537, 17538, 17539, 17541, 17542, 17543, 17544, 17545, 17546, 17547, 17548, 17549, 17551, 17552, 17553, 17554, 17556, 17557, 17558, 17559, 17560, 17562, 17566, 17567, 17568, 17569, 17570, 17571, 17572, 17573, 17575, 17577, 17578, 17583, 17584, 17585, 17586, 17587, 17589, 17592, 17593, 17594, 17595, 17597, 17598, 17599, 17602, 17604, 17606, 17607, 17608, 17609, 17610, 17611, 17612, 17614, 17615, 17617, 17618, 17619, 17620, 17621, 17622, 17623, 17627, 17629, 17630, 17631, 17632, 17636, 17637, 17638, 17639, 17640, 17641, 17642, 17646, 17648, 17649, 17650, 17651, 17652, 17653, 17654, 17657, 17658, 17662, 17664, 17665, 17666, 17667, 17668, 17669, 17671, 17672, 17674, 17675, 17677, 17678, 17679, 17680, 17681, 17682, 17683, 17684, 17685, 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17697, 17698, 17699, 17700, 17701, 17702, 17703, 17704, 17707, 17708, 17709, 17710, 17713, 17714, 17715, 17716, 17718, 17719, 17720, 17722, 17723, 17724, 17725, 17726, 17727, 17729, 17730, 17731, 17732, 17733, 17736, 17738, 17739, 17740, 17741, 17744, 17745, 17746, 17747, 17748, 17749, 17750, 17751, 17752, 17753, 17754, 17757, 17758, 17760, 17761, 17762, 17763, 17765, 17766, 17767, 17769, 17772, 17773, 17774, 17775, 17779, 17780, 17781, 17782, 17783, 17785, 17786, 17788, 17789, 17790, 17791, 17792, 17793, 17795, 17796, 17797, 17798, 17800, 17801, 17803, 17804, 17805, 17806, 17807, 17808, 17809, 17810, 17811, 17812, 17813, 17815, 17816, 17817, 17818, 17819, 17820, 17824, 17825, 17826, 17827, 17828, 17829, 17830, 17831, 17832, 17836, 17838, 17839, 17841, 17842, 17843, 17844, 17845, 17847, 17848, 17849, 17850, 17851, 17852, 17853, 17854, 17855, 17856, 17857, 17858, 17859, 17860, 17861, 17862, 17864, 17866, 17867, 17868, 17870, 17871, 17872, 17873, 17874, 17875, 17876, 17877, 17878, 17879, 17888, 17889, 17891, 17892, 17895, 17899, 17901, 17902, 17904, 17905, 17906, 17907, 17908, 17912, 17914, 17916, 17917, 17918, 17919, 17920, 17921, 17923, 17924, 17925, 17926, 17927, 17928, 17929, 17930, 17931, 17932, 17933, 17934, 17935, 17936, 17937, 17938, 17939, 17947, 17949, 17950, 17951, 17954, 17955, 17957, 17958, 17959, 17961, 17962, 17963, 17964, 17965, 17966, 17968, 17969, 17970, 17972, 17974, 17975, 17978, 17979, 17983, 17984, 17985, 17986, 17987, 17988, 17990, 17991, 17992, 17996, 17998, 18000, 18002, 18003, 18004, 18008, 18011, 18012, 18017, 18019, 18020, 18021, 18022, 18023, 18024, 18026, 18027, 18029, 18032, 18035, 18036, 18037, 18038, 18039, 18040, 18041, 18042, 18043, 18044, 18047, 18048, 18049, 18051, 18052, 18053, 18054, 18055, 18056, 18057, 18058, 18059, 18060, 18062, 18065, 18066, 18067, 18068, 18069, 18072, 18073, 18076, 18078, 18080, 18081, 18082, 18083, 18084, 18085, 18086, 18087, 18089, 18090, 18091, 18092, 18093, 18094, 18095, 18096, 18097, 18098, 18099, 18100, 18101, 18102, 18103, 18104, 18105, 18106, 18107, 18109, 18110, 18119, 18123, 18124, 18125, 18128, 18129, 18130, 18131, 18134, 18135, 18137, 18138, 18139, 18141, 18142, 18143, 18144, 18145, 18147, 18148, 18149, 18150, 18151, 18154, 18156, 18157, 18158, 18159, 18160, 18161, 18162, 18163, 18164, 18165, 18166, 18170, 18171, 18172, 18173, 18174, 18175, 18176, 18178, 18179, 18180, 18181, 18182, 18183, 18184, 18188, 18189, 18190, 18191, 18192, 18193, 18194, 18195, 18197, 18198, 18201, 18202, 18203, 18205, 18206, 18207, 18208, 18209, 18210, 18211, 18212, 18213, 18215, 18216, 18217, 18218, 18219, 18220, 18221, 18222, 18223, 18224, 18227, 18229, 18230, 18231, 18232, 18233, 18234, 18235, 18236, 18237, 18238, 18239, 18241, 18243, 18244, 18245, 18246, 18247, 18248, 18249, 18250, 18251, 18252, 18253, 18254, 18255, 18256, 18257, 18259, 18260, 18261, 18262, 18263, 18264, 18265, 18266, 18267, 18268, 18269, 18271, 18272, 18273, 18274, 18275, 18276, 18280, 18281, 18282, 18283, 18284, 18285, 18287, 18289, 18290, 18292, 18293, 18294, 18295, 18296, 18297, 18298, 18299, 18300, 18302, 18303, 18304, 18305, 18306, 18315, 18316, 18317, 18318, 18319, 18320, 18321, 18328, 18329, 18331, 18332, 18335, 18338, 18339, 18340, 18341, 18343, 18344, 18345, 18346, 18347, 18348, 18349, 18350, 18351, 18352, 18353, 18354, 18355, 18356, 18357, 18360, 18361, 18362, 18363, 18364, 18365, 18366, 18367, 18368, 18369, 18370, 18371, 18373, 18374, 18378, 18379, 18380, 18381, 18382, 18383, 18384, 18385, 18386, 18387, 18388, 18391, 18395, 18397, 18398, 18399, 18400, 18403, 18408, 18411, 18412, 18414, 18416, 18417, 18419, 18420, 18421, 18422, 18424, 18425, 18426, 18427, 18434, 18435, 18437, 18438, 18439, 18441, 18442, 18443, 18444, 18445, 18446, 18447, 18452, 18453, 18454, 18456, 18457, 18458, 18459, 18460, 18461, 18465, 18466, 18467, 18468, 18469, 18470, 18473, 18474, 18475, 18476, 18477, 18480, 18481, 18482, 18483, 18484, 18485, 18486, 18489, 18490, 18491, 18492, 18493, 18495, 18498, 18500, 18501, 18506, 18507, 18508, 18509, 18510, 18511, 18513, 18516, 18517, 18518, 18519, 18520, 18521, 18525, 18527, 18528, 18529, 18531, 18532, 18534, 18535, 18536, 18538, 18539, 18540, 18541, 18542, 18543, 18544, 18545, 18546, 18547, 18548, 18549, 18550, 18551, 18552, 18553, 18555, 18556, 18557, 18558, 18559, 18560, 18561, 18566, 18568, 18569, 18570, 18571, 18572, 18573, 18574, 18575, 18576, 18577, 18578, 18579, 18582, 18584, 18586, 18587, 18588, 18593, 18594, 18597, 18598, 18600, 18601, 18602, 18603, 18605, 18607, 18608, 18610, 18612, 18613, 18614, 18619, 18620, 18625, 18626, 18630, 18631, 18632, 18635, 18637, 18640, 18641, 18645, 18646, 18648, 18650, 18653, 18654, 18657, 18659, 18660, 18662, 18663, 18664, 18665, 18667, 18668, 18669, 18670, 18671, 18672, 18673, 18674, 18675, 18677, 18678, 18679, 18680, 18681, 18682, 18683, 18684, 18685, 18686, 18687, 18688, 18689, 18691, 18692, 18693, 18695, 18696, 18697, 18698, 18699, 18700, 18701, 18702, 18703, 18704, 18705, 18710, 18711, 18712, 18713, 18718, 18719, 18720, 18721, 18723, 18724, 18725, 18726, 18727, 18731, 18732, 18735, 18736, 18737, 18738, 18739, 18740, 18741, 18742, 18743, 18745, 18746, 18747, 18748, 18749, 18750, 18755, 18756, 18757, 18758, 18759, 18760, 18762, 18764, 18766, 18767, 18768, 18769, 18770, 18771, 18772, 18774, 18775, 18777, 18778, 18782, 18783, 18785, 18789, 18792, 18793, 18795, 18796, 18797, 18798, 18799, 18800, 18801, 18807, 18808, 18809, 18810, 18811, 18812, 18813, 18816, 18817, 18818, 18820, 18821, 18822, 18823, 18824, 18825, 18826, 18827, 18830, 18832, 18833, 18834, 18835, 18840, 18841, 18842, 18843, 18844, 18846, 18847, 18848, 18849, 18850, 18851, 18852, 18853, 18857, 18858, 18859, 18860, 18861, 18862, 18864, 18865, 18866, 18868, 18869, 18870, 18871, 18872, 18877, 18878, 18879, 18880, 18881, 18882, 18883, 18884, 18885, 18886, 18887, 18888, 18889, 18890, 18891, 18892, 18893, 18899, 18900, 18901, 18904, 18905, 18907, 18908, 18910, 18911, 18912, 18913, 18914, 18917, 18918, 18919, 18922, 18924, 18926, 18927, 18928, 18929, 18930, 18931, 18933, 18935, 18936, 18937, 18938, 18939, 18941, 18942, 18945, 18946, 18949, 18950, 18951, 18952, 18953, 18954, 18955, 18957, 18958, 18959, 18960, 18961, 18963, 18964, 18965, 18966, 18967, 18968, 18969, 18970, 18971, 18972, 18973, 18974, 18975, 18976, 18977, 18978, 18979, 18980, 18981, 18982, 18983, 18984, 18985, 18990, 18992, 18993, 18994, 18996, 19000, 19001, 19002, 19003, 19008, 19010, 19012, 19013, 19015, 19016, 19018, 19019, 19021, 19022, 19023, 19025, 19027, 19028, 19029, 19031, 19032, 19033, 19034, 19035, 19036, 19037, 19043, 19044, 19045, 19047, 19048, 19050, 19051, 19052, 19053, 19054, 19055, 19056, 19057, 19058, 19059, 19060, 19061, 19062, 19064, 19065, 19066, 19071, 19073, 19074, 19075, 19076, 19077, 19078, 19082, 19083, 19084, 19085, 19086, 19087, 19088, 19089, 19090, 19092, 19098, 19099, 19100, 19101, 19106, 19110, 19112, 19113, 19114, 19117, 19119, 19120, 19121, 19123, 19124, 19125, 19126, 19129, 19130, 19133, 19134, 19135, 19139, 19141, 19142, 19144, 19145, 19148, 19149, 19151, 19152, 19153, 19154, 19160, 19162, 19165, 19167, 19168, 19170, 19171, 19172, 19173, 19174, 19175, 19176, 19177, 19178, 19179, 19180, 19181, 19182, 19183, 19186, 19187, 19188, 19189, 19190, 19191, 19192, 19193, 19194, 19196, 19197, 19198, 19200, 19201, 19202, 19203, 19204, 19206, 19209, 19210, 19215, 19216, 19217, 19219, 19220, 19221, 19222, 19223, 19224, 19225, 19226, 19227, 19228, 19230, 19231, 19232, 19233, 19234, 19235, 19236, 19238, 19239, 19241, 19242, 19243, 19244, 19246, 19247, 19248, 19249, 19251, 19252, 19253, 19254, 19255, 19256, 19258, 19260, 19261, 19263, 19265, 19267, 19268, 19269, 19270, 19272, 19273, 19274, 19275, 19276, 19277, 19278, 19279, 19280, 19281, 19282, 19285, 19286, 19287, 19288, 19289, 19290, 19291, 19292, 19293, 19294, 19295, 19296, 19298, 19299, 19300, 19301, 19314, 19317, 19318, 19319, 19322, 19324, 19325, 19326, 19327, 19328, 19329, 19330, 19331, 19336, 19337, 19343, 19344, 19345, 19347, 19348, 19349, 19350, 19351, 19352, 19354, 19355, 19356, 19357, 19358, 19359, 19360, 19361, 19362, 19363, 19364, 19365, 19370, 19371, 19373, 19378, 19380, 19381, 19382, 19383, 19384, 19385, 19389, 19390, 19394, 19395, 19396, 19397, 19398, 19405, 19407, 19409, 19411, 19412, 19418, 19419, 19420, 19421, 19424, 19426, 19427, 19428, 19429, 19430, 19433, 19435, 19436, 19437, 19438, 19439, 19440, 19441, 19442, 19443, 19444, 19445, 19446, 19448, 19449, 19451, 19452, 19453, 19454, 19455, 19456, 19457, 19461, 19462, 19463, 19466, 19468, 19469, 19470, 19471, 19478, 19479, 19480, 19482, 19483, 19484, 19485, 19486, 19488, 19489, 19490, 19491, 19492, 19493, 19494, 19495, 19497, 19498, 19499, 19500, 19501, 19502, 19503, 19509, 19510, 19511, 19513, 19514, 19515, 19517, 19518, 19520, 19521, 19522, 19523, 19524, 19525, 19526, 19527, 19528, 19529, 19530, 19531, 19532, 19536, 19537, 19538, 19539, 19540, 19541, 19546, 19547, 19548, 19550, 19552, 19554, 19555, 19556, 19557, 19558, 19559, 19560, 19561, 19565, 19567, 19569, 19570, 19571, 19572, 19573, 19574, 19575, 19576, 19577, 19578, 19579, 19580, 19581, 19582, 19586, 19587, 19588, 19589, 19590, 19591, 19592, 19593, 19594, 19596, 19597, 19598, 19599, 19600, 19601, 19602, 19603, 19604, 19605, 19606, 19608, 19610, 19611, 19612, 19613, 19614, 19615, 19616, 19617, 19618, 19619, 19620, 19621, 19622, 19626, 19627, 19628, 19629, 19631, 19632, 19633, 19634, 19635, 19636, 19637, 19638, 19639, 19640, 19641, 19642, 19644, 19645, 19646, 19647, 19650, 19654, 19659, 19660, 19661, 19663, 19666, 19668, 19669, 19670, 19671, 19672, 19673, 19675, 19676, 19677, 19678, 19679, 19681, 19682, 19683, 19684, 19685, 19686, 19687, 19690, 19693, 19694, 19695, 19696, 19697, 19698, 19699, 19700, 19701, 19704, 19705, 19708, 19709, 19713, 19718, 19719, 19721, 19722, 19723, 19724, 19725, 19726, 19728, 19731, 19733, 19734, 19735, 19736, 19737, 19738, 19740, 19741, 19742, 19743, 19744, 19745, 19747, 19749, 19750, 19751, 19752, 19753, 19754, 19755, 19756, 19757, 19758, 19759, 19760, 19762, 19763, 19764, 19769, 19771, 19772, 19773, 19776, 19777, 19778, 19782, 19785, 19787, 19788, 19789, 19790, 19791, 19792, 19794, 19795, 19796, 19797, 19798, 19799, 19801, 19802, 19803, 19805, 19807, 19809, 19810, 19811, 19812, 19813, 19814, 19815, 19816, 19830, 19832, 19833, 19834, 19838, 19839, 19840, 19841, 19842, 19843, 19845, 19846, 19850, 19854, 19855, 19856, 19857, 19858, 19859, 19860, 19861, 19862, 19863, 19867, 19868, 19869, 19871, 19872, 19873, 19874, 19875, 19878, 19879, 19880, 19881, 19882, 19883, 19885, 19886, 19887, 19888, 19889, 19890, 19891, 19892, 19893, 19894, 19896, 19897, 19900, 19901, 19902, 19903, 19904, 19905, 19906, 19907, 19908, 19909, 19918, 19919, 19921, 19922, 19923, 19924, 19926, 19927, 19930, 19931, 19932, 19934, 19935, 19936, 19937, 19941, 19944, 19945, 19946, 19947, 19948, 19949, 19950, 19952, 19953, 19954, 19955, 19956, 19957, 19958, 19959, 19960, 19962, 19963, 19964, 19965, 19966, 19967, 19968, 19969, 19970, 19971, 19972, 19973, 19974, 19975, 19976, 19977, 19980, 19981, 19985, 19986, 19987, 19992, 19993, 19996, 19997, 19998, 19999, 20000, 20001, 20002, 20003, 20004, 20005, 20006, 20007, 20009, 20010, 20011, 20012, 20013, 20015, 20016, 20017, 20019, 20020, 20021, 20022, 20023, 20024, 20027, 20028, 20029, 20030, 20032, 20033, 20034, 20035, 20036, 20037, 20038, 20039, 20044, 20045, 20046, 20047, 20048, 20049, 20050, 20051, 20052, 20053, 20054, 20055, 20056, 20060, 20061, 20065, 20066, 20067, 20077, 20078, 20079, 20080, 20081, 20082, 20083, 20085, 20088, 20089, 20092, 20093, 20095, 20097, 20098, 20099, 20100, 20101, 20103, 20104, 20105, 20106, 20107, 20109, 20110, 20112, 20113, 20117, 20120, 20121, 20122, 20124, 20125, 20126, 20128, 20129, 20130, 20131, 20132, 20133, 20135, 20136, 20137, 20139, 20140, 20141, 20142, 20143, 20144, 20145, 20146, 20147, 20148, 20149, 20150, 20151, 20152, 20155, 20156, 20160, 20161, 20162, 20165, 20168, 20170, 20171, 20172, 20173, 20174, 20175, 20176, 20177, 20178, 20179, 20181, 20182, 20183, 20184, 20185, 20189, 20190, 20191, 20192, 20193, 20194, 20195, 20196, 20197, 20198, 20199, 20200, 20202, 20210, 20214, 20215, 20216, 20217, 20218, 20219, 20220, 20221, 20226, 20231, 20232, 20233, 20235, 20243, 20244, 20245, 20252, 20253, 20257, 20258, 20261, 20262, 20263, 20264, 20268, 20269, 20273, 20274, 20275, 20276, 20277, 20278, 20279, 20280, 20281, 20282, 20283, 20284, 20285, 20286, 20287, 20288, 20289, 20290, 20291, 20292, 20295, 20297, 20298, 20299, 20300, 20301, 20302, 20303, 20304, 20305, 20306, 20307, 20308, 20309, 20310, 20311, 20312, 20313, 20314, 20315, 20316, 20320, 20321, 20322, 20323, 20324, 20325, 20329, 20330, 20333, 20334, 20335, 20336, 20337, 20338, 20340, 20342, 20343, 20344, 20345, 20347, 20348, 20350, 20351, 20352, 20353, 20356, 20358, 20359, 20360, 20361, 20362, 20363, 20364, 20365, 20366, 20367, 20368, 20369, 20370, 20371, 20372, 20373, 20374, 20375, 20376, 20377, 20381, 20382, 20383, 20384, 20385, 20386, 20387, 20388, 20389, 20390, 20391, 20392, 20393, 20403, 20404, 20408, 20409, 20411, 20412, 20413, 20414, 20417, 20422, 20423, 20424, 20425, 20426, 20427, 20428, 20429, 20430, 20431, 20435, 20436, 20437, 20438, 20439, 20440, 20441, 20442, 20443, 20444, 20445, 20446, 20448, 20449, 20450, 20451, 20452, 20453, 20454, 20455, 20460, 20461, 20462, 20463, 20464, 20465, 20466, 20467, 20468, 20469, 20479, 20481, 20482, 20483, 20484, 20487, 20488, 20490, 20492, 20493, 20494, 20495, 20496, 20497, 20498, 20499, 20500, 20501, 20502, 20503, 20504, 20509, 20510, 20512, 20513, 20514, 20515, 20516, 20518, 20519, 20520, 20521, 20522, 20523, 20524, 20525, 20526, 20527, 20528, 20529, 20535, 20536, 20537, 20538, 20539, 20541, 20543, 20546, 20547, 20550, 20551, 20554, 20555, 20556, 20557, 20558, 20559, 20560, 20561, 20562, 20563, 20564, 20565, 20566, 20571, 20572, 20575, 20579, 20581, 20582, 20583, 20584, 20585, 20591, 20592, 20593, 20594, 20595, 20596, 20597, 20598, 20599, 20600, 20601, 20602, 20603, 20605, 20607, 20608, 20609, 20610, 20611, 20612, 20613, 20614, 20615, 20616, 20624, 20625, 20626, 20629, 20631, 20632, 20634, 20635, 20636, 20637, 20638, 20639, 20640, 20641, 20643, 20644, 20646, 20647, 20648, 20649, 20656, 20658, 20660, 20661, 20662, 20664, 20665, 20668, 20669, 20670, 20671, 20672, 20674, 20675, 20676, 20677, 20678, 20679, 20680, 20681, 20685, 20686, 20687, 20688, 20689, 20696, 20697, 20700, 20701, 20702, 20704, 20705, 20706, 20707, 20708, 20709, 20710, 20711, 20715, 20716, 20717, 20718, 20722, 20723, 20724, 20725, 20726, 20727, 20728, 20729, 20731, 20732, 20733, 20734, 20735, 20736, 20737, 20739, 20740, 20741, 20742, 20743, 20744, 20745, 20746, 20747, 20748, 20749, 20750, 20751, 20752, 20753, 20754, 20755, 20756, 20757, 20759, 20760, 20761, 20764, 20765, 20766, 20767, 20769, 20774, 20776, 20778, 20779, 20780, 20781, 20782, 20783, 20784, 20785, 20786, 20787, 20788, 20789, 20793, 20794, 20795, 20796, 20797, 20798, 20801, 20802, 20803, 20804, 20806, 20807, 20808, 20809, 20810, 20811, 20812, 20813, 20814, 20815, 20816, 20817, 20818, 20819, 20822, 20823, 20824, 20825, 20826, 20827, 20833, 20834, 20835, 20836, 20837, 20838, 20839, 20840, 20841, 20843, 20844, 20845, 20846, 20847, 20848, 20849, 20853, 20855, 20857, 20858, 20861, 20862, 20863, 20864, 20865, 20866, 20867, 20868, 20869, 20870, 20871, 20872, 20873, 20874, 20878, 20881, 20882, 20883, 20886, 20887, 20888, 20893, 20895, 20896, 20897, 20898, 20899, 20900, 20901, 20902, 20903, 20904, 20905, 20906, 20907, 20914, 20915, 20917, 20918, 20919, 20920, 20921, 20922, 20923, 20924, 20925, 20926, 20927, 20928, 20929, 20930, 20931, 20932, 20939, 20943, 20944, 20945, 20947, 20948, 20949, 20950, 20951, 20952, 20954, 20956, 20957, 20958, 20959, 20960, 20961, 20962, 20964, 20965, 20966, 20977, 20979, 20980, 20981, 20982, 20987, 20990, 20992, 20993, 20994, 20999, 21000, 21001, 21002, 21012, 21013, 21017, 21018, 21021, 21023, 21027, 21028, 21031, 21032, 21033, 21034, 21035, 21037, 21038, 21041, 21042, 21043, 21044, 21048, 21051, 21053, 21054, 21055, 21056, 21058, 21059, 21060, 21062, 21063, 21064, 21067, 21069, 21070, 21071, 21072, 21073, 21074, 21075, 21076, 21077, 21078, 21079, 21080, 21081, 21082, 21083, 21085, 21086, 21087, 21091, 21093, 21094, 21097, 21098, 21099, 21100, 21101, 21102, 21103, 21108, 21114, 21115, 21116, 21117, 21118, 21120, 21121, 21122, 21123, 21124, 21125, 21126, 21127, 21128, 21129, 21130, 21131, 21132, 21133, 21134, 21135, 21136, 21137, 21138, 21139, 21142, 21143, 21144, 21145, 21146, 21147, 21158, 21160, 21163, 21168, 21169, 21174, 21177, 21178, 21179, 21180, 21181, 21182, 21183, 21185, 21186, 21188, 21189, 21191, 21192, 21193, 21194, 21197, 21198, 21199, 21202, 21203, 21206, 21207, 21208, 21211, 21212, 21213, 21214, 21215, 21216, 21217, 21218, 21219, 21220, 21221, 21227, 21228, 21229, 21231, 21232, 21233, 21234, 21235, 21236, 21237, 21238, 21239, 21240, 21241, 21242, 21243, 21244, 21253, 21254, 21255, 21256, 21259, 21260, 21261, 21262, 21270, 21271, 21272, 21273, 21274, 21277, 21279, 21281, 21284, 21285, 21292, 21293, 21296, 21297, 21301, 21302, 21303, 21304, 21305, 21306, 21308, 21309, 21310, 21311, 21312, 21313, 21317, 21320, 21323, 21324, 21325, 21326, 21329, 21330, 21331, 21332, 21333, 21337, 21338, 21340, 21341, 21342, 21344, 21345, 21347, 21348, 21349, 21350, 21351, 21352, 21353, 21356, 21357, 21359, 21360, 21361, 21362, 21363, 21364, 21367, 21369, 21370, 21371, 21372, 21374, 21375, 21376, 21377, 21378, 21379, 21380, 21381, 21382, 21383, 21384, 21385, 21389, 21390, 21391, 21392, 21393, 21394, 21395, 21399, 21400, 21401, 21402, 21406, 21411, 21412, 21415, 21416, 21417, 21418, 21419, 21420, 21421, 21422, 21423, 21424, 21426, 21427, 21428, 21429, 21430, 21431, 21433, 21434, 21435, 21437, 21439, 21440, 21441, 21442, 21443, 21444, 21445, 21447, 21448, 21449, 21450, 21451, 21452, 21453, 21454, 21455, 21456, 21457, 21458, 21463, 21469, 21470, 21471, 21472, 21473, 21476, 21478, 21479, 21481, 21482, 21487, 21488, 21489, 21490, 21491, 21492, 21493, 21494, 21495, 21496, 21497, 21498, 21499, 21500, 21501, 21502, 21503, 21504, 21505, 21509, 21510, 21511, 21513, 21516, 21517, 21518, 21519, 21520, 21521, 21522, 21523, 21525, 21526, 21527, 21528, 21529, 21530, 21531, 21532, 21533, 21534, 21535, 21537, 21538, 21540, 21541, 21542, 21543, 21544, 21545, 21546, 21547, 21548, 21550, 21551, 21552, 21553, 21554, 21561, 21562, 21564, 21567, 21570, 21571, 21573, 21574, 21575, 21577, 21578, 21580, 21581, 21582, 21583, 21584, 21585, 21586, 21587, 21588, 21589, 21590, 21591, 21592, 21595, 21597, 21598, 21603, 21604, 21605, 21606, 21607, 21608, 21609, 21610, 21611, 21612, 21613, 21615, 21616, 21617, 21623, 21625, 21627, 21628, 21630, 21631, 21633, 21634, 21635, 21636, 21638, 21639, 21640, 21642, 21643, 21644, 21645, 21646, 21647, 21648, 21649, 21650, 21651, 21652, 21653, 21654, 21655, 21656, 21660, 21661, 21662, 21663, 21664, 21665, 21666, 21667, 21668, 21674, 21677, 21678, 21682, 21687, 21688, 21689, 21691, 21692, 21693, 21694, 21695, 21697, 21699, 21700, 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, 21709, 21710, 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, 21719, 21720, 21721, 21722, 21723, 21724, 21726, 21727, 21728, 21729, 21730, 21731, 21732, 21733, 21748, 21749, 21751, 21753, 21754, 21756, 21757, 21758, 21759, 21760, 21764, 21765, 21766, 21767, 21768, 21769, 21775, 21779, 21780, 21781, 21782, 21783, 21784, 21785, 21786, 21787, 21788, 21789, 21790, 21791, 21796, 21797, 21798, 21799, 21800, 21801, 21802, 21803, 21804, 21805, 21806, 21807, 21808, 21809, 21810, 21812, 21813, 21814, 21815, 21816, 21817, 21818, 21819, 21820, 21821, 21822, 21823, 21826, 21827, 21828, 21829, 21830, 21831, 21832, 21833, 21834, 21835, 21836, 21840, 21845, 21846, 21847, 21851, 21855, 21859, 21860, 21861, 21862, 21865, 21868, 21869, 21870, 21871, 21872, 21873, 21874, 21875, 21876, 21877, 21878, 21879, 21883, 21891, 21892, 21893, 21901, 21907, 21909, 21910, 21911, 21912, 21913, 21916, 21917, 21918, 21919, 21920, 21921, 21922, 21933, 21934, 21935, 21936, 21937, 21938, 21939, 21943, 21944, 21949, 21951, 21952, 21957, 21960, 21961, 21963, 21967, 21968, 21969, 21971, 21972, 21977, 21978, 21981, 21984, 21985, 21986, 21987, 21988, 21989, 21990, 21992, 21993, 21994, 21995, 21998, 22002, 22003, 22004, 22005, 22006, 22034, 22040, 22041, 22042, 22043, 22044, 22045, 22055, 22063, 22070, 22073, 22074, 22075, 22076, 22081, 22082, 22083, 22084, 22085, 22090, 22098, 22100, 22102, 22103, 22104, 22105, 22108, 22109, 22113, 22114, 22115, 22116, 22117, 22118, 22119, 22120, 22123, 22142, 22147, 22148, 22149, 22153, 22155, 22158, 22159, 22162, 22163, 22164, 22165, 22167, 22168, 22169, 22170, 22171, 22172, 22177, 22191, 22194, 22195, 22196, 22199, 22206, 22207, 22208, 22209, 22210, 22211, 22212, 22213, 22214, 22215, 22216, 22217, 22221, 22222, 22230, 22231, 22233, 22244, 22245, 22252, 22253, 22257, 22259, 22260, 22261, 22262, 22266, 22269, 22279, 22282, 22283, 22284, 22285, 22286, 22288, 22291, 22292, 22293, 22294, 22295, 22298, 22299, 22300, 22301, 22302, 22303, 22304, 22305, 22309, 22310, 22312, 22313, 22314, 22315, 22316, 22317, 22319, 22320, 22321, 22322, 22323, 22325, 22326, 22327, 22332, 22333, 22334, 22335, 22336, 22337, 22338, 22347, 22349, 22350, 22360, 22367, 22368, 22369, 22372, 22373, 22374, 22375, 22376, 22377, 22378, 22379, 22389, 22402, 22403, 22404, 22405, 22408, 22411, 22412, 22413, 22414, 22415, 22422, 22423, 22424, 22425, 22427, 22432, 22434, 22435, 22437, 22438, 22439, 22440, 22445, 22446, 22447, 22448, 22449, 22450, 22456, 22457, 22460, 22461, 22462, 22465, 22466, 22469, 22470, 22475, 22478, 22479, 22486, 22488, 22489, 22490, 22494, 22495, 22499, 22506, 22510, 22511, 22512, 22513, 22514, 22515, 22516, 22517, 22518, 22519, 22530, 22531, 22532, 22533, 22538, 22549, 22552, 22553, 22554, 22558, 22559, 22560, 22563, 22564, 22565, 22566, 22570, 22571, 22572, 22573, 22574, 22575, 22576, 22577, 22578, 22579, 22581, 22582, 22584, 22585, 22586, 22587, 22588, 22589, 22590, 22592, 22593, 22594, 22597, 22598, 22599, 22600, 22601, 22602, 22603, 22604, 22605, 22606, 22607, 22608, 22609, 22611, 22612, 22613, 22614, 22619, 22620, 22621, 22622, 22623, 22624, 22625, 22626, 22638, 22639, 22640, 22641, 22642, 22643, 22662, 22665, 22666, 22667, 22668, 22671, 22672, 22678, 22688, 22695, 22696, 22702, 22703, 22708, 22709, 22714, 22718, 22719, 22730, 22733, 22734, 22735, 22737, 22739, 22740, 22741, 22742, 22743, 22744, 22745, 22746, 22747, 22753, 22765, 22767, 22768, 22769, 22773, 22775, 22776, 22777, 22779, 22785, 22788, 22790, 22791, 22793, 22794, 22795, 22797, 22799, 22800, 22801, 22802, 22811, 22812, 22820, 22821, 22822, 22823, 22824, 22825, 22826, 22827, 22828, 22838, 22839, 22840, 22841, 22859, 22865, 22866, 22870, 22875, 22876, 22877, 22878, 22879, 22884, 22885, 22887, 22892, 22893, 22894, 22912, 22913, 22914, 22918, 22921, 22923, 22925, 22928, 22934, 22936, 22937, 22939, 22940, 22941, 22942, 22947, 22948, 22949, 22950, 22951, 22954, 22957, 22961, 22962, 22963, 22964, 22965, 22967, 22968, 22969, 22974, 22975, 22976, 22977, 22978, 22980, 22981, 22982, 22983, 22984, 22985, 22991, 22993, 22996, 22997, 22998, 23008, 23009, 23010, 23011, 23013, 23016, 23017, 23018, 23023, 23025, 23026, 23027, 23034, 23037, 23042, 23044, 23049, 23050, 23051, 23052, 23053, 23058, 23060, 23061, 23062, 23063, 23066, 23069, 23070, 23081, 23082, 23086, 23087, 23089, 23090, 23092, 23093, 23095, 23096, 23097, 23098, 23099, 23100, 23101, 23102, 23103, 23104, 23105, 23107, 23113, 23114, 23115, 23120, 23123, 23124, 23125, 23126, 23133, 23135, 23138, 23139, 23140, 23142, 23143, 23146, 23147, 23150, 23151, 23152, 23153, 23154, 23159, 23161, 23162, 23163, 23164, 23165, 23166, 23167, 23169, 23170, 23172, 23173, 23174, 23177, 23178, 23180, 23181, 23183, 23184, 23185, 23186, 23187, 23190, 23191, 23192, 23193, 23194, 23195, 23196, 23197, 23198, 23199, 23200, 23202, 23205, 23206, 23207, 23208, 23209, 23210, 23211, 23212, 23217, 23218, 23231, 23232, 23233, 23240, 23241, 23245, 23246, 23247, 23248, 23249, 23250, 23251, 23260, 23262, 23264, 23265, 23272, 23273, 23274, 23276, 23277, 23278, 23282, 23283, 23298, 23299, 23302, 23303, 23304, 23308, 23316, 23317, 23320, 23321, 23325, 23326, 23329, 23331, 23332, 23335, 23336, 23337, 23338, 23339, 23340, 23341, 23342, 23343, 23344, 23345, 23349, 23350, 23351, 23354, 23355, 23356, 23357, 23358, 23359, 23360, 23361, 23362, 23363, 23364, 23365, 23366, 23367, 23368, 23369, 23372, 23373, 23374, 23375, 23384, 23391, 23392, 23396, 23397, 23398, 23399, 23401, 23405, 23406, 23410, 23411, 23412, 23413, 23414, 23415, 23416, 23417, 23418, 23419, 23422, 23423, 23426, 23427, 23428, 23429, 23430, 23435, 23436, 23437, 23438, 23439, 23440, 23445, 23446, 23456, 23457, 23458, 23459, 23460, 23461, 23462, 23467, 23470, 23482, 23483, 23489, 23490, 23492, 23493, 23494, 23495, 23503, 23506, 23507, 23508, 23513, 23514, 23515, 23516, 23517, 23518, 23533, 23534, 23535, 23539, 23540, 23547, 23551, 23552, 23553, 23554, 23555, 23556, 23557, 23558, 23559, 23560, 23574, 23576, 23577, 23585, 23586, 23589, 23590, 23591, 23592, 23593, 23594, 23607, 23610, 23611, 23612, 23613, 23614, 23615, 23616, 23617, 23618, 23619, 23620, 23625, 23626, 23627, 23630, 23631, 23632, 23637, 23645, 23675, 23689, 23693, 23696, 23697, 23698, 23699, 23704, 23705, 23706, 23707, 23708, 23709, 23710, 23720, 23721, 23722, 23735, 23738, 23739, 23741, 23742, 23743, 23751, 23753, 23754, 23760, 23762, 23763, 23764, 23766, 23767, 23769, 23770, 23771, 23772, 23773, 23774, 23786, 23789, 23794, 23795, 23797, 23798, 23804, 23805, 23806, 23807, 23817, 23818, 23819, 23820, 23821, 23822, 23823, 23824, 23825, 23832, 23836, 23840, 23841, 23843, 23844, 23845, 23846, 23847, 23848, 23853, 23854, 23855, 23856, 23858, 23860, 23864, 23867, 23868, 23869, 23874, 23875, 23876, 23878, 23880, 23881, 23886, 23887, 23888, 23889, 23890, 23892, 23893, 23900, 23901, 23903, 23904, 23905, 23906, 23907, 23908, 23909, 23920, 23921, 23929, 23933, 23939, 23940, 23941, 23942, 23945, 23949, 23950, 23951, 23952, 23964, 23970, 23971, 23972, 23976, 23981, 23993, 23995, 23998, 23999, 24005, 24008, 24009, 24010, 24014, 24021, 24022, 24025, 24026, 24029, 24040, 24046, 24047, 24051, 24052, 24053, 24054, 24055, 24056, 24057, 24058, 24059, 24064, 24067, 24068, 24069, 24071, 24072, 24073, 24074, 24075, 24076, 24077, 24081, 24088, 24089, 24090, 24104, 24106, 24107, 24108, 24109, 24110, 24122, 24124, 24132, 24133, 24145, 24146, 24147, 24148, 24149, 24150, 24151, 24152, 24156, 24157, 24158, 24159, 24160, 24161, 24175, 24180, 24182, 24183, 24186, 24187, 24189, 24190, 24191, 24192, 24202, 24226, 24227, 24229, 24232, 24233, 24236, 24238, 24241, 24244, 24245, 24246, 24247, 24249, 24250, 24251, 24257, 24259, 24260, 24271, 24276, 24277, 24284, 24285, 24291, 24292, 24299, 24304, 24307, 24308, 24311, 24312, 24316, 24317, 24318, 24320, 24321, 24322, 24323, 24324, 24325, 24326, 24327, 24328, 24329, 24330, 24331, 24332, 24359, 24360, 24361, 24363, 24364, 24369, 24370, 24371, 24372, 24379, 24380, 24381, 24389, 24403, 24405, 24406, 24413, 24424, 24425, 24431, 24435, 24436, 24437, 24439, 24440, 24441, 24442, 24443, 24444, 24445, 24446, 24447, 24452, 24453, 24454, 24455, 24473, 24477, 24479, 24481, 24484, 24487, 24488, 24492, 24493, 24494, 24507, 24508, 24514, 24515, 24517, 24518, 24521, 24524, 24525, 24526, 24527, 24528, 24529, 24534, 24537, 24539, 24540, 24545, 24546, 24547, 24548, 24549, 24550, 24552, 24553, 24554, 24555, 24559, 24575, 24576, 24585, 24586, 24587, 24588, 24590, 24591, 24596, 24597, 24605, 24611, 24612, 24613, 24620, 24621, 24625, 24633, 24634, 24644, 24648, 24649, 24671, 24675, 24680, 24681, 24694, 24695, 24696, 24698, 24705, 24706, 24708, 24709, 24710, 24711, 24713, 24714, 24719, 24722, 24727, 24728, 24729, 24732, 24740, 24748, 24749, 24750, 24754, 24758, 24759, 24760, 24761, 24762, 24763, 24764, 24767, 24768, 24769, 24770, 24772, 24773, 24774, 24775, 24776, 24794, 24795, 24796, 24797, 24798, 24802, 24825, 24843, 24844, 24845, 24852, 24853, 24854, 24864, 24873, 24874, 24875, 24886, 24887, 24891, 24892, 24896, 24897, 24900, 24901, 24912, 24927, 24928, 24929, 24930, 24931, 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24940, 24941, 24942, 24943, 24949, 24950, 24951, 24952, 24953, 24954, 24955, 24961, 24966, 24968, 24969, 24970, 24971, 24972, 25004, 25005, 25006, 25009, 25022, 25024, 25030, 25034, 25035, 25038, 25040, 25057, 25058, 25060, 25061, 25062, 25063, 25064, 25077, 25078, 25079, 25080, 25081, 25086, 25087, 25095, 25096, 25117, 25118, 25119, 25122, 25138, 25149, 25150, 25151, 25152, 25153, 25154, 25155, 25158, 25159, 25161, 25175, 25176, 25177, 25178, 25188, 25201, 25202, 25209, 25212, 25214, 25217, 25218, 25220, 25222, 25223, 25224, 25225, 25226, 25240, 25241, 25242, 25243, 25244, 25259, 25267, 25272, 25273, 25274, 25275, 25276, 25288, 25290, 25291, 25296, 25297, 25321, 25322, 25325, 25334, 25335, 25336, 25338, 25339, 25341, 25342, 25348, 25349, 25350, 25351, 25352, 25365, 25371, 25372, 25377, 25379, 25381, 25382, 25383, 25384, 25385, 25387, 25388, 25392, 25393, 25394, 25395, 25396, 25397, 25398, 25399, 25403, 25406, 25415, 25416, 25417, 25419, 25420, 25431, 25442, 25445, 25450, 25451, 25456, 25457, 25458, 25459, 25462, 25465, 25466, 25467, 25468, 25469, 25470, 25471, 25472, 25473, 25477, 25486, 25487, 25488, 25489, 25504, 25506, 25507, 25508, 25509, 25528, 25529, 25547, 25548, 25550, 25554, 25555, 25556, 25557, 25558, 25559, 25588, 25590, 25591, 25592, 25593, 25594, 25595, 25596, 25624, 25635, 25636, 25643, 25653, 25655, 25663, 25664, 25665, 25666, 25667, 25668, 25669, 25670, 25671, 25672, 25677, 25685, 25686, 25695, 25698, 25699, 25700, 25701, 25702, 25703, 25704, 25705, 25706, 25707, 25708, 25718, 25721, 25722, 25723, 25724, 25725, 25735, 25739, 25740, 25743, 25748, 25749, 25762, 25763, 25764, 25769, 25770, 25784, 25786, 25787, 25788, 25789, 25790, 25796, 25801, 25827, 25828, 25840, 25845, 25846, 25847, 25848, 25849, 25857, 25860, 25861, 25869, 25871, 25872, 25873, 25885, 25891, 25899, 25918, 25930, 25931, 25937, 25940, 25941, 25947, 25960, 25963, 25967, 25968, 25969, 25970, 25971, 25972, 25978, 25979, 25980, 25981, 26006, 26007, 26015, 26017, 26018, 26019, 26021, 26028, 26055, 26066, 26067, 26073, 26084, 26086, 26087, 26096, 26097, 26098, 26099, 26100, 26101, 26102, 26130, 26136, 26137, 26147, 26148, 26160, 26171, 26173, 26174, 26178, 26183, 26184, 26185, 26186, 26188, 26190, 26191, 26193, 26194, 26195, 26205, 26218, 26220, 26222, 26225, 26226, 26227, 26236, 26237, 26238, 26243, 26269, 26279, 26280, 26281, 26285, 26297, 26310, 26318, 26319, 26331, 26341, 26343, 26345, 26346, 26347, 26378, 26379, 26391, 26392, 26408, 26418, 26419, 26420, 26425, 26442, 26443, 26454, 26455, 26456, 26469, 26473, 26486, 26487, 26488, 26490, 26494, 26495, 26496, 26520, 26521, 26522, 26529, 26532, 26533, 26537, 26539, 26541, 26542, 26543, 26544, 26545, 26560, 26568, 26569, 26570, 26571, 26572, 26582, 26589, 26592, 26609, 26610, 26611, 26612, 26613, 26614, 26615, 26625, 26632, 26649, 26662, 26676, 26677, 26678, 26685, 26692, 26696, 26697, 26701, 26702, 26703, 26704, 26717, 26718, 26719, 26724, 26725, 26726, 26737, 26738, 26760, 26761, 26777, 26778, 26786, 26790, 26794, 26803, 26811, 26815, 26833, 26836, 26837, 26841, 26842, 26848, 26849, 26850, 26857, 26884, 26890, 26894, 26895, 26901, 26902, 26906, 26907, 26914, 26916, 26921, 26922, 26923, 26925, 26926, 26940, 26950, 26951, 26953, 26954, 26955, 26956, 27001, 27002, 27003, 27004, 27005, 27009, 27010, 27024, 27026, 27029, 27040, 27041, 27042, 27046, 27047, 27068, 27069, 27070, 27085, 27086, 27103, 27115, 27120, 27123, 27124, 27150, 27155, 27171, 27172, 27179, 27197, 27201, 27206, 27207, 27208, 27224, 27225, 27226, 27227, 27243, 27244, 27245, 27246, 27280, 27284, 27285, 27293, 27299, 27300, 27319, 27321, 27322, 27323, 27324, 27330, 27331, 27343, 27344, 27349, 27351, 27352, 27353, 27354, 27361, 27370, 27381, 27409, 27412, 27413, 27415, 27417, 27418, 27460, 27471, 27472, 27493, 27524, 27539, 27543, 27544, 27551, 27553, 27554, 27555, 27556, 27575, 27578, 27579, 27580, 27596, 27601, 27620, 27624, 27625, 27626, 27627, 27628, 27668, 27669, 27684, 27687, 27690, 27699, 27714, 27724, 27726, 27727, 27729, 27730, 27734, 27738, 27743, 27744, 27746, 27762, 27763, 27764, 27792, 27796, 27797, 27807, 27810, 27822, 27824, 27825, 27838, 27844, 27845, 27853, 27854, 27855, 27864, 27865, 27924, 27926, 27928, 27929, 27940, 27941, 27948, 27949, 27977, 27978, 27991, 27997, 28007, 28010, 28013, 28027, 28028, 28029, 28030, 28031, 28073, 28074, 28075, 28100, 28101, 28108, 28115, 28126, 28127, 28128, 28142, 28153, 28154, 28169, 28179, 28182, 28189, 28210, 28211, 28212, 28216, 28223, 28224, 28225, 28243, 28277, 28311, 28332, 28333, 28336, 28337, 28338, 28352, 28353, 28358, 28364, 28370, 28374, 28398, 28407, 28408, 28419, 28424, 28463, 28482, 28491, 28492, 28496, 28513, 28514, 28534, 28548, 28551, 28552, 28553, 28562, 28572, 28593, 28594, 28595, 28607, 28629, 28640, 28647, 28650, 28651, 28656, 28661, 28741, 28758, 28759, 28786, 28792, 28793, 28795, 28796, 28801, 28802, 28804, 28807, 28814, 28825, 28826, 28844, 28855, 28856, 28872, 28906, 28916, 28926, 28942, 28953, 28954, 28964, 29013, 29021, 29052, 29053, 29082, 29104, 29115, 29131, 29177, 29192, 29243, 29249, 29250, 29269, 29298, 29320, 29321, 29345, 29372, 29377, 29430, 29447, 29450, 29493, 29518, 29519, 29520, 29546, 29560, 29605, 29616, 29620, 29628, 29632, 29633, 29640, 29644, 29645, 29653, 29664, 29680, 29735, 29736, 29748, 29791, 29816, 29831, 29868, 29871, 29876, 29877, 29886, 29897, 29898, 29905, 29908, 29958, 29982, 29984, 30046, 30082, 30086, 30120, 30121, 30133, 30172, 30183, 30184, 30191, 30217, 30236, 30358, 30359, 30366, 30406, 30446, 30451, 30553, 30569, 30597, 30598, 30599, 30695, 30696, 30701, 30796, 30853, 30879, 30880, 30883, 30968, 30984, 31030, 31031, 31043, 31153, 31189, 31196, 31348, 31401, 31440, 31725, 31726, 31768, 31958, 32017, 32358, 32397}, slot = 0 +cc d28b14f167a3950cfc2a5b82dff1e15c65e9ac23a5c249f812e69af96c3489ed # shrinks to coding_indices = 0..0, data_indices = 2984..15152, slot = 0 diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 2998172ff6cdc5..805ba539023e80 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -830,6 +830,7 @@ impl OptimisticSlotMetaVersioned { } } } + #[cfg(test)] mod test { use { @@ -901,23 +902,32 @@ mod test { } } + /// Generate a random Range. + fn rand_range(range: Range) -> impl Strategy> { + (range.clone(), range).prop_map( + // Avoid descending (empty) ranges + |(start, end)| { + if start > end { + end..start + } else { + start..end + } + }, + ) + } + proptest! { - // Property: ShredIndexV2 can be converted to ShredIndex and vice versa #[test] fn shred_index_legacy_compat( - shreds in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), - start in 0..MAX_DATA_SHREDS_PER_SLOT, - end in 0..MAX_DATA_SHREDS_PER_SLOT, + shreds in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), + range in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64) ) { let mut legacy = ShredIndex::default(); let mut v2 = ShredIndexV2::default(); for i in shreds { - v2.insert(i as u64); - legacy.insert(i as u64); + v2.insert(i); + legacy.insert(i); } for &i in legacy.index.iter() { @@ -925,7 +935,6 @@ mod test { } assert_eq!(v2.num_shreds(), legacy.num_shreds()); - let range = std::cmp::min(start, end) as u64..std::cmp::max(start, end) as u64; assert_eq!( v2.range(range.clone()).sum::(), @@ -939,19 +948,13 @@ mod test { // Property: Index cannot be deserialized from IndexV2 #[test] fn test_legacy_collision( - coding_indices in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), - data_indices in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), + coding_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), + data_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), slot in 0..u64::MAX ) { let index = IndexV2 { - coding: coding_indices.into_iter().map(|i| i as u64).collect(), - data: data_indices.into_iter().map(|i| i as u64).collect(), + coding: coding_indices.into_iter().collect(), + data: data_indices.into_iter().collect(), slot, }; let config = bincode::DefaultOptions::new().with_fixint_encoding().reject_trailing_bytes(); @@ -962,19 +965,13 @@ mod test { // Property: IndexV2 cannot be deserialized from Index #[test] fn test_legacy_collision_inverse( - coding_indices in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), - data_indices in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), + coding_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), + data_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), slot in 0..u64::MAX ) { let index = Index { - coding: coding_indices.into_iter().map(|i| i as u64).collect(), - data: data_indices.into_iter().map(|i| i as u64).collect(), + coding: coding_indices.into_iter().collect(), + data: data_indices.into_iter().collect(), slot, }; let config = bincode::DefaultOptions::new() @@ -987,32 +984,18 @@ mod test { // Property: range queries should return correct indices #[test] fn range_query_correctness( - indices in prop::collection::btree_set( - 0..MAX_DATA_SHREDS_PER_SLOT, - 0..MAX_DATA_SHREDS_PER_SLOT - ), - ranges in prop::collection::vec( - (0..MAX_DATA_SHREDS_PER_SLOT) - .prop_flat_map(|start| { - (start + 1..=MAX_DATA_SHREDS_PER_SLOT) - .prop_map(move |end| (start, end)) - }), - 1..1000 - ) + indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), ) { let mut index = ShredIndexV2::default(); - for &idx in &indices { - index.insert(idx as u64); + for idx in indices.clone() { + index.insert(idx); } - for (start, end) in ranges { - // Test indices match - assert_eq!( - index.range(start as u64..end as u64).collect::>(), - indices.range(start..end).map(|&i| i as u64).collect::>() - ); - } + assert_eq!( + index.range(indices.clone()).collect::>(), + indices.into_iter().collect::>() + ); } } From 674d6e739ea392a186762d0345ead8408487e9d3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 11 Jan 2025 17:15:57 +0000 Subject: [PATCH 42/44] Document (de)serialization failures between index types --- ledger/src/blockstore_meta.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore_meta.rs b/ledger/src/blockstore_meta.rs index 805ba539023e80..60d156b1642c00 100644 --- a/ledger/src/blockstore_meta.rs +++ b/ledger/src/blockstore_meta.rs @@ -945,7 +945,14 @@ mod test { assert_eq!(ShredIndex::from(v2), legacy); } - // Property: Index cannot be deserialized from IndexV2 + /// Property: [`Index`] cannot be deserialized from [`IndexV2`]. + /// + /// # Failure cases + /// 1. Empty [`IndexV2`] + /// - [`ShredIndex`] deserialization should fail due to trailing bytes of `num_shreds`. + /// 2. Non-empty [`IndexV2`] + /// - Encoded length of [`ShredIndexV2::index`] (`Vec`) will be relative to a sequence of `u8`, + /// resulting in not enough bytes when deserialized into sequence of `u64`. #[test] fn test_legacy_collision( coding_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), @@ -962,7 +969,14 @@ mod test { prop_assert!(legacy.is_err()); } - // Property: IndexV2 cannot be deserialized from Index + /// Property: [`IndexV2`] cannot be deserialized from [`Index`]. + /// + /// # Failure cases + /// 1. Empty [`Index`] + /// - [`ShredIndexV2`] deserialization should fail due to missing `num_shreds` (not enough bytes). + /// 2. Non-empty [`Index`] + /// - Encoded length of [`ShredIndex::index`] (`BTreeSet`) will be relative to a sequence of `u64`, + /// resulting in trailing bytes when deserialized into sequence of `u8`. #[test] fn test_legacy_collision_inverse( coding_indices in rand_range(0..MAX_DATA_SHREDS_PER_SLOT as u64), From c222ef25f0030b43ec0e96d8a2b44320f72ff63a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 11 Jan 2025 17:19:37 +0000 Subject: [PATCH 43/44] propagate Cargo.lock changes --- programs/sbf/Cargo.lock | 2 +- svm/examples/Cargo.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index f889756a190fc7..8666a479d748d1 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -3907,7 +3907,7 @@ checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.6.0", + "bitflags 2.7.0", "lazy_static", "num-traits", "rand 0.8.5", diff --git a/svm/examples/Cargo.lock b/svm/examples/Cargo.lock index db8b70566cec26..a2cbe7683d4584 100644 --- a/svm/examples/Cargo.lock +++ b/svm/examples/Cargo.lock @@ -3798,7 +3798,7 @@ checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.6.0", + "bitflags 2.7.0", "lazy_static", "num-traits", "rand 0.8.5", From 0b6928e42945532dc7788c96696dc0a757abd7fc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 11 Jan 2025 09:23:44 -0800 Subject: [PATCH 44/44] Revert "fix missing usage of `get_index` abstraction" This reverts commit b22c49ca72dc2b5fb38ac044bde44a2be057dcfd. --- ledger/src/blockstore.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 06fc486a9e46bd..4bff3aad447dba 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4711,7 +4711,8 @@ impl Blockstore { let mut total_start = Measure::start("Total elapsed"); let res = index_working_set.entry(slot).or_insert_with(|| { let newly_inserted_meta = self - .get_index(slot) + .index_cf + .get(slot) .unwrap() .unwrap_or_else(|| Index::new(slot)); IndexMetaWorkingSetEntry {