diff --git a/Cargo.lock b/Cargo.lock index eacb804bc3a0..9918088ddc04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,12 +20,12 @@ dependencies = [ "fil_types", "forest_address", "forest_bigint", - "forest_bitfield", "forest_cid", "forest_encoding", "forest_hash_utils", "forest_json_utils", "forest_vm", + "fvm_ipld_bitfield", "fvm_shared 0.7.1", "ipld_blockstore", "libp2p 0.40.0", @@ -3164,23 +3164,6 @@ dependencies = [ "serde_ipld_dagcbor 0.1.2", ] -[[package]] -name = "forest_bitfield" -version = "0.1.0" -dependencies = [ - "ahash 0.7.6", - "criterion", - "forest_encoding", - "fvm_ipld_bitfield", - "rand 0.8.5", - "rand_xorshift", - "serde", - "serde_bytes", - "serde_ipld_dagcbor 0.2.2", - "serde_json", - "unsigned-varint 0.7.1", -] - [[package]] name = "forest_blocks" version = "0.1.0" @@ -7073,7 +7056,6 @@ dependencies = [ "fil_types", "forest_address", "forest_bigint", - "forest_bitfield", "forest_blocks", "forest_cid", "forest_crypto", @@ -7083,6 +7065,7 @@ dependencies = [ "forest_libp2p", "forest_message", "futures", + "fvm_ipld_bitfield", "fvm_shared 0.7.1", "hex", "interpreter", @@ -7119,7 +7102,6 @@ dependencies = [ "fil_types", "forest_address", "forest_bigint", - "forest_bitfield", "forest_blocks", "forest_cid", "forest_crypto", @@ -7127,6 +7109,7 @@ dependencies = [ "forest_libp2p", "forest_message", "forest_vm", + "fvm_ipld_bitfield", "fvm_shared 0.7.1", "ipld_blockstore", "jsonrpc-v2", @@ -7868,7 +7851,6 @@ dependencies = [ "filecoin-proofs-api", "forest_address", "forest_bigint", - "forest_bitfield", "forest_blocks", "forest_cid", "forest_crypto", @@ -7879,6 +7861,7 @@ dependencies = [ "forest_vm", "futures", "fvm", + "fvm_ipld_bitfield", "fvm_shared 0.7.1", "interpreter", "ipld_amt 0.2.1", @@ -7905,13 +7888,13 @@ dependencies = [ "fil_types", "forest_address", "forest_bigint", - "forest_bitfield", "forest_cid", "forest_encoding", "forest_hash_utils", "forest_json_utils", "forest_vm", "futures", + "fvm_ipld_bitfield", "fvm_shared 0.7.1", "ipld_blockstore", "libp2p 0.40.0", diff --git a/Cargo.toml b/Cargo.toml index 8a790b0278f0..c09d7d725717 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ members = [ "types/networks", "utils/auth", "utils/bigint", - "utils/bitfield", "utils/genesis", "utils/hash_utils", "utils/json_utils", @@ -51,7 +50,6 @@ forest_db = { path = "./node/db" } forest_crypto = { path = "./crypto" } forest_address = { path = "./vm/address" } forest_bigint = { path = "./utils/bigint" } -forest_bitfield = { path = "./utils/bitfield" } forest_cid = { path = "./ipld/cid" } forest_ipld = { path = "./ipld" } forest_encoding = { path = "./encoding" } diff --git a/Makefile b/Makefile index eb7ba8e2245e..08c5b1ab87f3 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,6 @@ clean: @cargo clean -p forest_ipld @cargo clean -p ipld_amt @cargo clean -p forest_bigint - @cargo clean -p forest_bitfield @cargo clean -p fil_types @cargo clean -p ipld_blockstore @cargo clean -p rpc diff --git a/blockchain/state_manager/Cargo.toml b/blockchain/state_manager/Cargo.toml index ce5e48660012..48289a34d6dd 100644 --- a/blockchain/state_manager/Cargo.toml +++ b/blockchain/state_manager/Cargo.toml @@ -11,6 +11,7 @@ features = ["statediff"] anyhow = "1.0" fvm = "0.8" fvm_shared = { version = "0.7.1", default-features = false } +fvm_ipld_bitfield = "0.5.2" fil_actors_runtime = "=7.5.1" address = { package = "forest_address", version = "0.3" } actor = { package = "actor_interface", path = "../../vm/actor_interface" } @@ -39,7 +40,6 @@ message = { package = "forest_message", version = "0.7", features = [ "blst", ] } vm = { package = "forest_vm", version = "0.3.1" } -bitfield = { package = "forest_bitfield", version = "0.1" } serde = { version = "1.0", features = ["derive"] } num-traits = "0.2.11" tokio = { version = "1.0", features = ["sync"] } diff --git a/blockchain/state_manager/src/utils.rs b/blockchain/state_manager/src/utils.rs index 2e976086005e..5c9f7df50020 100644 --- a/blockchain/state_manager/src/utils.rs +++ b/blockchain/state_manager/src/utils.rs @@ -8,7 +8,6 @@ use actor::{ power, }; use address::Address; -use bitfield::BitField; use blockstore::BlockStore; use cid::Cid; use fil_types::{ @@ -16,6 +15,7 @@ use fil_types::{ SectorNumber, }; use forest_blocks::Tipset; +use fvm_ipld_bitfield::BitField; use interpreter::resolve_to_key_addr; use serde::Serialize; use state_tree::StateTree; diff --git a/node/rpc-api/Cargo.toml b/node/rpc-api/Cargo.toml index 12df11397616..6c91cfe233d8 100644 --- a/node/rpc-api/Cargo.toml +++ b/node/rpc-api/Cargo.toml @@ -9,7 +9,6 @@ edition = "2021" actor = { package = "actor_interface", path = "../../vm/actor_interface" } address = { package = "forest_address", version = "0.3", features = ["json"] } beacon = { package = "beacon", path = "../../blockchain/beacon", features = ["json"] } -bitfield = { package = "forest_bitfield", version = "0.1", features = ["json"] } blocks = { package = "forest_blocks", path = "../../blockchain/blocks", features = ["json"] } blockstore = { package = "ipld_blockstore", version = "0.1" } chain = { path = "../../blockchain/chain", features = ["json"] } @@ -32,6 +31,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = "1.0" libp2p = { version = "0.40.0-rc.1", default-features = false } fvm_shared = { version = "0.7.1", default-features = false } +fvm_ipld_bitfield = { version = "0.5.2", features = ["json"] } [dependencies.jsonrpc-v2] version = "0.10" diff --git a/node/rpc-api/src/data_types.rs b/node/rpc-api/src/data_types.rs index b1797257fec0..9ddba50882e6 100644 --- a/node/rpc-api/src/data_types.rs +++ b/node/rpc-api/src/data_types.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use actor::market::{DealProposal, DealState}; use address::{json::AddressJson, Address}; use beacon::{json::BeaconEntryJson, Beacon, BeaconSchedule}; -use bitfield::json::BitFieldJson; use blocks::{ election_proof::json::ElectionProofJson, ticket::json::TicketJson, tipset_keys_json::TipsetKeysJson, Tipset, @@ -23,6 +22,7 @@ use cid::{json::CidJson, Cid}; use fil_types::{json::SectorInfoJson, sector::post::json::PoStProofJson}; pub use forest_libp2p::{Multiaddr, Protocol}; use forest_libp2p::{Multihash, NetworkMessage}; +use fvm_ipld_bitfield::json::BitFieldJson; use fvm_shared::bigint::BigInt; use fvm_shared::clock::ChainEpoch; use ipld::json::IpldJson; diff --git a/node/rpc-api/src/lib.rs b/node/rpc-api/src/lib.rs index 15ae1a30eac8..e5ed22addfe9 100644 --- a/node/rpc-api/src/lib.rs +++ b/node/rpc-api/src/lib.rs @@ -360,12 +360,12 @@ pub mod state_api { MinerInfo, MinerPower, SectorOnChainInfo, SectorPreCommitInfo, SectorPreCommitOnChainInfo, }; use address::json::AddressJson; - use bitfield::json::BitFieldJson; use blocks::{ gossip_block::json::GossipBlockJson as BlockMsgJson, tipset_keys_json::TipsetKeysJson, }; use cid::json::CidJson; use fil_types::{deadlines::DeadlineInfo, NetworkVersion, SectorNumber}; + use fvm_ipld_bitfield::json::BitFieldJson; use fvm_shared::clock::ChainEpoch; use message::{ message_receipt::json::MessageReceiptJson, unsigned_message::json::UnsignedMessageJson, diff --git a/node/rpc/Cargo.toml b/node/rpc/Cargo.toml index c568a1dfcbe7..38457f465454 100644 --- a/node/rpc/Cargo.toml +++ b/node/rpc/Cargo.toml @@ -32,7 +32,6 @@ auth = { path = "../../utils/auth" } beacon = { package = "beacon", path = "../../blockchain/beacon", features = [ "json", ] } -bitfield = { package = "forest_bitfield", version = "0.1", features = ["json"] } blocks = { package = "forest_blocks", path = "../../blockchain/blocks", features = [ "json", ] } @@ -64,6 +63,7 @@ wallet = { package = "key_management", path = "../../key_management", features = "json", ] } fvm_shared = { version = "0.7.1", default-features = false } +fvm_ipld_bitfield = "0.5.2" [dependencies.jsonrpc-v2] version = "0.10" diff --git a/node/rpc/src/state_api.rs b/node/rpc/src/state_api.rs index 7b285d7d81a7..7ffd76e793ca 100644 --- a/node/rpc/src/state_api.rs +++ b/node/rpc/src/state_api.rs @@ -761,9 +761,9 @@ pub(crate) async fn state_miner_sector_allocated< miner::State::V7(m) => data .chain_store .db - .get::(&m.allocated_sectors)? + .get::(&m.allocated_sectors)? .ok_or("allocated sectors bitfield not found")? - .get(sector_num as usize), + .get(sector_num), }; Ok(allocated_sectors) diff --git a/utils/bitfield/Cargo.toml b/utils/bitfield/Cargo.toml deleted file mode 100644 index 6d1b723470f5..000000000000 --- a/utils/bitfield/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "forest_bitfield" -description = "Bitfield logic for use in Filecoin actors" -version = "0.1.0" -license = "MIT OR Apache-2.0" -authors = ["ChainSafe Systems "] -edition = "2021" -repository = "https://github.com/ChainSafe/forest" - -[dependencies] -fvm_ipld_bitfield = "0.5.2" -unsigned-varint = "0.7" -serde = { version = "1.0", features = ["derive"] } -serde_bytes = "0.11" -ahash = "0.7" -encoding = { package = "forest_encoding", version = "0.2" } - -[dev-dependencies] -rand_xorshift = "0.3" -rand = "0.8" -criterion = "0.3" -serde_json = "1.0" -serde_ipld_dagcbor = "0.2" - -[features] -json = [] - -[[bench]] -name = "benchmarks" -harness = false diff --git a/utils/bitfield/benches/benchmarks/examples.rs b/utils/bitfield/benches/benchmarks/examples.rs deleted file mode 100644 index f9f89014116c..000000000000 --- a/utils/bitfield/benches/benchmarks/examples.rs +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -use forest_bitfield::BitField; - -/// An example bit field. -pub fn example1() -> BitField { - BitField::from_bytes(EXAMPLE1).unwrap() -} - -/// Another example bit field. -pub fn example2() -> BitField { - BitField::from_bytes(EXAMPLE2).unwrap() -} - -// taken from go-bitfield -// properties: -// - 1362 runs -// - 54 single bit -// - 399 small block -// - 909 large block -// - 9955 set bits between 0 and ~1M -// - 681 ranges -const EXAMPLE1: &[u8] = &[ - 0x20, 0xfc, 0x40, 0xc2, 0xcc, 0xe5, 0xd8, 0xc1, 0xe1, 0x1e, 0x23, 0xd3, 0x02, 0x2e, 0xcd, 0x03, - 0x3e, 0x83, 0x16, 0x26, 0x3a, 0x5c, 0x30, 0x8e, 0x00, 0x05, 0xcc, 0x24, 0x0e, 0x96, 0x15, 0x48, - 0xa0, 0x2a, 0x40, 0x04, 0x92, 0x0e, 0x94, 0xc0, 0x48, 0x91, 0xee, 0x05, 0x28, 0x98, 0x55, 0x90, - 0xa0, 0xa4, 0x50, 0x98, 0x14, 0x40, 0x04, 0x59, 0x0a, 0x22, 0x00, 0x74, 0xb0, 0x40, 0x66, 0x30, - 0xf9, 0x66, 0x90, 0x51, 0xc7, 0x70, 0x74, 0x40, 0x48, 0x7f, 0xf0, 0x80, 0x24, 0x20, 0x85, 0x58, - 0x06, 0x07, 0x66, 0x04, 0x87, 0xe5, 0x05, 0x28, 0x00, 0xa4, 0xf0, 0x61, 0x2e, 0x90, 0x08, 0x44, - 0x70, 0x38, 0x34, 0xf0, 0x08, 0x4f, 0x70, 0x68, 0xad, 0xd0, 0x90, 0x1e, 0x90, 0x38, 0xc1, 0x85, - 0x76, 0x04, 0x15, 0x7c, 0x04, 0x28, 0x28, 0x17, 0x70, 0xe0, 0x15, 0x00, 0x82, 0xfb, 0x11, 0x0b, - 0x76, 0x09, 0x22, 0xb8, 0x2f, 0x90, 0x20, 0x5f, 0x80, 0x84, 0xc9, 0x10, 0x85, 0x66, 0x09, 0x05, - 0xc9, 0x03, 0x2d, 0x19, 0xa4, 0x5a, 0x70, 0x45, 0xa5, 0x40, 0xc2, 0x05, 0x6c, 0x4a, 0x0f, 0x64, - 0xf4, 0x19, 0xb0, 0x40, 0x28, 0x8b, 0x02, 0x6c, 0x20, 0x3e, 0x90, 0x40, 0x01, 0x19, 0xea, 0x09, - 0x20, 0x60, 0x2f, 0x50, 0x60, 0x15, 0x00, 0x04, 0x69, 0x06, 0xb1, 0x8c, 0xa9, 0x85, 0xc6, 0x1f, - 0x13, 0x54, 0x3e, 0x58, 0x40, 0x17, 0x60, 0x41, 0x2e, 0x8a, 0x42, 0x3c, 0x8b, 0x0b, 0x3f, 0x08, - 0x10, 0x5c, 0x32, 0x38, 0xd4, 0x1e, 0x68, 0x18, 0x5b, 0x70, 0xc1, 0x2c, 0x06, 0x17, 0x6c, 0x17, - 0xf8, 0x74, 0x36, 0x28, 0x9c, 0x0e, 0x20, 0x02, 0xa1, 0x84, 0x0f, 0xa2, 0x82, 0x0e, 0xbf, 0x82, - 0x45, 0x29, 0x84, 0x07, 0x16, 0x25, 0x10, 0x12, 0x38, 0x8c, 0x13, 0x50, 0x41, 0xff, 0x03, 0x13, - 0x70, 0x16, 0x80, 0x81, 0x63, 0x85, 0x8f, 0xa0, 0x81, 0x0a, 0xe1, 0x80, 0x84, 0xa3, 0x81, 0x89, - 0xff, 0x84, 0xc7, 0x3c, 0x87, 0x49, 0xbb, 0x81, 0x4b, 0x21, 0x82, 0x03, 0x2f, 0x83, 0x84, 0x29, - 0x83, 0x8c, 0xe0, 0x00, 0x40, 0x8c, 0xda, 0x20, 0x42, 0x06, 0x13, 0x64, 0x07, 0x24, 0x85, 0x0c, - 0xa6, 0x21, 0x24, 0x6c, 0x07, 0x74, 0x20, 0x13, 0x3c, 0x02, 0x13, 0x54, 0xba, 0x1d, 0x14, 0x9e, - 0x0b, 0xe0, 0x00, 0x39, 0x82, 0x0a, 0x72, 0x0b, 0x80, 0x60, 0x53, 0x20, 0xd5, 0x0e, 0x32, 0xd5, - 0x0e, 0x12, 0xb9, 0x01, 0x44, 0x90, 0x27, 0x51, 0x59, 0xe0, 0x12, 0xc3, 0x04, 0xe8, 0x01, 0x78, - 0x40, 0xd8, 0xe0, 0x03, 0x7b, 0x60, 0x93, 0x6b, 0x20, 0x72, 0xf8, 0x20, 0xf1, 0xe1, 0xe0, 0x4d, - 0x90, 0xdf, 0x02, 0x1f, 0x69, 0x03, 0x36, 0x30, 0x8d, 0x20, 0x82, 0x58, 0x0a, 0x8f, 0xec, 0x02, - 0x8d, 0xe8, 0x00, 0x11, 0x55, 0x81, 0x58, 0x17, 0xa0, 0x82, 0xa5, 0x81, 0x88, 0xb6, 0x83, 0x88, - 0xed, 0x86, 0x88, 0xe0, 0x86, 0x4b, 0x6e, 0x82, 0x45, 0xe1, 0x89, 0xc8, 0x2c, 0x41, 0x78, 0x09, - 0x12, 0xbd, 0x91, 0x50, 0x01, 0x42, 0x90, 0x85, 0x70, 0x69, 0xb5, 0x20, 0x83, 0x5e, 0x02, 0x8d, - 0xfb, 0x02, 0x0f, 0xfb, 0x0a, 0x09, 0xcc, 0x04, 0x8b, 0xe4, 0x05, 0x87, 0xc9, 0x02, 0x0f, 0xfc, - 0x01, 0x5c, 0x08, 0x0f, 0xd0, 0x01, 0x4e, 0xd0, 0x60, 0x4d, 0x50, 0xa1, 0x3c, 0xf0, 0x20, 0x6c, - 0x88, 0xe1, 0x41, 0x5c, 0x11, 0x1c, 0xc8, 0x25, 0x4c, 0xc0, 0x1d, 0x1c, 0x4e, 0x61, 0x54, 0x2a, - 0x05, 0x14, 0xf6, 0x54, 0x3c, 0x0b, 0x44, 0xe4, 0x05, 0x64, 0x42, 0x1f, 0x64, 0x32, 0x23, 0xa0, - 0x20, 0xf9, 0x01, 0x0c, 0x40, 0x27, 0xf8, 0x40, 0x3c, 0xc0, 0x82, 0x72, 0x01, 0x0d, 0xd1, 0x43, - 0x11, 0x01, 0x0d, 0xaa, 0x35, 0xc8, 0xe0, 0x1e, 0x42, 0x41, 0x5b, 0x20, 0x8b, 0x04, 0x0e, 0xc9, - 0x03, 0x0a, 0xea, 0x01, 0x3e, 0x5f, 0x12, 0xab, 0x07, 0x40, 0xf0, 0xad, 0x60, 0x42, 0x6f, 0xa1, - 0xd2, 0x39, 0x60, 0x91, 0x64, 0xf1, 0xe4, 0x10, 0x3f, 0x60, 0xc2, 0x5c, 0x94, 0x62, 0x07, 0x1f, - 0x67, 0x05, 0x87, 0xcc, 0x07, 0x9f, 0xee, 0x0e, 0x13, 0xd8, 0x02, 0x38, 0xa0, 0xb5, 0x20, 0x02, - 0x4e, 0x01, 0x85, 0x7c, 0x0c, 0x4e, 0xb0, 0xde, 0x90, 0x80, 0x2c, 0x50, 0xd8, 0x77, 0x88, 0xf6, - 0x80, 0x04, 0x22, 0x85, 0xce, 0xf2, 0x82, 0xcb, 0xb4, 0x05, 0x14, 0x68, 0xa9, 0x70, 0x77, 0xb8, - 0x34, 0x1b, 0xf0, 0x01, 0xec, 0x01, 0x13, 0x58, 0x1a, 0x78, 0x88, 0x0b, 0x58, 0x80, 0x27, 0x98, - 0x20, 0x4b, 0x50, 0x41, 0x69, 0x84, 0xc9, 0x63, 0x81, 0x08, 0x7b, 0x06, 0x10, 0xd8, 0x1e, 0x38, - 0x1c, 0x17, 0x68, 0x00, 0x2e, 0xa8, 0xe4, 0x23, 0x78, 0xdc, 0x0a, 0x28, 0x38, 0x0b, 0x28, 0xcc, - 0x1f, 0x40, 0x01, 0x6f, 0x83, 0x88, 0x75, 0x82, 0xc7, 0x8c, 0x04, 0xec, 0x08, 0x28, 0xcc, 0x7b, - 0xe4, 0xb8, 0x00, 0x0e, 0x46, 0x35, 0x2c, 0xc2, 0x2d, 0xb8, 0xe0, 0xb5, 0x42, 0xa2, 0x75, 0x40, - 0x02, 0xfc, 0xc4, 0x43, 0x98, 0x41, 0xa1, 0x56, 0x40, 0xc4, 0x5a, 0x01, 0x13, 0xda, 0x33, 0x5c, - 0xf2, 0x07, 0x44, 0x74, 0x3f, 0x5c, 0x5a, 0x49, 0x24, 0x64, 0x03, 0x2c, 0xd8, 0x23, 0x34, 0x78, - 0x19, 0xb8, 0x61, 0x15, 0x42, 0xa3, 0xc6, 0x46, 0x73, 0x80, 0x0a, 0x52, 0x17, 0x00, 0xa1, 0x7d, - 0x40, 0x26, 0x19, 0xc1, 0x06, 0xf0, 0x40, 0x45, 0x90, 0xc2, 0x81, 0xf6, 0xc0, 0x22, 0xf3, 0x80, - 0x15, 0x02, 0x07, 0x2c, 0x9a, 0x03, 0xd8, 0xc0, 0x95, 0x21, 0xbb, 0x04, 0x80, 0x00, 0x99, 0xe0, - 0x90, 0x19, 0x21, 0x93, 0xfe, 0x21, 0x43, 0x1a, 0x90, 0x64, 0x02, 0x2e, 0x20, 0x65, 0xc8, 0x79, - 0x86, 0xc4, 0x78, 0x04, 0x2a, 0x38, 0x9e, 0x50, 0x82, 0xaf, 0x05, 0x17, 0x50, 0x33, 0xa0, 0x42, - 0x21, 0x05, 0x24, 0xc0, 0x0f, 0x64, 0xb0, 0x40, 0xe5, 0x3a, 0x42, 0xc1, 0xfd, 0x02, 0x3c, 0x10, - 0x09, 0x88, 0x40, 0x5a, 0x40, 0x84, 0x11, 0x41, 0x83, 0x7f, 0x42, 0x82, 0x10, 0x03, 0x08, 0x8a, - 0x1b, 0xc8, 0xc0, 0xb2, 0x81, 0x16, 0x6e, 0x68, 0x21, 0x1a, 0x02, 0x11, 0x30, 0x00, 0x42, 0x71, - 0xc1, 0xe7, 0x89, 0x18, 0x60, 0x05, 0x3c, 0x70, 0x3c, 0xa2, 0x35, 0x14, 0xe6, 0x25, 0x98, 0x20, - 0x97, 0x20, 0xa8, 0x02, 0x26, 0x8e, 0x12, 0x6c, 0x30, 0x2a, 0xa0, 0x22, 0x5e, 0xc2, 0x05, 0x8f, - 0x08, 0x0a, 0xea, 0x04, 0x0a, 0x9b, 0x04, 0x3e, 0xf4, 0x03, 0x2e, 0x92, 0x05, 0x26, 0x65, 0xe0, - 0x28, 0x0e, 0x38, 0x50, 0xb0, 0x2f, 0x61, 0xe1, 0x6a, 0xa0, 0xf3, 0x2c, 0x12, 0x49, 0x06, 0x26, - 0x20, 0x84, 0x30, 0x51, 0x5d, 0x90, 0xb0, 0x57, 0x70, 0x11, 0x3d, 0x70, 0x10, 0x53, 0x1c, 0x98, - 0xf8, 0x54, 0x00, 0x02, 0xc3, 0x10, 0x58, 0xb0, 0x4d, 0xb0, 0x20, 0x0d, 0x71, 0x40, 0x4d, 0x20, - 0x04, 0x79, 0x05, 0x1f, 0x7b, 0x01, 0x89, 0x69, 0x02, 0x28, 0xc8, 0x17, 0x30, 0x81, 0x15, 0xd0, - 0xc0, 0x5f, 0x50, 0x19, 0x1e, 0x72, 0x4d, 0x8d, 0xc1, 0x01, 0x91, 0x7e, 0x05, 0x34, 0x30, 0x0e, - 0x91, 0xc9, 0x74, 0x20, 0x84, 0xd5, 0x13, 0x0b, 0x61, 0x0d, 0x8f, 0xd8, 0x00, 0x32, 0x88, 0x4e, - 0x20, 0x04, 0xd7, 0x84, 0xbc, 0x06, 0x38, 0x6c, 0x1f, 0xc8, 0xe0, 0x12, 0x68, 0xb4, 0x16, 0xa0, - 0x41, 0x3a, 0x07, 0x27, 0x90, 0x1a, 0x88, 0x18, 0x0a, 0xa5, 0x71, 0x25, 0xf1, 0x09, 0xa8, 0x50, - 0x9d, 0xe1, 0x82, 0xbc, 0x81, 0x04, 0xc9, 0x09, 0x7c, 0x10, 0x3e, 0x40, 0x05, 0xe9, 0x02, 0x3a, - 0xce, 0x0d, 0x5c, 0x70, 0xfa, 0x60, 0x91, 0x41, 0x09, 0xc7, 0x08, 0x0a, 0x93, 0x03, 0x7c, 0xa0, - 0x3f, 0xe0, 0x53, 0x4b, 0x40, 0x06, 0xea, 0x0d, 0x0a, 0xcd, 0x18, 0x0a, 0x83, 0x05, 0x2a, 0x29, - 0x0e, 0xc3, 0x01, 0x54, 0xb0, 0x7c, 0x41, 0x04, 0xaf, 0x02, 0x54, 0x40, 0xbe, 0x20, 0x41, 0xa7, - 0x21, 0x29, 0xe1, 0x62, 0x6a, 0x81, 0x08, 0xb8, 0x04, 0x64, 0xb0, 0x29, 0xa0, 0xb0, 0x7f, 0xe0, - 0x00, 0x3a, 0xc0, 0x05, 0xcb, 0x02, 0xb8, 0xe0, 0x6e, 0xc0, 0x08, 0xda, 0x13, 0x78, 0xe0, 0x1a, - 0x00, 0x04, 0xe2, 0x08, 0x0a, 0x1b, 0x16, 0x9d, 0x35, 0x88, 0x00, 0x0a, 0x61, 0xe3, 0x1d, 0x61, - 0xa1, 0x8f, 0xe0, 0xe3, 0x7d, 0x00, 0x08, 0x56, 0x16, 0xb9, 0x05, 0x7c, 0x50, 0x5a, 0xe2, 0xd1, - 0xc9, 0x40, 0x08, 0x93, 0x0a, 0x36, 0xc4, 0x02, 0x0e, 0xf7, 0x15, 0x0a, 0xae, 0x1b, 0x0a, 0x83, - 0x0d, 0x36, 0xe8, 0x0a, 0x16, 0xde, 0x11, 0x3e, 0x89, 0x0b, 0x50, 0x70, 0xd9, 0x10, 0xca, 0x00, - 0x38, 0xc0, 0x8f, 0xc1, 0x83, 0x77, 0x86, 0x0c, 0x1b, 0xe8, 0x00, 0x9e, 0x78, 0x08, 0x0b, 0xa0, - 0xc3, 0xb7, 0x84, 0x82, 0x18, 0x4b, 0x54, 0x3b, 0x70, 0x81, 0xbe, 0x41, 0xcc, 0x0d, 0x08, 0x61, - 0x8e, 0x17, 0x00, 0x0d, 0xd8, 0x40, 0x34, 0x20, 0xbd, 0x12, 0x0e, 0x83, 0x16, 0x3a, 0x8a, 0x16, - 0x48, 0xd0, 0x59, 0xe0, 0x21, 0x4c, 0x60, 0x42, 0xaa, 0xa0, 0x60, 0x69, 0x00, 0x05, 0xb9, 0x02, - 0x12, 0x94, 0x0f, 0x48, 0x60, 0xfe, 0x90, 0x66, 0x03, 0x4e, 0xf8, 0x4c, 0x50, 0xd8, 0x5f, 0x70, - 0x99, 0x1f, 0xd0, 0xf0, 0x55, 0x88, 0x74, 0x81, 0x0d, 0x6b, 0x80, 0x84, 0xb9, 0x8d, 0xc7, 0x34, - 0x81, 0x4e, 0xa4, 0x8a, 0x82, 0xa1, 0x02, 0x15, 0x40, 0x07, 0x70, 0x81, 0x7f, 0x8d, 0x03, 0x61, - 0x02, 0x11, 0x1c, 0x1e, 0x78, 0xb8, 0x1a, 0x80, 0x02, 0xa3, 0x80, 0x02, 0x69, 0x80, 0xc7, 0xf3, - 0x01, 0x2c, 0xdc, 0x13, 0x90, 0x81, 0xad, 0x81, 0x88, 0xb8, 0x81, 0x82, 0x78, 0x84, 0x4e, 0xb6, - 0x83, 0x45, 0xb3, 0x06, 0x37, 0x90, 0x5e, 0x68, 0x48, 0x82, 0x28, 0x00, 0xc1, 0x01, 0xe5, 0x80, - 0x86, 0xf2, 0x81, 0xcc, 0xb9, 0x81, 0x88, 0xeb, 0x80, 0x03, 0xb7, 0x87, 0x8d, 0x69, 0x83, 0x45, - 0x74, 0x05, 0x22, 0x84, 0x43, 0x48, 0x48, 0x9f, 0xa4, 0xf3, 0x40, 0xa4, 0x78, 0x80, 0x0b, 0xec, - 0x0b, 0x2c, 0x08, 0x07, 0x74, 0x88, 0x15, 0x34, 0xc2, 0xa2, 0xa1, 0x33, 0x0d, 0xc1, 0x08, 0x91, - 0x18, 0xf0, 0x60, 0x38, 0x40, 0x07, 0x83, 0x01, 0x8c, 0x10, 0x19, 0x40, 0x04, 0xbb, 0x1a, 0x68, - 0x30, 0x2b, 0x60, 0x92, 0xaa, 0x80, 0x08, 0xef, 0x02, 0x54, 0x50, 0x7f, 0x80, 0x0a, 0x89, 0x02, - 0x32, 0x94, 0x17, 0x1a, 0xe1, 0x04, 0x0e, 0xa6, 0x01, 0x0a, 0x97, 0x0b, 0x16, 0xe4, 0x0a, 0x91, - 0x3c, 0x90, 0xf9, 0x1e, 0x10, 0xf1, 0x25, 0x70, 0x29, 0x1d, 0xa0, 0x82, 0xf0, 0x04, 0x40, 0xe8, - 0x56, 0x40, 0x05, 0x54, 0x0f, 0x1b, 0xc6, 0x01, 0x2a, 0x98, 0x35, 0xf0, 0x30, 0xbd, 0x80, 0x07, - 0xd9, 0x06, 0x20, 0x90, 0x5e, 0x09, 0xf9, 0x0b, 0x1c, 0xac, 0x22, 0x20, 0xc1, 0x96, 0x4f, 0x70, - 0x80, 0x4b, 0xae, 0x81, 0x47, 0xb9, 0x8d, 0x8d, 0x3d, 0x82, 0x45, 0x65, 0x80, 0xca, 0xb9, 0x05, - 0x18, 0x6c, 0x26, 0xa4, 0x53, 0x40, 0xc7, 0xb2, 0xc2, 0x06, 0xdf, 0x80, 0x0a, 0xa0, 0x29, 0x30, - 0x23, 0x32, 0x81, 0x08, 0x7c, 0x3c, 0x3e, 0x37, 0xb8, 0x20, 0xd7, 0x80, 0x0b, 0xfc, 0x25, 0x24, - 0xe8, 0x1d, 0x2c, 0x50, 0x0d, 0x3c, 0xe0, 0x88, 0x60, 0x3a, 0x83, 0x0c, 0xa6, 0x11, 0xb2, 0x5c, - 0xc1, 0x07, 0xf2, 0x01, 0x2a, 0xce, 0x01, 0x12, 0xd7, 0x64, 0x0e, 0xb2, 0x07, 0xa9, 0x6e, 0xd0, - 0x71, 0x1d, 0x70, 0x80, 0xe5, 0x08, 0x6d, 0x83, 0x84, 0x6a, 0x00, 0x19, 0x78, 0x0e, 0x48, 0xc4, - 0x07, 0x80, 0x81, 0xae, 0x80, 0x44, 0x3f, 0x01, 0x13, 0x4c, 0x1f, 0x58, 0xe0, 0x4a, 0x48, 0x8c, - 0x2e, 0x04, 0x78, 0x81, 0x16, 0x7a, 0x05, 0x90, 0xa0, 0xdd, 0xc3, 0xa7, 0xd8, 0xc2, 0x44, 0xfc, - 0x40, 0x02, 0xda, 0x00, 0x0c, 0x54, 0x11, 0xe0, 0xc0, 0xba, 0x80, 0x09, 0x7a, 0x09, 0xa0, 0x00, - 0x52, 0x40, 0x61, 0x51, 0x44, 0x67, 0x12, 0x03, 0x0c, 0x00, 0x0f, 0x70, 0xe1, 0x31, 0xc0, 0xe1, - 0xb6, 0xc2, 0xa1, 0xb9, 0xc1, 0x46, 0x50, 0x40, 0x01, 0x9b, 0x01, 0x1a, 0xc2, 0x11, 0x72, 0x4c, - 0xa1, 0x02, 0x5b, 0x20, 0xb1, 0x28, 0xc0, 0x04, 0xc7, 0x12, 0x40, 0x00, 0xee, 0x00, 0x08, 0xd1, - 0x2a, 0x32, 0xd0, 0x11, 0x81, 0x7f, 0xa0, 0x84, 0x77, 0x0a, 0x93, 0xc8, 0x00, 0x97, 0xe7, 0x01, - 0x09, 0x50, 0x01, 0x05, 0xf5, 0x00, 0x85, 0xee, 0x03, 0x93, 0x59, 0x82, 0x3c, 0x59, 0x14, 0x52, - 0x28, 0xa4, 0x0b, 0x28, 0x30, 0x0a, 0x84, 0x4d, 0x81, 0x7f, 0x43, 0xa7, 0x56, 0x80, 0x0b, 0x92, - 0x0f, 0x3c, 0x98, 0x47, 0xa8, 0x00, 0x35, 0x20, 0xde, 0x03, 0x2a, 0x8b, 0x02, 0x2e, 0x8f, 0x06, - 0x58, 0x40, 0x58, 0xe0, 0x33, 0x4d, 0xc1, 0x0a, 0xc0, 0x06, 0xc0, 0xc0, 0x04, 0x05, 0xf5, 0x04, - 0x16, 0xfa, 0x26, 0x60, 0x60, 0x9c, 0xa0, 0x63, 0x2f, 0x20, 0x13, 0xbe, 0x81, 0x0c, 0x97, 0x0d, - 0x32, 0xd0, 0x01, 0x22, 0x3e, 0x80, 0x20, 0x49, 0xe0, 0x50, 0x18, 0x21, 0x02, 0x19, 0xe0, 0x20, - 0xbd, 0x10, 0xc8, 0x04, 0x9b, 0x40, 0x0d, 0x24, 0xe0, 0xcd, 0x90, 0xe9, 0x03, 0x82, 0x16, 0x11, - 0x28, 0x2a, 0xd0, 0xb5, 0x00, 0x83, 0x73, 0x0b, 0x19, 0xd6, 0x07, 0x1b, 0x71, 0x05, 0x26, 0x28, - 0x6d, 0x00, 0x82, 0x7f, 0x05, 0x2a, 0xb8, 0x56, 0x80, 0x84, 0x79, 0x04, 0x86, 0xa0, 0x7e, 0x50, - 0x20, 0xe3, 0x03, 0x20, 0x36, 0x88, 0xaf, 0x90, 0x91, 0xd5, 0x60, 0x82, 0xe4, 0x01, 0x93, 0xf4, - 0x08, 0x95, 0xc0, 0x05, 0x1b, 0x61, 0x06, 0x87, 0xc3, 0x01, 0x47, 0xb5, 0x41, 0x51, 0xc9, 0x26, - 0x00, 0x03, 0xc1, 0x00, 0x87, 0x46, 0x05, 0x8f, 0x5a, 0x01, 0x95, 0xef, 0x09, 0x89, 0xca, 0x00, - 0x8b, 0xf7, 0x02, 0x19, 0x5b, 0x0b, 0x32, 0x08, 0x82, 0x02, 0xec, 0x00, 0x13, 0x23, 0x11, 0x40, - 0x04, 0x20, 0xe0, 0x26, 0x51, 0x89, 0x85, 0x40, 0x84, 0xc2, 0x01, 0x0f, 0x51, 0x01, 0x0d, 0x08, - 0x6e, 0xf8, 0x6e, 0x00, 0x82, 0x6a, 0x01, 0x0f, 0x72, 0x04, 0x2c, 0x40, 0x4d, 0x50, 0x41, 0x16, - 0x08, 0x67, 0x46, 0x4a, 0x0d, 0x90, 0xc0, 0xfe, 0xc1, 0x42, 0x30, 0x01, 0x09, 0x50, 0x0d, 0x5c, - 0x16, 0x09, 0xc0, 0x00, 0xf6, 0x41, 0x27, 0x92, 0xc0, 0x63, 0x7a, 0xc2, 0xa1, 0x9c, 0x40, 0x21, - 0x27, 0xec, 0x25, 0x4c, 0x80, 0x1b, 0x20, 0x81, 0x2b, 0xa0, 0x00, 0x18, 0xe0, 0xe1, 0x6a, 0x21, - 0x02, 0xa3, 0xd2, 0xaf, 0xe0, 0x03, 0x7a, 0x00, 0x04, 0x99, 0x26, 0x50, 0x70, 0xcf, 0xa0, 0xe1, - 0x58, 0xa1, 0xd3, 0x38, 0xa0, 0x30, 0x79, 0x20, 0xd2, 0x3a, 0x20, 0xb3, 0x5e, 0x60, 0xa1, 0xeb, - 0x80, 0x05, 0xc8, 0x01, 0x4c, 0xc0, 0x5b, 0xa0, 0x60, 0xc8, 0x41, 0x04, 0x6d, 0xc8, 0x20, 0xab, - 0x10, 0xe3, 0x07, 0x1f, 0x4b, 0x07, 0x28, 0x70, 0x17, 0x80, 0x84, 0xf9, 0x06, 0x46, 0x90, 0x44, - 0xf0, 0x40, 0xf6, 0x60, 0x05, 0xca, 0x07, 0x1b, 0xcf, 0x02, 0x0b, 0x61, 0x12, 0x15, -]; - -// the same runs as the first one but shuffled (the runs of 1s are still runs of 1s, just in a -// different location), so it has the exact same properties -const EXAMPLE2: &[u8] = &[ - 0xa0, 0xb8, 0x81, 0x0e, 0x4a, 0x21, 0x5c, 0xaa, 0x05, 0x14, 0x60, 0x0f, 0x82, 0x1c, 0xa1, 0xc0, - 0x3c, 0x02, 0x06, 0xe9, 0x02, 0xdc, 0x90, 0x2f, 0xe0, 0x01, 0x21, 0x31, 0x3d, 0xe1, 0x60, 0x1c, - 0xe2, 0x63, 0x84, 0x0a, 0x93, 0x03, 0x1e, 0xe9, 0x1f, 0x12, 0x99, 0x0e, 0xf0, 0x70, 0xad, 0x80, - 0x04, 0xa9, 0x0a, 0x00, 0x91, 0x4d, 0xa0, 0xf3, 0x3d, 0xe1, 0xd1, 0x6a, 0xa1, 0xd3, 0x39, 0x80, - 0x8b, 0x68, 0x22, 0x2a, 0x90, 0x24, 0x05, 0xf6, 0x08, 0x8b, 0x54, 0x0b, 0x91, 0x3c, 0x9d, 0xe7, - 0x01, 0x0b, 0x52, 0x85, 0x4c, 0x2a, 0xf0, 0x82, 0xab, 0x01, 0x12, 0xa8, 0x63, 0xe4, 0x74, 0x20, - 0x29, 0x81, 0x3d, 0x40, 0x85, 0x5c, 0x02, 0x28, 0x90, 0x2c, 0x70, 0x98, 0xc5, 0x60, 0x82, 0x5a, - 0x01, 0x93, 0xcc, 0x07, 0x09, 0x77, 0x01, 0x15, 0x42, 0x0c, 0x8b, 0x49, 0x01, 0x2e, 0xe8, 0x56, - 0x20, 0x02, 0xde, 0x0c, 0x8d, 0xdc, 0x02, 0x87, 0x74, 0x0e, 0x07, 0x54, 0x0f, 0x28, 0x08, 0x44, - 0x30, 0xd9, 0x77, 0x50, 0x91, 0x3d, 0xe0, 0x02, 0xed, 0x09, 0x15, 0xd0, 0x01, 0x87, 0x99, 0x3c, - 0x8b, 0x10, 0x81, 0x74, 0x81, 0x87, 0xf8, 0x80, 0x08, 0x39, 0x82, 0xc6, 0xa0, 0x05, 0x21, 0xd4, - 0x13, 0x58, 0x44, 0x06, 0x58, 0xd4, 0x12, 0x88, 0x44, 0x57, 0x10, 0x01, 0x64, 0x82, 0x0e, 0x61, - 0x81, 0xc3, 0x38, 0x81, 0xc8, 0x2c, 0x81, 0x08, 0x1b, 0x14, 0x30, 0x23, 0x38, 0x8c, 0x6a, 0xd8, - 0x48, 0x9f, 0xf8, 0x5c, 0x36, 0x50, 0x82, 0x68, 0x00, 0x27, 0xb4, 0x0f, 0x80, 0x81, 0x78, 0x84, - 0xcb, 0xa2, 0x40, 0x58, 0x03, 0x24, 0x7a, 0x25, 0x4c, 0xc2, 0x13, 0x14, 0xe2, 0x59, 0x30, 0x41, - 0x58, 0x03, 0x08, 0xf6, 0x0f, 0x24, 0x6e, 0x24, 0x66, 0x09, 0x10, 0x01, 0x9a, 0xc2, 0x44, 0x36, - 0x40, 0x05, 0x57, 0x82, 0x12, 0x9e, 0x70, 0x21, 0x7a, 0xc3, 0x02, 0xf6, 0x81, 0x10, 0x90, 0x25, - 0xb8, 0x80, 0x57, 0xc0, 0xe3, 0xb6, 0x02, 0x19, 0xba, 0x0f, 0xa8, 0x80, 0xbd, 0xc0, 0x21, 0x9c, - 0x40, 0x41, 0xda, 0x00, 0x0e, 0x9c, 0x05, 0x7c, 0xe8, 0x07, 0x98, 0x80, 0x34, 0x84, 0x11, 0x34, - 0x14, 0x92, 0x1f, 0x2c, 0x0e, 0x0d, 0x14, 0x08, 0x09, 0x1c, 0x18, 0x07, 0x44, 0x86, 0x03, 0x90, - 0x40, 0xf2, 0x80, 0x0f, 0xdd, 0x23, 0xf4, 0xc1, 0x82, 0xb7, 0xc1, 0xc7, 0x34, 0x00, 0x11, 0x0a, - 0x11, 0x3c, 0x2a, 0x03, 0x02, 0x49, 0xa0, 0x23, 0x1e, 0x20, 0x33, 0x38, 0xe0, 0x10, 0x4b, 0x01, - 0x0c, 0xc5, 0x16, 0x16, 0xfc, 0x13, 0x90, 0xc0, 0xbf, 0x41, 0x05, 0x40, 0x3e, 0xfa, 0x08, 0x0e, - 0xa5, 0x06, 0x50, 0xa0, 0xfb, 0x81, 0x04, 0x99, 0x26, 0x12, 0xf7, 0x05, 0x7c, 0x40, 0x6b, 0x01, - 0x08, 0x85, 0x27, 0x0e, 0xd5, 0x0e, 0x0e, 0xb9, 0x02, 0x0a, 0x93, 0x07, 0x2a, 0xb2, 0x14, 0x94, - 0xe0, 0x9a, 0x60, 0x22, 0xbd, 0x60, 0x01, 0x5b, 0x80, 0x06, 0xe0, 0x27, 0x0a, 0x97, 0x0c, 0x32, - 0xd0, 0x01, 0x3a, 0xa4, 0x01, 0xa9, 0x14, 0xc8, 0xa2, 0x88, 0x84, 0x69, 0x84, 0x82, 0xb8, 0x81, - 0x82, 0x8f, 0x82, 0x75, 0x01, 0x1b, 0x3c, 0x11, 0xc1, 0x79, 0x06, 0x31, 0x2c, 0x0b, 0x80, 0x07, - 0xed, 0x81, 0x42, 0xf1, 0x00, 0x1e, 0x28, 0x22, 0xb8, 0xc4, 0x27, 0xc8, 0x1c, 0x17, 0x48, 0x9c, - 0x5b, 0x04, 0x56, 0x00, 0x0b, 0x48, 0x2f, 0x08, 0x01, 0x9b, 0x21, 0x91, 0x18, 0xb8, 0x20, 0x5e, - 0xe2, 0x22, 0xcc, 0xc0, 0x04, 0xb6, 0x06, 0x5c, 0xb0, 0x7a, 0x20, 0xa3, 0xba, 0x40, 0x05, 0xba, - 0x16, 0x88, 0xc0, 0x5b, 0x00, 0x09, 0xed, 0x08, 0x4c, 0x30, 0x7d, 0xe0, 0x80, 0x81, 0x08, 0xc0, - 0x06, 0x16, 0xe1, 0x09, 0x44, 0xa8, 0x0a, 0xc0, 0x0b, 0x80, 0x10, 0x1e, 0xa0, 0x00, 0x5b, 0xe0, - 0x60, 0xca, 0xe0, 0xa0, 0x8a, 0x40, 0x06, 0x9d, 0x35, 0x0a, 0x64, 0x70, 0xf0, 0xa1, 0xb1, 0xab, - 0xe1, 0xb3, 0x0a, 0x01, 0x09, 0xf8, 0x06, 0x12, 0x8e, 0x15, 0x48, 0xd0, 0x4b, 0x00, 0x05, 0x92, - 0x0e, 0x12, 0xc8, 0x1e, 0x60, 0x40, 0x3d, 0xe1, 0x40, 0x28, 0xc1, 0x0d, 0xa5, 0x03, 0x68, 0x60, - 0xc8, 0xc1, 0x0d, 0x85, 0x14, 0x3e, 0xcb, 0x0b, 0x1a, 0x8b, 0x02, 0x1e, 0xe1, 0x07, 0x3a, 0xf7, - 0x04, 0xac, 0xa0, 0x3f, 0xc0, 0x05, 0xf7, 0x18, 0x32, 0xb3, 0x06, 0x1a, 0x8c, 0x14, 0x26, 0xeb, - 0x05, 0x22, 0x9c, 0x19, 0x50, 0x80, 0xae, 0x90, 0x6f, 0x06, 0x15, 0x70, 0x07, 0x95, 0xea, 0x06, - 0x05, 0xe0, 0x04, 0x32, 0x50, 0x27, 0xd0, 0x69, 0xc6, 0xb0, 0x30, 0x6d, 0x40, 0x85, 0xfb, 0x02, - 0x8f, 0x66, 0x03, 0x85, 0x70, 0x01, 0xd1, 0x94, 0xd9, 0x2e, 0xb0, 0xb0, 0x34, 0x40, 0x84, 0x5a, - 0x01, 0x2a, 0x98, 0x26, 0xc8, 0xa8, 0x80, 0x45, 0x66, 0x84, 0x0e, 0x3e, 0x82, 0x4a, 0xb1, 0x43, - 0x2c, 0x2b, 0x64, 0xf6, 0x17, 0x3c, 0x24, 0x41, 0x54, 0x00, 0x17, 0x00, 0xc1, 0xb2, 0x82, 0x08, - 0x36, 0xd8, 0x00, 0x90, 0xc2, 0xc6, 0x3b, 0x42, 0x02, 0x32, 0x80, 0x0c, 0x8e, 0x0b, 0x80, 0x60, - 0x3a, 0x23, 0xd3, 0x16, 0x36, 0xd1, 0x1a, 0x0e, 0x80, 0x08, 0x0a, 0x81, 0x1a, 0x54, 0x80, 0x2c, - 0x80, 0x0c, 0xed, 0x1e, 0x44, 0xa0, 0x2a, 0x10, 0xf2, 0x17, 0x20, 0xc8, 0xb1, 0x90, 0x55, 0x90, - 0x68, 0x1d, 0xb0, 0x38, 0xc9, 0xea, 0x09, 0x17, 0x70, 0x77, 0x58, 0x50, 0x3f, 0x30, 0x01, 0x28, - 0x84, 0xc7, 0xb0, 0x81, 0xc7, 0xe1, 0x81, 0x8a, 0xa9, 0x85, 0x8c, 0x37, 0x81, 0x4f, 0xb3, 0x06, - 0x2a, 0x54, 0x3b, 0xb8, 0xf4, 0x0a, 0x50, 0x41, 0xed, 0x01, 0x17, 0xb8, 0x22, 0x90, 0x41, 0xb3, - 0x01, 0x10, 0x88, 0x2b, 0x20, 0x42, 0xea, 0x82, 0x04, 0xe0, 0x89, 0xc7, 0x64, 0x48, 0x8e, 0x1d, - 0x64, 0x3c, 0x07, 0x4c, 0xe6, 0x25, 0x14, 0x2c, 0x0d, 0x58, 0x01, 0x3f, 0x06, 0x0d, 0x9e, 0x0f, - 0x2c, 0xcc, 0x1f, 0x22, 0x2d, 0xa0, 0x52, 0xe3, 0x60, 0xaf, 0x00, 0x04, 0x91, 0x09, 0x3e, 0x88, - 0x04, 0xc1, 0x25, 0x90, 0xc8, 0x45, 0x51, 0xc1, 0x92, 0xc9, 0x00, 0x84, 0xcc, 0x03, 0x34, 0x88, - 0x1f, 0x60, 0x04, 0x7d, 0x03, 0x91, 0xc8, 0x00, 0x0d, 0x46, 0x01, 0x91, 0x66, 0x09, 0x87, 0x73, - 0x0b, 0x9b, 0x6c, 0x07, 0x17, 0xcd, 0x01, 0x9d, 0xc1, 0x80, 0xf4, 0x3f, 0x58, 0x18, 0x0e, 0x90, - 0xc1, 0x29, 0x8c, 0xc8, 0x26, 0x01, 0x1f, 0x80, 0x27, 0xc0, 0x04, 0x8a, 0xca, 0xe1, 0x83, 0x43, - 0x76, 0x41, 0x64, 0x21, 0x24, 0x50, 0x0d, 0x34, 0x5a, 0x07, 0xb2, 0x28, 0xa0, 0xf3, 0x2a, 0x00, - 0x04, 0xee, 0x06, 0x2e, 0xad, 0x15, 0x0a, 0x5b, 0x39, 0xa6, 0x80, 0x83, 0x63, 0x89, 0x84, 0x43, - 0xe8, 0xb0, 0x97, 0x58, 0x0c, 0x36, 0x78, 0x10, 0x0e, 0x30, 0x42, 0xe0, 0x02, 0x2d, 0x50, 0x3e, - 0x44, 0x30, 0xc1, 0x83, 0x11, 0x81, 0x10, 0xf2, 0x07, 0x64, 0xe6, 0x0f, 0x90, 0x60, 0x79, 0x81, - 0x0f, 0x62, 0x21, 0x1c, 0xc2, 0x07, 0x54, 0xa0, 0xb0, 0xa0, 0xcc, 0xe2, 0x31, 0x20, 0x99, 0x11, - 0x0e, 0x83, 0x16, 0x16, 0xe1, 0x1b, 0x70, 0xf0, 0x5e, 0x00, 0x0a, 0x9f, 0x1b, 0x1a, 0xf3, 0x1e, - 0x12, 0x8f, 0x05, 0x2e, 0x5f, 0x60, 0x00, 0x1d, 0x91, 0x6f, 0x09, 0x11, 0xeb, 0x0d, 0x0f, 0x58, - 0x0e, 0x4a, 0xa8, 0xd6, 0x70, 0x69, 0x1c, 0x20, 0x03, 0x7d, 0x13, 0x0f, 0x5d, 0x07, 0x54, 0x70, - 0x3c, 0x21, 0x85, 0x68, 0x15, 0x1b, 0x52, 0x06, 0x1f, 0xcc, 0x04, 0x87, 0xfb, 0x11, 0x95, 0x0c, - 0x05, 0xf6, 0x0c, 0x1b, 0x5f, 0x05, 0x05, 0x4e, 0x01, 0x8d, 0x7e, 0x05, 0x0b, 0xf5, 0x06, 0x28, - 0x90, 0x24, 0x90, 0x88, 0x0d, 0x00, 0x82, 0xcb, 0x06, 0x50, 0x78, 0x17, 0x70, 0x10, 0xdc, 0xb0, - 0x21, 0x16, 0x90, 0xf9, 0x4c, 0xa0, 0x82, 0xc2, 0x01, 0x30, 0x68, 0xcf, 0x30, 0x59, 0x45, 0x70, - 0xd8, 0x25, 0x40, 0x02, 0xfb, 0x0a, 0x07, 0xf9, 0x00, 0x07, 0x20, 0x0b, 0xd6, 0x07, 0x2a, 0xb0, - 0x3d, 0x00, 0x84, 0x20, 0x8f, 0xe5, 0x0a, 0x93, 0x44, 0x01, 0x2e, 0x68, 0x25, 0x11, 0x99, 0x53, - 0x10, 0x66, 0x10, 0x91, 0x1e, 0x50, 0xf8, 0x54, 0xd0, 0xe1, 0x26, 0x11, 0xb1, 0x7f, 0x20, 0x04, - 0x5b, 0x0b, 0x99, 0x7c, 0x0c, 0x09, 0xc8, 0x02, 0x17, 0x57, 0x0b, 0x0f, 0x26, 0x95, 0x73, 0x83, - 0x90, 0x1a, 0x00, 0x42, 0x67, 0x03, 0x1c, 0x70, 0x16, 0xd8, 0xb0, 0x4b, 0xd8, 0xc0, 0x3f, 0x40, - 0x01, 0xe8, 0x81, 0x02, 0xa4, 0x00, 0x22, 0xa0, 0x07, 0x78, 0xc4, 0x32, 0x68, 0xe4, 0x26, 0x30, - 0xc1, 0x66, 0x82, 0xc9, 0xe5, 0x02, 0x11, 0x80, 0x3b, 0xb8, 0xc4, 0x80, 0xc1, 0x34, 0x01, 0x1b, - 0xb8, 0x32, 0x68, 0x70, 0x7b, 0x70, 0x01, 0x72, 0x00, 0x10, 0x30, 0x0a, 0x10, 0x81, 0xed, 0x80, - 0x4f, 0xa4, 0x8a, 0x4a, 0x8b, 0xc5, 0xf6, 0x01, 0x4b, 0x4c, 0x0b, 0xe8, 0xd8, 0x17, 0xc0, 0x02, - 0xbd, 0x05, 0x11, 0xa9, 0xe0, 0x56, 0xc0, 0x43, 0xb6, 0x41, 0x42, 0x7a, 0x85, 0x0d, 0xc8, 0x25, - 0xf2, 0x1a, 0xe0, 0x30, 0x1e, 0x21, 0x61, 0x9b, 0x60, 0x53, 0x59, 0x60, 0xc3, 0x5c, 0x20, 0x31, - 0x89, 0xe1, 0x43, 0x38, 0xe0, 0x02, 0x3b, 0x02, 0x06, 0x84, 0x0d, 0x16, 0xe4, 0x0a, 0x40, 0x80, - 0x6a, 0x80, 0x19, 0x82, 0x14, 0x3e, 0xf7, 0x10, 0x4c, 0x00, 0x78, 0x00, 0x05, 0xa3, 0x36, 0x70, - 0x40, 0xef, 0x20, 0xe1, 0x28, 0xe1, 0x61, 0x2f, 0x90, 0x4d, 0x01, 0x87, 0x69, 0x08, 0x0b, 0xcb, - 0x06, 0x05, 0xc2, 0x04, 0x8f, 0x6b, 0x32, 0x22, 0x48, 0x46, 0x70, 0x18, 0x1e, 0x12, 0xc1, 0x51, - 0xc0, 0x71, 0x89, 0xaf, 0x08, 0xa8, 0x00, 0x10, 0x78, 0x16, 0x48, 0x0c, 0x16, 0xc4, 0x97, 0x80, - 0x12, 0xbc, 0x23, 0x38, 0x41, 0xd0, 0xc0, 0x83, 0x72, 0x00, 0x1e, 0x28, 0x0f, 0x6c, 0x70, 0x09, - 0xd8, 0xa0, 0x0d, 0x08, 0x04, 0x05, 0xa8, 0xc0, 0x50, 0xc1, 0xe1, 0x70, 0x00, 0x20, 0x68, 0x1b, - 0x24, 0x24, 0x25, 0xa8, 0x00, 0x91, 0x42, 0x81, 0x9a, 0xc1, 0x02, 0x32, 0x81, 0x09, 0xfa, 0x15, - 0x4c, 0xc4, 0x05, 0xb8, 0x20, 0x30, 0x01, 0x16, 0x1e, 0x0d, 0x64, 0xcc, 0xc8, 0xa0, 0xda, 0x01, - 0x08, 0x28, 0x29, 0x14, 0x0a, 0x23, 0x34, 0x58, 0x13, 0x34, 0xc8, 0x17, 0x3c, 0xe0, 0x07, 0x90, - 0x20, 0x1c, 0xc1, 0xc5, 0x30, 0x44, 0x83, 0xb9, 0x88, 0x09, 0xd2, 0x1f, 0x2c, 0x00, 0x1d, 0x14, - 0x52, 0x24, 0x5a, 0x0b, 0xf8, 0xa0, 0x34, 0x82, 0x21, 0x1e, 0x11, 0x1c, 0x22, 0x09, 0x44, 0x4c, - 0x2d, 0x24, 0x7c, 0x44, 0x28, 0x17, 0x54, 0x50, 0x13, 0x34, 0x5c, 0x07, 0xf0, 0xa0, 0x3a, 0x23, - 0xa8, 0x09, 0x54, 0x20, 0x7a, 0xa0, 0x91, 0x5b, 0x60, 0x33, 0x0d, 0xa1, 0x52, 0x5a, 0xc2, 0x05, - 0x80, 0x06, 0x0a, 0xd0, 0x06, 0x2e, 0xce, 0x0a, 0x40, 0xc0, 0x0f, 0x22, 0x53, 0x29, 0xe0, 0xe0, - 0x59, 0xa0, 0x90, 0x8f, 0xe0, 0xe1, 0x6a, 0xc0, 0x04, 0xe5, 0x04, 0x60, 0x50, 0x19, 0xc0, 0x07, - 0x93, 0x05, 0x1e, 0x3a, 0xc1, 0x0d, 0xb0, 0xe8, 0x5e, 0x10, 0xf9, 0xf3, 0xe1, 0x65, 0xf0, 0xe8, - 0x14, 0xb0, 0xb8, 0x67, 0xb0, 0x20, 0x84, 0xe0, 0x82, 0x5c, 0x03, 0x9b, 0xf8, 0x00, 0x05, 0xf2, - 0x05, 0x1b, 0x6d, 0x07, 0x26, 0x94, 0x44, 0x07, 0x80, 0x81, 0xb1, 0x05, 0x18, 0x14, 0x52, 0x50, - 0x81, 0xed, 0x86, 0x43, 0x3f, 0x01, 0x20, 0x98, 0x5e, 0xf8, 0xf8, 0x4b, 0x20, 0x81, 0xf6, 0x00, - 0x23, 0xb8, 0x0e, 0x44, 0x9f, 0x41, 0xc1, 0xdc, 0x40, 0xa1, 0xf4, 0xc0, 0x43, 0x30, 0x40, 0x82, - 0x55, 0x40, 0xa2, 0x71, 0x05, 0x1c, 0x16, 0x09, 0xb8, 0x60, 0x91, 0x20, 0xd3, 0x08, 0x26, 0xce, - 0x01, 0xf1, 0x3e, 0x20, 0x03, 0x38, 0x19, 0x47, 0x03, 0x9d, 0xca, 0x07, 0xd7, 0x11, 0x41, 0x15, - 0x90, 0x47, 0x98, 0x44, 0xa7, 0xf3, 0x40, 0x83, 0x58, 0x81, 0x16, 0x48, 0x03, 0xb8, 0xa0, 0x4f, - 0x44, 0xf9, 0xc0, 0x81, 0x32, 0x40, 0x81, 0x99, 0x04, 0x0a, 0xba, 0x3b, 0x50, 0x01, 0x3b, 0xc0, - 0x26, 0x37, 0x80, 0x10, 0xcc, 0x6d, 0xd0, 0xc0, 0x50, 0xc8, 0x47, 0x3d, 0x00, 0x0b, 0xba, 0x1d, - 0x4c, 0xa6, 0x29, 0x34, 0xbc, 0x07, 0x98, 0x40, 0x9a, 0x01, 0x17, 0xc6, 0x11, 0x24, 0x50, 0x03, - 0x5c, 0x70, 0x05, 0x64, 0xdc, 0x2f, 0x34, 0x5e, 0x2b, 0x00, 0x61, 0x7e, 0xc0, 0x85, 0x97, 0x00, - 0x0a, 0x54, 0x05, 0x92, 0x6b, 0xa0, 0xd0, 0xc9, 0x10, 0x49, 0x06, 0x85, 0xe6, 0x06, 0x07, 0x61, - 0x01, 0x32, 0x10, 0x93, 0xf8, 0x2d, 0x10, 0xe1, 0x65, 0x70, 0xb0, 0x47, 0x70, 0xb1, 0x74, 0xa0, - 0x82, 0x59, 0x05, 0x32, 0x48, 0x26, 0x90, 0x01, 0x3c, 0xf0, 0x80, 0x51, 0xb0, 0x52, 0x09, 0x0e, - 0xb0, 0x41, 0xde, 0x70, 0x91, 0xd5, 0x48, 0xad, 0x80, 0x02, 0x32, 0x83, 0x0e, 0xe5, 0x85, 0x43, - 0xa0, 0x00, 0x14, 0x74, 0x16, 0x24, 0xd7, 0x40, 0x46, 0x70, 0x40, 0x63, 0x74, 0x41, 0xa6, 0x79, - 0x00, 0x08, 0xfe, 0x15, 0xb0, 0xe0, 0x18, 0x81, 0x0d, 0xb2, 0x03, 0xd0, 0x61, 0x9e, 0x83, 0x0c, - 0xc2, 0x2d, 0xb0, 0x80, 0xb0, 0xc1, 0xe1, 0xf4, 0xc1, 0x42, 0x35, 0xc0, 0x23, 0x71, 0xc1, 0xc1, - 0xbf, 0x06, 0x0a, 0xe6, 0x1b, 0x54, 0xde, 0x19, 0x44, 0xe8, 0x4c, 0x60, 0x05, 0xa2, 0x2d, 0xa0, - 0x20, 0xaf, 0x60, 0x23, 0x8e, 0x80, 0x08, 0xd6, 0x09, 0x78, 0x70, 0x5f, 0x81, 0x08, 0x96, 0x06, - 0x2a, 0xf8, 0x0b, 0x12, 0xf8, 0x03, 0x0a, 0xfb, 0x12, 0x1a, 0x80, 0x01, 0x32, 0x90, 0x09, 0x2e, - 0x8a, 0x16, 0x3a, 0xe5, 0x36, 0x3e, 0xd1, 0x09, 0x2a, 0xf2, 0x02, 0x22, 0x61, 0x40, 0xf0, 0x5c, - 0x20, 0x61, 0x9c, 0x20, 0xf1, 0x3d, 0xa0, 0x60, 0x2f, 0x20, 0x63, 0x78, 0xc0, 0x05, 0xce, 0x0d, - 0x26, 0xe9, 0x02, 0x44, 0xf0, 0xad, 0x20, 0x52, 0x7f, 0xe0, 0x33, 0x2b, 0xa0, 0x93, 0x9c, 0xa0, - 0xb0, 0x29, 0xa0, 0xa0, 0xb8, 0x20, 0x41, 0xf9, 0x60, 0x92, 0x1e, 0x61, 0x12, 0x2a, 0x00, 0x05, - 0xc9, 0x03, 0x16, 0xd0, 0x01, 0x5c, 0x11, 0x38, 0x90, 0x72, 0x01, 0x32, 0x68, 0x37, 0x60, 0x02, - 0xe3, 0x07, 0x22, 0x00, 0x9d, 0x88, 0x6b, 0x40, 0x24, 0x11, 0x4c, 0x4a, 0x1b, 0x3c, 0x92, 0x07, - 0x7c, 0x1c, 0x2b, 0xa0, 0x40, 0x98, 0x40, 0xa5, 0x39, 0x40, 0x42, 0x88, 0x0f, 0x92, 0x0f, 0x7c, - 0xae, 0x23, 0xa0, 0x61, 0x1e, 0xc1, 0xc1, 0xd7, 0xc2, 0x43, 0x57, 0x40, 0xc6, 0xba, 0x00, 0x0a, - 0xa0, 0x0b, 0x20, 0xc1, 0x75, 0x23, 0xd6, 0x0a, 0x3e, 0xbd, 0x04, 0xac, 0x90, 0xbc, 0xa0, 0xf2, - 0x64, 0xa2, 0x1e, 0x40, 0x0b, 0xe2, 0x07, 0x70, 0xd0, 0xa8, 0xe0, 0x22, 0x26, 0x81, 0xa8, 0x20, - 0x11, 0x0d, 0x61, 0xf1, 0x4e, 0xa1, 0xb0, 0x07, 0x04, 0x5a, 0x1e, 0x83, 0x05, 0xe1, 0x57, 0xd0, - 0xf8, 0x6e, 0x30, 0x91, 0x3c, 0x60, 0x82, 0xe5, 0x0b, 0x2a, 0xf0, 0x9f, 0xe0, 0x04, -]; diff --git a/utils/bitfield/benches/benchmarks/main.rs b/utils/bitfield/benches/benchmarks/main.rs deleted file mode 100644 index 288738910d85..000000000000 --- a/utils/bitfield/benches/benchmarks/main.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -mod examples; - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use forest_bitfield::BitField; - -use examples::{example1, example2}; - -fn len(c: &mut Criterion) { - let bf = example1(); - c.bench_function("len", |b| b.iter(|| black_box(&bf).len())); -} - -fn bits(c: &mut Criterion) { - let bf = example1(); - c.bench_function("bits", |b| { - b.iter(|| black_box(&bf).iter().map(black_box).count()) - }); -} - -fn new(c: &mut Criterion) { - c.bench_function("new", |b| b.iter(example1)); -} - -fn decode_encode(c: &mut Criterion) { - let bf = example1(); - c.bench_function("decode_encode", |b| { - b.iter(|| BitField::from_ranges(bf.ranges())) - }); -} - -fn from_ranges(c: &mut Criterion) { - let vec: Vec<_> = example1().ranges().collect(); - let ranges = || forest_bitfield::iter::Ranges::new(vec.iter().cloned()); - c.bench_function("from_ranges", |b| { - b.iter(|| BitField::from_ranges(ranges())) - }); -} - -fn is_empty(c: &mut Criterion) { - let bf = example1(); - c.bench_function("is_empty", |b| b.iter(|| bf.is_empty())); -} - -fn intersection(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - c.bench_function("intersection", |b| b.iter(|| &bf1 & &bf2)); -} - -fn union(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - c.bench_function("union", |b| b.iter(|| &bf1 | &bf2)); -} - -fn difference(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - c.bench_function("difference", |b| b.iter(|| &bf1 - &bf2)); -} - -fn symmetric_difference(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - c.bench_function("symmetric_difference", |b| b.iter(|| &bf1 ^ &bf2)); -} - -fn cut(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - c.bench_function("cut", |b| b.iter(|| bf1.cut(&bf2))); -} - -fn contains_all(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - let intersection = &bf1 & &bf2; - c.bench_function("contains_all", |b| { - b.iter(|| bf1.contains_all(&intersection)) - }); -} - -fn contains_any(c: &mut Criterion) { - let bf1 = example1(); - let bf2 = example2(); - let difference = &bf1 - &bf2; - c.bench_function("contains_any", |b| b.iter(|| bf2.contains_any(&difference))); -} - -fn get(c: &mut Criterion) { - let bf = example1(); - c.bench_function("get", |b| b.iter(|| bf.get(500_000))); -} - -criterion_group!( - benches, - len, - bits, - new, - decode_encode, - from_ranges, - is_empty, - intersection, - union, - difference, - symmetric_difference, - cut, - contains_all, - contains_any, - get, -); -criterion_main!(benches); diff --git a/utils/bitfield/src/iter/combine.rs b/utils/bitfield/src/iter/combine.rs deleted file mode 100644 index 3a0d2874d125..000000000000 --- a/utils/bitfield/src/iter/combine.rs +++ /dev/null @@ -1,484 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -//! Combining two range iterators into a single new range iterator. -//! -//! This file contains the inner workings of the `BitField` combinators like -//! `merge` and `intersection`. The `Combinator` trait specifies how two range -//! iterators should be combined, and the `Combine` iterator lazily computes the -//! output ranges. -//! -//! The `Combine` iterator works at follows: -//! 1. it inspects the first range of each of the two input ranges -//! 2. it asks the corresponding combinator how these two ranges should be combined -//! 3. it discards the range with the lowest upper bound, and goes back to step 1 -//! -//! For example, given the iterators over the following ranges: -//! -//! ```ignore -//! lhs: -xx-xx -//! rhs: xxxxx- -//! ``` -//! -//! First `-xx---` and `xxxxx-` are passed to the combinator. Then `-xx---` is -//! discarded because it has the lowest upper bound, after which we are left with -//! -//! ```ignore -//! lhs: ----xx -//! rhs: xxxxx- -//! ``` -//! -//! Now `----xx` and `xxxxx-` are passed to the combinator. Finally, `xxxxx-` is -//! discarded, and the only remaining range `----xx` is passed to the combinator as -//! well. -//! -//! It is up to the specific combinator to decide which ranges to produce. For -//! example, the `Intersection` combinator would produce the following outputs -//! given the inputs from above: -//! -//! ```ignore -//! xxx--- -//! xxxxx- -//! ----xx -//! ``` -//! -//! These ranges are combined into a proper range iterator by merging overlapping -//! ranges. - -use super::RangeIterator; -use std::{cmp, iter, ops::Range}; - -/// A trait for defining how two range iterators can be combined into a single new range iterator. -/// -/// When returning a range, it is required that the lower bound of that range isn't smaller than -/// any previously returned range. The logic for stitching overlapping ranges together relies on -/// the lower bounds of the returned ranges to form a monotonically increasing sequence. -pub trait Combinator: Default { - /// Produces an output range for the two given input ranges. - /// - /// - It is guaranteed that `lhs.end <= rhs.end`. - /// - The `rhs` range can be mutated if necessary. - /// - Can return an empty range, those will be filtered out. - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range; - - /// Produces an output range for the two given input ranges. - /// - /// - It is guaranteed that `lhs.end > rhs.end`. - /// - The `lhs` range can be mutated if necessary. - /// - Can return an empty range, those will be filtered out. - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range; - - /// Produces an output range for the given input range. Called only when the - /// second input range iterator is empty. - fn advance_lhs_tail(&mut self, lhs: Range) -> Option>; - - /// Produces an output range for the given input range. Called only when the - /// first input range iterator is empty. - fn advance_rhs_tail(&mut self, rhs: Range) -> Option>; -} - -/// The union combinator. -/// -/// Produces ranges over the bits that are in one or both of the input range iterators. -#[derive(Default)] -pub struct Union; - -impl Combinator for Union { - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range { - // the returned range needs to start from the minimum lower bound of the two ranges, - // to ensure that the lower bounds are monotonically increasing - // - // e.g. `--xx--`, `xxxxxx` should first produce - // `xxxx--` and then `xxxxxx`, not - // `--xx--` and then `xxxxxx` - // - // lhs: xx---- xxxx-- --xx-- - // rhs: ----xx or --xxxx or xxxxxx - // output: xx---- xxxx-- xxxx-- - - cmp::min(lhs.start, rhs.start)..lhs.end - } - - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range { - cmp::min(lhs.start, rhs.start)..rhs.end - } - - fn advance_lhs_tail(&mut self, lhs: Range) -> Option> { - // the union of a range and an empty range is just that range - Some(lhs) - } - - fn advance_rhs_tail(&mut self, rhs: Range) -> Option> { - Some(rhs) - } -} - -/// The intersection combinator. -/// -/// Produces ranges over the bits that are in both of the input range iterators. -#[derive(Default)] -pub struct Intersection; - -impl Combinator for Intersection { - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range { - // lhs: xx---- xxxx-- --xx-- - // rhs: ----xx or --xxxx or xxxxxx - // output: ------ --xx-- --xx-- - - cmp::max(lhs.start, rhs.start)..lhs.end - } - - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range { - cmp::max(lhs.start, rhs.start)..rhs.end - } - - fn advance_lhs_tail(&mut self, _lhs: Range) -> Option> { - // the intersection of a range and an empty range is an empty range - None - } - - fn advance_rhs_tail(&mut self, _rhs: Range) -> Option> { - None - } -} - -/// The difference combinator. -/// -/// Produces ranges over the bits that are in the `lhs` range iterator, but not in the `rhs`. -#[derive(Default)] -pub struct Difference; - -impl Combinator for Difference { - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range { - // lhs: xx---- xxxx-- --xx-- - // rhs: ----xx or --xxxx or xxxxxx - // output: xx---- xx---- ------ - - lhs.start..cmp::min(lhs.end, rhs.start) - } - - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range { - // since we're advancing the rhs, we need to potentially shorten the lhs - // to avoid it from returning invalid bits in the next iteration - // - // e.g. `--xxxx`, `xxxx--` should first produce - // `------` and then `----xx`, not - // `------` and then `--xxxx` - // - // lhs: ----xx --xxxx xxxxxx - // rhs: xx---- or xxxx-- or --xx-- - // output: ------ ------ xx---- - // new lhs: ----xx ----xx ----xx - - let difference = lhs.start..cmp::min(lhs.end, rhs.start); - lhs.start = cmp::max(lhs.start, rhs.end); - difference - } - - fn advance_lhs_tail(&mut self, lhs: Range) -> Option> { - // the difference between a range and an empty range is just that range - Some(lhs) - } - - fn advance_rhs_tail(&mut self, _rhs: Range) -> Option> { - // the difference between an empty range and a range is an empty range - None - } -} - -/// The symmetric difference combinator. -/// -/// Produces ranges over the bits that are in one of the input range iterators, but not in both. -#[derive(Default)] -pub struct SymmetricDifference; - -impl SymmetricDifference { - /// Returns the symmetric difference of the two ranges where `left.end <= right.end`. - /// Adjusts `rhs` to not return invalid bits in the next iteration. - fn advance(left: Range, right: &mut Range) -> Range { - if left.start <= right.start { - // left: xxxx-- xx---- - // right: --xxxx or ----xx - // output: xx---- xx---- - // new right: ----xx ----xx - - let difference = left.start..cmp::min(left.end, right.start); - right.start = cmp::max(right.start, left.end); - difference - } else { - // left: --xx-- - // right: xxxxxx - // output: xx---- - // new right: ----xx - - let difference = right.start..left.start; - right.start = left.end; - difference - } - } -} - -impl Combinator for SymmetricDifference { - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range { - Self::advance(lhs, rhs) - } - - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range { - Self::advance(rhs, lhs) - } - - fn advance_lhs_tail(&mut self, lhs: Range) -> Option> { - // the symmetric difference of a range and an empty range is just that range - Some(lhs) - } - - fn advance_rhs_tail(&mut self, rhs: Range) -> Option> { - Some(rhs) - } -} - -/// The cut combinator. -/// -/// Produces ranges over the bits that remain after cutting the set bits of the `rhs` -/// out of the `lhs`, and shifting bits to the left to fill those gaps. -#[derive(Default)] -pub struct Cut { - /// Stores the number of bits that have been cut out so far, i.e. the number of bits - /// each output range needs to be shifted to the left by. - offset: usize, -} - -impl Cut { - /// Offsets an output range by the current offset. - fn offset(&self, range: Range) -> Range { - (range.start - self.offset)..(range.end - self.offset) - } -} - -impl Combinator for Cut { - fn advance_lhs(&mut self, lhs: Range, rhs: &mut Range) -> Range { - // apart from the offset, these implementations are identical to those of the `Difference` combinator - self.offset(lhs.start..cmp::min(lhs.end, rhs.start)) - } - - fn advance_rhs(&mut self, lhs: &mut Range, rhs: Range) -> Range { - let cut = self.offset(lhs.start..cmp::min(lhs.end, rhs.start)); - lhs.start = cmp::max(lhs.start, rhs.end); - self.offset += rhs.len(); - cut - } - - fn advance_lhs_tail(&mut self, lhs: Range) -> Option> { - Some(self.offset(lhs)) - } - - fn advance_rhs_tail(&mut self, _rhs: Range) -> Option> { - None - } -} - -/// Combines two range iterators according to the given combinator, and merges -/// the output ranges together. -pub struct Combine(Merge<_Combine>) -where - A: RangeIterator, - B: RangeIterator, - C: Combinator; - -impl Combine -where - A: RangeIterator, - B: RangeIterator, - C: Combinator, -{ - pub fn new(a: A, b: B) -> Self { - Self(Merge::new(_Combine::new(a, b))) - } -} - -impl Iterator for Combine -where - A: RangeIterator, - B: RangeIterator, - C: Combinator, -{ - type Item = Range; - - fn next(&mut self) -> Option { - self.0.next() - } -} - -impl RangeIterator for Combine -where - A: RangeIterator, - B: RangeIterator, - C: Combinator, -{ -} - -/// Combines two range iterators according to the given combinator, but does not -/// merge the output ranges together. Since the ranges can overlap, this does not -/// satisfy the `RangeIterator` requirements. -struct _Combine -where - A: RangeIterator, - B: RangeIterator, -{ - lhs: Lookahead, - rhs: Lookahead, - combinator: C, -} - -impl _Combine -where - A: RangeIterator, - B: RangeIterator, - C: Combinator, -{ - fn new(lhs: A, rhs: B) -> Self { - Self { - lhs: Lookahead::new(lhs), - rhs: Lookahead::new(rhs), - combinator: Default::default(), - } - } - - /// Computes the next range by inspecting the next range of each of the input - /// range iterators and passing them to the combinator. Also advances the range - /// iterator which corresponding range has the lowest upper bound. - fn next_range(&mut self) -> Option> { - let (range, advance_lhs) = match (self.lhs.peek(), self.rhs.peek()) { - (Some(lhs), Some(rhs)) => { - // if both iterators are non-empty, we advance the one whichever's - // corresponding range has a smaller upper bound - if lhs.end <= rhs.end { - (Some(self.combinator.advance_lhs(lhs.clone(), rhs)), true) - } else { - (Some(self.combinator.advance_rhs(lhs, rhs.clone())), false) - } - } - (Some(lhs), None) => (self.combinator.advance_lhs_tail(lhs.clone()), true), - (None, Some(rhs)) => (self.combinator.advance_rhs_tail(rhs.clone()), false), - (None, None) => return None, - }; - - if advance_lhs { - self.lhs.next(); - } else { - self.rhs.next(); - } - - range - } -} - -impl Iterator for _Combine -where - A: RangeIterator, - B: RangeIterator, - C: Combinator, -{ - type Item = Range; - - fn next(&mut self) -> Option { - // we repeatedly compute the next range until we find one that is non-empty - // TODO: use `!range.is_empty()` once it stabilizes in Rust 1.47 - iter::from_fn(|| self.next_range()).find(|range| range.start < range.end) - } -} - -/// A range iterator that wraps an iterator of ranges and merges the overlapping -/// (and touching) ranges together. -/// -/// For example, given the ranges: -/// -/// ```ignore -/// xx-------- -/// xxx------- -/// ---xx----- -/// ---x------ -/// -------xx- -/// --------xx -/// ``` -/// -/// `Merge` will produce -/// -/// ```ignore -/// xxxxx--xxx -/// ``` -/// -/// Since this is done lazily, it's required that the ranges of the underlying -/// iterator increase monotonically (i.e. are non-decreasing) in their lower bound. -/// Also requires that the underlying ranges are non-empty. -struct Merge { - iter: Lookahead, -} - -impl Merge -where - I: Iterator>, -{ - pub fn new(iter: I) -> Self { - Self { - iter: Lookahead::new(iter), - } - } -} - -impl Iterator for Merge -where - I: Iterator>, -{ - type Item = Range; - - fn next(&mut self) -> Option { - let mut range = self.iter.next()?; - - // as long as the next range overlaps with (or touches) current range, - // we merge it into the current range - while let Some(next) = self.iter.peek() { - if next.start > range.end { - break; - } - - range.end = cmp::max(range.end, next.end); - self.iter.next(); - } - - Some(range) - } -} - -impl RangeIterator for Merge where I: Iterator> {} - -/// An iterator wrapper that stores (and gives mutable access to) the next item of the iterator. -/// -/// Similar to `std::iter::Peekable`, but unlike `Peekable`, `Lookahead` stores the next item -/// unconditionally (if there is any). -struct Lookahead { - iter: I, - next: Option, -} - -impl Lookahead { - fn new(mut iter: I) -> Self { - let next = iter.next(); - Self { iter, next } - } - - fn peek(&mut self) -> Option<&mut I::Item> { - self.next.as_mut() - } -} - -impl Iterator for Lookahead { - type Item = I::Item; - - fn next(&mut self) -> Option { - // `self.next` always stores the next element, so if it is `None`, the iterator is empty - let next = self.next.take()?; - self.next = self.iter.next(); - Some(next) - } -} diff --git a/utils/bitfield/src/iter/mod.rs b/utils/bitfield/src/iter/mod.rs deleted file mode 100644 index dc36abc7cf3b..000000000000 --- a/utils/bitfield/src/iter/mod.rs +++ /dev/null @@ -1,517 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -mod combine; - -use combine::{Combine, Cut, Difference, Intersection, SymmetricDifference, Union}; -use std::{iter, ops::Range}; - -/// A trait for iterators over `Range`. -/// -/// Requirements: -/// - all ranges are non-empty -/// - the ranges are in ascending order -/// - no two ranges overlap or touch -pub trait RangeIterator: Iterator> + Sized { - /// Returns a new `RangeIterator` over the bits that are in `self`, in `other`, or in both. - fn union(self, other: R) -> Combine { - Combine::new(self, other) - } - - /// Returns a new `RangeIterator` over the bits that are in both `self` and `other`. - fn intersection(self, other: R) -> Combine { - Combine::new(self, other) - } - - /// Returns a new `RangeIterator` over the bits that are in `self` but not in `other`. - fn difference(self, other: R) -> Combine { - Combine::new(self, other) - } - - /// Returns a new `RangeIterator` over the bits that are in `self` or in `other`, but not in both. - fn symmetric_difference( - self, - other: R, - ) -> Combine { - Combine::new(self, other) - } - - /// Returns a new `RangeIterator` over the bits in `self` that remain after "cutting" out the - /// bits in `other`, and shifting remaining bits to the left if necessary. For example: - /// - /// ```ignore - /// lhs: xx-xxx--x - /// rhs: -xx-x---- - /// - /// cut: x x x--x - /// output: xxx--x - /// ``` - fn cut(self, other: R) -> Combine { - Combine::new(self, other) - } - - /// Returns a new `RangeIterator` over the bits in `self` after skipping the first `n` bits. - fn skip_bits(self, n: usize) -> Skip { - Skip { - iter: self, - skip: n, - } - } - - /// Returns a new `RangeIterator` over the first `n` bits in `self`. - fn take_bits(self, n: usize) -> Take { - Take { - iter: self, - take: n, - } - } -} - -/// A `RangeIterator` that skips over `n` bits of antoher `RangeIterator`. -pub struct Skip { - iter: I, - skip: usize, -} - -impl Iterator for Skip { - type Item = Range; - - fn next(&mut self) -> Option { - loop { - let mut range = self.iter.next()?; - - if range.len() > self.skip { - range.start += self.skip; - self.skip = 0; - return Some(range); - } else { - self.skip -= range.len(); - } - } - } -} - -impl RangeIterator for Skip {} - -/// A `RangeIterator` that iterates over the first `n` bits of antoher `RangeIterator`. -pub struct Take { - iter: I, - take: usize, -} - -impl Iterator for Take { - type Item = Range; - - fn next(&mut self) -> Option { - if self.take == 0 { - return None; - } - - let mut range = self.iter.next()?; - - if range.len() > self.take { - range.end = range.start + self.take; - } - - self.take -= range.len(); - Some(range) - } -} - -impl RangeIterator for Take {} - -/// A `RangeIterator` that wraps a regular iterator over `Range` as a way to explicitly -/// indicate that this iterator satisfies the requirements of the `RangeIterator` trait. -pub struct Ranges(I); - -impl Ranges -where - I: Iterator>, -{ - /// Creates a new `Ranges` instance. - pub fn new(iter: II) -> Self - where - II: IntoIterator>, - { - Self(iter.into_iter()) - } -} - -impl Iterator for Ranges -where - I: Iterator>, -{ - type Item = Range; - - fn next(&mut self) -> Option { - self.0.next() - } -} - -impl RangeIterator for Ranges where I: Iterator> {} - -/// Returns a `RangeIterator` which ranges contain the values from the provided iterator. -/// The values need to be in ascending order — if not, the returned iterator may not satisfy -/// all `RangeIterator` requirements. -pub fn ranges_from_bits(bits: impl IntoIterator) -> impl RangeIterator { - let mut iter = bits.into_iter().peekable(); - - Ranges::new(iter::from_fn(move || { - let start = iter.next()?; - let mut end = start + 1; - while iter.peek() == Some(&end) { - end += 1; - iter.next(); - } - Some(start..end) - })) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ranges(slice: &[Range]) -> impl RangeIterator + '_ { - Ranges::new(slice.iter().cloned()) - } - - #[test] - fn test_combinators() { - struct Case<'a> { - lhs: &'a [Range], - rhs: &'a [Range], - union: &'a [Range], - intersection: &'a [Range], - difference: &'a [Range], - symmetric_difference: &'a [Range], - cut: &'a [Range], - } - - for &Case { - lhs, - rhs, - union, - intersection, - difference, - symmetric_difference, - cut, - } in &[ - Case { - // --xxx - lhs: &[2..5], - - // ----- - rhs: &[], - - // --xxx - union: &[2..5], - - // ----- - intersection: &[], - - // --xxx - difference: &[2..5], - - // --xxx - symmetric_difference: &[2..5], - - // --xxx - cut: &[2..5], - }, - Case { - // xxx-------xxx - lhs: &[0..3, 10..13], - - // -----xxx----- - rhs: &[5..8], - - // xxx--xxx--xxx - union: &[0..3, 5..8, 10..13], - - // ------------- - intersection: &[], - - // xxx-------xxx - difference: &[0..3, 10..13], - - // xxx--xxx--xxx - symmetric_difference: &[0..3, 5..8, 10..13], - - // xxx-- --xxx - // xxx----xxx - cut: &[0..3, 7..10], - }, - Case { - // xxx-----xxx - lhs: &[0..3, 8..11], - - // --xxx------ - rhs: &[2..5], - - // xxxxx---xxx - union: &[0..5, 8..11], - - // --x-------- - intersection: &[2..3], - - // xx------xxx - difference: &[0..2, 8..11], - - // xx-xx---xxx - symmetric_difference: &[0..2, 3..5, 8..11], - - // xx ---xxx - // xx---xxx - cut: &[0..2, 5..8], - }, - Case { - // xxx-xxx-xxx-- - lhs: &[0..3, 4..7, 8..11], - - // --xxx-xxx-xxx - rhs: &[2..5, 6..9, 10..13], - - // xxxxxxxxxxxxx - union: &[0..13], - - // --x-x-x-x-x-- - intersection: &[2..3, 4..5, 6..7, 8..9, 10..11], - - // xx---x---x--- - difference: &[0..2, 5..6, 9..10], - - // xx-x-x-x-x-xx - symmetric_difference: &[0..2, 3..4, 5..6, 7..8, 9..10, 11..13], - - // xx x x - // xxxx - cut: &[0..4], - }, - Case { - // xxxxxx - lhs: &[0..6], - - // -xx--- - rhs: &[1..3], - - // xxxxxx - union: &[0..6], - - // -xx--- - intersection: &[1..3], - - // x--xxx - difference: &[0..1, 3..6], - - // x--xxx - symmetric_difference: &[0..1, 3..6], - - // x xxx - // xxxx - cut: &[0..4], - }, - Case { - // xxxxxx----- - lhs: &[0..6], - - // -xx--xx--xx - rhs: &[1..3, 5..7, 9..11], - - // xxxxxxx--xx - union: &[0..7, 9..11], - - // -xx--x----- - intersection: &[1..3, 5..6], - - // x--xx------ - difference: &[0..1, 3..5], - - // x--xx-x--xx - symmetric_difference: &[0..1, 3..5, 6..7, 9..11], - - // x xx -- - // xxx-- - cut: &[0..3], - }, - Case { - // ---xxx---- - lhs: &[3..6], - - // xx--x---xx - rhs: &[0..2, 4..5, 8..10], - - // xx-xxx--xx - union: &[0..2, 3..6, 8..10], - - // ----x----- - intersection: &[4..5], - - // ---x-x---- - difference: &[3..4, 5..6], - - // xx-x-x--xx - symmetric_difference: &[0..2, 3..4, 5..6, 8..10], - - // -x x-- - // -xx-- - cut: &[1..3], - }, - Case { - // ---xxx--xx- - lhs: &[3..6, 8..10], - - // --xxxxx-xxx - rhs: &[2..7, 8..11], - - // --xxxxx-xxx - union: &[2..7, 8..11], - - // ---xxx--xx- - intersection: &[3..6, 8..10], - - // ----------- - difference: &[], - - // --x---x---x - symmetric_difference: &[2..3, 6..7, 10..11], - - // -- - - // --- - cut: &[], - }, - Case { - // ---xxx--xx - lhs: &[3..6, 8..10], - - // --xx------ - rhs: &[2..4], - - // --xxxx--xx - union: &[2..6, 8..10], - - // ---x------ - intersection: &[3..4], - - // ----xx--xx - difference: &[4..6, 8..10], - - // --x-xx--xx - symmetric_difference: &[2..3, 4..6, 8..10], - - // -- xx--xx - // --xx--xx - cut: &[2..4, 6..8], - }, - ] { - assert_eq!(ranges(lhs).union(ranges(rhs)).collect::>(), union); - assert_eq!(ranges(rhs).union(ranges(lhs)).collect::>(), union); - - assert_eq!( - ranges(lhs).intersection(ranges(rhs)).collect::>(), - intersection - ); - assert_eq!( - ranges(rhs).intersection(ranges(lhs)).collect::>(), - intersection - ); - - assert_eq!( - ranges(lhs).difference(ranges(rhs)).collect::>(), - difference - ); - - assert_eq!( - ranges(lhs) - .symmetric_difference(ranges(rhs)) - .collect::>(), - symmetric_difference - ); - - assert_eq!(ranges(lhs).cut(ranges(rhs)).collect::>(), cut); - } - } - - #[test] - fn test_ranges_from_bits() { - struct Case<'a> { - input: &'a [usize], - output: &'a [Range], - } - for &Case { input, output } in &[ - Case { - input: &[], - output: &[], - }, - Case { - input: &[10], - output: &[10..11], - }, - Case { - input: &[2, 3, 4, 7, 9, 11, 12], - output: &[2..5, 7..8, 9..10, 11..13], - }, - ] { - assert_eq!( - ranges_from_bits(input.iter().copied()).collect::>(), - output - ); - } - } - - #[test] - fn test_skip_take() { - struct Case<'a> { - input: &'a [Range], - n: usize, - skip: &'a [Range], - take: &'a [Range], - } - - for &Case { - input, - n, - skip, - take, - } in &[ - Case { - input: &[], - n: 0, - skip: &[], - take: &[], - }, - Case { - input: &[], - n: 3, - skip: &[], - take: &[], - }, - Case { - input: &[1..3, 4..6], - n: 0, - skip: &[1..3, 4..6], - take: &[], - }, - Case { - input: &[1..3, 4..6], - n: 1, - skip: &[2..3, 4..6], - take: &[1..2], - }, - Case { - input: &[1..3, 4..6], - n: 2, - skip: &[4..6], - take: &[1..3], - }, - Case { - input: &[1..3, 4..6], - n: 3, - skip: &[5..6], - take: &[1..3, 4..5], - }, - ] { - assert_eq!(ranges(input).skip_bits(n).collect::>(), skip); - assert_eq!(ranges(input).take_bits(n).collect::>(), take); - } - } -} diff --git a/utils/bitfield/src/lib.rs b/utils/bitfield/src/lib.rs deleted file mode 100644 index 52fa054843cd..000000000000 --- a/utils/bitfield/src/lib.rs +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -pub mod iter; -mod rleplus; -mod unvalidated; - -pub use unvalidated::{UnvalidatedBitField, Validate}; - -use ahash::AHashSet; -use iter::{ranges_from_bits, RangeIterator}; -use std::{ - iter::FromIterator, - ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Range, Sub, SubAssign}, -}; - -type Result = std::result::Result; - -// MaxEncodedSize is the maximum encoded size of a bitfield. When expanded into -// a slice of runs, a bitfield of this size should not exceed 2MiB of memory. -// -// This bitfield can fit at least 3072 sparse elements. -const MAX_ENCODED_SIZE: usize = 32 << 10; - -/// A bit field with buffered insertion/removal that serializes to/from RLE+. Similar to -/// `HashSet`, but more memory-efficient when long runs of 1s and 0s are present. -#[derive(Debug, Default, Clone)] -pub struct BitField { - /// The underlying ranges of 1s. - ranges: Vec>, - /// Bits set to 1. Never overlaps with `unset`. - set: AHashSet, - /// Bits set to 0. Never overlaps with `set`. - unset: AHashSet, -} - -impl From for BitField { - fn from(bitfield: fvm_ipld_bitfield::BitField) -> Self { - bitfield.iter().map(|v| v as usize).collect() - } -} -impl From for fvm_ipld_bitfield::BitField { - fn from(bitfield: BitField) -> Self { - bitfield - .iter() - .map(|v| v as u64) - .collect::() - .unwrap() - } -} - -impl PartialEq for BitField { - fn eq(&self, other: &Self) -> bool { - Iterator::eq(self.ranges(), other.ranges()) - } -} - -impl FromIterator for BitField { - fn from_iter>(iter: I) -> Self { - let mut vec: Vec<_> = iter.into_iter().collect(); - vec.sort_unstable(); - Self::from_ranges(ranges_from_bits(vec)) - } -} - -impl FromIterator for BitField { - fn from_iter>(iter: I) -> Self { - let bits = iter - .into_iter() - .enumerate() - .filter(|&(_, b)| b) - .map(|(i, _)| i); - Self::from_ranges(ranges_from_bits(bits)) - } -} - -impl BitField { - /// Creates an empty bit field. - pub fn new() -> Self { - Self::default() - } - - /// Creates a new bit field from a `RangeIterator`. - pub fn from_ranges(iter: impl RangeIterator) -> Self { - Self { - ranges: iter.collect(), - ..Default::default() - } - } - - /// Adds the bit at a given index to the bit field. - pub fn set(&mut self, bit: usize) { - self.unset.remove(&bit); - self.set.insert(bit); - } - - /// Removes the bit at a given index from the bit field. - pub fn unset(&mut self, bit: usize) { - self.set.remove(&bit); - self.unset.insert(bit); - } - - /// Returns `true` if the bit field contains the bit at a given index. - pub fn get(&self, index: usize) -> bool { - if self.set.contains(&index) { - true - } else if self.unset.contains(&index) { - false - } else { - // since `self.ranges` is ordered, we can use a binary search to find out if - // any range in `self.ranges` contains `index` - use std::cmp::Ordering; - self.ranges - .binary_search_by(|range| { - if index < range.start { - Ordering::Greater - } else if index >= range.end { - Ordering::Less - } else { - // `index` is contained by this range - Ordering::Equal - } - }) - // Ok(range) is returned if the closure returns `Equal` for a certain range, - // meaning a range in `self.ranges` contains the given index - .is_ok() - } - } - - /// Returns the index of the lowest bit present in the bit field. - pub fn first(&self) -> Option { - // similar to `self.iter.next()`, but optimized using the fact that only the - // lowest bit in `self.set` is a candidate, and therefore there's no need to - // sort all bits in `self.set` - - let min_set_bit = self.set.iter().min(); - - // turns the `Option<&usize>` minimum set bit into an `Option>` - let min_range = min_set_bit.map(|&bit| bit..bit + 1); - - // turns this `Option>` into a `RangeIterator`, relying on the - // fact that `Option` is an `IntoIterator` over `T` with 0 or 1 items - let min_range_iterator = iter::Ranges::new(min_range); - - self.inner_ranges() - .union(min_range_iterator) - .flatten() - .find(|i| !self.unset.contains(i)) - } - - /// Returns the index of the highest bit present in the bit field. - /// Errors if no bits are set. Merges set/unset into ranges, so be cautious with use if set is pretty populated - pub fn last(&self) -> Result { - self.ranges() - .last() - .map(|r| r.end - 1) - .ok_or("no last bit set") - } - - /// Returns an iterator over the indices of the bit field's set bits. - pub fn iter(&self) -> impl Iterator + '_ { - // this code results in the same values as `self.ranges().flatten()`, but there's - // a key difference: - // - // `ranges()` needs to traverse both `self.set` and `self.unset` up front (so before - // iteration starts) in order to not have to visit each individual bit of `self.bitvec` - // during iteration, while here we can get away with only traversing `self.set` up - // front and checking `self.unset` containment for the candidate bits on the fly - // because we're visiting all bits either way - // - // consequently, the time complexity of `self.first()` is only linear in the length of - // `self.set`, not in the length of `self.unset` (as opposed to getting the first range - // with `self.ranges().next()` which is linear in both) - - let mut set_bits: Vec<_> = self.set.iter().copied().collect(); - set_bits.sort_unstable(); - - self.inner_ranges() - .union(ranges_from_bits(set_bits)) - .flatten() - .filter(move |i| !self.unset.contains(i)) - } - - /// Returns an iterator over the indices of the bit field's set bits if the number - /// of set bits in the bit field does not exceed `max`. Returns an error otherwise. - pub fn bounded_iter(&self, max: usize) -> Result + '_> { - if self.len() <= max { - Ok(self.iter()) - } else { - Err("Bits set exceeds max in retrieval") - } - } - - /// Returns an iterator over the ranges without applying the set/unset bits. - fn inner_ranges(&self) -> impl RangeIterator + '_ { - iter::Ranges::new(self.ranges.iter().cloned()) - } - - /// Returns an iterator over the ranges of set bits that make up the bit field. The - /// ranges are in ascending order, are non-empty, and don't overlap. - pub fn ranges(&self) -> impl RangeIterator + '_ { - let ranges = |set: &AHashSet| { - let mut vec: Vec<_> = set.iter().copied().collect(); - vec.sort_unstable(); - ranges_from_bits(vec) - }; - - self.inner_ranges() - .union(ranges(&self.set)) - .difference(ranges(&self.unset)) - } - - /// Returns `true` if the bit field is empty. - pub fn is_empty(&self) -> bool { - self.set.is_empty() - && self - .inner_ranges() - .flatten() - .all(|bit| self.unset.contains(&bit)) - } - - /// Returns a slice of the bit field with the start index of set bits - /// and number of bits to include in the slice. Returns an error if the - /// bit field contains fewer than `start + len` set bits. - pub fn slice(&self, start: usize, len: usize) -> Result { - let slice = BitField::from_ranges(self.ranges().skip_bits(start).take_bits(len)); - - if slice.len() == len { - Ok(slice) - } else { - Err("Not enough bits") - } - } - - /// Returns the number of set bits in the bit field. - pub fn len(&self) -> usize { - self.ranges().map(|range| range.len()).sum() - } - - /// Returns a new bit field containing the bits in `self` that remain - /// after "cutting" out the bits in `other`, and shifting remaining - /// bits to the left if necessary. For example: - /// - /// ```ignore - /// lhs: xx-xxx--x - /// rhs: -xx-x---- - /// - /// cut: x x x--x - /// output: xxx--x - /// ``` - pub fn cut(&self, other: &Self) -> Self { - Self::from_ranges(self.ranges().cut(other.ranges())) - } - - /// Returns the union of the given bit fields as a new bit field. - pub fn union<'a>(bitfields: impl IntoIterator) -> Self { - bitfields.into_iter().fold(Self::new(), |a, b| &a | b) - } - - /// Returns true if `self` overlaps with `other`. - pub fn contains_any(&self, other: &BitField) -> bool { - self.ranges().intersection(other.ranges()).next().is_some() - } - - /// Returns true if the `self` is a superset of `other`. - pub fn contains_all(&self, other: &BitField) -> bool { - other.ranges().difference(self.ranges()).next().is_none() - } -} - -impl BitOr<&BitField> for &BitField { - type Output = BitField; - - #[inline] - fn bitor(self, rhs: &BitField) -> Self::Output { - BitField::from_ranges(self.ranges().union(rhs.ranges())) - } -} - -impl BitOrAssign<&BitField> for BitField { - #[inline] - fn bitor_assign(&mut self, rhs: &BitField) { - *self = &*self | rhs; - } -} - -impl BitAnd<&BitField> for &BitField { - type Output = BitField; - - #[inline] - fn bitand(self, rhs: &BitField) -> Self::Output { - BitField::from_ranges(self.ranges().intersection(rhs.ranges())) - } -} - -impl BitAndAssign<&BitField> for BitField { - #[inline] - fn bitand_assign(&mut self, rhs: &BitField) { - *self = &*self & rhs; - } -} - -impl Sub<&BitField> for &BitField { - type Output = BitField; - - #[inline] - fn sub(self, rhs: &BitField) -> Self::Output { - BitField::from_ranges(self.ranges().difference(rhs.ranges())) - } -} - -impl SubAssign<&BitField> for BitField { - #[inline] - fn sub_assign(&mut self, rhs: &BitField) { - *self = &*self - rhs; - } -} - -impl BitXor<&BitField> for &BitField { - type Output = BitField; - - fn bitxor(self, rhs: &BitField) -> Self::Output { - BitField::from_ranges(self.ranges().symmetric_difference(rhs.ranges())) - } -} - -impl BitXorAssign<&BitField> for BitField { - fn bitxor_assign(&mut self, rhs: &BitField) { - *self = &*self ^ rhs; - } -} - -/// Constructs a `BitField` from a given list of 1s and 0s. -/// -/// # Examples -/// -/// ``` -/// use forest_bitfield::bitfield; -/// -/// let mut bf = bitfield![0, 1, 1, 0, 1, 0, 0, 0, 1, 1]; -/// assert!(bf.get(1)); -/// assert!(!bf.get(3)); -/// bf.set(3); -/// assert_eq!(bf.len(), 6); -/// assert_eq!(bf.ranges().next(), Some(1..5)); -/// ``` -#[macro_export] -macro_rules! bitfield { - (@iter) => { - std::iter::empty::() - }; - (@iter $head:literal $(, $tail:literal)*) => { - std::iter::once($head != 0_u32).chain(bitfield!(@iter $($tail),*)) - }; - ($($val:literal),* $(,)?) => { - bitfield!(@iter $($val),*).collect::<$crate::BitField>() - }; -} - -#[cfg(feature = "json")] -pub mod json { - use super::*; - use crate::iter::Ranges; - use serde::ser::SerializeSeq; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - #[derive(Deserialize, Serialize, Debug, PartialEq)] - #[serde(transparent)] - pub struct BitFieldJson(#[serde(with = "self")] pub BitField); - - /// Wrapper for serializing a UnsignedMessage reference to JSON. - #[derive(Serialize)] - #[serde(transparent)] - pub struct BitFieldJsonRef<'a>(#[serde(with = "self")] pub &'a BitField); - - impl From for BitField { - fn from(wrapper: BitFieldJson) -> Self { - wrapper.0 - } - } - - impl From for BitFieldJson { - fn from(wrapper: BitField) -> Self { - BitFieldJson(wrapper) - } - } - - fn serialize(m: &BitField, serializer: S) -> std::result::Result - where - S: Serializer, - { - let total: usize = m.len(); - - if !m.is_empty() { - let mut seq = serializer.serialize_seq(Some(total))?; - m.ranges().fold(Ok(0), |last_index, range| { - let last_index = last_index?; - let zero_index = (range.start - last_index) as u8; - let nonzero_index = (range.end - range.start) as u8; - seq.serialize_element(&zero_index)?; - seq.serialize_element(&nonzero_index)?; - Ok(range.end) - })?; - seq.end() - } else { - let mut seq = serializer.serialize_seq(Some(1))?; - seq.serialize_element(&0)?; - seq.end() - } - } - - fn deserialize<'de, D>(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let bitfield_bytes: Vec = Deserialize::deserialize(deserializer)?; - let mut ranges: Vec> = Vec::new(); - bitfield_bytes.iter().fold((false, 0), |last, index| { - let (should_set, last_index) = last; - let ending_index = index + last_index; - if should_set { - ranges.push(Range { - start: last_index, - end: ending_index, - }) - } - - (!should_set, ending_index) - }); - let ranges = Ranges::new(ranges.iter().cloned()); - Ok(BitField::from_ranges(ranges)) - } - - #[test] - fn serialization_starts_with_zeros() { - let bf = BitFieldJson(bitfield![0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1]); - let j = serde_json::to_string(&bf).unwrap(); - assert_eq!(j, "[2,4,3,2]"); - let bitfield: BitFieldJson = serde_json::from_str(&j).unwrap(); - assert_eq!(bf, bitfield); - } - - #[test] - fn serialization_starts_with_ones() { - let bf = BitFieldJson(bitfield![1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1]); - let j = serde_json::to_string(&bf).unwrap(); - assert_eq!(j, "[0,6,3,2]"); - let bitfield: BitFieldJson = serde_json::from_str(&j).unwrap(); - assert_eq!(bf, bitfield); - } - - #[test] - fn serialization_with_single_unut() { - let bf = BitFieldJson(bitfield![]); - let j = serde_json::to_string(&bf).unwrap(); - assert_eq!(j, "[0]"); - let bitfield: BitFieldJson = serde_json::from_str(&j).unwrap(); - assert_eq!(bf, bitfield); - } -} diff --git a/utils/bitfield/src/rleplus/mod.rs b/utils/bitfield/src/rleplus/mod.rs deleted file mode 100644 index 9ebd484ce000..000000000000 --- a/utils/bitfield/src/rleplus/mod.rs +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -//! # RLE+ Bitset Encoding -//! -//! (from https://github.com/filecoin-project/specs/blob/master/src/listings/data_structures.md) -//! -//! RLE+ is a lossless compression format based on [RLE](https://en.wikipedia.org/wiki/Run-length_encoding). -//! Its primary goal is to reduce the size in the case of many individual bits, where RLE breaks down quickly, -//! while keeping the same level of compression for large sets of contiguous bits. -//! -//! In tests it has shown to be more compact than RLE iteself, as well as [Concise](https://arxiv.org/pdf/1004.0403.pdf) and [Roaring](https://roaringbitmap.org/). -//! -//! ## Format -//! -//! The format consists of a header, followed by a series of blocks, of which there are three different types. -//! -//! The format can be expressed as the following [BNF](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form) grammar. -//! -//! ```text -//! ::=
-//!
::= -//! ::= "00" -//! ::= | "" -//! ::= | | -//! ::= "1" -//! ::= "01" -//! ::= "00" -//! ::= "0" | "1" -//! ``` -//! -//! An `` is defined as specified [here](https://github.com/multiformats/unsigned-varint). -//! -//! ### Header -//! -//! The header indiciates the very first bit of the bit vector to encode. This means the first bit is always -//! the same for the encoded and non encoded form. -//! -//! ### Blocks -//! -//! The blocks represent how many bits, of the current bit type there are. As `0` and `1` alternate in a bit vector -//! the inital bit, which is stored in the header, is enough to determine if a length is currently referencing -//! a set of `0`s, or `1`s. -//! -//! #### Block Single -//! -//! If the running length of the current bit is only `1`, it is encoded as a single set bit. -//! -//! #### Block Short -//! -//! If the running length is less than `16`, it can be encoded into up to four bits, which a short block -//! represents. The length is encoded into a 4 bits, and prefixed with `01`, to indicate a short block. -//! -//! #### Block Long -//! -//! If the running length is `16` or larger, it is encoded into a varint, and then prefixed with `00` to indicate -//! a long block. -//! -//! -//! > **Note:** The encoding is unique, so no matter which algorithm for encoding is used, it should produce -//! > the same encoding, given the same input. -//! - -mod reader; -mod writer; - -pub use reader::BitReader; -pub use writer::BitWriter; - -use super::{BitField, Result, MAX_ENCODED_SIZE}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::borrow::Cow; - -pub const VERSION: u8 = 0; - -impl Serialize for BitField { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - let bytes = self.to_bytes(); - if bytes.len() > MAX_ENCODED_SIZE { - return Err(serde::ser::Error::custom(format!( - "encoded bitfield was too large {}", - bytes.len() - ))); - } - serde_bytes::serialize(&bytes, serializer) - } -} - -impl<'de> Deserialize<'de> for BitField { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let bytes: Cow<'de, [u8]> = serde_bytes::deserialize(deserializer)?; - if bytes.len() > MAX_ENCODED_SIZE { - return Err(serde::de::Error::custom(format!( - "decoded bitfield was too large {}", - bytes.len() - ))); - } - Self::from_bytes(&bytes).map_err(serde::de::Error::custom) - } -} - -impl BitField { - /// Decodes RLE+ encoded bytes into a bit field. - pub fn from_bytes(bytes: &[u8]) -> Result { - if let Some(value) = bytes.last() { - if *value == 0 { - return Err("not minimally encoded"); - } - } - - let mut reader = BitReader::new(bytes); - - let version = reader.read(2); - if version != VERSION { - return Err("incorrect version"); - } - - let mut next_value = reader.read(1) == 1; - let mut ranges = Vec::new(); - let mut index = 0; - let mut total_len: u64 = 0; - - while let Some(len) = reader.read_len()? { - let (new_total_len, ovf) = total_len.overflowing_add(len as u64); - if ovf { - return Err("RLE+ overflow"); - } - total_len = new_total_len; - - let start = index; - index += len; - let end = index; - - if next_value { - ranges.push(start..end); - } - - next_value = !next_value; - } - - Ok(Self { - ranges, - ..Default::default() - }) - } - - /// Turns a bit field into its RLE+ encoded form. - pub fn to_bytes(&self) -> Vec { - let mut iter = self.ranges(); - - let first_range = match iter.next() { - Some(range) => range, - None => return Default::default(), - }; - - let mut writer = BitWriter::new(); - writer.write(0, 2); // version 00 - - if first_range.start == 0 { - writer.write(1, 1); // the first bit is a 1 - } else { - writer.write(0, 1); // the first bit is a 0 - writer.write_len(first_range.start); // the number of leading 0s - } - - writer.write_len(first_range.len()); - let mut index = first_range.end; - - // for each range of 1s we first encode the number of 0s that came prior - // before encoding the number of 1s - for range in iter { - writer.write_len(range.start - index); // zeros - writer.write_len(range.len()); // ones - index = range.end; - } - - writer.finish() - } -} - -#[cfg(test)] -mod tests { - use super::{ - super::{bitfield, ranges_from_bits}, - BitField, BitWriter, - }; - - use rand::{Rng, SeedableRng}; - use rand_xorshift::XorShiftRng; - - #[test] - fn test() { - for (bits, expected) in vec![ - (vec![], Ok(bitfield![])), - ( - vec![ - 1, 0, // incorrect version - 1, // starts with 1 - 0, 1, // fits into 4 bits - 0, 0, 0, 1, // 8 - 1 - ], - Err("incorrect version"), - ), - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 1, // fits into 4 bits - 0, 0, 0, 1, // 8 - 1 - ], - Ok(bitfield![1, 1, 1, 1, 1, 1, 1, 1]), - ), - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 1, // fits into 4 bits - 0, 0, 1, 0, // 4 - 1 - 1, // 1 - 0 - 0, 1, // fits into 4 bits - 1, 1, 0, 0, // 3 - 1 - ], - Ok(bitfield![1, 1, 1, 1, 0, 1, 1, 1]), - ), - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // does not fit into 4 bits - 1, 0, 0, 1, 1, 0, 0, 0, // 25 - 1 - ], - Ok(bitfield![ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - ]), - ), - // when a length of 0 is encountered, the rest of the encoded bits should be ignored - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 1, // 1 - 1 - 0, 1, // fits into 4 bits - 0, 0, 0, 0, // 0 - 0 - 1, // 1 - 1 - ], - Ok(bitfield![1]), - ), - // when a length of 0 is encountered, the rest of the encoded bits should be ignored - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 1, // 1 - 1 - 0, 0, // fits into a varint - 0, 0, 0, 0, 0, 0, 0, 0, // 0 - 0 - 1, // 1 - 1 - ], - Ok(bitfield![1]), - ), - // when the last byte is zero, this should fail - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 1, // fits into 4 bits - 1, 0, 1, // 5 - 1 - 0, 0, 0, 0, 0, 0, 0, 0, - ], - Err("not minimally encoded"), - ), - // a valid varint - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 0, 0, 0, 1, 0, 0, 0, // 17 - 1 - 0, 0, 0, - ], - Ok(bitfield![1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]), - ), - // a varint that is not minimally encoded - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 1, 0, 0, 0, 0, 0, 1, // 3 - 1 - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, - ], - Err("Invalid varint"), - ), - // a varint must not take more than 9 bytes - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 0, 0, 0, 0, 0, 0, 1, // 1 - 1 - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - ], - Err("Invalid varint"), - ), - // total running length should not overflow - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 1, 1, 1, 1, 1, 1, 1, // 9223372036854775807 - 1 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, // fits into a varint - 1, 1, 1, 1, 1, 1, 1, 1, // 9223372036854775807 - 0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, // fits into 4 bits - 0, 1, 0, 0, // 2 - 1 - ], - Err("RLE+ overflow"), - ), - // block_long that could have fit on block_short. TODO: is this legit? - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 1, 0, 0, 0, 0, 0, 0, // 3 - 1 - 1, 1, 1, - ], - Ok(bitfield![1, 1, 1, 0, 1, 0]), - ), - // block_long that could have fit on block_single. TODO: is this legit? - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 0, // fits into a varint - 1, 0, 0, 0, 0, 0, 0, 0, // 1 - 1 - 1, 1, 1, - ], - Ok(bitfield![1, 0, 1, 0]), - ), - // block_short that could have fit on block_single. TODO: is this legit? - ( - vec![ - 0, 0, // version - 1, // starts with 1 - 0, 1, // fits into 4 bits - 1, 0, 0, 0, // 1 - 1 - 1, 1, 1, 1, 1, 1, 1, - ], - Ok(bitfield![1, 0, 1, 0, 1, 0, 1, 0]), - ), - ] { - let mut writer = BitWriter::new(); - for bit in bits { - writer.write(bit, 1); - } - let res = BitField::from_bytes(&writer.finish_test()); - assert_eq!(res, expected); - } - } - - #[test] - fn roundtrip() { - let mut rng = XorShiftRng::seed_from_u64(1); - - for _i in 0..1000 { - let len: usize = rng.gen_range(0..1000); - let bits: Vec<_> = (0..len).filter(|_| rng.gen::()).collect(); - - let ranges: Vec<_> = ranges_from_bits(bits.clone()).collect(); - let bf = BitField::from_ranges(ranges_from_bits(bits)); - - assert_eq!(bf.ranges().collect::>(), ranges); - } - } - - // auditor says this: - // I used the ntest package to add a 60 sec timeout - // to the test, as it would otherwise not complete. - // may consider doing this eventually - //#[timeout(60000)] - #[test] - fn iter_last() { - // Create RLE with 2**64-2 set bits- tests timeout on the `let max` line with last - let rle: Vec = vec![ - 0xE4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2F, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, - ]; - let max = BitField::from_bytes(&rle).unwrap().last().unwrap(); - assert_eq!(max, 18446744073709551614); - } - - #[test] - fn test_unset_last() { - // Create a bitfield first 3 set bits - let ranges: Vec = vec![0, 1, 2, 3]; - let iter = ranges_from_bits(ranges); - let mut bf = BitField::from_ranges(iter); - // Unset bit at pos 3 - bf.unset(3); - - let last = bf.last().unwrap(); - assert_eq!(2, last); - } - - #[test] - fn test_zero_last() { - let mut bf = BitField::new(); - bf.set(0); - - let last = bf.last().unwrap(); - assert_eq!(0, last); - } -} diff --git a/utils/bitfield/src/rleplus/reader.rs b/utils/bitfield/src/rleplus/reader.rs deleted file mode 100644 index bcd68f75edab..000000000000 --- a/utils/bitfield/src/rleplus/reader.rs +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -use super::Result; - -// https://github.com/multiformats/unsigned-varint#practical-maximum-of-9-bytes-for-security -const VARINT_MAX_BYTES: usize = 9; - -/// A `BitReader` allows for efficiently reading bits from a byte buffer, up to a byte at a time. -/// -/// It works by always storing at least the next 8 bits in `bits`, which lets us conveniently -/// and efficiently read bits that cross a byte boundary. It's filled with the bits from `next_byte` -/// after every read operation, which is in turn replaced by the next byte from `bytes` as soon -/// as the next read might read bits from `next_byte`. -pub struct BitReader<'a> { - /// The bytes that have not been read from yet. - bytes: &'a [u8], - /// The next byte from `bytes` to be added to `bits`. - next_byte: u8, - /// The next bits to be read. - bits: u16, - /// The number of bits in `bits` from bytes that came before `next_byte` (at least 8, at most 15). - num_bits: u32, -} - -impl<'a> BitReader<'a> { - /// Creates a new `BitReader`. - pub fn new(bytes: &'a [u8]) -> Self { - let &byte1 = bytes.first().unwrap_or(&0); - let &byte2 = bytes.get(1).unwrap_or(&0); - let bytes = if bytes.len() > 2 { &bytes[2..] } else { &[] }; - - Self { - bytes, - bits: byte1 as u16, - next_byte: byte2, - num_bits: 8, - } - } - - /// Reads a given number of bits from the buffer. Will keep returning 0 once - /// the buffer has been exhausted. - pub fn read(&mut self, num_bits: u32) -> u8 { - debug_assert!(num_bits <= 8); - - // creates a mask with a `num_bits` number of 1s in order - // to get only the bits we need from `self.bits` - let mask = (1 << num_bits) - 1; - let res = (self.bits & mask) as u8; - - // removes the bits we've just read from local storage - // because we don't need them anymore - self.bits >>= num_bits; - self.num_bits -= num_bits; - - // this unconditionally adds the next byte to `bits`, - // regardless of whether there's enough space or not. the - // point is to make sure that `bits` always contains - // at least the next 8 bits to be read - self.bits |= (self.next_byte as u16) << self.num_bits; - - // if fewer than 8 bits remain, we increment `self.num_bits` - // to include the bits from `next_byte` (which is already - // contained in `bits`) and we update `next_byte` with the - // data to be read after that - if self.num_bits < 8 { - self.num_bits += 8; - - let (&next_byte, bytes) = self.bytes.split_first().unwrap_or((&0, &[])); - self.next_byte = next_byte; - self.bytes = bytes; - } - - res - } - - /// Reads a varint from the buffer. Returns an error if the - /// current position on the buffer contains no valid varint. - fn read_varint(&mut self) -> Result { - let mut len = 0; - - for i in 0..VARINT_MAX_BYTES { - let byte = self.read(8); - - // strip off the most significant bit and add - // it to the output - len |= (byte as usize & 0x7f) << (i * 7); - - // if the most significant bit is a 0, we've - // reached the end of the varint - if byte & 0x80 == 0 { - if i == 0 || byte != 0 { - // only the first byte can be zero in a varint - return Ok(len); - } - // not minimally encoded - break; - } - } - - Err("Invalid varint") - } - - /// Reads a length from the buffer according to RLE+ encoding. - pub fn read_len(&mut self) -> Result> { - let prefix_0 = self.read(1); - - let len = if prefix_0 == 1 { - // Block Single (prefix 1) - 1 - } else { - let prefix_1 = self.read(1); - - if prefix_1 == 1 { - // Block Short (prefix 01) - self.read(4) as usize - } else { - // Block Long (prefix 00) - self.read_varint()? - } - }; - - // decoding ends when a length of 0 is encountered, regardless of - // whether it is a short block or a long block - Ok(if len > 0 { Some(len) } else { None }) - } -} - -#[cfg(test)] -mod tests { - use super::BitReader; - - #[test] - fn read() { - let bytes = &[0b1011_1110, 0b0111_0010, 0b0010_1010]; - let mut reader = BitReader::new(bytes); - - assert_eq!(reader.read(0), 0); - assert_eq!(reader.read(1), 0); - assert_eq!(reader.read(3), 0b111); - assert_eq!(reader.read(6), 0b101011); - assert_eq!(reader.read(1), 0); - assert_eq!(reader.read(4), 0b1110); - assert_eq!(reader.read(3), 0b100); - assert_eq!(reader.read(2), 0b10); - assert_eq!(reader.read(3), 0b010); - assert_eq!(reader.read(4), 0); - assert_eq!(reader.read(8), 0); - assert_eq!(reader.read(0), 0); - } - - #[test] - fn read_len() { - let bytes = &[0b0001_0101, 0b1101_0111, 0b0110_0111, 0b00110010]; - let mut reader = BitReader::new(bytes); - - assert_eq!(reader.read_len().unwrap(), Some(1)); // prefix: 1 - assert_eq!(reader.read_len().unwrap(), Some(2)); // prefix: 01, value: 0100 (LSB to MSB) - assert_eq!(reader.read_len().unwrap(), Some(11)); // prefix: 01, value: 1101 - assert_eq!(reader.read_len().unwrap(), Some(15)); // prefix: 01, value: 1111 - assert_eq!(reader.read_len().unwrap(), Some(147)); // prefix: 00, value: 11001001 10000000 - assert_eq!(reader.read_len().unwrap(), None); - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "assertion failed")] - fn too_many_bits_at_once() { - let mut reader = BitReader::new(&[]); - reader.read(16); - } - - #[test] - fn roundtrip() { - use super::super::BitWriter; - use rand::{Rng, SeedableRng}; - use rand_xorshift::XorShiftRng; - - let mut rng = XorShiftRng::seed_from_u64(5); - - for _ in 0..100 { - let lengths: Vec<_> = std::iter::repeat_with(|| rng.gen_range(1..200)) - .take(100) - .collect(); - - let mut writer = BitWriter::new(); - - for &len in &lengths { - writer.write_len(len); - } - - let bytes = writer.finish(); - let mut reader = BitReader::new(&bytes); - - for &len in &lengths { - assert_eq!(reader.read_len().unwrap(), Some(len)); - } - - assert_eq!(reader.read_len().unwrap(), None); - } - } -} diff --git a/utils/bitfield/src/rleplus/writer.rs b/utils/bitfield/src/rleplus/writer.rs deleted file mode 100644 index 7c2784ba857d..000000000000 --- a/utils/bitfield/src/rleplus/writer.rs +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -/// The maximum varint that can be serialized on 9 bytes. -const MAX_VARINT: u64 = 2u64.pow(63) - 1; - -#[derive(Default, Clone, Debug)] -/// A `BitWriter` allows for efficiently writing bits to a byte buffer, up to a byte at a time. -pub struct BitWriter { - /// The buffer that is written to. - bytes: Vec, - /// The most recently written bits. Whenever this exceeds 8 bits, one byte is written to `bytes`. - bits: u16, - /// The number of bits currently stored in `bits`. - num_bits: u32, -} - -impl BitWriter { - /// Creates a new `BitWriter`. - pub fn new() -> Self { - Default::default() - } - - /// Writes a given number of bits from `byte` to the buffer. - pub fn write(&mut self, byte: u8, num_bits: u32) { - debug_assert!(num_bits <= 8); - debug_assert!(8 - byte.leading_zeros() <= num_bits); - - self.bits |= (byte as u16) << self.num_bits; - self.num_bits += num_bits; - - // when we have a full byte in `self.bits`, we write it to `self.bytes` - if self.num_bits >= 8 { - self.bytes.push(self.bits as u8); - self.bits >>= 8; - self.num_bits -= 8; - } - } - - /// Writes a given length to the buffer according to RLE+ encoding. - pub fn write_len(&mut self, len: usize) { - debug_assert!(len > 0); - debug_assert!(len <= (MAX_VARINT as usize)); - - if len == 1 { - // Block Single (prefix 1) - self.write(1, 1); - } else if len < 16 { - // Block Short (prefix 01) - self.write(2, 2); // 2 == 01 with the least significant bit first - self.write(len as u8, 4); - } else { - // Block Long (prefix 00) - self.write(0, 2); - - let mut buffer = unsigned_varint::encode::usize_buffer(); - for &byte in unsigned_varint::encode::usize(len, &mut buffer) { - self.write(byte, 8); - } - } - } - - /// Writes any remaining bits to the buffer and returns it. - pub fn finish(mut self) -> Vec { - if self.bits > 0 { - self.bytes.push(self.bits as u8); - } - - // This check should not be necessary, but as a sanity check to make sure 0 bytes - // aren't added at the end of the bytes - while let Some(0) = self.bytes.last() { - self.bytes.pop(); - } - self.bytes - } - - /// Writes any remaining bits to the buffer and returns it. - /// We write remaining bits even if they are are 0s. - /// This method is for testing purpose only. - #[cfg(test)] - pub fn finish_test(mut self) -> Vec { - if self.num_bits > 0 { - self.bytes.push(self.bits as u8); - } - - self.bytes - } -} - -#[cfg(test)] -mod tests { - use super::{BitWriter, MAX_VARINT}; - - #[test] - fn write() { - let mut writer = BitWriter::new(); - let empty_vec: Vec = Vec::new(); - assert_eq!(writer.clone().finish(), empty_vec); - - // Trailing 0 bits are not included - writer.write(0b0000_0000, 4); - assert_eq!(writer.clone().finish(), &[] as &[u8]); - - writer.write(0b0000_0000, 4); - assert_eq!(writer.clone().finish(), &[] as &[u8]); - - writer.write(0b0000_0001, 4); - assert_eq!(writer.clone().finish(), &[0b0000_0000, 0b0000_0001]); - // ^^^^ - - writer.write(0b0000_0011, 2); - assert_eq!(writer.clone().finish(), &[0b0000_0000, 0b0011_0001]); - // ^^ - - writer.write(0b0000_0110, 3); - assert_eq!( - writer.clone().finish(), - &[0b0000_0000, 0b1011_0001, 0b0000_0001] - ); // ^^ ^ - - writer.write(0b0111_0100, 8); - assert_eq!(writer.finish(), &[0b0000_0000, 0b1011_0001, 0b1110_1001]); - // ^^^^ ^^^ - } - - #[test] - fn write_len() { - let mut writer = BitWriter::new(); - - writer.write_len(1); // prefix: 1 - assert_eq!(writer.clone().finish(), &[0b0000_0001]); - // ^ - - writer.write_len(2); // prefix: 01, value: 0100 (LSB to MSB) - assert_eq!(writer.clone().finish(), &[0b0001_0101]); - // ^^^ ^^^ - - writer.write_len(11); // prefix: 01, value: 1101 - assert_eq!(writer.clone().finish(), &[0b0001_0101, 0b0001_0111]); - // ^ ^ ^^^^ - - writer.write_len(15); // prefix: 01, value: 1111 - assert_eq!( - writer.clone().finish(), - &[0b0001_0101, 0b1101_0111, 0b0000_0111] - ); // ^^^ ^^^ - - writer.write_len(147); // prefix: 00, value: 11001001 10000000 - assert_eq!( - writer.finish(), - &[ - 0b0001_0101, - 0b1101_0111, - 0b0110_0111, - //^^^^ ^ - 0b0011_0010, - //^^^^ ^^^^ - ] - ); - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "assertion failed")] - fn zero_len() { - let mut writer = BitWriter::new(); - writer.write_len(0); - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "assertion failed")] - fn more_bits_than_indicated() { - let mut writer = BitWriter::new(); - writer.write(100, 0); - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "assertion failed")] - fn too_many_bits_at_once() { - let mut writer = BitWriter::new(); - writer.write(0, 16); - } - - #[test] - fn test_write_max_varint() { - let mut writer = BitWriter::new(); - writer.write(0b_0000_0100, 3); - writer.write_len(MAX_VARINT as usize); - let rle = writer.finish(); - - crate::BitField::from_bytes(&rle).unwrap(); - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "assertion failed")] - fn test_write_succ_max_varint() { - let mut writer = BitWriter::new(); - writer.write(0b_0000_0100, 3); - writer.write_len(MAX_VARINT as usize + 1); - writer.finish(); - } -} diff --git a/utils/bitfield/src/unvalidated.rs b/utils/bitfield/src/unvalidated.rs deleted file mode 100644 index 5358e1737ee4..000000000000 --- a/utils/bitfield/src/unvalidated.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2019-2022 ChainSafe Systems -// SPDX-License-Identifier: Apache-2.0, MIT - -use super::rleplus::VERSION; -use super::{BitField, Result, MAX_ENCODED_SIZE}; -use encoding::serde_bytes; -use serde::{Deserialize, Deserializer, Serialize}; - -/// A trait for types that can produce a `&BitField` (or fail to do so). -/// Generalizes over `&BitField` and `&mut UnvalidatedBitField`. -pub trait Validate<'a> { - fn validate(self) -> Result<&'a BitField>; -} - -impl<'a> Validate<'a> for &'a mut UnvalidatedBitField { - /// Validates the RLE+ encoding of the bit field, returning a shared - /// reference to the decoded bit field. - fn validate(self) -> Result<&'a BitField> { - self.validate_mut().map(|bf| &*bf) - } -} - -impl<'a> Validate<'a> for &'a BitField { - fn validate(self) -> Result<&'a BitField> { - Ok(self) - } -} - -/// A bit field that may not yet have been validated for valid RLE+. -/// Used to defer this validation step until when the bit field is -/// first used, rather than at deserialization. -#[derive(Debug, Serialize)] -#[serde(untagged)] -pub enum UnvalidatedBitField { - Validated(BitField), - Unvalidated(#[serde(with = "serde_bytes")] Vec), -} - -impl UnvalidatedBitField { - /// Validates the RLE+ encoding of the bit field, returning a unique - /// reference to the decoded bit field. - pub fn validate_mut(&mut self) -> Result<&mut BitField> { - if let Self::Unvalidated(bytes) = self { - *self = Self::Validated(BitField::from_bytes(bytes)?); - } - - match self { - Self::Validated(bf) => Ok(bf), - Self::Unvalidated(_) => unreachable!(), - } - } -} - -impl From for UnvalidatedBitField { - fn from(bf: BitField) -> Self { - Self::Validated(bf) - } -} - -impl<'de> Deserialize<'de> for UnvalidatedBitField { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let bytes: Vec = serde_bytes::deserialize(deserializer)?; - if bytes.len() > MAX_ENCODED_SIZE { - return Err(serde::de::Error::custom(format!( - "decoded bitfield was too large {}", - bytes.len() - ))); - } - if !bytes.is_empty() && bytes[0] & 3 != VERSION { - return Err(serde::de::Error::custom("invalid RLE+ version".to_string())); - } - Ok(Self::Unvalidated(bytes)) - } -} diff --git a/vm/actor_interface/Cargo.toml b/vm/actor_interface/Cargo.toml index 48d112b35163..71d27466df7c 100644 --- a/vm/actor_interface/Cargo.toml +++ b/vm/actor_interface/Cargo.toml @@ -18,6 +18,7 @@ fil_actors_runtime_v7 = { package = "fil_actors_runtime", version = "=7.5.1" } anyhow = "1.0" fvm_shared = { version = "0.7.1", default-features = false } +fvm_ipld_bitfield = "0.5.2" fil_types = "0.2" vm = { package = "forest_vm", version = "0.3.1" } ipld_blockstore = "0.1" @@ -26,7 +27,6 @@ serde = { version = "1.0", features = ["derive"] } cid = { package = "forest_cid", version = "0.3", features = ["json"] } encoding = { package = "forest_encoding", version = "0.2" } libp2p = { version = "0.40.0-rc.1", default-features = false } -forest_bitfield = "0.1" num-bigint = { version = "0.1", package = "forest_bigint", features = ["json"] } forest_hash_utils = "0.1" forest_json_utils = "0.1" diff --git a/vm/actor_interface/src/builtin/miner/mod.rs b/vm/actor_interface/src/builtin/miner/mod.rs index 4475d1049adb..fec3ddf896a0 100644 --- a/vm/actor_interface/src/builtin/miner/mod.rs +++ b/vm/actor_interface/src/builtin/miner/mod.rs @@ -8,8 +8,8 @@ use encoding::BytesDe; use fil_types::{ deadlines::DeadlineInfo, RegisteredPoStProof, RegisteredSealProof, SectorNumber, SectorSize, }; -use forest_bitfield::BitField; use forest_json_utils::go_vec_visitor; +use fvm_ipld_bitfield::BitField; use fvm_shared::bigint::BigInt; use fvm_shared::clock::ChainEpoch; use ipld_blockstore::BlockStore; @@ -117,7 +117,7 @@ impl State { let fvm_store = ipld_blockstore::FvmRefStore::new(store); if let Some(sectors) = sectors { Ok(st - .load_sector_infos(&fvm_store, §ors.clone().into())? + .load_sector_infos(&fvm_store, sectors)? .into_iter() .map(From::from) .collect()) @@ -282,12 +282,12 @@ impl Partition<'_> { } pub fn live_sectors(&self) -> BitField { match self { - Partition::V7(dl) => dl.live_sectors().into(), + Partition::V7(dl) => dl.live_sectors(), } } pub fn active_sectors(&self) -> BitField { match self { - Partition::V7(dl) => dl.active_sectors().into(), + Partition::V7(dl) => dl.active_sectors(), } } } diff --git a/vm/state_migration/Cargo.toml b/vm/state_migration/Cargo.toml index cf02a948d1ca..c2b8a9d28e92 100644 --- a/vm/state_migration/Cargo.toml +++ b/vm/state_migration/Cargo.toml @@ -13,7 +13,7 @@ serde = { version = "1.0", features = ["derive"] } cid = { package = "forest_cid", version = "0.3", features = ["json"] } encoding = { package = "forest_encoding", version = "0.2" } libp2p = { version = "0.40.0-rc.1", default-features = false } -forest_bitfield = "0.1" +fvm_ipld_bitfield = "0.5.2" num-bigint = { version = "0.1", package = "forest_bigint", features = ["json"] } forest_hash_utils = "0.1" forest_json_utils = "0.1"