diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index 8ca713b1bbf61..6d9d81f385176 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -22,6 +22,7 @@ mod mock; +use sp_runtime::traits::{One, StaticLookup}; use sp_std::{prelude::*, vec}; use frame_benchmarking::benchmarks; @@ -30,12 +31,11 @@ use frame_support::{ traits::{KeyOwnerProofSystem, OnInitialize}, }; use frame_system::RawOrigin; -use pallet_session::{historical::Module as Historical, Pallet as Session, *}; +use pallet_session::{historical::Pallet as Historical, Pallet as Session, *}; use pallet_staking::{ benchmarking::create_validator_with_nominators, testing_utils::create_validators, RewardDestination, }; -use sp_runtime::traits::{One, StaticLookup}; const MAX_VALIDATORS: u32 = 1000; diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 0801b2aca1701..a3e64f4f9efa4 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -26,62 +26,74 @@ //! These roots and proofs of inclusion can be generated at any time during the current session. //! Afterwards, the proofs can be fed to a consensus module when reporting misbehavior. -use super::{Pallet as SessionModule, SessionIndex}; +pub mod offchain; +pub mod onchain; +mod shared; + use codec::{Decode, Encode}; -use frame_support::{ - decl_module, decl_storage, print, - traits::{ValidatorSet, ValidatorSetWithIdentification}, - Parameter, -}; use sp_runtime::{ traits::{Convert, OpaqueKeys}, KeyTypeId, }; use sp_session::{MembershipProof, ValidatorCount}; +use sp_staking::SessionIndex; use sp_std::prelude::*; use sp_trie::{ trie_types::{TrieDB, TrieDBMut}, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -pub mod offchain; -pub mod onchain; -mod shared; +use frame_support::{ + print, + traits::{KeyOwnerProofSystem, StorageVersion, ValidatorSet, ValidatorSetWithIdentification}, + Parameter, +}; -/// Config necessary for the historical module. -pub trait Config: super::Config { - /// Full identification of the validator. - type FullIdentification: Parameter; - - /// A conversion from validator ID to full identification. - /// - /// This should contain any references to economic actors associated with the - /// validator, since they may be outdated by the time this is queried from a - /// historical trie. - /// - /// It must return the identification for the current session index. - type FullIdentificationOf: Convert>; -} +use crate::{self as pallet_session, Pallet as Session}; + +pub use pallet::*; -decl_storage! { - trait Store for Module as Session { - /// Mapping from historical session indices to session-data root hash and validator count. - HistoricalSessions get(fn historical_root): - map hasher(twox_64_concat) SessionIndex => Option<(T::Hash, ValidatorCount)>; - /// The range of historical sessions we store. [first, last) - StoredRange: Option<(SessionIndex, SessionIndex)>; - /// Deprecated. - CachedObsolete: - map hasher(twox_64_concat) SessionIndex - => Option>; +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::storage_version(STORAGE_VERSION)] + pub struct Pallet(_); + + /// Config necessary for the historical pallet. + #[pallet::config] + pub trait Config: pallet_session::Config + frame_system::Config { + /// Full identification of the validator. + type FullIdentification: Parameter; + + /// A conversion from validator ID to full identification. + /// + /// This should contain any references to economic actors associated with the + /// validator, since they may be outdated by the time this is queried from a + /// historical trie. + /// + /// It must return the identification for the current session index. + type FullIdentificationOf: Convert>; } -} -decl_module! { - pub struct Module for enum Call where origin: T::Origin {} + /// Mapping from historical session indices to session-data root hash and validator count. + #[pallet::storage] + #[pallet::getter(fn historical_root)] + pub type HistoricalSessions = + StorageMap<_, Twox64Concat, SessionIndex, (T::Hash, ValidatorCount), OptionQuery>; + + /// The range of historical sessions we store. [first, last) + #[pallet::storage] + pub type StoredRange = StorageValue<_, (SessionIndex, SessionIndex), OptionQuery>; } -impl Module { +impl Pallet { /// Prune historical stored session roots up to (but not including) /// `up_to`. pub fn prune_up_to(up_to: SessionIndex) { @@ -109,7 +121,7 @@ impl Module { } } -impl ValidatorSet for Module { +impl ValidatorSet for Pallet { type ValidatorId = T::ValidatorId; type ValidatorIdOf = T::ValidatorIdOf; @@ -122,7 +134,7 @@ impl ValidatorSet for Module { } } -impl ValidatorSetWithIdentification for Module { +impl ValidatorSetWithIdentification for Pallet { type Identification = T::FullIdentification; type IdentificationOf = T::FullIdentificationOf; } @@ -130,7 +142,7 @@ impl ValidatorSetWithIdentification for Module { /// Specialization of the crate-level `SessionManager` which returns the set of full identification /// when creating a new session. pub trait SessionManager: - crate::SessionManager + pallet_session::SessionManager { /// If there was a validator set change, its returns the set of new validators along with their /// full identifications. @@ -150,7 +162,7 @@ pub struct NoteHistoricalRoot(sp_std::marker::PhantomData<(T, I)>); impl> NoteHistoricalRoot { fn do_new_session(new_index: SessionIndex, is_genesis: bool) -> Option> { - StoredRange::mutate(|range| { + >::mutate(|range| { range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1; }); @@ -183,7 +195,7 @@ impl> NoteHi } } -impl crate::SessionManager for NoteHistoricalRoot +impl pallet_session::SessionManager for NoteHistoricalRoot where I: SessionManager, { @@ -207,7 +219,7 @@ where /// A tuple of the validator's ID and their full identification. pub type IdentificationTuple = - (::ValidatorId, ::FullIdentification); + (::ValidatorId, ::FullIdentification); /// A trie instance for checking and generating proofs. pub struct ProvingTrie { @@ -227,7 +239,7 @@ impl ProvingTrie { let mut trie = TrieDBMut::new(&mut db, &mut root); for (i, (validator, full_id)) in validators.into_iter().enumerate() { let i = i as u32; - let keys = match >::load_keys(&validator) { + let keys = match >::load_keys(&validator) { None => continue, Some(k) => k, }; @@ -304,15 +316,13 @@ impl ProvingTrie { } } -impl> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)> - for Module -{ +impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet { type Proof = MembershipProof; type IdentificationTuple = IdentificationTuple; fn prove(key: (KeyTypeId, D)) -> Option { - let session = >::current_index(); - let validators = >::validators() + let session = >::current_index(); + let validators = >::validators() .into_iter() .filter_map(|validator| { T::FullIdentificationOf::convert(validator.clone()) @@ -335,10 +345,10 @@ impl> frame_support::traits::KeyOwnerProofSystem<(KeyT fn check_proof(key: (KeyTypeId, D), proof: Self::Proof) -> Option> { let (id, data) = key; - if proof.session == >::current_index() { - >::key_owner(id, data.as_ref()).and_then(|owner| { + if proof.session == >::current_index() { + >::key_owner(id, data.as_ref()).and_then(|owner| { T::FullIdentificationOf::convert(owner.clone()).and_then(move |id| { - let count = >::validators().len() as ValidatorCount; + let count = >::validators().len() as ValidatorCount; if count != proof.validator_count { return None @@ -374,7 +384,7 @@ pub(crate) mod tests { BasicExternalities, }; - type Historical = Module; + type Historical = Pallet; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); @@ -386,7 +396,9 @@ pub(crate) mod tests { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); + pallet_session::GenesisConfig:: { keys } + .assimilate_storage(&mut t) + .unwrap(); sp_io::TestExternalities::new(t) } @@ -436,27 +448,27 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((0, 100))); + assert_eq!(>::get(), Some((0, 100))); for i in 0..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(10); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(>::get(), Some((10, 100))); Historical::prune_up_to(9); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(>::get(), Some((10, 100))); for i in 10..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(99); - assert_eq!(StoredRange::get(), Some((99, 100))); + assert_eq!(>::get(), Some((99, 100))); Historical::prune_up_to(100); - assert_eq!(StoredRange::get(), None); + assert_eq!(>::get(), None); for i in 99..199u64 { set_next_validators(vec![i]); @@ -466,14 +478,14 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((100, 200))); + assert_eq!(>::get(), Some((100, 200))); for i in 100..200 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(9999); - assert_eq!(StoredRange::get(), None); + assert_eq!(>::get(), None); for i in 100..200 { assert!(Historical::historical_root(i).is_none()) diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index b646ecc2764f7..0b292b57658d0 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -140,7 +140,7 @@ pub fn keep_newest(n_to_keep: usize) { mod tests { use super::*; use crate::{ - historical::{onchain, Module}, + historical::{onchain, Pallet}, mock::{force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS}, }; @@ -156,7 +156,7 @@ mod tests { BasicExternalities, }; - type Historical = Module; + type Historical = Pallet; pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default() diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 6779285ee3187..2fd34365705bb 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -108,6 +108,7 @@ #[cfg(feature = "historical")] pub mod historical; +pub mod migrations; #[cfg(test)] mod mock; #[cfg(test)] diff --git a/frame/session/src/migrations/mod.rs b/frame/session/src/migrations/mod.rs new file mode 100644 index 0000000000000..ccc5ee3c2e525 --- /dev/null +++ b/frame/session/src/migrations/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Version 1. +/// +/// In version 0 session historical pallet uses `Session` for storage module prefix. +/// In version 1 it uses its name as configured in `construct_runtime`. +/// This migration moves session historical pallet storages from old prefix to new prefix. +#[cfg(feature = "historical")] +pub mod v1; diff --git a/frame/session/src/migrations/v1.rs b/frame/session/src/migrations/v1.rs new file mode 100644 index 0000000000000..1de199fe7bedd --- /dev/null +++ b/frame/session/src/migrations/v1.rs @@ -0,0 +1,194 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sp_io::hashing::twox_128; +use sp_std::str; + +use frame_support::{ + storage::{generator::StorageValue, StoragePrefixedMap}, + traits::{ + Get, GetStorageVersion, PalletInfoAccess, StorageVersion, + STORAGE_VERSION_STORAGE_KEY_POSTFIX, + }, + weights::Weight, +}; + +use crate::historical as pallet_session_historical; + +const OLD_PREFIX: &str = "Session"; + +/// Migrate the entire storage of this pallet to a new prefix. +/// +/// This new prefix must be the same as the one set in construct_runtime. +/// +/// The migration will look into the storage version in order not to trigger a migration on an up +/// to date storage. Thus the on chain storage version must be less than 1 in order to trigger the +/// migration. +pub fn migrate( +) -> Weight { + let new_pallet_name =

::name(); + + if new_pallet_name == OLD_PREFIX { + log::info!( + target: "runtime::session_historical", + "New pallet name is equal to the old prefix. No migration needs to be done.", + ); + return 0 + } + + let on_chain_storage_version =

::on_chain_storage_version(); + log::info!( + target: "runtime::session_historical", + "Running migration to v1 for session_historical with storage version {:?}", + on_chain_storage_version, + ); + + if on_chain_storage_version < 1 { + let storage_prefix = pallet_session_historical::HistoricalSessions::::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + let storage_prefix = pallet_session_historical::StoredRange::::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + StorageVersion::new(1).put::

(); + ::BlockWeights::get().max_block + } else { + log::warn!( + target: "runtime::session_historical", + "Attempted to apply migration to v1 but failed because storage version is {:?}", + on_chain_storage_version, + ); + 0 + } +} + +/// Some checks prior to migration. This can be linked to +/// [`frame_support::traits::OnRuntimeUpgrade::pre_upgrade`] for further testing. +/// +/// Panics if anything goes wrong. +pub fn pre_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name =

::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::::storage_prefix(); + + log_migration("pre-migration", storage_prefix_historical_sessions, OLD_PREFIX, new_pallet_name); + log_migration("pre-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let storage_version_key = twox_128(STORAGE_VERSION_STORAGE_KEY_POSTFIX); + + let mut new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |key| Ok(key.to_vec()), + ); + + // Ensure nothing except the storage_version_key is stored in the new prefix. + assert!(new_pallet_prefix_iter.all(|key| key == storage_version_key)); + + assert!(

::on_chain_storage_version() < 1); +} + +/// Some checks for after migration. This can be linked to +/// [`frame_support::traits::OnRuntimeUpgrade::post_upgrade`] for further testing. +/// +/// Panics if anything goes wrong. +pub fn post_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name =

::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::::storage_prefix(); + + log_migration( + "post-migration", + storage_prefix_historical_sessions, + OLD_PREFIX, + new_pallet_name, + ); + log_migration("post-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + // Assert that no `HistoricalSessions` and `StoredRange` storages remains at the old prefix. + let old_pallet_prefix = twox_128(OLD_PREFIX.as_bytes()); + let old_historical_sessions_key = + [&old_pallet_prefix, &twox_128(storage_prefix_historical_sessions)[..]].concat(); + let old_historical_sessions_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_historical_sessions_key.to_vec(), + old_historical_sessions_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_historical_sessions_key_iter.count(), 0); + + let old_stored_range_key = + [&old_pallet_prefix, &twox_128(storage_prefix_stored_range)[..]].concat(); + let old_stored_range_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_stored_range_key.to_vec(), + old_stored_range_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_stored_range_key_iter.count(), 0); + + // Assert that the `HistoricalSessions` and `StoredRange` storages (if they exist) have been + // moved to the new prefix. + // NOTE: storage_version_key is already in the new prefix. + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |_| Ok(()), + ); + assert!(new_pallet_prefix_iter.count() >= 1); + + assert_eq!(

::on_chain_storage_version(), 1); +} + +fn log_migration(stage: &str, storage_prefix: &[u8], old_pallet_name: &str, new_pallet_name: &str) { + log::info!( + target: "runtime::session_historical", + "{} prefix of storage '{}': '{}' ==> '{}'", + stage, + str::from_utf8(storage_prefix).unwrap_or(""), + old_pallet_name, + new_pallet_name, + ); +} diff --git a/frame/session/src/tests.rs b/frame/session/src/tests.rs index 308ed7c5e5487..cc0606edf499d 100644 --- a/frame/session/src/tests.rs +++ b/frame/session/src/tests.rs @@ -453,3 +453,30 @@ fn upgrade_keys() { } }) } + +#[cfg(feature = "historical")] +#[test] +fn test_migration_v1() { + use crate::{ + historical::{HistoricalSessions, StoredRange}, + mock::Historical, + }; + use frame_support::traits::PalletInfoAccess; + + new_test_ext().execute_with(|| { + assert!(>::iter_values().count() > 0); + assert!(>::exists()); + + let old_pallet = "Session"; + let new_pallet = ::name(); + frame_support::storage::migration::move_pallet( + new_pallet.as_bytes(), + old_pallet.as_bytes(), + ); + StorageVersion::new(0).put::(); + + crate::migrations::v1::pre_migrate::(); + crate::migrations::v1::migrate::(); + crate::migrations::v1::post_migrate::(); + }); +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 95d397359f8d6..e5a3e49033934 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -96,6 +96,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + Historical: pallet_session::historical::{Pallet, Storage}, BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } );