Allow transaction for offchain indexing

client/api/src/backend.rs

@@ -21,12 +21,12 @@
 use std::sync::Arc;
 use std::collections::{HashMap, HashSet};
 use sp_core::ChangesTrieConfigurationRange;
-use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges};
+use sp_core::offchain::OffchainStorage;
 use sp_runtime::{generic::BlockId, Justification, Storage};
 use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor};
 use sp_state_machine::{
 	ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction,
-	StorageCollection, ChildStorageCollection,
+	StorageCollection, ChildStorageCollection, OffchainChangesCollection,
 };
 use sp_storage::{StorageData, StorageKey, PrefixedStorageKey, ChildInfo};
 use crate::{
@@ -174,7 +174,7 @@ pub trait BlockImportOperation<Block: BlockT> {
 	/// Write offchain storage changes to the database.
 	fn update_offchain_storage(
 		&mut self,
-		_offchain_update: OffchainOverlayedChanges,
+		_offchain_update: OffchainChangesCollection,
 	) -> sp_blockchain::Result<()> {
 		 Ok(())
 	}

client/api/src/call_executor.rs

@@ -28,7 +28,7 @@ use sp_state_machine::{
 };
 use sc_executor::{RuntimeVersion, NativeVersion};
 use sp_externalities::Extensions;
-use sp_core::{NativeOrEncoded,offchain::storage::OffchainOverlayedChanges};
+use sp_core::NativeOrEncoded;
 
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use crate::execution_extensions::ExecutionExtensions;
@@ -86,7 +86,6 @@ pub trait CallExecutor<B: BlockT> {
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<B, <Self::Backend as crate::backend::Backend<B>>::State>,
 		>>,

client/db/src/lib.rs

@@ -67,7 +67,7 @@ use hash_db::Prefix;
 use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
 use sp_database::Transaction;
 use sp_core::{Hasher, ChangesTrieConfiguration};
-use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges};
+use sp_core::offchain::OffchainOverlayedChange;
 use sp_core::storage::{well_known_keys, ChildInfo};
 use sp_arithmetic::traits::Saturating;
 use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage};
@@ -76,7 +76,7 @@ use sp_runtime::traits::{
 };
 use sp_state_machine::{
 	DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, UsageInfo as StateUsageInfo,
-	StorageCollection, ChildStorageCollection,
+	StorageCollection, ChildStorageCollection, OffchainChangesCollection,
 	backend::Backend as StateBackend, StateMachineStats,
 };
 use crate::utils::{DatabaseType, Meta, meta_keys, read_db, read_meta};
@@ -667,7 +667,7 @@ pub struct BlockImportOperation<Block: BlockT> {
 	db_updates: PrefixedMemoryDB<HashFor<Block>>,
 	storage_updates: StorageCollection,
 	child_storage_updates: ChildStorageCollection,
-	offchain_storage_updates: OffchainOverlayedChanges,
+	offchain_storage_updates: OffchainChangesCollection,
 	changes_trie_updates: MemoryDB<HashFor<Block>>,
 	changes_trie_build_cache_update: Option<ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>>,
 	changes_trie_config_update: Option<Option<ChangesTrieConfiguration>>,
@@ -680,7 +680,7 @@ pub struct BlockImportOperation<Block: BlockT> {
 
 impl<Block: BlockT> BlockImportOperation<Block> {
 	fn apply_offchain(&mut self, transaction: &mut Transaction<DbHash>) {
-		for ((prefix, key), value_operation) in self.offchain_storage_updates.drain() {
+		for ((prefix, key), value_operation) in self.offchain_storage_updates.drain(..) {
 			let key: Vec<u8> = prefix
 				.into_iter()
 				.chain(sp_core::sp_std::iter::once(b'/'))
@@ -800,7 +800,7 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block> for Bloc
 
 	fn update_offchain_storage(
 		&mut self,
-		offchain_update: OffchainOverlayedChanges,
+		offchain_update: OffchainChangesCollection,
 	) -> ClientResult<()> {
 		self.offchain_storage_updates = offchain_update;
 		Ok(())

client/executor/src/integration_tests/mod.rs

@@ -475,7 +475,7 @@ fn offchain_index(wasm_method: WasmExecutionMethod) {
 		&mut ext.ext(),
 	).unwrap();
 
-	use sp_core::offchain::storage::OffchainOverlayedChange;
+	use sp_core::offchain::OffchainOverlayedChange;
 	assert_eq!(
 		ext.ext()
 			.get_offchain_storage_changes()

client/light/src/call_executor.rs

@@ -25,7 +25,6 @@ use std::{
 use codec::{Encode, Decode};
 use sp_core::{
 	convert_hash, NativeOrEncoded, traits::{CodeExecutor, SpawnNamed},
-	offchain::storage::OffchainOverlayedChanges,
 };
 use sp_runtime::{
 	generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor},
@@ -113,7 +112,6 @@ impl<Block, B, Local> CallExecutor<Block> for
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		_: Option<&RefCell<StorageTransactionCache<Block, B::State>>>,
 		initialize_block: InitializeBlock<'a, Block>,
 		_manager: ExecutionManager<EM>,
@@ -140,7 +138,6 @@ impl<Block, B, Local> CallExecutor<Block> for
 				method,
 				call_data,
 				changes,
-				offchain_changes,
 				None,
 				initialize_block,
 				ExecutionManager::NativeWhenPossible,

client/service/src/client/call_executor.rs

@@ -29,7 +29,6 @@ use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use sp_externalities::Extensions;
 use sp_core::{
 	NativeOrEncoded, NeverNativeValue, traits::{CodeExecutor, SpawnNamed, RuntimeCode},
-	offchain::storage::OffchainOverlayedChanges,
 };
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use sc_client_api::{backend, call_executor::CallExecutor};
@@ -126,11 +125,10 @@ where
 		strategy: ExecutionStrategy,
 		extensions: Option<Extensions>,
 	) -> sp_blockchain::Result<Vec<u8>> {
-		let mut changes = OverlayedChanges::default();
-		let mut offchain_changes = if self.client_config.offchain_indexing_api {
-			OffchainOverlayedChanges::enabled()
+		let mut changes = if self.client_config.offchain_indexing_api {
+			OverlayedChanges::with_offchain_indexing()
 		} else {
-			OffchainOverlayedChanges::disabled()
+			OverlayedChanges::default()
 		};
 		let changes_trie = backend::changes_tries_state_at_block(
 			id, self.backend.changes_trie_storage()
@@ -145,7 +143,6 @@ where
 			&state,
 			changes_trie,
 			&mut changes,
-			&mut offchain_changes,
 			&self.executor,
 			method,
 			call_data,
@@ -176,7 +173,6 @@ where
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<Block, B::State>
 		>>,
@@ -201,7 +197,6 @@ where
 		let mut state = self.backend.state_at(*at)?;
 
 		let changes = &mut *changes.borrow_mut();
-		let offchain_changes = &mut *offchain_changes.borrow_mut();
 
 		match recorder {
 			Some(recorder) => {
@@ -227,7 +222,6 @@ where
 					&backend,
 					changes_trie_state,
 					changes,
-					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -249,7 +243,6 @@ where
 					&state,
 					changes_trie_state,
 					changes,
-					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -264,7 +257,6 @@ where
 
 	fn runtime_version(&self, id: &BlockId<Block>) -> sp_blockchain::Result<RuntimeVersion> {
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let changes_trie_state = backend::changes_tries_state_at_block(
 			id,
 			self.backend.changes_trie_storage(),
@@ -273,7 +265,6 @@ where
 		let mut cache = StorageTransactionCache::<Block, B::State>::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&state,
 			changes_trie_state,

client/service/src/client/client.rs

@@ -1654,7 +1654,6 @@ impl<B, E, Block, RA> CallApiAt<Block> for Client<B, E, Block, RA> where
 			params.function,
 			&params.arguments,
 			params.overlayed_changes,
-			params.offchain_changes,
 			Some(params.storage_transaction_cache),
 			params.initialize_block,
 			manager,

client/service/test/src/client/light.rs

@@ -36,7 +36,7 @@ use parking_lot::Mutex;
 use substrate_test_runtime_client::{
 	runtime::{Hash, Block, Header}, TestClient, ClientBlockImportExt,
 };
-use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder, OffchainOverlayedChanges};
+use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder};
 use sp_consensus::BlockOrigin;
 use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVersion};
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
@@ -223,7 +223,6 @@ impl CallExecutor<Block> for DummyCallExecutor {
 		_method: &str,
 		_call_data: &[u8],
 		_changes: &RefCell<OverlayedChanges>,
-		_offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		_storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<
 				Block,

client/service/test/src/client/mod.rs

@@ -41,7 +41,7 @@ use sp_runtime::traits::{
 };
 use substrate_test_runtime::TestAPI;
 use sp_state_machine::backend::Backend as _;
-use sp_api::{ProvideRuntimeApi, OffchainOverlayedChanges};
+use sp_api::ProvideRuntimeApi;
 use sp_core::{H256, ChangesTrieConfiguration, blake2_256, testing::TaskExecutor};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
@@ -163,7 +163,6 @@ fn construct_block(
 	};
 	let hash = header.hash();
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 	let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend);
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 	let task_executor = Box::new(TaskExecutor::new());
@@ -172,7 +171,6 @@ fn construct_block(
 		backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_initialize_block",
 		&header.encode(),
@@ -188,7 +186,6 @@ fn construct_block(
 			backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
-			&mut offchain_overlay,
 			&executor(),
 			"BlockBuilder_apply_extrinsic",
 			&tx.encode(),
@@ -204,7 +201,6 @@ fn construct_block(
 		backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"BlockBuilder_finalize_block",
 		&[],
@@ -252,13 +248,11 @@ fn construct_genesis_should_work_with_native() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let _ = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,
@@ -288,13 +282,11 @@ fn construct_genesis_should_work_with_wasm() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let _ = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,
@@ -324,13 +316,11 @@ fn construct_genesis_with_bad_transaction_should_panic() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let r = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,

primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -409,7 +409,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
-				offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 				storage_transaction_cache: &std::cell::RefCell<
 					#crate_::StorageTransactionCache<Block, T::StateBackend>
 				>,
@@ -439,7 +438,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 							native_call: None,
 							arguments: args,
 							overlayed_changes: changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialize_block,
 							context,
@@ -460,7 +458,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 					native_call,
 					arguments: args,
 					overlayed_changes: changes,
-					offchain_changes,
 					storage_transaction_cache,
 					initialize_block,
 					context,

primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -208,7 +208,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 			commit_on_success: std::cell::RefCell<bool>,
 			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 			storage_transaction_cache: std::cell::RefCell<
 				#crate_::StorageTransactionCache<Block, C::StateBackend>
 			>,
@@ -338,7 +337,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					commit_on_success: true.into(),
 					initialized_block: None.into(),
 					changes: Default::default(),
-					offchain_changes: Default::default(),
 					recorder: Default::default(),
 					storage_transaction_cache: Default::default(),
 				}.into()
@@ -357,7 +355,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&C,
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
-					&std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
 					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 					&Option<#crate_::ProofRecorder<Block>>,
@@ -374,7 +371,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&self.call,
 					self,
 					&self.changes,
-					&self.offchain_changes,
 					&self.storage_transaction_cache,
 					&self.initialized_block,
 					&self.recorder,
@@ -531,7 +527,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 							call_runtime_at,
 							core_api,
 							changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialized_block,
 							recorder
@@ -542,7 +537,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 								at,
 								params_encoded,
 								changes,
-								offchain_changes,
 								storage_transaction_cache,
 								initialized_block,
 								params.map(|p| {

primitives/api/src/lib.rs

@@ -47,8 +47,6 @@ pub use sp_core::NativeOrEncoded;
 #[doc(hidden)]
 #[cfg(feature = "std")]
 pub use hash_db::Hasher;
-#[cfg(feature = "std")]
-pub use sp_core::offchain::storage::OffchainOverlayedChanges;
 #[doc(hidden)]
 #[cfg(not(feature = "std"))]
 pub use sp_core::to_substrate_wasm_fn_return_value;
@@ -521,8 +519,6 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HashF
 	pub arguments: Vec<u8>,
 	/// The overlayed changes that are on top of the state.
 	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
-	/// The overlayed changes to be applied to the offchain worker database.
-	pub offchain_changes: &'a RefCell<OffchainOverlayedChanges>,
 	/// The cache for storage transactions.
 	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
 	/// Determines if the function requires that `initialize_block` should be called before calling