schemas: PoV compatible changes (#1743)

# Goal
The goal of this PR is to split schemas and the model into 2 separate
storages so that we can limit the size of PoV being accessed from other
pallets.

Closes #1742 


# Checklist
- [x] Chain spec updated
- [x] Design doc(s) updated
- [x] Tests added
- [x] Benchmarks added
- [x] Weights updated

---------

Co-authored-by: Frequency CI [bot] <[email protected]>

common/primitives/src/schema.rs

@@ -73,10 +73,27 @@ pub struct SchemaResponse {
 	pub settings: Vec<SchemaSetting>,
 }
 
+/// RPC Response form for a Schema Info
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+#[derive(Clone, Encode, Decode, PartialEq, Debug, TypeInfo, Eq)]
+pub struct SchemaInfoResponse {
+	/// The unique identifier for this Schema
+	pub schema_id: SchemaId,
+	/// The model format type for how the schema model is represented
+	pub model_type: ModelType,
+	/// The payload location
+	pub payload_location: PayloadLocation,
+	/// grants for the schema
+	pub settings: Vec<SchemaSetting>,
+}
+
 /// This allows other pallets to resolve Schema information. With generic SchemaId
 pub trait SchemaProvider<SchemaId> {
 	/// Gets the Schema details associated with this `SchemaId` if any
 	fn get_schema_by_id(schema_id: SchemaId) -> Option<SchemaResponse>;
+
+	/// Gets the Schema Info associated with this `SchemaId` if any
+	fn get_schema_info_by_id(schema_id: SchemaId) -> Option<SchemaInfoResponse>;
 }
 
 /// This allows other Pallets to check validity of schema ids.

designdocs/schema_v2.md

@@ -0,0 +1,46 @@
+# On Chain Message Storage
+
+## Context and Scope
+The proposed feature consists of changes that is going to be one (or more) pallet(s) in runtime of a
+Substrate based blockchain, and it will be used in all environments including production.
+
+## Problem Statement
+After introduction of **Proof of Validity** or **PoV** in runtime weights, all pallets should be
+re-evaluated and refactored if necessary to minimize the usage of **PoV**. This is to ensure all
+important operations are scalable.
+This document tries to propose some changes on **Schemas** pallet to optimize the **PoV** size.
+
+## Goals
+- Minimizing Weights including **execution times** and **PoV** size.
+
+## Proposal
+Split Schemas into `SchemaInfo` and `payload` would allow lower **PoV** when verifying schema existence
+or compatibility.
+
+### Main Storage types
+- **SchemaInfos**
+    - _Type_: `StorageMap<SchemaId, SchemaInfo>`
+    - _Purpose_: Main structure To store related properties of any schema
+      index
+- **SchemaPayloads**
+    -  _Type_: `StorageMap<SchemaId, BoundedVec<u8>>`
+    - _Purpose_: Stores the payload or model for each schema
+
+
+### On Chain Structure
+Following is a proposed data structure for storing schema information on chain.
+```rust
+pub struct SchemaInfo {
+    /// The type of model (AvroBinary, Parquet, etc.)
+    pub model_type: ModelType,
+    /// The payload location
+    pub payload_location: PayloadLocation,
+    /// additional control settings for the schema
+    pub settings: SchemaSettings,
+}
+```
+### Expected PoV improvements
+This PoV improvement would not affect extrinsic weights in this pallet, but it would directly affect any
+pallet that is dependent on **Schemas** pallet. Some of these pallets are **Messages** and
+**Stateful-Storage**. After these changes we are expecting see to see around 30-60KiB decrease in PoV
+for `MaxEncodedLen` mode.

pallets/messages/src/lib.rs

@@ -234,7 +234,7 @@ pub mod pallet {
 				.try_into()
 				.map_err(|_| Error::<T>::ExceedsMaxMessagePayloadSizeBytes)?;
 
-			if let Some(schema) = T::SchemaProvider::get_schema_by_id(schema_id) {
+			if let Some(schema) = T::SchemaProvider::get_schema_info_by_id(schema_id) {
 				ensure!(
 					schema.payload_location == PayloadLocation::IPFS,
 					Error::<T>::InvalidPayloadLocation
@@ -283,7 +283,7 @@ pub mod pallet {
 			let bounded_payload: BoundedVec<u8, T::MessagesMaxPayloadSizeBytes> =
 				payload.try_into().map_err(|_| Error::<T>::ExceedsMaxMessagePayloadSizeBytes)?;
 
-			if let Some(schema) = T::SchemaProvider::get_schema_by_id(schema_id) {
+			if let Some(schema) = T::SchemaProvider::get_schema_info_by_id(schema_id) {
 				ensure!(
 					schema.payload_location == PayloadLocation::OnChain,
 					Error::<T>::InvalidPayloadLocation

pallets/messages/src/tests/mock.rs

@@ -213,6 +213,17 @@ impl SchemaProvider<u16> for SchemaHandler {
 			settings: Vec::new(),
 		})
 	}
+
+	fn get_schema_info_by_id(schema_id: u16) -> Option<SchemaInfoResponse> {
+		Self::get_schema_by_id(schema_id).and_then(|schema| {
+			Some(SchemaInfoResponse {
+				schema_id: schema.schema_id,
+				settings: schema.settings,
+				model_type: schema.model_type,
+				payload_location: schema.payload_location,
+			})
+		})
+	}
 }
 
 impl pallet_messages::Config for Test {

pallets/messages/src/weights.rs

@@ -18,9 +18,9 @@
 //! Autogenerated weights for pallet_messages
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2023-10-30, STEPS: `20`, REPEAT: `10`, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2023-11-10, STEPS: `20`, REPEAT: `10`, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! WORST CASE MAP SIZE: `1000000`
-//! HOSTNAME: `benchmark-runner-44wtw-bw25f`, CPU: `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz`
+//! HOSTNAME: `benchmark-runner-44wtw-d4nrm`, CPU: `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz`
 //! EXECUTION: , WASM-EXECUTION: Compiled, CHAIN: Some("frequency-bench"), DB CACHE: 1024
 
 // Executed Command:
@@ -56,78 +56,78 @@ pub trait WeightInfo {
 /// Weights for pallet_messages using the Substrate node and recommended hardware.
 pub struct SubstrateWeight<T>(PhantomData<T>);
 impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
-	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
+	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::DelegatorAndProviderToDelegation` (r:1 w:0)
-	/// Proof: `Msa::DelegatorAndProviderToDelegation` (`max_values`: None, `max_size`: Some(217), added: 2692, mode: `Measured`)
-	/// Storage: `Messages::Messages` (r:1 w:1)
-	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
+	/// Proof: `Msa::DelegatorAndProviderToDelegation` (`max_values`: None, `max_size`: Some(217), added: 2692, mode: `MaxEncodedLen`)
+	/// Storage: `Messages::MessagesV2` (r:0 w:1)
+	/// Proof: `Messages::MessagesV2` (`max_values`: None, `max_size`: Some(3123), added: 5598, mode: `MaxEncodedLen`)
 	/// The range of component `n` is `[0, 3071]`.
 	fn add_onchain_message(n: u32, ) -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `9664`
-		//  Estimated: `22039`
-		// Minimum execution time: 47_640_000 picoseconds.
-		Weight::from_parts(49_189_621, 22039)
-			// Standard Error: 82
-			.saturating_add(Weight::from_parts(938, 0).saturating_mul(n.into()))
-			.saturating_add(T::DbWeight::get().reads(4_u64))
+		//  Measured:  `402`
+		//  Estimated: `12592`
+		// Minimum execution time: 32_164_000 picoseconds.
+		Weight::from_parts(33_345_645, 12592)
+			// Standard Error: 43
+			.saturating_add(Weight::from_parts(848, 0).saturating_mul(n.into()))
+			.saturating_add(T::DbWeight::get().reads(3_u64))
 			.saturating_add(T::DbWeight::get().writes(1_u64))
 	}
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
-	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
-	/// Storage: `Messages::Messages` (r:1 w:1)
-	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
+	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`)
+	/// Storage: `Messages::MessagesV2` (r:0 w:1)
+	/// Proof: `Messages::MessagesV2` (`max_values`: None, `max_size`: Some(3123), added: 5598, mode: `MaxEncodedLen`)
 	fn add_ipfs_message() -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `7958`
-		//  Estimated: `20333`
-		// Minimum execution time: 43_681_000 picoseconds.
-		Weight::from_parts(45_218_000, 20333)
-			.saturating_add(T::DbWeight::get().reads(3_u64))
+		//  Measured:  `790`
+		//  Estimated: `12423`
+		// Minimum execution time: 31_839_000 picoseconds.
+		Weight::from_parts(32_576_000, 12423)
+			.saturating_add(T::DbWeight::get().reads(2_u64))
 			.saturating_add(T::DbWeight::get().writes(1_u64))
 	}
 }
 
 // For backwards compatibility and tests
 impl WeightInfo for () {
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
-	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
+	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::DelegatorAndProviderToDelegation` (r:1 w:0)
-	/// Proof: `Msa::DelegatorAndProviderToDelegation` (`max_values`: None, `max_size`: Some(217), added: 2692, mode: `Measured`)
-	/// Storage: `Messages::Messages` (r:1 w:1)
-	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
+	/// Proof: `Msa::DelegatorAndProviderToDelegation` (`max_values`: None, `max_size`: Some(217), added: 2692, mode: `MaxEncodedLen`)
+	/// Storage: `Messages::MessagesV2` (r:0 w:1)
+	/// Proof: `Messages::MessagesV2` (`max_values`: None, `max_size`: Some(3123), added: 5598, mode: `MaxEncodedLen`)
 	/// The range of component `n` is `[0, 3071]`.
 	fn add_onchain_message(n: u32, ) -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `9664`
-		//  Estimated: `22039`
-		// Minimum execution time: 47_640_000 picoseconds.
-		Weight::from_parts(49_189_621, 22039)
-			// Standard Error: 82
-			.saturating_add(Weight::from_parts(938, 0).saturating_mul(n.into()))
-			.saturating_add(RocksDbWeight::get().reads(4_u64))
+		//  Measured:  `402`
+		//  Estimated: `12592`
+		// Minimum execution time: 32_164_000 picoseconds.
+		Weight::from_parts(33_345_645, 12592)
+			// Standard Error: 43
+			.saturating_add(Weight::from_parts(848, 0).saturating_mul(n.into()))
+			.saturating_add(RocksDbWeight::get().reads(3_u64))
 			.saturating_add(RocksDbWeight::get().writes(1_u64))
 	}
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
-	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
-	/// Storage: `Messages::Messages` (r:1 w:1)
-	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
+	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`)
+	/// Storage: `Messages::MessagesV2` (r:0 w:1)
+	/// Proof: `Messages::MessagesV2` (`max_values`: None, `max_size`: Some(3123), added: 5598, mode: `MaxEncodedLen`)
 	fn add_ipfs_message() -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `7958`
-		//  Estimated: `20333`
-		// Minimum execution time: 43_681_000 picoseconds.
-		Weight::from_parts(45_218_000, 20333)
-			.saturating_add(RocksDbWeight::get().reads(3_u64))
+		//  Measured:  `790`
+		//  Estimated: `12423`
+		// Minimum execution time: 31_839_000 picoseconds.
+		Weight::from_parts(32_576_000, 12423)
+			.saturating_add(RocksDbWeight::get().reads(2_u64))
 			.saturating_add(RocksDbWeight::get().writes(1_u64))
 	}
 }

pallets/schemas/Cargo.toml

@@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [
   "derive",
 ] }
+log = { version = "0.4.17", default-features = false }
 frame-benchmarking = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", optional = true, branch = "release-polkadot-v1.1.0" }
 frame-support = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", branch = "release-polkadot-v1.1.0" }
 frame-system = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", branch = "release-polkadot-v1.1.0" }

pallets/schemas/src/benchmarking.rs

@@ -42,7 +42,7 @@ benchmarks! {
 	}: _(RawOrigin::Signed(sender), schema_input, model_type, payload_location)
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	create_schema_via_governance {
@@ -55,7 +55,7 @@ benchmarks! {
 	}: _(RawOrigin::Root, sender.clone(), schema_input, model_type, payload_location, BoundedVec::default())
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	propose_to_create_schema {
@@ -80,7 +80,7 @@ benchmarks! {
 	}: _(RawOrigin::Signed(sender), schema_input, model_type, payload_location, BoundedVec::default())
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	set_max_schema_model_bytes {