Metadata V15: Expose pallet documentation (paritytech#13452)

* frame/proc: Helpers to parse pallet documentation attributes

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/proc: Expand pallet with runtime metadata documentation

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/dispatch: Implement doc function getter for dispatch

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Check exposed runtime metadata documentation

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/tests: Add UI tests for `pallet_doc` attribute

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/proc: Document pallet_doc attribute

Signed-off-by: Alexandru Vasile <[email protected]>

* frame/support: Use `derive_syn_parse`

Signed-off-by: Alexandru Vasile <[email protected]>

* Update frame/support/procedural/src/lib.rs

Co-authored-by: Niklas Adolfsson <[email protected]>

* frame/support: Improve documentation

Signed-off-by: Alexandru Vasile <[email protected]>

---------

Signed-off-by: Alexandru Vasile <[email protected]>
Co-authored-by: parity-processbot <>
Co-authored-by: Niklas Adolfsson <[email protected]>

frame/support/procedural/src/lib.rs

@@ -481,6 +481,57 @@ pub fn construct_runtime(input: TokenStream) -> TokenStream {
 /// </pre></div>
 ///
 /// See `frame_support::pallet` docs for more info.
+///
+/// ## Runtime Metadata Documentation
+///
+/// The documentation added to this pallet is included in the runtime metadata.
+///
+/// The documentation can be defined in the following ways:
+///
+/// ```ignore
+/// #[pallet::pallet]
+/// /// Documentation for pallet 1
+/// #[doc = "Documentation for pallet 2"]
+/// #[doc = include_str!("../README.md")]
+/// #[pallet_doc("../doc1.md")]
+/// #[pallet_doc("../doc2.md")]
+/// pub struct Pallet<T>(_);
+/// ```
+///
+/// The runtime metadata for this pallet contains the following
+///  - " Documentation for pallet 1" (captured from `///`)
+///  - "Documentation for pallet 2"  (captured from `#[doc]`)
+///  - content of ../README.md       (captured from `#[doc]` with `include_str!`)
+///  - content of "../doc1.md"       (captured from `pallet_doc`)
+///  - content of "../doc2.md"       (captured from `pallet_doc`)
+///
+/// ### `doc` attribute
+///
+/// The value of the `doc` attribute is included in the runtime metadata, as well as
+/// expanded on the pallet module. The previous example is expanded to:
+///
+/// ```ignore
+/// /// Documentation for pallet 1
+/// /// Documentation for pallet 2
+/// /// Content of README.md
+/// pub struct Pallet<T>(_);
+/// ```
+///
+/// If you want to specify the file from which the documentation is loaded, you can use the
+/// `include_str` macro. However, if you only want the documentation to be included in the
+/// runtime metadata, use the `pallet_doc` attribute.
+///
+/// ### `pallet_doc` attribute
+///
+/// Unlike the `doc` attribute, the documentation provided to the `pallet_doc` attribute is
+/// not inserted on the module.
+///
+/// The `pallet_doc` attribute can only be provided with one argument,
+/// which is the file path that holds the documentation to be added to the metadata.
+///
+/// This approach is beneficial when you use the `include_str` macro at the beginning of the file
+/// and want that documentation to extend to the runtime metadata, without reiterating the
+/// documentation on the module itself.
 #[proc_macro_attribute]
 pub fn pallet(attr: TokenStream, item: TokenStream) -> TokenStream {
 	pallet::pallet(attr, item)

frame/support/procedural/src/pallet/expand/documentation.rs

@@ -0,0 +1,221 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::pallet::Def;
+use derive_syn_parse::Parse;
+use proc_macro2::TokenStream;
+use quote::ToTokens;
+use syn::{
+	parse::{self, Parse, ParseStream},
+	spanned::Spanned,
+	Attribute, Lit,
+};
+
+const DOC: &'static str = "doc";
+const PALLET_DOC: &'static str = "pallet_doc";
+
+mod keywords {
+	syn::custom_keyword!(include_str);
+}
+
+/// Get the documentation file path from the `pallet_doc` attribute.
+///
+/// Supported format:
+/// `#[pallet_doc(PATH)]`: The path of the file from which the documentation is loaded
+fn parse_pallet_doc_value(attr: &Attribute) -> syn::Result<DocMetaValue> {
+	let span = attr.span();
+
+	let meta = attr.parse_meta()?;
+	let syn::Meta::List(metalist) = meta else {
+		let msg = "The `pallet_doc` attribute must receive arguments as a list. Supported format: `pallet_doc(PATH)`";
+		return Err(syn::Error::new(span, msg))
+	};
+
+	let paths: Vec<_> = metalist
+		.nested
+		.into_iter()
+		.map(|nested| {
+			let syn::NestedMeta::Lit(lit) = nested else {
+				let msg = "The `pallet_doc` received an unsupported argument. Supported format: `pallet_doc(PATH)`";
+				return Err(syn::Error::new(span, msg))
+			};
+
+			Ok(lit)
+		})
+		.collect::<syn::Result<_>>()?;
+
+	if paths.len() != 1 {
+		let msg = "The `pallet_doc` attribute must receive only one argument. Supported format: `pallet_doc(PATH)`";
+		return Err(syn::Error::new(span, msg))
+	}
+
+	Ok(DocMetaValue::Path(paths[0].clone()))
+}
+
+/// Get the value from the `doc` comment attribute:
+///
+/// Supported formats:
+/// - `#[doc = "A doc string"]`: Documentation as a string literal
+/// - `#[doc = include_str!(PATH)]`: Documentation obtained from a path
+fn parse_doc_value(attr: &Attribute) -> Option<DocMetaValue> {
+	let Some(ident) = attr.path.get_ident() else {
+		return None
+	};
+	if ident != DOC {
+		return None
+	}
+
+	let parser = |input: ParseStream| DocParser::parse(input);
+	let result = parse::Parser::parse2(parser, attr.tokens.clone()).ok()?;
+
+	if let Some(lit) = result.lit {
+		Some(DocMetaValue::Lit(lit))
+	} else if let Some(include_doc) = result.include_doc {
+		Some(DocMetaValue::Path(include_doc.lit))
+	} else {
+		None
+	}
+}
+
+/// Parse the include_str attribute.
+#[derive(Debug, Parse)]
+struct IncludeDocParser {
+	_include_str: keywords::include_str,
+	_eq_token: syn::token::Bang,
+	#[paren]
+	_paren: syn::token::Paren,
+	#[inside(_paren)]
+	lit: Lit,
+}
+
+/// Parse the doc literal.
+#[derive(Debug, Parse)]
+struct DocParser {
+	_eq_token: syn::token::Eq,
+	#[peek(Lit)]
+	lit: Option<Lit>,
+	#[parse_if(lit.is_none())]
+	include_doc: Option<IncludeDocParser>,
+}
+
+/// Supported documentation tokens.
+#[derive(Debug)]
+enum DocMetaValue {
+	/// Documentation with string literals.
+	///
+	/// `#[doc = "Lit"]`
+	Lit(Lit),
+	/// Documentation with `include_str!` macro.
+	///
+	/// The string literal represents the file `PATH`.
+	///
+	/// `#[doc = include_str!(PATH)]`
+	Path(Lit),
+}
+
+impl ToTokens for DocMetaValue {
+	fn to_tokens(&self, tokens: &mut TokenStream) {
+		match self {
+			DocMetaValue::Lit(lit) => lit.to_tokens(tokens),
+			DocMetaValue::Path(path_lit) => {
+				let decl = quote::quote!(include_str!(#path_lit));
+				tokens.extend(decl)
+			},
+		}
+	}
+}
+
+/// Extract the documentation from the given pallet definition
+/// to include in the runtime metadata.
+///
+/// Implement a `pallet_documentation_metadata` function to fetch the
+/// documentation that is included in the metadata.
+///
+/// The documentation is placed at the top of the module similar to:
+///
+/// ```ignore
+/// #[pallet]
+/// /// Documentation for pallet
+/// #[doc = "Documentation for pallet"]
+/// #[doc = include_str!("../README.md")]
+/// #[pallet_doc("../documentation1.md")]
+/// #[pallet_doc("../documentation2.md")]
+/// pub mod pallet {}
+/// ```
+///
+/// # pallet_doc
+///
+/// The `pallet_doc` attribute can only be provided with one argument,
+/// which is the file path that holds the documentation to be added to the metadata.
+///
+/// Unlike the `doc` attribute, the documentation provided to the `proc_macro` attribute is
+/// not inserted at the beginning of the module.
+pub fn expand_documentation(def: &mut Def) -> proc_macro2::TokenStream {
+	let frame_support = &def.frame_support;
+	let type_impl_gen = &def.type_impl_generics(proc_macro2::Span::call_site());
+	let type_use_gen = &def.type_use_generics(proc_macro2::Span::call_site());
+	let pallet_ident = &def.pallet_struct.pallet;
+	let where_clauses = &def.config.where_clause;
+
+	// TODO: Use [drain_filter](https://doc.rust-lang.org/std/vec/struct.Vec.html#method.drain_filter) when it is stable.
+
+	// The `pallet_doc` attributes are excluded from the generation of the pallet,
+	// but they are included in the runtime metadata.
+	let mut pallet_docs = Vec::with_capacity(def.item.attrs.len());
+	let mut index = 0;
+	while index < def.item.attrs.len() {
+		let attr = &def.item.attrs[index];
+		if let Some(ident) = attr.path.get_ident() {
+			if ident == PALLET_DOC {
+				let elem = def.item.attrs.remove(index);
+				pallet_docs.push(elem);
+				// Do not increment the index, we have just removed the
+				// element from the attributes.
+				continue
+			}
+		}
+
+		index += 1;
+	}
+
+	// Capture the `#[doc = include_str!("../README.md")]` and `#[doc = "Documentation"]`.
+	let mut docs: Vec<_> = def.item.attrs.iter().filter_map(parse_doc_value).collect();
+
+	// Capture the `#[pallet_doc("../README.md")]`.
+	let pallet_docs: Vec<_> = match pallet_docs
+		.into_iter()
+		.map(|attr| parse_pallet_doc_value(&attr))
+		.collect::<syn::Result<_>>()
+	{
+		Ok(docs) => docs,
+		Err(err) => return err.into_compile_error(),
+	};
+
+	docs.extend(pallet_docs);
+
+	quote::quote!(
+		impl<#type_impl_gen> #pallet_ident<#type_use_gen> #where_clauses{
+
+			#[doc(hidden)]
+			pub fn pallet_documentation_metadata()
+				-> #frame_support::sp_std::vec::Vec<&'static str>
+			{
+				#frame_support::sp_std::vec![ #( #docs ),* ]
+			}
+		}
+	)
+}

frame/support/procedural/src/pallet/expand/mod.rs

@@ -18,6 +18,7 @@
 mod call;
 mod config;
 mod constants;
+mod documentation;
 mod error;
 mod event;
 mod genesis_build;
@@ -52,6 +53,8 @@ pub fn merge_where_clauses(clauses: &[&Option<syn::WhereClause>]) -> Option<syn:
 /// * create some new types,
 /// * impl stuff on them.
 pub fn expand(mut def: Def) -> proc_macro2::TokenStream {
+	// Remove the `pallet_doc` attribute first.
+	let metadata_docs = documentation::expand_documentation(&mut def);
 	let constants = constants::expand_constants(&mut def);
 	let pallet_struct = pallet_struct::expand_pallet_struct(&mut def);
 	let config = config::expand_config(&mut def);
@@ -82,6 +85,7 @@ pub fn expand(mut def: Def) -> proc_macro2::TokenStream {
 	}
 
 	let new_items = quote::quote!(
+		#metadata_docs
 		#constants
 		#pallet_struct
 		#config

frame/support/src/dispatch.rs

@@ -2948,6 +2948,10 @@ macro_rules! decl_module {
 			{ $( $other_where_bounds )* }
 			$( $error_type )*
 		}
+		$crate::__impl_docs_metadata! {
+			$mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>
+			{ $( $other_where_bounds )* }
+		}
 		$crate::__impl_module_constants_metadata ! {
 			$mod_type<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?>
 			{ $( $other_where_bounds )* }
@@ -3018,6 +3022,26 @@ macro_rules! __impl_error_metadata {
 	};
 }
 
+/// Implement metadata for pallet documentation.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! __impl_docs_metadata {
+	(
+		$mod_type:ident<$trait_instance:ident: $trait_name:ident$(<I>, $instance:ident: $instantiable:path)?>
+		{ $( $other_where_bounds:tt )* }
+	) => {
+		impl<$trait_instance: $trait_name $(<I>, $instance: $instantiable)?> $mod_type<$trait_instance $(, $instance)?>
+			where $( $other_where_bounds )*
+		{
+			#[doc(hidden)]
+			#[allow(dead_code)]
+			pub fn pallet_documentation_metadata() -> $crate::sp_std::vec::Vec<&'static str> {
+				$crate::sp_std::vec![]
+			}
+		}
+	};
+}
+
 /// Implement metadata for module constants.
 #[macro_export]
 #[doc(hidden)]

frame/support/test/tests/pallet.rs

@@ -101,6 +101,10 @@ impl SomeAssociation2 for u64 {
 }
 
 #[frame_support::pallet]
+/// Pallet documentation
+// Comments should not be included in the pallet documentation
+#[pallet_doc("../../README.md")]
+#[doc = include_str!("../../README.md")]
 pub mod pallet {
 	use super::*;
 	use frame_support::pallet_prelude::*;
@@ -1589,6 +1593,14 @@ fn metadata() {
 	pretty_assertions::assert_eq!(actual_metadata.pallets, expected_metadata.pallets);
 }
 
+#[test]
+fn test_pallet_runtime_docs() {
+	let docs = crate::pallet::Pallet::<Runtime>::pallet_documentation_metadata();
+	let readme = "Support code for the runtime.\n\nLicense: Apache-2.0";
+	let expected = vec![" Pallet documentation", readme, readme];
+	assert_eq!(docs, expected);
+}
+
 #[test]
 fn test_pallet_info_access() {
 	assert_eq!(<System as frame_support::traits::PalletInfoAccess>::name(), "System");

frame/support/test/tests/pallet_ui/pallet_doc_arg_non_path.rs

@@ -0,0 +1,16 @@
+#[frame_support::pallet]
+// Must receive a string literal pointing to a path
+#[pallet_doc(X)]
+mod pallet {
+	#[pallet::config]
+	pub trait Config: frame_system::Config
+	where
+		<Self as frame_system::Config>::Index: From<u128>,
+	{
+	}
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(core::marker::PhantomData<T>);
+}
+
+fn main() {}

frame/support/test/tests/pallet_ui/pallet_doc_arg_non_path.stderr

@@ -0,0 +1,5 @@
+error: The `pallet_doc` received an unsupported argument. Supported format: `pallet_doc(PATH)`
+ --> tests/pallet_ui/pallet_doc_arg_non_path.rs:3:1
+  |
+3 | #[pallet_doc(X)]
+  | ^