From ac408e41f54164ab1791bde4515cd5c97723d84d Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Mon, 3 Aug 2020 12:45:27 -0500 Subject: [PATCH 01/20] Add json backend --- src/librustdoc/clean/types.rs | 5 + src/librustdoc/json/conversions.rs | 597 +++++++++++++++++++++++++++++ src/librustdoc/json/mod.rs | 185 ++++++++- src/librustdoc/json/types.rs | 485 +++++++++++++++++++++++ 4 files changed, 1258 insertions(+), 14 deletions(-) create mode 100644 src/librustdoc/json/conversions.rs create mode 100644 src/librustdoc/json/types.rs diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 32b3f69ecd4f0..e25f076147821 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -1576,6 +1576,11 @@ impl Path { pub fn last_name(&self) -> &str { self.segments.last().expect("segments were empty").name.as_str() } + + pub fn whole_name(&self) -> String { + String::from(if self.global { "::" } else { "" }) + + &self.segments.iter().map(|s| s.name.clone()).collect::>().join("::") + } } #[derive(Clone, PartialEq, Eq, Debug, Hash)] diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs new file mode 100644 index 0000000000000..6855931def1e6 --- /dev/null +++ b/src/librustdoc/json/conversions.rs @@ -0,0 +1,597 @@ +use std::convert::From; + +use rustc_ast::ast; +use rustc_span::def_id; + +use crate::clean; +use crate::doctree; +use crate::formats::item_type::ItemType; +use crate::json::types::*; + +impl From for Item { + fn from(item: clean::Item) -> Self { + let item_type = ItemType::from(&item); + let clean::Item { + source, + name, + attrs, + inner, + visibility, + def_id, + stability: _, + deprecation, + } = item; + Item { + crate_num: def_id.krate.as_u32(), + name, + source: source.into(), + visibility: visibility.into(), + docs: attrs.collapsed_doc_value().unwrap_or_default(), + links: attrs + .links + .into_iter() + .filter_map(|(a, b, _)| b.map(|b| (a, b.into()))) + .collect(), + attrs: attrs + .other_attrs + .iter() + .map(rustc_ast_pretty::pprust::attribute_to_string) + .collect(), + deprecation: deprecation.map(Into::into), + kind: item_type.into(), + inner: inner.into(), + } + } +} + +impl From for Option { + fn from(span: clean::Span) -> Self { + let clean::Span { loline, locol, hiline, hicol, .. } = span; + match span.filename { + rustc_span::FileName::Real(name) => Some(Span { + filename: match name { + rustc_span::RealFileName::Named(path) => path, + rustc_span::RealFileName::Devirtualized { local_path, virtual_name: _ } => { + local_path + } + }, + begin: (loline, locol), + end: (hiline, hicol), + }), + _ => None, + } + } +} + +impl From for Deprecation { + fn from(deprecation: clean::Deprecation) -> Self { + Deprecation { since: deprecation.since, note: deprecation.note } + } +} + +impl From for Visibility { + fn from(v: clean::Visibility) -> Self { + use clean::Visibility::*; + match v { + Public => Visibility::Public, + Inherited => Visibility::Default, + Crate => Visibility::Crate, + Restricted(_did, _path) => unimplemented!(), + // Visibility::Restricted(did.into(), path.whole_name()), + } + } +} + +impl From for GenericArgs { + fn from(args: clean::GenericArgs) -> Self { + use clean::GenericArgs::*; + match args { + AngleBracketed { args, bindings } => GenericArgs::AngleBracketed { + args: args.into_iter().map(Into::into).collect(), + bindings: bindings.into_iter().map(Into::into).collect(), + }, + Parenthesized { inputs, output } => GenericArgs::Parenthesized { + inputs: inputs.into_iter().map(Into::into).collect(), + output: output.map(Into::into), + }, + } + } +} + +impl From for GenericArg { + fn from(arg: clean::GenericArg) -> Self { + use clean::GenericArg::*; + match arg { + Lifetime(l) => GenericArg::Lifetime(l.0), + Type(t) => GenericArg::Type(t.into()), + Const(c) => GenericArg::Const(c.into()), + } + } +} + +impl From for Constant { + fn from(constant: clean::Constant) -> Self { + let clean::Constant { type_, expr, value, is_literal } = constant; + Constant { type_: type_.into(), expr, value, is_literal } + } +} + +impl From for TypeBinding { + fn from(binding: clean::TypeBinding) -> Self { + TypeBinding { name: binding.name, binding: binding.kind.into() } + } +} + +impl From for TypeBindingKind { + fn from(kind: clean::TypeBindingKind) -> Self { + use clean::TypeBindingKind::*; + match kind { + Equality { ty } => TypeBindingKind::Equality(ty.into()), + Constraint { bounds } => { + TypeBindingKind::Constraint(bounds.into_iter().map(Into::into).collect()) + } + } + } +} + +impl From for Id { + fn from(did: def_id::DefId) -> Self { + Id(format!("{}:{}", did.krate.as_u32(), u32::from(did.index))) + } +} + +impl From for ItemEnum { + fn from(item: clean::ItemEnum) -> Self { + use clean::ItemEnum::*; + match item { + ModuleItem(m) => ItemEnum::ModuleItem(m.into()), + ExternCrateItem(c, a) => ItemEnum::ExternCrateItem { name: c, rename: a }, + ImportItem(i) => ItemEnum::ImportItem(i.into()), + StructItem(s) => ItemEnum::StructItem(s.into()), + UnionItem(u) => ItemEnum::StructItem(u.into()), + StructFieldItem(f) => ItemEnum::StructFieldItem(f.into()), + EnumItem(e) => ItemEnum::EnumItem(e.into()), + VariantItem(v) => ItemEnum::VariantItem(v.into()), + FunctionItem(f) => ItemEnum::FunctionItem(f.into()), + ForeignFunctionItem(f) => ItemEnum::FunctionItem(f.into()), + TraitItem(t) => ItemEnum::TraitItem(t.into()), + TraitAliasItem(t) => ItemEnum::TraitAliasItem(t.into()), + MethodItem(m) => ItemEnum::MethodItem(m.into()), + TyMethodItem(m) => ItemEnum::MethodItem(m.into()), + ImplItem(i) => ItemEnum::ImplItem(i.into()), + StaticItem(s) => ItemEnum::StaticItem(s.into()), + ForeignStaticItem(s) => ItemEnum::StaticItem(s.into()), + ForeignTypeItem => ItemEnum::ForeignTypeItem, + TypedefItem(t, _) => ItemEnum::TypedefItem(t.into()), + OpaqueTyItem(t, _) => ItemEnum::OpaqueTyItem(t.into()), + ConstantItem(c) => ItemEnum::ConstantItem(c.into()), + MacroItem(m) => ItemEnum::MacroItem(m.source), + ProcMacroItem(m) => ItemEnum::ProcMacroItem(m.into()), + AssocConstItem(t, s) => ItemEnum::AssocConstItem { type_: t.into(), default: s }, + AssocTypeItem(g, t) => ItemEnum::AssocTypeItem { + bounds: g.into_iter().map(Into::into).collect(), + default: t.map(Into::into), + }, + StrippedItem(inner) => ItemEnum::StrippedItem(Box::new((*inner).into())), + _ => panic!("{:?} is not supported for JSON output", item), + } + } +} + +impl From for Module { + fn from(module: clean::Module) -> Self { + Module { + is_crate: module.is_crate, + items: module.items.into_iter().map(|i| i.def_id.into()).collect(), + } + } +} + +impl From for Struct { + fn from(struct_: clean::Struct) -> Self { + let clean::Struct { struct_type, generics, fields, fields_stripped } = struct_; + Struct { + struct_type: struct_type.into(), + generics: generics.into(), + fields_stripped, + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), + impls: Vec::new(), // Added in JsonRenderer::insert + } + } +} + +impl From for Struct { + fn from(struct_: clean::Union) -> Self { + let clean::Union { struct_type, generics, fields, fields_stripped } = struct_; + Struct { + struct_type: struct_type.into(), + generics: generics.into(), + fields_stripped, + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), + impls: Vec::new(), // Added in JsonRenderer::insert + } + } +} + +impl From for StructType { + fn from(struct_type: doctree::StructType) -> Self { + use doctree::StructType::*; + match struct_type { + Plain => StructType::Plain, + Tuple => StructType::Tuple, + Unit => StructType::Unit, + } + } +} + +fn stringify_header(header: &rustc_hir::FnHeader) -> String { + let mut s = String::from(header.unsafety.prefix_str()); + if header.asyncness == rustc_hir::IsAsync::Async { + s.push_str("async ") + } + if header.constness == rustc_hir::Constness::Const { + s.push_str("const ") + } + s +} + +impl From for Function { + fn from(function: clean::Function) -> Self { + let clean::Function { decl, generics, header, all_types: _, ret_types: _ } = function; + Function { + decl: decl.into(), + generics: generics.into(), + header: stringify_header(&header), + abi: header.abi.to_string(), + } + } +} + +impl From for Generics { + fn from(generics: clean::Generics) -> Self { + Generics { + params: generics.params.into_iter().map(Into::into).collect(), + where_predicates: generics.where_predicates.into_iter().map(Into::into).collect(), + } + } +} + +impl From for GenericParamDef { + fn from(generic_param: clean::GenericParamDef) -> Self { + GenericParamDef { name: generic_param.name, kind: generic_param.kind.into() } + } +} + +impl From for GenericParamDefKind { + fn from(kind: clean::GenericParamDefKind) -> Self { + use clean::GenericParamDefKind::*; + match kind { + Lifetime => GenericParamDefKind::Lifetime, + Type { did: _, bounds, default, synthetic: _ } => GenericParamDefKind::Type { + bounds: bounds.into_iter().map(Into::into).collect(), + default: default.map(Into::into), + }, + Const { did: _, ty } => GenericParamDefKind::Const(ty.into()), + } + } +} + +impl From for WherePredicate { + fn from(predicate: clean::WherePredicate) -> Self { + use clean::WherePredicate::*; + match predicate { + BoundPredicate { ty, bounds } => WherePredicate::BoundPredicate { + ty: ty.into(), + bounds: bounds.into_iter().map(Into::into).collect(), + }, + RegionPredicate { lifetime, bounds } => WherePredicate::RegionPredicate { + lifetime: lifetime.0, + bounds: bounds.into_iter().map(Into::into).collect(), + }, + EqPredicate { lhs, rhs } => { + WherePredicate::EqPredicate { lhs: lhs.into(), rhs: rhs.into() } + } + } + } +} + +impl From for GenericBound { + fn from(bound: clean::GenericBound) -> Self { + use clean::GenericBound::*; + match bound { + TraitBound(clean::PolyTrait { trait_, generic_params }, modifier) => { + GenericBound::TraitBound { + trait_: trait_.into(), + generic_params: generic_params.into_iter().map(Into::into).collect(), + modifier: modifier.into(), + } + } + Outlives(lifetime) => GenericBound::Outlives(lifetime.0), + } + } +} + +impl From for TraitBoundModifier { + fn from(modifier: rustc_hir::TraitBoundModifier) -> Self { + use rustc_hir::TraitBoundModifier::*; + match modifier { + None => TraitBoundModifier::None, + Maybe => TraitBoundModifier::Maybe, + MaybeConst => TraitBoundModifier::MaybeConst, + } + } +} + +impl From for Type { + fn from(ty: clean::Type) -> Self { + use clean::Type::*; + match ty { + ResolvedPath { path, param_names, did, is_generic: _ } => Type::ResolvedPath { + name: path.whole_name(), + id: did.into(), + args: Box::new(path.segments.last().map(|args| args.clone().args.into())), + param_names: param_names + .map(|v| v.into_iter().map(Into::into).collect()) + .unwrap_or_default(), + }, + Generic(s) => Type::Generic(s), + Primitive(p) => Type::Primitive(p.as_str().to_string()), + // TODO: check if there's a more idiomatic way of calling `into` on Box + BareFunction(f) => Type::FunctionPointer(Box::new((*f).into())), + Tuple(t) => Type::Tuple(t.into_iter().map(Into::into).collect()), + Slice(t) => Type::Slice(Box::new((*t).into())), + Array(t, s) => Type::Array { type_: Box::new((*t).into()), len: s }, + ImplTrait(g) => Type::ImplTrait(g.into_iter().map(Into::into).collect()), + Never => Type::Never, + Infer => Type::Infer, + RawPointer(mutability, type_) => Type::RawPointer { + mutable: mutability == ast::Mutability::Mut, + type_: Box::new((*type_).into()), + }, + BorrowedRef { lifetime, mutability, type_ } => Type::BorrowedRef { + lifetime: lifetime.map(|l| l.0), + mutable: mutability == ast::Mutability::Mut, + type_: Box::new((*type_).into()), + }, + QPath { name, self_type, trait_ } => Type::QualifiedPath { + name, + self_type: Box::new((*self_type).into()), + trait_: Box::new((*trait_).into()), + }, + } + } +} + +impl From for FunctionPointer { + fn from(bare_decl: clean::BareFunctionDecl) -> Self { + let clean::BareFunctionDecl { unsafety, generic_params, decl, abi } = bare_decl; + FunctionPointer { + is_unsafe: unsafety == rustc_hir::Unsafety::Unsafe, + generic_params: generic_params.into_iter().map(Into::into).collect(), + decl: decl.into(), + abi: abi.to_string(), + } + } +} + +impl From for FnDecl { + fn from(decl: clean::FnDecl) -> Self { + let clean::FnDecl { inputs, output, c_variadic, attrs: _ } = decl; + FnDecl { + inputs: inputs.values.into_iter().map(|arg| (arg.name, arg.type_.into())).collect(), + output: match output { + clean::FnRetTy::Return(t) => Some(t.into()), + clean::FnRetTy::DefaultReturn => None, + }, + c_variadic, + } + } +} + +impl From for Trait { + fn from(trait_: clean::Trait) -> Self { + let clean::Trait { auto, unsafety, items, generics, bounds, is_spotlight: _, is_auto: _ } = + trait_; + Trait { + is_auto: auto, + is_unsafe: unsafety == rustc_hir::Unsafety::Unsafe, + items: items.into_iter().map(|i| i.def_id.into()).collect(), + generics: generics.into(), + bounds: bounds.into_iter().map(Into::into).collect(), + implementors: Vec::new(), // Added in JsonRenderer::insert + } + } +} + +impl From for Impl { + fn from(impl_: clean::Impl) -> Self { + let clean::Impl { + unsafety, + generics, + provided_trait_methods, + trait_, + for_, + items, + polarity, + synthetic, + blanket_impl, + } = impl_; + Impl { + is_unsafe: unsafety == rustc_hir::Unsafety::Unsafe, + generics: generics.into(), + provided_trait_methods: provided_trait_methods.into_iter().collect(), + trait_: trait_.map(Into::into), + for_: for_.into(), + items: items.into_iter().map(|i| i.def_id.into()).collect(), + negative: polarity == Some(clean::ImplPolarity::Negative), + synthetic, + blanket_impl: blanket_impl.map(Into::into), + } + } +} + +impl From for Method { + fn from(method: clean::TyMethod) -> Self { + let clean::TyMethod { header, decl, generics, all_types: _, ret_types: _ } = method; + Method { + decl: decl.into(), + generics: generics.into(), + header: stringify_header(&header), + has_body: false, + } + } +} + +impl From for Method { + fn from(method: clean::Method) -> Self { + let clean::Method { header, decl, generics, defaultness: _, all_types: _, ret_types: _ } = + method; + Method { + decl: decl.into(), + generics: generics.into(), + header: stringify_header(&header), + has_body: true, + } + } +} + +impl From for Enum { + fn from(enum_: clean::Enum) -> Self { + let clean::Enum { variants, generics, variants_stripped } = enum_; + Enum { + generics: generics.into(), + variants_stripped, + variants: variants.into_iter().map(|i| i.def_id.into()).collect(), + impls: Vec::new(), // Added in JsonRenderer::insert + } + } +} + +impl From for Struct { + fn from(struct_: clean::VariantStruct) -> Self { + let clean::VariantStruct { struct_type, fields, fields_stripped } = struct_; + Struct { + struct_type: struct_type.into(), + generics: Default::default(), + fields_stripped, + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), + impls: Vec::new(), + } + } +} + +impl From for Variant { + fn from(variant: clean::Variant) -> Self { + use clean::VariantKind::*; + match variant.kind { + CLike => Variant::Plain, + Tuple(t) => Variant::Tuple(t.into_iter().map(Into::into).collect()), + Struct(s) => Variant::Struct(s.fields.into_iter().map(|i| i.def_id.into()).collect()), + } + } +} + +impl From for Import { + fn from(import: clean::Import) -> Self { + use clean::Import::*; + match import { + Simple(s, i) => Import { + source: i.path.whole_name(), + name: s, + id: i.did.map(Into::into), + glob: false, + }, + Glob(i) => Import { + source: i.path.whole_name(), + name: i.path.last_name().to_string(), + id: i.did.map(Into::into), + glob: true, + }, + } + } +} + +impl From for ProcMacro { + fn from(mac: clean::ProcMacro) -> Self { + ProcMacro { kind: mac.kind.into(), helpers: mac.helpers } + } +} + +impl From for MacroKind { + fn from(kind: rustc_span::hygiene::MacroKind) -> Self { + use rustc_span::hygiene::MacroKind::*; + match kind { + Bang => MacroKind::Bang, + Attr => MacroKind::Attr, + Derive => MacroKind::Derive, + } + } +} + +impl From for Typedef { + fn from(typedef: clean::Typedef) -> Self { + let clean::Typedef { type_, generics, item_type: _ } = typedef; + Typedef { type_: type_.into(), generics: generics.into() } + } +} + +impl From for OpaqueTy { + fn from(opaque: clean::OpaqueTy) -> Self { + OpaqueTy { + bounds: opaque.bounds.into_iter().map(Into::into).collect(), + generics: opaque.generics.into(), + } + } +} + +impl From for Static { + fn from(stat: clean::Static) -> Self { + Static { + type_: stat.type_.into(), + mutable: stat.mutability == ast::Mutability::Mut, + expr: stat.expr, + } + } +} + +impl From for TraitAlias { + fn from(alias: clean::TraitAlias) -> Self { + TraitAlias { + generics: alias.generics.into(), + bounds: alias.bounds.into_iter().map(Into::into).collect(), + } + } +} + +impl From for ItemKind { + fn from(kind: ItemType) -> Self { + use ItemType::*; + match kind { + Module => ItemKind::Module, + ExternCrate => ItemKind::ExternCrate, + Import => ItemKind::Import, + Struct => ItemKind::Struct, + Union => ItemKind::Union, + Enum => ItemKind::Enum, + Function => ItemKind::Function, + Typedef => ItemKind::Typedef, + OpaqueTy => ItemKind::OpaqueTy, + Static => ItemKind::Static, + Constant => ItemKind::Constant, + Trait => ItemKind::Trait, + Impl => ItemKind::Impl, + TyMethod | Method => ItemKind::Method, + StructField => ItemKind::StructField, + Variant => ItemKind::Variant, + Macro => ItemKind::Macro, + Primitive => ItemKind::Primitive, + AssocConst => ItemKind::AssocConst, + AssocType => ItemKind::AssocType, + ForeignType => ItemKind::ForeignType, + Keyword => ItemKind::Keyword, + TraitAlias => ItemKind::TraitAlias, + ProcAttribute => ItemKind::ProcAttribute, + ProcDerive => ItemKind::ProcDerive, + } + } +} diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 14f87ec2aa9bb..5f4034d8753de 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -1,47 +1,204 @@ +mod conversions; +mod types; + +use std::cell::RefCell; +use std::fs::File; +use std::rc::Rc; + +use rustc_data_structures::fx::FxHashMap; +use rustc_span::edition::Edition; + use crate::clean; use crate::config::{RenderInfo, RenderOptions}; use crate::error::Error; use crate::formats::cache::Cache; use crate::formats::FormatRenderer; - -use rustc_span::edition::Edition; +use crate::html::render::cache::ExternalLocation; #[derive(Clone)] -pub struct JsonRenderer {} +pub struct JsonRenderer { + index: Rc>>, +} + +impl JsonRenderer { + fn insert(&self, item: clean::Item, cache: &Cache) { + let id = item.def_id; + let mut new_item: types::Item = item.into(); + if let types::ItemEnum::TraitItem(ref mut t) = new_item.inner { + t.implementors = self.get_trait_implementors(id, cache) + } else if let types::ItemEnum::StructItem(ref mut s) = new_item.inner { + s.impls = self.get_impls(id, cache) + } else if let types::ItemEnum::EnumItem(ref mut e) = new_item.inner { + e.impls = self.get_impls(id, cache) + } + self.index.borrow_mut().insert(id.into(), new_item); + } + + fn get_trait_implementors( + &self, + id: rustc_span::def_id::DefId, + cache: &Cache, + ) -> Vec { + cache + .implementors + .get(&id) + .map(|implementors| { + implementors + .iter() + .map(|i| { + let item = &i.impl_item; + self.insert(item.clone(), cache); + item.def_id.into() + }) + .collect() + }) + .unwrap_or_default() + } + + fn get_impls(&self, id: rustc_span::def_id::DefId, cache: &Cache) -> Vec { + cache + .impls + .get(&id) + .map(|impls| { + impls + .iter() + .filter_map(|i| { + let item = &i.impl_item; + if item.def_id.is_local() { + self.insert(item.clone(), cache); + Some(item.def_id.into()) + } else { + None + } + }) + .collect() + }) + .unwrap_or_default() + } +} impl FormatRenderer for JsonRenderer { fn init( - _krate: clean::Crate, + krate: clean::Crate, _options: RenderOptions, _render_info: RenderInfo, _edition: Edition, _cache: &mut Cache, ) -> Result<(Self, clean::Crate), Error> { - unimplemented!() + debug!("Initializing json renderer"); + Ok((JsonRenderer { index: Rc::new(RefCell::new(FxHashMap::default())) }, krate)) } - fn item(&mut self, _item: clean::Item, _cache: &Cache) -> Result<(), Error> { - unimplemented!() + fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { + use clean::ItemEnum::*; + // Flatten items that recursively store other items by putting their children in the index + match item.inner.clone() { + StructItem(s) => s.fields.into_iter().for_each(|i| self.insert(i, cache)), + UnionItem(u) => u.fields.into_iter().for_each(|i| self.insert(i, cache)), + VariantItem(clean::Variant { kind: clean::VariantKind::Struct(v) }) => { + v.fields.into_iter().for_each(|i| self.insert(i, cache)); + } + EnumItem(e) => e.variants.into_iter().for_each(|i| self.item(i, cache).unwrap()), + TraitItem(t) => t.items.into_iter().for_each(|i| self.insert(i, cache)), + ImplItem(i) => i.items.into_iter().for_each(|i| self.insert(i, cache)), + _ => {} + } + self.insert(item.clone(), cache); + Ok(()) } fn mod_item_in( &mut self, - _item: &clean::Item, + item: &clean::Item, _item_name: &str, - _cache: &Cache, + cache: &Cache, ) -> Result<(), Error> { - unimplemented!() + self.insert(item.clone(), cache); + Ok(()) } fn mod_item_out(&mut self, _item_name: &str) -> Result<(), Error> { - unimplemented!() + Ok(()) } - fn after_krate(&mut self, _krate: &clean::Crate, _cache: &Cache) -> Result<(), Error> { - unimplemented!() + fn after_krate(&mut self, krate: &clean::Crate, cache: &Cache) -> Result<(), Error> { + debug!("Done with crate"); + let mut index = (*self.index).clone().into_inner(); + let trait_items = cache.traits.iter().filter_map(|(id, trait_item)| { + // only need to synthesize items for external traits + if !id.is_local() { + trait_item.items.clone().into_iter().for_each(|i| self.insert(i, cache)); + Some(( + (*id).into(), + types::Item { + crate_num: id.krate.as_u32(), + name: cache + .paths + .get(&id) + .unwrap_or_else(|| { + cache + .external_paths + .get(&id) + .expect("Trait should either be in local or external paths") + }) + .0 + .last() + .map(Clone::clone), + visibility: types::Visibility::Public, + kind: types::ItemKind::Trait, + inner: types::ItemEnum::TraitItem(trait_item.clone().into()), + source: None, + docs: Default::default(), + links: Default::default(), + attrs: Default::default(), + deprecation: Default::default(), + }, + )) + } else { + None + } + }); + index.extend(trait_items); + let output = types::Crate { + root: types::Id(String::from("0:0")), + version: krate.version.clone(), + includes_private: cache.document_private, + index, + paths: cache + .paths + .clone() + .into_iter() + .chain(cache.external_paths.clone().into_iter()) + .map(|(k, (path, kind))| { + ( + k.into(), + types::ItemSummary { crate_num: k.krate.as_u32(), path, kind: kind.into() }, + ) + }) + .collect(), + external_crates: cache + .extern_locations + .iter() + .map(|(k, v)| { + ( + k.as_u32(), + types::ExternalCrate { + name: v.0.clone(), + html_root_url: match &v.2 { + ExternalLocation::Remote(s) => Some(s.clone()), + _ => None, + }, + }, + ) + }) + .collect(), + format_version: 1, + }; + serde_json::ser::to_writer_pretty(&File::create("test.json").unwrap(), &output).unwrap(); + Ok(()) } fn after_run(&mut self, _diag: &rustc_errors::Handler) -> Result<(), Error> { - unimplemented!() + Ok(()) } } diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs new file mode 100644 index 0000000000000..f888d00b95158 --- /dev/null +++ b/src/librustdoc/json/types.rs @@ -0,0 +1,485 @@ +//! Rustdoc's JSON output interface +//! +//! These types are the public API exposed through the `--output-format json` flag. The [`Crate`][] +//! struct is the root of the JSON blob and all other items are contained within. + +use std::path::PathBuf; + +use rustc_data_structures::fx::FxHashMap; +use serde::{Deserialize, Serialize}; + +/// A `Crate` is the root of the emitted JSON blob. It contains all type/documentation information +/// about the language items in the local crate, as well as info about external items to allow +/// tools to find or link to them. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Crate { + /// The id of the root [`Module`][] item of the local crate. + pub root: Id, + /// The version string given to `--crate-version`, if any. + pub version: Option, + /// Whether or not the output includes private items. + pub includes_private: bool, + /// A collection of all items in the local crate as well as some external traits and their + /// items that are referenced locally. + pub index: FxHashMap, + /// Maps ids to fully qualified paths (e.g. `["std", "io", "lazy", "Lazy"]` for + /// `std::io::lazy::Lazy`) as well as their `ItemKind` + pub paths: FxHashMap, + /// Maps `crate_num` of items to a crate name and html_root_url if it exists + pub external_crates: FxHashMap, + /// A single version number to be used in the future when making backwards incompatible changes + /// to the JSON output. + pub format_version: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ExternalCrate { + pub name: String, + pub html_root_url: Option, +} + +/// For external items (stuff not defined in the local crate), you don't get the same level of +/// information. This struct should contain enough to generate a link/reference to the item in +/// question, or can be used by a tool that takes the json output of multiple crates to find +/// the actual item definition with all the relevant info. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ItemSummary { + pub crate_num: u32, + pub path: Vec, + pub kind: ItemKind, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Item { + /// This can be used as a key to the `external_crates` map of [`Crate`][] to see which crate + /// this item came from. + pub crate_num: u32, + /// Some items such as impls don't have names. + pub name: Option, + /// The source location of this item. May not be present if it came from a macro expansion, + /// inline assembly, other "virtual" files. + pub source: Option, + /// Usually documented items are all public, but you can tell rustdoc to output private items + /// so this field is needed to differentiate. + pub visibility: Visibility, + /// The full docstring of this item. + pub docs: String, + /// This mapping resolves [intradoc links](https://github.com/rust-lang/rfcs/blob/master/text/1946-intra-rustdoc-links.md) from the docstring to their IDs + pub links: FxHashMap, + /// Stringified versions of the attributes on this item (e.g. `"#[inline]"`) + pub attrs: Vec, + pub deprecation: Option, + pub kind: ItemKind, + pub inner: ItemEnum, + // TODO: should we stringify the cfg attrs as well, or should we preserve their structure so + // the consumer doesn't have to parse an arbitrarily nested tree to figure out what platforms + // the item is available on? + // TODO: should we have a "stability" field if it's only used by the standard library? +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Span { + /// The path to the source file for this span relative to the crate root. + pub filename: PathBuf, + /// Zero indexed Line and Column of the first character of the `Span` + pub begin: (usize, usize), + /// Zero indexed Line and Column of the last character of the `Span` + pub end: (usize, usize), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Deprecation { + pub since: Option, + pub note: Option, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Visibility { + Public, + Default, + Crate, + // TODO: Restricted(Id, String), +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum GenericArgs { + /// <'a, 32, B: Copy, C = u32> + AngleBracketed { args: Vec, bindings: Vec }, + /// Fn(A, B) -> C + Parenthesized { inputs: Vec, output: Option }, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum GenericArg { + Lifetime(String), + Type(Type), + Const(Constant), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Constant { + #[serde(rename = "type")] + pub type_: Type, + pub expr: String, + pub value: Option, + pub is_literal: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TypeBinding { + pub name: String, + pub binding: TypeBindingKind, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum TypeBindingKind { + Equality(Type), + Constraint(Vec), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Id(pub String); + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum ItemKind { + Module, + ExternCrate, + Import, + Struct, + StructField, + Union, + Enum, + Variant, + Function, + Typedef, + OpaqueTy, + Constant, + Trait, + TraitAlias, + Method, + Impl, + Static, + ForeignType, + Macro, + ProcAttribute, + ProcDerive, + AssocConst, + AssocType, + Primitive, + Keyword, +} + +#[serde(untagged)] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum ItemEnum { + ModuleItem(Module), + ExternCrateItem { + name: String, + rename: Option, + }, + ImportItem(Import), + + StructItem(Struct), + StructFieldItem(Type), + EnumItem(Enum), + VariantItem(Variant), + + FunctionItem(Function), + + TypedefItem(Typedef), + OpaqueTyItem(OpaqueTy), + ConstantItem(Constant), + + TraitItem(Trait), + TraitAliasItem(TraitAlias), + MethodItem(Method), + ImplItem(Impl), + + StaticItem(Static), + + /// `type`s from an extern block + ForeignTypeItem, + + /// Declarative macro_rules! macro + MacroItem(String), + ProcMacroItem(ProcMacro), + + AssocConstItem { + #[serde(rename = "type")] + type_: Type, + default: Option, + }, + AssocTypeItem { + bounds: Vec, + default: Option, + }, + + /// An item that has been stripped by a rustdoc pass + StrippedItem(Box), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Module { + pub is_crate: bool, + pub items: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Struct { + pub struct_type: StructType, + pub generics: Generics, + pub fields_stripped: bool, + pub fields: Vec, + pub impls: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Enum { + pub generics: Generics, + pub variants_stripped: bool, + pub variants: Vec, + pub impls: Vec, +} + +#[serde(rename_all = "snake_case")] +#[serde(tag = "variant_kind", content = "variant_inner")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Variant { + Plain, + Tuple(Vec), + Struct(Vec), +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum StructType { + Plain, + Tuple, + Unit, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Function { + pub decl: FnDecl, + pub generics: Generics, + pub header: String, + pub abi: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Method { + pub decl: FnDecl, + pub generics: Generics, + pub header: String, + pub has_body: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Generics { + pub params: Vec, + pub where_predicates: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GenericParamDef { + pub name: String, + pub kind: GenericParamDefKind, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum GenericParamDefKind { + Lifetime, + Type { bounds: Vec, default: Option }, + Const(Type), +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum WherePredicate { + BoundPredicate { ty: Type, bounds: Vec }, + RegionPredicate { lifetime: String, bounds: Vec }, + EqPredicate { lhs: Type, rhs: Type }, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum GenericBound { + TraitBound { + #[serde(rename = "trait")] + trait_: Type, + /// Used for HRTBs + generic_params: Vec, + modifier: TraitBoundModifier, + }, + Outlives(String), +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum TraitBoundModifier { + None, + Maybe, + MaybeConst, +} + +#[serde(rename_all = "snake_case")] +#[serde(tag = "kind", content = "inner")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Type { + /// Structs, enums, and traits + ResolvedPath { + name: String, + id: Id, + args: Box>, + param_names: Vec, + }, + /// Parameterized types + Generic(String), + /// Fixed-size numeric types (plus int/usize/float), char, arrays, slices, and tuples + Primitive(String), + /// `extern "ABI" fn` + FunctionPointer(Box), + /// `(String, u32, Box)` + Tuple(Vec), + /// `[u32]` + Slice(Box), + /// [u32; 15] + Array { + #[serde(rename = "type")] + type_: Box, + len: String, + }, + /// `impl TraitA + TraitB + ...` + ImplTrait(Vec), + /// `!` + Never, + /// `_` + Infer, + /// `*mut u32`, `*u8`, etc. + RawPointer { + mutable: bool, + #[serde(rename = "type")] + type_: Box, + }, + /// `&'a mut String`, `&str`, etc. + BorrowedRef { + lifetime: Option, + mutable: bool, + #[serde(rename = "type")] + type_: Box, + }, + /// `::Name` or associated types like `T::Item` where `T: Iterator` + QualifiedPath { + name: String, + self_type: Box, + #[serde(rename = "trait")] + trait_: Box, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FunctionPointer { + pub is_unsafe: bool, + pub generic_params: Vec, + pub decl: FnDecl, + pub abi: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FnDecl { + pub inputs: Vec<(String, Type)>, + pub output: Option, + pub c_variadic: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Trait { + pub is_auto: bool, + pub is_unsafe: bool, + pub items: Vec, + pub generics: Generics, + pub bounds: Vec, + pub implementors: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TraitAlias { + pub generics: Generics, + pub bounds: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Impl { + pub is_unsafe: bool, + pub generics: Generics, + pub provided_trait_methods: Vec, + #[serde(rename = "trait")] + pub trait_: Option, + #[serde(rename = "for")] + pub for_: Type, + pub items: Vec, + pub negative: bool, + pub synthetic: bool, + pub blanket_impl: Option, +} + +// TODO: this is currently broken because imports have the same ID as the module that contains +// them. The only obvious fix is to modify the clean types to renumber imports so that IDs are +// actually unique. +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Import { + /// The full path being imported. + pub source: String, + /// May be different from the last segment of `source` when renaming imports: + /// `use source as name;` + pub name: String, + /// The ID of the item being imported. + pub id: Option, // TODO: when is this None? + /// Whether this import uses a glob: `use source::*;` + pub glob: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ProcMacro { + pub kind: MacroKind, + pub helpers: Vec, +} + +#[serde(rename_all = "snake_case")] +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MacroKind { + /// A bang macro `foo!()`. + Bang, + /// An attribute macro `#[foo]`. + Attr, + /// A derive macro `#[derive(Foo)]` + Derive, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Typedef { + #[serde(rename = "type")] + pub type_: Type, + pub generics: Generics, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OpaqueTy { + pub bounds: Vec, + pub generics: Generics, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Static { + #[serde(rename = "type")] + pub type_: Type, + pub mutable: bool, + pub expr: String, +} From 3506135751c9877b94e89163c84394900a200225 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Wed, 12 Aug 2020 12:22:45 -0500 Subject: [PATCH 02/20] Respond to comments and start adding tests --- src/librustdoc/clean/mod.rs | 2 +- src/librustdoc/clean/types.rs | 36 ++++ src/librustdoc/json/conversions.rs | 9 +- src/librustdoc/json/mod.rs | 124 +++++++------ src/librustdoc/json/types.rs | 41 +++-- .../run-make-fulldeps/rustdoc-json/Makefile | 5 + .../rustdoc-json/check_missing_items.py | 170 ++++++++++++++++++ .../run-make-fulldeps/rustdoc-json/test.rs | 17 ++ 8 files changed, 329 insertions(+), 75 deletions(-) create mode 100644 src/test/run-make-fulldeps/rustdoc-json/Makefile create mode 100644 src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py create mode 100644 src/test/run-make-fulldeps/rustdoc-json/test.rs diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index bfc747058185e..3145d5e306d03 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -2314,7 +2314,7 @@ impl Clean> for doctree::Import<'_> { name: None, attrs: self.attrs.clean(cx), source: self.span.clean(cx), - def_id: DefId::local(CRATE_DEF_INDEX), + def_id: cx.tcx.hir().local_def_id(self.id).to_def_id(), visibility: self.vis.clean(cx), stability: None, deprecation: None, diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index e25f076147821..fbd6853388cc3 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -287,6 +287,42 @@ pub enum ItemEnum { } impl ItemEnum { + /// Some items contain others such as structs (for their fields) and Enums + /// (for their variants). This method returns those contained items. + pub fn inner_items(&self) -> impl Iterator { + match self { + StructItem(s) => s.fields.iter(), + UnionItem(u) => u.fields.iter(), + VariantItem(Variant { kind: VariantKind::Struct(v) }) => v.fields.iter(), + EnumItem(e) => e.variants.iter(), + TraitItem(t) => t.items.iter(), + ImplItem(i) => i.items.iter(), + ModuleItem(m) => m.items.iter(), + ExternCrateItem(_, _) + | ImportItem(_) + | FunctionItem(_) + | TypedefItem(_, _) + | OpaqueTyItem(_, _) + | StaticItem(_) + | ConstantItem(_) + | TraitAliasItem(_) + | TyMethodItem(_) + | MethodItem(_) + | StructFieldItem(_) + | VariantItem(_) + | ForeignFunctionItem(_) + | ForeignStaticItem(_) + | ForeignTypeItem + | MacroItem(_) + | ProcMacroItem(_) + | PrimitiveItem(_) + | AssocConstItem(_,_) + | AssocTypeItem(_,_) + | StrippedItem(_) + | KeywordItem(_) => [].iter(), + } + } + pub fn is_type_alias(&self) -> bool { match *self { ItemEnum::TypedefItem(_, _) | ItemEnum::AssocTypeItem(_, _) => true, diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 6855931def1e6..d0e38ceb8a023 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -1,3 +1,7 @@ +//! These from impls are used to create the JSON types which get serialized. They're very close to +//! the `clean` types but with some fields removed or stringified to simplify the output and not +//! expose unstable compiler internals. + use std::convert::From; use rustc_ast::ast; @@ -22,7 +26,7 @@ impl From for Item { deprecation, } = item; Item { - crate_num: def_id.krate.as_u32(), + crate_id: def_id.krate.as_u32(), name, source: source.into(), visibility: visibility.into(), @@ -329,14 +333,13 @@ impl From for Type { ResolvedPath { path, param_names, did, is_generic: _ } => Type::ResolvedPath { name: path.whole_name(), id: did.into(), - args: Box::new(path.segments.last().map(|args| args.clone().args.into())), + args: path.segments.last().map(|args| Box::new(args.clone().args.into())), param_names: param_names .map(|v| v.into_iter().map(Into::into).collect()) .unwrap_or_default(), }, Generic(s) => Type::Generic(s), Primitive(p) => Type::Primitive(p.as_str().to_string()), - // TODO: check if there's a more idiomatic way of calling `into` on Box BareFunction(f) => Type::FunctionPointer(Box::new((*f).into())), Tuple(t) => Type::Tuple(t.into_iter().map(Into::into).collect()), Slice(t) => Type::Slice(Box::new((*t).into())), diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 5f4034d8753de..de858ff79aa2a 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -1,8 +1,15 @@ +//! Rustdoc's JSON backend +//! +//! This module contains the logic for rendering a crate as JSON rather than the normal static HTML +//! output. See [the RFC](https://github.com/rust-lang/rfcs/pull/2963) and the [`types`] module +//! docs for usage and details. + mod conversions; mod types; use std::cell::RefCell; use std::fs::File; +use std::path::PathBuf; use std::rc::Rc; use rustc_data_structures::fx::FxHashMap; @@ -17,10 +24,17 @@ use crate::html::render::cache::ExternalLocation; #[derive(Clone)] pub struct JsonRenderer { + /// A mapping of IDs that contains all local items for this crate which gets output as a top + /// level field of the JSON blob. index: Rc>>, + /// The directory where the blob will be written to. + out_path: PathBuf, } impl JsonRenderer { + /// Inserts an item into the index. This should be used rather than directly calling insert on + /// the hashmap because certain items (traits and types) need to have their mappings for trait + /// implementations filled out before they're inserted. fn insert(&self, item: clean::Item, cache: &Cache) { let id = item.def_id; let mut new_item: types::Item = item.into(); @@ -75,33 +89,74 @@ impl JsonRenderer { }) .unwrap_or_default() } + + fn get_trait_items(&self, cache: &Cache) -> Vec<(types::Id, types::Item)> { + cache + .traits + .iter() + .filter_map(|(id, trait_item)| { + // only need to synthesize items for external traits + if !id.is_local() { + trait_item.items.clone().into_iter().for_each(|i| self.insert(i, cache)); + Some(( + (*id).into(), + types::Item { + crate_id: id.krate.as_u32(), + name: cache + .paths + .get(&id) + .unwrap_or_else(|| { + cache + .external_paths + .get(&id) + .expect("Trait should either be in local or external paths") + }) + .0 + .last() + .map(Clone::clone), + visibility: types::Visibility::Public, + kind: types::ItemKind::Trait, + inner: types::ItemEnum::TraitItem(trait_item.clone().into()), + source: None, + docs: Default::default(), + links: Default::default(), + attrs: Default::default(), + deprecation: Default::default(), + }, + )) + } else { + None + } + }) + .collect() + } } impl FormatRenderer for JsonRenderer { fn init( krate: clean::Crate, - _options: RenderOptions, + options: RenderOptions, _render_info: RenderInfo, _edition: Edition, _cache: &mut Cache, ) -> Result<(Self, clean::Crate), Error> { debug!("Initializing json renderer"); - Ok((JsonRenderer { index: Rc::new(RefCell::new(FxHashMap::default())) }, krate)) + Ok(( + JsonRenderer { + index: Rc::new(RefCell::new(FxHashMap::default())), + out_path: options.output, + }, + krate, + )) } fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { use clean::ItemEnum::*; - // Flatten items that recursively store other items by putting their children in the index - match item.inner.clone() { - StructItem(s) => s.fields.into_iter().for_each(|i| self.insert(i, cache)), - UnionItem(u) => u.fields.into_iter().for_each(|i| self.insert(i, cache)), - VariantItem(clean::Variant { kind: clean::VariantKind::Struct(v) }) => { - v.fields.into_iter().for_each(|i| self.insert(i, cache)); - } - EnumItem(e) => e.variants.into_iter().for_each(|i| self.item(i, cache).unwrap()), - TraitItem(t) => t.items.into_iter().for_each(|i| self.insert(i, cache)), - ImplItem(i) => i.items.into_iter().for_each(|i| self.insert(i, cache)), - _ => {} + // Flatten items that recursively store other items by inserting them into the index + if let ModuleItem(_) = &item.inner { + // but ignore modules because we handle recursing into them separately + } else { + item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()) } self.insert(item.clone(), cache); Ok(()) @@ -124,41 +179,7 @@ impl FormatRenderer for JsonRenderer { fn after_krate(&mut self, krate: &clean::Crate, cache: &Cache) -> Result<(), Error> { debug!("Done with crate"); let mut index = (*self.index).clone().into_inner(); - let trait_items = cache.traits.iter().filter_map(|(id, trait_item)| { - // only need to synthesize items for external traits - if !id.is_local() { - trait_item.items.clone().into_iter().for_each(|i| self.insert(i, cache)); - Some(( - (*id).into(), - types::Item { - crate_num: id.krate.as_u32(), - name: cache - .paths - .get(&id) - .unwrap_or_else(|| { - cache - .external_paths - .get(&id) - .expect("Trait should either be in local or external paths") - }) - .0 - .last() - .map(Clone::clone), - visibility: types::Visibility::Public, - kind: types::ItemKind::Trait, - inner: types::ItemEnum::TraitItem(trait_item.clone().into()), - source: None, - docs: Default::default(), - links: Default::default(), - attrs: Default::default(), - deprecation: Default::default(), - }, - )) - } else { - None - } - }); - index.extend(trait_items); + index.extend(self.get_trait_items(cache)); let output = types::Crate { root: types::Id(String::from("0:0")), version: krate.version.clone(), @@ -172,7 +193,7 @@ impl FormatRenderer for JsonRenderer { .map(|(k, (path, kind))| { ( k.into(), - types::ItemSummary { crate_num: k.krate.as_u32(), path, kind: kind.into() }, + types::ItemSummary { crate_id: k.krate.as_u32(), path, kind: kind.into() }, ) }) .collect(), @@ -194,7 +215,10 @@ impl FormatRenderer for JsonRenderer { .collect(), format_version: 1, }; - serde_json::ser::to_writer_pretty(&File::create("test.json").unwrap(), &output).unwrap(); + let mut p = self.out_path.clone(); + p.push(output.index.get(&output.root).unwrap().name.clone().unwrap()); + p.set_extension("json"); + serde_json::ser::to_writer_pretty(&File::create(p).unwrap(), &output).unwrap(); Ok(()) } diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index f888d00b95158..b3db6ff33f24f 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -1,6 +1,6 @@ //! Rustdoc's JSON output interface //! -//! These types are the public API exposed through the `--output-format json` flag. The [`Crate`][] +//! These types are the public API exposed through the `--output-format json` flag. The [`Crate`] //! struct is the root of the JSON blob and all other items are contained within. use std::path::PathBuf; @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; /// tools to find or link to them. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Crate { - /// The id of the root [`Module`][] item of the local crate. + /// The id of the root [`Module`] item of the local crate. pub root: Id, /// The version string given to `--crate-version`, if any. pub version: Option, @@ -22,10 +22,9 @@ pub struct Crate { /// A collection of all items in the local crate as well as some external traits and their /// items that are referenced locally. pub index: FxHashMap, - /// Maps ids to fully qualified paths (e.g. `["std", "io", "lazy", "Lazy"]` for - /// `std::io::lazy::Lazy`) as well as their `ItemKind` + /// Maps IDs to fully qualified paths and other info helpful for generating links. pub paths: FxHashMap, - /// Maps `crate_num` of items to a crate name and html_root_url if it exists + /// Maps `crate_id` of items to a crate name and html_root_url if it exists. pub external_crates: FxHashMap, /// A single version number to be used in the future when making backwards incompatible changes /// to the JSON output. @@ -38,31 +37,36 @@ pub struct ExternalCrate { pub html_root_url: Option, } -/// For external items (stuff not defined in the local crate), you don't get the same level of +/// For external (not defined in the local crate) items, you don't get the same level of /// information. This struct should contain enough to generate a link/reference to the item in /// question, or can be used by a tool that takes the json output of multiple crates to find /// the actual item definition with all the relevant info. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ItemSummary { - pub crate_num: u32, + /// Can be used to look up the name and html_root_url of the crate this item came from in the + /// `external_crates` map. + pub crate_id: u32, + /// The list of path components for the fully qualified path of this item (e.g. + /// `["std", "io", "lazy", "Lazy"]` for `std::io::lazy::Lazy`). pub path: Vec, + /// Whether this item is a struct, trait, macro, etc. pub kind: ItemKind, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Item { - /// This can be used as a key to the `external_crates` map of [`Crate`][] to see which crate + /// This can be used as a key to the `external_crates` map of [`Crate`] to see which crate /// this item came from. - pub crate_num: u32, + pub crate_id: u32, /// Some items such as impls don't have names. pub name: Option, - /// The source location of this item. May not be present if it came from a macro expansion, - /// inline assembly, other "virtual" files. + /// The source location of this item (absent if it came from a macro expansion or inline + /// assembly). pub source: Option, - /// Usually documented items are all public, but you can tell rustdoc to output private items + /// By default all documented items are public, but you can tell rustdoc to output private items /// so this field is needed to differentiate. pub visibility: Visibility, - /// The full docstring of this item. + /// The full markdown docstring of this item. pub docs: String, /// This mapping resolves [intradoc links](https://github.com/rust-lang/rfcs/blob/master/text/1946-intra-rustdoc-links.md) from the docstring to their IDs pub links: FxHashMap, @@ -71,10 +75,6 @@ pub struct Item { pub deprecation: Option, pub kind: ItemKind, pub inner: ItemEnum, - // TODO: should we stringify the cfg attrs as well, or should we preserve their structure so - // the consumer doesn't have to parse an arbitrarily nested tree to figure out what platforms - // the item is available on? - // TODO: should we have a "stability" field if it's only used by the standard library? } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -97,6 +97,8 @@ pub struct Deprecation { #[derive(Clone, Debug, Serialize, Deserialize)] pub enum Visibility { Public, + /// For the most part items are private by default. The exceptions are associated items of + /// public traits and variants of public enums. Default, Crate, // TODO: Restricted(Id, String), @@ -336,7 +338,7 @@ pub enum Type { ResolvedPath { name: String, id: Id, - args: Box>, + args: Option>, param_names: Vec, }, /// Parameterized types @@ -429,9 +431,6 @@ pub struct Impl { pub blanket_impl: Option, } -// TODO: this is currently broken because imports have the same ID as the module that contains -// them. The only obvious fix is to modify the clean types to renumber imports so that IDs are -// actually unique. #[serde(rename_all = "snake_case")] #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Import { diff --git a/src/test/run-make-fulldeps/rustdoc-json/Makefile b/src/test/run-make-fulldeps/rustdoc-json/Makefile new file mode 100644 index 0000000000000..b66bcb8035daa --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/Makefile @@ -0,0 +1,5 @@ +-include ../tools.mk + +tests: *.rs + $(RUSTDOC) $< -o $(TMPDIR) --output-format json + $(PYTHON) check_missing_items.py $(TMPDIR)/$(basename $<).json diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py new file mode 100644 index 0000000000000..224b8f82ca9cc --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python + +import sys +import json + +crate = json.load(open(sys.argv[1])) + + +def get_local_item(item_id): + if item_id in crate["index"]: + return crate["index"][item_id] + print("Missing local ID:", item_id) + sys.exit(1) + + +# local IDs have to be in `index`, external ones can sometimes be in `index` but otherwise have +# to be in `paths` +def valid_id(item_id): + return item_id in crate["index"] or item_id[0] != "0" and item_id in crate["paths"] + + +def check_generics(generics): + for param in generics["params"]: + check_generic_param(param) + for where_predicate in generics["where_predicates"]: + if "bound_predicate" in where_predicate: + pred = where_predicate["bound_predicate"] + check_type(pred["ty"]) + for bound in pred["bounds"]: + check_generic_bound(bound) + elif "region_predicate" in where_predicate: + pred = where_predicate["region_predicate"] + for bound in pred["bounds"]: + check_generic_bound(bound) + elif "eq_predicate" in where_predicate: + pred = where_predicate["eq_predicate"] + check_type(pred["rhs"]) + check_type(pred["lhs"]) + + +def check_generic_param(param): + if "type" in param["kind"]: + ty = param["kind"]["type"] + if ty["default"]: + check_type(ty["default"]) + for bound in ty["bounds"]: + check_generic_bound(bound) + elif "const" in param["kind"]: + check_type(param["kind"]["const"]) + + +def check_generic_bound(bound): + if "trait_bound" in bound: + for param in bound["trait_bound"]["generic_params"]: + check_generic_param(param) + check_type(bound["trait_bound"]["trait"]) + + +def check_decl(decl): + for (_name, ty) in decl["inputs"]: + check_type(ty) + if decl["output"]: + check_type(decl["output"]) + + +def check_type(ty): + if ty["kind"] == "resolved_path": + for bound in ty["inner"]["param_names"]: + check_generic_bound(bound) + if args := ty["inner"]["args"]: + if "angle_bracketed" in args: + for arg in args["angle_bracketed"]["args"]: + if "type" in arg: + check_type(arg["type"]) + elif "const" in arg: + check_type(arg["const"]["type"]) + for binding in args["angle_bracketed"]["bindings"]: + if "equality" in binding["binding"]: + check_type(binding["binding"]["equality"]) + elif "constraint" in binding["binding"]: + for bound in binding["binding"]["constraint"]: + check_generic_bound(bound) + elif "parenthesized" in args: + for ty in args["parenthesized"]["inputs"]: + check_type(ty) + if args["parenthesized"]["output"]: + ckeck_ty(args["parenthesized"]["output"]) + if not valid_id(ty["inner"]["id"]): + print("Type contained an invalid ID:", ty["inner"]["id"]) + sys.exit(1) + elif ty["kind"] == "tuple": + for ty in ty["inner"]: + check_type(ty) + elif ty["kind"] == "slice": + check_type(ty["inner"]) + elif ty["kind"] == "impl_trait": + for bound in ty["inner"]: + check_generic_bound(bound) + elif ty["kind"] in ("raw_pointer", "borrowed_ref", "array"): + check_type(ty["inner"]["type"]) + elif ty["kind"] == "function_pointer": + for param in ty["inner"]["generic_params"]: + check_generic_param(param) + check_decl(ty["inner"]["inner"]) + elif ty["kind"] == "qualified_path": + check_type(ty["inner"]["self_type"]) + check_type(ty["inner"]["trait"]) + + +work_list = set([crate["root"]]) +visited = work_list.copy() + +while work_list: + current = work_list.pop() + visited.add(current) + item = get_local_item(current) + # check intradoc links + work_list |= set(item["links"].values()) - visited + + # check all fields that reference types such as generics as well as nested items + # (modules, structs, traits, and enums) + if item["kind"] == "module": + work_list |= set(item["inner"]["items"]) - visited + elif item["kind"] == "struct": + check_generics(item["inner"]["generics"]) + work_list |= (set(item["inner"]["fields"]) | set(item["inner"]["impls"])) - visited + elif item["kind"] == "struct_field": + check_type(item["inner"]) + elif item["kind"] == "enum": + check_generics(item["inner"]["generics"]) + work_list |= (set(item["inner"]["variants"]) | set(item["inner"]["impls"])) - visited + elif item["kind"] == "variant": + if item["inner"]["variant_kind"] == "tuple": + for ty in item["inner"]["variant_inner"]: + check_type(ty) + elif item["inner"]["variant_kind"] == "struct": + work_list |= set(item["inner"]["variant_inner"]) - visited + elif item["kind"] in ("function", "method"): + check_generics(item["inner"]["generics"]) + check_decl(item["inner"]["decl"]) + elif item["kind"] in ("static", "constant", "assoc_const"): + check_type(item["inner"]["type"]) + elif item["kind"] == "typedef": + check_type(item["inner"]["type"]) + check_generics(item["inner"]["generics"]) + elif item["kind"] in ("opaque_ty", "trait_alias"): + check_generics(item["inner"]["generics"]) + for bound in item["inner"]["bounds"]: + check_generic_bound(bound) + elif item["kind"] == "trait": + check_generics(item["inner"]["generics"]) + for bound in item["inner"]["bounds"]: + check_generic_bound(bound) + work_list |= (set(item["inner"]["items"]) | set(item["inner"]["implementors"])) - visited + elif item["kind"] == "impl": + check_generics(item["inner"]["generics"]) + if item["inner"]["trait"]: + check_type(item["inner"]["trait"]) + if item["inner"]["blanket_impl"]: + check_type(item["inner"]["blanket_impl"]) + check_type(item["inner"]["for"]) + for assoc_item in item["inner"]["items"]: + if not valid_id(assoc_item): + print("Impl block referenced a missing ID:", assoc_item) + sys.exit(1) + elif item["kind"] == "assoc_type": + for bound in item["inner"]["bounds"]: + check_generic_bound(bound) + if item["inner"]["default"]: + check_type(item["inner"]["default"]) diff --git a/src/test/run-make-fulldeps/rustdoc-json/test.rs b/src/test/run-make-fulldeps/rustdoc-json/test.rs new file mode 100644 index 0000000000000..e467f0d4cccc2 --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/test.rs @@ -0,0 +1,17 @@ +use std::collections::HashMap; + +pub struct Normal {} + +pub struct Tuple(); + +pub struct Unit; + +pub struct WithPrimitives<'a> { + num: u32, + s: &'a str, +} + +pub struct WithGenerics { + stuff: Vec, + things: HashMap, +} From 8156b9a9ff20ce703a1a63f25dd6d343f8b11e28 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 14 Aug 2020 07:09:01 -0500 Subject: [PATCH 03/20] Fix re-exports and extern-crate --- src/librustdoc/json/mod.rs | 20 ++++++++++++------- src/librustdoc/json/types.rs | 4 ++-- .../rustdoc-json/check_missing_items.py | 4 +++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index de858ff79aa2a..2f22e7f2473e3 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -151,13 +151,8 @@ impl FormatRenderer for JsonRenderer { } fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { - use clean::ItemEnum::*; // Flatten items that recursively store other items by inserting them into the index - if let ModuleItem(_) = &item.inner { - // but ignore modules because we handle recursing into them separately - } else { - item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()) - } + item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()); self.insert(item.clone(), cache); Ok(()) } @@ -165,9 +160,20 @@ impl FormatRenderer for JsonRenderer { fn mod_item_in( &mut self, item: &clean::Item, - _item_name: &str, + _module_name: &str, cache: &Cache, ) -> Result<(), Error> { + use clean::types::ItemEnum::*; + if let ModuleItem(m) = &item.inner { + for item in &m.items { + match item.inner { + // These don't have names so they don't get added to the output by default + ImportItem(_) => self.insert(item.clone(), cache), + ExternCrateItem(_, _) => self.insert(item.clone(), cache), + _ => {} + } + } + } self.insert(item.clone(), cache); Ok(()) } diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index b3db6ff33f24f..6adc226e69bc0 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -101,7 +101,7 @@ pub enum Visibility { /// public traits and variants of public enums. Default, Crate, - // TODO: Restricted(Id, String), + // FIXME(pineapple): add support for restricted paths } #[serde(rename_all = "snake_case")] @@ -440,7 +440,7 @@ pub struct Import { /// `use source as name;` pub name: String, /// The ID of the item being imported. - pub id: Option, // TODO: when is this None? + pub id: Option, // TODO when is this None? /// Whether this import uses a glob: `use source::*;` pub glob: bool, } diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py index 224b8f82ca9cc..7c3e389cfd468 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -115,7 +115,9 @@ def check_type(ty): visited.add(current) item = get_local_item(current) # check intradoc links - work_list |= set(item["links"].values()) - visited + for (_name, link) in item["links"].items(): + if not valid_id(link): + print("Intra-doc link contains invalid ID:", link) # check all fields that reference types such as generics as well as nested items # (modules, structs, traits, and enums) From c8b4e3cb065bcc657bbf15482176df02309681e0 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 14 Aug 2020 08:39:02 -0500 Subject: [PATCH 04/20] Add expected output tests --- src/librustdoc/json/mod.rs | 2 +- .../run-make-fulldeps/rustdoc-json/Makefile | 1 + .../rustdoc-json/check_missing_items.py | 4 + .../run-make-fulldeps/rustdoc-json/compare.py | 105 +++++++ .../rustdoc-json/modules.expected | 171 +++++++++++ .../run-make-fulldeps/rustdoc-json/modules.rs | 8 + .../rustdoc-json/structs.expected | 277 ++++++++++++++++++ .../rustdoc-json/{test.rs => structs.rs} | 0 8 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 src/test/run-make-fulldeps/rustdoc-json/compare.py create mode 100644 src/test/run-make-fulldeps/rustdoc-json/modules.expected create mode 100644 src/test/run-make-fulldeps/rustdoc-json/modules.rs create mode 100644 src/test/run-make-fulldeps/rustdoc-json/structs.expected rename src/test/run-make-fulldeps/rustdoc-json/{test.rs => structs.rs} (100%) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 2f22e7f2473e3..58b5671e8668d 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -5,7 +5,7 @@ //! docs for usage and details. mod conversions; -mod types; +pub mod types; use std::cell::RefCell; use std::fs::File; diff --git a/src/test/run-make-fulldeps/rustdoc-json/Makefile b/src/test/run-make-fulldeps/rustdoc-json/Makefile index b66bcb8035daa..ad517ae95ebb6 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/Makefile +++ b/src/test/run-make-fulldeps/rustdoc-json/Makefile @@ -3,3 +3,4 @@ tests: *.rs $(RUSTDOC) $< -o $(TMPDIR) --output-format json $(PYTHON) check_missing_items.py $(TMPDIR)/$(basename $<).json + $(PYTHON) compare.py $(basename $<).expected $(TMPDIR)/$(basename $<).json diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py index 7c3e389cfd468..c110477faa97c 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -1,5 +1,9 @@ #!/usr/bin/env python +# This test ensures that every ID in the produced json actually resolves to an item either in `index` or `paths`. +# It DOES NOT check that the structure of the produced json is actually in any way correct, +# for example an empty map would pass. + import sys import json diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py new file mode 100644 index 0000000000000..d9a2232888556 --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +# This script can check that an expected json blob is a subset of what actually gets produced. +# The comparison is independent of the value of IDs (which are unstable) and instead uses their +# relative ordering to check them against eachother by looking them up in their respective blob's +# `index` or `paths` mappings. To add a new test run `rustdoc --output-format json -o . yourtest.rs` +# and then create `yourtest.expected` by stripping unnecessary details from `yourtest.json`. + +import sys +import json +import types + +# Used instead of the string ids when used as references. +# Not used as keys in `index` or `paths` +class ID(str): + pass + + +class SubsetException(Exception): + def __init__(self, msg, trace): + self.msg = msg + self.trace = msg + super().__init__("{}: {}".format(trace, msg)) + + +def check_subset(expected_main, actual_main): + expected_index = expected_main["index"] + expected_paths = expected_main["paths"] + actual_index = actual_main["index"] + actual_paths = actual_main["paths"] + already_checked = set() + + def _check_subset(expected, actual, trace): + expected_type = type(expected) + actual_type = type(actual) + if expected_type is not actual_type: + raise SubsetException( + "expected type `{}`, got `{}`".format(expected_type, actual_type), trace + ) + if expected_type in (str, int, bool) and expected != actual: + raise SubsetException("expected `{}`, got: `{}`".format(expected, actual), trace) + if expected_type is dict: + for key in expected: + if key not in actual: + raise SubsetException("Key `{}` not found in output".format(key)) + new_trace = trace.copy() + new_trace.append(key) + _check_subset(expected[key], actual[key], new_trace) + elif expected_type is list: + expected_elements = len(expected) + actual_elements = len(actual) + if expected_elements != actual_elements: + raise SubsetException( + "Found {} items, expected {}".format(expected_elements, actual_elements) + ) + for expected, actual in zip(expected, actual): + new_trace = trace.copy() + new_trace.append(expected) + _check_subset(expected, actual, new_trace) + elif expected_type is ID and expected not in already_checked: + already_checked.add(expected) + _check_subset(expected_index.get(expected, {}), actual_index.get(actual, {}), trace) + _check_subset(expected_paths.get(expected, {}), actual_paths.get(actual, {}), trace) + + _check_subset(expected_main, actual_main, []) + + +def rustdoc_object_hook(obj): + # No need to convert paths, index and external_crates keys to ids, since + # they are the target of resolution, and never a source itself. + if "id" in obj: + obj["id"] = ID(id) + if "root" in obj: + obj["root"] = ID(id) + if "items" in obj: + obj["items"] = [ID(id) for id in obj["items"]] + if "variants" in obj: + obj["variants"] = [ID(id) for id in obj["variants"]] + if "fields" in obj: + obj["fields"] = [ID(id) for id in obj["fields"]] + if "impls" in obj: + obj["impls"] = [ID(id) for id in obj["impls"]] + if "implementors" in obj: + obj["implementors"] = [ID(id) for id in obj["implementors"]] + if "links" in obj: + obj["links"] = {s: ID(id) for s, id in obj["links"]} + if "variant_kind" in obj and obj["variant_kind"] == "struct": + obj["variant_inner"] = [ID(id) for id in obj["variant_inner"]] + return obj + + +def main(expected_fpath, actual_fpath): + print("checking that {} is a logical subset of {}".format(expected_fpath, actual_fpath)) + with open(expected_fpath) as expected_file: + expected_main = json.load(expected_file, object_hook=rustdoc_object_hook) + with open(actual_fpath) as actual_file: + actual_main = json.load(actual_file, object_hook=rustdoc_object_hook) + check_subset(expected_main, actual_main) + print("all checks passed") + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: `compare.py expected.json actual.json`") + main(sys.argv[1], sys.argv[2]) diff --git a/src/test/run-make-fulldeps/rustdoc-json/modules.expected b/src/test/run-make-fulldeps/rustdoc-json/modules.expected new file mode 100644 index 0000000000000..93b7e3a5cd922 --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/modules.expected @@ -0,0 +1,171 @@ +{ + "root": "0:0", + "version": null, + "includes_private": false, + "index": { + "0:6": { + "crate_id": 0, + "name": "bar", + "source": { + "filename": "modules.rs", + "begin": [ + 5, + 4 + ], + "end": [ + 7, + 5 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "module", + "inner": { + "is_crate": false, + "items": [ + "0:7" + ] + } + }, + "0:3": { + "crate_id": 0, + "name": null, + "source": { + "filename": "modules.rs", + "begin": [ + 1, + 0 + ], + "end": [ + 1, + 27 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "import", + "inner": { + "source": "foo::bar", + "name": "foobar", + "id": "0:6", + "glob": false + } + }, + "0:0": { + "crate_id": 0, + "name": "modules", + "source": { + "filename": "modules.rs", + "begin": [ + 1, + 0 + ], + "end": [ + 8, + 1 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "module", + "inner": { + "is_crate": true, + "items": [ + "0:3", + "0:4", + "0:5" + ] + } + }, + "0:5": { + "crate_id": 0, + "name": "foo", + "source": { + "filename": "modules.rs", + "begin": [ + 4, + 0 + ], + "end": [ + 8, + 1 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "module", + "inner": { + "is_crate": false, + "items": [ + "0:6" + ] + } + }, + "0:4": { + "crate_id": 0, + "name": null, + "source": { + "filename": "modules.rs", + "begin": [ + 2, + 0 + ], + "end": [ + 2, + 23 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "import", + "inner": { + "source": "foobar::baz", + "name": "baz", + "id": "0:7", + "glob": true + } + }, + "0:7": { + "crate_id": 0, + "name": "baz", + "source": { + "filename": "modules.rs", + "begin": [ + 6, + 8 + ], + "end": [ + 6, + 22 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "module", + "inner": { + "is_crate": false, + "items": [] + } + } + }, + "paths": {}, + "format_version": 1 +} diff --git a/src/test/run-make-fulldeps/rustdoc-json/modules.rs b/src/test/run-make-fulldeps/rustdoc-json/modules.rs new file mode 100644 index 0000000000000..b9fcac4760584 --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/modules.rs @@ -0,0 +1,8 @@ +pub use foo::bar as foobar; +pub use foobar::baz::*; + +pub mod foo { + pub mod bar { + pub mod baz {} + } +} diff --git a/src/test/run-make-fulldeps/rustdoc-json/structs.expected b/src/test/run-make-fulldeps/rustdoc-json/structs.expected new file mode 100644 index 0000000000000..aaaa7495dce56 --- /dev/null +++ b/src/test/run-make-fulldeps/rustdoc-json/structs.expected @@ -0,0 +1,277 @@ +{ + "root": "0:0", + "version": null, + "includes_private": false, + "index": { + "0:9": { + "crate_id": 0, + "name": "WithPrimitives", + "source": { + "filename": "structs.rs", + "begin": [ + 9, + 0 + ], + "end": [ + 12, + 1 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "plain", + "generics": { + "params": [ + { + "name": "'a", + "kind": "lifetime" + } + ], + "where_predicates": [] + }, + "fields_stripped": true, + "fields": [ + "0:11", + "0:12" + ] + } + }, + "0:0": { + "crate_id": 0, + "name": "structs", + "source": { + "filename": "structs.rs", + "begin": [ + 1, + 0 + ], + "end": [ + 17, + 1 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "module", + "inner": { + "is_crate": true, + "items": [ + "0:4", + "0:5", + "0:7", + "0:9", + "0:13" + ] + } + }, + "0:13": { + "crate_id": 0, + "name": "WithGenerics", + "source": { + "filename": "structs.rs", + "begin": [ + 14, + 0 + ], + "end": [ + 17, + 1 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "plain", + "generics": { + "params": [ + { + "name": "T", + "kind": { + "type": { + "bounds": [], + "default": null + } + } + }, + { + "name": "U", + "kind": { + "type": { + "bounds": [], + "default": null + } + } + } + ], + "where_predicates": [] + }, + "fields_stripped": true, + "fields": [ + "0:16", + "0:17" + ] + } + }, + "0:5": { + "crate_id": 0, + "name": "Tuple", + "source": { + "filename": "structs.rs", + "begin": [ + 5, + 0 + ], + "end": [ + 5, + 19 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "tuple", + "generics": { + "params": [], + "where_predicates": [] + }, + "fields_stripped": false, + "fields": [] + } + }, + "0:4": { + "crate_id": 0, + "name": "Normal", + "source": { + "filename": "structs.rs", + "begin": [ + 3, + 0 + ], + "end": [ + 3, + 20 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "plain", + "generics": { + "params": [], + "where_predicates": [] + }, + "fields_stripped": false, + "fields": [] + } + }, + "0:11": { + "crate_id": 0, + "name": "num", + "source": { + "filename": "structs.rs", + "begin": [ + 10, + 4 + ], + "end": [ + 10, + 12 + ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "primitive", + "inner": "u32" + } + }, + "0:12": { + "crate_id": 0, + "name": "s", + "source": { + "filename": "structs.rs", + "begin": [ + 11, + 4 + ], + "end": [ + 11, + 14 + ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "borrowed_ref", + "inner": { + "lifetime": "'a", + "mutable": false, + "type": { + "kind": "primitive", + "inner": "str" + } + } + } + }, + "0:7": { + "crate_id": 0, + "name": "Unit", + "source": { + "filename": "structs.rs", + "begin": [ + 7, + 0 + ], + "end": [ + 7, + 16 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "unit", + "generics": { + "params": [], + "where_predicates": [] + }, + "fields_stripped": false, + "fields": [] + } + } + }, + "paths": {}, + "format_version": 1 +} diff --git a/src/test/run-make-fulldeps/rustdoc-json/test.rs b/src/test/run-make-fulldeps/rustdoc-json/structs.rs similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/test.rs rename to src/test/run-make-fulldeps/rustdoc-json/structs.rs From ce4404a21a55c392bf2d24d31301caba6fd3e166 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 14 Aug 2020 09:29:58 -0500 Subject: [PATCH 05/20] Add restricted paths --- src/librustdoc/json/conversions.rs | 3 +-- src/librustdoc/json/types.rs | 9 +++++++-- .../rustdoc-json/check_missing_items.py | 6 +++--- src/test/run-make-fulldeps/rustdoc-json/compare.py | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index d0e38ceb8a023..8477de822f1c1 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -80,8 +80,7 @@ impl From for Visibility { Public => Visibility::Public, Inherited => Visibility::Default, Crate => Visibility::Crate, - Restricted(_did, _path) => unimplemented!(), - // Visibility::Restricted(did.into(), path.whole_name()), + Restricted(did, path) => Visibility::Restricted{parent: did.into(), path: path.whole_name()}, } } } diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index 6adc226e69bc0..08748d2c9e6e0 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -101,7 +101,12 @@ pub enum Visibility { /// public traits and variants of public enums. Default, Crate, - // FIXME(pineapple): add support for restricted paths + /// For `pub(in path)` visibility. `parent` is the module it's restricted to and `path` is how + /// that module was referenced (like `"super::super"` or `"crate::foo::bar"`). + Restricted { + parent: Id, + path: String, + }, } #[serde(rename_all = "snake_case")] @@ -440,7 +445,7 @@ pub struct Import { /// `use source as name;` pub name: String, /// The ID of the item being imported. - pub id: Option, // TODO when is this None? + pub id: Option, // FIXME is this actually ever None? /// Whether this import uses a glob: `use source::*;` pub glob: bool, } diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py index c110477faa97c..73f42384bd9b7 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -# This test ensures that every ID in the produced json actually resolves to an item either in `index` or `paths`. -# It DOES NOT check that the structure of the produced json is actually in any way correct, -# for example an empty map would pass. +# This test ensures that every ID in the produced json actually resolves to an item either in +# `index` or `paths`. It DOES NOT check that the structure of the produced json is actually in +# any way correct, for example an empty map would pass. import sys import json diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py index d9a2232888556..7e1eb830fc90e 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/compare.py +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -42,7 +42,7 @@ def _check_subset(expected, actual, trace): if expected_type is dict: for key in expected: if key not in actual: - raise SubsetException("Key `{}` not found in output".format(key)) + raise SubsetException("Key `{}` not found in output".format(key), trace) new_trace = trace.copy() new_trace.append(key) _check_subset(expected[key], actual[key], new_trace) @@ -51,7 +51,7 @@ def _check_subset(expected, actual, trace): actual_elements = len(actual) if expected_elements != actual_elements: raise SubsetException( - "Found {} items, expected {}".format(expected_elements, actual_elements) + "Found {} items, expected {}".format(expected_elements, actual_elements), trace ) for expected, actual in zip(expected, actual): new_trace = trace.copy() From 5673bc5641646707cd204e3954fb15d2191311be Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 14 Aug 2020 09:33:56 -0500 Subject: [PATCH 06/20] Format --- src/librustdoc/clean/types.rs | 4 ++-- src/librustdoc/json/conversions.rs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index fbd6853388cc3..32f6f43dc62c5 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -316,8 +316,8 @@ impl ItemEnum { | MacroItem(_) | ProcMacroItem(_) | PrimitiveItem(_) - | AssocConstItem(_,_) - | AssocTypeItem(_,_) + | AssocConstItem(_, _) + | AssocTypeItem(_, _) | StrippedItem(_) | KeywordItem(_) => [].iter(), } diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 8477de822f1c1..5d2c70f1b195f 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -80,7 +80,9 @@ impl From for Visibility { Public => Visibility::Public, Inherited => Visibility::Default, Crate => Visibility::Crate, - Restricted(did, path) => Visibility::Restricted{parent: did.into(), path: path.whole_name()}, + Restricted(did, path) => { + Visibility::Restricted { parent: did.into(), path: path.whole_name() } + } } } } From ad67aea5b7ce15bcd78275192d7cd332aa2d5444 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 14 Aug 2020 12:40:18 -0500 Subject: [PATCH 07/20] Fix: associated methods missing in output --- src/librustdoc/json/mod.rs | 57 ++++++++++--------- .../rustdoc-json/check_missing_items.py | 3 +- .../run-make-fulldeps/rustdoc-json/compare.py | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 58b5671e8668d..cf1db86187751 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -32,24 +32,8 @@ pub struct JsonRenderer { } impl JsonRenderer { - /// Inserts an item into the index. This should be used rather than directly calling insert on - /// the hashmap because certain items (traits and types) need to have their mappings for trait - /// implementations filled out before they're inserted. - fn insert(&self, item: clean::Item, cache: &Cache) { - let id = item.def_id; - let mut new_item: types::Item = item.into(); - if let types::ItemEnum::TraitItem(ref mut t) = new_item.inner { - t.implementors = self.get_trait_implementors(id, cache) - } else if let types::ItemEnum::StructItem(ref mut s) = new_item.inner { - s.impls = self.get_impls(id, cache) - } else if let types::ItemEnum::EnumItem(ref mut e) = new_item.inner { - e.impls = self.get_impls(id, cache) - } - self.index.borrow_mut().insert(id.into(), new_item); - } - fn get_trait_implementors( - &self, + &mut self, id: rustc_span::def_id::DefId, cache: &Cache, ) -> Vec { @@ -61,7 +45,7 @@ impl JsonRenderer { .iter() .map(|i| { let item = &i.impl_item; - self.insert(item.clone(), cache); + self.item(item.clone(), cache).unwrap(); item.def_id.into() }) .collect() @@ -69,7 +53,7 @@ impl JsonRenderer { .unwrap_or_default() } - fn get_impls(&self, id: rustc_span::def_id::DefId, cache: &Cache) -> Vec { + fn get_impls(&mut self, id: rustc_span::def_id::DefId, cache: &Cache) -> Vec { cache .impls .get(&id) @@ -79,7 +63,7 @@ impl JsonRenderer { .filter_map(|i| { let item = &i.impl_item; if item.def_id.is_local() { - self.insert(item.clone(), cache); + self.item(item.clone(), cache).unwrap(); Some(item.def_id.into()) } else { None @@ -90,14 +74,14 @@ impl JsonRenderer { .unwrap_or_default() } - fn get_trait_items(&self, cache: &Cache) -> Vec<(types::Id, types::Item)> { + fn get_trait_items(&mut self, cache: &Cache) -> Vec<(types::Id, types::Item)> { cache .traits .iter() .filter_map(|(id, trait_item)| { // only need to synthesize items for external traits if !id.is_local() { - trait_item.items.clone().into_iter().for_each(|i| self.insert(i, cache)); + trait_item.items.clone().into_iter().for_each(|i| self.item(i, cache).unwrap()); Some(( (*id).into(), types::Item { @@ -150,10 +134,24 @@ impl FormatRenderer for JsonRenderer { )) } + /// Inserts an item into the index. This should be used rather than directly calling insert on + /// the hashmap because certain items (traits and types) need to have their mappings for trait + /// implementations filled out before they're inserted. fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { - // Flatten items that recursively store other items by inserting them into the index + // Flatten items that recursively store other items item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()); - self.insert(item.clone(), cache); + + let id = item.def_id; + let mut new_item: types::Item = item.into(); + if let types::ItemEnum::TraitItem(ref mut t) = new_item.inner { + t.implementors = self.get_trait_implementors(id, cache) + } else if let types::ItemEnum::StructItem(ref mut s) = new_item.inner { + s.impls = self.get_impls(id, cache) + } else if let types::ItemEnum::EnumItem(ref mut e) = new_item.inner { + e.impls = self.get_impls(id, cache) + } + + self.index.borrow_mut().insert(id.into(), new_item); Ok(()) } @@ -166,15 +164,18 @@ impl FormatRenderer for JsonRenderer { use clean::types::ItemEnum::*; if let ModuleItem(m) = &item.inner { for item in &m.items { - match item.inner { + match &item.inner { // These don't have names so they don't get added to the output by default - ImportItem(_) => self.insert(item.clone(), cache), - ExternCrateItem(_, _) => self.insert(item.clone(), cache), + ImportItem(_) => self.item(item.clone(), cache).unwrap(), + ExternCrateItem(_, _) => self.item(item.clone(), cache).unwrap(), + ImplItem(i) => { + i.items.iter().for_each(|i| self.item(i.clone(), cache).unwrap()) + } _ => {} } } } - self.insert(item.clone(), cache); + self.item(item.clone(), cache).unwrap(); Ok(()) } diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py index 73f42384bd9b7..791436f9b0bb8 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -71,7 +71,8 @@ def check_type(ty): if ty["kind"] == "resolved_path": for bound in ty["inner"]["param_names"]: check_generic_bound(bound) - if args := ty["inner"]["args"]: + args = ty["inner"]["args"] + if args: if "angle_bracketed" in args: for arg in args["angle_bracketed"]["args"]: if "type" in arg: diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py index 7e1eb830fc90e..f039351c1c928 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/compare.py +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -68,7 +68,7 @@ def _check_subset(expected, actual, trace): def rustdoc_object_hook(obj): # No need to convert paths, index and external_crates keys to ids, since # they are the target of resolution, and never a source itself. - if "id" in obj: + if "id" in obj and obj["id"]: obj["id"] = ID(id) if "root" in obj: obj["root"] = ID(id) From e9af2d60bdd772f3a70be3c7197c706a86eb539c Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Mon, 7 Sep 2020 08:41:38 -0500 Subject: [PATCH 08/20] Ignore stripped items --- src/librustdoc/json/conversions.rs | 43 ++- src/librustdoc/json/mod.rs | 16 +- src/librustdoc/json/types.rs | 18 +- .../rustdoc-json/check_missing_items.py | 8 +- .../run-make-fulldeps/rustdoc-json/compare.py | 7 +- .../rustdoc-json/modules.expected | 171 -------- .../run-make-fulldeps/rustdoc-json/modules.rs | 8 - .../rustdoc-json/structs.expected | 365 +++++++++++++----- .../run-make-fulldeps/rustdoc-json/structs.rs | 4 +- 9 files changed, 340 insertions(+), 300 deletions(-) delete mode 100644 src/test/run-make-fulldeps/rustdoc-json/modules.expected delete mode 100644 src/test/run-make-fulldeps/rustdoc-json/modules.rs diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 5d2c70f1b195f..db3a3a6f0dcff 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -26,8 +26,10 @@ impl From for Item { deprecation, } = item; Item { + id: def_id.into(), crate_id: def_id.krate.as_u32(), name, + stripped: inner == clean::StrippedItem(_), source: source.into(), visibility: visibility.into(), docs: attrs.collapsed_doc_value().unwrap_or_default(), @@ -183,12 +185,21 @@ impl From for ItemEnum { } } +fn remove_stripped(items: &[clean::Item]) -> Vec { + items + .into_iter() + .filter_map(|i| { + if let clean::StrippedItem(_) = i.inner { + return None; + } + Some(i.def_id.into()) + }) + .collect() +} + impl From for Module { fn from(module: clean::Module) -> Self { - Module { - is_crate: module.is_crate, - items: module.items.into_iter().map(|i| i.def_id.into()).collect(), - } + Module { is_crate: module.is_crate, items: remove_stripped(&module.items) } } } @@ -199,8 +210,8 @@ impl From for Struct { struct_type: struct_type.into(), generics: generics.into(), fields_stripped, - fields: fields.into_iter().map(|i| i.def_id.into()).collect(), - impls: Vec::new(), // Added in JsonRenderer::insert + fields: remove_stripped(&fields), + impls: Vec::new(), // Added in JsonRenderer::item } } } @@ -212,8 +223,8 @@ impl From for Struct { struct_type: struct_type.into(), generics: generics.into(), fields_stripped, - fields: fields.into_iter().map(|i| i.def_id.into()).collect(), - impls: Vec::new(), // Added in JsonRenderer::insert + fields: remove_stripped(&fields), + impls: Vec::new(), // Added in JsonRenderer::item } } } @@ -399,10 +410,10 @@ impl From for Trait { Trait { is_auto: auto, is_unsafe: unsafety == rustc_hir::Unsafety::Unsafe, - items: items.into_iter().map(|i| i.def_id.into()).collect(), + items: remove_stripped(&items), generics: generics.into(), bounds: bounds.into_iter().map(Into::into).collect(), - implementors: Vec::new(), // Added in JsonRenderer::insert + implementors: Vec::new(), // Added in JsonRenderer::item } } } @@ -426,7 +437,7 @@ impl From for Impl { provided_trait_methods: provided_trait_methods.into_iter().collect(), trait_: trait_.map(Into::into), for_: for_.into(), - items: items.into_iter().map(|i| i.def_id.into()).collect(), + items: remove_stripped(&items), negative: polarity == Some(clean::ImplPolarity::Negative), synthetic, blanket_impl: blanket_impl.map(Into::into), @@ -465,8 +476,8 @@ impl From for Enum { Enum { generics: generics.into(), variants_stripped, - variants: variants.into_iter().map(|i| i.def_id.into()).collect(), - impls: Vec::new(), // Added in JsonRenderer::insert + variants: remove_stripped(&variants.into_iter().collect::>()), + impls: Vec::new(), // Added in JsonRenderer::item } } } @@ -478,7 +489,7 @@ impl From for Struct { struct_type: struct_type.into(), generics: Default::default(), fields_stripped, - fields: fields.into_iter().map(|i| i.def_id.into()).collect(), + fields: remove_stripped(&fields), impls: Vec::new(), } } @@ -490,7 +501,7 @@ impl From for Variant { match variant.kind { CLike => Variant::Plain, Tuple(t) => Variant::Tuple(t.into_iter().map(Into::into).collect()), - Struct(s) => Variant::Struct(s.fields.into_iter().map(|i| i.def_id.into()).collect()), + Struct(s) => Variant::Struct(remove_stripped(&s.fields)), } } } @@ -562,7 +573,7 @@ impl From for TraitAlias { fn from(alias: clean::TraitAlias) -> Self { TraitAlias { generics: alias.generics.into(), - bounds: alias.bounds.into_iter().map(Into::into).collect(), + params: alias.bounds.into_iter().map(Into::into).collect(), } } } diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index cf1db86187751..7002f2854ab4f 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -78,13 +78,14 @@ impl JsonRenderer { cache .traits .iter() - .filter_map(|(id, trait_item)| { + .filter_map(|(&id, trait_item)| { // only need to synthesize items for external traits if !id.is_local() { trait_item.items.clone().into_iter().for_each(|i| self.item(i, cache).unwrap()); Some(( - (*id).into(), + id.into(), types::Item { + id: id.into(), crate_id: id.krate.as_u32(), name: cache .paths @@ -138,8 +139,17 @@ impl FormatRenderer for JsonRenderer { /// the hashmap because certain items (traits and types) need to have their mappings for trait /// implementations filled out before they're inserted. fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { + if let clean::StrippedItem(_) = item.inner { + return Ok(()); + } + // Flatten items that recursively store other items - item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()); + item.inner.inner_items().for_each(|i| { + if let clean::StrippedItem(_) = i.inner { + } else { + self.item(i.clone(), cache).unwrap() + } + }); let id = item.def_id; let mut new_item: types::Item = item.into(); diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index 08748d2c9e6e0..cd8410b0af3f1 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -55,11 +55,16 @@ pub struct ItemSummary { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Item { + /// The unique identifier of this item. Can be used to find this item in various mappings. + pub id: Id, /// This can be used as a key to the `external_crates` map of [`Crate`] to see which crate /// this item came from. pub crate_id: u32, /// Some items such as impls don't have names. pub name: Option, + /// Whether this item is meant to be omitted from the generated documentation due to `#doc(hidden)`, + /// because it is private, or because it was inlined. + pub stripped: bool, /// The source location of this item (absent if it came from a macro expansion or inline /// assembly). pub source: Option, @@ -198,15 +203,15 @@ pub enum ItemEnum { FunctionItem(Function), - TypedefItem(Typedef), - OpaqueTyItem(OpaqueTy), - ConstantItem(Constant), - TraitItem(Trait), TraitAliasItem(TraitAlias), MethodItem(Method), ImplItem(Impl), + TypedefItem(Typedef), + OpaqueTyItem(OpaqueTy), + ConstantItem(Constant), + StaticItem(Static), /// `type`s from an extern block @@ -225,9 +230,6 @@ pub enum ItemEnum { bounds: Vec, default: Option, }, - - /// An item that has been stripped by a rustdoc pass - StrippedItem(Box), } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -418,7 +420,7 @@ pub struct Trait { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TraitAlias { pub generics: Generics, - pub bounds: Vec, + pub params: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py index 791436f9b0bb8..0004dc8fb147c 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py +++ b/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py @@ -89,7 +89,7 @@ def check_type(ty): for ty in args["parenthesized"]["inputs"]: check_type(ty) if args["parenthesized"]["output"]: - ckeck_ty(args["parenthesized"]["output"]) + check_type(args["parenthesized"]["output"]) if not valid_id(ty["inner"]["id"]): print("Type contained an invalid ID:", ty["inner"]["id"]) sys.exit(1) @@ -150,10 +150,14 @@ def check_type(ty): elif item["kind"] == "typedef": check_type(item["inner"]["type"]) check_generics(item["inner"]["generics"]) - elif item["kind"] in ("opaque_ty", "trait_alias"): + elif item["kind"] == "opaque_ty": check_generics(item["inner"]["generics"]) for bound in item["inner"]["bounds"]: check_generic_bound(bound) + elif item["kind"] == "trait_alias": + check_generics(item["inner"]["params"]) + for bound in item["inner"]["bounds"]: + check_generic_bound(bound) elif item["kind"] == "trait": check_generics(item["inner"]["generics"]) for bound in item["inner"]["bounds"]: diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py index f039351c1c928..aa1a06b2921e0 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/compare.py +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -69,9 +69,9 @@ def rustdoc_object_hook(obj): # No need to convert paths, index and external_crates keys to ids, since # they are the target of resolution, and never a source itself. if "id" in obj and obj["id"]: - obj["id"] = ID(id) + obj["id"] = ID(obj["id"]) if "root" in obj: - obj["root"] = ID(id) + obj["root"] = ID(obj["root"]) if "items" in obj: obj["items"] = [ID(id) for id in obj["items"]] if "variants" in obj: @@ -102,4 +102,5 @@ def main(expected_fpath, actual_fpath): if __name__ == "__main__": if len(sys.argv) < 3: print("Usage: `compare.py expected.json actual.json`") - main(sys.argv[1], sys.argv[2]) + else: + main(sys.argv[1], sys.argv[2]) diff --git a/src/test/run-make-fulldeps/rustdoc-json/modules.expected b/src/test/run-make-fulldeps/rustdoc-json/modules.expected deleted file mode 100644 index 93b7e3a5cd922..0000000000000 --- a/src/test/run-make-fulldeps/rustdoc-json/modules.expected +++ /dev/null @@ -1,171 +0,0 @@ -{ - "root": "0:0", - "version": null, - "includes_private": false, - "index": { - "0:6": { - "crate_id": 0, - "name": "bar", - "source": { - "filename": "modules.rs", - "begin": [ - 5, - 4 - ], - "end": [ - 7, - 5 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "module", - "inner": { - "is_crate": false, - "items": [ - "0:7" - ] - } - }, - "0:3": { - "crate_id": 0, - "name": null, - "source": { - "filename": "modules.rs", - "begin": [ - 1, - 0 - ], - "end": [ - 1, - 27 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "import", - "inner": { - "source": "foo::bar", - "name": "foobar", - "id": "0:6", - "glob": false - } - }, - "0:0": { - "crate_id": 0, - "name": "modules", - "source": { - "filename": "modules.rs", - "begin": [ - 1, - 0 - ], - "end": [ - 8, - 1 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "module", - "inner": { - "is_crate": true, - "items": [ - "0:3", - "0:4", - "0:5" - ] - } - }, - "0:5": { - "crate_id": 0, - "name": "foo", - "source": { - "filename": "modules.rs", - "begin": [ - 4, - 0 - ], - "end": [ - 8, - 1 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "module", - "inner": { - "is_crate": false, - "items": [ - "0:6" - ] - } - }, - "0:4": { - "crate_id": 0, - "name": null, - "source": { - "filename": "modules.rs", - "begin": [ - 2, - 0 - ], - "end": [ - 2, - 23 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "import", - "inner": { - "source": "foobar::baz", - "name": "baz", - "id": "0:7", - "glob": true - } - }, - "0:7": { - "crate_id": 0, - "name": "baz", - "source": { - "filename": "modules.rs", - "begin": [ - 6, - 8 - ], - "end": [ - 6, - 22 - ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "module", - "inner": { - "is_crate": false, - "items": [] - } - } - }, - "paths": {}, - "format_version": 1 -} diff --git a/src/test/run-make-fulldeps/rustdoc-json/modules.rs b/src/test/run-make-fulldeps/rustdoc-json/modules.rs deleted file mode 100644 index b9fcac4760584..0000000000000 --- a/src/test/run-make-fulldeps/rustdoc-json/modules.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub use foo::bar as foobar; -pub use foobar::baz::*; - -pub mod foo { - pub mod bar { - pub mod baz {} - } -} diff --git a/src/test/run-make-fulldeps/rustdoc-json/structs.expected b/src/test/run-make-fulldeps/rustdoc-json/structs.expected index aaaa7495dce56..45b23534bc77b 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/structs.expected +++ b/src/test/run-make-fulldeps/rustdoc-json/structs.expected @@ -4,6 +4,113 @@ "includes_private": false, "index": { "0:9": { + "crate_id": 0, + "name": "Unit", + "source": { + "filename": "structs.rs", + "begin": [ + 7, + 0 + ], + "end": [ + 7, + 16 + ] + }, + "visibility": "public", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct", + "inner": { + "struct_type": "unit", + "generics": { + "params": [], + "where_predicates": [] + }, + "fields_stripped": false, + "fields": [] + } + }, + "0:8": { + "crate_id": 0, + "name": "1", + "source": { + "filename": "structs.rs", + "begin": [ + 5, + 22 + ], + "end": [ + 5, + 28 + ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "resolved_path", + "inner": { + "name": "String", + "id": "5:5035", + "args": { + "angle_bracketed": { + "args": [], + "bindings": [] + } + }, + "param_names": [] + } + } + }, + "0:18": { + "crate_id": 0, + "name": "stuff", + "source": { + "filename": "structs.rs", + "begin": [ + 15, + 4 + ], + "end": [ + 15, + 17 + ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "resolved_path", + "inner": { + "name": "Vec", + "id": "5:4322", + "args": { + "angle_bracketed": { + "args": [ + { + "type": { + "kind": "generic", + "inner": "T" + } + } + ], + "bindings": [] + } + }, + "param_names": [] + } + } + }, + "0:11": { "crate_id": 0, "name": "WithPrimitives", "source": { @@ -36,43 +143,92 @@ }, "fields_stripped": true, "fields": [ - "0:11", - "0:12" + "0:13", + "0:14" ] } }, - "0:0": { + "0:14": { "crate_id": 0, - "name": "structs", + "name": "s", "source": { "filename": "structs.rs", "begin": [ - 1, - 0 + 11, + 4 ], "end": [ - 17, - 1 + 11, + 14 ] }, - "visibility": "public", + "visibility": "default", "docs": "", "links": {}, "attrs": [], "deprecation": null, - "kind": "module", + "kind": "struct_field", "inner": { - "is_crate": true, - "items": [ - "0:4", - "0:5", - "0:7", - "0:9", - "0:13" + "kind": "borrowed_ref", + "inner": { + "lifetime": "'a", + "mutable": false, + "type": { + "kind": "primitive", + "inner": "str" + } + } + } + }, + "0:19": { + "crate_id": 0, + "name": "things", + "source": { + "filename": "structs.rs", + "begin": [ + 16, + 4 + ], + "end": [ + 16, + 25 ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "resolved_path", + "inner": { + "name": "HashMap", + "id": "1:6600", + "args": { + "angle_bracketed": { + "args": [ + { + "type": { + "kind": "generic", + "inner": "U" + } + }, + { + "type": { + "kind": "generic", + "inner": "U" + } + } + ], + "bindings": [] + } + }, + "param_names": [] + } } }, - "0:13": { + "0:15": { "crate_id": 0, "name": "WithGenerics", "source": { @@ -119,23 +275,23 @@ }, "fields_stripped": true, "fields": [ - "0:16", - "0:17" + "0:18", + "0:19" ] } }, - "0:5": { + "0:0": { "crate_id": 0, - "name": "Tuple", + "name": "structs", "source": { "filename": "structs.rs", "begin": [ - 5, + 1, 0 ], "end": [ - 5, - 19 + 17, + 1 ] }, "visibility": "public", @@ -143,48 +299,19 @@ "links": {}, "attrs": [], "deprecation": null, - "kind": "struct", + "kind": "module", "inner": { - "struct_type": "tuple", - "generics": { - "params": [], - "where_predicates": [] - }, - "fields_stripped": false, - "fields": [] - } - }, - "0:4": { - "crate_id": 0, - "name": "Normal", - "source": { - "filename": "structs.rs", - "begin": [ - 3, - 0 - ], - "end": [ - 3, - 20 + "is_crate": true, + "items": [ + "0:4", + "0:5", + "0:9", + "0:11", + "0:15" ] - }, - "visibility": "public", - "docs": "", - "links": {}, - "attrs": [], - "deprecation": null, - "kind": "struct", - "inner": { - "struct_type": "plain", - "generics": { - "params": [], - "where_predicates": [] - }, - "fields_stripped": false, - "fields": [] } }, - "0:11": { + "0:13": { "crate_id": 0, "name": "num", "source": { @@ -209,50 +336,51 @@ "inner": "u32" } }, - "0:12": { + "0:5": { "crate_id": 0, - "name": "s", + "name": "Tuple", "source": { "filename": "structs.rs", "begin": [ - 11, - 4 + 5, + 0 ], "end": [ - 11, - 14 + 5, + 30 ] }, - "visibility": "default", + "visibility": "public", "docs": "", "links": {}, "attrs": [], "deprecation": null, - "kind": "struct_field", + "kind": "struct", "inner": { - "kind": "borrowed_ref", - "inner": { - "lifetime": "'a", - "mutable": false, - "type": { - "kind": "primitive", - "inner": "str" - } - } + "struct_type": "tuple", + "generics": { + "params": [], + "where_predicates": [] + }, + "fields_stripped": true, + "fields": [ + "0:7", + "0:8" + ] } }, - "0:7": { + "0:4": { "crate_id": 0, - "name": "Unit", + "name": "PlainEmpty", "source": { "filename": "structs.rs", "begin": [ - 7, + 3, 0 ], "end": [ - 7, - 16 + 3, + 24 ] }, "visibility": "public", @@ -262,7 +390,7 @@ "deprecation": null, "kind": "struct", "inner": { - "struct_type": "unit", + "struct_type": "plain", "generics": { "params": [], "where_predicates": [] @@ -270,8 +398,71 @@ "fields_stripped": false, "fields": [] } + }, + "0:7": { + "crate_id": 0, + "name": "0", + "source": { + "filename": "structs.rs", + "begin": [ + 5, + 17 + ], + "end": [ + 5, + 20 + ] + }, + "visibility": "default", + "docs": "", + "links": {}, + "attrs": [], + "deprecation": null, + "kind": "struct_field", + "inner": { + "kind": "primitive", + "inner": "u32" + } + } + }, + "paths": { + "5:4322": { + "crate_id": 5, + "path": [ + "alloc", + "vec", + "Vec" + ], + "kind": "struct" + }, + "5:5035": { + "crate_id": 5, + "path": [ + "alloc", + "string", + "String" + ], + "kind": "struct" + }, + "1:6600": { + "crate_id": 1, + "path": [ + "std", + "collections", + "hash", + "map", + "HashMap" + ], + "kind": "struct" + } + }, + "external_crates": { + "1": { + "name": "std" + }, + "5": { + "name": "alloc" } }, - "paths": {}, "format_version": 1 } diff --git a/src/test/run-make-fulldeps/rustdoc-json/structs.rs b/src/test/run-make-fulldeps/rustdoc-json/structs.rs index e467f0d4cccc2..43fc4743503aa 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/structs.rs +++ b/src/test/run-make-fulldeps/rustdoc-json/structs.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; -pub struct Normal {} +pub struct PlainEmpty {} -pub struct Tuple(); +pub struct Tuple(u32, String); pub struct Unit; From 5ff33ca9805655e859a9c95ef4bc170cc693a631 Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Fri, 11 Sep 2020 08:18:08 -0500 Subject: [PATCH 09/20] Mark stripped items removing them creates dangling references --- src/librustdoc/json/conversions.rs | 38 +++++++++++++----------------- src/librustdoc/json/mod.rs | 12 ++-------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index db3a3a6f0dcff..458f5c3042dae 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -29,7 +29,10 @@ impl From for Item { id: def_id.into(), crate_id: def_id.krate.as_u32(), name, - stripped: inner == clean::StrippedItem(_), + stripped: match inner { + clean::StrippedItem(_) => true, + _ => false, + }, source: source.into(), visibility: visibility.into(), docs: attrs.collapsed_doc_value().unwrap_or_default(), @@ -179,27 +182,18 @@ impl From for ItemEnum { bounds: g.into_iter().map(Into::into).collect(), default: t.map(Into::into), }, - StrippedItem(inner) => ItemEnum::StrippedItem(Box::new((*inner).into())), + StrippedItem(inner) => (*inner).into(), _ => panic!("{:?} is not supported for JSON output", item), } } } -fn remove_stripped(items: &[clean::Item]) -> Vec { - items - .into_iter() - .filter_map(|i| { - if let clean::StrippedItem(_) = i.inner { - return None; - } - Some(i.def_id.into()) - }) - .collect() -} - impl From for Module { fn from(module: clean::Module) -> Self { - Module { is_crate: module.is_crate, items: remove_stripped(&module.items) } + Module { + is_crate: module.is_crate, + items: module.items.into_iter().map(|i| i.def_id.into()).collect(), + } } } @@ -210,7 +204,7 @@ impl From for Struct { struct_type: struct_type.into(), generics: generics.into(), fields_stripped, - fields: remove_stripped(&fields), + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), impls: Vec::new(), // Added in JsonRenderer::item } } @@ -223,7 +217,7 @@ impl From for Struct { struct_type: struct_type.into(), generics: generics.into(), fields_stripped, - fields: remove_stripped(&fields), + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), impls: Vec::new(), // Added in JsonRenderer::item } } @@ -410,7 +404,7 @@ impl From for Trait { Trait { is_auto: auto, is_unsafe: unsafety == rustc_hir::Unsafety::Unsafe, - items: remove_stripped(&items), + items: items.into_iter().map(|i| i.def_id.into()).collect(), generics: generics.into(), bounds: bounds.into_iter().map(Into::into).collect(), implementors: Vec::new(), // Added in JsonRenderer::item @@ -437,7 +431,7 @@ impl From for Impl { provided_trait_methods: provided_trait_methods.into_iter().collect(), trait_: trait_.map(Into::into), for_: for_.into(), - items: remove_stripped(&items), + items: items.into_iter().map(|i| i.def_id.into()).collect(), negative: polarity == Some(clean::ImplPolarity::Negative), synthetic, blanket_impl: blanket_impl.map(Into::into), @@ -476,7 +470,7 @@ impl From for Enum { Enum { generics: generics.into(), variants_stripped, - variants: remove_stripped(&variants.into_iter().collect::>()), + variants: variants.into_iter().map(|i| i.def_id.into()).collect(), impls: Vec::new(), // Added in JsonRenderer::item } } @@ -489,7 +483,7 @@ impl From for Struct { struct_type: struct_type.into(), generics: Default::default(), fields_stripped, - fields: remove_stripped(&fields), + fields: fields.into_iter().map(|i| i.def_id.into()).collect(), impls: Vec::new(), } } @@ -501,7 +495,7 @@ impl From for Variant { match variant.kind { CLike => Variant::Plain, Tuple(t) => Variant::Tuple(t.into_iter().map(Into::into).collect()), - Struct(s) => Variant::Struct(remove_stripped(&s.fields)), + Struct(s) => Variant::Struct(s.fields.into_iter().map(|i| i.def_id.into()).collect()), } } } diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 7002f2854ab4f..d67122ed311e8 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -99,6 +99,7 @@ impl JsonRenderer { .0 .last() .map(Clone::clone), + stripped: false, visibility: types::Visibility::Public, kind: types::ItemKind::Trait, inner: types::ItemEnum::TraitItem(trait_item.clone().into()), @@ -139,17 +140,8 @@ impl FormatRenderer for JsonRenderer { /// the hashmap because certain items (traits and types) need to have their mappings for trait /// implementations filled out before they're inserted. fn item(&mut self, item: clean::Item, cache: &Cache) -> Result<(), Error> { - if let clean::StrippedItem(_) = item.inner { - return Ok(()); - } - // Flatten items that recursively store other items - item.inner.inner_items().for_each(|i| { - if let clean::StrippedItem(_) = i.inner { - } else { - self.item(i.clone(), cache).unwrap() - } - }); + item.inner.inner_items().for_each(|i| self.item(i.clone(), cache).unwrap()); let id = item.def_id; let mut new_item: types::Item = item.into(); From d99ec1d790c9d84e24b7f39147416b671c34d72e Mon Sep 17 00:00:00 2001 From: Joseph Ryan Date: Wed, 14 Oct 2020 18:57:12 -0500 Subject: [PATCH 10/20] Fix tests and update conversions --- src/librustdoc/clean/types.rs | 2 +- src/librustdoc/json/conversions.rs | 22 +++++++++---------- .../run-make-fulldeps/rustdoc-json/compare.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 32f6f43dc62c5..3a86e0959cdf1 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -302,7 +302,7 @@ impl ItemEnum { | ImportItem(_) | FunctionItem(_) | TypedefItem(_, _) - | OpaqueTyItem(_, _) + | OpaqueTyItem(_) | StaticItem(_) | ConstantItem(_) | TraitAliasItem(_) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 458f5c3042dae..2eb2a95e2b3e2 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -39,7 +39,7 @@ impl From for Item { links: attrs .links .into_iter() - .filter_map(|(a, b, _)| b.map(|b| (a, b.into()))) + .filter_map(|clean::ItemLink { link, did, .. }| did.map(|did| (link, did.into()))) .collect(), attrs: attrs .other_attrs @@ -173,7 +173,7 @@ impl From for ItemEnum { ForeignStaticItem(s) => ItemEnum::StaticItem(s.into()), ForeignTypeItem => ItemEnum::ForeignTypeItem, TypedefItem(t, _) => ItemEnum::TypedefItem(t.into()), - OpaqueTyItem(t, _) => ItemEnum::OpaqueTyItem(t.into()), + OpaqueTyItem(t) => ItemEnum::OpaqueTyItem(t.into()), ConstantItem(c) => ItemEnum::ConstantItem(c.into()), MacroItem(m) => ItemEnum::MacroItem(m.source), ProcMacroItem(m) => ItemEnum::ProcMacroItem(m.into()), @@ -502,18 +502,18 @@ impl From for Variant { impl From for Import { fn from(import: clean::Import) -> Self { - use clean::Import::*; - match import { - Simple(s, i) => Import { - source: i.path.whole_name(), + use clean::ImportKind::*; + match import.kind { + Simple(s) => Import { + source: import.source.path.whole_name(), name: s, - id: i.did.map(Into::into), + id: import.source.did.map(Into::into), glob: false, }, - Glob(i) => Import { - source: i.path.whole_name(), - name: i.path.last_name().to_string(), - id: i.did.map(Into::into), + Glob => Import { + source: import.source.path.whole_name(), + name: import.source.path.last_name().to_string(), + id: import.source.did.map(Into::into), glob: true, }, } diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py index aa1a06b2921e0..85aa064aa6722 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/compare.py +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -62,7 +62,7 @@ def _check_subset(expected, actual, trace): _check_subset(expected_index.get(expected, {}), actual_index.get(actual, {}), trace) _check_subset(expected_paths.get(expected, {}), actual_paths.get(actual, {}), trace) - _check_subset(expected_main, actual_main, []) + _check_subset(expected_main["root"], actual_main["root"], []) def rustdoc_object_hook(obj): From 9ca0856f40070386627163f68e692d267df511c9 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 14:36:50 -0500 Subject: [PATCH 11/20] Don't panic if JSON backend fails to create a file --- src/librustdoc/json/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index d67122ed311e8..941f661874dee 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -227,7 +227,8 @@ impl FormatRenderer for JsonRenderer { let mut p = self.out_path.clone(); p.push(output.index.get(&output.root).unwrap().name.clone().unwrap()); p.set_extension("json"); - serde_json::ser::to_writer_pretty(&File::create(p).unwrap(), &output).unwrap(); + let file = File::create(&p).map_err(|error| Error { error: error.to_string(), file: p })?; + serde_json::ser::to_writer_pretty(&file, &output).unwrap(); Ok(()) } From 99d4784d3e71b3d1e17e3ba4ae9e7e3c48b5fecd Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 14:53:15 -0500 Subject: [PATCH 12/20] Fix attribute error in JSON testsuite --- src/test/run-make-fulldeps/rustdoc-json/compare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/run-make-fulldeps/rustdoc-json/compare.py index 85aa064aa6722..5daf8903e2030 100644 --- a/src/test/run-make-fulldeps/rustdoc-json/compare.py +++ b/src/test/run-make-fulldeps/rustdoc-json/compare.py @@ -6,6 +6,7 @@ # `index` or `paths` mappings. To add a new test run `rustdoc --output-format json -o . yourtest.rs` # and then create `yourtest.expected` by stripping unnecessary details from `yourtest.json`. +import copy import sys import json import types @@ -43,7 +44,7 @@ def _check_subset(expected, actual, trace): for key in expected: if key not in actual: raise SubsetException("Key `{}` not found in output".format(key), trace) - new_trace = trace.copy() + new_trace = copy.deepcopy(trace) new_trace.append(key) _check_subset(expected[key], actual[key], new_trace) elif expected_type is list: @@ -54,7 +55,7 @@ def _check_subset(expected, actual, trace): "Found {} items, expected {}".format(expected_elements, actual_elements), trace ) for expected, actual in zip(expected, actual): - new_trace = trace.copy() + new_trace = copy.deepcopy(trace) new_trace.append(expected) _check_subset(expected, actual, new_trace) elif expected_type is ID and expected not in already_checked: From a66a97ac588f26350cab886904ddbe70ce17d339 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 14:56:33 -0500 Subject: [PATCH 13/20] Move rustdoc-json to rustdoc/ This way it doesn't have to build rustc twice. Eventually it should probably get its own suite, like rustdoc-js, but that can be fixed in a follow-up. --- src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/Makefile | 0 .../rustdoc-json/check_missing_items.py | 0 src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/compare.py | 0 .../{run-make-fulldeps => rustdoc}/rustdoc-json/structs.expected | 0 src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/structs.rs | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/Makefile (100%) rename src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/check_missing_items.py (100%) rename src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/compare.py (100%) rename src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/structs.expected (100%) rename src/test/{run-make-fulldeps => rustdoc}/rustdoc-json/structs.rs (100%) diff --git a/src/test/run-make-fulldeps/rustdoc-json/Makefile b/src/test/rustdoc/rustdoc-json/Makefile similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/Makefile rename to src/test/rustdoc/rustdoc-json/Makefile diff --git a/src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py b/src/test/rustdoc/rustdoc-json/check_missing_items.py similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/check_missing_items.py rename to src/test/rustdoc/rustdoc-json/check_missing_items.py diff --git a/src/test/run-make-fulldeps/rustdoc-json/compare.py b/src/test/rustdoc/rustdoc-json/compare.py similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/compare.py rename to src/test/rustdoc/rustdoc-json/compare.py diff --git a/src/test/run-make-fulldeps/rustdoc-json/structs.expected b/src/test/rustdoc/rustdoc-json/structs.expected similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/structs.expected rename to src/test/rustdoc/rustdoc-json/structs.expected diff --git a/src/test/run-make-fulldeps/rustdoc-json/structs.rs b/src/test/rustdoc/rustdoc-json/structs.rs similarity index 100% rename from src/test/run-make-fulldeps/rustdoc-json/structs.rs rename to src/test/rustdoc/rustdoc-json/structs.rs From 338aceea3ca5573515cd78553dd4b8cae5285cad Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 15:10:09 -0500 Subject: [PATCH 14/20] Small cleanups --- src/librustdoc/clean/types.rs | 2 +- src/librustdoc/json/conversions.rs | 4 +++- src/librustdoc/json/types.rs | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 3a86e0959cdf1..3ad25e72f53c8 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -1613,7 +1613,7 @@ impl Path { self.segments.last().expect("segments were empty").name.as_str() } - pub fn whole_name(&self) -> String { + crate fn whole_name(&self) -> String { String::from(if self.global { "::" } else { "" }) + &self.segments.iter().map(|s| s.name.clone()).collect::>().join("::") } diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 2eb2a95e2b3e2..1b3534df2342c 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -183,7 +183,9 @@ impl From for ItemEnum { default: t.map(Into::into), }, StrippedItem(inner) => (*inner).into(), - _ => panic!("{:?} is not supported for JSON output", item), + PrimitiveItem(_) | KeywordItem(_) => { + panic!("{:?} is not supported for JSON output", item) + } } } } diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index cd8410b0af3f1..91c17af5144ef 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -73,7 +73,7 @@ pub struct Item { pub visibility: Visibility, /// The full markdown docstring of this item. pub docs: String, - /// This mapping resolves [intradoc links](https://github.com/rust-lang/rfcs/blob/master/text/1946-intra-rustdoc-links.md) from the docstring to their IDs + /// This mapping resolves [intra-doc links](https://github.com/rust-lang/rfcs/blob/master/text/1946-intra-rustdoc-links.md) from the docstring to their IDs pub links: FxHashMap, /// Stringified versions of the attributes on this item (e.g. `"#[inline]"`) pub attrs: Vec, @@ -84,7 +84,7 @@ pub struct Item { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Span { - /// The path to the source file for this span relative to the crate root. + /// The path to the source file for this span relative to the path `rustdoc` was invoked with. pub filename: PathBuf, /// Zero indexed Line and Column of the first character of the `Span` pub begin: (usize, usize), From 47acf9ae5e92e29d6531fae2a266172cd9d1c1a4 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 15:15:42 -0500 Subject: [PATCH 15/20] Don't prettify json before printing This fully halves the size of the emitted JSON. --- src/librustdoc/json/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 941f661874dee..b102aacbb0030 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -228,7 +228,7 @@ impl FormatRenderer for JsonRenderer { p.push(output.index.get(&output.root).unwrap().name.clone().unwrap()); p.set_extension("json"); let file = File::create(&p).map_err(|error| Error { error: error.to_string(), file: p })?; - serde_json::ser::to_writer_pretty(&file, &output).unwrap(); + serde_json::ser::to_writer(&file, &output).unwrap(); Ok(()) } From cc853759cdeeba2dc19295524904ed69d2291e32 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 15:33:30 -0500 Subject: [PATCH 16/20] Add comments --- src/librustdoc/json/types.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index 91c17af5144ef..d795d64e7732f 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -224,10 +224,12 @@ pub enum ItemEnum { AssocConstItem { #[serde(rename = "type")] type_: Type, + /// e.g. `const X: usize = 5;` default: Option, }, AssocTypeItem { bounds: Vec, + /// e.g. `type X = usize;` default: Option, }, } From 9147fd7aae13fe3d37cf843c0041b25c67acd458 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 15:37:21 -0500 Subject: [PATCH 17/20] [BREAKING CHANGE] rename version -> crate_version --- src/librustdoc/json/mod.rs | 2 +- src/librustdoc/json/types.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index b102aacbb0030..d1a2c86683ca2 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -191,7 +191,7 @@ impl FormatRenderer for JsonRenderer { index.extend(self.get_trait_items(cache)); let output = types::Crate { root: types::Id(String::from("0:0")), - version: krate.version.clone(), + crate_version: krate.version.clone(), includes_private: cache.document_private, index, paths: cache diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index d795d64e7732f..f08c931bf7cb2 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -16,7 +16,7 @@ pub struct Crate { /// The id of the root [`Module`] item of the local crate. pub root: Id, /// The version string given to `--crate-version`, if any. - pub version: Option, + pub crate_version: Option, /// Whether or not the output includes private items. pub includes_private: bool, /// A collection of all items in the local crate as well as some external traits and their From 463e8b8c355d29ae0b60b3ea175bfc6774ece4df Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 15:47:22 -0500 Subject: [PATCH 18/20] [BREAKING CHANGE] rename source -> span --- src/librustdoc/json/conversions.rs | 4 ++-- src/librustdoc/json/types.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 1b3534df2342c..935c8bb1f4ef7 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -507,13 +507,13 @@ impl From for Import { use clean::ImportKind::*; match import.kind { Simple(s) => Import { - source: import.source.path.whole_name(), + span: import.source.path.whole_name(), name: s, id: import.source.did.map(Into::into), glob: false, }, Glob => Import { - source: import.source.path.whole_name(), + span: import.source.path.whole_name(), name: import.source.path.last_name().to_string(), id: import.source.did.map(Into::into), glob: true, diff --git a/src/librustdoc/json/types.rs b/src/librustdoc/json/types.rs index f08c931bf7cb2..62705affafce1 100644 --- a/src/librustdoc/json/types.rs +++ b/src/librustdoc/json/types.rs @@ -444,7 +444,7 @@ pub struct Impl { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Import { /// The full path being imported. - pub source: String, + pub span: String, /// May be different from the last segment of `source` when renaming imports: /// `use source as name;` pub name: String, From ec7a660c87092153653f407b402fd757c2a711e6 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 16:05:09 -0500 Subject: [PATCH 19/20] Use exhaustive matches --- src/librustdoc/json/conversions.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 935c8bb1f4ef7..7ba3068faa892 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -74,7 +74,8 @@ impl From for Option { impl From for Deprecation { fn from(deprecation: clean::Deprecation) -> Self { - Deprecation { since: deprecation.since, note: deprecation.note } + let clean::Deprecation { since, note, is_since_rustc_version: _ } = deprecation; + Deprecation { since, note } } } From 7ad40b5ad519cf8d80d98b8d8d39e1c368369ff0 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 8 Nov 2020 16:10:02 -0500 Subject: [PATCH 20/20] Don't qualify imports for DefId --- src/librustdoc/json/conversions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs index 7ba3068faa892..740e494e9f10a 100644 --- a/src/librustdoc/json/conversions.rs +++ b/src/librustdoc/json/conversions.rs @@ -5,7 +5,7 @@ use std::convert::From; use rustc_ast::ast; -use rustc_span::def_id; +use rustc_span::def_id::DefId; use crate::clean; use crate::doctree; @@ -145,8 +145,8 @@ impl From for TypeBindingKind { } } -impl From for Id { - fn from(did: def_id::DefId) -> Self { +impl From for Id { + fn from(did: DefId) -> Self { Id(format!("{}:{}", did.krate.as_u32(), u32::from(did.index))) } }