From 27d15b8e152d591132dae72f20918e9002f760ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 07:24:00 +0000 Subject: [PATCH 1/5] build(deps): bump the xunit group with 1 update (#9822) Bumps the xunit group with 1 update: [Verify.Xunit](https://github.com/VerifyTests/Verify). Updates `Verify.Xunit` from 23.5.2 to 23.6.0 - [Release notes](https://github.com/VerifyTests/Verify/releases) - [Commits](https://github.com/VerifyTests/Verify/compare/23.5.2...23.6.0) --- updated-dependencies: - dependency-name: Verify.Xunit dependency-type: direct:production update-type: version-update:semver-minor dependency-group: xunit ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 05e0c604aa2..29cf2b06999 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -37,7 +37,7 @@ - + From e4d06da4d68ca9d49ad9488a1a124974a885d1e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 07:41:58 +0000 Subject: [PATCH 2/5] build(deps): bump HtmlAgilityPack from 1.11.59 to 1.11.60 (#9823) Bumps [HtmlAgilityPack](https://github.com/zzzprojects/html-agility-pack) from 1.11.59 to 1.11.60. - [Release notes](https://github.com/zzzprojects/html-agility-pack/releases) - [Commits](https://github.com/zzzprojects/html-agility-pack/compare/v1.11.59...v1.11.60) --- updated-dependencies: - dependency-name: HtmlAgilityPack dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 29cf2b06999..8a77e05dd75 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ true - + From 009dd5e4022d61b1b1db1aef2fede3c9922d0d26 Mon Sep 17 00:00:00 2001 From: Pavel Voronin Date: Mon, 1 Apr 2024 04:16:37 +0200 Subject: [PATCH 3/5] chore: Add a warning about relative path resolution. (#9828) Add a warning about relative path resolution. It's not obvious whether paths are resolved based on current working directory where docfx is called or based on location of `docfx.json` file. It's worth to be explicit about it in the documentation. --- docs/reference/docfx-json-reference.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/docfx-json-reference.md b/docs/reference/docfx-json-reference.md index 35dd78001dc..2c02d6366d0 100644 --- a/docs/reference/docfx-json-reference.md +++ b/docs/reference/docfx-json-reference.md @@ -2,6 +2,9 @@ The `docfx.json` file indicates that the directory is the root of a docfx project. +> [!IMPORTANT] +> All relative paths specified in config file are relative to the location of `docfx.json` + ```json { "build": { }, From 2799e6f77248dcae13ca00ed39a42ea86b7d302b Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:18:31 +0900 Subject: [PATCH 4/5] feat: Add DLL references support for file-based API metadata generation (#9825) feat: Add DLL references support for file-based metadata generation --- src/Docfx.Dotnet/CompilationHelper.cs | 16 ++++----- src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs | 18 +++++++--- .../GenerateMetadataFromCSUnitTest.cs | 30 ++++++++++++++-- .../GenerateMetadataFromVBUnitTest.cs | 36 +++++++++++++++++-- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/Docfx.Dotnet/CompilationHelper.cs b/src/Docfx.Dotnet/CompilationHelper.cs index 8de142094d1..405e416407c 100644 --- a/src/Docfx.Dotnet/CompilationHelper.cs +++ b/src/Docfx.Dotnet/CompilationHelper.cs @@ -54,7 +54,7 @@ public static bool CheckDiagnostics(this Compilation compilation, bool errorAsWa return errorCount > 0; } - public static Compilation CreateCompilationFromCSharpFiles(IEnumerable files, IDictionary msbuildProperties) + public static Compilation CreateCompilationFromCSharpFiles(IEnumerable files, IDictionary msbuildProperties, MetadataReference[] references) { var parserOption = GetCSharpParseOptions(msbuildProperties); var syntaxTrees = files.Select(path => CS.CSharpSyntaxTree.ParseText(File.ReadAllText(path), parserOption, path: path)); @@ -63,7 +63,7 @@ public static Compilation CreateCompilationFromCSharpFiles(IEnumerable f assemblyName: null, options: new CS.CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, xmlReferenceResolver: XmlFileResolver.Default), syntaxTrees: syntaxTrees, - references: GetDefaultMetadataReferences("C#")); + references: GetDefaultMetadataReferences("C#").Concat(references)); } public static Compilation CreateCompilationFromCSharpCode(string code, IDictionary msbuildProperties, string? name = null, params MetadataReference[] references) @@ -75,10 +75,10 @@ public static Compilation CreateCompilationFromCSharpCode(string code, IDictiona name, options: new CS.CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, xmlReferenceResolver: XmlFileResolver.Default), syntaxTrees: [syntaxTree], - references: GetDefaultMetadataReferences("C#").Concat(references)); + references: GetDefaultMetadataReferences("C#").Concat(references ?? [])); } - public static Compilation CreateCompilationFromVBFiles(IEnumerable files, IDictionary msbuildProperties) + public static Compilation CreateCompilationFromVBFiles(IEnumerable files, IDictionary msbuildProperties, MetadataReference[] references) { var parserOption = GetVisualBasicParseOptions(msbuildProperties); var syntaxTrees = files.Select(path => VB.VisualBasicSyntaxTree.ParseText(File.ReadAllText(path), parserOption, path: path)); @@ -87,7 +87,7 @@ public static Compilation CreateCompilationFromVBFiles(IEnumerable files assemblyName: null, options: new VB.VisualBasicCompilationOptions(OutputKind.DynamicallyLinkedLibrary, globalImports: GetVBGlobalImports(), xmlReferenceResolver: XmlFileResolver.Default), syntaxTrees: syntaxTrees, - references: GetDefaultMetadataReferences("VB")); + references: GetDefaultMetadataReferences("VB").Concat(references)); } public static Compilation CreateCompilationFromVBCode(string code, IDictionary msbuildProperties, string? name = null, params MetadataReference[] references) @@ -99,10 +99,10 @@ public static Compilation CreateCompilationFromVBCode(string code, IDictionary? references = null) + public static (Compilation, IAssemblySymbol) CreateCompilationFromAssembly(string assemblyPath, params MetadataReference[] references) { var metadataReference = CreateMetadataReference(assemblyPath); var compilation = CS.CSharpCompilation.Create( @@ -110,8 +110,8 @@ public static (Compilation, IAssemblySymbol) CreateCompilationFromAssembly(strin options: new CS.CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary), syntaxTrees: s_assemblyBootstrap, references: GetReferenceAssemblies(assemblyPath) - .Concat(references ?? Enumerable.Empty()) .Select(CreateMetadataReference) + .Concat(references ?? []) .Append(metadataReference)); var assembly = (IAssemblySymbol)compilation.GetAssemblyOrModuleSymbol(metadataReference)!; diff --git a/src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs b/src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs index 3473f2b6182..ae7a801f30e 100644 --- a/src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs +++ b/src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs @@ -40,7 +40,7 @@ partial class DotnetApiCatalog _ => LoggerVerbosity.Quiet, }); - var workspace = MSBuildWorkspace.Create(msbuildProperties); + using var workspace = MSBuildWorkspace.Create(msbuildProperties); workspace.WorkspaceFailed += (sender, e) => Logger.LogWarning($"{e.Diagnostic}"); if (files.TryGetValue(FileType.NotSupported, out var unsupportedFiles)) @@ -88,26 +88,36 @@ await LoadCompilationFromProject(project.AbsolutePath) is { } compilation) assemblies.Add((compilation.Assembly, compilation)); } + var references = config.References ?? []; + var metadataReferences = references.Select(assemblyPath => + { + var documentation = XmlDocumentationProvider.CreateFromFile(Path.ChangeExtension(assemblyPath, ".xml")); + return MetadataReference.CreateFromFile(assemblyPath, documentation: documentation); + }).ToArray(); + + // LoadCompilationFrom C# source files if (files.TryGetValue(FileType.CSSourceCode, out var csFiles)) { - var compilation = CompilationHelper.CreateCompilationFromCSharpFiles(csFiles.Select(f => f.NormalizedPath), msbuildProperties); + var compilation = CompilationHelper.CreateCompilationFromCSharpFiles(csFiles.Select(f => f.NormalizedPath), msbuildProperties, metadataReferences); hasCompilationError |= compilation.CheckDiagnostics(config.AllowCompilationErrors); assemblies.Add((compilation.Assembly, compilation)); } + // LoadCompilationFrom VB source files if (files.TryGetValue(FileType.VBSourceCode, out var vbFiles)) { - var compilation = CompilationHelper.CreateCompilationFromVBFiles(vbFiles.Select(f => f.NormalizedPath), msbuildProperties); + var compilation = CompilationHelper.CreateCompilationFromVBFiles(vbFiles.Select(f => f.NormalizedPath), msbuildProperties, metadataReferences); hasCompilationError |= compilation.CheckDiagnostics(config.AllowCompilationErrors); assemblies.Add((compilation.Assembly, compilation)); } + // Load Compilation from assembly files if (files.TryGetValue(FileType.Assembly, out var assemblyFiles)) { foreach (var assemblyFile in assemblyFiles) { Logger.LogInfo($"Loading assembly {assemblyFile.NormalizedPath}"); - var (compilation, assembly) = CompilationHelper.CreateCompilationFromAssembly(assemblyFile.NormalizedPath, config.References); + var (compilation, assembly) = CompilationHelper.CreateCompilationFromAssembly(assemblyFile.NormalizedPath, metadataReferences); hasCompilationError |= compilation.CheckDiagnostics(config.AllowCompilationErrors); assemblies.Add((assembly, compilation)); } diff --git a/test/Docfx.Dotnet.Tests/GenerateMetadataFromCSUnitTest.cs b/test/Docfx.Dotnet.Tests/GenerateMetadataFromCSUnitTest.cs index 3a2e89e05dd..3f266d5453f 100644 --- a/test/Docfx.Dotnet.Tests/GenerateMetadataFromCSUnitTest.cs +++ b/test/Docfx.Dotnet.Tests/GenerateMetadataFromCSUnitTest.cs @@ -14,9 +14,9 @@ public class GenerateMetadataFromCSUnitTest { private static readonly Dictionary EmptyMSBuildProperties = new(); - private static MetadataItem Verify(string code, ExtractMetadataConfig config = null, IDictionary msbuildProperties = null) + private static MetadataItem Verify(string code, ExtractMetadataConfig config = null, IDictionary msbuildProperties = null, MetadataReference[] references = null) { - var compilation = CompilationHelper.CreateCompilationFromCSharpCode(code, msbuildProperties ?? EmptyMSBuildProperties, "test.dll"); + var compilation = CompilationHelper.CreateCompilationFromCSharpCode(code, msbuildProperties ?? EmptyMSBuildProperties, "test.dll", references); var extensionMethods = compilation.Assembly.FindExtensionMethods(new(new(), new())).ToArray(); return compilation.Assembly.GenerateMetadataItem(compilation, config, extensionMethods: extensionMethods); } @@ -3771,4 +3771,30 @@ public void F1() {} Assert.Empty(foo.Items); } } + + [Fact] + public void TestGenerateMetadataWithReference() + { + string code = @" +namespace Test +{ + public class Foo + { + public TupleLibrary.XmlTasks Tasks{ get;set; } + } +} +"; + var references = new string[] { "TestData/TupleLibrary.dll" }.Select(assemblyPath => + { + var documentation = XmlDocumentationProvider.CreateFromFile(Path.ChangeExtension(assemblyPath, ".xml")); + return MetadataReference.CreateFromFile(assemblyPath, documentation: documentation); + }).ToArray(); + + // Act + var output = Verify(code, references: references); + + // Assert + Assert.Contains("TupleLibrary", output.References.Keys); + Assert.Contains("TupleLibrary.XmlTasks", output.References.Keys); + } } diff --git a/test/Docfx.Dotnet.Tests/GenerateMetadataFromVBUnitTest.cs b/test/Docfx.Dotnet.Tests/GenerateMetadataFromVBUnitTest.cs index e3c5463bdca..ea798501ca4 100644 --- a/test/Docfx.Dotnet.Tests/GenerateMetadataFromVBUnitTest.cs +++ b/test/Docfx.Dotnet.Tests/GenerateMetadataFromVBUnitTest.cs @@ -13,10 +13,10 @@ public class GenerateMetadataFromVBUnitTest { private static readonly Dictionary EmptyMSBuildProperties = new(); - private static MetadataItem Verify(string code, ExtractMetadataConfig options = null, IDictionary msbuildProperties = null, params MetadataReference[] references) + private static MetadataItem Verify(string code, ExtractMetadataConfig config = null, IDictionary msbuildProperties = null, MetadataReference[] references = null) { var compilation = CompilationHelper.CreateCompilationFromVBCode(code, msbuildProperties ?? EmptyMSBuildProperties, "test.dll", references); - return compilation.Assembly.GenerateMetadataItem(compilation, options); + return compilation.Assembly.GenerateMetadataItem(compilation, config); } [Trait("Related", "Generic")] @@ -1643,4 +1643,36 @@ End Namespace Assert.Empty(foo.Items); } } + + [Fact] + public void TestGenerateMetadataWithReference() + { + string code = @" +Namespace Test + Public Class Foo + Property Tasks As TupleLibrary.XmlTasks + End Class +End Namespace +namespace Test +{ + public class Foo + { + public TupleLibrary.XmlTasks Tasks { get;set; } + } +} +"; + + var references = new string[] { "TestData/TupleLibrary.dll" }.Select(assemblyPath => + { + var documentation = XmlDocumentationProvider.CreateFromFile(Path.ChangeExtension(assemblyPath, ".xml")); + return MetadataReference.CreateFromFile(assemblyPath, documentation: documentation); + }).ToArray(); + + // Act + var output = Verify(code, references: references); + + // Assert + Assert.Contains("TupleLibrary", output.References.Keys); + Assert.Contains("TupleLibrary.XmlTasks", output.References.Keys); + } } From aa982be31b80a2ef2b73a37b9e35557f8452ad21 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:20:44 +0900 Subject: [PATCH 5/5] perf: Optimize xrefmap.json file deserialization performance (#9824) perf: optimize xrefmap.json file deserialization --- src/Docfx.Build/XRefMaps/XRefMapDownloader.cs | 39 ++++++++-- .../XRefMaps/XRefMapRedirection.cs | 6 ++ test/Docfx.Build.Tests/TestData/xrefmap.json | 10 +++ .../XRefMapDownloaderTest.cs | 15 ++++ .../XRefMapSerializationTest.cs | 74 +++++++++++++++++++ test/docfx.Tests/Api.verified.cs | 4 + 6 files changed, 140 insertions(+), 8 deletions(-) create mode 100644 test/Docfx.Build.Tests/TestData/xrefmap.json create mode 100644 test/Docfx.Build.Tests/XRefMapSerializationTest.cs diff --git a/src/Docfx.Build/XRefMaps/XRefMapDownloader.cs b/src/Docfx.Build/XRefMaps/XRefMapDownloader.cs index 586c93167b9..55d477394b5 100644 --- a/src/Docfx.Build/XRefMaps/XRefMapDownloader.cs +++ b/src/Docfx.Build/XRefMaps/XRefMapDownloader.cs @@ -111,13 +111,25 @@ protected static IXRefContainer DownloadFromLocal(Uri uri) private static IXRefContainer ReadLocalFile(string filePath) { Logger.LogVerbose($"Reading from file: {filePath}"); - if (".zip".Equals(Path.GetExtension(filePath), StringComparison.OrdinalIgnoreCase)) + + switch (Path.GetExtension(filePath).ToLowerInvariant()) { - return XRefArchive.Open(filePath, XRefArchiveMode.Read); - } + case ".zip": + return XRefArchive.Open(filePath, XRefArchiveMode.Read); - using var sr = File.OpenText(filePath); - return YamlUtility.Deserialize(sr); + case ".json": + { + using var stream = File.OpenText(filePath); + return JsonUtility.Deserialize(stream); + } + + case ".yml": + default: + { + using var sr = File.OpenText(filePath); + return YamlUtility.Deserialize(sr); + } + } } protected static async Task DownloadFromWebAsync(Uri uri) @@ -134,10 +146,21 @@ protected static async Task DownloadFromWebAsync(Uri uri) }; using var stream = await httpClient.GetStreamAsync(uri); - using var sr = new StreamReader(stream, bufferSize: 81920); // Default :1024 byte - var map = YamlUtility.Deserialize(sr); - return map; + switch (Path.GetExtension(uri.AbsolutePath).ToLowerInvariant()) + { + case ".json": + { + using var sr = new StreamReader(stream, bufferSize: 81920); // Default :1024 byte + return JsonUtility.Deserialize(sr); + } + case ".yml": + default: + { + using var sr = new StreamReader(stream, bufferSize: 81920); // Default :1024 byte + return YamlUtility.Deserialize(sr); + } + } } public static void UpdateHref(XRefMap map, Uri uri) diff --git a/src/Docfx.Build/XRefMaps/XRefMapRedirection.cs b/src/Docfx.Build/XRefMaps/XRefMapRedirection.cs index 09ba60984f8..1035a7f0650 100644 --- a/src/Docfx.Build/XRefMaps/XRefMapRedirection.cs +++ b/src/Docfx.Build/XRefMaps/XRefMapRedirection.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json.Serialization; +using Newtonsoft.Json; using YamlDotNet.Serialization; namespace Docfx.Build.Engine; @@ -8,8 +10,12 @@ namespace Docfx.Build.Engine; public class XRefMapRedirection { [YamlMember(Alias = "uidPrefix")] + [JsonProperty("uidPrefix")] + [JsonPropertyName("uidPrefix")] public string UidPrefix { get; set; } [YamlMember(Alias = "href")] + [JsonProperty("Href")] + [JsonPropertyName("href")] public string Href { get; set; } } diff --git a/test/Docfx.Build.Tests/TestData/xrefmap.json b/test/Docfx.Build.Tests/TestData/xrefmap.json new file mode 100644 index 00000000000..7a3aad507fb --- /dev/null +++ b/test/Docfx.Build.Tests/TestData/xrefmap.json @@ -0,0 +1,10 @@ +{ + "references": [ + { + "fullName": "str", + "href": "https://docs.python.org/3.5/library/stdtypes.html#str", + "name": "str", + "uid": "str" + } + ] +} diff --git a/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs b/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs index e7536736935..c8ae4b7ec57 100644 --- a/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs +++ b/test/Docfx.Build.Tests/XRefMapDownloaderTest.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Net; +using FluentAssertions; using Xunit; namespace Docfx.Build.Engine.Tests; @@ -35,4 +36,18 @@ public async Task ReadLocalXRefMapWithFallback() Assert.NotNull(xrefSpec); Assert.Equal("https://docs.python.org/3.5/library/stdtypes.html#str", xrefSpec.Href); } + + [Fact] + public async Task ReadLocalXRefMapJsonFileTest() + { + // Arrange + var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", "xrefmap.json"); + + XRefMapDownloader downloader = new XRefMapDownloader(); + var xrefMap = await downloader.DownloadAsync(new Uri(path)) as XRefMap; + + // Assert + xrefMap.Should().NotBeNull(); + xrefMap.References.Should().HaveCount(1); + } } diff --git a/test/Docfx.Build.Tests/XRefMapSerializationTest.cs b/test/Docfx.Build.Tests/XRefMapSerializationTest.cs new file mode 100644 index 00000000000..6f6cac18377 --- /dev/null +++ b/test/Docfx.Build.Tests/XRefMapSerializationTest.cs @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Docfx.Common; +using Docfx.Plugins; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Docfx.Build.Engine.Tests; + +public class XRefMapSerializationTest +{ + [Fact] + public void XRefMapSerializationRoundTripTest() + { + var model = new XRefMap + { + BaseUrl = "http://localhost", + Sorted = true, + HrefUpdated = null, + Redirections = new List + { + new XRefMapRedirection + { + Href = "Dummy", + UidPrefix = "Dummy" + }, + }, + References = new List + { + new XRefSpec(new Dictionary + { + ["Additional1"] = "Dummy", + }) + { + Uid = "Dummy", + Name = "Dummy", + Href = "Dummy", + CommentId ="Dummy", + IsSpec = true, + }, + }, + Others = new Dictionary + { + ["Other1"] = "Dummy", + } + }; + + // Arrange + var jsonResult = RoundtripByNewtonsoftJson(model); + var yamlResult = RoundtripWithYamlDotNet(model); + + // Assert + jsonResult.Should().BeEquivalentTo(model); + yamlResult.Should().BeEquivalentTo(model); + } + + private static T RoundtripByNewtonsoftJson(T model) + { + var json = JsonUtility.Serialize(model); + return JsonUtility.Deserialize(new StringReader(json)); + } + + private static T RoundtripWithYamlDotNet(T model) + { + var sb = new StringBuilder(); + using var sw = new StringWriter(sb); + YamlUtility.Serialize(sw, model); + var json = sb.ToString(); + return YamlUtility.Deserialize(new StringReader(json)); + } +} diff --git a/test/docfx.Tests/Api.verified.cs b/test/docfx.Tests/Api.verified.cs index 0933d8366bb..348e099c9b3 100644 --- a/test/docfx.Tests/Api.verified.cs +++ b/test/docfx.Tests/Api.verified.cs @@ -486,8 +486,12 @@ protected override Docfx.Build.Engine.IXRefContainer GetMap(string name) { } public class XRefMapRedirection { public XRefMapRedirection() { } + [Newtonsoft.Json.JsonProperty("Href")] + [System.Text.Json.Serialization.JsonPropertyName("href")] [YamlDotNet.Serialization.YamlMember(Alias="href")] public string Href { get; set; } + [Newtonsoft.Json.JsonProperty("uidPrefix")] + [System.Text.Json.Serialization.JsonPropertyName("uidPrefix")] [YamlDotNet.Serialization.YamlMember(Alias="uidPrefix")] public string UidPrefix { get; set; } }