From 5be412ce8d0c355e2db918b1ca8f5550a02a2b4a Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Thu, 2 Sep 2021 04:47:39 -0700 Subject: [PATCH 1/5] Only pass perfmap argument for Linux --- src/coreclr/crossgen-corelib.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj index 847066a891c07c..49b2dba17b191e 100644 --- a/src/coreclr/crossgen-corelib.proj +++ b/src/coreclr/crossgen-corelib.proj @@ -97,7 +97,6 @@ $(CrossGenDllCmd) -o:$(CoreLibOutputPath) $(CrossGenDllCmd) -r:$([MSBuild]::NormalizePath('$(BinDir)', 'IL', '*.dll')) $(CrossGenDllCmd) --targetarch:$(TargetArchitecture) - $(CrossGenDllCmd) --perfmap-format-version:1 @(OptimizationMibcFiles->'-m:$(MergedMibcPath)', ' ') $(CrossGenDllCmd) $(MibcArgs) --embed-pgo-data $(CrossGenDllCmd) -O @@ -110,6 +109,7 @@ + $(CrossGenDllCmd) --perfmap-format-version:1 $(CrossGenDllCmd) --perfmap --perfmap-path:$(BinDir) From 42fc682aa9655f27f8aa2caa895750e59d4a6965 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Wed, 8 Sep 2021 21:28:39 -0700 Subject: [PATCH 2/5] Enhance CrossGen2 to emit PerfMap debug directory entry This is used to correlate PE's with their corresponding PerfMaps. For example the header in the perfmap could be: ``` FFFFFFFF 00 026D4D21B3EE3D93843FF7A964235822 FFFFFFFE 00 1 FFFFFFFD 00 1 FFFFFFFC 00 3 FFFFFFFB 00 1 ``` And the PE will have the corresponding entries in the PE as: ``` PerfMap (Type 21): System.Private.CoreLib.ni.r2rmap, Signature = 026d4d21b3ee3d93843ff7a964235822, Version = 1 ``` --- .../ILCompiler.Diagnostics/PerfMapWriter.cs | 115 ++++++++++++++---- .../ReadyToRunDiagnosticsConstants.cs | 40 ++++++ .../CodeGen/ReadyToRunObjectWriter.cs | 56 ++++++--- .../ReadyToRun/DebugDirectoryEntryNode.cs | 76 +++++++++++- .../ReadyToRun/DebugDirectoryNode.cs | 56 ++++++--- .../Compiler/ReadyToRunCodegenCompilation.cs | 10 +- .../ReadyToRunCodegenCompilationBuilder.cs | 2 +- .../ObjectWriter/SymbolFileBuilder.cs | 4 +- src/coreclr/tools/r2rdump/R2RDump.cs | 8 +- 9 files changed, 302 insertions(+), 65 deletions(-) create mode 100644 src/coreclr/tools/aot/ILCompiler.Diagnostics/ReadyToRunDiagnosticsConstants.cs diff --git a/src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs b/src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs index dd494b9fdcf9a8..bba40fd1df33a4 100644 --- a/src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs +++ b/src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Security.Cryptography; +using Internal.ReadyToRunDiagnosticsConstants; using Internal.TypeSystem; namespace ILCompiler.Diagnostics @@ -16,14 +17,8 @@ public class PerfMapWriter public const int LegacyCrossgen1FormatVersion = 0; public const int CurrentFormatVersion = 1; - - public enum PseudoRVA : uint - { - OutputGuid = 0xFFFFFFFF, - TargetOS = 0xFFFFFFFE, - TargetArchitecture = 0xFFFFFFFD, - FormatVersion = 0xFFFFFFFC, - } + + const int HeaderEntriesPseudoLength = 0; private TextWriter _writer; @@ -32,7 +27,7 @@ private PerfMapWriter(TextWriter writer) _writer = writer; } - public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnumerable methods, IEnumerable inputAssemblies, TargetOS targetOS, TargetArchitecture targetArch) + public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnumerable methods, IEnumerable inputAssemblies, TargetDetails details) { if (perfMapFormatVersion > CurrentFormatVersion) { @@ -41,22 +36,10 @@ public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnum using (TextWriter writer = new StreamWriter(perfMapFileName)) { - IEnumerable orderedInputs = inputAssemblies.OrderBy(asm => asm.Name, StringComparer.OrdinalIgnoreCase); PerfMapWriter perfMapWriter = new PerfMapWriter(writer); - - List inputHash = new List(); - foreach (AssemblyInfo inputAssembly in orderedInputs) - { - inputHash.AddRange(inputAssembly.Mvid.ToByteArray()); - } - inputHash.Add((byte)targetOS); - inputHash.Add((byte)targetArch); - Guid outputGuid = new Guid(MD5.HashData(inputHash.ToArray())); - perfMapWriter.WriteLine(outputGuid.ToString(), (uint)PseudoRVA.OutputGuid, 0); - perfMapWriter.WriteLine(targetOS.ToString(), (uint)PseudoRVA.TargetOS, 0); - perfMapWriter.WriteLine(targetArch.ToString(), (uint)PseudoRVA.TargetArchitecture, 0); - perfMapWriter.WriteLine(CurrentFormatVersion.ToString(), (uint)PseudoRVA.FormatVersion, 0); + byte[] signature = PerfMapV1SignatureHelper(inputAssemblies, details); + WritePerfMapV1Header(inputAssemblies, details, perfMapWriter); foreach (MethodInfo methodInfo in methods) { @@ -72,6 +55,92 @@ public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnum } } + private static void WritePerfMapV1Header(IEnumerable inputAssemblies, TargetDetails details, PerfMapWriter perfMapWriter) + { + byte[] signature = PerfMapV1SignatureHelper(inputAssemblies, details); + + // Make sure these get emitted in this order, other tools in the ecosystem like the symbol uploader and PerfView rely on this. + // In particular, the order of it. Append only. + string signatureFormatted = Convert.ToHexString(signature); + + PerfmapTokensForTarget targetTokens = TranslateTargetDetailsToPerfmapConstants(details); + + perfMapWriter.WriteLine(signatureFormatted, (uint)PerfMapPseudoRVAToken.OutputSignature, HeaderEntriesPseudoLength); + perfMapWriter.WriteLine(CurrentFormatVersion.ToString(), (uint)PerfMapPseudoRVAToken.FormatVersion, HeaderEntriesPseudoLength); + perfMapWriter.WriteLine(((uint)targetTokens.OperatingSystem).ToString(), (uint)PerfMapPseudoRVAToken.TargetOS, HeaderEntriesPseudoLength); + perfMapWriter.WriteLine(((uint)targetTokens.Architecture).ToString(), (uint)PerfMapPseudoRVAToken.TargetArchitecture, HeaderEntriesPseudoLength); + perfMapWriter.WriteLine(((uint)targetTokens.Abi).ToString(), (uint)PerfMapPseudoRVAToken.TargetABI, HeaderEntriesPseudoLength); + } + + public static byte[] PerfMapV1SignatureHelper(IEnumerable inputAssemblies, TargetDetails details) + { + IEnumerable orderedInputs = inputAssemblies.OrderBy(asm => asm.Name, StringComparer.OrdinalIgnoreCase); + List inputHash = new List(); + foreach (AssemblyInfo inputAssembly in orderedInputs) + { + inputHash.AddRange(inputAssembly.Mvid.ToByteArray()); + } + + PerfmapTokensForTarget targetTokens = TranslateTargetDetailsToPerfmapConstants(details); + + byte[] buffer = new byte[12]; + if (!BitConverter.TryWriteBytes(buffer.AsSpan(0, sizeof(uint)), (uint)targetTokens.OperatingSystem) + || !BitConverter.TryWriteBytes(buffer.AsSpan(4, sizeof(uint)), (uint)targetTokens.Architecture) + || !BitConverter.TryWriteBytes(buffer.AsSpan(8, sizeof(uint)), (uint)targetTokens.Abi)) + { + throw new InvalidOperationException(); + } + + if (!BitConverter.IsLittleEndian) + { + buffer.AsSpan(0, sizeof(uint)).Reverse(); + buffer.AsSpan(4, sizeof(uint)).Reverse(); + buffer.AsSpan(8, sizeof(uint)).Reverse(); + } + + inputHash.AddRange(buffer); + byte[] hash = MD5.HashData(inputHash.ToArray()); + + return hash; + } + + internal record struct PerfmapTokensForTarget(PerfMapOSToken OperatingSystem, PerfMapArchitectureToken Architecture, PerfMapAbiToken Abi); + + private static PerfmapTokensForTarget TranslateTargetDetailsToPerfmapConstants(TargetDetails details) + { + PerfMapOSToken osToken = details.OperatingSystem switch + { + TargetOS.Unknown => PerfMapOSToken.Unknown, + TargetOS.Windows => PerfMapOSToken.Windows, + TargetOS.Linux => PerfMapOSToken.Linux, + TargetOS.OSX => PerfMapOSToken.OSX, + TargetOS.FreeBSD => PerfMapOSToken.FreeBSD, + TargetOS.NetBSD => PerfMapOSToken.NetBSD, + TargetOS.SunOS => PerfMapOSToken.SunOS, + _ => throw new NotImplementedException(details.OperatingSystem.ToString()) + }; + + PerfMapAbiToken abiToken = details.Abi switch + { + TargetAbi.Unknown => PerfMapAbiToken.Unknown, + TargetAbi.CoreRT => PerfMapAbiToken.Default, + TargetAbi.CoreRTArmel => PerfMapAbiToken.Armel, + _ => throw new NotImplementedException(details.Abi.ToString()) + }; + + PerfMapArchitectureToken archToken = details.Architecture switch + { + TargetArchitecture.Unknown => PerfMapArchitectureToken.Unknown, + TargetArchitecture.ARM => PerfMapArchitectureToken.ARM, + TargetArchitecture.ARM64 => PerfMapArchitectureToken.ARM64, + TargetArchitecture.X64 => PerfMapArchitectureToken.X64, + TargetArchitecture.X86 => PerfMapArchitectureToken.X86, + _ => throw new NotImplementedException(details.Architecture.ToString()) + }; + + return new PerfmapTokensForTarget(osToken, archToken, abiToken); + } + private void WriteLine(string methodName, uint rva, uint length) { _writer.WriteLine($@"{rva:X8} {length:X2} {methodName}"); diff --git a/src/coreclr/tools/aot/ILCompiler.Diagnostics/ReadyToRunDiagnosticsConstants.cs b/src/coreclr/tools/aot/ILCompiler.Diagnostics/ReadyToRunDiagnosticsConstants.cs new file mode 100644 index 00000000000000..fea26f9db02897 --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.Diagnostics/ReadyToRunDiagnosticsConstants.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Internal.ReadyToRunDiagnosticsConstants; + +public enum PerfMapPseudoRVAToken : uint +{ + OutputSignature = 0xFFFFFFFF, + FormatVersion = 0xFFFFFFFE, + TargetOS = 0xFFFFFFFD, + TargetArchitecture = 0xFFFFFFFC, + TargetABI = 0xFFFFFFFB, +} + +public enum PerfMapArchitectureToken : uint +{ + Unknown = 0, + ARM = 1, + ARM64 = 2, + X64 = 3, + X86 = 4, +} + +public enum PerfMapOSToken : uint +{ + Unknown = 0, + Windows = 1, + Linux = 2, + OSX = 3, + FreeBSD = 4, + NetBSD = 5, + SunOS = 6, +} + +public enum PerfMapAbiToken : uint +{ + Unknown = 0, + Default = 1, + Armel = 2, +} diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs index 6404f77fbac06d..d704579efd7661 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs @@ -27,7 +27,7 @@ namespace ILCompiler.DependencyAnalysis internal class ReadyToRunObjectWriter { /// - /// Nodefactory for which ObjectWriter is instantiated for. + /// Nodefactory for which ObjectWriter is instantiated for. /// private readonly NodeFactory _nodeFactory; @@ -234,6 +234,7 @@ public void EmitPortableExecutable() peIdProvider); NativeDebugDirectoryEntryNode nativeDebugDirectoryEntryNode = null; + PerfMapDebugDirectoryEntryNode perfMapDebugDirectoryEntryNode = null; ISymbolDefinitionNode firstImportThunk = null; ISymbolDefinitionNode lastImportThunk = null; ObjectNode lastWrittenObjectNode = null; @@ -261,6 +262,13 @@ public void EmitPortableExecutable() nativeDebugDirectoryEntryNode = nddeNode; } + if (node is PerfMapDebugDirectoryEntryNode pmdeNode) + { + // There should be only one PerfMapDebugDirectoryEntryNode. + Debug.Assert(perfMapDebugDirectoryEntryNode is null); + perfMapDebugDirectoryEntryNode = pmdeNode; + } + if (node is ImportThunk importThunkNode) { Debug.Assert(firstImportThunk == null || lastWrittenObjectNode is ImportThunk, @@ -305,7 +313,7 @@ public void EmitPortableExecutable() { r2rPeBuilder.AddSymbolForRange(_nodeFactory.DelayLoadMethodCallThunks, firstImportThunk, lastImportThunk); } - + if (_nodeFactory.Win32ResourcesNode != null) { @@ -313,6 +321,14 @@ public void EmitPortableExecutable() r2rPeBuilder.SetWin32Resources(_nodeFactory.Win32ResourcesNode, _nodeFactory.Win32ResourcesNode.Size); } + if (_outputInfoBuilder != null) + { + foreach (string inputFile in _inputFiles) + { + _outputInfoBuilder.AddInputModule(_nodeFactory.TypeSystemContext.GetModuleFromPath(inputFile)); + } + } + using (var peStream = File.Create(_objectFilePath)) { r2rPeBuilder.Write(peStream, timeDateStamp); @@ -322,26 +338,36 @@ public void EmitPortableExecutable() _mapFileBuilder.SetFileSize(peStream.Length); } - // Compute MD5 hash of the output image and store that in the native DebugDirectory entry - using (var md5Hash = MD5.Create()) + if (nativeDebugDirectoryEntryNode is not null) { - peStream.Seek(0, SeekOrigin.Begin); - byte[] hash = md5Hash.ComputeHash(peStream); - byte[] rsdsEntry = nativeDebugDirectoryEntryNode.GenerateRSDSEntryData(hash); + Debug.Assert(_generatePdbFile); + // Compute MD5 hash of the output image and store that in the native DebugDirectory entry + using (var md5Hash = MD5.Create()) + { + peStream.Seek(0, SeekOrigin.Begin); + byte[] hash = md5Hash.ComputeHash(peStream); + byte[] rsdsEntry = nativeDebugDirectoryEntryNode.GenerateRSDSEntryData(hash); + + int offsetToUpdate = r2rPeBuilder.GetSymbolFilePosition(nativeDebugDirectoryEntryNode); + peStream.Seek(offsetToUpdate, SeekOrigin.Begin); + peStream.Write(rsdsEntry); + } + } + + if (perfMapDebugDirectoryEntryNode is not null) + { + Debug.Assert(_generatePerfMapFile && _outputInfoBuilder is not null && _outputInfoBuilder.EnumerateInputAssemblies().Any()); + byte[] perfmapSig = PerfMapWriter.PerfMapV1SignatureHelper(_outputInfoBuilder.EnumerateInputAssemblies(), _nodeFactory.Target); + byte[] perfMapEntry = perfMapDebugDirectoryEntryNode.GeneratePerfMapEntryData(perfmapSig, _perfMapFormatVersion); - int offsetToUpdate = r2rPeBuilder.GetSymbolFilePosition(nativeDebugDirectoryEntryNode); + int offsetToUpdate = r2rPeBuilder.GetSymbolFilePosition(perfMapDebugDirectoryEntryNode); peStream.Seek(offsetToUpdate, SeekOrigin.Begin); - peStream.Write(rsdsEntry); + peStream.Write(perfMapEntry); } } if (_outputInfoBuilder != null) { - foreach (string inputFile in _inputFiles) - { - _outputInfoBuilder.AddInputModule(_nodeFactory.TypeSystemContext.GetModuleFromPath(inputFile)); - } - r2rPeBuilder.AddSections(_outputInfoBuilder); if (_generateMapFile) @@ -374,7 +400,7 @@ public void EmitPortableExecutable() { path = Path.GetDirectoryName(_objectFilePath); } - _symbolFileBuilder.SavePerfMap(path, _perfMapFormatVersion, _objectFilePath, _nodeFactory.Target.OperatingSystem, _nodeFactory.Target.Architecture); + _symbolFileBuilder.SavePerfMap(path, _perfMapFormatVersion, _objectFilePath, _nodeFactory.Target); } if (_profileFileBuilder != null) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs index 9aa6e6bbc46099..ae6323797d2c31 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs @@ -43,6 +43,76 @@ public override int CompareToImpl(ISortableNode other, CompilerComparer comparer } } + public class PerfMapDebugDirectoryEntryNode : DebugDirectoryEntryNode + { + const int PerfMapEntrySize = + sizeof(uint) + // Magic + SignatureSize + // Signature + sizeof(uint) + // Age + 260; // FileName + + public const uint PerfMapMagic = 0x4D523252;// R2RM + + public const int PerfMapEntryType = 21; // DebugDirectoryEntryType for this entry. + + private const int SignatureSize = 16; + + public override int ClassCode => 813123850; + + public unsafe int Size => PerfMapEntrySize; + + public PerfMapDebugDirectoryEntryNode(string entryName) + : base(null) + { + _entryName = entryName; + } + + private string _entryName; + + public override void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) + { + sb.Append(nameMangler.CompilationUnitPrefix); + sb.Append($"__PerfMapDebugDirectoryEntryNode_{_entryName.Replace('.','_')}"); + } + + public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) + { + ObjectDataBuilder builder = new ObjectDataBuilder(factory, relocsOnly); + builder.RequireInitialPointerAlignment(); + builder.AddSymbol(this); + + // Emit empty entry. This will be filled with data after the output image is emitted + builder.EmitZeros(PerfMapEntrySize); + + return builder.ToObjectData(); + } + + public byte[] GeneratePerfMapEntryData(byte[] signature, int version) + { + Debug.Assert(SignatureSize == signature.Length); + MemoryStream perfmapEntry = new MemoryStream(PerfMapEntrySize); + + using (BinaryWriter writer = new BinaryWriter(perfmapEntry)) + { + writer.Write(PerfMapMagic); + writer.Write(signature); + writer.Write(version); + + byte[] perfmapNameBytes = Encoding.UTF8.GetBytes(_entryName); + writer.Write(perfmapNameBytes); + writer.Write(0); // Null terminator + + Debug.Assert(perfmapEntry.Length <= PerfMapEntrySize); + return perfmapEntry.ToArray(); + } + } + + public override int CompareToImpl(ISortableNode other, CompilerComparer comparer) + { + return _entryName.CompareTo(((PerfMapDebugDirectoryEntryNode)other)._entryName); + } + } + public class NativeDebugDirectoryEntryNode : DebugDirectoryEntryNode { const int RSDSSize = @@ -55,6 +125,8 @@ public class NativeDebugDirectoryEntryNode : DebugDirectoryEntryNode public unsafe int Size => RSDSSize; + public const uint RsdsMagic = 0x53445352;// R2RM + public NativeDebugDirectoryEntryNode(string pdbName) : base(null) { @@ -87,8 +159,7 @@ public byte[] GenerateRSDSEntryData(byte[] md5Hash) using (BinaryWriter writer = new BinaryWriter(rsdsEntry)) { - // Magic "RSDS" - writer.Write((uint)0x53445352); + writer.Write(RsdsMagic); // The PDB signature will be the same as our NGEN signature. // However we want the printed version of the GUID to be the same as the @@ -105,6 +176,7 @@ public byte[] GenerateRSDSEntryData(byte[] md5Hash) string pdbFileName = _pdbName; byte[] pdbFileNameBytes = Encoding.UTF8.GetBytes(pdbFileName); writer.Write(pdbFileNameBytes); + writer.Write(0); // Null terminator Debug.Assert(rsdsEntry.Length <= RSDSSize); return rsdsEntry.ToArray(); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs index 175025449039bd..fd1fb49d25febd 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs @@ -1,13 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. - using System; using System.Collections.Immutable; using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection.PortableExecutable; using Internal.Text; +using Internal.TypeSystem; using Internal.TypeSystem.Ecma; namespace ILCompiler.DependencyAnalysis.ReadyToRun @@ -26,9 +27,11 @@ public class DebugDirectoryNode : ObjectNode, ISymbolDefinitionNode private EcmaModule _module; private NativeDebugDirectoryEntryNode _nativeEntry; + private PerfMapDebugDirectoryEntryNode _perfMapEntry; + private bool _insertDeterministicEntry; - public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName) + public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName, bool shouldAddNiPdb, bool shouldGeneratePerfmap) { _module = sourceModule; _insertDeterministicEntry = sourceModule == null; // Mark module as deterministic if generating composite image @@ -37,7 +40,16 @@ public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName) { pdbNameRoot = sourceModule.Assembly.GetName().Name; } - _nativeEntry = new NativeDebugDirectoryEntryNode(pdbNameRoot + ".ni.pdb"); + + if (shouldAddNiPdb) + { + _nativeEntry = new NativeDebugDirectoryEntryNode(pdbNameRoot + ".ni.pdb"); + } + + if (shouldGeneratePerfmap) + { + _perfMapEntry = new PerfMapDebugDirectoryEntryNode(pdbNameRoot + ".ni.r2rmap"); + } } public override ObjectNodeSection Section => ObjectNodeSection.TextSection; @@ -52,7 +64,10 @@ public DebugDirectoryNode(EcmaModule sourceModule, string outputFileName) public int Offset => 0; - public int Size => (GetNumDebugDirectoryEntriesInModule() + 1 + (_insertDeterministicEntry ? 1 : 0)) * ImageDebugDirectorySize; + public int Size => (GetNumDebugDirectoryEntriesInModule() + + (_nativeEntry is not null ? 1 : 0) + + (_perfMapEntry is not null ? 1 : 0) + + (_insertDeterministicEntry ? 1 : 0)) * ImageDebugDirectorySize; public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) { @@ -83,27 +98,23 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) builder.RequireInitialPointerAlignment(); builder.AddSymbol(this); - ImmutableArray entries = default(ImmutableArray); + ImmutableArray entries = ImmutableArray.Empty; if (_module != null) entries = _module.PEReader.ReadDebugDirectory(); int numEntries = GetNumDebugDirectoryEntriesInModule(); + // Reuse the module's PDB entry + DebugDirectoryEntry pdbEntry = entries.Where(s => s.Type == DebugDirectoryEntryType.CodeView).FirstOrDefault(); + // First, write the native debug directory entry + if (_nativeEntry is not null) { var entry = _nativeEntry; builder.EmitUInt(0 /* Characteristics */); - if (numEntries > 0) - { - builder.EmitUInt(entries[0].Stamp); - builder.EmitUShort(entries[0].MajorVersion); - } - else - { - builder.EmitUInt(0); - builder.EmitUShort(0); - } + builder.EmitUInt(pdbEntry.Stamp); + builder.EmitUShort(pdbEntry.MajorVersion); // Make sure the "is portable pdb" indicator (MinorVersion == 0x504d) is clear // for the NGen debug directory entry since this debug directory can be copied // from an existing entry which could be a portable pdb. @@ -114,6 +125,21 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE); } + // Second emit a record for the perfmap + if (_perfMapEntry is not null) + { + var entry = _perfMapEntry; + + builder.EmitUInt(0); /* Characteristics */ + builder.EmitUInt(0); /* Stamp */ + builder.EmitUShort(1); /* Major */ + builder.EmitUShort(0); /* Minor */ + builder.EmitInt((int)PerfMapDebugDirectoryEntryNode.PerfMapEntryType); + builder.EmitInt(entry.Size); + builder.EmitReloc(entry, RelocType.IMAGE_REL_BASED_ADDR32NB); + builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE); + } + // If generating a composite image, emit the deterministic marker if (_insertDeterministicEntry) { diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs index de7ada8b21e01a..a5c7359a048b04 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs @@ -236,7 +236,7 @@ public sealed class ReadyToRunCodegenCompilation : Compilation private readonly IEnumerable _inputFiles; private readonly string _compositeRootPath; - + private readonly bool _resilient; private readonly int _parallelism; @@ -400,7 +400,9 @@ private void RewriteComponentFile(string inputFile, string outputFile, string ow } CopiedCorHeaderNode copiedCorHeader = new CopiedCorHeaderNode(inputModule); - DebugDirectoryNode debugDirectory = new DebugDirectoryNode(inputModule, outputFile); + // Re-written components shouldn't have any additional diagnostic information - only information about the forwards. + // Even with all of this, we might be modifying the image in a silly manner - adding a directory when if didn't have one. + DebugDirectoryNode debugDirectory = new DebugDirectoryNode(inputModule, outputFile, shouldAddNiPdb: false, shouldGeneratePerfmap: false); NodeFactory componentFactory = new NodeFactory( _nodeFactory.TypeSystemContext, _nodeFactory.CompilationModuleGroup, @@ -490,7 +492,7 @@ private bool IsLayoutFixedInCurrentVersionBubbleInternal(TypeDesc type) var fieldType = field.FieldType; if (!fieldType.IsValueType) continue; - + if (!IsLayoutFixedInCurrentVersionBubble(fieldType)) { return false; @@ -515,7 +517,7 @@ public bool IsInheritanceChainLayoutFixedInCurrentVersionBubble(TypeDesc type) { return false; } - + type = type.BaseType; if (type != null) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs index c1207a4b5b2f00..34aced76b44b01 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs @@ -219,7 +219,7 @@ public override ICompilation ToCompilation() EcmaModule singleModule = _compilationGroup.IsCompositeBuildMode ? null : inputModules.First(); CopiedCorHeaderNode corHeaderNode = new CopiedCorHeaderNode(singleModule); // TODO: proper support for multiple input files - DebugDirectoryNode debugDirectoryNode = new DebugDirectoryNode(singleModule, _outputFile); + DebugDirectoryNode debugDirectoryNode = new DebugDirectoryNode(singleModule, _outputFile, _generatePdbFile, _generatePerfMapFile); // Produce a ResourceData where the IBC PROFILE_DATA entry has been filtered out // TODO: proper support for multiple input files diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/SymbolFileBuilder.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/SymbolFileBuilder.cs index 8cd0ad481cc6d2..d46dab26c7fd92 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/SymbolFileBuilder.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/SymbolFileBuilder.cs @@ -29,7 +29,7 @@ public void SavePdb(string pdbPath, string dllFileName) new PdbWriter(pdbPath, PDBExtraData.None).WritePDBData(dllFileName, _outputInfoBuilder.EnumerateMethods()); } - public void SavePerfMap(string perfMapPath, int perfMapFormatVersion, string dllFileName, TargetOS targetOS, TargetArchitecture targetArch) + public void SavePerfMap(string perfMapPath, int perfMapFormatVersion, string dllFileName, TargetDetails details) { string perfMapExtension; if (perfMapFormatVersion == PerfMapWriter.LegacyCrossgen1FormatVersion) @@ -56,7 +56,7 @@ public void SavePerfMap(string perfMapPath, int perfMapFormatVersion, string dll string perfMapFileName = Path.Combine(perfMapPath, Path.GetFileNameWithoutExtension(dllFileName) + perfMapExtension); Console.WriteLine("Emitting PerfMap file: {0}", perfMapFileName); - PerfMapWriter.Write(perfMapFileName, perfMapFormatVersion, _outputInfoBuilder.EnumerateMethods(), _outputInfoBuilder.EnumerateInputAssemblies(), targetOS, targetArch); + PerfMapWriter.Write(perfMapFileName, perfMapFormatVersion, _outputInfoBuilder.EnumerateMethods(), _outputInfoBuilder.EnumerateInputAssemblies(), details); } } } diff --git a/src/coreclr/tools/r2rdump/R2RDump.cs b/src/coreclr/tools/r2rdump/R2RDump.cs index 94d71835966e5a..db9ecd31d3ade4 100644 --- a/src/coreclr/tools/r2rdump/R2RDump.cs +++ b/src/coreclr/tools/r2rdump/R2RDump.cs @@ -146,7 +146,7 @@ private static unsafe IAssemblyMetadata Open(string filename) return new StandaloneAssemblyMetadata(peReader); } - + public SignatureFormattingOptions GetSignatureFormattingOptions() { if (signatureFormattingOptions == null) @@ -434,7 +434,9 @@ public void Dump(ReadyToRunReader r2r) { perfmapPath = Path.ChangeExtension(r2r.Filename, ".r2rmap"); } - PerfMapWriter.Write(perfmapPath, _options.PerfmapFormatVersion, ProduceDebugInfoMethods(r2r), ProduceDebugInfoAssemblies(r2r), r2r.TargetOperatingSystem, r2r.TargetArchitecture); + // TODO: can't seem to find any place that surfaces the ABI. This is for debugging purposes, so may not be as relevant to be correct. + TargetDetails details = new TargetDetails(r2r.TargetArchitecture, r2r.TargetOperatingSystem, TargetAbi.CoreRT); + PerfMapWriter.Write(perfmapPath, _options.PerfmapFormatVersion, ProduceDebugInfoMethods(r2r), ProduceDebugInfoAssemblies(r2r), details); } if (standardDump) @@ -459,7 +461,7 @@ IEnumerable ProduceDebugInfoMethods(ReadyToRunReader r2r) mi.AssemblyName = method.ComponentReader.MetadataReader.GetString(method.ComponentReader.MetadataReader.GetAssemblyDefinition().Name); mi.ColdRVA = 0; mi.ColdLength = 0; - + yield return mi; } } From de38fc01076c15955507550aa701ad4030f32b66 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Wed, 8 Sep 2021 21:51:03 -0700 Subject: [PATCH 3/5] Emit headers for debug directory entries from the nodes themselves. --- .../ReadyToRun/DebugDirectoryEntryNode.cs | 41 ++++++++++++++++++ .../ReadyToRun/DebugDirectoryNode.cs | 43 ++----------------- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs index ae6323797d2c31..3780c4ea191212 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryEntryNode.cs @@ -43,6 +43,21 @@ public override int CompareToImpl(ISortableNode other, CompilerComparer comparer } } + public static class DeterministicDebugDirectoryEntry + { + internal static void EmitHeader(ref ObjectDataBuilder builder) + { + builder.EmitUInt(0 /* Characteristics */); + builder.EmitUInt(0); + builder.EmitUShort(0); + builder.EmitUShort(0); + builder.EmitInt((int)DebugDirectoryEntryType.Reproducible); + builder.EmitInt(0); + builder.EmitUInt(0); + builder.EmitUInt(0); + } + } + public class PerfMapDebugDirectoryEntryNode : DebugDirectoryEntryNode { const int PerfMapEntrySize = @@ -107,6 +122,18 @@ public byte[] GeneratePerfMapEntryData(byte[] signature, int version) } } + internal void EmitHeader(ref ObjectDataBuilder builder) + { + builder.EmitUInt(0); /* Characteristics */ + builder.EmitUInt(0); /* Stamp */ + builder.EmitUShort(1); /* Major */ + builder.EmitUShort(0); /* Minor */ + builder.EmitInt((int)PerfMapEntryType); + builder.EmitInt(Size); + builder.EmitReloc(this, RelocType.IMAGE_REL_BASED_ADDR32NB); + builder.EmitReloc(this, RelocType.IMAGE_REL_FILE_ABSOLUTE); + } + public override int CompareToImpl(ISortableNode other, CompilerComparer comparer) { return _entryName.CompareTo(((PerfMapDebugDirectoryEntryNode)other)._entryName); @@ -187,6 +214,20 @@ public override int CompareToImpl(ISortableNode other, CompilerComparer comparer { return _pdbName.CompareTo(((NativeDebugDirectoryEntryNode)other)._pdbName); } + + internal void EmitHeader(ref ObjectDataBuilder builder, uint stamp, ushort majorVersion) + { + builder.EmitUInt(0); /* Characteristics */ + builder.EmitUInt(stamp); + builder.EmitUShort(majorVersion); + // Make sure the "is portable pdb" indicator (MinorVersion == 0x504d) is clear. + // The NI PDB generated currently is a full PDB. + builder.EmitUShort(0 /* MinorVersion */); + builder.EmitInt((int)DebugDirectoryEntryType.CodeView); + builder.EmitInt(Size); + builder.EmitReloc(this, RelocType.IMAGE_REL_BASED_ADDR32NB); + builder.EmitReloc(this, RelocType.IMAGE_REL_FILE_ABSOLUTE); + } } public class CopiedDebugDirectoryEntryNode : DebugDirectoryEntryNode diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs index fd1fb49d25febd..eea27f5ec798dc 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/DebugDirectoryNode.cs @@ -107,50 +107,15 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) // Reuse the module's PDB entry DebugDirectoryEntry pdbEntry = entries.Where(s => s.Type == DebugDirectoryEntryType.CodeView).FirstOrDefault(); - // First, write the native debug directory entry - if (_nativeEntry is not null) - { - var entry = _nativeEntry; + // NI PDB entry + _nativeEntry?.EmitHeader(ref builder, pdbEntry.Stamp, pdbEntry.MajorVersion); - builder.EmitUInt(0 /* Characteristics */); - builder.EmitUInt(pdbEntry.Stamp); - builder.EmitUShort(pdbEntry.MajorVersion); - // Make sure the "is portable pdb" indicator (MinorVersion == 0x504d) is clear - // for the NGen debug directory entry since this debug directory can be copied - // from an existing entry which could be a portable pdb. - builder.EmitUShort(0 /* MinorVersion */); - builder.EmitInt((int)DebugDirectoryEntryType.CodeView); - builder.EmitInt(entry.Size); - builder.EmitReloc(entry, RelocType.IMAGE_REL_BASED_ADDR32NB); - builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE); - } - - // Second emit a record for the perfmap - if (_perfMapEntry is not null) - { - var entry = _perfMapEntry; - - builder.EmitUInt(0); /* Characteristics */ - builder.EmitUInt(0); /* Stamp */ - builder.EmitUShort(1); /* Major */ - builder.EmitUShort(0); /* Minor */ - builder.EmitInt((int)PerfMapDebugDirectoryEntryNode.PerfMapEntryType); - builder.EmitInt(entry.Size); - builder.EmitReloc(entry, RelocType.IMAGE_REL_BASED_ADDR32NB); - builder.EmitReloc(entry, RelocType.IMAGE_REL_FILE_ABSOLUTE); - } + _perfMapEntry?.EmitHeader(ref builder); // If generating a composite image, emit the deterministic marker if (_insertDeterministicEntry) { - builder.EmitUInt(0 /* Characteristics */); - builder.EmitUInt(0); - builder.EmitUShort(0); - builder.EmitUShort(0); - builder.EmitInt((int)DebugDirectoryEntryType.Reproducible); - builder.EmitInt(0); - builder.EmitUInt(0); - builder.EmitUInt(0); + DeterministicDebugDirectoryEntry.EmitHeader(ref builder); } // Second, copy existing entries from input module From 72a528910ace126c8fbc82e5b440ac64195357d2 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Thu, 2 Sep 2021 05:01:52 -0700 Subject: [PATCH 4/5] Add PerfMap Debug Directory Entry spec --- docs/design/specs/PE-COFF.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/design/specs/PE-COFF.md b/docs/design/specs/PE-COFF.md index dff7248a326c9f..79f4c89a7ee737 100644 --- a/docs/design/specs/PE-COFF.md +++ b/docs/design/specs/PE-COFF.md @@ -107,3 +107,15 @@ When validating that Windows PDB matches the debug directory record check that t > Note that when the debugger (or other tool) searches for the PDB only GUID and Age fields are used to match the PDB, but the timestamp of the CodeView debug directory entry does not need to match the timestamp stored in the PDB. Therefore, to verify byte-for-byte identity of the PDB, the timestamp field should also be checked. +### R2R PerfMap Debug Directory Entry (type 21) + +Declares that the image has an associated PerfMap file containing a table mapping symbols to offsets for ready to run compilations. + +*Version Major=0x0001, Minor=0x0000* of the entry data format is following: + +| Offset | Size | Field | Description | +|:-------|:-----|:------------------|-----------------------------------------------------------------------| +| 0 | 4 | Magic | 0x52 0x32 0x52 0x4D (ASCII string: "R2RM") | +| 4 | 16 | Signature | Byte sequence uniquely identifying the associated PerfMap | +| 20 | 4 | Version | Version number of the PerfMap. Currently only version 1 is supported. | +| 24 | | Path | UTF-8 NUL-terminated path to the associated `.r2rmap` file. | From 7adb8937f84c43afaef3bd262430c59b0dc9809f Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Wed, 8 Sep 2021 22:00:18 -0700 Subject: [PATCH 5/5] Extend perfmap and r2r documentation --- .../design/coreclr/botr/r2r-perfmap-format.md | 57 +++++++++++++++++++ docs/design/coreclr/botr/readytorun-format.md | 7 +++ 2 files changed, 64 insertions(+) create mode 100644 docs/design/coreclr/botr/r2r-perfmap-format.md diff --git a/docs/design/coreclr/botr/r2r-perfmap-format.md b/docs/design/coreclr/botr/r2r-perfmap-format.md new file mode 100644 index 00000000000000..56516dd6c5ec34 --- /dev/null +++ b/docs/design/coreclr/botr/r2r-perfmap-format.md @@ -0,0 +1,57 @@ +# Ready to run PerfMap format + +Traditionally in .NET symbols have been described using PDBs. These are used to map IL to source lines for code that the JIT will compile. The JIT usually emits the data that can then map from IL to a native address for symbolication purposes. + +Ready to run, however, avoids this IL to native code translation at runtime. For this reason, tools that emit R2R images often need to emit auxiliary artifacts to facilitate the mapping between source and native addresses. The Ready to Run PerfMap format describes such one map - where any method in the source code is associated with a region within the R2R image. That way any region from such image that gets executed can be linked back to a method at the source level. This facilitates tasks like stack symbolication for performance oriented investigations, although it is not appropriate to aid in tasks such as debugging at the source line level. + +## Version 1 + +R2R PerfMaps of version 1 are usually found in files with the extension `.ni.r2rmap`. It's a plain text UTF-8 format where each entry is on a separate line. Each entry is composed of a triplet: an offset relative to the beginning of the image, a length, and a name. The file is laid out in the following as follows. + +### Header + +The header leads the file and is composed by special entries. Each entry contains a 4 byte integer token in place of an RVA signifying the type of information in the entry, a length that is always 0, and the entry data. The entries are emitted in the following order. + +| Token | Description | +|:-----------|-----------------------------------------------------------------------| +| 0xFFFFFFFF | A 16 byte sequence representing a signature to correlate the perfmap with the r2r image. | +| 0xFFFFFFFE | The version of the perfmap being emitted as a unsigned 4 byte integer. | +| 0xFFFFFFFD | An unsigned 4 byte unsigned integer representing the OS the image targets. See [enumerables section](#enumerables-used-in-headers) | +| 0xFFFFFFFC | An unsigned 4 byte unsigned integer representing the architecture the image targets. See [enumerables section](#enumerables-used-in-headers) | +| 0xFFFFFFFB | An unsigned 4 byte unsigned integer representing the ABI of the image. See [enumerables section](#enumerables-used-in-headers) | + +These entries contain information about the compilation that can be useful to tools and identifiers that can be used to correlate a perfmap with an image as described in ["Ready to Run format - debug directory entries"](./readytorun-format.md#additions-to-the-debug-directory). + + +### Content + +Each entry is a triplet - the relative address of a method with respect to the image start as an unsigned 4 byte integer, the number of bytes used by the native code represented by an unsigned 2 byte integer, and the name of the method. There's one entry per line after the header, and a method can appear more than once since if may have gone through cold/hot path splitting. + +## Enumerables used in headers. + +``` +PerfMapArchitectureToken + Unknown = 0, + ARM = 1, + ARM64 = 2, + X64 = 3, + X86 = 4, +``` + +``` +PerfMapOSToken + Unknown = 0, + Windows = 1, + Linux = 2, + OSX = 3, + FreeBSD = 4, + NetBSD = 5, + SunOS = 6, +``` + +``` +PerfMapAbiToken + Unknown = 0, + Default = 1, + Armel = 2, +``` diff --git a/docs/design/coreclr/botr/readytorun-format.md b/docs/design/coreclr/botr/readytorun-format.md index 63b032326403e4..bcbe91245f84bd 100644 --- a/docs/design/coreclr/botr/readytorun-format.md +++ b/docs/design/coreclr/botr/readytorun-format.md @@ -44,6 +44,13 @@ without MSIL embedding are copied to the output folder next to the composite R2R and are rewritten by the compiler to include a formal ReadyToRun header with forwarding information pointing to the owner composite R2R executable (section `OwnerCompositeExecutable`). +# Additions to the debug directory + +Currently shipping PE envelopes - both single-file and composite - can contain records for additional +debug information in the debug directory. One such entry specific to R2R images is the one for R2R PerfMaps. +The format of the auxiliary file is described [R2R perfmap format](./r2r-perfmap-format.md) and the corresponding +debug directory entry is described in [PE COFF](../../../design/specs/PE-COFF.md#r2r-perfmap-debug-directory-entry-type-21). + ## Future Improvements The limitations of the current format are: