Add code coverage to the consolidated pipeline

.azure-pipelines/ultimate-pipeline.yml

@@ -200,7 +200,7 @@ stages:
       - template: steps/install-dotnet.yml
       - template: steps/restore-working-directory.yml
 
-      - script: build.cmd BuildAndRunManagedUnitTests
+      - script: build.cmd BuildAndRunManagedUnitTests --code-coverage
         displayName: Build and Test
 
       - publish: build_data
@@ -213,7 +213,7 @@ stages:
           testResultsFormat: VSTest
           testResultsFiles: build_data/results/**/*.trx
         condition: succeededOrFailed()
-
+        
     - job: native
       steps:
       - template: steps/install-dotnet.yml
@@ -241,7 +241,7 @@ stages:
         parameters:
           artifact: build-macos-working-directory
 
-      - script: ./build.sh BuildAndRunManagedUnitTests
+      - script: ./build.sh BuildAndRunManagedUnitTests --code-coverage
         displayName: Build and Test
 
       - publish: build_data
@@ -254,6 +254,8 @@ stages:
           testResultsFormat: VSTest
           testResultsFiles: build_data/results/**/*.trx
         condition: succeededOrFailed()
+
+
 
 - stage: unit_tests_linux
   condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'))
@@ -278,7 +280,7 @@ stages:
       parameters:
         build: true
         baseImage: $(baseImage)
-        command: "BuildAndRunManagedUnitTests"
+        command: "BuildAndRunManagedUnitTests --code-coverage"
 
     - publish: build_data
       artifact: profiler-logs_unit_tests_linux_$(Agent.JobName)_$(System.JobAttempt)
@@ -290,6 +292,11 @@ stages:
         testResultsFormat: VSTest
         testResultsFiles: build_data/results/**/*.trx
       condition: succeededOrFailed()
+
+    - task: PublishPipelineArtifact@1
+      inputs:
+        targetPath: 'build_data/results/'
+      continueOnError: true
 
 - stage: unit_tests_arm64
   condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'))
@@ -350,7 +357,7 @@ stages:
       displayName: 'Start CosmosDB Emulator'
       workingDirectory: $(Pipeline.Workspace)
 
-    - script: build.cmd BuildAndRunWindowsIntegrationTests --PrintDriveSpace
+    - script: build.cmd BuildAndRunWindowsIntegrationTests --PrintDriveSpace --code-coverage
       displayName: Run integration tests
 
     - task: PublishTestResults@2
@@ -359,6 +366,11 @@ stages:
         testResultsFormat: VSTest
         testResultsFiles: build_data/results/**/*.trx
       condition: succeededOrFailed()
+
+    - task: PublishPipelineArtifact@1
+      inputs:
+        targetPath: 'build_data/results/'
+      continueOnError: true
 
 - stage: integration_tests_windows_iis
   condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'))
@@ -408,13 +420,7 @@ stages:
         containerregistrytype: Container Registry
         dockerComposeCommand: up -d IntegrationTests.IIS
 
-    - powershell: |
-        [System.Reflection.Assembly]::Load("System.EnterpriseServices, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")
-        $publish = New-Object System.EnterpriseServices.Internal.Publish
-        Get-ChildItem $(tracerHome)/net45 -Filter *.dll | Foreach-Object { $publish.GacInstall($_.FullName) }
-      displayName: Add net45 Datadog.Trace.ClrProfiler.Managed assets to the GAC
-
-    - script: build.cmd RunWindowsIisIntegrationTests
+    - script: build.cmd RunWindowsIisIntegrationTests --code-coverage
       displayName: RunWindowsIisIntegrationTests
 
     - task: PublishTestResults@2
@@ -424,6 +430,11 @@ stages:
         testResultsFiles: build_data/results/**/*.trx
       condition: succeededOrFailed()
 
+    - task: PublishPipelineArtifact@1
+      inputs:
+        targetPath: 'build_data/results/'
+      continueOnError: true
+
     - task: DockerCompose@0
       displayName: docker-compose stop services
       inputs:
@@ -932,3 +943,33 @@ stages:
       displayName: Crank
       env:
         DD_SERVICE: dd-trace-dotnet
+- stage: coverage
+  condition: and(succeeded(), eq(variables['isScheduledBuild'], 'False'))
+  dependsOn: [integration_tests_windows, integration_tests_windows_iis, integration_tests_linux, unit_tests_linux, unit_tests_macos, unit_tests_windows]
+  jobs:
+    - job: Windows
+      timeoutInMinutes: 30
+
+      pool:
+        vmImage: windows-2019
+
+      steps:
+      - template: steps/restore-working-directory.yml
+
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          patterns: '**/coverage.cobertura.xml'
+          path: $(Build.SourcesDirectory)/cover
+
+      - task: reportgenerator@4
+        inputs:
+          reports: '$(Build.SourcesDirectory)\cover\**\coverage.cobertura.xml'
+          targetdir: '$(Build.SourcesDirectory)\coveragereport'
+          sourcedirs: '$(Build.SourcesDirectory);..'
+          reporttypes: 'Cobertura'
+
+      - task: PublishCodeCoverageResults@1
+        inputs:
+          codeCoverageTool: 'Cobertura'
+          summaryFileLocation: '$(Build.SourcesDirectory)/coveragereport/Cobertura.xml'
+          pathToSources: '$(Build.SourcesDirectory)'

build/_build/Build.Steps.cs

@@ -57,32 +57,32 @@ partial class Build
 
     IEnumerable<MSBuildTargetPlatform> ArchitecturesForPlatform =>
         Equals(Platform, MSBuildTargetPlatform.x64)
-            ? new[] {MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86}
-            : new[] {MSBuildTargetPlatform.x86};
+            ? new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 }
+            : new[] { MSBuildTargetPlatform.x86 };
 
     bool IsArm64 => RuntimeInformation.ProcessArchitecture == Architecture.Arm64;
     string LinuxArchitectureIdentifier => IsArm64 ? "arm64" : Platform.ToString();
 
-    IEnumerable<string> LinuxPackageTypes => IsAlpine ? new[] {"tar"} : new[] {"deb", "rpm", "tar"};
+    IEnumerable<string> LinuxPackageTypes => IsAlpine ? new[] { "tar" } : new[] { "deb", "rpm", "tar" };
 
-    IEnumerable<Project> ProjectsToPack => new []
+    IEnumerable<Project> ProjectsToPack => new[]
     {
         Solution.GetProject(Projects.DatadogTrace),
         Solution.GetProject(Projects.DatadogTraceOpenTracing),
     };
 
-    Project[] ParallelIntegrationTests => new []
+    Project[] ParallelIntegrationTests => new[]
     {
         Solution.GetProject(Projects.TraceIntegrationTests),
         Solution.GetProject(Projects.OpenTracingIntegrationTests),
     };
 
-    Project[] ClrProfilerIntegrationTests => new []
+    Project[] ClrProfilerIntegrationTests => new[]
     {
         Solution.GetProject(Projects.ClrProfilerIntegrationTests)
     };
 
-    readonly IEnumerable<TargetFramework> TargetFrameworks = new []
+    readonly IEnumerable<TargetFramework> TargetFrameworks = new[]
     {
         TargetFramework.NET45,
         TargetFramework.NET461,
@@ -439,7 +439,7 @@ partial class Build
 
                 foreach (var packageType in LinuxPackageTypes)
                 {
-                    var args = new []
+                    var args = new[]
                     {
                         "-f",
                         "-s dir",
@@ -522,6 +522,7 @@ partial class Build
                     .SetTargetPlatformAnyCPU()
                     .SetDDEnvironmentVariables("dd-tracer-dotnet")
                     .EnableMemoryDumps()
+                    .When(CodeCoverage, ConfigureCodeCoverage)
                     .CombineWith(testProjects, (x, project) => x
                         .EnableTrxLogOutput(GetResultsDirectory(project))
                         .SetProjectFile(project)));
@@ -623,18 +624,18 @@ partial class Build
                             && !x.Contains("StackExchange.Redis.AssemblyConflict.LegacyProject")
                             && !x.Contains("dependency-libs"));
 
-             // Allow restore here, otherwise things go wonky with runtime identifiers
-             // in some target frameworks. No, I don't know why
-             DotNetBuild(x => x
-                 // .EnableNoRestore()
-                 .EnableNoDependencies()
-                 .SetConfiguration(BuildConfiguration)
-                 .SetTargetPlatform(Platform)
-                 .SetNoWarnDotNetCore3()
-                 .When(!string.IsNullOrEmpty(NugetPackageDirectory), o =>
-                     o.SetPackageDirectory(NugetPackageDirectory))
-                 .CombineWith(regressionLibs, (x, project) => x
-                     .SetProjectFile(project)));
+            // Allow restore here, otherwise things go wonky with runtime identifiers
+            // in some target frameworks. No, I don't know why
+            DotNetBuild(x => x
+                // .EnableNoRestore()
+                .EnableNoDependencies()
+                .SetConfiguration(BuildConfiguration)
+                .SetTargetPlatform(Platform)
+                .SetNoWarnDotNetCore3()
+                .When(!string.IsNullOrEmpty(NugetPackageDirectory), o =>
+                    o.SetPackageDirectory(NugetPackageDirectory))
+                .CombineWith(regressionLibs, (x, project) => x
+                    .SetProjectFile(project)));
         });
 
     Target CompileFrameworkReproductions => _ => _
@@ -734,7 +735,7 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                 .SetProperty("DeployOnBuild", true)
                 .SetProperty("PublishProfile", publishProfile)
                 .SetMaxCpuCount(null)
-                .CombineWith(aspnetProjects, (c, project ) => c
+                .CombineWith(aspnetProjects, (c, project) => c
                     .SetTargetPath(project))
             );
         });
@@ -745,6 +746,7 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
         .After(CompileIntegrationTests)
         .After(CompileSamples)
         .After(CompileFrameworkReproductions)
+        .After(BuildWindowsIntegrationTests)
         .Requires(() => IsWin)
         .Executes(() =>
         {
@@ -760,6 +762,7 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                     .EnableNoRestore()
                     .EnableNoBuild()
                     .When(!string.IsNullOrEmpty(Filter), c => c.SetFilter(Filter))
+                    .When(CodeCoverage, ConfigureCodeCoverage)
                     .CombineWith(ParallelIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
                         .SetProjectFile(project)), degreeOfParallelism: 4);
@@ -774,6 +777,7 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                     .EnableNoRestore()
                     .EnableNoBuild()
                     .SetFilter(Filter ?? "(RunOnWindows=True|Category=Smoke)&LoadFromGAC!=True&IIS!=True")
+                    .When(CodeCoverage, ConfigureCodeCoverage)
                     .CombineWith(ClrProfilerIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
                         .SetProjectFile(project)));
@@ -801,9 +805,11 @@ _ when projectPath.ToString().Contains("Samples.OracleMDA") => false,
                     .SetDotnetPath(Platform)
                     .SetConfiguration(BuildConfiguration)
                     .SetTargetPlatform(Platform)
+                    .When(Framework != null, o => o.SetFramework(Framework))
                     .EnableNoRestore()
                     .EnableNoBuild()
                     .SetFilter(Filter ?? "(RunOnWindows=True|Category=Smoke)&LoadFromGAC=True")
+                    .When(CodeCoverage, ConfigureCodeCoverage)
                     .CombineWith(ClrProfilerIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
                         .SetProjectFile(project)));
@@ -903,6 +909,8 @@ var name when multiApiProjects.Contains(name) => false,
                     .SetFramework(Framework)
                     // .SetTargetPlatform(Platform)
                     .SetNoWarnDotNetCore3()
+                    .SetProperty("ExcludeManagedProfiler", "true")
+                    .SetProperty("ExcludeNativeProfiler", "true")
                     .SetProperty("ManagedProfilerOutputDirectory", TracerHomeDirectory)
                     .When(TestAllPackageVersions, o => o.SetProperty("TestAllPackageVersions", "true"))
                     .When(!string.IsNullOrEmpty(NugetPackageDirectory), o => o.SetPackageDirectory(NugetPackageDirectory))
@@ -939,7 +947,7 @@ var name when multiApiProjects.Contains(name) => false,
         {
             // Build and restore for all versions
             // Annoyingly this rebuilds everything again and again.
-            var targets = new [] { "RestoreSamplesForPackageVersionsOnly", "RestoreAndBuildSamplesForPackageVersionsOnly" };
+            var targets = new[] { "RestoreSamplesForPackageVersionsOnly", "RestoreAndBuildSamplesForPackageVersionsOnly" };
 
             // /nowarn:NU1701 - Package 'x' was restored using '.NETFramework,Version=v4.6.1' instead of the project target framework '.NETCoreApp,Version=v2.1'.
             DotNetMSBuild(x => x
@@ -949,6 +957,8 @@ var name when multiApiProjects.Contains(name) => false,
                 .SetProperty("TargetFramework", Framework.ToString())
                 .SetProperty("ManagedProfilerOutputDirectory", TracerHomeDirectory)
                 .SetProperty("BuildInParallel", "true")
+                .SetProperty("ExcludeManagedProfiler", "true")
+                .SetProperty("ExcludeNativeProfiler", "true")
                 .SetProcessArgumentConfigurator(arg => arg.Add("/nowarn:NU1701"))
                 .AddProcessEnvironmentVariable("TestAllPackageVersions", "true")
                 .When(TestAllPackageVersions, o => o.SetProperty("TestAllPackageVersions", "true"))
@@ -1031,6 +1041,7 @@ var name when multiApiProjects.Contains(name) => false,
                         .SetFilter(filter)
                         .When(TestAllPackageVersions, o => o
                             .SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
+                        .When(CodeCoverage, ConfigureCodeCoverage)
                         .CombineWith(ParallelIntegrationTests, (s, project) => s
                             .EnableTrxLogOutput(GetResultsDirectory(project))
                             .SetProjectFile(project)),
@@ -1047,6 +1058,7 @@ var name when multiApiProjects.Contains(name) => false,
                     .SetFilter(filter)
                     .When(TestAllPackageVersions, o => o
                         .SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
+                    .When(CodeCoverage, ConfigureCodeCoverage)
                     .CombineWith(ClrProfilerIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
                         .SetProjectFile(project))
@@ -1079,6 +1091,22 @@ private void MoveLogsToBuildData()
         }
     }
 
+    private DotNetTestSettings ConfigureCodeCoverage(DotNetTestSettings settings)
+    {
+        var strongNameKeyPath = Solution.Directory / "Datadog.Trace.snk";
+
+        return settings.SetDataCollector("XPlat Code Coverage")
+                .SetProcessArgumentConfigurator(
+                     args =>
+                         args.Add("--")
+                             .Add("RunConfiguration.DisableAppDomain=true") // https://github.com/coverlet-coverage/coverlet/issues/347
+                             .Add("DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.SkipAutoProps=true")
+                             .Add("DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura")
+                             .Add($"DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.StrongNameKey=\"{strongNameKeyPath}\"")
+                             .Add("DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Exclude=[*]Datadog.Trace.Vendors.*,")
+                             .Add("DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Include=\"[Datadog.Trace.ClrProfiler.*]*,[Datadog.Trace]*,[Datadog.Trace.AspNet]*\""));
+    }
+
     protected override void OnTargetStart(string target)
     {
         if (PrintDriveSpace)
@@ -1092,7 +1120,7 @@ protected override void OnTargetStart(string target)
 
         static string PrettyPrint(long bytes)
         {
-            var power = Math.Min((int) Math.Log(bytes, 1000), 4);
+            var power = Math.Min((int)Math.Log(bytes, 1000), 4);
             var normalised = bytes / Math.Pow(1000, power);
             return power switch
             {

build/_build/Build.cs

@@ -59,6 +59,9 @@ partial class Build : NukeBuild
     [Parameter("Override the default test filters for integration tests. (Optional)")]
     readonly string Filter;
 
+    [Parameter("Enables code coverage")]
+    readonly bool CodeCoverage;
+
     Target Info => _ => _
         .Description("Describes the current configuration")
         .Before(Clean, Restore, BuildTracerHome)

docker-compose.yml

@@ -217,7 +217,7 @@ services:
       args:
         - DOTNETSDK_VERSION=${dotnetCoreSdk5Version:-5.0.103}
     image: dd-trace-dotnet/${baseImage:-debian}-tester
-    command: dotnet /build/bin/Debug/_build.dll RunLinuxIntegrationTests --framework ${framework:-netcoreapp3.1}
+    command: dotnet /build/bin/Debug/_build.dll RunLinuxIntegrationTests --framework ${framework:-netcoreapp3.1} --code-coverage
     volumes:
       - ./:/project
     environment:

nuget.config

@@ -3,6 +3,7 @@
   <packageSources>
     <!--To inherit the global NuGet package sources remove the <clear/> line below -->
     <clear />
+    <add key="Azure" value="https://pkgs.dev.azure.com/datadoghq/dd-trace-dotnet/_packaging/Public_Feed/nuget/v3/index.json" />
     <add key="nuget" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
 </configuration>

test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc4Tests.cs

@@ -106,6 +106,7 @@ public AspNetMvc4Tests(IisFixture iisFixture, ITestOutputHelper output, bool ena
             }
 
             _iisFixture = iisFixture;
+            _iisFixture.ShutdownPath = "/home/shutdown";
             _iisFixture.TryStartIis(this, classicMode);
             _testName = nameof(AspNetMvc4Tests)
                       + (enableCallTarget ? ".CallSite" : ".CallTarget")

test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetMvc5Tests.cs

@@ -106,6 +106,7 @@ public AspNetMvc5Tests(IisFixture iisFixture, ITestOutputHelper output, bool ena
             }
 
             _iisFixture = iisFixture;
+            _iisFixture.ShutdownPath = "/home/shutdown";
             _iisFixture.TryStartIis(this, classicMode);
             _testName = nameof(AspNetMvc5Tests)
                       + (enableCallTarget ? ".CallSite" : ".CallTarget")

test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetWebApi2Tests.cs

@@ -106,6 +106,7 @@ public AspNetWebApi2Tests(IisFixture iisFixture, ITestOutputHelper output, bool
             }
 
             _iisFixture = iisFixture;
+            _iisFixture.ShutdownPath = "/home/shutdown";
             _iisFixture.TryStartIis(this, classicMode);
             _testName = nameof(AspNetWebApi2Tests)
                       + (enableCallTarget ? ".CallSite" : ".CallTarget")