Merge pull request #2 from tonerdo/master

Update fork

Documentation/VSTestIntegration.md

@@ -1,30 +1,22 @@
 # Coverlet Integration with VSTest
 
-## Motivation
-
-The cross platform solution for Code Coverage generation for .NET Core projects (in a consistent manner).
-The asks for Code Coverage support for .NET Core on Linux is the most commented issue on vstest repo:
-https://github.com/Microsoft/vstest/issues/981#issuecomment-320311552
-
-## Summary
-
-What would integrating Coverlet with Microsoft Test Platform mean:
-
-1. Coverlet based coverage solution is available as a data collector that does the instrumentation of the necessary modules before test execution and appropriate restore after.
-2. Authoring - When new test projects are created (dotnet mstest), references to the data collector package are added by default. This reduces adoption friction for customers.
-3. Test execution - Today Coverlet is invoked as a msbuild target and as such only works with dotnet test (and requires a csproj). With the data collector becoming available, coverage can also be collected when tests are run on built binaries (dotnet vstest)
-
-## Proposed Solution
-
-### Scenarios to support
-
-The following table summarizes the support that needs to be added for a seamless code coverage collection for .NET Core on both Linux and Windows platforms:
 
 | Entry point | How will code coverage be enabled? | Syntax                                                               |
 |-------------|------------------------------------|----------------------------------------------------------------------|
 |dotnet test CLI              | Through a switch to condition data collection | `dotnet test --collect:"XPlat Code Coverage"`   |
 |dotnet vstest CLI            | Through a switch to condition data collection | `dotnet vstest --collect:"XPlat Code Coverage"` |
 
+NB. If you're using `dotnet vstest` you MUST `publish` your test project before i.e.
+```bash
+C:\project
+dotnet publish
+...
+  vstest -> C:\project\bin\Debug\netcoreapp3.0\testdll.dll
+  vstest -> C:\project\bin\Debug\netcoreapp3.0\publish\
+...
+dotnet vstest C:\project\bin\Debug\netcoreapp3.0\publish\testdll.dll --collect:"XPlat Code Coverage"
+```
+
 ### Coverlet Options Supported with VSTest
 
 #### Default
@@ -75,11 +67,6 @@ This runsettings file can easily be provided using command line option as given
 
 2. `dotnet vstest --settings coverletArgs.runsettings`
 
-
-#### Scope of Enhancement
-
-Currently, advanced options are supported via runsettings. Providing support through additional command line arguments in vstest can be taken up separately.
-
 ## Implementation Details
 
 The proposed solution is implemented with the help of [datacollectors](https://github.com/Microsoft/vstest-docs/blob/master/docs/extensions/datacollector.md).

src/coverlet.collector/DataCollection/CoverletSettingsParser.cs

@@ -131,13 +131,6 @@ private string[] ParseExcludeFilters(XmlElement configurationElement)
                 }
             }
 
-            // if we've only one element mean that we only added CoverletConstants.DefaultExcludeFilter
-            // so add default exclusions
-            if (excludeFilters.Count == 1)
-            {
-                excludeFilters.Add("[xunit*]*");
-            }
-
             return excludeFilters.ToArray();
         }
 

src/coverlet.console/Program.cs

@@ -59,12 +59,6 @@ static int Main(string[] args)
                     logger.Level = verbosity.ParsedValue;
                 }
 
-                // We add default exclusion filter if no specified
-                if (excludeFilters.Values.Count == 0)
-                {
-                    excludeFilters.Values.Add("[xunit*]*");
-                }
-
                 Coverage coverage = new Coverage(module.Value,
                     includeFilters.Values.ToArray(),
                     includeDirectories.Values.ToArray(),

src/coverlet.core/Instrumentation/InstrumenterResult.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Runtime.Serialization;
 
 namespace Coverlet.Core.Instrumentation
@@ -17,6 +18,7 @@ public class Line
         public int Hits;
     }
 
+    [DebuggerDisplay("Line = {Number} Offset = {Offset} EndOffset = {EndOffset} Path = {Path} Ordinal = {Ordinal} Hits = {Hits}")]
     [DataContract]
     public class Branch : Line
     {
@@ -30,6 +32,7 @@ public class Branch : Line
         public uint Ordinal;
     }
 
+    [DebuggerDisplay("line = {Line} Ordinal = {Ordinal}")]
     // Implements IEquatable because is used by dictionary key https://docs.microsoft.com/en-us/dotnet/api/system.iequatable-1?view=netcore-2.2#remarks
     [DataContract]
     public class BranchKey : IEquatable<BranchKey>

src/coverlet.msbuild.tasks/InstrumentationTask.cs

@@ -104,12 +104,6 @@ public override bool Execute()
                 var excludedSourceFiles = _excludeByFile?.Split(',');
                 var excludeAttributes = _excludeByAttribute?.Split(',');
 
-                // We add default exclusion filter if no specified
-                if (excludeFilters is null || excludeFilters.Length == 0)
-                {
-                    excludeFilters = new string[] { "[xunit*]*" };
-                }
-
                 Coverage coverage = new Coverage(_path, includeFilters, includeDirectories, excludeFilters, excludedSourceFiles, excludeAttributes, _includeTestAssembly, _singleHit, _mergeWith, _useSourceLink, _logger);
                 CoveragePrepareResult prepareResult = coverage.PrepareModules();
                 InstrumenterState = new TaskItem(System.IO.Path.GetTempFileName());

test/coverlet.collector.tests/CoverletSettingsParserTests.cs

@@ -78,33 +78,5 @@ private void CreateCoverletNodes(XmlDocument doc, XmlElement configElement, stri
             node.InnerText = nodeValue;
             configElement.AppendChild(node);
         }
-
-        [Fact]
-        public void ParseShouldSkipXunitModulesIfEmptyExclude()
-        {
-            var testModules = new List<string> { "abc.dll" };
-
-            CoverletSettings coverletSettings = _coverletSettingsParser.Parse(null, testModules);
-
-            Assert.Equal("[coverlet.*]*", coverletSettings.ExcludeFilters[0]);
-            Assert.Equal("[xunit*]*", coverletSettings.ExcludeFilters[1]);
-            Assert.Equal(2, coverletSettings.ExcludeFilters.Length);
-        }
-
-        [Fact]
-        public void ParseShouldNotSkipXunitModulesIfNotEmptyExclude()
-        {
-            var testModules = new List<string> { "abc.dll" };
-            var doc = new XmlDocument();
-            var configElement = doc.CreateElement("Configuration");
-            this.CreateCoverletNodes(doc, configElement, CoverletConstants.ExcludeFiltersElementName, "[coverlet.*.tests?]*");
-
-            CoverletSettings coverletSettings = _coverletSettingsParser.Parse(configElement, testModules);
-
-            Assert.Equal("[coverlet.*]*", coverletSettings.ExcludeFilters[0]);
-            Assert.Equal("[coverlet.*.tests?]*", coverletSettings.ExcludeFilters[1]);
-            Assert.Equal(2, coverletSettings.ExcludeFilters.Length);
-            Assert.DoesNotContain("[xunit*]*", coverletSettings.ExcludeFilters);
-        }
     }
 }

test/coverlet.core.tests/CoverageTests.cs

@@ -62,7 +62,7 @@ public void TestCoverageWithTestAssembly()
         }
 
         [Fact]
-        public void Condition_If()
+        public void SelectionStatements_If()
         {
             // We need to pass file name to remote process where it save instrumentation result
             // Similar to msbuild input/output
@@ -73,7 +73,7 @@ public void Condition_If()
                 RemoteExecutor.Invoke(async pathSerialize =>
                 {
                     // Run load and call a delegate passing class as dynamic to simplify method call
-                    CoveragePrepareResult coveragePrepareResult = await TestInstrumentationHelper.Run<Conditions>(instance =>
+                    CoveragePrepareResult coveragePrepareResult = await TestInstrumentationHelper.Run<SelectionStatements>(instance =>
                     {
                         // We call method to trigger coverage hits
                         instance.If(true);
@@ -91,7 +91,7 @@ public void Condition_If()
                 CoverageResult result = TestInstrumentationHelper.GetCoverageResult(path);
 
                 // Asserts on doc/lines/branches
-                result.Document("Instrumentation.cs")
+                result.Document("Instrumentation.SelectionStatements.cs")
                       // (line, hits)
                       .AssertLinesCovered((11, 1), (15, 0))
                       // (line,ordinal,hits)
@@ -106,5 +106,35 @@ public void Condition_If()
                 File.Delete(path);
             }
         }
+
+        [Fact]
+        public void SelectionStatements_Switch()
+        {
+            string path = Path.GetTempFileName();
+            try
+            {
+                RemoteExecutor.Invoke(async pathSerialize =>
+                {
+                    CoveragePrepareResult coveragePrepareResult = await TestInstrumentationHelper.Run<SelectionStatements>(instance =>
+                    {
+                        instance.Switch(1);
+                        return Task.CompletedTask;
+                    }, pathSerialize);
+                    return 0;
+                }, path).Dispose();
+
+                CoverageResult result = TestInstrumentationHelper.GetCoverageResult(path);
+
+                result.Document("Instrumentation.SelectionStatements.cs")
+                      .AssertLinesCovered(BuildConfiguration.Release, (24, 1), (26, 0), (28, 0))
+                      .AssertBranchesCovered(BuildConfiguration.Release, (24, 1, 1))
+                      .AssertLinesCovered(BuildConfiguration.Debug, (20, 1), (21, 1), (24, 1), (30, 1))
+                      .AssertBranchesCovered(BuildConfiguration.Debug, (21, 0, 0), (21, 1, 1), (21, 2, 0), (21, 3, 0));
+            }
+            finally
+            {
+                File.Delete(path);
+            }
+        }
     }
 }

test/coverlet.core.tests/Instrumentation/InstrumenterTests.cs

@@ -42,10 +42,12 @@ public void TestCoreLibInstrumentation()
             Assert.NotNull(result);
         }
 
-        [Fact]
-        public void TestInstrument()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestInstrument(bool singleHit)
         {
-            var instrumenterTest = CreateInstrumentor();
+            var instrumenterTest = CreateInstrumentor(singleHit: singleHit);
 
             var result = instrumenterTest.Instrumenter.Instrument();
 
@@ -55,10 +57,12 @@ public void TestInstrument()
             instrumenterTest.Directory.Delete(true);
         }
 
-        [Fact]
-        public void TestInstrumentCoreLib()
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void TestInstrumentCoreLib(bool singleHit)
         {
-            var instrumenterTest = CreateInstrumentor(fakeCoreLibModule: true);
+            var instrumenterTest = CreateInstrumentor(fakeCoreLibModule: true, singleHit: singleHit);
 
             var result = instrumenterTest.Instrumenter.Instrument();
 
@@ -144,7 +148,7 @@ public void TestInstrument_ClassesWithPropertyWithCustomExcludeAttributeAreExclu
             instrumenterTest.Directory.Delete(true);
         }
 
-        private InstrumenterTest CreateInstrumentor(bool fakeCoreLibModule = false, string[] attributesToIgnore = null)
+        private InstrumenterTest CreateInstrumentor(bool fakeCoreLibModule = false, string[] attributesToIgnore = null, string[] excludedFiles = null, bool singleHit = false)
         {
             string module = GetType().Assembly.Location;
             string pdb = Path.Combine(Path.GetDirectoryName(module), Path.GetFileNameWithoutExtension(module) + ".pdb");
@@ -168,7 +172,8 @@ private InstrumenterTest CreateInstrumentor(bool fakeCoreLibModule = false, stri
             File.Copy(pdb, Path.Combine(directory.FullName, destPdb), true);
 
             module = Path.Combine(directory.FullName, destModule);
-            Instrumenter instrumenter = new Instrumenter(module, identifier, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), attributesToIgnore, false, new Mock<ILogger>().Object);
+            Instrumenter instrumenter = new Instrumenter(module, identifier, Array.Empty<string>(), Array.Empty<string>(), excludedFiles, attributesToIgnore, singleHit, new Mock<ILogger>().Object);
+
             return new InstrumenterTest
             {
                 Instrumenter = instrumenter,
@@ -253,5 +258,14 @@ public void SkipEmbeddedPpdbWithoutLocalSource()
             loggerMock.VerifyNoOtherCalls();
         }
 
+        [Fact]
+        public void TestInstrument_MissingModule()
+        {
+            var loggerMock = new Mock<ILogger>();
+            var instrumenter = new Instrumenter("test", "_test_instrumented", Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), false, loggerMock.Object);
+            Assert.False(instrumenter.CanInstrument());
+            loggerMock.Verify(l => l.LogWarning(It.IsAny<string>()));
+        }
+
     }
-}
+}

test/coverlet.core.tests/InstrumenterHelper.cs

@@ -18,6 +18,13 @@
 
 namespace Coverlet.Core.Tests
 {
+    [Flags]
+    public enum BuildConfiguration
+    {
+        Debug = 1,
+        Release = 2
+    }
+
     public static class TestInstrumentationAssert
     {
         public static Document Document(this CoverageResult coverageResult, string docName)
@@ -42,12 +49,24 @@ public static Document Document(this CoverageResult coverageResult, string docNa
         }
 
         public static Document AssertBranchesCovered(this Document document, params (int line, int ordinal, int hits)[] lines)
+        {
+            return AssertBranchesCovered(document, BuildConfiguration.Debug | BuildConfiguration.Release, lines);
+        }
+
+        public static Document AssertBranchesCovered(this Document document, BuildConfiguration configuration, params (int line, int ordinal, int hits)[] lines)
         {
             if (document is null)
             {
                 throw new ArgumentNullException(nameof(document));
             }
 
+            BuildConfiguration buildConfiguration = GetAssemblyBuildConfiguration();
+
+            if ((buildConfiguration & configuration) != buildConfiguration)
+            {
+                return document;
+            }
+
             List<string> branchesToCover = new List<string>(lines.Select(b => $"[line {b.line} ordinal {b.ordinal}]"));
             foreach (KeyValuePair<BranchKey, Branch> branch in document.Branches)
             {
@@ -77,12 +96,24 @@ public static Document AssertBranchesCovered(this Document document, params (int
         }
 
         public static Document AssertLinesCovered(this Document document, params (int line, int hits)[] lines)
+        {
+            return AssertLinesCovered(document, BuildConfiguration.Debug | BuildConfiguration.Release, lines);
+        }
+
+        public static Document AssertLinesCovered(this Document document, BuildConfiguration configuration, params (int line, int hits)[] lines)
         {
             if (document is null)
             {
                 throw new ArgumentNullException(nameof(document));
             }
 
+            BuildConfiguration buildConfiguration = GetAssemblyBuildConfiguration();
+
+            if ((buildConfiguration & configuration) != buildConfiguration)
+            {
+                return document;
+            }
+
             List<int> linesToCover = new List<int>(lines.Select(l => l.line));
             foreach (KeyValuePair<int, Line> line in document.Lines)
             {
@@ -106,6 +137,23 @@ public static Document AssertLinesCovered(this Document document, params (int li
 
             return document;
         }
+
+        private static BuildConfiguration GetAssemblyBuildConfiguration()
+        {
+            var configurationAttribute = Assembly.GetExecutingAssembly().GetCustomAttribute<AssemblyConfigurationAttribute>();
+            if (configurationAttribute.Configuration.Equals("Debug", StringComparison.InvariantCultureIgnoreCase))
+            {
+                return Tests.BuildConfiguration.Debug;
+            }
+            else if (configurationAttribute.Configuration.Equals("Release", StringComparison.InvariantCultureIgnoreCase))
+            {
+                return Tests.BuildConfiguration.Release;
+            }
+            else
+            {
+                throw new NotSupportedException($"Build configuration '{configurationAttribute.Configuration}' not supported");
+            }
+        }
     }
 
     public static class TestInstrumentationHelper