diff --git a/src/vstest.console/Internal/ConsoleLogger.cs b/src/vstest.console/Internal/ConsoleLogger.cs index 4ef52ff60a..846ad79c7e 100644 --- a/src/vstest.console/Internal/ConsoleLogger.cs +++ b/src/vstest.console/Internal/ConsoleLogger.cs @@ -11,14 +11,13 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal using System.Globalization; using System.Linq; using System.Text; - + using System.Xml.XPath; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.Utilities; - + using NuGet.Frameworks; using CommandLineResources = Resources.Resources; - /// /// Logger for sending output to the console. /// All the console logger messages prints to Standard Output with respective color, except OutputLevel.Error messages @@ -81,6 +80,16 @@ internal class ConsoleLogger : ITestLoggerWithParameters /// public const string ExecutionIdPropertyIdentifier = "ExecutionId"; + // Figure out the longest result string (+1 for ! where applicable), so we don't + // get misaligned output on non-english systems + private static int LongestResultIndicator = new[] + { + CommandLineResources.FailedTestIndicator.Length + 1, + CommandLineResources.PassedTestIndicator.Length + 1, + CommandLineResources.SkippedTestIndicator.Length + 1, + CommandLineResources.None.Length + }.Max(); + #endregion internal enum Verbosity @@ -104,7 +113,11 @@ internal enum Verbosity #endif private bool testRunHasErrorMessages = false; - private ConcurrentDictionary leafExecutionIdAndTestOutcomePairDictionary = new ConcurrentDictionary(); + + /// + /// Framework on which the test runs. + /// + private string targetFramework; #endregion @@ -148,6 +161,12 @@ protected static IOutput Output /// public Verbosity VerbosityLevel => verbosityLevel; + /// + /// Tracks leaf test outcomes per source. This is needed to correctly count hierarchical tests as well as + /// tracking counts per source for the minimal and quiet output. + /// + private ConcurrentDictionary leafTestResults { get; set; } + #endregion #region ITestLoggerWithParameters @@ -183,6 +202,7 @@ public void Initialize(TestLoggerEvents events, string testRunDirectory) // Register for the discovery events. events.DiscoveryMessage += this.TestMessageHandler; + this.leafTestResults = new ConcurrentDictionary(); // TODO Get changes from https://github.com/Microsoft/vstest/pull/1111/ // events.DiscoveredTests += DiscoveredTestsHandler; @@ -218,6 +238,9 @@ public void Initialize(TestLoggerEvents events, Dictionary param bool.TryParse(enableProgress, out EnableProgress); } + parameters.TryGetValue(DefaultLoggerParameterNames.TargetFramework, out this.targetFramework); + this.targetFramework = !string.IsNullOrEmpty(this.targetFramework) ? NuGetFramework.Parse(this.targetFramework).GetShortFolderName() : this.targetFramework; + Initialize(events, String.Empty); } #endregion @@ -410,7 +433,6 @@ private void TestRunStartHandler(object sender, TestRunStartEventArgs e) ValidateArg.NotNull(e, "e"); // Print all test containers. - Output.WriteLine(string.Empty, OutputLevel.Information); Output.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestSourcesDiscovered, CommandLineOptions.Instance.Sources.Count()), OutputLevel.Information); if (verbosityLevel == Verbosity.Detailed) { @@ -510,12 +532,21 @@ private void TestResultHandler(object sender, TestResultEventArgs e) var executionId = GetExecutionId(e.Result); var parentExecutionId = GetParentExecutionId(e.Result); - if (parentExecutionId != Guid.Empty && leafExecutionIdAndTestOutcomePairDictionary.ContainsKey(parentExecutionId)) + if (parentExecutionId != Guid.Empty) { - leafExecutionIdAndTestOutcomePairDictionary.TryRemove(parentExecutionId, out _); + // Not checking the result value. + // This would return false if the id did not exist, + // or true if it did exist. In either case the id is not in the dictionary + // which is our goal. + leafTestResults.TryRemove(parentExecutionId, out _); } - leafExecutionIdAndTestOutcomePairDictionary.TryAdd(executionId, e.Result.Outcome); + if (!leafTestResults.TryAdd(executionId, e.Result)) + { + // This would happen if the key already exists. This should not happen, because we are + // inserting by GUID key, so this would mean an error in our code. + throw new InvalidOperationException($"ExecutionId {executionId} already exists."); + }; switch (e.Result.Outcome) { @@ -618,28 +649,28 @@ private string GetFormattedDurationString(TimeSpan duration) var time = new List(); if (duration.Hours > 0) { - time.Add(duration.Hours + "h"); + time.Add(duration.Hours + " h"); } if (duration.Minutes > 0) { - time.Add(duration.Minutes + "m"); + time.Add(duration.Minutes + " m"); } if (duration.Hours == 0) { if (duration.Seconds > 0) { - time.Add(duration.Seconds + "s"); + time.Add(duration.Seconds + " s"); } - if (duration.Milliseconds > 0 && duration.Minutes == 0) + if (duration.Milliseconds > 0 && duration.Minutes == 0 && duration.Seconds == 0) { - time.Add(duration.Milliseconds + "ms"); + time.Add(duration.Milliseconds + " ms"); } } - return time.Count == 0 ? "< 1ms" : string.Join(" ", time); + return time.Count == 0 ? "< 1 ms" : string.Join(" ", time); } /// @@ -647,12 +678,12 @@ private string GetFormattedDurationString(TimeSpan duration) /// private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) { - var testsTotal = 0; - var testsPassed = 0; - var testsFailed = 0; - var testsSkipped = 0; // Stop the progress indicator as we are about to print the summary this.progressIndicator?.Stop(); + var passedTests = 0; + var failedTests = 0; + var skippedTests = 0; + var totalTests = 0; Output.WriteLine(string.Empty, OutputLevel.Information); // Printing Run-level Attachments @@ -670,23 +701,138 @@ private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) } } - foreach (KeyValuePair entry in leafExecutionIdAndTestOutcomePairDictionary) + var leafTestResultsPerSource = this.leafTestResults.Select(p => p.Value).GroupBy(r => r.TestCase.Source); + foreach (var sd in leafTestResultsPerSource) { - testsTotal++; - switch (entry.Value) + var source = sd.Key; + var sourceSummary = new SourceSummary(); + + foreach (var result in sd.ToArray()) { - case TestOutcome.Failed: - testsFailed++; - break; - case TestOutcome.Passed: - testsPassed++; - break; - case TestOutcome.Skipped: - testsSkipped++; - break; - default: - break; + sourceSummary.Duration += result.Duration; + switch (result.Outcome) + { + case TestOutcome.Passed: + sourceSummary.TotalTests++; + sourceSummary.PassedTests++; + break; + case TestOutcome.Failed: + sourceSummary.TotalTests++; + sourceSummary.FailedTests++; + break; + case TestOutcome.Skipped: + sourceSummary.TotalTests++; + sourceSummary.SkippedTests++; + break; + default: + break; + } + } + + if (verbosityLevel == Verbosity.Quiet || verbosityLevel == Verbosity.Minimal) + { + TestOutcome sourceOutcome = TestOutcome.None; + if (sourceSummary.FailedTests > 0) + { + sourceOutcome = TestOutcome.Failed; + } + else if (sourceSummary.PassedTests > 0) + { + sourceOutcome = TestOutcome.Passed; + } + else if (sourceSummary.SkippedTests > 0) + { + sourceOutcome = TestOutcome.Skipped; + } + + + string resultString; + switch (sourceOutcome) + { + case TestOutcome.Failed: + resultString = (CommandLineResources.FailedTestIndicator + "!").PadRight(LongestResultIndicator); + break; + case TestOutcome.Passed: + resultString = (CommandLineResources.PassedTestIndicator + "!").PadRight(LongestResultIndicator); + break; + case TestOutcome.Skipped: + resultString = (CommandLineResources.SkippedTestIndicator + "!").PadRight(LongestResultIndicator); + break; + default: + resultString = CommandLineResources.None.PadRight(LongestResultIndicator); + break; + }; + + var failed = sourceSummary.FailedTests.ToString().PadLeft(5); + var passed = sourceSummary.PassedTests.ToString().PadLeft(5); + var skipped = sourceSummary.SkippedTests.ToString().PadLeft(5); + var total = sourceSummary.TotalTests.ToString().PadLeft(5); + + + var frameworkString = string.IsNullOrEmpty(targetFramework) + ? string.Empty + : $"({targetFramework})"; + + var duration = GetFormattedDurationString(sourceSummary.Duration); + var sourceName = sd.Key.Split('\\').Last(); + + var outputLine = string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, + resultString, + failed, + passed, + skipped, + total, + duration, + sourceName, + frameworkString); + + + ConsoleColor? color = null; + if (sourceOutcome == TestOutcome.Failed) + { + color = ConsoleColor.Red; + } + else if (sourceOutcome == TestOutcome.Passed) + { + color = ConsoleColor.Green; + } + else if (sourceOutcome == TestOutcome.Skipped) + { + color = ConsoleColor.Yellow; + } + + if (color != null) + { + Output.Write(outputLine, OutputLevel.Information, color.Value); + } + else + { + Output.Write(outputLine, OutputLevel.Information); + } + + Output.Information(false, CommandLineResources.TestRunSummaryAssemblyAndFramework, + sourceName, + frameworkString); } + + passedTests += sourceSummary.PassedTests; + failedTests += sourceSummary.FailedTests; + skippedTests += sourceSummary.SkippedTests; + totalTests += sourceSummary.TotalTests; + } + + if (verbosityLevel == Verbosity.Quiet || verbosityLevel == Verbosity.Minimal) + { + if (e.IsCanceled) + { + Output.Error(false, CommandLineResources.TestRunCanceled); + } + else if (e.IsAborted) + { + Output.Error(false, CommandLineResources.TestRunAborted); + } + + return; } if (e.IsCanceled) @@ -697,36 +843,36 @@ private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) { Output.Error(false, CommandLineResources.TestRunAborted); } - else if (testsFailed > 0 || this.testRunHasErrorMessages) + else if (failedTests > 0 || this.testRunHasErrorMessages) { Output.Error(false, CommandLineResources.TestRunFailed); } - else if (testsTotal > 0) + else if (totalTests > 0) { Output.Information(false, ConsoleColor.Green, CommandLineResources.TestRunSuccessful); } // Output a summary. - if (testsTotal > 0) + if (totalTests > 0) { string totalTestsformat = (e.IsAborted || e.IsCanceled) ? CommandLineResources.TestRunSummaryForCanceledOrAbortedRun : CommandLineResources.TestRunSummaryTotalTests; - Output.Information(false, string.Format(CultureInfo.CurrentCulture, totalTestsformat, testsTotal)); + Output.Information(false, string.Format(CultureInfo.CurrentCulture, totalTestsformat, totalTests)); - if (testsPassed > 0) + if (passedTests > 0) { - Output.Information(false, ConsoleColor.Green, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, testsPassed)); + Output.Information(false, ConsoleColor.Green, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, passedTests)); } - if (testsFailed > 0) + if (failedTests > 0) { - Output.Information(false, ConsoleColor.Red, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, testsFailed)); + Output.Information(false, ConsoleColor.Red, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, failedTests)); } - if (testsSkipped > 0) + if (skippedTests > 0) { - Output.Information(false, ConsoleColor.Yellow, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummarySkippedTests, testsSkipped)); + Output.Information(false, ConsoleColor.Yellow, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummarySkippedTests, skippedTests)); } } - if (testsTotal > 0) + if (totalTests > 0) { if (e.ElapsedTimeInRunningTests.Equals(TimeSpan.Zero)) { diff --git a/src/vstest.console/Internal/SourceSummary.cs b/src/vstest.console/Internal/SourceSummary.cs new file mode 100644 index 0000000000..d0f228b6de --- /dev/null +++ b/src/vstest.console/Internal/SourceSummary.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal +{ + using System; + + /// + /// Summary of test results per source. + /// + internal class SourceSummary + { + /// + /// Total tests of a test run. + /// + public int TotalTests { get; set; } + + /// + /// Passed tests of a test run. + /// + public int PassedTests { get; set; } + + /// + /// Failed tests of a test run. + /// + public int FailedTests { get; set; } + + /// + /// Skipped tests of a test run. + /// + public int SkippedTests { get; set; } + + /// + /// Duration of the test run. + /// + public TimeSpan Duration { get; set; } + } +} diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs index db78353506..d4db636f10 100644 --- a/src/vstest.console/Resources/Resources.Designer.cs +++ b/src/vstest.console/Resources/Resources.Designer.cs @@ -1028,6 +1028,15 @@ internal static string NonDefaultFrameworkAndOrArchDetected { } } + /// + /// Looks up a localized string similar to None. + /// + internal static string None { + get { + return ResourceManager.GetString("None", resourceCulture); + } + } + /// /// Looks up a localized string similar to App package '{0}' does not has test executor entry point. For running unit tests for Windows Store apps, create app package using Windows Store app Unit Test Library project.. /// @@ -1567,6 +1576,24 @@ internal static string TestRunSuccessful { } } + /// + /// Looks up a localized string similar to {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5}. + /// + internal static string TestRunSummary { + get { + return ResourceManager.GetString("TestRunSummary", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to - {0} {1}. + /// + internal static string TestRunSummaryAssemblyAndFramework { + get { + return ResourceManager.GetString("TestRunSummaryAssemblyAndFramework", resourceCulture); + } + } + /// /// Looks up a localized string similar to Failed: {0}. /// diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx index 5fb7d82647..1e4808fbf4 100644 --- a/src/vstest.console/Resources/Resources.resx +++ b/src/vstest.console/Resources/Resources.resx @@ -734,10 +734,19 @@ A total of {0} test files matched the specified pattern. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + The test run parameter argument '{0}' is invalid. Please use the format below. Format: TestRunParameters.Parameter(name=\"<name>\", value=\"<value>\") + + None + + + - {0} {1} + Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. diff --git a/src/vstest.console/Resources/xlf/Resources.cs.xlf b/src/vstest.console/Resources/xlf/Resources.cs.xlf index 3cafe3c695..a9c537603d 100644 --- a/src/vstest.console/Resources/xlf/Resources.cs.xlf +++ b/src/vstest.console/Resources/xlf/Resources.cs.xlf @@ -1668,6 +1668,21 @@ Shromažďování výpisů stavu systému při zablokování pomocí možnosti CollectDump s TestTimeout pro Blame se pro tuto platformu nepodporuje. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.de.xlf b/src/vstest.console/Resources/xlf/Resources.de.xlf index edd2298e65..84e784589e 100644 --- a/src/vstest.console/Resources/xlf/Resources.de.xlf +++ b/src/vstest.console/Resources/xlf/Resources.de.xlf @@ -1668,6 +1668,21 @@ Das Sammeln von Abbildern bei Reaktionsunfähigkeit über die Option "CollectDump" mit "TestTimeout" für "Blame" wird für diese Plattform nicht unterstützt. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.es.xlf b/src/vstest.console/Resources/xlf/Resources.es.xlf index 9a695e6dbb..8673f05e98 100644 --- a/src/vstest.console/Resources/xlf/Resources.es.xlf +++ b/src/vstest.console/Resources/xlf/Resources.es.xlf @@ -1671,6 +1671,21 @@ La recopilación de volcados de memoria mediante la opción CollectDump con TestTimeout para Blame no se admite en esta plataforma. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.fr.xlf b/src/vstest.console/Resources/xlf/Resources.fr.xlf index c8597416c9..08dd4979d9 100644 --- a/src/vstest.console/Resources/xlf/Resources.fr.xlf +++ b/src/vstest.console/Resources/xlf/Resources.fr.xlf @@ -1668,6 +1668,21 @@ La collecte des vidages sur blocage par l'option CollectDump avec TestTimeout pour Blame n'est pas prise en charge pour cette plateforme. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.it.xlf b/src/vstest.console/Resources/xlf/Resources.it.xlf index e877cb591c..359f587dbd 100644 --- a/src/vstest.console/Resources/xlf/Resources.it.xlf +++ b/src/vstest.console/Resources/xlf/Resources.it.xlf @@ -1668,6 +1668,21 @@ L'opzione CollectDump con TestTimeout per Blame non supporta la raccolta dei dump di blocco per questa piattaforma. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ja.xlf b/src/vstest.console/Resources/xlf/Resources.ja.xlf index f689672d85..68dec3d54e 100644 --- a/src/vstest.console/Resources/xlf/Resources.ja.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ja.xlf @@ -1668,6 +1668,21 @@ このプラットフォームでは、Blame の TestTimeout を使用する CollectDump オプションによるハング ダンプの収集はサポートされていません。 + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ko.xlf b/src/vstest.console/Resources/xlf/Resources.ko.xlf index 787b373600..d645a2eec7 100644 --- a/src/vstest.console/Resources/xlf/Resources.ko.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ko.xlf @@ -1668,6 +1668,21 @@ Blame에 대한 TestTimeout이 있는 CollectDump 옵션에 의한 중단 덤프 수집은 이 플랫폼에서 지원되지 않습니다. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.pl.xlf b/src/vstest.console/Resources/xlf/Resources.pl.xlf index 4c2712c3ec..600e065613 100644 --- a/src/vstest.console/Resources/xlf/Resources.pl.xlf +++ b/src/vstest.console/Resources/xlf/Resources.pl.xlf @@ -1668,6 +1668,21 @@ Zbieranie zrzutów po zawieszeniu przez opcję CollectDump z opcją TestTimeout programu Blame nie jest obsługiwane w przypadku tej platformy. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf b/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf index 26369df67e..839d1a867b 100644 --- a/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf +++ b/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf @@ -1668,6 +1668,21 @@ Altere o prefixo de nível de diagnóstico do agente de console, como mostrado a A coleta de despejos de trava pela opção CollectDump com TestTimeout para Blame não é tem suporte para esta plataforma. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.ru.xlf b/src/vstest.console/Resources/xlf/Resources.ru.xlf index 6ff8179a71..6389f284c0 100644 --- a/src/vstest.console/Resources/xlf/Resources.ru.xlf +++ b/src/vstest.console/Resources/xlf/Resources.ru.xlf @@ -1668,6 +1668,21 @@ Сбор дампов зависаний с помощью параметра CollectDump с TestTimeout для Blame не поддерживается для этой платформы. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.tr.xlf b/src/vstest.console/Resources/xlf/Resources.tr.xlf index 8a6d7adf95..46d2f82170 100644 --- a/src/vstest.console/Resources/xlf/Resources.tr.xlf +++ b/src/vstest.console/Resources/xlf/Resources.tr.xlf @@ -1668,6 +1668,21 @@ Günlükler için izleme düzeyini aşağıda gösterildiği gibi değiştirin Blame için TestTimeout ile CollectDump seçeneğine göre asılı kalma bilgi dökümlerinin toplanması bu platform için desteklenmiyor. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.xlf b/src/vstest.console/Resources/xlf/Resources.xlf index 6f53284141..f2ccd897b2 100644 --- a/src/vstest.console/Resources/xlf/Resources.xlf +++ b/src/vstest.console/Resources/xlf/Resources.xlf @@ -859,6 +859,21 @@ Format : TestRunParameters.Parameter(name="<name>", value="<value>") Collecting hang dumps by option CollectDump with TestTimeout for Blame is not supported for this platform. + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf index c8edefd47b..d37030370b 100644 --- a/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf +++ b/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf @@ -1668,6 +1668,21 @@ 此平台不支持通过 CollectDump with TestTimeout for Blame 选项收集挂起转储。 + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf index 1bfdbda0d5..0bcc68aa0f 100644 --- a/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf +++ b/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf @@ -1669,6 +1669,21 @@ 此平台不支援使用 Blame 的 CollectDump 及 TestTimeout 選項收集懸置傾印。 + + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + {0} - Failed: {1}, Passed: {2}, Skipped: {3}, Total: {4}, Duration: {5} + + + + None + None + + + + - {0} {1} + - {0} {1} + + \ No newline at end of file diff --git a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs index 9477385886..eeaa16aeba 100644 --- a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs +++ b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs @@ -21,7 +21,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Internal using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using vstest.console.Internal; - using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; + using CommandLineResources = Resources.Resources; [TestClass] public class ConsoleLoggerTests @@ -63,10 +63,13 @@ public void InitializeShouldNotThrowExceptionIfEventsIsNotNull() [TestMethod] public void InitializeWithParametersShouldThrowExceptionIfEventsIsNull() { + var parameters = new Dictionary + { + { "param1", "value" }, + }; + Assert.ThrowsException(() => { - var parameters = new Dictionary(); - parameters.Add("parma1", "value"); this.consoleLogger.Initialize(null, parameters); }); } @@ -92,8 +95,11 @@ public void InitializeWithParametersShouldThrowExceptionIfParametersIsNull() [TestMethod] public void InitializeWithParametersShouldSetVerbosityLevel() { - var parameters = new Dictionary(); - parameters.Add("verbosity", "minimal"); + var parameters = new Dictionary + { + { "verbosity", "minimal" }, + { DefaultLoggerParameterNames.TargetFramework , "net451"} + }; this.consoleLogger.Initialize(new Mock().Object, parameters); Assert.AreEqual(ConsoleLogger.Verbosity.Minimal, this.consoleLogger.VerbosityLevel); @@ -102,8 +108,11 @@ public void InitializeWithParametersShouldSetVerbosityLevel() [TestMethod] public void InitializeWithParametersShouldDefaultToNormalVerbosityLevelForInvalidVerbosity() { - var parameters = new Dictionary(); - parameters.Add("verbosity", "random"); + var parameters = new Dictionary + { + { "verbosity", "" }, + }; + this.consoleLogger.Initialize(new Mock().Object, parameters); #if NET451 @@ -116,15 +125,15 @@ public void InitializeWithParametersShouldDefaultToNormalVerbosityLevelForInvali [TestMethod] public void InitializeWithParametersShouldSetPrefixValue() { - var parameters = new Dictionary(); - + var parameters = new Dictionary + { + { "prefix", "true" }, + }; Assert.IsFalse(ConsoleLogger.AppendPrefix); - parameters.Add("prefix", "true"); this.consoleLogger.Initialize(new Mock().Object, parameters); Assert.IsTrue(ConsoleLogger.AppendPrefix); - ConsoleLogger.AppendPrefix = false; } @@ -347,14 +356,15 @@ public void TestRunErrorMessageShowShouldTestRunFailed() } [TestMethod] - public void InQuietModeTestErrorMessageShowShouldShowTestRunFailed() + public void InQuietModeTestErrorMessageShouldShowTestRunFailed() { // Setup var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); loggerEvents.EnableEvents(); var parameters = new Dictionary { - { "verbosity", "quiet" } + { "verbosity", "quiet" }, + { DefaultLoggerParameterNames.TargetFramework , "abc" } }; this.consoleLogger.Initialize(loggerEvents, parameters); @@ -366,7 +376,6 @@ public void InQuietModeTestErrorMessageShowShouldShowTestRunFailed() loggerEvents.WaitForEventCompletion(); // Verify - this.mockOutput.Verify(o => o.WriteLine(CommandLineResources.TestRunFailed, OutputLevel.Error), Times.Once()); this.mockOutput.Verify(o => o.WriteLine(message, OutputLevel.Error), Times.Once()); } @@ -378,7 +387,8 @@ public void InQuietModeTestWarningMessageShouldNotShow() loggerEvents.EnableEvents(); var parameters = new Dictionary { - { "verbosity", "quiet" } + { "verbosity", "quiet" }, + { DefaultLoggerParameterNames.TargetFramework , "abc" } }; this.consoleLogger.Initialize(loggerEvents, parameters); @@ -548,7 +558,7 @@ public void TestResultHandlerShouldNotShowAdditionalInfoBannerIfAdditionalInfoIs } [TestMethod] - public void TestResultHandlerShouldWriteToConsoleShouldShowPassedTestsForNormalVebosity() + public void TestResultHandlerShouldShowPassedTestsForNormalVebosity() { var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); loggerEvents.EnableEvents(); @@ -563,15 +573,100 @@ public void TestResultHandlerShouldWriteToConsoleShouldShowPassedTestsForNormalV loggerEvents.WaitForEventCompletion(); this.mockOutput.Verify(o => o.Write(PassedTestIndicator, OutputLevel.Information), Times.Once); - this.mockOutput.Verify(o => o.WriteLine("TestName [1h 2m]", OutputLevel.Information), Times.Once); + this.mockOutput.Verify(o => o.WriteLine("TestName [1 h 2 m]", OutputLevel.Information), Times.Once); this.mockOutput.Verify(o => o.Write(FailedTestIndicator, OutputLevel.Information), Times.Once); - this.mockOutput.Verify(o => o.WriteLine("TestName [4m 5s]", OutputLevel.Information), Times.Once()); + this.mockOutput.Verify(o => o.WriteLine("TestName [4 m 5 s]", OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.Write(SkippedTestIndicator, OutputLevel.Information), Times.Exactly(3)); this.mockOutput.Verify(o => o.WriteLine("TestName", OutputLevel.Information), Times.Exactly(3)); this.mockProgressIndicator.Verify(pi => pi.Pause(), Times.Exactly(5)); this.mockProgressIndicator.Verify(pi => pi.Start(), Times.Exactly(5)); } + [DataRow(".NETFramework,version=v4.5.1", "(net451)", "quiet")] + [DataRow(".NETFramework,version=v4.5.1", "(net451)", "minimal")] + [DataRow(null, null, "quiet")] + [DataRow(null, null, "minimal")] + [TestMethod] + public void TestResultHandlerShouldShowFailedTestsAndPassedTestsForQuietVebosity(string framework, string expectedFramework, string verbosityLevel) + { + var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); + loggerEvents.EnableEvents(); + var parameters = new Dictionary + { + { "verbosity", verbosityLevel }, + { DefaultLoggerParameterNames.TargetFramework , framework} + }; + this.consoleLogger.Initialize(loggerEvents, parameters); + + foreach (var testResult in this.GetTestResultsObject()) + { + loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); + } + + foreach (var testResult in this.GetPassedTestResultsObject()) + { + loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); + } + + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.WaitForEventCompletion(); + + this.mockOutput.Verify(o => o.Write(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, + (CommandLineResources.PassedTestIndicator + "!").PadRight(8), + 0.ToString().PadLeft(5), + 1.ToString().PadLeft(5), + 1.ToString().PadLeft(5), 2 + .ToString().PadLeft(5), + "1 m 2 s"), OutputLevel.Information), Times.Once); + + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryAssemblyAndFramework, + "TestSourcePassed", + expectedFramework), OutputLevel.Information), Times.Once); + + this.mockOutput.Verify(o => o.Write(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, + (CommandLineResources.FailedTestIndicator + "!").PadRight(8), + 1.ToString().PadLeft(5), + 1.ToString().PadLeft(5), + 1.ToString().PadLeft(5), + 3.ToString().PadLeft(5), + "1 h 6 m"), OutputLevel.Information), Times.Once); + + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryAssemblyAndFramework, + "TestSource", + expectedFramework), OutputLevel.Information), Times.Once); + } + + [TestMethod] + [DataRow("normal")] + [DataRow("detailed")] + public void TestResultHandlerShouldNotShowformattedFailedTestsAndPassedTestsForOtherThanQuietVebosity(string verbosityLevel) + { + var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); + loggerEvents.EnableEvents(); + var parameters = new Dictionary + { + { "verbosity", verbosityLevel }, + { DefaultLoggerParameterNames.TargetFramework , "net451"} + }; + this.consoleLogger.Initialize(loggerEvents, parameters); + + foreach (var testResult in this.GetTestResultsObject()) + { + loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); + } + + foreach (var testResult in this.GetPassedTestResultsObject()) + { + loggerEvents.RaiseTestResult(new TestResultEventArgs(testResult)); + } + + loggerEvents.RaiseTestRunComplete(new TestRunCompleteEventArgs(new Mock().Object, false, false, null, new Collection(), TimeSpan.FromSeconds(1))); + loggerEvents.WaitForEventCompletion(); + + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, CommandLineResources.PassedTestIndicator, 2, 1, 0, 1, "1 m 2 s", "TestSourcePassed", "(net451)"), OutputLevel.Information), Times.Never); + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary, CommandLineResources.FailedTestIndicator, 5, 1, 1, 1, "1 h 6 m", "TestSource", "(net451)"), OutputLevel.Information), Times.Never); + } + [TestMethod] public void TestResultHandlerShouldNotShowNotStdOutMsgOfPassedTestIfVerbosityIsNormal() { @@ -627,8 +722,11 @@ public void TestResultHandlerShouldWriteToConsoleButSkipPassedTestsForMinimalVer { var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); loggerEvents.EnableEvents(); - var parameters = new Dictionary(); - parameters.Add("verbosity", "minimal"); + var parameters = new Dictionary + { + { "verbosity", "minimal" }, + { DefaultLoggerParameterNames.TargetFramework , "net451"} + }; this.consoleLogger.Initialize(loggerEvents, parameters); foreach (var testResult in this.GetTestResultsObject()) @@ -638,9 +736,9 @@ public void TestResultHandlerShouldWriteToConsoleButSkipPassedTestsForMinimalVer loggerEvents.WaitForEventCompletion(); this.mockOutput.Verify(o => o.Write(PassedTestIndicator, OutputLevel.Information), Times.Never); - this.mockOutput.Verify(o => o.WriteLine("TestName [1h 2m]", OutputLevel.Information), Times.Never); + this.mockOutput.Verify(o => o.WriteLine("TestName [1 h 2 m]", OutputLevel.Information), Times.Never); this.mockOutput.Verify(o => o.Write(FailedTestIndicator, OutputLevel.Information), Times.Once); - this.mockOutput.Verify(o => o.WriteLine("TestName [4m 5s]", OutputLevel.Information), Times.Once()); + this.mockOutput.Verify(o => o.WriteLine("TestName [4 m 5 s]", OutputLevel.Information), Times.Once()); this.mockOutput.Verify(o => o.Write(SkippedTestIndicator, OutputLevel.Information), Times.Exactly(3)); this.mockOutput.Verify(o => o.WriteLine("TestName", OutputLevel.Information), Times.Exactly(3)); } @@ -650,8 +748,11 @@ public void TestResultHandlerShouldWriteToNoTestResultForQuietVerbosity() { var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); loggerEvents.EnableEvents(); - var parameters = new Dictionary(); - parameters.Add("verbosity", "Quiet"); + var parameters = new Dictionary + { + { "verbosity", "quiet" }, + { DefaultLoggerParameterNames.TargetFramework , "net451"} + }; this.consoleLogger.Initialize(loggerEvents, parameters); foreach (var testResult in this.GetTestResultsObject()) @@ -660,18 +761,19 @@ public void TestResultHandlerShouldWriteToNoTestResultForQuietVerbosity() } loggerEvents.WaitForEventCompletion(); - this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName [1h 2m]"), OutputLevel.Information), Times.Never); - this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.FailedTestIndicator, "TestName [4m 5s]"), OutputLevel.Information), Times.Never); + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName [1 h 2 m]"), OutputLevel.Information), Times.Never); + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.FailedTestIndicator, "TestName [4 m 5 s]"), OutputLevel.Information), Times.Never); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Warning), Times.Never); this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.NotRunTestIndicator, "TestName"), OutputLevel.Information), Times.Never); } - [DataRow("[1h 2m]", new int[5] { 0, 1, 2, 3, 78 })] - [DataRow("[4m 3s]", new int[5] { 0, 0, 4, 3, 78 })] - [DataRow("[3s 78ms]", new int[5] { 0, 0, 0, 3, 78 })] - [DataRow("[78ms]", new int[5] { 0, 0, 0, 0, 78 })] - [DataRow("[1h]", new int[5] { 0, 1, 0, 5, 78 })] - [DataRow("[5m]", new int[5] { 0, 0, 5, 0, 78 })] + [DataRow("[1 h 2 m]", new int[5] { 0, 1, 2, 3, 78 })] + [DataRow("[4 m 3 s]", new int[5] { 0, 0, 4, 3, 78 })] + [DataRow("[3 s]", new int[5] { 0, 0, 0, 3, 78 })] + [DataRow("[78 ms]", new int[5] { 0, 0, 0, 0, 78 })] + [DataRow("[1 h]", new int[5] { 0, 1, 0, 5, 78 })] + [DataRow("[5 m]", new int[5] { 0, 0, 5, 0, 78 })] + [DataRow("[4 s]", new int[5] { 0, 0, 0, 4, 0 })] [DataTestMethod] public void TestResultHandlerForTestResultWithDurationShouldPrintDurationInfo(string expectedDuration, int[] timeSpanArgs) { @@ -711,7 +813,7 @@ public void TestResultHandlerForTestResultWithDurationLessThanOneMsShouldPrintDu loggerEvents.WaitForEventCompletion(); this.mockOutput.Verify(o => o.Write(PassedTestIndicator, OutputLevel.Information), Times.Once()); - this.mockOutput.Verify(o => o.WriteLine("TestName [< 1ms]", OutputLevel.Information), Times.Once()); + this.mockOutput.Verify(o => o.WriteLine("TestName [< 1 ms]", OutputLevel.Information), Times.Once()); } [TestMethod] @@ -977,7 +1079,7 @@ public void DisplayFullInformationShouldWriteStdMessageWithNewLine() this.consoleLogger.Initialize(loggerEvents, parameters); var testresults = this.GetTestResultObject(TestOutcome.Passed); - testresults[0].Messages.Add(new TestResultMessage (TestResultMessage.StandardOutCategory, "Hello")); + testresults[0].Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, "Hello")); foreach (var testResult in testresults) { @@ -1022,13 +1124,17 @@ public void GetTestMessagesShouldWriteMessageAndStackTraceToConsole() this.mockOutput.Verify(o => o.WriteLine(" AdditionalInfoCategory AnotherAdditionalInfoCategory", OutputLevel.Information), Times.Once()); } + [DataRow("quiet")] + [DataRow("Normal")] + [DataRow("minimal")] + [DataRow("detailed")] [TestMethod] - public void AttachmentInformationShouldBeWrittenToConsoleIfAttachmentsArePresent() + public void AttachmentInformationShouldBeWrittenToConsoleIfAttachmentsArePresent(string verbosityLevel) { var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); loggerEvents.EnableEvents(); var parameters = new Dictionary(); - parameters.Add("verbosity", "normal"); + parameters.Add("verbosity", verbosityLevel); this.consoleLogger.Initialize(loggerEvents, parameters); var attachmentSet = new AttachmentSet(new Uri("test://uri"), "myattachmentset"); @@ -1141,6 +1247,30 @@ private void Setup() return testresultList; } + private List GetPassedTestResultsObject() + { + var testcase = new TestCase("DymmyNamespace.DummyClass.TestName", new Uri("some://uri"), "TestSourcePassed") + { + DisplayName = "TestName" + }; + + var testresult = new ObjectModel.TestResult(testcase) + { + Outcome = TestOutcome.Passed, + Duration = new TimeSpan(0, 0, 1, 2, 3) + }; + + var testresult1 = new ObjectModel.TestResult(testcase) + { + Outcome = TestOutcome.Skipped + }; + + var testresultList = new List { testresult, testresult1 }; + + return testresultList; + } + + private List GetTestResultObject(TestOutcome outcome) { var testcase = new TestCase("TestName", new Uri("some://uri"), "TestSource");