diff --git a/src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets b/src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets index 6fc4cf12dd..86ff8004c3 100644 --- a/src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets +++ b/src/Microsoft.TestPlatform.Build/Microsoft.TestPlatform.targets @@ -39,6 +39,7 @@ Copyright (c) .NET Foundation. All rights reserved. VSTestLogger="$(VSTestLogger)" VSTestListTests="$(VSTestListTests)" VSTestDiag="$(VSTestDiag)" + VSTestCLIRunSettings="$(VSTestCLIRunSettings)" /> @@ -72,6 +73,7 @@ Copyright (c) .NET Foundation. All rights reserved. + diff --git a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs index 14ea1c2e94..b4e2076d40 100644 --- a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs +++ b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs @@ -70,6 +70,12 @@ public string VSTestDiag set; } + public string[] VSTestCLIRunSettings + { + get; + set; + } + public override bool Execute() { var traceEnabledValue = Environment.GetEnvironmentVariable("VSTEST_BUILD_TRACE"); @@ -159,6 +165,12 @@ private IEnumerable CreateArgument() } } + if (this.VSTestCLIRunSettings!=null && this.VSTestCLIRunSettings.Length>0) + { + allArgs.Add("--"); + allArgs.AddRange(this.VSTestCLIRunSettings); + } + return allArgs; } } diff --git a/src/vstest.console/CommandLine/Executor.cs b/src/vstest.console/CommandLine/Executor.cs index 74f299fc19..aa9eaef58c 100644 --- a/src/vstest.console/CommandLine/Executor.cs +++ b/src/vstest.console/CommandLine/Executor.cs @@ -50,7 +50,7 @@ internal class Executor /// /// Default constructor. /// - public Executor(IOutput output): this(output, TestPlatformEventSource.Instance) + public Executor(IOutput output) : this(output, TestPlatformEventSource.Instance) { } @@ -152,8 +152,18 @@ private int GetArgumentProcessors(string[] args, out List pr int result = 0; var processorFactory = ArgumentProcessorFactory.Create(); - foreach (string arg in args) + for (var index = 0; index < args.Length; index++) { + var arg = args[index]; + + // If argument is '--', following arguments are key=value pairs for run settings. + if (arg.Equals("--")) + { + var cliRunSettingsProcessor = processorFactory.CreateArgumentProcessor(arg, args.Skip(index + 1).ToArray()); + processors.Add(cliRunSettingsProcessor); + break; + } + var processor = processorFactory.CreateArgumentProcessor(arg); if (processor != null) diff --git a/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs b/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs new file mode 100644 index 0000000000..ac41bcb65f --- /dev/null +++ b/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + using System.Xml; + using System.Xml.XPath; + + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + + using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; + + /// + /// The argument processor for runsettings passed as argument through cli + /// + internal class CLIRunSettingsArgumentProcessor : IArgumentProcessor + { + #region Constants + + /// + /// The name of the command line argument that the PortArgumentExecutor handles. + /// + public const string CommandName = "--"; + + #endregion + + private Lazy metadata; + + private Lazy executor; + + /// + /// Gets the metadata. + /// + public Lazy Metadata + { + get + { + if (this.metadata == null) + { + this.metadata = new Lazy(() => new CLIRunSettingsArgumentProcessorCapabilities()); + } + + return this.metadata; + } + } + + /// + /// Gets or sets the executor. + /// + public Lazy Executor + { + get + { + if (this.executor == null) + { + this.executor = new Lazy(() => new CLIRunSettingsArgumentExecutor(RunSettingsManager.Instance)); + } + + return this.executor; + } + + set + { + this.executor = value; + } + } + } + + internal class CLIRunSettingsArgumentProcessorCapabilities : BaseArgumentProcessorCapabilities + { + public override string CommandName => CLIRunSettingsArgumentProcessor.CommandName; + + public override bool AllowMultiple => false; + + public override bool IsAction => false; + + public override ArgumentProcessorPriority Priority => ArgumentProcessorPriority.CLIRunSettings; + + public override string HelpContentResourceName => CommandLineResources.CLIRunSettingsArgumentHelp; + + public override HelpContentPriority HelpPriority => HelpContentPriority.CLIRunSettingsArgumentProcessorHelpPriority; + } + + internal class CLIRunSettingsArgumentExecutor : IArgumentsExecutor + { + private IRunSettingsProvider runSettingsManager; + + internal CLIRunSettingsArgumentExecutor(IRunSettingsProvider runSettingsManager) + { + this.runSettingsManager = runSettingsManager; + } + + public void Initialize(string argument) + { + throw new NotImplementedException(); + } + + public void Initialize(string[] arguments) + { + // if argument is null or doesn't contain any element, don't do anything. + if (arguments == null || arguments.Length == 0) + { + return; + } + + Contract.EndContractBlock(); + + // Load up the run settings and set it as the active run settings. + try + { + var doc = new XmlDocument(); + + if (this.runSettingsManager.ActiveRunSettings != null && !string.IsNullOrEmpty(this.runSettingsManager.ActiveRunSettings.SettingsXml)) + { + var settingsXml = this.runSettingsManager.ActiveRunSettings.SettingsXml; + +#if net46 + using (var reader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) + { +#else + using (var reader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) + { +#endif + doc.Load(reader); + } + } + else + { +#if net46 + doc = (XmlDocument)XmlRunSettingsUtilities.CreateDefaultRunSettings(); +#else + using (var reader = XmlReader.Create(new StringReader(XmlRunSettingsUtilities.CreateDefaultRunSettings().CreateNavigator().OuterXml), new XmlReaderSettings() { CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) + { + doc.Load(reader); + } +#endif + } + + // Append / Override run settings supplied in CLI + CreateOrOverwriteRunSettings(doc, arguments); + + // Set Active Run Settings. + var runSettings = new RunSettings(); + runSettings.LoadSettingsXml(doc.OuterXml); + this.runSettingsManager.SetActiveRunSettings(runSettings); + } + catch (XPathException exception) + { + throw new CommandLineException(CommandLineResources.MalformedRunSettingsKey, exception); + } + catch (SettingsException exception) + { + throw new CommandLineException(exception.Message, exception); + } + } + + public ArgumentProcessorResult Execute() + { + // Nothing to do here, the work was done in initialization. + return ArgumentProcessorResult.Success; + } + + private void CreateOrOverwriteRunSettings(XmlDocument xmlDoc, string[] args) + { + var length = args.Length; + + for (int index = 0; index < length; index++) + { + var keyValuePair = args[index]; + var indexOfSeparator = keyValuePair.IndexOf("="); + if (indexOfSeparator <= 0 || indexOfSeparator >= keyValuePair.Length - 1) + { + continue; + } + var key = keyValuePair.Substring(0, indexOfSeparator).Trim(); + var value = keyValuePair.Substring(indexOfSeparator + 1); + + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + // Check if the key exists. + var xPath = key.Replace('.', '/'); + var node = xmlDoc.SelectSingleNode(string.Format("//RunSettings/{0}", xPath)); + + if (node == null) + { + node = CreateNode(xmlDoc, key.Split('.')); + } + + node.InnerText = value; + } + } + + private XmlNode CreateNode(XmlDocument doc, string[] xPath) + { + XmlNode node = null; + XmlNode parent = doc.DocumentElement; + + for (int i = 0; i < xPath.Length; i++) + { + node = parent.SelectSingleNode(xPath[i]); + + if (node == null) + { + node = parent.AppendChild(doc.CreateElement(xPath[i])); + } + + parent = node; + } + + return node; + } + } +} diff --git a/src/vstest.console/Processors/Interfaces/IArgumentsExecutor.cs b/src/vstest.console/Processors/Interfaces/IArgumentsExecutor.cs new file mode 100644 index 0000000000..2c1e968eab --- /dev/null +++ b/src/vstest.console/Processors/Interfaces/IArgumentsExecutor.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors +{ + /// + /// Defines interface for interacting with a command line arguments executor. + /// Items exporting this interface will be used in processing command line arguments. + /// + internal interface IArgumentsExecutor : IArgumentExecutor + { + /// + /// Initializes the Argument Processor with the arguments that was provided with the command. + /// + /// Arguments that are provided with the command. + void Initialize(string[] arguments); + } +} diff --git a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs index 1e72812b9a..e0683c555d 100644 --- a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs +++ b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs @@ -60,7 +60,7 @@ protected ArgumentProcessorFactory(IEnumerable argumentProce #endregion #region Static Methods - + /// /// Creates ArgumentProcessorFactory. /// @@ -160,6 +160,33 @@ public IArgumentProcessor CreateArgumentProcessor(string argument) return argumentProcessor; } + /// + /// Creates the argument processor associated with the provided command line argument. + /// The Lazy that is returned will initialize the underlying argument processor when it is first accessed. + /// + /// Command name of the argument processor. + /// Command line arguments to create the argument processor for. + /// The argument processor or null if one was not found. + public IArgumentProcessor CreateArgumentProcessor(string command, string[] arguments) + { + if (arguments == null || arguments.Length == 0) + { + throw new ArgumentException("Cannot be null or empty", "argument"); + } + Contract.EndContractBlock(); + + // Find the associated argument processor. + IArgumentProcessor argumentProcessor; + CommandToProcessorMap.TryGetValue(command, out argumentProcessor); + + if (argumentProcessor != null) + { + argumentProcessor = WrapLazyProcessorToInitializeOnInstantiation(argumentProcessor, arguments); + } + + return argumentProcessor; + } + /// /// Creates the default action argument processor. /// The Lazy that is returned will initialize the underlying argument processor when it is first accessed. @@ -168,7 +195,7 @@ public IArgumentProcessor CreateArgumentProcessor(string argument) public IArgumentProcessor CreateDefaultActionArgumentProcessor() { var argumentProcessor = SpecialCommandToProcessorMap[RunTestsArgumentProcessor.CommandName]; - return WrapLazyProcessorToInitializeOnInstantiation(argumentProcessor, null); + return WrapLazyProcessorToInitializeOnInstantiation(argumentProcessor); } /// @@ -201,7 +228,8 @@ public IEnumerable GetArgumentProcessorsToAlwaysExecute() new FrameworkArgumentProcessor(), new EnableLoggerArgumentProcessor(), new ParallelArgumentProcessor(), - new EnableDiagArgumentProcessor() + new EnableDiagArgumentProcessor(), + new CLIRunSettingsArgumentProcessor() }; /// @@ -246,7 +274,7 @@ private void BuildCommandMaps() /// The decorated lazy processor. private static IArgumentProcessor WrapLazyProcessorToInitializeOnInstantiation( IArgumentProcessor processor, - string initArg) + string initArg = null) { var processorExecutor = processor.Executor; var lazyArgumentProcessor = new Lazy(() => @@ -288,6 +316,56 @@ private static IArgumentProcessor WrapLazyProcessorToInitializeOnInstantiation( return processor; } + /// + /// Decorates a lazy argument processor so that the real processor is initialized when the lazy value is obtained. + /// + /// The lazy processor. + /// The argument with which the real processor should be initialized. + /// The decorated lazy processor. + private static IArgumentProcessor WrapLazyProcessorToInitializeOnInstantiation( + IArgumentProcessor processor, + string[] initArgs) + { + var processorExecutor = processor.Executor; + var lazyArgumentProcessor = new Lazy(() => + { + IArgumentsExecutor instance = null; + try + { + instance = (IArgumentsExecutor)processorExecutor.Value; + } + catch (Exception e) + { + if (EqtTrace.IsErrorEnabled) + { + EqtTrace.Error("ArgumentProcessorFactory.WrapLazyProcessorToInitializeOnInstantiation: Exception creating argument processor: {0}", e); + } + throw; + } + + if (instance != null) + { + try + { + instance.Initialize(initArgs); + } + catch (Exception e) + { + if (EqtTrace.IsErrorEnabled) + { + EqtTrace.Error("ArgumentProcessorFactory.WrapLazyProcessorToInitializeOnInstantiation: Exception initializing argument processor: {0}", e); + } + throw; + } + } + + return instance; + }, System.Threading.LazyThreadSafetyMode.PublicationOnly); + processor.Executor = lazyArgumentProcessor; + + return processor; + } + #endregion } } diff --git a/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs b/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs index c2c422122e..a551970e8d 100644 --- a/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs +++ b/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs @@ -42,6 +42,11 @@ internal enum ArgumentProcessorPriority /// RunSettings = 5, + /// + /// Priority of processors related to CLI Run Settings. + /// + CLIRunSettings = 6, + /// /// Priority of processors related to logging. /// diff --git a/src/vstest.console/Processors/Utilities/HelpContentPriority.cs b/src/vstest.console/Processors/Utilities/HelpContentPriority.cs index 586085fcf8..16b515bc95 100644 --- a/src/vstest.console/Processors/Utilities/HelpContentPriority.cs +++ b/src/vstest.console/Processors/Utilities/HelpContentPriority.cs @@ -77,12 +77,12 @@ internal enum HelpContentPriority /// HelpArgumentExecutor /// HelpArgumentProcessorHelpPriority, - + /// /// EnableLoggerArgumentProcessor Help /// EnableLoggerArgumentProcessorHelpPriority, - + /// /// ListTestsArgumentExecutor Help /// @@ -92,7 +92,7 @@ internal enum HelpContentPriority /// ListDiscoverersArgumentProcessor Help /// ListDiscoverersArgumentProcessorHelpPriority, - + /// /// ListExecutorsArgumentProcessor Help /// @@ -121,6 +121,11 @@ internal enum HelpContentPriority /// /// EnableDiagArgumentProcessor Help /// - EnableDiagArgumentProcessorHelpPriority + EnableDiagArgumentProcessorHelpPriority, + + /// + /// CLIRunSettingsArgumentProcessor Help + /// + CLIRunSettingsArgumentProcessorHelpPriority } } diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs index 4db509668a..3cbe0f1e62 100644 --- a/src/vstest.console/Resources/Resources.Designer.cs +++ b/src/vstest.console/Resources/Resources.Designer.cs @@ -11,8 +11,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Resources { using System; using System.Reflection; - - + + /// /// A strongly-typed resource class, for looking up localized strings, etc. /// @@ -215,6 +215,16 @@ public static string CannotBeNullOrEmpty { } } + /// + /// Looks up a localized string similar to Args: + /// Any extra arguments that should be passed to adapter. Arguments may be specified as name-value pair of the form <n>=<v>, where <n> is the argument name, and <v> is the argument value. Use a space to separate multiple arguments.. + /// + public static string CLIRunSettingsArgumentHelp { + get { + return ResourceManager.GetString("CLIRunSettingsArgumentHelp", resourceCulture); + } + } + /// /// Looks up a localized string similar to Error: {0}. /// @@ -527,7 +537,7 @@ public static string HelpOptionsText { } /// - /// Looks up a localized string similar to Usage: vstest.console.exe [Arguments] [Options]. + /// Looks up a localized string similar to Usage: vstest.console.exe [Arguments] [Options] [[--] <args>...]]. /// public static string HelpUsageText { get { @@ -777,6 +787,15 @@ public static string MalformedRunSettingsFile { } } + /// + /// Looks up a localized string similar to One or more runsettings provided contain invalid token. + /// + public static string MalformedRunSettingsKey { + get { + return ResourceManager.GetString("MalformedRunSettingsKey", resourceCulture); + } + } + /// /// Looks up a localized string similar to Microsoft (R) Test Execution Command Line Tool Version {0}. /// diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx index 09b946025b..f657d11e6d 100644 --- a/src/vstest.console/Resources/Resources.resx +++ b/src/vstest.console/Resources/Resources.resx @@ -297,7 +297,7 @@ Section Header for subsequent command help listing - Usage: vstest.console.exe [Arguments] [Options] + Usage: vstest.console.exe [Arguments] [Options] [[--] <args>...]] Hours @@ -616,4 +616,11 @@ The /UseVsixExtensions parameter requires a value. If 'true', the installed VSIX extensions (if any) will be used in the test run. If false, they will be ignored. Example: /UseVsixExtensions:true + + Args: + Any extra arguments that should be passed to adapter. Arguments may be specified as name-value pair of the form <n>=<v>, where <n> is the argument name, and <v> is the argument value. Use a space to separate multiple arguments. + + + One or more runsettings provided contain invalid token + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 0b5670db0c..484cf7db8d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -53,7 +53,93 @@ public void RunTestExecutionWithRunSettingsWithoutParallelAndPlatformX86(string this.RunTestWithRunSettings(runConfigurationDictionary, testhostProcessName, expectedProcessCreated); } - [Ignore] + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void RunTestExecutionWithRunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + + string testhostProcessName; + int expectedProcessCreated; + if (this.IsDesktopTargetFramework()) + { + testhostProcessName = "testhost.x86"; + expectedProcessCreated = 1; + } + else + { + testhostProcessName = "dotnet"; + if (this.IsDesktopRunner()) + { + expectedProcessCreated = 2; + } + else + { + // includes launcher dotnet process + expectedProcessCreated = 3; + } + } + + var runSettingsArgs = String.Join( + " ", + new string[] + { + "RunConfiguration.MaxCpuCount=1", "RunConfiguration.TargetPlatform=x86", + string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), + string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) + }); + + this.RunTestWithRunSettingsAndRunSettingsParamsAsArguments(null, runSettingsArgs, testhostProcessName, expectedProcessCreated); + } + + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void RunTestExecutionWithRunSettingsAndRunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + + string testhostProcessName; + int expectedProcessCreated; + if (this.IsDesktopTargetFramework()) + { + testhostProcessName = "testhost.x86"; + expectedProcessCreated = 1; + } + else + { + testhostProcessName = "dotnet"; + if (this.IsDesktopRunner()) + { + expectedProcessCreated = 2; + } + else + { + // includes launcher dotnet process + expectedProcessCreated = 3; + } + } + var runConfigurationDictionary = new Dictionary + { + { "MaxCpuCount", "2" }, + { "TargetPlatform", "x86" }, + { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings() }, + { "TestAdaptersPaths", this.GetTestAdapterPath() } + }; + + var runSettingsArgs = String.Join( + " ", + new string[] + { + "RunConfiguration.MaxCpuCount=1", "RunConfiguration.TargetPlatform=x86", + string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), + string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) + }); + + this.RunTestWithRunSettingsAndRunSettingsParamsAsArguments(runConfigurationDictionary, runSettingsArgs, testhostProcessName, expectedProcessCreated); + } + [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] @@ -144,5 +230,37 @@ private void RunTestWithRunSettings( this.ValidateSummaryStatus(2, 2, 2); File.Delete(runsettingsPath); } + + private void RunTestWithRunSettingsAndRunSettingsParamsAsArguments( + Dictionary runConfigurationDictionary, + string runSettingsArgs, + string testhostProcessName, + int expectedProcessCreated) + { + var assemblyPaths = + this.BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + string runsettingsPath = string.Empty; + + if (runConfigurationDictionary != null) + { + runsettingsPath = this.GetRunsettingsFilePath(runConfigurationDictionary); + } + + var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), runsettingsPath, this.FrameworkArgValue); + arguments = string.Concat(arguments, " -- ", runSettingsArgs); + var cts = new CancellationTokenSource(); + var numOfProcessCreatedTask = NumberOfProcessLaunchedUtility.NumberOfProcessCreated( + cts, + testhostProcessName); + + this.InvokeVsTest(arguments); + cts.Cancel(); + + Assert.AreEqual( + expectedProcessCreated, + numOfProcessCreatedTask.Result, + $"Number of {testhostProcessName} process created, expected: {expectedProcessCreated} actual: {numOfProcessCreatedTask.Result} args: {arguments} runner path: {this.testEnvironment.GetConsoleRunnerPath()}"); + this.ValidateSummaryStatus(2, 2, 2); + } } } diff --git a/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs new file mode 100644 index 0000000000..1896d368ac --- /dev/null +++ b/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.console.UnitTests.Processors +{ + using System; + using System.Collections.Generic; + using System.Text; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.CommandLine; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CLIRunSettingsArgumentProcessorTests + { + private const string DefaultRunSettings = "\r\n\r\n \r\n \r\n \r\n"; + private const string RunSettingsWithDeploymentDisabled = "\r\n\r\n \r\n \r\n \r\n \r\n False\r\n \r\n"; + private const string RunSettingsWithDeploymentEnabled = "\r\n\r\n \r\n \r\n \r\n \r\n True\r\n \r\n"; + + [TestMethod] + public void GetMetadataShouldReturnRunSettingsArgumentProcessorCapabilities() + { + var processor = new CLIRunSettingsArgumentProcessor(); + Assert.IsTrue(processor.Metadata.Value is CLIRunSettingsArgumentProcessorCapabilities); + } + + [TestMethod] + public void GetExecuterShouldReturnRunSettingsArgumentProcessorCapabilities() + { + var processor = new CLIRunSettingsArgumentProcessor(); + Assert.IsTrue(processor.Executor.Value is CLIRunSettingsArgumentExecutor); + } + + #region CLIRunSettingsArgumentProcessorCapabilities tests + + [TestMethod] + public void CapabilitiesShouldReturnAppropriateProperties() + { + var capabilities = new CLIRunSettingsArgumentProcessorCapabilities(); + + Assert.AreEqual("--", capabilities.CommandName); + Assert.AreEqual("Args:\n Any extra arguments that should be passed to adapter. Arguments may be specified as name-value pair of the form =, where is the argument name, and is the argument value. Use a space to separate multiple arguments.", capabilities.HelpContentResourceName); + + Assert.AreEqual(HelpContentPriority.CLIRunSettingsArgumentProcessorHelpPriority, capabilities.HelpPriority); + Assert.AreEqual(false, capabilities.IsAction); + Assert.AreEqual(ArgumentProcessorPriority.CLIRunSettings, capabilities.Priority); + + Assert.AreEqual(false, capabilities.AllowMultiple); + Assert.AreEqual(false, capabilities.AlwaysExecute); + Assert.AreEqual(false, capabilities.IsSpecialCommand); + } + + #endregion + + #region CLIRunSettingsArgumentExecutor tests + + [TestMethod] + public void InitializeShouldNotThrowExceptionIfArgumentIsNull() + { + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(null); + + executor.Initialize((string[])null); + + Assert.IsNull(settingsProvider.ActiveRunSettings); + } + + [TestMethod] + public void InitializeShouldNotThrowExceptionIfArgumentIsEmpty() + { + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + executor.Initialize(new string[0]); + + Assert.IsNull(settingsProvider.ActiveRunSettings); + } + + [TestMethod] + public void InitializeShouldCreateDefaultRunSettingsIfArgumentsHasOnlyWhiteSpace() + { + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + executor.Initialize(new string[] { " " }); + + Assert.AreEqual(DefaultRunSettings, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldSetValueInRunSettings() + { + var args = new string[] { "MSTest.DeploymentEnabled=False" }; + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldIgnoreKeyIfValueIsNotPassed() + { + var args = new string[] { "MSTest.DeploymentEnabled=False", "MSTest1" }; + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldIgnoreWhiteSpaceInBeginningOrEndOfKey() + { + var args = new string[] { " MSTest.DeploymentEnabled =False" }; + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldIgnoreThrowExceptionIfKeyHasWhiteSpace() + { + var args = new string[] { "MST est.DeploymentEnabled=False" }; + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + Action action = () => executor.Initialize(args); + + ExceptionUtilities.ThrowsException( + action, + "One or more runsettings provided contain invalid token"); + } + + [TestMethod] + public void InitializeShouldEncodeXMLIfInvalidXMLCharsArePresent() + { + var args = new string[] { "MSTest.DeploymentEnabled=F>a>\r\n\r\n \r\n \r\n \r\n \r\n F>a><l<se\r\n \r\n", settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldIgnoreIfKeyIsNotPassed() + { + var args = new string[] { "MSTest.DeploymentEnabled=False", "=value" }; + var settingsProvider = new TestableRunSettingsProvider(); + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldIgnoreIfEmptyValueIsPassed() + { + var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); + runSettings.LoadSettingsXml(DefaultRunSettings); + settingsProvider.SetActiveRunSettings(runSettings); + + var args = new string[] { "MSTest.DeploymentEnabled=" }; + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(DefaultRunSettings, settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldOverwriteValueIfNodeAlreadyExists() + { + var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); + runSettings.LoadSettingsXml(DefaultRunSettings); + settingsProvider.SetActiveRunSettings(runSettings); + + var args = new string[] { "MSTest.DeploymentEnabled=True" }; + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual(RunSettingsWithDeploymentEnabled, settingsProvider.ActiveRunSettings.SettingsXml); + } + + + [TestMethod] + public void InitializeShouldOverwriteValueIfWhitSpaceIsPassedAndNodeAlreadyExists() + { + var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); + runSettings.LoadSettingsXml(DefaultRunSettings); + settingsProvider.SetActiveRunSettings(runSettings); + + var args = new string[] { "MSTest.DeploymentEnabled= " }; + var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + executor.Initialize(args); + + Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n", settingsProvider.ActiveRunSettings.SettingsXml); + } + + #endregion + + #region private + + private class TestableRunSettingsProvider : IRunSettingsProvider + { + public RunSettings ActiveRunSettings + { + get; + set; + } + + public void SetActiveRunSettings(RunSettings runSettings) + { + this.ActiveRunSettings = runSettings; + } + } + + #endregion + } +} diff --git a/test/vstest.console.UnitTests/Processors/EnableLoggersArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/EnableLoggersArgumentProcessorTests.cs index 767abdca01..bb27527228 100644 --- a/test/vstest.console.UnitTests/Processors/EnableLoggersArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/EnableLoggersArgumentProcessorTests.cs @@ -39,7 +39,7 @@ public void CapabilitiesShouldAppropriateProperties() { EnableLoggerArgumentProcessorCapabilities capabilities = new EnableLoggerArgumentProcessorCapabilities(); Assert.AreEqual("/Logger", capabilities.CommandName); - Assert.AreEqual("--logger|/logger:\n Specify a logger for test results. For example, to log results into a \n Visual Studio Test Results File (TRX) use /logger:trx.\n To publish test results to Team Foundation Server, use TfsPublisher as shown below\n Example: /logger:TfsPublisher;\n Collection=;\n BuildName=;\n TeamProject=\n [;Platform=]\n [;Flavor=]\n [;RunTitle=]", capabilities.HelpContentResourceName); + Assert.AreEqual("--logger|/logger:<Logger Uri/FriendlyName>\n Specify a logger for test results. For example, to log results into a \n Visual Studio Test Results File (TRX) use /logger:trx [;LogFileName=<Defaults to unique file name>]\n Creates file in TestResults directory with given LogFileName.\n\n To publish test results to Team Foundation Server, use TfsPublisher as shown below\n Example: /logger:TfsPublisher;\n Collection=<team project collection url>;\n BuildName=<build name>;\n TeamProject=<team project name>\n [;Platform=<Defaults to \"Any CPU\">]\n [;Flavor=<Defaults to \"Debug\">]\n [;RunTitle=<title>]", capabilities.HelpContentResourceName); Assert.AreEqual(HelpContentPriority.EnableLoggerArgumentProcessorHelpPriority, capabilities.HelpPriority); Assert.AreEqual(false, capabilities.IsAction); diff --git a/test/vstest.console.UnitTests/Processors/HelpArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/HelpArgumentProcessorTests.cs index 254b68b157..3f02cc7110 100644 --- a/test/vstest.console.UnitTests/Processors/HelpArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/HelpArgumentProcessorTests.cs @@ -67,7 +67,7 @@ public void ExecuterExecuteWritesAppropriateDataToConsole() var output = new DummyConsoleOutput(); executor.Output = output; var result = executor.Execute(); - Assert.IsTrue(output.Lines.Contains("Usage: vstest.console.exe [Arguments] [Options]")); + Assert.IsTrue(output.Lines.Contains("Usage: vstest.console.exe [Arguments] [Options] [[--] <args>...]]")); Assert.IsTrue(output.Lines.Contains("Arguments:")); Assert.IsTrue(output.Lines.Contains("Options:")); Assert.IsTrue(output.Lines.Contains("Description: Runs tests from the specified files.")); diff --git a/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs b/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs index be3e91b9e4..24ff790fab 100644 --- a/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs +++ b/test/vstest.console.UnitTests/Processors/Utilities/ArgumentProcessorFactoryTests.cs @@ -62,6 +62,43 @@ public void CreateArgumentProcessorShouldReturnPlatformArgumentProcessorWhenArgu Assert.AreEqual(typeof(PlatformArgumentProcessor), result.GetType()); } + [TestMethod] + public void CreateArgumentProcessorShouldReturnThrowExceptionIfArgumentsIsNull() + { + var command = "--"; + + ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(); + Action action = () => factory.CreateArgumentProcessor(command, null); + + ExceptionUtilities.ThrowsException<ArgumentException>( + action, + "Cannot be null or empty", "argument"); + } + + [TestMethod] + public void CreateArgumentProcessorShouldReturnNullIfInvalidCommandIsPassed() + { + var command = "/-"; + + ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(); + + IArgumentProcessor result = factory.CreateArgumentProcessor(command, new string[] { "" }); + + Assert.IsNull(result); + } + + [TestMethod] + public void CreateArgumentProcessorShouldReturnCLIRunSettingsArgumentProcessorIfCommandIsGiven() + { + var command = "--"; + + ArgumentProcessorFactory factory = ArgumentProcessorFactory.Create(); + + IArgumentProcessor result = factory.CreateArgumentProcessor(command, new string[] { "" }); + + Assert.AreEqual(typeof(CLIRunSettingsArgumentProcessor), result.GetType()); + } + [TestMethod] public void BuildCommadMapsForProcessorWithIsSpecialCommandSetAddsProcessorToSpecialMap() {