Logging VSTestTask

VSTestTask.cs

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.TestPlatform.Build.Tasks
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Runtime.InteropServices;
    using Microsoft.Build.Framework;
    using Microsoft.Build.Utilities;
    using Microsoft.TestPlatform.Build.Resources;

    public class VSTestTask : ToolTask
    {

        [Required]
        //TODO: rename, as relative paths are allowed
        public ITaskItem TestFileFullPath
        {
            get;
            set;
        }

        public ITaskItem VSTestSetting
        {
            get;
            set;
        }

        public string[] VSTestTestAdapterPath
        {
            get;
            set;
        }

        public string VSTestFramework
        {
            get;
            set;
        }

        public string VSTestPlatform
        {
            get;
            set;
        }

        public string VSTestTestCaseFilter
        {
            get;
            set;
        }
        public string[] VSTestLogger
        {
            get;
            set;
        }

        public string VSTestListTests
        {
            get;
            set;
        }

        public ITaskItem VSTestDiag
        {
            get;
            set;
        }

        public string[] VSTestCLIRunSettings
        {
            get;
            set;
        }

        [Required]
        public ITaskItem VSTestConsolePath
        {
            get;
            set;
        }

        public string VSTestResultsDirectory
        {
            get;
            set;
        }

        public string VSTestVerbosity
        {
            get;
            set;
        }

        public string[] VSTestCollect
        {
            get;
            set;
        }

        public string VSTestBlame
        {
            get;
            set;
        }

        public string VSTestTraceDataCollectorDirectoryPath
        {
            get;
            set;
        }

        //TODO: should be a bool
        public string VSTestNoLogo
        {
            get;
            set;
        }
        protected override string ToolName
        {
            get
            {
                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                    return "dotnet.exe";
                else
                    return "dotnet";
            }
        }

        public VSTestTask()
        {
            StandardOutputImportance = "Low";
        }

        protected override string GenerateCommandLineCommands()
        {
            var isConsoleLoggerSpecifiedByUser = false;
            var isCollectCodeCoverageEnabled = false;
            var isRunSettingsEnabled = false;

            var builder = new CommandLineBuilder();
            builder.AppendSwitch("exec");
            builder.AppendSwitchIfNotNull("", this.VSTestConsolePath);

            if (this.VSTestSetting != null)
            {
                isRunSettingsEnabled = true;
                builder.AppendSwitchIfNotNull("--settings:", this.VSTestSetting);
            }

            if (this.VSTestTestAdapterPath != null && this.VSTestTestAdapterPath.Any())
            {
                foreach (var arg in this.VSTestTestAdapterPath)
                {
                    builder.AppendSwitchUnquotedIfNotNull("--testAdapterPath:", Utils.ArgumentEscaper.HandleEscapeSequenceInArgForProcessStart(arg));
                }
            }

            if (!string.IsNullOrEmpty(this.VSTestFramework))
            {
                builder.AppendSwitchIfNotNull("--framework:", this.VSTestFramework);
            }

            // vstest.console only support x86 and x64 for argument platform
            if (!string.IsNullOrEmpty(this.VSTestPlatform) && !this.VSTestPlatform.Contains("AnyCPU"))
            {
                builder.AppendSwitchIfNotNull("--platform:", this.VSTestPlatform);
            }

            if (!string.IsNullOrEmpty(this.VSTestTestCaseFilter))
            {
                builder.AppendSwitchIfNotNull("--testCaseFilter:", this.VSTestTestCaseFilter);
            }

            if (this.VSTestLogger != null && this.VSTestLogger.Length > 0)
            {
                foreach (var arg in this.VSTestLogger)
                {
                    builder.AppendSwitchUnquotedIfNotNull("--logger:", Utils.ArgumentEscaper.HandleEscapeSequenceInArgForProcessStart(arg));

                    if (arg.StartsWith("console", StringComparison.OrdinalIgnoreCase))
                    {
                        isConsoleLoggerSpecifiedByUser = true;
                    }
                }
            }

            if (!string.IsNullOrEmpty(this.VSTestResultsDirectory))
            {
                builder.AppendSwitchUnquotedIfNotNull("--resultsDirectory:", Utils.ArgumentEscaper.HandleEscapeSequenceInArgForProcessStart(this.VSTestResultsDirectory));
            }

            if (!string.IsNullOrEmpty(this.VSTestListTests))
            {
                builder.AppendSwitch("--listTests");
            }

            if (this.VSTestDiag != null)
            {
                builder.AppendSwitchIfNotNull("--Diag:", this.VSTestDiag);
            }

            if (this.TestFileFullPath != null)
            {
                builder.AppendFileNameIfNotNull(this.TestFileFullPath);
            }

            // Console logger was not specified by user, but verbosity was, hence add default console logger with verbosity as specified
            if (!string.IsNullOrWhiteSpace(this.VSTestVerbosity) && !isConsoleLoggerSpecifiedByUser)
            {
                var normalTestLogging = new List<string>() { "n", "normal", "d", "detailed", "diag", "diagnostic" };
                var quietTestLogging = new List<string>() { "q", "quiet" };

                string vsTestVerbosity = "minimal";
                if (normalTestLogging.Contains(this.VSTestVerbosity))
                {
                    vsTestVerbosity = "normal";
                } else if (quietTestLogging.Contains(this.VSTestVerbosity))
                {
                    vsTestVerbosity = "quiet";
                }

                builder.AppendSwitchUnquotedIfNotNull("--logger:", $"Console;Verbosity={vsTestVerbosity}");
            }

            if (!string.IsNullOrEmpty(this.VSTestBlame))
            {
                builder.AppendSwitch("--Blame");
            }

            if (this.VSTestCollect != null && this.VSTestCollect.Length > 0)
            {
                foreach (var arg in this.VSTestCollect)
                {
                    if (arg.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase))
                    {
                        isCollectCodeCoverageEnabled = true;
                    }

                    builder.AppendSwitchIfNotNull("--collect:", arg);
                }
            }

            if (isCollectCodeCoverageEnabled || isRunSettingsEnabled)
            {
                // Pass TraceDataCollector path to vstest.console as TestAdapterPath if --collect "Code Coverage"
                // or --settings (User can enable code coverage from runsettings) option given.
                // Not parsing the runsettings for two reason:
                //    1. To keep no knowledge of runsettings structure in VSTestTask.
                //    2. Impact of adding adapter path always is minimal. (worst case: loads additional data collector assembly in datacollector process.)
                // This is required due to currently trace datacollector not ships with dotnet sdk, can be remove once we have
                // go code coverage x-plat.
                if (!string.IsNullOrEmpty(this.VSTestTraceDataCollectorDirectoryPath))
                {
                    builder.AppendSwitchUnquotedIfNotNull("--testAdapterPath:", Utils.ArgumentEscaper.HandleEscapeSequenceInArgForProcessStart(this.VSTestTraceDataCollectorDirectoryPath));
                } else
                {
                    if (isCollectCodeCoverageEnabled)
                    {
                        // Not showing message in runsettings scenario, because we are not sure that code coverage is enabled.
                        // User might be using older Microsoft.NET.Test.Sdk which don't have CodeCoverage infra.
                        this.Log.LogWarning(Resources.UpdateTestSdkForCollectingCodeCoverage);
                    }
                }
            }

            if (!string.IsNullOrWhiteSpace(this.VSTestNoLogo))
            {
                builder.AppendSwitch("--nologo");
            }

            // VSTestCLIRunSettings should be last argument as vstest.console ignore options after "--"(CLIRunSettings option).
            if (this.VSTestCLIRunSettings != null && this.VSTestCLIRunSettings.Any())
            {
                builder.AppendSwitch("--");

                foreach (var arg in this.VSTestCLIRunSettings)
                {
                    builder.AppendSwitchIfNotNull(string.Empty, arg);
                }
            }

            return builder.ToString();
        }

        protected override string GenerateFullPathToTool()
        {
            string path = null;

            if (!string.IsNullOrEmpty(ToolPath))
            {
                path = Path.Combine(Path.GetDirectoryName(Path.GetFullPath(ToolPath)), ToolExe);
            } else
            {
                if (File.Exists(ToolExe))
                    path = Path.GetFullPath(ToolExe);

                var values = Environment.GetEnvironmentVariable("PATH");
                foreach (var p in values.Split(Path.PathSeparator))
                {
                    var fullPath = Path.Combine(p, ToolExe);
                    if (File.Exists(fullPath))
                        path = fullPath;
                }
            }

            return path;
        }

        internal protected string GenerateCommandLineCommandsForUnitTests()
        {
            return this.GenerateCommandLineCommands();
        }
    }
}

VSTestTaskTests.cs

// Copyright (c) Microsoft. All rights reserved.

namespace Microsoft.TestPlatform.Build.UnitTests
{
    using System.Collections;
    using System.Text.RegularExpressions;
    using Microsoft.Build.Framework;
    using Microsoft.Build.Utilities;
    using Microsoft.TestPlatform.Build.Tasks;
    using Microsoft.VisualStudio.TestTools.UnitTesting;

    [TestClass]
    public class VSTestTaskTests
    {
        private readonly VSTestTask vsTestTask;

        public VSTestTaskTests()
        {
            this.vsTestTask = new VSTestTask
            {
                BuildEngine = new FakeBuildEngine(),
                TestFileFullPath = new TaskItem(@"C:\path\to\test-assembly.dll"),
                VSTestFramework = ".NETCoreapp,Version2.0"
            };
        }

        [TestMethod]
        public void CreateArgumentShouldAddOneEntryForCLIRunSettings()
        {
            const string arg1 = "RunConfiguration.ResultsDirectory=Path having Space";
            const string arg2 = "MSTest.DeploymentEnabled";

            this.vsTestTask.VSTestCLIRunSettings = new string[2];
            this.vsTestTask.VSTestCLIRunSettings[0] = arg1;
            this.vsTestTask.VSTestCLIRunSettings[1] = arg2;

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, " -- ");
            StringAssert.Contains(commandline, $"\"{arg1}\"");
            StringAssert.Contains(commandline, $"{arg2}");
        }

        [TestMethod]
        public void CreateArgumentShouldAddCLIRunSettingsArgAtEnd()
        {
            const string codeCoverageOption = "Code Coverage";

            this.vsTestTask.VSTestCollect = new string[] { codeCoverageOption };
            this.vsTestTask.VSTestBlame = "Blame";

            const string arg1 = "RunConfiguration.ResultsDirectory=Path having Space";
            const string arg2 = "MSTest.DeploymentEnabled";

            this.vsTestTask.VSTestCLIRunSettings = new string[2];
            this.vsTestTask.VSTestCLIRunSettings[0] = arg1;
            this.vsTestTask.VSTestCLIRunSettings[1] = arg2;

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, " -- ");
            StringAssert.Contains(commandline, $"\"{arg1}\"");
            StringAssert.Contains(commandline, $"{arg2}");
        }

        [TestMethod]
        public void CreateArgumentShouldPassResultsDirectoryCorrectly()
        {
            const string resultsDirectoryValue = @"C:\tmp\Results Directory";
            this.vsTestTask.VSTestResultsDirectory = resultsDirectoryValue;

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, $"--resultsDirectory:\"{resultsDirectoryValue}\"");
        }

        [TestMethod]
        public void CreateArgumentShouldNotSetConsoleLoggerVerbosityIfConsoleLoggerIsGivenInArgs()
        {
            this.vsTestTask.VSTestVerbosity = "diag";
            this.vsTestTask.VSTestLogger = new string[] { "Console;Verbosity=quiet" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.DoesNotMatch(commandline, new Regex(@"(--logger:Console;Verbosity=normal)"));
            StringAssert.Contains(commandline, "--logger:Console;Verbosity=quiet");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsn()
        {
            this.vsTestTask.VSTestVerbosity = "n";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsnormal()
        {
            this.vsTestTask.VSTestVerbosity = "normal";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsd()
        {
            this.vsTestTask.VSTestVerbosity = "d";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsdetailed()
        {
            this.vsTestTask.VSTestVerbosity = "detailed";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsdiag()
        {
            this.vsTestTask.VSTestVerbosity = "diag";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToNormalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsdiagnostic()
        {
            this.vsTestTask.VSTestVerbosity = "diagnostic";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=normal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToQuietIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsq()
        {
            this.vsTestTask.VSTestVerbosity = "q";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=quiet");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToQuietIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsquiet()
        {
            this.vsTestTask.VSTestVerbosity = "quiet";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=quiet");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToMinimalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsm()
        {
            this.vsTestTask.VSTestVerbosity = "m";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=minimal");
        }

        [TestMethod]
        public void CreateArgumentShouldSetConsoleLoggerVerbosityToMinimalIfConsoleLoggerIsNotGivenInArgsAndVerbosityIsminimal()
        {
            this.vsTestTask.VSTestVerbosity = "minimal";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:Console;Verbosity=minimal");
        }

        [TestMethod]
        public void CreateArgumentShouldPreserveWhiteSpaceInLogger()
        {
            this.vsTestTask.VSTestLogger = new string[] { "trx;LogFileName=foo bar.trx" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:\"trx;LogFileName=foo bar.trx\"");
        }

        [TestMethod]
        public void CreateArgumentShouldAddOneCollectArgumentForEachCollect()
        {
            this.vsTestTask.VSTestCollect = new string[2];

            this.vsTestTask.VSTestCollect[0] = "name1";
            this.vsTestTask.VSTestCollect[1] = "name 2";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--collect:name1");
            StringAssert.Contains(commandline, "--collect:\"name 2\"");
        }

        [TestMethod]
        public void CreateArgumentShouldAddMultipleTestAdapterPaths()
        {
            this.vsTestTask.VSTestTestAdapterPath = new string[] { "path1", "path2" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--testAdapterPath:path1");
            StringAssert.Contains(commandline, "--testAdapterPath:path2");
        }

        [TestMethod]
        public void CreateArgumentShouldAddMultipleLoggers()
        {
            this.vsTestTask.VSTestLogger = new string[] { "trx;LogFileName=foo bar.trx", "console" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--logger:\"trx;LogFileName=foo bar.trx\"");
            StringAssert.Contains(commandline, "--logger:console");
        }

        [TestMethod]
        public void CreateArgumentShouldAddTraceCollectorDirectoryPathAsTestAdapterForCodeCoverageCollect()
        {
            const string traceDataCollectorDirectoryPath = @"c:\path\to\tracedata collector";
            this.vsTestTask.VSTestTraceDataCollectorDirectoryPath = traceDataCollectorDirectoryPath;
            this.vsTestTask.VSTestCollect = new string[] { "code coverage" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            const string expectedArg = "--testAdapterPath:\"c:\\path\\to\\tracedata collector\"";
            StringAssert.Contains(commandline, expectedArg);
        }

        [TestMethod]
        public void CreateArgumentShouldNotAddTraceCollectorDirectoryPathAsTestAdapterForNonCodeCoverageCollect()
        {
            const string traceDataCollectorDirectoryPath = @"c:\path\to\tracedata collector";
            this.vsTestTask.VSTestTraceDataCollectorDirectoryPath = traceDataCollectorDirectoryPath;
            this.vsTestTask.VSTestCollect = new string[] { "not code coverage" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            const string notExpectedArg = "--testAdapterPath:\"c:\\path\\to\\tracedata collector\"";
            StringAssert.DoesNotMatch(commandline, new Regex(Regex.Escape(notExpectedArg)));
        }

        [TestMethod]
        public void CreateArgumentShouldAddTraceCollectorDirectoryPathAsTestAdapterIfSettingsGiven()
        {
            const string traceDataCollectorDirectoryPath = @"c:\path\to\tracedatacollector\";
            this.vsTestTask.VSTestTraceDataCollectorDirectoryPath = traceDataCollectorDirectoryPath;
            this.vsTestTask.VSTestSetting = new TaskItem(@"c:\path\to\sample.runsettings");

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            const string expectedArg = "--testAdapterPath:c:\\path\\to\\tracedatacollector\\";
            StringAssert.Contains(commandline, expectedArg);
        }

        [TestMethod]
        public void CreateArgumentShouldNotAddTestAdapterPathIfVSTestTraceDataCollectorDirectoryPathIsEmpty()
        {
            this.vsTestTask.VSTestTraceDataCollectorDirectoryPath = string.Empty;
            this.vsTestTask.VSTestSetting = new TaskItem(@"c:\path\to\sample.runsettings");
            this.vsTestTask.VSTestCollect = new string[] { "code coverage" };

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();
            StringAssert.DoesNotMatch(commandline, new Regex(@"(--testAdapterPath:)"));
        }

        [TestMethod]
        public void CreateArgumentShouldAddNoLogoOptionIfSpecifiedByUser()
        {
            this.vsTestTask.VSTestNoLogo = "--nologo";

            var commandline = this.vsTestTask.GenerateCommandLineCommandsForUnitTests();

            StringAssert.Contains(commandline, "--nologo");
        }

        private class FakeBuildEngine : IBuildEngine
        {
            public bool ContinueOnError
            {
                get
                {
                    return false;
                }
            }

            public int LineNumberOfTaskNode
            {
                get
                {
                    return 0;
                }
            }

            public int ColumnNumberOfTaskNode
            {
                get
                {
                    return 0;
                }
            }

            public string ProjectFileOfTaskNode
            {
                get
                {
                    return "Fake";
                }
            }

            public bool BuildProjectFile(string projectFileName, string[] targetNames, IDictionary globalProperties, IDictionary targetOutputs)
            {
                return true;
            }

            public void LogCustomEvent(CustomBuildEventArgs e)
            {
            }

            public void LogErrorEvent(BuildErrorEventArgs e)
            {
            }

            public void LogMessageEvent(BuildMessageEventArgs e)
            {
            }

            public void LogWarningEvent(BuildWarningEventArgs e)
            {
            }
        }
    }
}