diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 00000000000..d7769013100 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +''' +Builds the CoreClr Benchmarks +''' + +from subprocess import CalledProcessError +from traceback import format_exc +from typing import Tuple + +import argparse +import datetime +import logging +import os +import sys + +from build.common import get_logging_console_handler +from build.common import get_logging_file_handler +from build.common import get_repo_root_path +from build.common import is_supported_version +from build.common import log_start_message +from build.common import LAUNCH_TIME +from build.common import LOGGING_FORMATTER +from build.exception.FatalError import FatalError +from build.parser.TargetFrameworkAction import TargetFrameworkAction +from build.process.DotNet import DotNet +from build.runner.RunCommand import RunCommand + + +def generate_log_file_name() -> str: + '''Generates a unique log file name for the current script''' + log_dir = os.path.join(get_repo_root_path(), 'logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] + timestamp = datetime.datetime.fromtimestamp(LAUNCH_TIME).strftime( + "%Y%m%d%H%M%S") + log_file_name = '{}-{}-pid{}.log'.format( + timestamp, script_name, os.getpid()) + return os.path.join(log_dir, log_file_name) + + +def init_logging(verbose: bool) -> str: + '''Initializes the loggers used by the script.''' + logging.getLogger().setLevel(logging.INFO) + + log_file_name = generate_log_file_name() + + for logger in ['shell', 'script']: + logging.getLogger(logger).addHandler(get_logging_console_handler( + LOGGING_FORMATTER, verbose)) + logging.getLogger(logger).addHandler(get_logging_file_handler( + log_file_name, LOGGING_FORMATTER)) + logging.getLogger(logger).setLevel(logging.INFO) + + return log_file_name + + +def check_requirements(log_file: str, verbose: bool) -> None: + ''' + Checks that the requirements needs to build the CoreClr benchmarks are met. + ''' + logging.getLogger('script').info("Making sure dotnet exists...") + try: + cmdline = ['dotnet', '--info'] + RunCommand(log_file, cmdline, verbose=verbose).run('dotnet-info') + except Exception: + raise FatalError("Cannot find dotnet.") + + +def process_arguments() -> Tuple[str, list, bool]: + ''' + Function used to parse the command line arguments passed to this script + through the cli. + ''' + parser = argparse.ArgumentParser( + description="Builds the CoreClr benchmarks.", + ) + parser.add_argument( + '-c', '--configuration', + metavar='CONFIGURATION', + required=False, + default='release', + choices=['debug', 'release'], + type=str.casefold, + help='Configuration use for building the project (default "release").', + ) + parser.add_argument( + '-f', '--frameworks', + metavar='FRAMEWORK', + required=False, + nargs='*', + action=TargetFrameworkAction, + default=TargetFrameworkAction.supported_target_frameworks(), + help='Target frameworks to publish for (default all).', + ) + parser.add_argument( + '-v', '--verbose', + required=False, + default=False, + action='store_true', + help='Turns on verbosity (default "False")', + ) + + # --verbosity + # ['quiet', 'minimal', 'normal', 'detailed', 'diagnostic'] + + args = parser.parse_args() + return ( + args.configuration, + args.frameworks, + args.verbose + ) + + +def build_coreclr( + log_file: str, + configuration: str, + frameworks: list, + verbose: bool) -> None: + '''Builds the CoreClr set of benchmarks (Code Quality).''' + working_directory = os.path.join( + get_repo_root_path(), 'src', 'coreclr', 'PerformanceHarness') + csproj_file = 'PerformanceHarness.csproj' + + dotnet = DotNet(log_file, working_directory, csproj_file, verbose) + dotnet.restore() + for framework in frameworks: + dotnet.publish(configuration, framework, 'CoreClr-Benchmarks') + + +def main() -> int: + '''Script main entry point.''' + try: + if not is_supported_version(): + raise FatalError("Unsupported python version.") + + args = process_arguments() + configuration, frameworks, verbose = args + log_file = init_logging(verbose) + + log_start_message('script') + check_requirements(log_file, verbose) + build_coreclr(log_file, configuration, frameworks, verbose) + + return 0 + except FatalError as ex: + logging.getLogger('script').error(str(ex)) + except CalledProcessError as ex: + logging.getLogger('script').error( + 'Command: "%s", exited with status: %s', ex.cmd, ex.returncode) + except IOError as ex: + logging.getLogger('script').error( + "I/O error (%s): %s", ex.errno, ex.strerror) + except SystemExit: # Argparse throws this exception when it exits. + pass + except Exception: + logging.getLogger('script')( + 'Unexpected error: {}'.format(sys.exc_info()[0])) + logging.getLogger('script')(format_exc()) + raise + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build/__init__.py b/scripts/build/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/common.py b/scripts/build/common.py new file mode 100644 index 00000000000..b811025dfc7 --- /dev/null +++ b/scripts/build/common.py @@ -0,0 +1,88 @@ +''' +Common functionality used by the .NET Performance Repository build scripts. +''' + +from contextlib import contextmanager + +import datetime +import logging +import os +import sys +import time + + +LAUNCH_TIME = time.time() +LOGGING_FORMATTER = logging.Formatter( + fmt='[%(asctime)s][%(levelname)s] %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + + +def is_supported_version() -> bool: + '''Checks if the script is running on the supported version (>=3.5).''' + return sys.version_info.major > 2 and sys.version_info.minor > 4 + + +def log_start_message(name) -> None: + '''Used to log a start event message header.''' + start_msg = "Script started at {}".format( + str(datetime.datetime.fromtimestamp(LAUNCH_TIME))) + logging.getLogger(name).info('-' * len(start_msg)) + logging.getLogger(name).info(start_msg) + logging.getLogger(name).info('-' * len(start_msg)) + + +def get_script_path() -> str: + '''Gets this script directory.''' + return sys.path[0] + + +def get_repo_root_path() -> str: + '''Gets repository root directory.''' + return os.path.abspath(os.path.join(get_script_path(), '..')) + + +@contextmanager +def push_dir(path: str = None) -> None: + ''' + Adds the specified location to the top of a location stack, then changes to + the specified directory. + ''' + if path: + prev = os.getcwd() + try: + logging.getLogger('shell').info('pushd "%s"', path) + os.chdir(path) + yield + finally: + logging.getLogger('shell').info('popd') + os.chdir(prev) + else: + yield + + +def get_logging_console_handler( + fmt: logging.Formatter, + verbose: bool) -> logging.StreamHandler: + ''' + Gets a logging console handler (logging.StreamHandler) based on the + specified formatter (logging.Formatter) and verbosity. + ''' + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO if verbose else logging.WARNING) + console_handler.setFormatter(fmt) + return console_handler + + +def get_logging_file_handler( + file: str, + fmt: logging.Formatter, + set_formatter: bool = True) -> logging.FileHandler: + ''' + Gets a logging file handler (logging.FileHandler) based on the specified + formatter (logging.Formatter). + ''' + file_handler = logging.FileHandler(file) + file_handler.setLevel(logging.INFO) + if set_formatter: + file_handler.setFormatter(fmt) + return file_handler diff --git a/scripts/build/exception/FatalError.py b/scripts/build/exception/FatalError.py new file mode 100644 index 00000000000..85ea14bbc78 --- /dev/null +++ b/scripts/build/exception/FatalError.py @@ -0,0 +1,10 @@ +''' +Contains the definition of the FatalError exception. +''' + + +class FatalError(Exception): + ''' + Raised for various script errors regarding environment and build + requirements. + ''' diff --git a/scripts/build/exception/__init__.py b/scripts/build/exception/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/parser/TargetFrameworkAction.py b/scripts/build/parser/TargetFrameworkAction.py new file mode 100644 index 00000000000..51b44101c89 --- /dev/null +++ b/scripts/build/parser/TargetFrameworkAction.py @@ -0,0 +1,30 @@ +''' +Contains the definition of the TargetFrameworkAction type used to parse +the .NET Cli the supported target frameworks. +''' + +import argparse + + +class TargetFrameworkAction(argparse.Action): + ''' + Used by the ArgumentParser to represent the information needed to parse the + supported .NET Core target frameworks argument from the command line. + ''' + + def __call__(self, parser, namespace, values, option_string=None): + if values: + wrong_choices = [] + for value in values: + if value not in self.supported_target_frameworks(): + wrong_choices.append(value) + if wrong_choices: + message = ', '.join(wrong_choices) + message = 'Invalid choice(s): {}'.format(message) + raise argparse.ArgumentError(self, message) + setattr(namespace, self.dest, values) + + @staticmethod + def supported_target_frameworks() -> list: + '''List of supported .NET Core target frameworks.''' + return ['netcoreapp1.1', 'netcoreapp2.0', 'netcoreapp2.1', 'net461'] diff --git a/scripts/build/parser/__init__.py b/scripts/build/parser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/process/DotNet.py b/scripts/build/process/DotNet.py new file mode 100644 index 00000000000..54144aa0c32 --- /dev/null +++ b/scripts/build/process/DotNet.py @@ -0,0 +1,106 @@ +''' +Contains the definition of DotNet process. +''' + +import os + +from ..common import get_repo_root_path +from ..runner.RunCommand import RunCommand + + +class DotNet(object): + ''' + This is a class wrapper around the `dotnet` command line interface. + ''' + + def __init__( + self, + log_file: str, + working_directory: str, + csproj_file: str, + verbose: bool): + if not log_file: + raise TypeError('Unspecified log file.') + if not working_directory: + raise TypeError('Unspecified working directory.') + if not os.path.isdir(working_directory): + raise ValueError( + 'Specified working directory: {}, does not exist.'.format( + working_directory)) + + if os.path.isabs(csproj_file) and not os.path.exists(csproj_file): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + elif not os.path.exists(os.path.join(working_directory, csproj_file)): + raise ValueError( + 'Specified project file: {}, does not exist.'.format( + csproj_file)) + + self.__log_file = log_file + self.__working_directory = working_directory + self.__csproj_file = csproj_file + self.__verbose = verbose + + @property + def log_file(self) -> str: + '''Gets the log file name to write to.''' + return self.__log_file + + @property + def working_directory(self) -> str: + '''Gets the working directory for the dotnet process to be started.''' + return self.__working_directory + + @property + def csproj_file(self) -> str: + '''Gets the project file to run the dotnet cli against.''' + return self.__csproj_file + + @property + def verbose(self) -> bool: + '''Gets a flag to whether verbosity if turned on or off.''' + return self.__verbose + + @property + def packages_path(self) -> str: + '''Gets the folder to restore packages to.''' + return os.path.join(get_repo_root_path(), 'packages') + + @property + def bin_path(self) -> str: + '''Gets the directory in which the built binaries will be placed.''' + return os.path.join(get_repo_root_path(), 'bin') + + def restore(self) -> None: + ''' + Calls dotnet to restore the dependencies and tools of the specified + project. + ''' + cmdline = ['dotnet', 'restore', + '--packages', self.packages_path, + self.csproj_file] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-restore', self.working_directory) + + def publish(self, + configuration: str, + framework: str, + product: str) -> None: + ''' + Calls dotnet to pack the specified application and its dependencies + into the repo bin folder for deployment to a hosting system. + ''' + if not product: + raise TypeError('Unspecified product name.') + base_output_path = '{}{}'.format( + os.path.join(self.bin_path, product), os.path.sep) + + cmdline = ['dotnet', 'publish', + '--no-restore', + '--configuration', configuration, + '--framework', framework, + self.csproj_file, + '/p:BaseOutputPath={}'.format(base_output_path)] + RunCommand(self.log_file, cmdline, verbose=self.verbose).run( + 'dotnet-publish', self.working_directory) diff --git a/scripts/build/process/__init__.py b/scripts/build/process/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/build/runner/RunCommand.py b/scripts/build/runner/RunCommand.py new file mode 100644 index 00000000000..c935bc4df55 --- /dev/null +++ b/scripts/build/runner/RunCommand.py @@ -0,0 +1,122 @@ +''' +Contains the definition of RunCommand runner object. +''' + +import logging +import os +import subprocess + +from ..common import push_dir +from ..common import get_logging_console_handler +from ..common import get_logging_file_handler +from ..common import log_start_message +from ..common import LOGGING_FORMATTER + + +class RunCommand(object): + ''' + This is a class wrapper around `subprocess.Popen` with an additional set + of logging features. + ''' + + def __init__( + self, + log_file, + cmdline: list, + success_exit_codes: list = None, + verbose: bool = False): + if not log_file: + raise TypeError('Unspecified log file.') + if cmdline is None: + raise TypeError('Unspecified command line to be executed.') + if not cmdline: + raise ValueError('Specified command line is empty.') + + self.__log_file = log_file + self.__cmdline = cmdline + self.__verbose = verbose + + if success_exit_codes is None: + self.__success_exit_codes = [0] + else: + self.__success_exit_codes = success_exit_codes + + @property + def log_file(self) -> str: + '''Log file name to write to.''' + return self.__log_file + + @property + def cmdline(self) -> str: + '''Command-line to use when starting the application.''' + return self.__cmdline + + @property + def success_exit_codes(self) -> list: + ''' + The successful exit codes that the associated process specifies when it + terminated. + ''' + return self.__success_exit_codes + + @property + def verbose(self) -> bool: + '''Enables/Disables verbosity.''' + return self.__verbose + + def run(self, suffix: str = None, working_directory: str = None) -> None: + ''' + Executes specified shell command. + ''' + should_pipe = self.verbose + with push_dir(working_directory): + quoted_cmdline = subprocess.list2cmdline(self.cmdline) + quoted_cmdline += ' > {}'.format( + os.devnull) if not should_pipe else '' + + logging.getLogger('shell').info(quoted_cmdline) + exe_name = os.path.basename(self.cmdline[0]).replace('.', '_') + + exe_log_file = self.log_file + if suffix is not None: + exe_log_file = exe_log_file.replace( + '.log', '.{}.log'.format(suffix)) + + exe_logger = logging.getLogger(exe_name) + exe_logger.handlers = [] + + file_handler = get_logging_file_handler( + exe_log_file, + LOGGING_FORMATTER, + set_formatter=(suffix is None)) + exe_logger.addHandler(file_handler) + + if suffix is not None: + log_start_message(exe_name) + + console_handler = get_logging_console_handler( + LOGGING_FORMATTER, self.verbose) + exe_logger.addHandler(console_handler) + + with open(os.devnull) as devnull: + with subprocess.Popen( + self.cmdline, + stdout=subprocess.PIPE if should_pipe else devnull, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) as proc: + + if proc.stdout is not None: + with proc.stdout: + for line in iter(proc.stdout.readline, ''): + line = line.rstrip() + exe_logger.info(line) + + proc.wait() + # FIXME: dotnet child processes are still running. + + if proc.returncode not in self.success_exit_codes: + exe_logger.error( + "Process exited with status %s", proc.returncode) + raise subprocess.CalledProcessError( + proc.returncode, quoted_cmdline) diff --git a/scripts/build/runner/__init__.py b/scripts/build/runner/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/coreclr/README.md b/src/coreclr/README.md index d147748e1d7..b3ed850cd7e 100644 --- a/src/coreclr/README.md +++ b/src/coreclr/README.md @@ -4,6 +4,7 @@ This is a collection of micro benchmarks, ported from [CoreClr](https://github.c ## Supported target frameworks ## +- net461 - netcoreapp1.1 - netcoreapp2.0 - netcoreapp2.1 @@ -77,3 +78,21 @@ REM WARNING! The command below will run all the CoreClr benchmarks. cd PerformanceHarness/bin/x64/Release/netcoreapp2.0/publish dotnet PerformanceHarness.dll --perf:collect stopwatch+gcapi ``` + +### Example 4: Running with CoreClr's CoreRun.exe ### + +For this you will need to patch Core_Root with few assemblies: + +Managed (the assemblies used by the benchmarks are newer than the ones on Core_Root): + +- Microsoft.CodeAnalysis*.dll +- Newtonsoft.Json*.dll + +**Affected benchmarks: Serialization, and Roslyn. + +Native (assemblies used by TraceEvent not deployed with Core_Root): + +- \amd64\*.* +- \x86\*.* + +This [batch script](run-with-coreclr-corerun.cmd) is an example of the steps needed to run the CoreClr benchmarks with CoreRun. diff --git a/src/coreclr/common.props b/src/coreclr/common.props index 37d0c2da33d..0862e50af1b 100644 --- a/src/coreclr/common.props +++ b/src/coreclr/common.props @@ -23,6 +23,16 @@ $RepositoryUrl + + + true + + All diff --git a/src/coreclr/run-with-coreclr-corerun.cmd b/src/coreclr/run-with-coreclr-corerun.cmd new file mode 100644 index 00000000000..f8b6cc48a16 --- /dev/null +++ b/src/coreclr/run-with-coreclr-corerun.cmd @@ -0,0 +1,190 @@ +@REM This is a template script to run .NET Performance benchmarks with CoreRun.exe +@if not defined _echo echo off + + +setlocal + set "ERRORLEVEL=" + set "USAGE_DISPLAYED=" + set "PUBLISH_DIR=" + set "BENCHMARK_ASSEMBLY=" + + set "ARCHITECTURE=x64" + set "CONFIGURATION=Release" + set "TARGET_FRAMEWORK=netcoreapp2.0" + + call :parse_command_line_arguments %* || exit /b 1 + if defined USAGE_DISPLAYED exit /b %ERRORLEVEL% + cd "%PUBLISH_DIR%" + call :patch_core_root || exit /b 1 + call :common_env || exit /b 1 + + call :run_command %STABILITY_PREFIX% "%CORE_ROOT%\CoreRun.exe" PerformanceHarness.dll %BENCHMARK_ASSEMBLY% --perf:collect %COLLECTION_FLAGS% +endlocal& exit /b %ERRORLEVEL% + + +:common_env + echo/ Common .NET environment variables set. + set COMPlus_ + set DOTNET_ + set UseSharedCompilation + set XUNIT_ + exit /b 0 + + +:patch_core_root +rem **************************************************************************** +rem Copies latest managed binaries needed by the benchmarks, plus native +rem binaries used by TraceEvent. +rem **************************************************************************** + REM Managed binaries + for %%f in (Microsoft.CodeAnalysis Newtonsoft.Json) do ( + call :run_command xcopy.exe /VYRQKZ "%CD%\%%f*.dll" "%CORE_ROOT%\" || ( + call :print_error Failed to copy from: "%CD%\%%f*.dll", to: "%CORE_ROOT%\" + exit /b 1 + ) + ) + + REM Copy native libraries used by Microsoft.Diagnostics.Tracing.TraceEvent.dll + for %%a in (amd64 x86) do ( + for %%f in (KernelTraceControl msdia140) do ( + call :run_command xcopy.exe /VYRQKZ "%CD%\%%a\%%f.dll" "%CORE_ROOT%\%%a\" || ( + call :print_error Failed to copy from: "%CD%\%%a\%%f.dll", to: "%CORE_ROOT%\%%a\" + exit /b 1 + ) + ) + ) + exit /b %ERRORLEVEL% + + +:parse_command_line_arguments +rem **************************************************************************** +rem Parses the script's command line arguments. +rem **************************************************************************** + IF /I [%~1] == [--core-root] ( + set "CORE_ROOT=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--stability-prefix] ( + set "STABILITY_PREFIX=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--collection-flags] ( + set "COLLECTION_FLAGS=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--publish-dir] ( + set "PUBLISH_DIR=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + IF /I [%~1] == [--assembly] ( + set "BENCHMARK_ASSEMBLY=%~2" + shift + shift + goto :parse_command_line_arguments + ) + + if /I [%~1] == [-?] ( + call :usage + exit /b 0 + ) + if /I [%~1] == [-h] ( + call :usage + exit /b 0 + ) + if /I [%~1] == [--help] ( + call :usage + exit /b 0 + ) + + if not defined CORE_ROOT ( + call :print_error CORE_ROOT was not defined. + exit /b 1 + ) + + if not defined PUBLISH_DIR ( + call :print_error --publish-dir was not specified. + exit /b 1 + ) + if not exist "%PUBLISH_DIR%" ( + call :print_error Specified published directory: %PUBLISH_DIR%, does not exist. + exit /b 1 + ) + + if not defined BENCHMARK_ASSEMBLY ( + call :print_to_console --assembly DOTNET_ASSEMBLY_NAME was not specified, all benchmarks will be run. + ) + + if not defined COLLECTION_FLAGS ( + call :print_to_console COLLECTION_FLAGS was not defined. Defaulting to stopwatch + set "COLLECTION_FLAGS=stopwatch" + ) + + exit /b %ERRORLEVEL% + + +:usage +rem **************************************************************************** +rem Script's usage. +rem **************************************************************************** + set "USAGE_DISPLAYED=1" + echo/ %~nx0 [OPTIONS] + echo/ + echo/ Options: + echo/ --collection-flags COLLECTION_FLAGS + echo/ xUnit-Performance Api valid performance collection flags: ^ + echo/ --core-root CORE_ROOT + echo/ CoreClr's CORE_ROOT directory (Binaries to be tested). + echo/ --publish-dir PUBLISH_DIR + echo/ Directory that contains the published .NET benchmarks. + echo/ --stability-prefix STABILITY_PREFIX + echo/ Command to prepend to the benchmark execution. + echo/ --assembly DOTNET_ASSEMBLY_NAME + echo/ .NET assembly to be tested. + exit /b %ERRORLEVEL% + + +:run_command +rem **************************************************************************** +rem Function wrapper used to send the command line being executed to the +rem console screen, before the command is executed. +rem **************************************************************************** + if "%~1" == "" ( + call :print_error No command was specified. + exit /b 1 + ) + + call :print_to_console $ %* + call %* + exit /b %ERRORLEVEL% + + +:print_error +rem **************************************************************************** +rem Function wrapper that unifies how errors are output by the script. +rem Functions output to the standard error. +rem **************************************************************************** + call :print_to_console [ERROR] %* 1>&2 + exit /b %ERRORLEVEL% + + +:print_to_console +rem **************************************************************************** +rem Sends text to the console screen. This can be useful to provide +rem information on where the script is executing. +rem **************************************************************************** + echo/ + echo/%USERNAME%@%COMPUTERNAME% "%CD%" + echo/[%DATE%][%TIME:~0,-3%] %* + exit /b %ERRORLEVEL%