Fix optdata restore functionality (#11935)

Fix optdata restore functionality (#11935)

Bring back optdata restore functionality following the move to 2.0
BuildTools and csproj-based CLI. Disable a harmless warning that broke
the Linux build when consuming PGO counts due to -Werr.

Fix #11796 for master

build.cmd

@@ -236,21 +236,6 @@ REM === Restore optimization profile data
 REM ===
 REM =========================================================================================
 
-REM Parse the package version out of project.json so that we can pass it on to CMake
-where /q python || (
-    echo %__MsgPrefix%Error: Python not found on PATH, please make sure that it is installed.
-    exit /b 1
-)
-set OptDataProjectJsonPath=%__ProjectDir%\src\.nuget\optdata\project.json
-if EXIST "%OptDataProjectJsonPath%" (
-    for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.PGO.CoreCLR') do @(
-        set __PgoOptDataVersion=%%s
-    )
-    for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.IBC.CoreCLR') do @(
-        set __IbcOptDataVersion=%%s
-    )
-)
-
 if %__RestoreOptData% EQU 1 (
     echo %__MsgPrefix%Restoring the OptimizationData Package
     @call %__ProjectDir%\run.cmd sync -optdata
@@ -260,6 +245,20 @@ if %__RestoreOptData% EQU 1 (
     )
 )
 
+REM Parse the optdata package versions out of msbuild so that we can pass them on to CMake
+set DotNetCli=%__ProjectDir%\Tools\dotnetcli\dotnet.exe
+if not exist "%DotNetCli%" (
+    echo Assertion failed: dotnet.exe not found at path "%DotNetCli%"
+    exit /b 1
+)
+set OptDataProjectFilePath=%__ProjectDir%\src\.nuget\optdata\optdata.csproj
+for /f "tokens=*" %%s in ('%DotNetCli% msbuild "%OptDataProjectFilePath%" /t:DumpPgoDataPackageVersion /nologo') do @(
+    set __PgoOptDataVersion=%%s
+)
+for /f "tokens=*" %%s in ('%DotNetCli% msbuild "%OptDataProjectFilePath%" /t:DumpIbcDataPackageVersion /nologo') do @(
+    set __IbcOptDataVersion=%%s
+)
+
 REM =========================================================================================
 REM ===
 REM === Build the CLR VM

build.proj

@@ -24,12 +24,8 @@
     <Delete Files="$(BinDir)System.Private.CoreLib.*" />
   </Target>
 
-  <PropertyGroup>
-    <OptDataProjectJson>$(SourceDir).nuget/optdata/project.json</OptDataProjectJson>
-    <OptDataPackageFeed>https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json</OptDataPackageFeed>
-  </PropertyGroup>
   <Target Name="RestoreOptData">
-    <Exec Condition="Exists('$(OptDataProjectJson)')" Command="$(DnuRestoreCommand) &quot;$(OptDataProjectJson)&quot; --source &quot;$(OptDataPackageFeed)&quot;" />
+    <Exec Command="$(DotnetRestoreCommand) $(SourceDir).nuget/optdata/optdata.csproj" />
   </Target>
 
   <!--

build.sh

@@ -141,6 +141,16 @@ restore_optdata()
             echo "Failed to restore the optimization data package."
             exit 1
         fi
+
+        # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
+        local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
+        if [ ! -f $DotNetCli ]; then
+            echo "Assertion failed: dotnet CLI not found at '$DotNetCli'"
+            exit 1
+        fi
+        local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
+        __PgoOptDataVersion=$($DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
+        __IbcOptDataVersion=$($DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^\s*//')
     fi
 }
 
@@ -897,13 +907,6 @@ if [ $__CrossBuild == 1 ]; then
     fi
 fi
 
-# Parse the optdata package version from its project.json file
-optDataProjectJsonPath="$__ProjectRoot/src/.nuget/optdata/project.json"
-if [ -f $optDataProjectJsonPath ]; then
-    __PgoOptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.PGO.CoreCLR)
-    __IbcOptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.IBC.CoreCLR)
-fi
-
 # init the target distro name
 initTargetDistroRid
 

dir.props

@@ -21,6 +21,12 @@
     <RoslynPackageName>Microsoft.Net.ToolsetCompilers</RoslynPackageName>
   </PropertyGroup>
 
+  <!-- Profile-based optimization data package versions -->
+  <PropertyGroup>
+    <PgoDataPackageVersion>99.99.99-master-20170531-3000</PgoDataPackageVersion>
+    <!--<IbcDataPackageVersion></IbcDataPackageVersion>-->
+  </PropertyGroup>
+
   <!--
      Switching to the .NET Core version of the BuildTools tasks seems to break numerous scenarios, such as VS intellisense and resource designer
      as well as runnning the build on mono. Until we can get these sorted out we will continue using the .NET 4.5 version of the tasks.

pgosupport.cmake

@@ -20,6 +20,8 @@ function(add_pgo TargetName)
         "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
         ProfilePath
     )
+    # NuGet packages are restored to lowercase paths
+    string(TOLOWER "${ProfilePath}" ProfilePath)
 
     if(CLR_CMAKE_PGO_INSTRUMENT)
         if(WIN32)
@@ -40,7 +42,7 @@ function(add_pgo TargetName)
             else(WIN32)
                 if(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)
                     if(HAVE_LTO)
-                        target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath})
+                        target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath} -Wno-profile-instr-out-of-date)
                         set_property(TARGET ${TargetName} APPEND_STRING PROPERTY LINK_FLAGS " -flto -fuse-ld=gold -fprofile-instr-use=${ProfilePath}")
                     endif(HAVE_LTO)
                 endif(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)

src/.nuget/optdata/nuget.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+ <packageSources>
+    <add key="myget.org dotnet-core-optimization-data" value="https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json" />
+ </packageSources>
+</configuration>

src/.nuget/optdata/optdata.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="optimization.PGO.CoreCLR" Version="$(PgoDataPackageVersion)" Condition="'$(PgoDataPackageVersion)'!=''" />
+    <PackageReference Include="optimization.IBC.CoreCLR" Version="$(IbcDataPackageVersion)" Condition="'$(IbcDataPackageVersion)'!=''" />
+  </ItemGroup>
+
+  <Target Name="DumpPgoDataPackageVersion">
+    <Message Importance="high" Text="$(PgoDataPackageVersion)" />
+  </Target>
+
+  <Target Name="DumpIbcDataPackageVersion">
+    <Message Importance="high" Text="$(IbcDataPackageVersion)" />
+  </Target>
+
+</Project>

tests/scripts/optdata/bootstrap.py

@@ -7,12 +7,12 @@
 """
 
 import argparse
-import json
 import os
 from os import path
 import shutil
 import subprocess
 import sys
+import xml.etree.ElementTree as ET
 
 # Display the docstring if the user passes -h|--help
 argparse.ArgumentParser(description=__doc__).parse_args()
@@ -24,8 +24,8 @@
 assert path.exists(NUGET_SRC_DIR), \
     "Expected %s to exist; please check whether REPO_ROOT is really %s" % (NUGET_SRC_DIR, REPO_ROOT)
 
-ORIGIN_FILE = path.join(SCRIPT_ROOT, 'project.json')
-TARGET_FILE = path.join(NUGET_SRC_DIR, 'optdata', 'project.json')
+ORIGIN_FILE = path.join(SCRIPT_ROOT, 'optdata.csproj')
+TARGET_FILE = path.join(NUGET_SRC_DIR, 'optdata', 'optdata.csproj')
 
 ARCH_LIST = ['x64', 'x86']
 TOOL_LIST = ['IBC', 'PGO']
@@ -40,9 +40,12 @@ def get_buildos():
 
 def get_optdata_version(tool):
     """Returns the version string specified in project.json for the given tool."""
-    package_name = 'optimization.%s.CoreCLR' % (tool)
-    with open(ORIGIN_FILE) as json_file:
-        return json.load(json_file)['dependencies'][package_name]
+    element_name = {
+        'IBC': 'IbcDataPackageVersion',
+        'PGO': 'PgoDataPackageVersion',
+    }[tool]
+    root = ET.parse(ORIGIN_FILE)
+    return root.findtext('./PropertyGroup/{}'.format(element_name))
 
 def get_optdata_dir(tool, arch):
     """Returns an absolute path to the directory that should contain optdata given a tool,arch"""

tests/scripts/optdata/optdata.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <PgoDataPackageVersion>99.99.99-test</PgoDataPackageVersion>
+    <IbcDataPackageVersion>99.99.99-test</IbcDataPackageVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="optimization.PGO.CoreCLR" Version="$(PgoDataPackageVersion)" Condition="'$(PgoDataPackageVersion)'!=''" />
+    <PackageReference Include="optimization.IBC.CoreCLR" Version="$(IbcDataPackageVersion)" Condition="'$(IbcDataPackageVersion)'!=''" />
+  </ItemGroup>
+
+  <Target Name="DumpPgoDataPackageVersion">
+    <Message Importance="high" Text="$(PgoDataPackageVersion)" />
+  </Target>
+
+  <Target Name="DumpIbcDataPackageVersion">
+    <Message Importance="high" Text="$(IbcDataPackageVersion)" />
+  </Target>
+
+</Project>