From 96f20275be683866cea28068189c66b9fe9a9a38 Mon Sep 17 00:00:00 2001
From: Daniel Podder <dapodd@microsoft.com>
Date: Thu, 1 Jun 2017 22:21:32 -0500
Subject: [PATCH] Fix optdata restore functionality (port #11935) (#12044)

* Port [Fix optdata restore functionality (#11935)] from master

* Fixup optdata package version
---
 build.cmd                            | 29 +++++++-------
 build.proj                           |  6 +--
 build.sh                             | 17 +++++----
 dir.props                            |  6 +++
 extract-from-json.py                 | 56 ----------------------------
 pgosupport.cmake                     |  4 +-
 src/.nuget/optdata/nuget.config      |  6 +++
 src/.nuget/optdata/optdata.csproj    | 24 ++++++++++++
 tests/scripts/optdata/bootstrap.py   | 15 +++++---
 tests/scripts/optdata/optdata.csproj | 29 ++++++++++++++
 tests/scripts/optdata/project.json   | 12 ------
 11 files changed, 102 insertions(+), 102 deletions(-)
 delete mode 100755 extract-from-json.py
 create mode 100644 src/.nuget/optdata/nuget.config
 create mode 100644 src/.nuget/optdata/optdata.csproj
 create mode 100644 tests/scripts/optdata/optdata.csproj
 delete mode 100644 tests/scripts/optdata/project.json

diff --git a/build.cmd b/build.cmd
index b3706baefa90..61d5536143d2 100644
--- a/build.cmd
+++ b/build.cmd
@@ -236,21 +236,6 @@ REM === Restore optimization profile data
 REM ===
 REM =========================================================================================
 
-REM Parse the package version out of project.json so that we can pass it on to CMake
-where /q python || (
-    echo %__MsgPrefix%Error: Python not found on PATH, please make sure that it is installed.
-    exit /b 1
-)
-set OptDataProjectJsonPath=%__ProjectDir%\src\.nuget\optdata\project.json
-if EXIST "%OptDataProjectJsonPath%" (
-    for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.PGO.CoreCLR') do @(
-        set __PgoOptDataVersion=%%s
-    )
-    for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.IBC.CoreCLR') do @(
-        set __IbcOptDataVersion=%%s
-    )
-)
-
 if %__RestoreOptData% EQU 1 (
     echo %__MsgPrefix%Restoring the OptimizationData Package
     @call %__ProjectDir%\run.cmd sync -optdata
@@ -260,6 +245,20 @@ if %__RestoreOptData% EQU 1 (
     )
 )
 
+REM Parse the optdata package versions out of msbuild so that we can pass them on to CMake
+set DotNetCli=%__ProjectDir%\Tools\dotnetcli\dotnet.exe
+if not exist "%DotNetCli%" (
+    echo Assertion failed: dotnet.exe not found at path "%DotNetCli%"
+    exit /b 1
+)
+set OptDataProjectFilePath=%__ProjectDir%\src\.nuget\optdata\optdata.csproj
+for /f "tokens=*" %%s in ('%DotNetCli% msbuild "%OptDataProjectFilePath%" /t:DumpPgoDataPackageVersion /nologo') do @(
+    set __PgoOptDataVersion=%%s
+)
+for /f "tokens=*" %%s in ('%DotNetCli% msbuild "%OptDataProjectFilePath%" /t:DumpIbcDataPackageVersion /nologo') do @(
+    set __IbcOptDataVersion=%%s
+)
+
 REM =========================================================================================
 REM ===
 REM === Build the CLR VM
diff --git a/build.proj b/build.proj
index 7325e6ff3049..8d15cbcd6e2c 100644
--- a/build.proj
+++ b/build.proj
@@ -24,12 +24,8 @@
     <Delete Files="$(BinDir)System.Private.CoreLib.*" />
   </Target>
 
-  <PropertyGroup>
-    <OptDataProjectJson>$(SourceDir).nuget/optdata/project.json</OptDataProjectJson>
-    <OptDataPackageFeed>https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json</OptDataPackageFeed>
-  </PropertyGroup>
   <Target Name="RestoreOptData">
-    <Exec Condition="Exists('$(OptDataProjectJson)')" Command="$(DnuRestoreCommand) &quot;$(OptDataProjectJson)&quot; --source &quot;$(OptDataPackageFeed)&quot;" />
+    <Exec Command="$(DotnetRestoreCommand) $(SourceDir).nuget/optdata/optdata.csproj" />
   </Target>
 
   <!--
diff --git a/build.sh b/build.sh
index 976e18bda664..55b4ba320081 100755
--- a/build.sh
+++ b/build.sh
@@ -141,6 +141,16 @@ restore_optdata()
             echo "Failed to restore the optimization data package."
             exit 1
         fi
+
+        # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
+        local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
+        if [ ! -f $DotNetCli ]; then
+            echo "Assertion failed: dotnet CLI not found at '$DotNetCli'"
+            exit 1
+        fi
+        local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
+        __PgoOptDataVersion=$($DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
+        __IbcOptDataVersion=$($DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^\s*//')
     fi
 }
 
@@ -899,13 +909,6 @@ if [ $__CrossBuild == 1 ]; then
     fi
 fi
 
-# Parse the optdata package version from its project.json file
-optDataProjectJsonPath="$__ProjectRoot/src/.nuget/optdata/project.json"
-if [ -f $optDataProjectJsonPath ]; then
-    __PgoOptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.PGO.CoreCLR)
-    __IbcOptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.IBC.CoreCLR)
-fi
-
 # init the target distro name
 initTargetDistroRid
 
diff --git a/dir.props b/dir.props
index cfc6a602bbe8..4f62d0e907ac 100644
--- a/dir.props
+++ b/dir.props
@@ -21,6 +21,12 @@
     <RoslynPackageName>Microsoft.Net.ToolsetCompilers</RoslynPackageName>
   </PropertyGroup>
 
+  <!-- Profile-based optimization data package versions -->
+  <PropertyGroup>
+    <PgoDataPackageVersion>2.0.0-release-20170531-3000</PgoDataPackageVersion>
+    <!--<IbcDataPackageVersion></IbcDataPackageVersion>-->
+  </PropertyGroup>
+
   <!--
      Switching to the .NET Core version of the BuildTools tasks seems to break numerous scenarios, such as VS intellisense and resource designer
      as well as runnning the build on mono. Until we can get these sorted out we will continue using the .NET 4.5 version of the tasks.
diff --git a/extract-from-json.py b/extract-from-json.py
deleted file mode 100755
index e432b2b067f0..000000000000
--- a/extract-from-json.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/python
-
-import argparse
-import json
-import sys
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        description="""Extracts information from a json file by navigating the JSON object using a
-            sequence of property accessors and returning the JSON subtree, or the raw data, found
-            at that location."""
-    )
-
-    parser.add_argument(
-        '-f', '--file',
-        metavar='<project.json>',
-        help="Path to project.json file to parse",
-        required=True,
-    )
-
-    parser.add_argument(
-        'property',
-        metavar='property_name',
-        help="""Name of property to extract using object notation.
-            Pass multiple values to drill down into nested objects (in order).""",
-        nargs='*',
-    )
-
-    parser.add_argument(
-        '-r', '--raw',
-        help="""Dumps the raw object found at the requested location.
-            If omitted, returns a JSON formatted object instead.""",
-        action='store_true',
-        default=False
-    )
-
-    return parser.parse_args()
-
-def main():
-    args = parse_args()
-
-    with open(args.file) as json_file:
-        selected_property = json.load(json_file)
-
-    for prop in args.property:
-        selected_property = selected_property[prop]
-
-    if args.raw:
-        print(selected_property)
-    else:
-        print(json.dumps(selected_property))
-
-    return 0
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/pgosupport.cmake b/pgosupport.cmake
index dbba415a6185..90bcbc387677 100644
--- a/pgosupport.cmake
+++ b/pgosupport.cmake
@@ -20,6 +20,8 @@ function(add_pgo TargetName)
         "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
         ProfilePath
     )
+    # NuGet packages are restored to lowercase paths
+    string(TOLOWER "${ProfilePath}" ProfilePath)
 
     if(CLR_CMAKE_PGO_INSTRUMENT)
         if(WIN32)
@@ -40,7 +42,7 @@ function(add_pgo TargetName)
             else(WIN32)
                 if(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)
                     if(HAVE_LTO)
-                        target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath})
+                        target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath} -Wno-profile-instr-out-of-date)
                         set_property(TARGET ${TargetName} APPEND_STRING PROPERTY LINK_FLAGS " -flto -fuse-ld=gold -fprofile-instr-use=${ProfilePath}")
                     endif(HAVE_LTO)
                 endif(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)
diff --git a/src/.nuget/optdata/nuget.config b/src/.nuget/optdata/nuget.config
new file mode 100644
index 000000000000..e747f7ecb9f6
--- /dev/null
+++ b/src/.nuget/optdata/nuget.config
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+ <packageSources>
+    <add key="myget.org dotnet-core-optimization-data" value="https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json" />
+ </packageSources>
+</configuration>
diff --git a/src/.nuget/optdata/optdata.csproj b/src/.nuget/optdata/optdata.csproj
new file mode 100644
index 000000000000..20e2a40adae9
--- /dev/null
+++ b/src/.nuget/optdata/optdata.csproj
@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="optimization.PGO.CoreCLR" Version="$(PgoDataPackageVersion)" Condition="'$(PgoDataPackageVersion)'!=''" />
+    <PackageReference Include="optimization.IBC.CoreCLR" Version="$(IbcDataPackageVersion)" Condition="'$(IbcDataPackageVersion)'!=''" />
+  </ItemGroup>
+
+  <Target Name="DumpPgoDataPackageVersion">
+    <Message Importance="high" Text="$(PgoDataPackageVersion)" />
+  </Target>
+
+  <Target Name="DumpIbcDataPackageVersion">
+    <Message Importance="high" Text="$(IbcDataPackageVersion)" />
+  </Target>
+
+</Project>
diff --git a/tests/scripts/optdata/bootstrap.py b/tests/scripts/optdata/bootstrap.py
index 1cf55fa70cef..8dcecca77924 100755
--- a/tests/scripts/optdata/bootstrap.py
+++ b/tests/scripts/optdata/bootstrap.py
@@ -7,12 +7,12 @@
 """
 
 import argparse
-import json
 import os
 from os import path
 import shutil
 import subprocess
 import sys
+import xml.etree.ElementTree as ET
 
 # Display the docstring if the user passes -h|--help
 argparse.ArgumentParser(description=__doc__).parse_args()
@@ -24,8 +24,8 @@
 assert path.exists(NUGET_SRC_DIR), \
     "Expected %s to exist; please check whether REPO_ROOT is really %s" % (NUGET_SRC_DIR, REPO_ROOT)
 
-ORIGIN_FILE = path.join(SCRIPT_ROOT, 'project.json')
-TARGET_FILE = path.join(NUGET_SRC_DIR, 'optdata', 'project.json')
+ORIGIN_FILE = path.join(SCRIPT_ROOT, 'optdata.csproj')
+TARGET_FILE = path.join(NUGET_SRC_DIR, 'optdata', 'optdata.csproj')
 
 ARCH_LIST = ['x64', 'x86']
 TOOL_LIST = ['IBC', 'PGO']
@@ -40,9 +40,12 @@ def get_buildos():
 
 def get_optdata_version(tool):
     """Returns the version string specified in project.json for the given tool."""
-    package_name = 'optimization.%s.CoreCLR' % (tool)
-    with open(ORIGIN_FILE) as json_file:
-        return json.load(json_file)['dependencies'][package_name]
+    element_name = {
+        'IBC': 'IbcDataPackageVersion',
+        'PGO': 'PgoDataPackageVersion',
+    }[tool]
+    root = ET.parse(ORIGIN_FILE)
+    return root.findtext('./PropertyGroup/{}'.format(element_name))
 
 def get_optdata_dir(tool, arch):
     """Returns an absolute path to the directory that should contain optdata given a tool,arch"""
diff --git a/tests/scripts/optdata/optdata.csproj b/tests/scripts/optdata/optdata.csproj
new file mode 100644
index 000000000000..ac7360643f22
--- /dev/null
+++ b/tests/scripts/optdata/optdata.csproj
@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard</TargetFramework>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <PgoDataPackageVersion>99.99.99-test</PgoDataPackageVersion>
+    <IbcDataPackageVersion>99.99.99-test</IbcDataPackageVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="optimization.PGO.CoreCLR" Version="$(PgoDataPackageVersion)" Condition="'$(PgoDataPackageVersion)'!=''" />
+    <PackageReference Include="optimization.IBC.CoreCLR" Version="$(IbcDataPackageVersion)" Condition="'$(IbcDataPackageVersion)'!=''" />
+  </ItemGroup>
+
+  <Target Name="DumpPgoDataPackageVersion">
+    <Message Importance="high" Text="$(PgoDataPackageVersion)" />
+  </Target>
+
+  <Target Name="DumpIbcDataPackageVersion">
+    <Message Importance="high" Text="$(IbcDataPackageVersion)" />
+  </Target>
+
+</Project>
diff --git a/tests/scripts/optdata/project.json b/tests/scripts/optdata/project.json
deleted file mode 100644
index ae8f9463c458..000000000000
--- a/tests/scripts/optdata/project.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-    "dependencies": {
-        "optimization.IBC.CoreCLR": "99.99.99-test",
-        "optimization.PGO.CoreCLR": "99.99.99-test"
-    },
-    "frameworks": {
-        "netstandard": {}
-    },
-    "runtimes": {
-        "win7-x64": {}
-    }
-}