[master] Add incremental build (#1740)

* Add incremental build (#1700) * Add semaphores for incremental build * Updates based on PR review comments * One more PR comment update Conflicts: build.proj repos/Directory.Build.targets repos/core-sdk.proj repos/core-setup.common.targets repos/coreclr-portable.proj repos/toolset.proj support/tarball/build.sh tools-local/init-build.proj * Trim trailing whitespace * Fix spacing issue
dotnet · Sep 17, 2020 · 952e162 · 952e162
1 parent 4cd1226
commit 952e162
Show file tree

Hide file tree

Showing 16 changed files with 231 additions and 71 deletions.
diff --git a/.gitignore b/.gitignore
@@ -281,3 +281,4 @@ testing-smoke/
 # UpdateDependenciesAndSubmitPullRequest: if no upgrades are performed, the
 # sanity check fails when "git status" shows this file.
 msbuild.ProjectImports.zip
+Build_*.ProjectImports.zip
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -211,9 +211,12 @@
     <RootRepoTests>known-good-tests</RootRepoTests>
   </PropertyGroup>
 
-  <ItemGroup>
-    <SourceBuiltPackage Include="$(SourceBuiltPackagesPath)/*.nupkg" />
-  </ItemGroup>
+  <!--
+    Semaphore path for incremental builds
+  -->
+  <PropertyGroup>
+    <CompletedSemaphorePath>$(BaseIntermediatePath)semaphores/</CompletedSemaphorePath>
+  </PropertyGroup>
 
   <!-- CLI internal version is statically set by us to a version that will never show up in the wild.
        This ensures we will never restore a public version instead of our source-built version.  We

diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -13,10 +13,14 @@
   <Target Name="AcquireDarc"
           Condition="
             '$(OfflineBuild)' != 'true' and
-            '$(SkipAcquireDarc)' != 'true'">
+            '$(SkipAcquireDarc)' != 'true'"
+          Inputs="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
+          Outputs="$(CompletedSemaphorePath)AcquireDarc.complete">
     <MSBuild
       Projects="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
       Targets="AcquireDarcCore" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)AcquireDarc.complete" Overwrite="true" />
   </Target>
 
   <Target Name="GetCommonDarcCloneCommand" DependsOnTargets="AcquireDarc">

diff --git a/build-source-tarball.sh b/build-source-tarball.sh
@@ -143,6 +143,6 @@ mkdir -p "$FULL_TARBALL_ROOT"
 MAIN_BUILD_ARGS+=( "/p:TarballRoot=$FULL_TARBALL_ROOT" )
 MAIN_BUILD_ARGS+=( "/p:PackSourceBuildTarball=true" )
 
-$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} "$@"
+$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTarball_$(date +"%m%d%H%M%S").binlog "$@"
 
 echo "Done. Tarball created: $TARBALL_ROOT"
diff --git a/build.proj b/build.proj
@@ -44,7 +44,7 @@
   <Target Name="InitBuild">
     <MSBuild Projects="tools-local/init-build.proj" />
   </Target>
-  
+
   <Target Name="Clone">
     <MSBuild Projects="tools-local/init-build.proj" Targets="FetchSources" />
   </Target>
@@ -55,18 +55,36 @@
 
   <Target Name="DownloadSourceBuildReferencePackages"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
-    <DownloadFileSB 
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'"
+          Inputs="$(MSBuildProjectFullPath)"
+          Outputs="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" >
+    <PropertyGroup Condition="'$(DownloadSourceBuildReferencePackagesTimeoutSeconds)' == ''">
+      <DownloadSourceBuildReferencePackagesTimeoutSeconds>600</DownloadSourceBuildReferencePackagesTimeoutSeconds>
+    </PropertyGroup>
+
+    <DownloadFileSB
           SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
-          DestinationFolder="$(ExternalTarballsDir)" />
+          DestinationFolder="$(ExternalTarballsDir)"
+          TimeoutSeconds="$(DownloadSourceBuildReferencePackagesTimeoutSeconds)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" Overwrite="true" />
   </Target>
 
   <Target Name="DownloadSourceBuiltArtifacts"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
-    <DownloadFileSB 
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'"
+          Inputs="$(MSBuildProjectFullPath)"
+          Outputs="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" >
+    <PropertyGroup Condition="'$(DownloadSourceBuiltArtifactsTimeoutSeconds)' == ''">
+      <DownloadSourceBuiltArtifactsTimeoutSeconds>1500</DownloadSourceBuiltArtifactsTimeoutSeconds>
+    </PropertyGroup>
+
+    <DownloadFileSB
           SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
-          DestinationFolder="$(ExternalTarballsDir)" />
+          DestinationFolder="$(ExternalTarballsDir)"
+          TimeoutSeconds="$(DownloadSourceBuiltArtifactsTimeoutSeconds)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" Overwrite="true" />
   </Target>
 
   <!-- After building, generate a prebuilt usage report. -->
@@ -88,23 +106,31 @@
 
   <Target Name="ReportPoisonUsage"
           AfterTargets="Build"
-          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'"
+          Inputs="$(MSBuildProjectFullPath)"
+          Outputs="$(CompletedSemaphorePath)ReportPoisonUsage.complete" >
     <ItemGroup>
       <FinalCliTarball Include="$(SourceBuiltTarBallPath)**/*$(TarBallExtension)" />
     </ItemGroup>
     <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
                     PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
   </Target>
 
-  <Target Name="GeneratePrebuiltBurndownData">
+  <Target Name="GeneratePrebuiltBurndownData"
+          Inputs="$(MSBuildProjectFullPath)"
+          Outputs="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" >
     <WriteUsageBurndownData RootDirectory="$(ProjectDir)"
                             PrebuiltBaselineFile="$(OnlineBaselineDataFile)"
                             OutputFilePath="$(OnlinePrebuiltBurndownDataFile)" />
     <WriteUsageBurndownData RootDirectory="$(ProjectDir)"
                             PrebuiltBaselineFile="$(OfflineBaselineDataFile)"
                             OutputFilePath="$(OfflinePrebuiltBurndownDataFile)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" Overwrite="true" />
   </Target>
 
   <Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">

diff --git a/build.sh b/build.sh
@@ -111,5 +111,5 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 if [ "$alternateTarget" == "true" ]; then
   "$CLIPATH/dotnet" $SDKPATH/MSBuild.dll "$scriptroot/build.proj" /bl:source-build-test.binlog /flp:v=diag /clp:v=m "$@"
 else
-  "$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false -bl /flp:v=diag "$@"
+  "$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false /bl:$scriptroot/artifacts/log/Debug/Build_$(date +"%m%d%H%M%S").binlog /flp:v=diag "$@"
 fi
diff --git a/repos/Directory.Build.props b/repos/Directory.Build.props
@@ -17,6 +17,11 @@
     <RepoConsoleLogFile>$(LoggingDir)$(RepositoryName).log</RepoConsoleLogFile>
     <RedirectRepoOutputToLog Condition="'$(MinimalConsoleLogOutput)' == 'true'">&gt;&gt; $(RepoConsoleLogFile) 2&gt;&amp;1</RedirectRepoOutputToLog>
     <IsStable Condition="'$(UseStableVersions)' == 'true'">true</IsStable>
+
+    <!--
+      Repo specific semaphore path for incremental build
+    -->
+    <RepoCompletedSemaphorePath>$(CompletedSemaphorePath)$(RepositoryName)/</RepoCompletedSemaphorePath>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(OS)' == 'Windows_NT'">
@@ -77,8 +82,8 @@
     <!-- We pass '-ci', but also apply ci mode via env var for edge cases. (E.g. misbehaving inner builds.) -->
     <EnvironmentVariables Include="ContinuousIntegrationBuild=true" />
 
-    <!-- Turn off node reuse for source build because repos use conflicting versions 
-         of compilers which cause assembly load errors.  
+    <!-- Turn off node reuse for source build because repos use conflicting versions
+         of compilers which cause assembly load errors.
          See https://github.com/dotnet/source-build/issues/541 -->
     <EnvironmentVariables Include="MSBUILDDISABLENODEREUSE=1" />