Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce disk space usage by using hard links in a few copy operations and improve repo clean #46208

Merged
merged 3 commits into from
Jan 22, 2025
Merged

Reduce disk space usage by using hard links in a few copy operations and improve repo clean #46208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fb0b79d
Reduce disk space usage by using hard links in a few copy operations
ViktorHofer Jan 22, 2025
d621260
Fix typo in 'UseHardLinksIfPossible' attribute
ViktorHofer Jan 22, 2025
ded787e
Don't project.assets.json files in non-source-only builds & make repo…
ViktorHofer Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/SourceBuild/content/Directory.Build.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
Inputs="@(BinPlaceDir);%(BinPlaceDir.Identity)"
Outputs="unused">
<PropertyGroup>
<BinPlaceUseHardlinksIfPossible Condition="'$(BinPlaceUseHardlinksIfPossible)' == ''">true</BinPlaceUseHardlinksIfPossible>
<_BinPlaceDir>%(BinPlaceDir.Identity)</_BinPlaceDir>
</PropertyGroup>

Expand All @@ -62,7 +61,7 @@
OverwriteReadOnlyFiles="true"
Retries="$(CopyRetryCount)"
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
UseHardlinksIfPossible="$(BinPlaceUseHardlinksIfPossible)" />
UseHardlinksIfPossible="true" />
</Target>

</Project>
64 changes: 35 additions & 29 deletions src/SourceBuild/content/repo-projects/Directory.Build.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,8 @@

<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(NuGetConfigFile)'))" />

<Copy
SourceFiles="$(OriginalNuGetConfigFile)"
DestinationFiles="$(NuGetConfigFile)">
<Copy SourceFiles="$(OriginalNuGetConfigFile)"
DestinationFiles="$(NuGetConfigFile)">
<Output TaskParameter="CopiedFiles" ItemName="FileWrites" />
</Copy>
</Target>
Expand Down Expand Up @@ -602,6 +601,7 @@
DestinationFiles="$(NuGetPackageRoot)%(RecursiveDir)%(Filename)%(Extension)"
SkipUnchangedFiles="true"
Retries="5"
UseHardlinksIfPossible="true"
Condition="'@(_InnerPackageCacheFiles)' != ''" />
</Target>

Expand All @@ -625,9 +625,28 @@
Condition="'@(PrebuiltReportsToMove)' != ''" />
</Target>

<!-- Make a copy of project.assets.json files for prebuilt report generation. -->
<Target Name="BackupProjectAssetsJsonFiles"
Condition="'$(DotNetBuildSourceOnly)' == 'true'">
<PropertyGroup>
<ProjectAssetsJsonBackupDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'project-assets-json'))</ProjectAssetsJsonBackupDir>
</PropertyGroup>

<ItemGroup>
<ProjectAssetsJsonFile Include="$(RepoArtifactsDir)**/project.assets.json" />
</ItemGroup>

<Copy SourceFiles="@(ProjectAssetsJsonFile)"
DestinationFolder="$(ProjectAssetsJsonBackupDir)%(RecursiveDir)"
UseHardlinksIfPossible="true"
SkipUnchangedFiles="true"
Condition="'@(ProjectAssetsJsonFile)' != ''" />
</Target>

<Target Name="CleanupRepo"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)CleanupRepo.complete"
DependsOnTargets="BackupProjectAssetsJsonFiles"
Condition="'$(IsUtilityProject)' != 'true' and
'$(CleanWhileBuilding)' == 'true' and
Exists('$(RepoArtifactsDir)')">
Expand All @@ -646,35 +665,20 @@
IgnoreStandardErrorWarningFormat="true"
IgnoreExitCode="true" />

<Message Text="DirSize After Building $(RepositoryName)" Importance="High" Condition="'$(BuildOS)' != 'windows'" />
<Exec Command="df -h $(RepoRoot)" Condition="'$(BuildOS)' != 'windows'" />

<!-- Cleanup the entire repo artifacts dir. Ignore failures as this is best effort.
Don't use Removedir as ContinueOnError on it doesn't work when using warnaserror/TreatWarningsAsErrors. -->
<PropertyGroup>
<BuildObjDir>$([MSBuild]::NormalizeDirectory('$(ProjectDirectory)', 'artifacts', 'buildObj'))</BuildObjDir>
<RepoCleanCommand Condition="'$(BuildOS)' == 'windows'">rmdir "$(RepoArtifactsDir.TrimEnd('\'))" /s /q</RepoCleanCommand>
<RepoCleanCommand Condition="'$(BuildOS)' != 'windows'">rm -rf "$(RepoArtifactsDir)"</RepoCleanCommand>
</PropertyGroup>

<ItemGroup>
<ObjFilesToCopy Include="$(RepoArtifactsDir)**/project.assets.json" />
</ItemGroup>

<!-- Make a copy of project.assets.json files -->
<Copy SourceFiles="@(ObjFilesToCopy)"
DestinationFolder="$(BuildObjDir)%(RecursiveDir)"
Condition="'@(ObjFilesToCopy)' != ''" />

<ItemGroup>
<DirsToDelete Include="$([System.IO.Directory]::GetDirectories('$(RepoArtifactsDir)'))" />

<DirsToDeleteWithTrailingSeparator Include="$([MSBuild]::EnsureTrailingSlash('%(DirsToDelete.Identity)'))" />
<DirsToDeleteWithTrailingSeparator Remove="$(BuildObjDir)" />
</ItemGroup>

<Message Text="DirSize After Building $(RepositoryName)" Importance="High" Condition="'$(BuildOS)' != 'windows' and '@(DirsToDeleteWithTrailingSeparator)' != ''" />
<Exec Command="df -h $(RepoRoot)" Condition="'$(BuildOS)' != 'windows' and '@(DirsToDeleteWithTrailingSeparator)' != ''" />

<!-- Cleanup everything else. Ignore errors as out-of-band build servers don't reliably shut down, even with the build-server shutdown command.
https://github.com/dotnet/source-build/issues/4175 tracks a long term fix. -->
<RemoveDir Directories="@(DirsToDeleteWithTrailingSeparator)" ContinueOnError="true" />
<Exec Command="$(RepoCleanCommand)" IgnoreExitCode="true" IgnoreStandardErrorWarningFormat="true" />

<Message Text="DirSize After CleanupRepo $(RepositoryName)" Importance="High" Condition="'$(BuildOS)' != 'windows'and '@(DirsToDeleteWithTrailingSeparator)' != ''" />
<Exec Command="df -h $(RepoRoot)" Condition="'$(BuildOS)' != 'windows'and '@(DirsToDeleteWithTrailingSeparator)' != ''" />
<Message Text="DirSize After CleanupRepo $(RepositoryName)" Importance="High" Condition="'$(BuildOS)' != 'windows'" />
<Exec Command="df -h $(RepoRoot)" Condition="'$(BuildOS)' != 'windows'" />

<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)CleanupRepo.complete" AlwaysCreate="true">
Expand Down Expand Up @@ -720,7 +724,9 @@
<ExtractedToolFiles Include="$(SourceBuiltSdksDir)%(_ToolPackage.Id)/**/*netcore*/*.dll" />
</ItemGroup>

<Copy SourceFiles="@(ExtractedToolFiles)" DestinationFolder="$(SourceBuiltSdksDir)/" />
<Copy SourceFiles="@(ExtractedToolFiles)"
DestinationFolder="$(SourceBuiltSdksDir)/"
UseHardlinksIfPossible="true" />

<MakeDir Directories="$(BaseIntermediateOutputPath)" />
<Touch Files="$(BaseIntermediateOutputPath)ExtractToolPackage.complete" AlwaysCreate="true">
Expand Down
Loading