Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[vcpkg] Add write-only binary caching for CI. #12130

Merged
merged 4 commits into from
Jun 26, 2020
Merged

[vcpkg] Add write-only binary caching for CI. #12130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
70e002d
[vcpkg] Add write-only binary caching for CI.
BillyONeal Jun 26, 2020
eca8b1f
Delete accidentally doubled line.
BillyONeal Jun 26, 2020
3bf72ff
Merge remote-tracking branch 'origin/master' into read_write_caching
BillyONeal Jun 26, 2020
59a87bf
readwrite
BillyONeal Jun 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions scripts/azure-pipelines/test-modified-ports.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ supplied, binary caching will be used.

[CmdletBinding()]
Param(
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
[string]$Triplet,
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
$ArchivesRoot,
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
$WorkingRoot,
[ValidateNotNullOrEmpty()]
Expand All @@ -53,33 +53,34 @@ $buildtreesRoot = Join-Path $WorkingRoot 'buildtrees'
$installRoot = Join-Path $WorkingRoot 'installed'
$packagesRoot = Join-Path $WorkingRoot 'packages'
$commonArgs = @(
'--binarycaching',
"--x-buildtrees-root=$buildtreesRoot",
"--x-install-root=$installRoot",
"--x-packages-root=$packagesRoot"
)

$binaryCaching = $false
$binaryCachingMode = 'readwrite'
if ([string]::IsNullOrWhiteSpace($BuildReason)) {
Write-Host 'Build reason not specified, defaulting to using binary caching.'
$binaryCaching = $true
} elseif ($BuildReason -eq 'PullRequest') {
Write-Host 'Build reason was Pull Request, using binary caching.'
$binaryCaching = $true
Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.'
}

if ($binaryCaching) {
$commonArgs += @(
'--binarycaching',
"--x-binarysource=clear;files,$ArchivesRoot,upload"
)
elseif ($BuildReason -eq 'PullRequest') {
Write-Host 'Build reason was Pull Request, using binary caching in read write mode.'
}
else {
Write-Host "Build reason was $BuildReason, using binary caching in write only mode."
$binaryCachingMode = 'write'
}

$commonArgs += @("--x-binarysource=clear;files,$ArchivesRoot,$binaryCachingMode")

if ($Triplet -eq 'x64-linux') {
$env:HOME = '/home/agent'
$executableExtension = [string]::Empty
} elseif ($Triplet -eq 'x64-osx') {
}
elseif ($Triplet -eq 'x64-osx') {
$executableExtension = [string]::Empty
} else {
}
else {
$executableExtension = '.exe'
}

Expand Down
28 changes: 22 additions & 6 deletions toolsrc/src/vcpkg-test/binaryconfigparser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,19 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]")
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload", {});
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",read", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload,extra", {});
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",write", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",readwrite", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",readwrite,extra", {});
REQUIRE(!parsed.has_value());
}
{
Expand Down Expand Up @@ -140,11 +148,19 @@ TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]")
REQUIRE(!parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("default,upload", {});
auto parsed = create_binary_provider_from_configs_pure("default,read", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("default,readwrite", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("default,write", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("default,upload,extra", {});
auto parsed = create_binary_provider_from_configs_pure("default,read,extra", {});
REQUIRE(!parsed.has_value());
}
}
Expand Down Expand Up @@ -180,11 +196,11 @@ TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]")
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("clear;default,upload", {});
auto parsed = create_binary_provider_from_configs_pure("clear;default,readwrite", {});
REQUIRE(parsed.has_value());
}
{
auto parsed = create_binary_provider_from_configs_pure("clear;default,upload;clear;clear", {});
auto parsed = create_binary_provider_from_configs_pure("clear;default,readwrite;clear;clear", {});
REQUIRE(parsed.has_value());
}
{
Expand Down
99 changes: 66 additions & 33 deletions toolsrc/src/vcpkg/binarycaching.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ namespace
}
}
void push_failure(const VcpkgPaths&, const std::string&, const PackageSpec&) override {}
RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&, bool) override
RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
{
return RestoreResult::missing;
}
Expand Down Expand Up @@ -572,12 +572,11 @@ namespace
}
}
RestoreResult precheck(const VcpkgPaths& paths,
const Dependencies::InstallPlanAction& action,
bool purge_tombstones) override
const Dependencies::InstallPlanAction& action) override
{
for (auto&& provider : m_providers)
{
auto result = provider->precheck(paths, action, purge_tombstones);
auto result = provider->precheck(paths, action);
switch (result)
{
case RestoreResult::build_failed:
Expand All @@ -600,8 +599,8 @@ namespace
{
return RestoreResult::missing;
}
void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override {}
void push_failure(const VcpkgPaths&, const std::string&, const PackageSpec&) override {}
void push_success(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override { }
void push_failure(const VcpkgPaths&, const std::string&, const PackageSpec&) override { }
RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
{
return RestoreResult::missing;
Expand Down Expand Up @@ -815,33 +814,48 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
else if (segments[0].second == "files")
{
if (segments.size() < 2)
{
return add_error("expected arguments: binary config 'files' requires at least a path argument",
segments[0].first);
}

auto p = fs::u8path(segments[1].second);
if (!p.is_absolute())
{
return add_error("expected arguments: path arguments for binary config strings must be absolute",
segments[1].first);
}

if (segments.size() > 3)
std::string mode;
switch (segments.size())
{
return add_error("unexpected arguments: binary config 'files' does not take more than 2 arguments",
segments[3].first);
case 2: mode = "read"; break;
case 3: mode = segments[2].second; break;
default:
return add_error("unexpected arguments: binary config 'files' requires 1 or 2 arguments",
segments[3].first);
}
else if (segments.size() == 3)

if (mode == "read")
{
if (segments[2].second != "upload")
{
return add_error("unexpected arguments: binary config 'files' can only accept 'upload' as "
"a second argument",
segments[2].first);
}
else
{
state->archives_to_write.push_back(p);
}
state->archives_to_read.push_back(std::move(p));
}
else if (mode == "write")
{
state->archives_to_write.push_back(std::move(p));
}
else if (mode == "readwrite")
{
state->archives_to_read.push_back(p);
state->archives_to_write.push_back(std::move(p));
}
else
{
Checks::check_exit(VCPKG_LINE_INFO, segments.size() > 2);
return add_error("unexpected arguments: binary config 'files' can only accept"
" 'read', readwrite', or 'write' as a second argument",
segments[2].first);
}
state->archives_to_read.push_back(std::move(p));
}
else if (segments[0].second == "interactive")
{
Expand Down Expand Up @@ -916,30 +930,49 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
else if (segments[0].second == "default")
{
if (segments.size() > 2)
{
return add_error("unexpected arguments: binary config 'default' does not take more than 1 argument",
segments[0].first);
}

auto&& maybe_home = System::get_platform_cache_home();
if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first);

auto p = *maybe_home.get();
p /= fs::u8path("vcpkg/archives");
if (!p.is_absolute())
{
return add_error("default path was not absolute: " + p.u8string(), segments[0].first);
if (segments.size() == 2)
}

std::string mode;
switch (segments.size())
{
if (segments[1].second != "upload")
{
return add_error(
"unexpected arguments: binary config 'default' can only accept 'upload' as an argument",
segments[1].first);
}
else
{
state->archives_to_write.push_back(p);
}
case 1: mode = "read"; break;
case 2: mode = segments[1].second; break;
default: Checks::unreachable(VCPKG_LINE_INFO);
}

if (mode == "read")
{
state->archives_to_read.push_back(std::move(p));
}
else if (mode == "write")
{
state->archives_to_write.push_back(std::move(p));
}
else if (mode == "readwrite")
{
state->archives_to_read.push_back(p);
state->archives_to_write.push_back(std::move(p));
}
else
{
Checks::check_exit(VCPKG_LINE_INFO, segments.size() > 1);
return add_error("unexpected arguments: binary config 'default' can only accept"
" 'read', readwrite', or 'write' as a first argument",
segments[1].first);
}
state->archives_to_read.push_back(std::move(p));
}
else
{
Expand Down