Skip to content

Commit

Permalink
[vcpkg] Enable binary caching by default (microsoft#12370)
Browse files Browse the repository at this point in the history 
* [vcpkg] Enable binary caching by default. Support `x-` migration.

Fix passing multiple copies of single parameter arguments.

* [vcpkg] Handle x- prefixes for general arguments

* [vcpkg] Fix microsoft#12285 and improve documentation of default binary cache path

* [vcpkg] Revert x- prefix homogenization for per-command arguments

* [vcpkg] Only use accelerated compiler detection for Windows Desktop + Ninja. Improve breadcrumbs for users encountering issues.

* [vcpkg] Fix compiler tracking not pre-downloading Ninja. Fix compiler tracking not looking in -err.log.

* [vcpkg] Update toolsrc/src/vcpkg/binarycaching.cpp

Co-authored-by: Billy O'Neal <[email protected]>

* [vcpkg] Format

Co-authored-by: Robert Schumacher <[email protected]>
Co-authored-by: Billy O'Neal <[email protected]>
  • Loading branch information
@ras0219 @ras0219-msft @BillyONeal
3 people authored Jul 13, 2020
1 parent 64429fa commit 488e154
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 33 deletions.
6 changes: 3 additions & 3 deletions include/vcpkg/vcpkgcmdarguments.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ namespace vcpkg
constexpr static StringLiteral OVERLAY_TRIPLETS_ARG = "overlay-triplets";
std::vector<std::string> overlay_triplets;

constexpr static StringLiteral BINARY_SOURCES_ARG = "x-binarysource";
constexpr static StringLiteral BINARY_SOURCES_ARG = "binarysource";
std::vector<std::string> binary_sources;

constexpr static StringLiteral DEBUG_SWITCH = "debug";
Expand Down Expand Up @@ -166,8 +166,8 @@ namespace vcpkg
constexpr static StringLiteral MANIFEST_MODE_FEATURE = "manifests";
Optional<bool> manifest_mode = nullopt;

bool binary_caching_enabled() const { return binary_caching.value_or(false); }
bool compiler_tracking_enabled() const { return compiler_tracking.value_or(false); }
bool binary_caching_enabled() const { return binary_caching.value_or(true); }
bool compiler_tracking_enabled() const { return compiler_tracking.value_or(true); }

std::string command;
std::vector<std::string> command_arguments;
Expand Down
15 changes: 14 additions & 1 deletion src/vcpkg/base/system.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,20 @@ namespace vcpkg
static ExpectedS<fs::path> s_home = []() -> ExpectedS<fs::path> {
auto maybe_home = System::get_environment_variable("LOCALAPPDATA");
if (!maybe_home.has_value() || maybe_home.get()->empty())
return {"unable to read %LOCALAPPDATA%", ExpectedRightTag{}};
{
// Consult %APPDATA% as a workaround for Service accounts
// Microsoft/vcpkg#12285
maybe_home = System::get_environment_variable("APPDATA");
if (!maybe_home.has_value() || maybe_home.get()->empty())
{
return {"unable to read %LOCALAPPDATA% or %APPDATA%", ExpectedRightTag{}};
}

auto p = fs::u8path(*maybe_home.get()).parent_path();
p /= "Local";
if (!p.is_absolute()) return {"%APPDATA% was not an absolute path", ExpectedRightTag{}};
return {std::move(p), ExpectedLeftTag{}};
}

auto p = fs::u8path(*maybe_home.get());
if (!p.is_absolute()) return {"%LOCALAPPDATA% was not an absolute path", ExpectedRightTag{}};
Expand Down
66 changes: 41 additions & 25 deletions src/vcpkg/binarycaching.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,22 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
}
namespace
{
const ExpectedS<fs::path>& default_cache_path()
{
static auto cachepath = System::get_platform_cache_home().then([](fs::path p) -> ExpectedS<fs::path> {
p /= fs::u8path("vcpkg/archives");
if (p.is_absolute())
{
return {std::move(p), expected_left_tag};
}
else
{
return {"default path was not absolute: " + p.u8string(), expected_right_tag};
}
});
return cachepath;
}

struct State
{
bool m_cleared = false;
Expand Down Expand Up @@ -856,17 +872,11 @@ namespace
segments[0].first);
}

auto&& maybe_home = System::get_platform_cache_home();
const auto& maybe_home = default_cache_path();
if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first);

auto p = *maybe_home.get();
p /= fs::u8path("vcpkg/archives");
if (!p.is_absolute())
{
return add_error("default path was not absolute: " + p.u8string(), segments[0].first);
}

handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 1);
handle_readwrite(
state->archives_to_read, state->archives_to_write, fs::path(*maybe_home.get()), segments, 1);
}
else
{
Expand Down Expand Up @@ -986,38 +996,44 @@ std::string vcpkg::generate_nuspec(const VcpkgPaths& paths,

void vcpkg::help_topic_binary_caching(const VcpkgPaths&)
{
System::print2(
System::Color::warning,
"** The following help documentation covers an experimental feature that will change at any time **\n\n");

HelpTableFormatter tbl;
tbl.text(
"Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network."
" This functionality is currently disabled by default and must be enabled by either passing `--binarycaching` "
"to every vcpkg command line or setting the environment variable `VCPKG_FEATURE_FLAGS` to `binarycaching`.");
tbl.text("Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network."
" This functionality is currently enabled by default and can be disabled by either passing "
"`--no-binarycaching` to every vcpkg command line or setting the environment variable "
"`VCPKG_FEATURE_FLAGS` to `-binarycaching`.");
tbl.blank();
tbl.blank();
tbl.text(
"Once caching is enabled, it can be further configured by either passing `--x-binarysource=<source>` options "
"Once caching is enabled, it can be further configured by either passing `--binarysource=<source>` options "
"to every command line or setting the `VCPKG_BINARY_SOURCES` environment variable to a set of sources (Ex: "
"\"<source>;<source>;...\"). Command line sources are interpreted after environment sources.");
tbl.blank();
tbl.blank();
tbl.header("Valid source strings");
tbl.format("clear", "Removes all previous sources");
tbl.format("default[,upload]", "Adds the default file-based source location (~/.vcpkg/archives).");
tbl.format("files,<path>[,upload]", "Adds a custom file-based source location.");
tbl.format("nuget,<uri>[,upload]",
tbl.format("default[,<rw>]", "Adds the default file-based location.");
tbl.format("files,<path>[,<rw>]", "Adds a custom file-based location.");
tbl.format("nuget,<uri>[,<rw>]",
"Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI.");
tbl.format("nugetconfig,<path>[,upload]",
tbl.format("nugetconfig,<path>[,<rw>]",
"Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This "
"config should specify `defaultPushSource` for uploads.");
tbl.format("interactive", "Enables interactive credential management for some source types");
tbl.blank();
tbl.text("The `upload` optional parameter for certain source strings controls whether on-demand builds will be "
"uploaded to that remote.");

tbl.text("The `<rw>` optional parameter for certain strings controls whether they will be consulted for "
"downloading binaries and whether on-demand builds will be uploaded to that remote. It can be specified "
"as 'read', 'write', or 'readwrite'.");
tbl.blank();
System::print2(tbl.m_str);
const auto& maybe_cachepath = default_cache_path();
if (auto p = maybe_cachepath.get())
{
auto p_preferred = *p;
System::print2(
"\nBased on your system settings, the default path to store binaries is\n ",
p_preferred.make_preferred().u8string(),
"\n\nThis consults %LOCALAPPDATA%/%APPDATA% on Windows and $XDG_CACHE_HOME or $HOME on other platforms.");
}
}

std::string vcpkg::generate_nuget_packages_config(const Dependencies::ActionPlan& action)
Expand Down
13 changes: 13 additions & 0 deletions src/vcpkg/build.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,11 @@ namespace vcpkg::Build
System::print2("Detecting compiler hash for triplet ", triplet, "...\n");
auto buildpath = paths.buildtrees / "detect_compiler";

#if !defined(_WIN32)
// TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107.
// bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted.
vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA));
#endif
std::vector<System::CMakeVariable> cmake_args{
{"CURRENT_PORT_DIR", paths.scripts / "detect_compiler"},
{"CURRENT_BUILDTREES_DIR", buildpath},
Expand Down Expand Up @@ -501,6 +506,14 @@ namespace vcpkg::Build
env);
out_file.close();

if (compiler_hash.empty())
{
Debug::print("Compiler information tracking can be disabled by passing --",
VcpkgCmdArguments::FEATURE_FLAGS_ARG,
"=-",
VcpkgCmdArguments::COMPILER_TRACKING_FEATURE,
"\n");
}
Checks::check_exit(VCPKG_LINE_INFO,
!compiler_hash.empty(),
"Error occured while detecting compiler information. Pass `--debug` for more information.");
Expand Down
32 changes: 28 additions & 4 deletions src/vcpkg/vcpkgcmdarguments.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,17 @@ namespace vcpkg
static bool try_parse_argument_as_option(StringView option, StringView argument, T& place, F parser)
{
// remove the first two '-'s
const auto arg = argument.substr(2);
auto arg = argument.substr(2);
if (arg.size() <= option.size() + 1)
{
// it is impossible for this argument to be this option
return false;
}

if (Strings::starts_with(arg, "x-") && !Strings::starts_with(option, "x-"))
{
arg = arg.substr(2);
}
if (Strings::starts_with(arg, option) && arg.byte_at_index(option.size()) == '=')
{
parser(arg.substr(option.size() + 1), option, place);
Expand All @@ -198,21 +202,33 @@ namespace vcpkg
return false;
}

static bool equals_modulo_experimental(StringView arg, StringView option)
{
if (Strings::starts_with(arg, "x-") && !Strings::starts_with(option, "x-"))
{
return arg.substr(2) == option;
}
else
{
return arg == option;
}
}

// returns true if this does parse this argument as this option
// REQUIRES: Strings::starts_with(argument, "--");
template<class T>
static bool try_parse_argument_as_switch(StringView option, StringView argument, T& place)
{
// remove the first two '-'s
const auto arg = argument.substr(2);
auto arg = argument.substr(2);

if (arg == option)
if (equals_modulo_experimental(arg, option))
{
parse_switch(true, option, place);
return true;
}

if (Strings::starts_with(arg, "no-") && arg.substr(3) == option)
if (Strings::starts_with(arg, "no-") && equals_modulo_experimental(arg.substr(3), option))
{
parse_switch(false, option, place);
return true;
Expand Down Expand Up @@ -451,6 +467,12 @@ namespace vcpkg
System::Color::error, "Error: The option '%s' must be passed an argument.\n", option.name);
failed = true;
}
else if (value.size() > 1)
{
System::printf(
System::Color::error, "Error: The option '%s' can only be passed once.\n", option.name);
failed = true;
}
else
{
output.settings.emplace(option.name, value.front());
Expand Down Expand Up @@ -588,6 +610,8 @@ namespace vcpkg
table.format("", "(default: " + format_environment_variable("VCPKG_DEFAULT_TRIPLET") + ')');
table.format(opt(OVERLAY_PORTS_ARG, "=", "<path>"), "Specify directories to be used when searching for ports");
table.format(opt(OVERLAY_TRIPLETS_ARG, "=", "<path>"), "Specify directories containing triplets files");
table.format(opt(BINARY_SOURCES_ARG, "=", "<path>"),
"Add sources for binary caching. See 'vcpkg help binarycaching'");
table.format(opt(DOWNLOADS_ROOT_DIR_ARG, "=", "<path>"), "Specify the downloads root directory");
table.format("", "(default: " + format_environment_variable("VCPKG_DOWNLOADS") + ')');
table.format(opt(VCPKG_ROOT_DIR_ARG, " ", "<path>"), "Specify the vcpkg root directory");
Expand Down

0 comments on commit 488e154

Please sign in to comment.