Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deduplicate Platform and PlatformConstraint #11157

Merged
merged 3 commits into from
Nov 12, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions src/python/pants/backend/python/util_rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
MergeDigests,
PathGlobs,
)
from pants.engine.platform import Platform, PlatformConstraint
from pants.engine.platform import Platform
from pants.engine.process import (
MultiPlatformProcess,
Process,
Expand Down Expand Up @@ -636,9 +636,7 @@ async def create_pex(
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(
ProcessResult, MultiPlatformProcess({PlatformConstraint(platform.value): process})
)
result = await Get(ProcessResult, MultiPlatformProcess({platform: process}))

if pex_runtime_env.verbosity > 0:
log_output = result.stderr.decode()
Expand Down
6 changes: 3 additions & 3 deletions src/python/pants/engine/internals/engine_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
)
from pants.engine.internals.scheduler import ExecutionError
from pants.engine.internals.scheduler_test_base import SchedulerTestBase
from pants.engine.platform import PlatformConstraint, create_platform_rules
from pants.engine.platform import rules as platform_rules
from pants.engine.process import MultiPlatformProcess, Process, ProcessResult
from pants.engine.process import rules as process_rules
from pants.engine.rules import Get, MultiGet, rule
Expand Down Expand Up @@ -657,10 +657,10 @@ async def a_rule() -> TrueResult:
["/bin/sh", "-c", "true"],
description="always true",
)
_ = await Get(ProcessResult, MultiPlatformProcess({PlatformConstraint.none: proc}))
_ = await Get(ProcessResult, MultiPlatformProcess({None: proc}))
return TrueResult()

rules = [a_rule, QueryRule(TrueResult, tuple()), *process_rules(), *create_platform_rules()]
rules = [a_rule, QueryRule(TrueResult, tuple()), *process_rules(), *platform_rules()]

scheduler = self.mk_scheduler(
rules, include_trace_on_error=False, should_report_workunits=True
Expand Down
14 changes: 2 additions & 12 deletions src/python/pants/engine/platform.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,12 @@ def current(cls) -> "Platform":
return Platform(get_normalized_os_name())


class PlatformConstraint(Enum):
darwin = "darwin"
linux = "linux"
none = "none"

@memoized_classproperty
def local_platform(cls) -> "PlatformConstraint":
return PlatformConstraint(Platform.current.value)


# TODO We will want to allow users to specify the execution platform for rules,
# which means replacing this singleton rule with a RootRule populated by an option.
@rule
def materialize_platform() -> Platform:
def current_platform() -> Platform:
return Platform.current


def create_platform_rules() -> Iterable[Rule]:
def rules() -> Iterable[Rule]:
return collect_rules()
16 changes: 9 additions & 7 deletions src/python/pants/engine/process.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent
from pants.engine.internals.selectors import MultiGet
from pants.engine.internals.uuid import UUIDRequest, UUIDScope
from pants.engine.platform import Platform, PlatformConstraint
from pants.engine.platform import Platform
from pants.engine.rules import Get, collect_rules, rule, side_effecting
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
Expand Down Expand Up @@ -117,20 +117,22 @@ def __init__(
@frozen_after_init
@dataclass(unsafe_hash=True)
class MultiPlatformProcess:
platform_constraints: Tuple[str, ...]
platform_constraints: Tuple[Optional[str], ...]
processes: Tuple[Process, ...]

def __init__(self, request_dict: Dict[PlatformConstraint, Process]) -> None:
def __init__(self, request_dict: Dict[Optional[Platform], Process]) -> None:
if len(request_dict) == 0:
raise ValueError("At least one platform constrained Process must be passed.")
validated_constraints = tuple(constraint.value for constraint in request_dict)
raise ValueError("At least one platform-constrained Process must be passed.")
serialized_constraints = tuple(
constraint.value if constraint else None for constraint in request_dict
)
if len([req.description for req in request_dict.values()]) != 1:
raise ValueError(
f"The `description` of all processes in a {MultiPlatformProcess.__name__} must "
f"be identical, but got: {list(request_dict.values())}."
)

self.platform_constraints = validated_constraints
self.platform_constraints = serialized_constraints
self.processes = tuple(request_dict.values())

@property
Expand Down Expand Up @@ -211,7 +213,7 @@ def get_multi_platform_request_description(req: MultiPlatformProcess) -> Product
@rule
def upcast_process(req: Process) -> MultiPlatformProcess:
"""This rule allows an Process to be run as a platform compatible MultiPlatformProcess."""
return MultiPlatformProcess({PlatformConstraint.none: req})
return MultiPlatformProcess({None: req})


@rule
Expand Down
5 changes: 2 additions & 3 deletions src/python/pants/init/engine_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pants.base.exiter import PANTS_SUCCEEDED_EXIT_CODE
from pants.base.specs import Specs
from pants.build_graph.build_configuration import BuildConfiguration
from pants.engine import desktop, fs, process
from pants.engine import desktop, fs, platform, process
from pants.engine.console import Console
from pants.engine.fs import PathGlobs, Snapshot, Workspace
from pants.engine.goal import Goal
Expand All @@ -21,7 +21,6 @@
from pants.engine.internals.scheduler import Scheduler, SchedulerSession
from pants.engine.internals.selectors import Params
from pants.engine.internals.session import SessionValues
from pants.engine.platform import create_platform_rules
from pants.engine.process import InteractiveRunner
from pants.engine.rules import QueryRule, collect_rules, rule
from pants.engine.target import RegisteredTargetTypes
Expand Down Expand Up @@ -240,7 +239,7 @@ def build_root_singleton() -> BuildRoot:
*uuid.rules(),
*options_parsing.rules(),
*process.rules(),
*create_platform_rules(),
*platform.rules(),
*changed_rules(),
*specs_calculator.rules(),
*rules,
Expand Down
69 changes: 16 additions & 53 deletions src/rust/engine/process_execution/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ impl Platform {
pub fn current() -> Result<Platform, String> {
let platform_info =
uname::uname().map_err(|_| "Failed to get local platform info!".to_string())?;

match platform_info {
uname::Info { ref sysname, .. } if sysname.to_lowercase() == "darwin" => Ok(Platform::Darwin),
uname::Info { ref sysname, .. } if sysname.to_lowercase() == "linux" => Ok(Platform::Linux),
Expand All @@ -93,66 +92,29 @@ impl Platform {
}
}

#[derive(PartialOrd, Ord, Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub enum PlatformConstraint {
Darwin,
Linux,
None,
}

impl PlatformConstraint {
pub fn current_platform_constraint() -> Result<PlatformConstraint, String> {
Platform::current().map(|p: Platform| p.into())
}
}

impl From<Platform> for PlatformConstraint {
fn from(platform: Platform) -> PlatformConstraint {
impl From<Platform> for String {
fn from(platform: Platform) -> String {
match platform {
Platform::Linux => PlatformConstraint::Linux,
Platform::Darwin => PlatformConstraint::Darwin,
Platform::Linux => "linux".to_string(),
Platform::Darwin => "darwin".to_string(),
}
}
}

impl TryFrom<&String> for PlatformConstraint {
impl TryFrom<String> for Platform {
type Error = String;
///
/// This is a helper method to convert values from the python/engine/platform.py::PlatformConstraint enum,
/// which have been serialized, into the rust PlatformConstraint enum.
///
fn try_from(variant_candidate: &String) -> Result<Self, Self::Error> {
fn try_from(variant_candidate: String) -> Result<Self, Self::Error> {
match variant_candidate.as_ref() {
"darwin" => Ok(PlatformConstraint::Darwin),
"linux" => Ok(PlatformConstraint::Linux),
"none" => Ok(PlatformConstraint::None),
"darwin" => Ok(Platform::Darwin),
"linux" => Ok(Platform::Linux),
other => Err(format!(
"Unknown, platform {:?} encountered in parsing",
"Unknown platform {:?} encountered in parsing",
other
)),
}
}
}

impl From<Platform> for String {
fn from(platform: Platform) -> String {
match platform {
Platform::Linux => "linux".to_string(),
Platform::Darwin => "darwin".to_string(),
}
}
}

impl From<PlatformConstraint> for String {
fn from(platform: PlatformConstraint) -> String {
match platform {
PlatformConstraint::Linux => "linux".to_string(),
PlatformConstraint::Darwin => "darwin".to_string(),
PlatformConstraint::None => "none".to_string(),
}
}
}

///
/// A process to be executed.
///
Expand Down Expand Up @@ -222,7 +184,8 @@ pub struct Process {
/// see https://github.com/pantsbuild/pants/issues/6416.
///
pub jdk_home: Option<PathBuf>,
pub target_platform: PlatformConstraint,

pub platform_constraint: Option<Platform>,

pub is_nailgunnable: bool,

Expand Down Expand Up @@ -253,7 +216,7 @@ impl Process {
level: log::Level::Info,
append_only_caches: BTreeMap::new(),
jdk_home: None,
target_platform: PlatformConstraint::None,
platform_constraint: None,
is_nailgunnable: false,
execution_slot_variable: None,
cache_failures: false,
Expand Down Expand Up @@ -300,7 +263,7 @@ impl TryFrom<MultiPlatformProcess> for Process {
type Error = String;

fn try_from(req: MultiPlatformProcess) -> Result<Self, Self::Error> {
match req.0.get(&PlatformConstraint::None) {
match req.0.get(&None) {
Some(crossplatform_req) => Ok(crossplatform_req.clone()),
None => Err(String::from(
"Cannot coerce to a simple Process, no cross platform request exists.",
Expand All @@ -313,7 +276,7 @@ impl TryFrom<MultiPlatformProcess> for Process {
/// A container of platform constrained processes.
///
#[derive(Derivative, Clone, Debug, Eq, PartialEq, Hash)]
pub struct MultiPlatformProcess(pub BTreeMap<PlatformConstraint, Process>);
pub struct MultiPlatformProcess(pub BTreeMap<Option<Platform>, Process>);

impl MultiPlatformProcess {
pub fn user_facing_name(&self) -> String {
Expand All @@ -340,8 +303,8 @@ impl MultiPlatformProcess {
}

impl From<Process> for MultiPlatformProcess {
fn from(req: Process) -> Self {
MultiPlatformProcess(vec![(PlatformConstraint::None, req)].into_iter().collect())
fn from(proc: Process) -> Self {
MultiPlatformProcess(vec![(None, proc)].into_iter().collect())
}
}

Expand Down
5 changes: 2 additions & 3 deletions src/rust/engine/process_execution/src/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ use tokio::time::{timeout, Duration};
use tokio_util::codec::{BytesCodec, FramedRead};

use crate::{
Context, FallibleProcessResultWithPlatform, MultiPlatformProcess, NamedCaches, Platform,
PlatformConstraint, Process,
Context, FallibleProcessResultWithPlatform, MultiPlatformProcess, NamedCaches, Platform, Process,
};

use bytes::{Bytes, BytesMut};
Expand Down Expand Up @@ -220,7 +219,7 @@ impl ChildResults {
#[async_trait]
impl super::CommandRunner for CommandRunner {
fn extract_compatible_request(&self, req: &MultiPlatformProcess) -> Option<Process> {
for compatible_constraint in vec![PlatformConstraint::None, self.platform.into()].iter() {
for compatible_constraint in vec![None, self.platform.into()].iter() {
if let Some(compatible_req) = req.0.get(compatible_constraint) {
return Some(compatible_req.clone());
}
Expand Down
16 changes: 10 additions & 6 deletions src/rust/engine/process_execution/src/nailgun/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ use tokio::net::TcpStream;
use crate::local::CapturedWorkdir;
use crate::nailgun::nailgun_pool::NailgunProcessName;
use crate::{
Context, FallibleProcessResultWithPlatform, MultiPlatformProcess, NamedCaches, Platform,
PlatformConstraint, Process, ProcessMetadata,
Context, FallibleProcessResultWithPlatform, MultiPlatformProcess, NamedCaches, Platform, Process,
ProcessMetadata,
};

#[cfg(test)]
Expand Down Expand Up @@ -44,7 +44,7 @@ fn construct_nailgun_server_request(
nailgun_name: &str,
args_for_the_jvm: Vec<String>,
jdk: PathBuf,
platform_constraint: PlatformConstraint,
platform_constraint: Option<Platform>,
) -> Process {
let mut full_args = args_for_the_jvm;
full_args.push(NAILGUN_MAIN_CLASS.to_string());
Expand All @@ -62,7 +62,7 @@ fn construct_nailgun_server_request(
level: log::Level::Info,
append_only_caches: BTreeMap::new(),
jdk_home: Some(jdk),
target_platform: platform_constraint,
platform_constraint,
is_nailgunnable: true,
execution_slot_variable: None,
cache_failures: false,
Expand Down Expand Up @@ -213,8 +213,12 @@ impl CapturedWorkdir for CommandRunner {
.jdk_home
.clone()
.ok_or("JDK home must be specified for all nailgunnable requests.")?;
let nailgun_req =
construct_nailgun_server_request(&nailgun_name, nailgun_args, jdk_home, req.target_platform);
let nailgun_req = construct_nailgun_server_request(
&nailgun_name,
nailgun_args,
jdk_home,
req.platform_constraint,
);
trace!("Extracted nailgun request:\n {:#?}", &nailgun_req);

let nailgun_req_digest = crate::digest(
Expand Down
6 changes: 3 additions & 3 deletions src/rust/engine/process_execution/src/nailgun/tests.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::nailgun::{CommandRunner, ARGS_TO_START_NAILGUN, NAILGUN_MAIN_CLASS};
use crate::{NamedCaches, PlatformConstraint, Process, ProcessMetadata};
use crate::{NamedCaches, Platform, Process, ProcessMetadata};
use futures::compat::Future01CompatExt;
use hashing::EMPTY_DIGEST;
use std::fs::read_link;
Expand Down Expand Up @@ -41,7 +41,7 @@ fn mock_nailgunnable_request(jdk_home: Option<PathBuf>) -> Process {
let mut process = Process::new(vec![]);
process.jdk_home = jdk_home;
process.is_nailgunnable = true;
process.target_platform = PlatformConstraint::Darwin;
process.platform_constraint = Some(Platform::Darwin);
process
}

Expand Down Expand Up @@ -85,7 +85,7 @@ async fn creating_nailgun_server_request_updates_the_cli() {
&NAILGUN_MAIN_CLASS.to_string(),
Vec::new(),
PathBuf::from(""),
PlatformConstraint::None,
None,
);
assert_eq!(req.argv[0], NAILGUN_MAIN_CLASS);
assert_eq!(req.argv[1..], ARGS_TO_START_NAILGUN);
Expand Down
9 changes: 6 additions & 3 deletions src/rust/engine/process_execution/src/remote.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use workunit_store::{with_workunit, Metric, SpanId, WorkunitMetadata, WorkunitSt

use crate::{
Context, ExecutionStats, FallibleProcessResultWithPlatform, MultiPlatformProcess, Platform,
PlatformConstraint, Process, ProcessMetadata,
Process, ProcessMetadata,
};
use bazel_protos::remote_execution_grpc::ActionCacheClient;

Expand Down Expand Up @@ -790,7 +790,7 @@ impl crate::CommandRunner for CommandRunner {

// TODO: This is a copy of the same method on crate::remote::CommandRunner.
fn extract_compatible_request(&self, req: &MultiPlatformProcess) -> Option<Process> {
for compatible_constraint in vec![PlatformConstraint::None, self.platform.into()].iter() {
for compatible_constraint in vec![None, self.platform.into()].iter() {
if let Some(compatible_req) = req.0.get(compatible_constraint) {
return Some(compatible_req.clone());
}
Expand Down Expand Up @@ -873,7 +873,10 @@ pub fn make_execute_request(
{
let mut env = bazel_protos::remote_execution::Command_EnvironmentVariable::new();
env.set_name(CACHE_KEY_TARGET_PLATFORM_ENV_VAR_NAME.to_string());
env.set_value(req.target_platform.into());
env.set_value(match req.platform_constraint {
Some(plat) => plat.into(),
None => "none".to_string(),
});
command.mut_environment_variables().push(env);
}

Expand Down
Loading