Skip to content

Commit

Permalink
Add helper for abstracting over Python provider representation and de…
Browse files Browse the repository at this point in the history
…faults

This creates PyProviderUtils, a collection of helpers that take in a TransitiveInfoCollection and return fields obtained from the Python provider. Currently this just dispatches to PyStructUtils to retrieve from the legacy "py" struct provider. In a follow-up CL, we'll add a modern PyInfo type, and PyProviderUtils will access either one as appropriate.

PyProviderUtils also takes over the default logic from PyCommon, for when no Python provider information is available.

Work toward bazelbuild#7010.

RELNOTES: None
PiperOrigin-RevId: 230791581
  • Loading branch information
brandjon authored and weixiao-huang committed Jan 31, 2019
1 parent c119d2a commit f4492e4
Show file tree
Hide file tree
Showing 8 changed files with 388 additions and 213 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import com.google.devtools.build.lib.packages.BuildType;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException;
import com.google.devtools.build.lib.packages.StructImpl;
import com.google.devtools.build.lib.rules.cpp.CcInfo;
import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
Expand Down Expand Up @@ -230,20 +229,11 @@ private static NestedSet<Artifact> initTransitivePythonSources(RuleContext ruleC
private static void collectTransitivePythonSourcesFromDeps(
RuleContext ruleContext, NestedSetBuilder<Artifact> builder) {
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
StructImpl info = PyStructUtils.getProvider(dep);
NestedSet<Artifact> sources = PyStructUtils.getTransitiveSources(info);
builder.addTransitive(sources);
} catch (EvalException e) {
// Either the provider type or field type is bad.
ruleContext.ruleError(e.getMessage());
}
} else {
// TODO(bazel-team): We also collect .py source files from deps (e.g. for proto_library
// rules). We should have rules implement a "PythonSourcesProvider" instead.
FileProvider provider = dep.getProvider(FileProvider.class);
builder.addAll(FileType.filter(provider.getFilesToBuild(), PyRuleClasses.PYTHON_SOURCE));
try {
builder.addTransitive(PyProviderUtils.getTransitiveSources(dep));
} catch (EvalException e) {
// Either the provider type or field type is bad.
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
}
Expand All @@ -268,48 +258,36 @@ private static boolean initUsesSharedLibraries(RuleContext ruleContext) {
} else {
targets = ruleContext.getPrerequisites("deps", Mode.TARGET);
}
try {
for (TransitiveInfoCollection target : targets) {
if (checkForSharedLibraries(target)) {
for (TransitiveInfoCollection target : targets) {
try {
if (PyProviderUtils.getUsesSharedLibraries(target)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", target.getLabel(), e.getMessage()));
}
return false;
} catch (EvalException e) {
ruleContext.ruleError(e.getMessage());
return false;
}
}

private static boolean checkForSharedLibraries(TransitiveInfoCollection target)
throws EvalException {
if (PyStructUtils.hasProvider(target)) {
return PyStructUtils.getUsesSharedLibraries(PyStructUtils.getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return FileType.contains(files, CppFileTypes.SHARED_LIBRARY);
}
return false;
}

private static NestedSet<String> initImports(RuleContext ruleContext, PythonSemantics semantics) {
NestedSetBuilder<String> builder = NestedSetBuilder.compileOrder();
builder.addAll(semantics.getImports(ruleContext));

for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (dep.getProvider(PythonImportsProvider.class) != null) {
PythonImportsProvider provider = dep.getProvider(PythonImportsProvider.class);
NestedSet<String> imports = provider.getTransitivePythonImports();
try {
NestedSet<String> imports = PyProviderUtils.getImports(dep);
if (!builder.getOrder().isCompatible(imports.getOrder())) {
// TODO(brandjon): Add test case for this error, once we replace PythonImportsProvider
// with the normal Python provider and once we clean up our provider merge logic.
// TODO(brandjon): We should make order an invariant of the Python provider. Then once we
/// remove PythonImportsProvider we can move this check into PyProvider/PyStructUtils.
ruleContext.ruleError(
getOrderErrorMessage(PyStructUtils.IMPORTS, builder.getOrder(), imports.getOrder()));
} else {
builder.addTransitive(imports);
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}

return builder.build();
}

Expand All @@ -324,14 +302,12 @@ private static boolean initHasPy2OnlySources(
return true;
}
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
if (PyStructUtils.getHasPy2OnlySources(PyStructUtils.getProvider(dep))) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
try {
if (PyProviderUtils.getHasPy2OnlySources(dep)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
return false;
Expand All @@ -347,14 +323,12 @@ private static boolean initHasPy3OnlySources(
return true;
}
for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) {
if (PyStructUtils.hasProvider(dep)) {
try {
if (PyStructUtils.getHasPy3OnlySources(PyStructUtils.getProvider(dep))) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
try {
if (PyProviderUtils.getHasPy3OnlySources(dep)) {
return true;
}
} catch (EvalException e) {
ruleContext.ruleError(String.format("In dep '%s': %s", dep.getLabel(), e.getMessage()));
}
}
return false;
Expand Down Expand Up @@ -617,10 +591,9 @@ public void addCommonTransitiveInfoProviders(
*/
public List<Artifact> validateSrcs() {
List<Artifact> sourceFiles = new ArrayList<>();
// TODO(bazel-team): Need to get the transitive deps closure, not just the
// sources of the rule.
for (TransitiveInfoCollection src : ruleContext
.getPrerequisitesIf("srcs", Mode.TARGET, FileProvider.class)) {
// TODO(bazel-team): Need to get the transitive deps closure, not just the sources of the rule.
for (TransitiveInfoCollection src :
ruleContext.getPrerequisitesIf("srcs", Mode.TARGET, FileProvider.class)) {
// Make sure that none of the sources contain hyphens.
if (Util.containsHyphen(src.getLabel().getPackageFragment())) {
ruleContext.attributeError("srcs",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.rules.python;

import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.FileProvider;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.packages.StructImpl;
import com.google.devtools.build.lib.rules.cpp.CppFileTypes;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.SkylarkType;
import com.google.devtools.build.lib.util.FileType;

/**
* Static helper class for creating and accessing Python provider information.
*
* <p>This class exposes a unified view over both the legacy and modern Python providers.
*/
public class PyProviderUtils {

// Disable construction.
private PyProviderUtils() {}

/** Returns whether a given target has the py provider. */
public static boolean hasProvider(TransitiveInfoCollection target) {
return target.get(PyStructUtils.PROVIDER_NAME) != null;
}

/**
* Returns the struct representing the py provider, from the given target info.
*
* @throws EvalException if the provider does not exist or has the wrong type.
*/
public static StructImpl getProvider(TransitiveInfoCollection target) throws EvalException {
Object info = target.get(PyStructUtils.PROVIDER_NAME);
if (info == null) {
throw new EvalException(/*location=*/ null, "Target does not have 'py' provider");
}
return SkylarkType.cast(
info,
StructImpl.class,
null,
"'%s' provider should be a struct",
PyStructUtils.PROVIDER_NAME);
}

/**
* Returns the transitive sources of a given target.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we fall
* back on collecting .py source files from the target's filesToBuild.
*
* @throws EvalException if the legacy struct provider is present but malformed
*/
// TODO(bazel-team): Eliminate the fallback behavior by returning an appropriate py provider from
// the relevant rules.
public static NestedSet<Artifact> getTransitiveSources(TransitiveInfoCollection target)
throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getTransitiveSources(getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return NestedSetBuilder.<Artifact>compileOrder()
.addAll(FileType.filter(files, PyRuleClasses.PYTHON_SOURCE))
.build();
}
}

/**
* Returns whether a target uses shared libraries.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we fall
* back on checking whether the target's filesToBuild contains a shared library file type (e.g., a
* .so file).
*
* @throws EvalException if the legacy struct provider is present but malformed
*/
public static boolean getUsesSharedLibraries(TransitiveInfoCollection target)
throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getUsesSharedLibraries(getProvider(target));
} else {
NestedSet<Artifact> files = target.getProvider(FileProvider.class).getFilesToBuild();
return FileType.contains(files, CppFileTypes.SHARED_LIBRARY);
}
}

/**
* Returns the transitive import paths of a target.
*
* <p>Imports are not currently propagated correctly (#7054). Currently the behavior is to return
* the imports contained in the target's {@link PythonImportsProvider}, ignoring the py provider,
* or no imports if there is no {@code PythonImportsProvider}. When #7054 is fixed, we'll instead
* return the imports specified by the py provider, or those from {@code PythonImportsProvider} if
* the py provider is not present, with an eventual goal of removing {@code
* PythonImportsProvider}.
*/
// TODO(#7054) Implement the above change.
public static NestedSet<String> getImports(TransitiveInfoCollection target) throws EvalException {
PythonImportsProvider importsProvider = target.getProvider(PythonImportsProvider.class);
if (importsProvider != null) {
return importsProvider.getTransitivePythonImports();
} else {
return NestedSetBuilder.emptySet(Order.COMPILE_ORDER);
}
}

/**
* Returns whether the target has transitive sources requiring Python 2.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we default
* to false.
*/
public static boolean getHasPy2OnlySources(TransitiveInfoCollection target) throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getHasPy2OnlySources(getProvider(target));
} else {
return false;
}
}

/**
* Returns whether the target has transitive sources requiring Python 3.
*
* <p>If the target has a py provider, the value from that provider is used. Otherwise, we default
* to false.
*/
public static boolean getHasPy3OnlySources(TransitiveInfoCollection target) throws EvalException {
if (hasProvider(target)) {
return PyStructUtils.getHasPy3OnlySources(getProvider(target));
} else {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
Expand All @@ -28,7 +27,7 @@
import com.google.devtools.build.lib.syntax.SkylarkNestedSet;
import com.google.devtools.build.lib.syntax.SkylarkType;

/** Static helper class for creating and accessing instances of the "py" legacy struct provider. */
/** Static helper class for creating and accessing instances of the legacy "py" struct provider. */
// TODO(#7010): Replace this with a real provider.
public class PyStructUtils {

Expand All @@ -54,9 +53,7 @@ private PyStructUtils() {}
* target).
*/
// TODO(brandjon): Make this a pre-order depset, since higher-level targets should get precedence
// on PYTHONPATH.
// TODO(brandjon): Add assertions that this depset and transitive_sources have an order compatible
// with the one expected by the rules.
// on PYTHONPATH. Add assertions on its order compatibility.
public static final String IMPORTS = "imports";

/**
Expand Down Expand Up @@ -85,25 +82,6 @@ private PyStructUtils() {}
DEFAULTS = builder.build();
}

/** Returns whether a given dependency has the py provider. */
public static boolean hasProvider(TransitiveInfoCollection dep) {
return dep.get(PROVIDER_NAME) != null;
}

/**
* Returns the struct representing the py provider, from the given target info.
*
* @throws EvalException if the provider does not exist or has the wrong type.
*/
public static StructImpl getProvider(TransitiveInfoCollection dep) throws EvalException {
Object info = dep.get(PROVIDER_NAME);
if (info == null) {
throw new EvalException(/*location=*/ null, "Target does not have 'py' provider");
}
return SkylarkType.cast(
info, StructImpl.class, null, "'%s' provider should be a struct", PROVIDER_NAME);
}

private static Object getValue(StructImpl info, String fieldName) throws EvalException {
Object fieldValue = info.getValue(fieldName);
if (fieldValue == null) {
Expand Down Expand Up @@ -220,7 +198,7 @@ public static Builder builder() {
return new Builder();
}

/** Builder for a py provider struct. */
/** Builder for a legacy py provider struct. */
public static class Builder {
SkylarkNestedSet transitiveSources = null;
Boolean usesSharedLibraries = null;
Expand Down
Loading

0 comments on commit f4492e4

Please sign in to comment.