diff --git a/tensorflow/compiler/tests/plugin.bzl b/tensorflow/compiler/tests/plugin.bzl index fbc8781a3e59fa..46a854d1459b7e 100644 --- a/tensorflow/compiler/tests/plugin.bzl +++ b/tensorflow/compiler/tests/plugin.bzl @@ -18,13 +18,12 @@ # git update-index --assume-unchanged tensorflow/compiler/tests/plugin.bzl plugins = { - #"example": { - # "device":"XLA_MY_DEVICE", - # "types":"DT_FLOAT,DT_HALF,DT_INT32", - # "tags":[], - # "args":["--disabled_manifest=tensorflow/compiler/plugin/example/disabled_manifest.txt"], - # "data":["//tensorflow/compiler/plugin/example:disabled_manifest.txt"], - # "deps":[], - #}, + #"example": { + # "device":"XLA_MY_DEVICE", + # "types":"DT_FLOAT,DT_HALF,DT_INT32", + # "tags":[], + # "args":["--disabled_manifest=tensorflow/compiler/plugin/example/disabled_manifest.txt"], + # "data":["//tensorflow/compiler/plugin/example:disabled_manifest.txt"], + # "deps":[], + #}, } - diff --git a/tensorflow/compiler/xla/service/cpu/build_defs.bzl b/tensorflow/compiler/xla/service/cpu/build_defs.bzl index e78330b21689fd..ffa1cd4ec8e26e 100644 --- a/tensorflow/compiler/xla/service/cpu/build_defs.bzl +++ b/tensorflow/compiler/xla/service/cpu/build_defs.bzl @@ -1,12 +1,11 @@ """build_defs for service/cpu.""" - def runtime_copts(): - """Returns copts used for CPU runtime libraries.""" - return (["-DEIGEN_AVOID_STL_ARRAY"] + select({ - "//tensorflow:android_arm": ["-mfpu=neon"], - "//conditions:default": [] - }) + select({ - "//tensorflow:android": ["-O2"], - "//conditions:default": [] - })) + """Returns copts used for CPU runtime libraries.""" + return (["-DEIGEN_AVOID_STL_ARRAY"] + select({ + "//tensorflow:android_arm": ["-mfpu=neon"], + "//conditions:default": [], + }) + select({ + "//tensorflow:android": ["-O2"], + "//conditions:default": [], + })) diff --git a/tensorflow/compiler/xla/tests/plugin.bzl b/tensorflow/compiler/xla/tests/plugin.bzl index 8a5d91363b619c..107869fe59d43d 100644 --- a/tensorflow/compiler/xla/tests/plugin.bzl +++ b/tensorflow/compiler/xla/tests/plugin.bzl @@ -33,4 +33,3 @@ # } plugins = {} - diff --git a/tensorflow/core/platform/default/platform.bzl b/tensorflow/core/platform/default/platform.bzl index 20ab441bf43e19..76bfaa896efa2f 100644 --- a/tensorflow/core/platform/default/platform.bzl +++ b/tensorflow/core/platform/default/platform.bzl @@ -5,55 +5,52 @@ CUDNN_VERSION = "" PLATFORM = "" def cuda_sdk_version(): - return CUDA_VERSION + return CUDA_VERSION def cudnn_sdk_version(): - return CUDNN_VERSION + return CUDNN_VERSION def cuda_library_path(name, version = cuda_sdk_version()): - if PLATFORM == "Darwin": - if not version: - return "lib/lib{}.dylib".format(name) + if PLATFORM == "Darwin": + if not version: + return "lib/lib{}.dylib".format(name) + else: + return "lib/lib{}.{}.dylib".format(name, version) + elif not version: + return "lib64/lib{}.so".format(name) else: - return "lib/lib{}.{}.dylib".format(name, version) - else: - if not version: - return "lib64/lib{}.so".format(name) - else: - return "lib64/lib{}.so.{}".format(name, version) + return "lib64/lib{}.so.{}".format(name, version) def cuda_static_library_path(name): - if PLATFORM == "Darwin": - return "lib/lib{}_static.a".format(name) - else: - return "lib64/lib{}_static.a".format(name) + if PLATFORM == "Darwin": + return "lib/lib{}_static.a".format(name) + else: + return "lib64/lib{}_static.a".format(name) def cudnn_library_path(version = cudnn_sdk_version()): - if PLATFORM == "Darwin": - if not version: - return "lib/libcudnn.dylib" - else: - return "lib/libcudnn.{}.dylib".format(version) - else: - if not version: - return "lib64/libcudnn.so" + if PLATFORM == "Darwin": + if not version: + return "lib/libcudnn.dylib" + else: + return "lib/libcudnn.{}.dylib".format(version) + elif not version: + return "lib64/libcudnn.so" else: - return "lib64/libcudnn.so.{}".format(version) + return "lib64/libcudnn.so.{}".format(version) def cupti_library_path(version = cuda_sdk_version()): - if PLATFORM == "Darwin": - if not version: - return "extras/CUPTI/lib/libcupti.dylib" + if PLATFORM == "Darwin": + if not version: + return "extras/CUPTI/lib/libcupti.dylib" + else: + return "extras/CUPTI/lib/libcupti.{}.dylib".format(version) + elif not version: + return "extras/CUPTI/lib64/libcupti.so" else: - return "extras/CUPTI/lib/libcupti.{}.dylib".format(version) - else: - if not version: - return "extras/CUPTI/lib64/libcupti.so" - else: - return "extras/CUPTI/lib64/libcupti.so.{}".format(version) + return "extras/CUPTI/lib64/libcupti.so.{}".format(version) def readlink_command(): - if PLATFORM == "Darwin": - return "greadlink" - else: - return "readlink" + if PLATFORM == "Darwin": + return "greadlink" + else: + return "readlink" diff --git a/tensorflow/java/build_defs.bzl b/tensorflow/java/build_defs.bzl index e1916ca4d9d6aa..f423cc4d827750 100644 --- a/tensorflow/java/build_defs.bzl +++ b/tensorflow/java/build_defs.bzl @@ -18,7 +18,7 @@ XLINT_OPTS = [ "-Xlint:-processing", "-Xlint:-serial", "-Xlint:-try", - "-Xlint:-classfile", # see b/32750402, go/javac-warnings#classfile + "-Xlint:-classfile", # see b/32750402, go/javac-warnings#classfile ] # The bazel errorprone plugin currently only enables default errorChecks diff --git a/tensorflow/java/src/gen/gen_ops.bzl b/tensorflow/java/src/gen/gen_ops.bzl index f4ff34ea0361fb..b46721a93dcbd1 100644 --- a/tensorflow/java/src/gen/gen_ops.bzl +++ b/tensorflow/java/src/gen/gen_ops.bzl @@ -17,46 +17,48 @@ load( # and then archive those source files into # ops/gen_sources.srcjar # -def tf_java_op_gen_srcjar(name, - gen_tool, - base_package, - api_def_srcs=[], - out_dir="ops/", - out_src_dir="src/main/java/", - visibility=["//tensorflow/java:__pkg__"]): +def tf_java_op_gen_srcjar( + name, + gen_tool, + base_package, + api_def_srcs = [], + out_dir = "ops/", + out_src_dir = "src/main/java/", + visibility = ["//tensorflow/java:__pkg__"]): + gen_cmds = ["rm -rf $(@D)"] # Always start from fresh when generating source files + srcs = api_def_srcs[:] - gen_cmds = ["rm -rf $(@D)"] # Always start from fresh when generating source files - srcs = api_def_srcs[:] + if not api_def_srcs: + api_def_args_str = "," + else: + api_def_args = [] + for api_def_src in api_def_srcs: + # Add directory of the first ApiDef source to args. + # We are assuming all ApiDefs in a single api_def_src are in the + # same directory. + api_def_args.append( + "$$(dirname $$(echo $(locations " + api_def_src + + ") | cut -d\" \" -f1))", + ) + api_def_args_str = ",".join(api_def_args) - if not api_def_srcs: - api_def_args_str = "," - else: - api_def_args = [] - for api_def_src in api_def_srcs: - # Add directory of the first ApiDef source to args. - # We are assuming all ApiDefs in a single api_def_src are in the - # same directory. - api_def_args.append( - "$$(dirname $$(echo $(locations " + api_def_src + - ") | cut -d\" \" -f1))") - api_def_args_str = ",".join(api_def_args) + gen_cmds += ["$(location " + gen_tool + ")" + + " --output_dir=$(@D)/" + out_src_dir + + " --base_package=" + base_package + + " --api_dirs=" + api_def_args_str] - gen_cmds += ["$(location " + gen_tool + ")" + - " --output_dir=$(@D)/" + out_src_dir + - " --base_package=" + base_package + - " --api_dirs=" + api_def_args_str] + # Generate a source archive containing generated code for these ops. + gen_srcjar = out_dir + name + ".srcjar" + gen_cmds += ["$(location @local_jdk//:jar) cMf $(location :" + gen_srcjar + ") -C $(@D) src"] - # Generate a source archive containing generated code for these ops. - gen_srcjar = out_dir + name + ".srcjar" - gen_cmds += ["$(location @local_jdk//:jar) cMf $(location :" + gen_srcjar + ") -C $(@D) src"] - - native.genrule( - name=name, - srcs=srcs, - outs=[gen_srcjar], - tools=[ - "@local_jdk//:jar", - "@local_jdk//:jdk", - gen_tool - ] + tf_binary_additional_srcs(), - cmd=" && ".join(gen_cmds)) + native.genrule( + name = name, + srcs = srcs, + outs = [gen_srcjar], + tools = [ + "@local_jdk//:jar", + "@local_jdk//:jdk", + gen_tool, + ] + tf_binary_additional_srcs(), + cmd = " && ".join(gen_cmds), + ) diff --git a/tensorflow/python/build_defs.bzl b/tensorflow/python/build_defs.bzl index b9056f86e6d046..244820f41a8577 100644 --- a/tensorflow/python/build_defs.bzl +++ b/tensorflow/python/build_defs.bzl @@ -12,22 +12,26 @@ load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py") # consumers of the tf_gen_op_wrapper_py rule would be simplified if we don't # hard code the ops/ directory. -def tf_gen_op_wrapper_private_py(name, out=None, deps=[], - require_shape_functions=True, - visibility=[]): - if not name.endswith("_gen"): - fail("name must end in _gen") - if not visibility: - visibility = ["//visibility:private"] - bare_op_name = name[:-4] # Strip off the _gen - tf_gen_op_wrapper_py(name=bare_op_name, - out=out, - visibility=visibility, - deps=deps, - require_shape_functions=require_shape_functions, - generated_target_name=name, - api_def_srcs = [ - "//tensorflow/core/api_def:base_api_def", - "//tensorflow/core/api_def:python_api_def", - ], - ) +def tf_gen_op_wrapper_private_py( + name, + out = None, + deps = [], + require_shape_functions = True, + visibility = []): + if not name.endswith("_gen"): + fail("name must end in _gen") + if not visibility: + visibility = ["//visibility:private"] + bare_op_name = name[:-4] # Strip off the _gen + tf_gen_op_wrapper_py( + name = bare_op_name, + out = out, + visibility = visibility, + deps = deps, + require_shape_functions = require_shape_functions, + generated_target_name = name, + api_def_srcs = [ + "//tensorflow/core/api_def:base_api_def", + "//tensorflow/core/api_def:python_api_def", + ], + ) diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl index 1d8a7ec62a958c..b2da13e5ae708e 100644 --- a/tensorflow/stream_executor/build_defs.bzl +++ b/tensorflow/stream_executor/build_defs.bzl @@ -17,5 +17,5 @@ def tf_additional_cudnn_plugin_deps(): # Returns whether any GPU backend is configuered. def if_gpu_is_configured(x): if cuda_is_configured() or rocm_is_configured(): - return x + return x return [] diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 988b1e21591b0a..a7205a42ee5cb0 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -45,6 +45,7 @@ load( "//third_party/ngraph:build_defs.bzl", "if_ngraph", ) + def register_extension_info(**kwargs): pass @@ -1463,7 +1464,7 @@ def cc_header_only_library(name, deps = [], includes = [], extra_deps = [], **kw def tf_custom_op_library_additional_deps(): return [ - "@protobuf_archive//:protobuf_headers", + "@protobuf_archive//:protobuf_headers", clean_dep("//third_party/eigen3"), clean_dep("//tensorflow/core:framework_headers_lib"), ] + if_windows(["//tensorflow/python:pywrap_tensorflow_import_lib"]) @@ -1473,8 +1474,8 @@ def tf_custom_op_library_additional_deps(): # exporting symbols from _pywrap_tensorflow.dll on Windows. def tf_custom_op_library_additional_deps_impl(): return [ - "@protobuf_archive//:protobuf", - "@nsync//:nsync_cpp", + "@protobuf_archive//:protobuf", + "@nsync//:nsync_cpp", # for //third_party/eigen3 clean_dep("//third_party/eigen3"), # for //tensorflow/core:framework_headers_lib diff --git a/tensorflow/tools/test/performance.bzl b/tensorflow/tools/test/performance.bzl index 3486871080c78d..97861110346b62 100644 --- a/tensorflow/tools/test/performance.bzl +++ b/tensorflow/tools/test/performance.bzl @@ -4,60 +4,66 @@ load("//tensorflow:tensorflow.bzl", "tf_py_test") # Create a benchmark test target of a TensorFlow C++ test (tf_cc_*_test) def tf_cc_logged_benchmark( - name=None, - target=None, - benchmarks="..", - tags=[], - test_log_output_prefix="", - benchmark_type="cpp_microbenchmark"): - if not name: - fail("Must provide a name") - if not target: - fail("Must provide a target") - if (not ":" in target - or not target.startswith("//") - or target.endswith(":all") - or target.endswith(".")): - fail(" ".join(("Target must be a single well-defined test, e.g.,", - "//path/to:test. Received: %s" % target))) + name = None, + target = None, + benchmarks = "..", + tags = [], + test_log_output_prefix = "", + benchmark_type = "cpp_microbenchmark"): + if not name: + fail("Must provide a name") + if not target: + fail("Must provide a target") + if (not ":" in target or + not target.startswith("//") or + target.endswith(":all") or + target.endswith(".")): + fail(" ".join(( + "Target must be a single well-defined test, e.g.,", + "//path/to:test. Received: %s" % target, + ))) - all_tags = ( - depset(tags) + depset( - ["benchmark-test", "local", "manual", "regression-test"])).to_list() + all_tags = ( + depset(tags) + depset( + ["benchmark-test", "local", "manual", "regression-test"], + ) + ).to_list() - tf_py_test( - name = name, - tags = all_tags, - size = "large", - srcs = ["//tensorflow/tools/test:run_and_gather_logs"], - args = [ - "--name=//%s:%s" % (native.package_name(), name), - "--test_name=" + target, - "--test_args=--benchmarks=%s" % benchmarks, - "--benchmark_type=%s" % benchmark_type, - ], - data = [ - target, - ], - main = "run_and_gather_logs.py", - additional_deps = [ - "//tensorflow/tools/test:run_and_gather_logs" - ]) + tf_py_test( + name = name, + tags = all_tags, + size = "large", + srcs = ["//tensorflow/tools/test:run_and_gather_logs"], + args = [ + "--name=//%s:%s" % (native.package_name(), name), + "--test_name=" + target, + "--test_args=--benchmarks=%s" % benchmarks, + "--benchmark_type=%s" % benchmark_type, + ], + data = [ + target, + ], + main = "run_and_gather_logs.py", + additional_deps = [ + "//tensorflow/tools/test:run_and_gather_logs", + ], + ) # Create a benchmark test target of a TensorFlow python test (*py_tests) def tf_py_logged_benchmark( - name=None, - target=None, - benchmarks="..", - tags=[], - test_log_output_prefix=""): - # For now generating a py benchmark is the same as generating a C++ - # benchmark target. In the future this may change, so we have - # two macros just in case - tf_cc_logged_benchmark( - name=name, - target=target, - benchmarks=benchmarks, - tags=tags, - test_log_output_prefix=test_log_output_prefix, - benchmark_type="python_benchmark") + name = None, + target = None, + benchmarks = "..", + tags = [], + test_log_output_prefix = ""): + # For now generating a py benchmark is the same as generating a C++ + # benchmark target. In the future this may change, so we have + # two macros just in case + tf_cc_logged_benchmark( + name = name, + target = target, + benchmarks = benchmarks, + tags = tags, + test_log_output_prefix = test_log_output_prefix, + benchmark_type = "python_benchmark", + ) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 99027a07889bb7..dd67f887ddbb84 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -7,7 +7,6 @@ load("//third_party:nccl/nccl_configure.bzl", "nccl_configure") load("//third_party/mkl:build_defs.bzl", "mkl_repository") load("//third_party/git:git_configure.bzl", "git_configure") load("//third_party/py:python_configure.bzl", "python_configure") - load("//third_party/sycl:sycl_configure.bzl", "sycl_configure") load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure") load("//third_party/toolchains/remote:configure.bzl", "remote_execution_configure") diff --git a/third_party/android/android.bzl.tpl b/third_party/android/android.bzl.tpl index e6ed4994f3ba6d..c498f585abfa52 100644 --- a/third_party/android/android.bzl.tpl +++ b/third_party/android/android.bzl.tpl @@ -1,9 +1,9 @@ """Set up configurable Android SDK and NDK dependencies.""" def android_workspace(): - # String for replacement in Bazel template. - # These will either be replaced by android_sdk_repository if various ENV - # variables are set when `local_config_android` repo_rule is run, or they - # will be replaced by noops otherwise. - MAYBE_ANDROID_SDK_REPOSITORY - MAYBE_ANDROID_NDK_REPOSITORY + # String for replacement in Bazel template. + # These will either be replaced by android_sdk_repository if various ENV + # variables are set when `local_config_android` repo_rule is run, or they + # will be replaced by noops otherwise. + MAYBE_ANDROID_SDK_REPOSITORY + MAYBE_ANDROID_NDK_REPOSITORY diff --git a/third_party/android/android_configure.bzl b/third_party/android/android_configure.bzl index da09bdf39eed90..646ed732a1c7c7 100644 --- a/third_party/android/android_configure.bzl +++ b/third_party/android/android_configure.bzl @@ -36,33 +36,39 @@ _ANDROID_NDK_REPO_TEMPLATE = """ """ def _android_autoconf_impl(repository_ctx): - """Implementation of the android_autoconf repository rule.""" - sdk_home = repository_ctx.os.environ.get(_ANDROID_SDK_HOME) - sdk_api_level = repository_ctx.os.environ.get(_ANDROID_SDK_API_VERSION) - build_tools_version = repository_ctx.os.environ.get( - _ANDROID_BUILD_TOOLS_VERSION) - ndk_home = repository_ctx.os.environ.get(_ANDROID_NDK_HOME) - ndk_api_level = repository_ctx.os.environ.get(_ANDROID_NDK_API_VERSION) + """Implementation of the android_autoconf repository rule.""" + sdk_home = repository_ctx.os.environ.get(_ANDROID_SDK_HOME) + sdk_api_level = repository_ctx.os.environ.get(_ANDROID_SDK_API_VERSION) + build_tools_version = repository_ctx.os.environ.get( + _ANDROID_BUILD_TOOLS_VERSION, + ) + ndk_home = repository_ctx.os.environ.get(_ANDROID_NDK_HOME) + ndk_api_level = repository_ctx.os.environ.get(_ANDROID_NDK_API_VERSION) - sdk_rule = "pass" - if all([sdk_home, sdk_api_level, build_tools_version]): - sdk_rule = _ANDROID_SDK_REPO_TEMPLATE % ( - sdk_home, sdk_api_level, build_tools_version) + sdk_rule = "pass" + if all([sdk_home, sdk_api_level, build_tools_version]): + sdk_rule = _ANDROID_SDK_REPO_TEMPLATE % ( + sdk_home, + sdk_api_level, + build_tools_version, + ) - ndk_rule = "pass" - if all([ndk_home, ndk_api_level]): - ndk_rule = _ANDROID_NDK_REPO_TEMPLATE % (ndk_home, ndk_api_level) + ndk_rule = "pass" + if all([ndk_home, ndk_api_level]): + ndk_rule = _ANDROID_NDK_REPO_TEMPLATE % (ndk_home, ndk_api_level) - repository_ctx.template( - "BUILD", - Label("//third_party/android:android_configure.BUILD.tpl")) - repository_ctx.template( - "android.bzl", - Label("//third_party/android:android.bzl.tpl"), - substitutions={ - "MAYBE_ANDROID_SDK_REPOSITORY": sdk_rule, - "MAYBE_ANDROID_NDK_REPOSITORY": ndk_rule, - }) + repository_ctx.template( + "BUILD", + Label("//third_party/android:android_configure.BUILD.tpl"), + ) + repository_ctx.template( + "android.bzl", + Label("//third_party/android:android.bzl.tpl"), + substitutions = { + "MAYBE_ANDROID_SDK_REPOSITORY": sdk_rule, + "MAYBE_ANDROID_NDK_REPOSITORY": ndk_rule, + }, + ) android_configure = repository_rule( implementation = _android_autoconf_impl, diff --git a/third_party/common.bzl b/third_party/common.bzl index db981a5e314c08..8134bf3d2562f5 100644 --- a/third_party/common.bzl +++ b/third_party/common.bzl @@ -21,11 +21,11 @@ # substitutions: A dictionary mapping strings to their substitutions def template_rule_impl(ctx): - ctx.template_action( - template = ctx.file.src, - output = ctx.outputs.out, - substitutions = ctx.attr.substitutions, - ) + ctx.template_action( + template = ctx.file.src, + output = ctx.outputs.out, + substitutions = ctx.attr.substitutions, + ) template_rule = rule( attrs = { diff --git a/third_party/git/git_configure.bzl b/third_party/git/git_configure.bzl index 8e2839bdc254ac..fc18fdb98837e7 100644 --- a/third_party/git/git_configure.bzl +++ b/third_party/git/git_configure.bzl @@ -8,49 +8,57 @@ _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" def _fail(msg): - """Output failure message when auto configuration fails.""" - red = "\033[0;31m" - no_color = "\033[0m" - fail("%sGit Configuration Error:%s %s\n" % (red, no_color, msg)) + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sGit Configuration Error:%s %s\n" % (red, no_color, msg)) def _get_python_bin(repository_ctx): - """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_path = repository_ctx.which("python") - if python_bin_path != None: - return str(python_bin_path) - _fail("Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or --define " + - "%s='/something/else'.\nPATH=%s" % ( - _PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) - + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = repository_ctx.which("python") + if python_bin_path != None: + return str(python_bin_path) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % ( + _PYTHON_BIN_PATH, + repository_ctx.os.environ.get("PATH", ""), + )) def _git_conf_impl(repository_ctx): - repository_ctx.template( - "BUILD", - Label("//third_party/git:BUILD.tpl")) - - tensorflow_root_path = str(repository_ctx.path( - Label("@org_tensorflow//:BUILD")))[:-len("BUILD")] - python_script_path = repository_ctx.path( - Label("@org_tensorflow//tensorflow/tools/git:gen_git_source.py")) - generated_files_path = repository_ctx.path("gen") + repository_ctx.template( + "BUILD", + Label("//third_party/git:BUILD.tpl"), + ) - r = repository_ctx.execute( - ["test", "-f", "%s/.git/logs/HEAD" % tensorflow_root_path]) - if r.return_code == 0: - unused_var = repository_ctx.path(Label("//:.git/HEAD")) # pylint: disable=unused-variable + tensorflow_root_path = str(repository_ctx.path( + Label("@org_tensorflow//:BUILD"), + ))[:-len("BUILD")] + python_script_path = repository_ctx.path( + Label("@org_tensorflow//tensorflow/tools/git:gen_git_source.py"), + ) + generated_files_path = repository_ctx.path("gen") - result = repository_ctx.execute([ - _get_python_bin(repository_ctx), - python_script_path, "--configure", tensorflow_root_path, - "--gen_root_path", generated_files_path], quiet=False) + r = repository_ctx.execute( + ["test", "-f", "%s/.git/logs/HEAD" % tensorflow_root_path], + ) + if r.return_code == 0: + unused_var = repository_ctx.path(Label("//:.git/HEAD")) # pylint: disable=unused-variable - if not result.return_code == 0: - _fail(result.stderr) + result = repository_ctx.execute([ + _get_python_bin(repository_ctx), + python_script_path, + "--configure", + tensorflow_root_path, + "--gen_root_path", + generated_files_path, + ], quiet = False) + if not result.return_code == 0: + _fail(result.stderr) git_configure = repository_rule( implementation = _git_conf_impl, diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl index 40c5e373ede149..f13aacb93d59a4 100644 --- a/third_party/gpus/cuda_configure.bzl +++ b/third_party/gpus/cuda_configure.bzl @@ -126,141 +126,142 @@ load( ) def _get_python_bin(repository_ctx): - """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python" - python_bin_path = repository_ctx.which(python_bin_name) - if python_bin_path != None: - return str(python_bin_path) - auto_configure_fail( - "Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or --define " + - "%s='/something/else'.\nPATH=%s" % ( - _PYTHON_BIN_PATH, - repository_ctx.os.environ.get("PATH", ""), - )) - + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python" + python_bin_path = repository_ctx.which(python_bin_name) + if python_bin_path != None: + return str(python_bin_path) + auto_configure_fail( + "Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % ( + _PYTHON_BIN_PATH, + repository_ctx.os.environ.get("PATH", ""), + ), + ) def _get_nvcc_tmp_dir_for_windows(repository_ctx): - """Return the tmp directory for nvcc to generate intermediate source files.""" - escaped_tmp_dir = escape_string( - get_env_var(repository_ctx, "TMP", "C:\\Windows\\Temp").replace( - "\\", "\\\\"),) - return escaped_tmp_dir + "\\\\nvcc_inter_files_tmp_dir" - + """Return the tmp directory for nvcc to generate intermediate source files.""" + escaped_tmp_dir = escape_string( + get_env_var(repository_ctx, "TMP", "C:\\Windows\\Temp").replace( + "\\", + "\\\\", + ), + ) + return escaped_tmp_dir + "\\\\nvcc_inter_files_tmp_dir" def _get_msvc_compiler(repository_ctx): - vc_path = find_vc_path(repository_ctx) - return find_msvc_tool(repository_ctx, vc_path, "cl.exe").replace("\\", "/") - + vc_path = find_vc_path(repository_ctx) + return find_msvc_tool(repository_ctx, vc_path, "cl.exe").replace("\\", "/") def _get_win_cuda_defines(repository_ctx): - """Return CROSSTOOL defines for Windows""" + """Return CROSSTOOL defines for Windows""" + + # If we are not on Windows, return empty vaules for Windows specific fields. + # This ensures the CROSSTOOL file parser is happy. + if not _is_windows(repository_ctx): + return { + "%{msvc_env_tmp}": "", + "%{msvc_env_path}": "", + "%{msvc_env_include}": "", + "%{msvc_env_lib}": "", + "%{msvc_cl_path}": "", + "%{msvc_ml_path}": "", + "%{msvc_link_path}": "", + "%{msvc_lib_path}": "", + "%{cxx_builtin_include_directory}": "", + } + + vc_path = find_vc_path(repository_ctx) + if not vc_path: + auto_configure_fail( + "Visual C++ build tools not found on your machine." + + "Please check your installation following https://docs.bazel.build/versions/master/windows.html#using", + ) + return {} + + env = setup_vc_env_vars(repository_ctx, vc_path) + escaped_paths = escape_string(env["PATH"]) + escaped_include_paths = escape_string(env["INCLUDE"]) + escaped_lib_paths = escape_string(env["LIB"]) + escaped_tmp_dir = escape_string( + get_env_var(repository_ctx, "TMP", "C:\\Windows\\Temp").replace( + "\\", + "\\\\", + ), + ) + + msvc_cl_path = _get_python_bin(repository_ctx) + msvc_ml_path = find_msvc_tool(repository_ctx, vc_path, "ml64.exe").replace( + "\\", + "/", + ) + msvc_link_path = find_msvc_tool(repository_ctx, vc_path, "link.exe").replace( + "\\", + "/", + ) + msvc_lib_path = find_msvc_tool(repository_ctx, vc_path, "lib.exe").replace( + "\\", + "/", + ) + + # nvcc will generate some temporary source files under %{nvcc_tmp_dir} + # The generated files are guranteed to have unique name, so they can share the same tmp directory + escaped_cxx_include_directories = [ + "cxx_builtin_include_directory: \"%s\"" % + _get_nvcc_tmp_dir_for_windows(repository_ctx), + ] + for path in escaped_include_paths.split(";"): + if path: + escaped_cxx_include_directories.append( + "cxx_builtin_include_directory: \"%s\"" % path, + ) - # If we are not on Windows, return empty vaules for Windows specific fields. - # This ensures the CROSSTOOL file parser is happy. - if not _is_windows(repository_ctx): return { - "%{msvc_env_tmp}": "", - "%{msvc_env_path}": "", - "%{msvc_env_include}": "", - "%{msvc_env_lib}": "", - "%{msvc_cl_path}": "", - "%{msvc_ml_path}": "", - "%{msvc_link_path}": "", - "%{msvc_lib_path}": "", - "%{cxx_builtin_include_directory}": "", + "%{msvc_env_tmp}": escaped_tmp_dir, + "%{msvc_env_path}": escaped_paths, + "%{msvc_env_include}": escaped_include_paths, + "%{msvc_env_lib}": escaped_lib_paths, + "%{msvc_cl_path}": msvc_cl_path, + "%{msvc_ml_path}": msvc_ml_path, + "%{msvc_link_path}": msvc_link_path, + "%{msvc_lib_path}": msvc_lib_path, + "%{cxx_builtin_include_directory}": "\n".join(escaped_cxx_include_directories), } - vc_path = find_vc_path(repository_ctx) - if not vc_path: - auto_configure_fail( - "Visual C++ build tools not found on your machine." + - "Please check your installation following https://docs.bazel.build/versions/master/windows.html#using" - ) - return {} - - env = setup_vc_env_vars(repository_ctx, vc_path) - escaped_paths = escape_string(env["PATH"]) - escaped_include_paths = escape_string(env["INCLUDE"]) - escaped_lib_paths = escape_string(env["LIB"]) - escaped_tmp_dir = escape_string( - get_env_var(repository_ctx, "TMP", "C:\\Windows\\Temp").replace( - "\\", "\\\\"),) - - msvc_cl_path = _get_python_bin(repository_ctx) - msvc_ml_path = find_msvc_tool(repository_ctx, vc_path, "ml64.exe").replace( - "\\", "/") - msvc_link_path = find_msvc_tool(repository_ctx, vc_path, "link.exe").replace( - "\\", "/") - msvc_lib_path = find_msvc_tool(repository_ctx, vc_path, "lib.exe").replace( - "\\", "/") - - # nvcc will generate some temporary source files under %{nvcc_tmp_dir} - # The generated files are guranteed to have unique name, so they can share the same tmp directory - escaped_cxx_include_directories = [ - "cxx_builtin_include_directory: \"%s\"" % - _get_nvcc_tmp_dir_for_windows(repository_ctx) - ] - for path in escaped_include_paths.split(";"): - if path: - escaped_cxx_include_directories.append( - "cxx_builtin_include_directory: \"%s\"" % path) - - return { - "%{msvc_env_tmp}": - escaped_tmp_dir, - "%{msvc_env_path}": - escaped_paths, - "%{msvc_env_include}": - escaped_include_paths, - "%{msvc_env_lib}": - escaped_lib_paths, - "%{msvc_cl_path}": - msvc_cl_path, - "%{msvc_ml_path}": - msvc_ml_path, - "%{msvc_link_path}": - msvc_link_path, - "%{msvc_lib_path}": - msvc_lib_path, - "%{cxx_builtin_include_directory}": - "\n".join(escaped_cxx_include_directories), - } - # TODO(dzc): Once these functions have been factored out of Bazel's # cc_configure.bzl, load them from @bazel_tools instead. # BEGIN cc_configure common functions. def find_cc(repository_ctx): - """Find the C++ compiler.""" - if _is_windows(repository_ctx): - return _get_msvc_compiler(repository_ctx) - - if _use_cuda_clang(repository_ctx): - target_cc_name = "clang" - cc_path_envvar = _CLANG_CUDA_COMPILER_PATH - if _flag_enabled(repository_ctx, _TF_DOWNLOAD_CLANG): - return "extra_tools/bin/clang" - else: - target_cc_name = "gcc" - cc_path_envvar = _GCC_HOST_COMPILER_PATH - cc_name = target_cc_name - - if cc_path_envvar in repository_ctx.os.environ: - cc_name_from_env = repository_ctx.os.environ[cc_path_envvar].strip() - if cc_name_from_env: - cc_name = cc_name_from_env - if cc_name.startswith("/"): - # Absolute path, maybe we should make this supported by our which function. - return cc_name - cc = repository_ctx.which(cc_name) - if cc == None: - fail(("Cannot find {}, either correct your path or set the {}" + - " environment variable").format(target_cc_name, cc_path_envvar)) - return cc - + """Find the C++ compiler.""" + if _is_windows(repository_ctx): + return _get_msvc_compiler(repository_ctx) + + if _use_cuda_clang(repository_ctx): + target_cc_name = "clang" + cc_path_envvar = _CLANG_CUDA_COMPILER_PATH + if _flag_enabled(repository_ctx, _TF_DOWNLOAD_CLANG): + return "extra_tools/bin/clang" + else: + target_cc_name = "gcc" + cc_path_envvar = _GCC_HOST_COMPILER_PATH + cc_name = target_cc_name + + if cc_path_envvar in repository_ctx.os.environ: + cc_name_from_env = repository_ctx.os.environ[cc_path_envvar].strip() + if cc_name_from_env: + cc_name = cc_name_from_env + if cc_name.startswith("/"): + # Absolute path, maybe we should make this supported by our which function. + return cc_name + cc = repository_ctx.which(cc_name) + if cc == None: + fail(("Cannot find {}, either correct your path or set the {}" + + " environment variable").format(target_cc_name, cc_path_envvar)) + return cc _INC_DIR_MARKER_BEGIN = "#include <...>" @@ -269,480 +270,484 @@ _OSX_FRAMEWORK_SUFFIX = " (framework directory)" _OSX_FRAMEWORK_SUFFIX_LEN = len(_OSX_FRAMEWORK_SUFFIX) def _cxx_inc_convert(path): - """Convert path returned by cc -E xc++ in a complete path.""" - path = path.strip() - if path.endswith(_OSX_FRAMEWORK_SUFFIX): - path = path[:-_OSX_FRAMEWORK_SUFFIX_LEN].strip() - return path - + """Convert path returned by cc -E xc++ in a complete path.""" + path = path.strip() + if path.endswith(_OSX_FRAMEWORK_SUFFIX): + path = path[:-_OSX_FRAMEWORK_SUFFIX_LEN].strip() + return path def _normalize_include_path(repository_ctx, path): - """Normalizes include paths before writing them to the crosstool. - - If path points inside the 'crosstool' folder of the repository, a relative - path is returned. - If path points outside the 'crosstool' folder, an absolute path is returned. - """ - path = str(repository_ctx.path(path)) - crosstool_folder = str(repository_ctx.path(".").get_child("crosstool")) - - if path.startswith(crosstool_folder): - # We drop the path to "$REPO/crosstool" and a trailing path separator. - return path[len(crosstool_folder) + 1:] - return path - + """Normalizes include paths before writing them to the crosstool. + + If path points inside the 'crosstool' folder of the repository, a relative + path is returned. + If path points outside the 'crosstool' folder, an absolute path is returned. + """ + path = str(repository_ctx.path(path)) + crosstool_folder = str(repository_ctx.path(".").get_child("crosstool")) + + if path.startswith(crosstool_folder): + # We drop the path to "$REPO/crosstool" and a trailing path separator. + return path[len(crosstool_folder) + 1:] + return path def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp): - """Compute the list of default C or C++ include directories.""" - if lang_is_cpp: - lang = "c++" - else: - lang = "c" - result = repository_ctx.execute([cc, "-E", "-x" + lang, "-", "-v"]) - index1 = result.stderr.find(_INC_DIR_MARKER_BEGIN) - if index1 == -1: - return [] - index1 = result.stderr.find("\n", index1) - if index1 == -1: - return [] - index2 = result.stderr.rfind("\n ") - if index2 == -1 or index2 < index1: - return [] - index2 = result.stderr.find("\n", index2 + 1) - if index2 == -1: - inc_dirs = result.stderr[index1 + 1:] - else: - inc_dirs = result.stderr[index1 + 1:index2].strip() - - return [ - _normalize_include_path(repository_ctx, _cxx_inc_convert(p)) - for p in inc_dirs.split("\n") - ] - + """Compute the list of default C or C++ include directories.""" + if lang_is_cpp: + lang = "c++" + else: + lang = "c" + result = repository_ctx.execute([cc, "-E", "-x" + lang, "-", "-v"]) + index1 = result.stderr.find(_INC_DIR_MARKER_BEGIN) + if index1 == -1: + return [] + index1 = result.stderr.find("\n", index1) + if index1 == -1: + return [] + index2 = result.stderr.rfind("\n ") + if index2 == -1 or index2 < index1: + return [] + index2 = result.stderr.find("\n", index2 + 1) + if index2 == -1: + inc_dirs = result.stderr[index1 + 1:] + else: + inc_dirs = result.stderr[index1 + 1:index2].strip() + + return [ + _normalize_include_path(repository_ctx, _cxx_inc_convert(p)) + for p in inc_dirs.split("\n") + ] def get_cxx_inc_directories(repository_ctx, cc): - """Compute the list of default C and C++ include directories.""" - - # For some reason `clang -xc` sometimes returns include paths that are - # different from the ones from `clang -xc++`. (Symlink and a dir) - # So we run the compiler with both `-xc` and `-xc++` and merge resulting lists - includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True) - includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False) - - includes_cpp_set = depset(includes_cpp) - return includes_cpp + [ - inc for inc in includes_c if inc not in includes_cpp_set - ] - + """Compute the list of default C and C++ include directories.""" + + # For some reason `clang -xc` sometimes returns include paths that are + # different from the ones from `clang -xc++`. (Symlink and a dir) + # So we run the compiler with both `-xc` and `-xc++` and merge resulting lists + includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True) + includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False) + + includes_cpp_set = depset(includes_cpp) + return includes_cpp + [ + inc + for inc in includes_c + if inc not in includes_cpp_set + ] def auto_configure_fail(msg): - """Output failure message when cuda configuration fails.""" - red = "\033[0;31m" - no_color = "\033[0m" - fail("\n%sCuda Configuration Error:%s %s\n" % (red, no_color, msg)) + """Output failure message when cuda configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("\n%sCuda Configuration Error:%s %s\n" % (red, no_color, msg)) # END cc_configure common functions (see TODO above). def _host_compiler_includes(repository_ctx, cc): - """Generates the cxx_builtin_include_directory entries for gcc inc dirs. - - Args: - repository_ctx: The repository context. - cc: The path to the gcc host compiler. - - Returns: - A string containing the cxx_builtin_include_directory for each of the gcc - host compiler include directories, which can be added to the CROSSTOOL - file. - """ - inc_dirs = get_cxx_inc_directories(repository_ctx, cc) - inc_entries = [] - for inc_dir in inc_dirs: - inc_entries.append(" cxx_builtin_include_directory: \"%s\"" % inc_dir) - return "\n".join(inc_entries) - + """Generates the cxx_builtin_include_directory entries for gcc inc dirs. + + Args: + repository_ctx: The repository context. + cc: The path to the gcc host compiler. + + Returns: + A string containing the cxx_builtin_include_directory for each of the gcc + host compiler include directories, which can be added to the CROSSTOOL + file. + """ + inc_dirs = get_cxx_inc_directories(repository_ctx, cc) + inc_entries = [] + for inc_dir in inc_dirs: + inc_entries.append(" cxx_builtin_include_directory: \"%s\"" % inc_dir) + return "\n".join(inc_entries) def _cuda_include_path(repository_ctx, cuda_config): - """Generates the cxx_builtin_include_directory entries for cuda inc dirs. - - Args: - repository_ctx: The repository context. - cc: The path to the gcc host compiler. - - Returns: - A string containing the cxx_builtin_include_directory for each of the gcc - host compiler include directories, which can be added to the CROSSTOOL - file. - """ - nvcc_path = repository_ctx.path("%s/bin/nvcc%s" % ( - cuda_config.cuda_toolkit_path, - ".exe" if cuda_config.cpu_value == "Windows" else "", - )) - result = repository_ctx.execute([ - nvcc_path, - "-v", - "/dev/null", - "-o", - "/dev/null", - ]) - target_dir = "" - for one_line in result.stderr.splitlines(): - if one_line.startswith("#$ _TARGET_DIR_="): - target_dir = ( - cuda_config.cuda_toolkit_path + "/" + one_line.replace( - "#$ _TARGET_DIR_=", "") + "/include") - inc_entries = [] - if target_dir != "": - inc_entries.append(" cxx_builtin_include_directory: \"%s\"" % target_dir) - default_include = cuda_config.cuda_toolkit_path + "/include" - inc_entries.append( - " cxx_builtin_include_directory: \"%s\"" % default_include) - return "\n".join(inc_entries) - + """Generates the cxx_builtin_include_directory entries for cuda inc dirs. + + Args: + repository_ctx: The repository context. + cc: The path to the gcc host compiler. + + Returns: + A string containing the cxx_builtin_include_directory for each of the gcc + host compiler include directories, which can be added to the CROSSTOOL + file. + """ + nvcc_path = repository_ctx.path("%s/bin/nvcc%s" % ( + cuda_config.cuda_toolkit_path, + ".exe" if cuda_config.cpu_value == "Windows" else "", + )) + result = repository_ctx.execute([ + nvcc_path, + "-v", + "/dev/null", + "-o", + "/dev/null", + ]) + target_dir = "" + for one_line in result.stderr.splitlines(): + if one_line.startswith("#$ _TARGET_DIR_="): + target_dir = ( + cuda_config.cuda_toolkit_path + "/" + one_line.replace( + "#$ _TARGET_DIR_=", + "", + ) + "/include" + ) + inc_entries = [] + if target_dir != "": + inc_entries.append(" cxx_builtin_include_directory: \"%s\"" % target_dir) + default_include = cuda_config.cuda_toolkit_path + "/include" + inc_entries.append( + " cxx_builtin_include_directory: \"%s\"" % default_include, + ) + return "\n".join(inc_entries) def enable_cuda(repository_ctx): - if "TF_NEED_CUDA" in repository_ctx.os.environ: - enable_cuda = repository_ctx.os.environ["TF_NEED_CUDA"].strip() - return enable_cuda == "1" - return False - + if "TF_NEED_CUDA" in repository_ctx.os.environ: + enable_cuda = repository_ctx.os.environ["TF_NEED_CUDA"].strip() + return enable_cuda == "1" + return False def cuda_toolkit_path(repository_ctx): - """Finds the cuda toolkit directory. - - Args: - repository_ctx: The repository context. + """Finds the cuda toolkit directory. - Returns: - A speculative real path of the cuda toolkit install directory. - """ - cuda_toolkit_path = _DEFAULT_CUDA_TOOLKIT_PATH - if _CUDA_TOOLKIT_PATH in repository_ctx.os.environ: - cuda_toolkit_path = repository_ctx.os.environ[_CUDA_TOOLKIT_PATH].strip() - if not repository_ctx.path(cuda_toolkit_path).exists: - auto_configure_fail("Cannot find cuda toolkit path.") - return str(repository_ctx.path(cuda_toolkit_path).realpath) + Args: + repository_ctx: The repository context. + Returns: + A speculative real path of the cuda toolkit install directory. + """ + cuda_toolkit_path = _DEFAULT_CUDA_TOOLKIT_PATH + if _CUDA_TOOLKIT_PATH in repository_ctx.os.environ: + cuda_toolkit_path = repository_ctx.os.environ[_CUDA_TOOLKIT_PATH].strip() + if not repository_ctx.path(cuda_toolkit_path).exists: + auto_configure_fail("Cannot find cuda toolkit path.") + return str(repository_ctx.path(cuda_toolkit_path).realpath) def _cudnn_install_basedir(repository_ctx): - """Finds the cudnn install directory.""" - cudnn_install_path = _DEFAULT_CUDNN_INSTALL_PATH - if _CUDNN_INSTALL_PATH in repository_ctx.os.environ: - cudnn_install_path = repository_ctx.os.environ[_CUDNN_INSTALL_PATH].strip() - if not repository_ctx.path(cudnn_install_path).exists: - auto_configure_fail("Cannot find cudnn install path.") - return cudnn_install_path - + """Finds the cudnn install directory.""" + cudnn_install_path = _DEFAULT_CUDNN_INSTALL_PATH + if _CUDNN_INSTALL_PATH in repository_ctx.os.environ: + cudnn_install_path = repository_ctx.os.environ[_CUDNN_INSTALL_PATH].strip() + if not repository_ctx.path(cudnn_install_path).exists: + auto_configure_fail("Cannot find cudnn install path.") + return cudnn_install_path def matches_version(environ_version, detected_version): - """Checks whether the user-specified version matches the detected version. - - This function performs a weak matching so that if the user specifies only - the - major or major and minor versions, the versions are still considered - matching - if the version parts match. To illustrate: - - environ_version detected_version result - ----------------------------------------- - 5.1.3 5.1.3 True - 5.1 5.1.3 True - 5 5.1 True - 5.1.3 5.1 False - 5.2.3 5.1.3 False - - Args: - environ_version: The version specified by the user via environment - variables. - detected_version: The version autodetected from the CUDA installation on - the system. - Returns: True if user-specified version matches detected version and False - otherwise. - """ - environ_version_parts = environ_version.split(".") - detected_version_parts = detected_version.split(".") - if len(detected_version_parts) < len(environ_version_parts): - return False - for i, part in enumerate(detected_version_parts): - if i >= len(environ_version_parts): - break - if part != environ_version_parts[i]: - return False - return True - + """Checks whether the user-specified version matches the detected version. + + This function performs a weak matching so that if the user specifies only + the + major or major and minor versions, the versions are still considered + matching + if the version parts match. To illustrate: + + environ_version detected_version result + ----------------------------------------- + 5.1.3 5.1.3 True + 5.1 5.1.3 True + 5 5.1 True + 5.1.3 5.1 False + 5.2.3 5.1.3 False + + Args: + environ_version: The version specified by the user via environment + variables. + detected_version: The version autodetected from the CUDA installation on + the system. + Returns: True if user-specified version matches detected version and False + otherwise. + """ + environ_version_parts = environ_version.split(".") + detected_version_parts = detected_version.split(".") + if len(detected_version_parts) < len(environ_version_parts): + return False + for i, part in enumerate(detected_version_parts): + if i >= len(environ_version_parts): + break + if part != environ_version_parts[i]: + return False + return True _NVCC_VERSION_PREFIX = "Cuda compilation tools, release " def _cuda_version(repository_ctx, cuda_toolkit_path, cpu_value): - """Detects the version of CUDA installed on the system. - - Args: - repository_ctx: The repository context. - cuda_toolkit_path: The CUDA install directory. - - Returns: - String containing the version of CUDA. - """ + """Detects the version of CUDA installed on the system. - # Run nvcc --version and find the line containing the CUDA version. - nvcc_path = repository_ctx.path("%s/bin/nvcc%s" % ( - cuda_toolkit_path, - ".exe" if cpu_value == "Windows" else "", - )) - if not nvcc_path.exists: - auto_configure_fail("Cannot find nvcc at %s" % str(nvcc_path)) - result = repository_ctx.execute([str(nvcc_path), "--version"]) - if result.stderr: - auto_configure_fail("Error running nvcc --version: %s" % result.stderr) - lines = result.stdout.splitlines() - version_line = lines[len(lines) - 1] - if version_line.find(_NVCC_VERSION_PREFIX) == -1: - auto_configure_fail( - "Could not parse CUDA version from nvcc --version. Got: %s" % - result.stdout,) - - # Parse the CUDA version from the line containing the CUDA version. - prefix_removed = version_line.replace(_NVCC_VERSION_PREFIX, "") - parts = prefix_removed.split(",") - if len(parts) != 2 or len(parts[0]) < 2: - auto_configure_fail( - "Could not parse CUDA version from nvcc --version. Got: %s" % - result.stdout,) - full_version = parts[1].strip() - if full_version.startswith("V"): - full_version = full_version[1:] - - # Check whether TF_CUDA_VERSION was set by the user and fail if it does not - # match the detected version. - environ_version = "" - if _TF_CUDA_VERSION in repository_ctx.os.environ: - environ_version = repository_ctx.os.environ[_TF_CUDA_VERSION].strip() - if environ_version and not matches_version(environ_version, full_version): - auto_configure_fail( - ("CUDA version detected from nvcc (%s) does not match " + - "TF_CUDA_VERSION (%s)") % (full_version, environ_version),) + Args: + repository_ctx: The repository context. + cuda_toolkit_path: The CUDA install directory. - # We only use the version consisting of the major and minor version numbers. - version_parts = full_version.split(".") - if len(version_parts) < 2: - auto_configure_fail("CUDA version detected from nvcc (%s) is incomplete.") - if cpu_value == "Windows": - version = "64_%s%s" % (version_parts[0], version_parts[1]) - else: - version = "%s.%s" % (version_parts[0], version_parts[1]) - return version + Returns: + String containing the version of CUDA. + """ + # Run nvcc --version and find the line containing the CUDA version. + nvcc_path = repository_ctx.path("%s/bin/nvcc%s" % ( + cuda_toolkit_path, + ".exe" if cpu_value == "Windows" else "", + )) + if not nvcc_path.exists: + auto_configure_fail("Cannot find nvcc at %s" % str(nvcc_path)) + result = repository_ctx.execute([str(nvcc_path), "--version"]) + if result.stderr: + auto_configure_fail("Error running nvcc --version: %s" % result.stderr) + lines = result.stdout.splitlines() + version_line = lines[len(lines) - 1] + if version_line.find(_NVCC_VERSION_PREFIX) == -1: + auto_configure_fail( + "Could not parse CUDA version from nvcc --version. Got: %s" % + result.stdout, + ) + + # Parse the CUDA version from the line containing the CUDA version. + prefix_removed = version_line.replace(_NVCC_VERSION_PREFIX, "") + parts = prefix_removed.split(",") + if len(parts) != 2 or len(parts[0]) < 2: + auto_configure_fail( + "Could not parse CUDA version from nvcc --version. Got: %s" % + result.stdout, + ) + full_version = parts[1].strip() + if full_version.startswith("V"): + full_version = full_version[1:] + + # Check whether TF_CUDA_VERSION was set by the user and fail if it does not + # match the detected version. + environ_version = "" + if _TF_CUDA_VERSION in repository_ctx.os.environ: + environ_version = repository_ctx.os.environ[_TF_CUDA_VERSION].strip() + if environ_version and not matches_version(environ_version, full_version): + auto_configure_fail( + ("CUDA version detected from nvcc (%s) does not match " + + "TF_CUDA_VERSION (%s)") % (full_version, environ_version), + ) + + # We only use the version consisting of the major and minor version numbers. + version_parts = full_version.split(".") + if len(version_parts) < 2: + auto_configure_fail("CUDA version detected from nvcc (%s) is incomplete.") + if cpu_value == "Windows": + version = "64_%s%s" % (version_parts[0], version_parts[1]) + else: + version = "%s.%s" % (version_parts[0], version_parts[1]) + return version _DEFINE_CUDNN_MAJOR = "#define CUDNN_MAJOR" _DEFINE_CUDNN_MINOR = "#define CUDNN_MINOR" _DEFINE_CUDNN_PATCHLEVEL = "#define CUDNN_PATCHLEVEL" def find_cuda_define(repository_ctx, header_dir, header_file, define): - """Returns the value of a #define in a header file. - - Greps through a header file and returns the value of the specified #define. - If the #define is not found, then raise an error. - - Args: - repository_ctx: The repository context. - header_dir: The directory containing the header file. - header_file: The header file name. - define: The #define to search for. - - Returns: - The value of the #define found in the header. - """ - - # Confirm location of the header and grep for the line defining the macro. - h_path = repository_ctx.path("%s/%s" % (header_dir, header_file)) - if not h_path.exists: - auto_configure_fail("Cannot find %s at %s" % (header_file, str(h_path))) - result = repository_ctx.execute( - # Grep one more lines as some #defines are splitted into two lines. - ["grep", "--color=never", "-A1", "-E", define, - str(h_path)],) - if result.stderr: - auto_configure_fail("Error reading %s: %s" % (str(h_path), result.stderr)) - - # Parse the version from the line defining the macro. - if result.stdout.find(define) == -1: - auto_configure_fail( - "Cannot find line containing '%s' in %s" % (define, h_path)) - - # Split results to lines - lines = result.stdout.split("\n") - num_lines = len(lines) - for l in range(num_lines): - line = lines[l] - if define in line: # Find the line with define - version = line - if l != num_lines - 1 and line[-1] == "\\": # Add next line, if multiline - version = version[:-1] + lines[l + 1] - break - - # Remove any comments - version = version.split("//")[0] - - # Remove define name - version = version.replace(define, "").strip() - - # Remove the code after the version number. - version_end = version.find(" ") - if version_end != -1: - if version_end == 0: - auto_configure_fail( - "Cannot extract the version from line containing '%s' in %s" % - (define, str(h_path)),) - version = version[:version_end].strip() - return version - + """Returns the value of a #define in a header file. + + Greps through a header file and returns the value of the specified #define. + If the #define is not found, then raise an error. + + Args: + repository_ctx: The repository context. + header_dir: The directory containing the header file. + header_file: The header file name. + define: The #define to search for. + + Returns: + The value of the #define found in the header. + """ + + # Confirm location of the header and grep for the line defining the macro. + h_path = repository_ctx.path("%s/%s" % (header_dir, header_file)) + if not h_path.exists: + auto_configure_fail("Cannot find %s at %s" % (header_file, str(h_path))) + result = repository_ctx.execute( + # Grep one more lines as some #defines are splitted into two lines. + [ + "grep", + "--color=never", + "-A1", + "-E", + define, + str(h_path), + ], + ) + if result.stderr: + auto_configure_fail("Error reading %s: %s" % (str(h_path), result.stderr)) + + # Parse the version from the line defining the macro. + if result.stdout.find(define) == -1: + auto_configure_fail( + "Cannot find line containing '%s' in %s" % (define, h_path), + ) + + # Split results to lines + lines = result.stdout.split("\n") + num_lines = len(lines) + for l in range(num_lines): + line = lines[l] + if define in line: # Find the line with define + version = line + if l != num_lines - 1 and line[-1] == "\\": # Add next line, if multiline + version = version[:-1] + lines[l + 1] + break + + # Remove any comments + version = version.split("//")[0] + + # Remove define name + version = version.replace(define, "").strip() + + # Remove the code after the version number. + version_end = version.find(" ") + if version_end != -1: + if version_end == 0: + auto_configure_fail( + "Cannot extract the version from line containing '%s' in %s" % + (define, str(h_path)), + ) + version = version[:version_end].strip() + return version def _cudnn_version(repository_ctx, cudnn_install_basedir, cpu_value): - """Detects the version of cuDNN installed on the system. + """Detects the version of cuDNN installed on the system. - Args: - repository_ctx: The repository context. - cpu_value: The name of the host operating system. - cudnn_install_basedir: The cuDNN install directory. - - Returns: - A string containing the version of cuDNN. - """ - cudnn_header_dir = _find_cudnn_header_dir( - repository_ctx, - cudnn_install_basedir, - ) - major_version = find_cuda_define( - repository_ctx, - cudnn_header_dir, - "cudnn.h", - _DEFINE_CUDNN_MAJOR, - ) - minor_version = find_cuda_define( - repository_ctx, - cudnn_header_dir, - "cudnn.h", - _DEFINE_CUDNN_MINOR, - ) - patch_version = find_cuda_define( - repository_ctx, - cudnn_header_dir, - "cudnn.h", - _DEFINE_CUDNN_PATCHLEVEL, - ) - full_version = "%s.%s.%s" % (major_version, minor_version, patch_version) - - # Check whether TF_CUDNN_VERSION was set by the user and fail if it does not - # match the detected version. - environ_version = "" - if _TF_CUDNN_VERSION in repository_ctx.os.environ: - environ_version = repository_ctx.os.environ[_TF_CUDNN_VERSION].strip() - if environ_version and not matches_version(environ_version, full_version): - cudnn_h_path = repository_ctx.path( - "%s/include/cudnn.h" % cudnn_install_basedir) - auto_configure_fail(("cuDNN version detected from %s (%s) does not match " + - "TF_CUDNN_VERSION (%s)") % - (str(cudnn_h_path), full_version, environ_version),) - # Only use the major version to match the SONAME of the library. - version = major_version - if cpu_value == "Windows": - version = "64_" + version - return version + Args: + repository_ctx: The repository context. + cpu_value: The name of the host operating system. + cudnn_install_basedir: The cuDNN install directory. + Returns: + A string containing the version of cuDNN. + """ + cudnn_header_dir = _find_cudnn_header_dir( + repository_ctx, + cudnn_install_basedir, + ) + major_version = find_cuda_define( + repository_ctx, + cudnn_header_dir, + "cudnn.h", + _DEFINE_CUDNN_MAJOR, + ) + minor_version = find_cuda_define( + repository_ctx, + cudnn_header_dir, + "cudnn.h", + _DEFINE_CUDNN_MINOR, + ) + patch_version = find_cuda_define( + repository_ctx, + cudnn_header_dir, + "cudnn.h", + _DEFINE_CUDNN_PATCHLEVEL, + ) + full_version = "%s.%s.%s" % (major_version, minor_version, patch_version) + + # Check whether TF_CUDNN_VERSION was set by the user and fail if it does not + # match the detected version. + environ_version = "" + if _TF_CUDNN_VERSION in repository_ctx.os.environ: + environ_version = repository_ctx.os.environ[_TF_CUDNN_VERSION].strip() + if environ_version and not matches_version(environ_version, full_version): + cudnn_h_path = repository_ctx.path( + "%s/include/cudnn.h" % cudnn_install_basedir, + ) + auto_configure_fail(("cuDNN version detected from %s (%s) does not match " + + "TF_CUDNN_VERSION (%s)") % + (str(cudnn_h_path), full_version, environ_version)) + + # Only use the major version to match the SONAME of the library. + version = major_version + if cpu_value == "Windows": + version = "64_" + version + return version def compute_capabilities(repository_ctx): - """Returns a list of strings representing cuda compute capabilities.""" - if _TF_CUDA_COMPUTE_CAPABILITIES not in repository_ctx.os.environ: - return _DEFAULT_CUDA_COMPUTE_CAPABILITIES - capabilities_str = repository_ctx.os.environ[_TF_CUDA_COMPUTE_CAPABILITIES] - capabilities = capabilities_str.split(",") - for capability in capabilities: - # Workaround for Skylark's lack of support for regex. This check should - # be equivalent to checking: - # if re.match("[0-9]+.[0-9]+", capability) == None: - parts = capability.split(".") - if len(parts) != 2 or not parts[0].isdigit() or not parts[1].isdigit(): - auto_configure_fail("Invalid compute capability: %s" % capability) - return capabilities - + """Returns a list of strings representing cuda compute capabilities.""" + if _TF_CUDA_COMPUTE_CAPABILITIES not in repository_ctx.os.environ: + return _DEFAULT_CUDA_COMPUTE_CAPABILITIES + capabilities_str = repository_ctx.os.environ[_TF_CUDA_COMPUTE_CAPABILITIES] + capabilities = capabilities_str.split(",") + for capability in capabilities: + # Workaround for Skylark's lack of support for regex. This check should + # be equivalent to checking: + # if re.match("[0-9]+.[0-9]+", capability) == None: + parts = capability.split(".") + if len(parts) != 2 or not parts[0].isdigit() or not parts[1].isdigit(): + auto_configure_fail("Invalid compute capability: %s" % capability) + return capabilities def get_cpu_value(repository_ctx): - """Returns the name of the host operating system. - - Args: - repository_ctx: The repository context. - - Returns: - A string containing the name of the host operating system. - """ - os_name = repository_ctx.os.name.lower() - if os_name.startswith("mac os"): - return "Darwin" - if os_name.find("windows") != -1: - return "Windows" - result = repository_ctx.execute(["uname", "-s"]) - return result.stdout.strip() - + """Returns the name of the host operating system. + + Args: + repository_ctx: The repository context. + + Returns: + A string containing the name of the host operating system. + """ + os_name = repository_ctx.os.name.lower() + if os_name.startswith("mac os"): + return "Darwin" + if os_name.find("windows") != -1: + return "Windows" + result = repository_ctx.execute(["uname", "-s"]) + return result.stdout.strip() def _is_windows(repository_ctx): - """Returns true if the host operating system is windows.""" - return get_cpu_value(repository_ctx) == "Windows" - + """Returns true if the host operating system is windows.""" + return get_cpu_value(repository_ctx) == "Windows" def lib_name(base_name, cpu_value, version = None, static = False): - """Constructs the platform-specific name of a library. + """Constructs the platform-specific name of a library. - Args: - base_name: The name of the library, such as "cudart" - cpu_value: The name of the host operating system. - version: The version of the library. - static: True the library is static or False if it is a shared object. + Args: + base_name: The name of the library, such as "cudart" + cpu_value: The name of the host operating system. + version: The version of the library. + static: True the library is static or False if it is a shared object. + + Returns: + The platform-specific name of the library. + """ + version = "" if not version else "." + version + if cpu_value in ("Linux", "FreeBSD"): + if static: + return "lib%s.a" % base_name + return "lib%s.so%s" % (base_name, version) + elif cpu_value == "Windows": + return "%s.lib" % base_name + elif cpu_value == "Darwin": + if static: + return "lib%s.a" % base_name + return "lib%s%s.dylib" % (base_name, version) + else: + auto_configure_fail("Invalid cpu_value: %s" % cpu_value) - Returns: - The platform-specific name of the library. +def find_lib(repository_ctx, paths, check_soname = True): """ - version = "" if not version else "." + version - if cpu_value in ("Linux", "FreeBSD"): - if static: - return "lib%s.a" % base_name - return "lib%s.so%s" % (base_name, version) - elif cpu_value == "Windows": - return "%s.lib" % base_name - elif cpu_value == "Darwin": - if static: - return "lib%s.a" % base_name - return "lib%s%s.dylib" % (base_name, version) - else: - auto_configure_fail("Invalid cpu_value: %s" % cpu_value) + Finds a library among a list of potential paths. -def find_lib(repository_ctx, paths, check_soname = True): - """ - Finds a library among a list of potential paths. - - Args: - paths: List of paths to inspect. - - Returns: - Returns the first path in paths that exist. - """ - objdump = repository_ctx.which("objdump") - mismatches = [] - for path in [repository_ctx.path(path) for path in paths]: - if not path.exists: - continue - if check_soname and objdump != None: - output = repository_ctx.execute([objdump, "-p", str(path)]).stdout - output = [line for line in output.splitlines() if "SONAME" in line] - sonames = [line.strip().split(" ")[-1] for line in output] - if not any([soname == path.basename for soname in sonames]): - mismatches.append(str(path)) - continue - return path - if mismatches: - auto_configure_fail( - "None of the libraries match their SONAME: " + ", ".join(mismatches)) - auto_configure_fail("No library found under: " + ", ".join(paths)) + Args: + paths: List of paths to inspect. + Returns: + Returns the first path in paths that exist. + """ + objdump = repository_ctx.which("objdump") + mismatches = [] + for path in [repository_ctx.path(path) for path in paths]: + if not path.exists: + continue + if check_soname and objdump != None: + output = repository_ctx.execute([objdump, "-p", str(path)]).stdout + output = [line for line in output.splitlines() if "SONAME" in line] + sonames = [line.strip().split(" ")[-1] for line in output] + if not any([soname == path.basename for soname in sonames]): + mismatches.append(str(path)) + continue + return path + if mismatches: + auto_configure_fail( + "None of the libraries match their SONAME: " + ", ".join(mismatches), + ) + auto_configure_fail("No library found under: " + ", ".join(paths)) def _find_cuda_lib( lib, @@ -751,263 +756,257 @@ def _find_cuda_lib( basedir, version, static = False): - """Finds the given CUDA or cuDNN library on the system. - - Args: - lib: The name of the library, such as "cudart" - repository_ctx: The repository context. - cpu_value: The name of the host operating system. - basedir: The install directory of CUDA or cuDNN. - version: The version of the library. - static: True if static library, False if shared object. - - Returns: - Returns the path to the library. - """ - file_name = lib_name(lib, cpu_value, version, static) - return find_lib(repository_ctx, [ - "%s/%s%s" % (basedir, path, file_name) for path in CUDA_LIB_PATHS - ], check_soname = version and not static) + """Finds the given CUDA or cuDNN library on the system. + Args: + lib: The name of the library, such as "cudart" + repository_ctx: The repository context. + cpu_value: The name of the host operating system. + basedir: The install directory of CUDA or cuDNN. + version: The version of the library. + static: True if static library, False if shared object. + + Returns: + Returns the path to the library. + """ + file_name = lib_name(lib, cpu_value, version, static) + return find_lib(repository_ctx, [ + "%s/%s%s" % (basedir, path, file_name) + for path in CUDA_LIB_PATHS + ], check_soname = version and not static) def _find_cupti_header_dir(repository_ctx, cuda_config): - """Returns the path to the directory containing cupti.h - - On most systems, the cupti library is not installed in the same directory as - the other CUDA libraries but rather in a special extras/CUPTI directory. - - Args: - repository_ctx: The repository context. - cuda_config: The CUDA config as returned by _get_cuda_config - - Returns: - The path of the directory containing the cupti header. - """ - cuda_toolkit_path = cuda_config.cuda_toolkit_path - for relative_path in CUPTI_HEADER_PATHS: - if repository_ctx.path( - "%s/%scupti.h" % (cuda_toolkit_path, relative_path)).exists: - return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] - auto_configure_fail("Cannot find cupti.h under %s" % ", ".join( - [cuda_toolkit_path + "/" + s for s in CUPTI_HEADER_PATHS])) - + """Returns the path to the directory containing cupti.h + + On most systems, the cupti library is not installed in the same directory as + the other CUDA libraries but rather in a special extras/CUPTI directory. + + Args: + repository_ctx: The repository context. + cuda_config: The CUDA config as returned by _get_cuda_config + + Returns: + The path of the directory containing the cupti header. + """ + cuda_toolkit_path = cuda_config.cuda_toolkit_path + for relative_path in CUPTI_HEADER_PATHS: + if repository_ctx.path( + "%s/%scupti.h" % (cuda_toolkit_path, relative_path), + ).exists: + return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] + auto_configure_fail("Cannot find cupti.h under %s" % ", ".join( + [cuda_toolkit_path + "/" + s for s in CUPTI_HEADER_PATHS], + )) def _find_cupti_lib(repository_ctx, cuda_config): - """Finds the cupti library on the system. - - On most systems, the cupti library is not installed in the same directory as - the other CUDA libraries but rather in a special extras/CUPTI directory. - - Args: - repository_ctx: The repository context. - cuda_config: The cuda configuration as returned by _get_cuda_config. - - Returns: - Returns the path to the library. - """ - file_name = lib_name( - "cupti", - cuda_config.cpu_value, - cuda_config.cuda_version, - ) - basedir = cuda_config.cuda_toolkit_path - return find_lib(repository_ctx, [ - "%s/%s%s" % (basedir, path, file_name) for path in CUPTI_LIB_PATHS - ]) - + """Finds the cupti library on the system. + + On most systems, the cupti library is not installed in the same directory as + the other CUDA libraries but rather in a special extras/CUPTI directory. + + Args: + repository_ctx: The repository context. + cuda_config: The cuda configuration as returned by _get_cuda_config. + + Returns: + Returns the path to the library. + """ + file_name = lib_name( + "cupti", + cuda_config.cpu_value, + cuda_config.cuda_version, + ) + basedir = cuda_config.cuda_toolkit_path + return find_lib(repository_ctx, [ + "%s/%s%s" % (basedir, path, file_name) + for path in CUPTI_LIB_PATHS + ]) def _find_libs(repository_ctx, cuda_config): - """Returns the CUDA and cuDNN libraries on the system. - - Args: - repository_ctx: The repository context. - cuda_config: The CUDA config as returned by _get_cuda_config + """Returns the CUDA and cuDNN libraries on the system. - Returns: - Map of library names to structs of filename and path. - """ - cpu_value = cuda_config.cpu_value - return { - "cuda": - _find_cuda_lib( - "cuda", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - None), - "cudart": - _find_cuda_lib( - "cudart", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - ), - "cudart_static": - _find_cuda_lib( - "cudart_static", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - static=True, - ), - "cublas": - _find_cuda_lib( - "cublas", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - ), - "cusolver": - _find_cuda_lib( - "cusolver", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - ), - "curand": - _find_cuda_lib( - "curand", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - ), - "cufft": - _find_cuda_lib( - "cufft", - repository_ctx, - cpu_value, - cuda_config.cuda_toolkit_path, - cuda_config.cuda_version, - ), - "cudnn": - _find_cuda_lib( - "cudnn", - repository_ctx, - cpu_value, - cuda_config.cudnn_install_basedir, - cuda_config.cudnn_version, - ), - "cupti": - _find_cupti_lib(repository_ctx, cuda_config), - } + Args: + repository_ctx: The repository context. + cuda_config: The CUDA config as returned by _get_cuda_config + Returns: + Map of library names to structs of filename and path. + """ + cpu_value = cuda_config.cpu_value + return { + "cuda": _find_cuda_lib( + "cuda", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + None, + ), + "cudart": _find_cuda_lib( + "cudart", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + ), + "cudart_static": _find_cuda_lib( + "cudart_static", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + static = True, + ), + "cublas": _find_cuda_lib( + "cublas", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + ), + "cusolver": _find_cuda_lib( + "cusolver", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + ), + "curand": _find_cuda_lib( + "curand", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + ), + "cufft": _find_cuda_lib( + "cufft", + repository_ctx, + cpu_value, + cuda_config.cuda_toolkit_path, + cuda_config.cuda_version, + ), + "cudnn": _find_cuda_lib( + "cudnn", + repository_ctx, + cpu_value, + cuda_config.cudnn_install_basedir, + cuda_config.cudnn_version, + ), + "cupti": _find_cupti_lib(repository_ctx, cuda_config), + } def _find_cuda_include_path(repository_ctx, cuda_config): - """Returns the path to the directory containing cuda.h - - Args: - repository_ctx: The repository context. - cuda_config: The CUDA config as returned by _get_cuda_config - - Returns: - The path of the directory containing the CUDA headers. - """ - cuda_toolkit_path = cuda_config.cuda_toolkit_path - for relative_path in CUDA_INCLUDE_PATHS: - if repository_ctx.path( - "%s/%scuda.h" % (cuda_toolkit_path, relative_path)).exists: - return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] - auto_configure_fail("Cannot find cuda.h under %s" % cuda_toolkit_path) - + """Returns the path to the directory containing cuda.h + + Args: + repository_ctx: The repository context. + cuda_config: The CUDA config as returned by _get_cuda_config + + Returns: + The path of the directory containing the CUDA headers. + """ + cuda_toolkit_path = cuda_config.cuda_toolkit_path + for relative_path in CUDA_INCLUDE_PATHS: + if repository_ctx.path( + "%s/%scuda.h" % (cuda_toolkit_path, relative_path), + ).exists: + return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] + auto_configure_fail("Cannot find cuda.h under %s" % cuda_toolkit_path) def _find_cudnn_header_dir(repository_ctx, cudnn_install_basedir): - """Returns the path to the directory containing cudnn.h - - Args: - repository_ctx: The repository context. - cudnn_install_basedir: The cudnn install directory as returned by - _cudnn_install_basedir. - - Returns: - The path of the directory containing the cudnn header. - """ - for relative_path in CUDA_INCLUDE_PATHS: - if repository_ctx.path( - "%s/%scudnn.h" % (cudnn_install_basedir, relative_path)).exists: - return ("%s/%s" % (cudnn_install_basedir, relative_path))[:-1] - if repository_ctx.path("/usr/include/cudnn.h").exists: - return "/usr/include" - auto_configure_fail("Cannot find cudnn.h under %s" % cudnn_install_basedir) - + """Returns the path to the directory containing cudnn.h + + Args: + repository_ctx: The repository context. + cudnn_install_basedir: The cudnn install directory as returned by + _cudnn_install_basedir. + + Returns: + The path of the directory containing the cudnn header. + """ + for relative_path in CUDA_INCLUDE_PATHS: + if repository_ctx.path( + "%s/%scudnn.h" % (cudnn_install_basedir, relative_path), + ).exists: + return ("%s/%s" % (cudnn_install_basedir, relative_path))[:-1] + if repository_ctx.path("/usr/include/cudnn.h").exists: + return "/usr/include" + auto_configure_fail("Cannot find cudnn.h under %s" % cudnn_install_basedir) def _find_nvvm_libdevice_dir(repository_ctx, cuda_config): - """Returns the path to the directory containing libdevice in bitcode format. - - Args: - repository_ctx: The repository context. - cuda_config: The CUDA config as returned by _get_cuda_config - - Returns: - The path of the directory containing the CUDA headers. - """ - cuda_toolkit_path = cuda_config.cuda_toolkit_path - for libdevice_file in NVVM_LIBDEVICE_FILES: - for relative_path in NVVM_LIBDEVICE_PATHS: - if repository_ctx.path("%s/%s%s" % (cuda_toolkit_path, relative_path, - libdevice_file)).exists: - return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] - auto_configure_fail( - "Cannot find libdevice*.bc files under %s" % cuda_toolkit_path) - + """Returns the path to the directory containing libdevice in bitcode format. + + Args: + repository_ctx: The repository context. + cuda_config: The CUDA config as returned by _get_cuda_config + + Returns: + The path of the directory containing the CUDA headers. + """ + cuda_toolkit_path = cuda_config.cuda_toolkit_path + for libdevice_file in NVVM_LIBDEVICE_FILES: + for relative_path in NVVM_LIBDEVICE_PATHS: + if repository_ctx.path("%s/%s%s" % ( + cuda_toolkit_path, + relative_path, + libdevice_file, + )).exists: + return ("%s/%s" % (cuda_toolkit_path, relative_path))[:-1] + auto_configure_fail( + "Cannot find libdevice*.bc files under %s" % cuda_toolkit_path, + ) def _cudart_static_linkopt(cpu_value): - """Returns additional platform-specific linkopts for cudart.""" - return "" if cpu_value == "Darwin" else "\"-lrt\"," - + """Returns additional platform-specific linkopts for cudart.""" + return "" if cpu_value == "Darwin" else "\"-lrt\"," def _get_cuda_config(repository_ctx): - """Detects and returns information about the CUDA installation on the system. - - Args: - repository_ctx: The repository context. - - Returns: - A struct containing the following fields: - cuda_toolkit_path: The CUDA toolkit installation directory. - cudnn_install_basedir: The cuDNN installation directory. - cuda_version: The version of CUDA on the system. - cudnn_version: The version of cuDNN on the system. - compute_capabilities: A list of the system's CUDA compute capabilities. - cpu_value: The name of the host operating system. - """ - cpu_value = get_cpu_value(repository_ctx) - toolkit_path = cuda_toolkit_path(repository_ctx) - cuda_version = _cuda_version(repository_ctx, toolkit_path, cpu_value) - cudnn_install_basedir = _cudnn_install_basedir(repository_ctx) - cudnn_version = _cudnn_version(repository_ctx, cudnn_install_basedir, - cpu_value) - return struct( - cuda_toolkit_path=toolkit_path, - cudnn_install_basedir=cudnn_install_basedir, - cuda_version=cuda_version, - cudnn_version=cudnn_version, - compute_capabilities=compute_capabilities(repository_ctx), - cpu_value=cpu_value, - ) - + """Detects and returns information about the CUDA installation on the system. + + Args: + repository_ctx: The repository context. + + Returns: + A struct containing the following fields: + cuda_toolkit_path: The CUDA toolkit installation directory. + cudnn_install_basedir: The cuDNN installation directory. + cuda_version: The version of CUDA on the system. + cudnn_version: The version of cuDNN on the system. + compute_capabilities: A list of the system's CUDA compute capabilities. + cpu_value: The name of the host operating system. + """ + cpu_value = get_cpu_value(repository_ctx) + toolkit_path = cuda_toolkit_path(repository_ctx) + cuda_version = _cuda_version(repository_ctx, toolkit_path, cpu_value) + cudnn_install_basedir = _cudnn_install_basedir(repository_ctx) + cudnn_version = _cudnn_version( + repository_ctx, + cudnn_install_basedir, + cpu_value, + ) + return struct( + cuda_toolkit_path = toolkit_path, + cudnn_install_basedir = cudnn_install_basedir, + cuda_version = cuda_version, + cudnn_version = cudnn_version, + compute_capabilities = compute_capabilities(repository_ctx), + cpu_value = cpu_value, + ) def _tpl(repository_ctx, tpl, substitutions = {}, out = None): - if not out: - out = tpl.replace(":", "/") - repository_ctx.template( - out, - Label("//third_party/gpus/%s.tpl" % tpl), - substitutions, - ) - + if not out: + out = tpl.replace(":", "/") + repository_ctx.template( + out, + Label("//third_party/gpus/%s.tpl" % tpl), + substitutions, + ) def _file(repository_ctx, label): - repository_ctx.template( - label.replace(":", "/"), - Label("//third_party/gpus/%s.tpl" % label), - {}, - ) - + repository_ctx.template( + label.replace(":", "/"), + Label("//third_party/gpus/%s.tpl" % label), + {}, + ) _DUMMY_CROSSTOOL_BZL_FILE = """ def error_gpu_disabled(): @@ -1035,99 +1034,83 @@ error_gpu_disabled() """ def _create_dummy_repository(repository_ctx): - cpu_value = get_cpu_value(repository_ctx) - - # Set up BUILD file for cuda/. - _tpl( - repository_ctx, - "cuda:build_defs.bzl", - { - "%{cuda_is_configured}": "False", - "%{cuda_extra_copts}": "[]", - }, - ) - _tpl( - repository_ctx, - "cuda:BUILD", - { - "%{cuda_driver_lib}": - lib_name("cuda", cpu_value), - "%{cudart_static_lib}": - lib_name( - "cudart_static", - cpu_value, - static=True, - ), - "%{cudart_static_linkopt}": - _cudart_static_linkopt(cpu_value), - "%{cudart_lib}": - lib_name("cudart", cpu_value), - "%{cublas_lib}": - lib_name("cublas", cpu_value), - "%{cusolver_lib}": - lib_name("cusolver", cpu_value), - "%{cudnn_lib}": - lib_name("cudnn", cpu_value), - "%{cufft_lib}": - lib_name("cufft", cpu_value), - "%{curand_lib}": - lib_name("curand", cpu_value), - "%{cupti_lib}": - lib_name("cupti", cpu_value), - "%{copy_rules}": - "", - "%{cuda_headers}": - "", - }, - ) + cpu_value = get_cpu_value(repository_ctx) - # Create dummy files for the CUDA toolkit since they are still required by - # tensorflow/core/platform/default/build_config:cuda. - repository_ctx.file("cuda/cuda/include/cuda.h") - repository_ctx.file("cuda/cuda/include/cublas.h") - repository_ctx.file("cuda/cuda/include/cudnn.h") - repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h") - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cuda", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudart", cpu_value)) - repository_ctx.file( - "cuda/cuda/lib/%s" % lib_name("cudart_static", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cublas", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusolver", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudnn", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("curand", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cufft", cpu_value)) - repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cupti", cpu_value)) - - # Set up cuda_config.h, which is used by - # tensorflow/stream_executor/dso_loader.cc. - _tpl( - repository_ctx, - "cuda:cuda_config.h", - { - "%{cuda_version}": - _DEFAULT_CUDA_VERSION, - "%{cudnn_version}": - _DEFAULT_CUDNN_VERSION, - "%{cuda_compute_capabilities}": - ",".join([ - "CudaVersion(\"%s\")" % c - for c in _DEFAULT_CUDA_COMPUTE_CAPABILITIES - ]), - "%{cuda_toolkit_path}": - _DEFAULT_CUDA_TOOLKIT_PATH, - }, - "cuda/cuda/cuda_config.h", - ) + # Set up BUILD file for cuda/. + _tpl( + repository_ctx, + "cuda:build_defs.bzl", + { + "%{cuda_is_configured}": "False", + "%{cuda_extra_copts}": "[]", + }, + ) + _tpl( + repository_ctx, + "cuda:BUILD", + { + "%{cuda_driver_lib}": lib_name("cuda", cpu_value), + "%{cudart_static_lib}": lib_name( + "cudart_static", + cpu_value, + static = True, + ), + "%{cudart_static_linkopt}": _cudart_static_linkopt(cpu_value), + "%{cudart_lib}": lib_name("cudart", cpu_value), + "%{cublas_lib}": lib_name("cublas", cpu_value), + "%{cusolver_lib}": lib_name("cusolver", cpu_value), + "%{cudnn_lib}": lib_name("cudnn", cpu_value), + "%{cufft_lib}": lib_name("cufft", cpu_value), + "%{curand_lib}": lib_name("curand", cpu_value), + "%{cupti_lib}": lib_name("cupti", cpu_value), + "%{copy_rules}": "", + "%{cuda_headers}": "", + }, + ) - # If cuda_configure is not configured to build with GPU support, and the user - # attempts to build with --config=cuda, add a dummy build rule to intercept - # this and fail with an actionable error message. - repository_ctx.file( - "crosstool/error_gpu_disabled.bzl", - _DUMMY_CROSSTOOL_BZL_FILE, - ) - repository_ctx.file("crosstool/BUILD", _DUMMY_CROSSTOOL_BUILD_FILE) + # Create dummy files for the CUDA toolkit since they are still required by + # tensorflow/core/platform/default/build_config:cuda. + repository_ctx.file("cuda/cuda/include/cuda.h") + repository_ctx.file("cuda/cuda/include/cublas.h") + repository_ctx.file("cuda/cuda/include/cudnn.h") + repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h") + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cuda", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudart", cpu_value)) + repository_ctx.file( + "cuda/cuda/lib/%s" % lib_name("cudart_static", cpu_value), + ) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cublas", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusolver", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudnn", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("curand", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cufft", cpu_value)) + repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cupti", cpu_value)) + + # Set up cuda_config.h, which is used by + # tensorflow/stream_executor/dso_loader.cc. + _tpl( + repository_ctx, + "cuda:cuda_config.h", + { + "%{cuda_version}": _DEFAULT_CUDA_VERSION, + "%{cudnn_version}": _DEFAULT_CUDNN_VERSION, + "%{cuda_compute_capabilities}": ",".join([ + "CudaVersion(\"%s\")" % c + for c in _DEFAULT_CUDA_COMPUTE_CAPABILITIES + ]), + "%{cuda_toolkit_path}": _DEFAULT_CUDA_TOOLKIT_PATH, + }, + "cuda/cuda/cuda_config.h", + ) + # If cuda_configure is not configured to build with GPU support, and the user + # attempts to build with --config=cuda, add a dummy build rule to intercept + # this and fail with an actionable error message. + repository_ctx.file( + "crosstool/error_gpu_disabled.bzl", + _DUMMY_CROSSTOOL_BZL_FILE, + ) + repository_ctx.file("crosstool/BUILD", _DUMMY_CROSSTOOL_BUILD_FILE) def _execute( repository_ctx, @@ -1135,43 +1118,44 @@ def _execute( error_msg = None, error_details = None, empty_stdout_fine = False): - """Executes an arbitrary shell command. - - Args: - repository_ctx: the repository_ctx object - cmdline: list of strings, the command to execute - error_msg: string, a summary of the error if the command fails - error_details: string, details about the error or steps to fix it - empty_stdout_fine: bool, if True, an empty stdout result is fine, - otherwise it's an error - Return: the result of repository_ctx.execute(cmdline) - """ - result = repository_ctx.execute(cmdline) - if result.stderr or not (empty_stdout_fine or result.stdout): - auto_configure_fail( - "\n".join([ - error_msg.strip() if error_msg else "Repository command failed", - result.stderr.strip(), - error_details if error_details else "", - ]),) - return result - + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, + otherwise it's an error + Return: the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + auto_configure_fail( + "\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else "", + ]), + ) + return result def _norm_path(path): - """Returns a path with '/' and remove the trailing slash.""" - path = path.replace("\\", "/") - if path[-1] == "/": - path = path[:-1] - return path + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path def make_copy_files_rule(repository_ctx, name, srcs, outs): - """Returns a rule to copy a set of files.""" - cmds = [] - # Copy files. - for src, out in zip(srcs, outs): - cmds.append('cp -f "%s" $(location %s)' % (src, out)) - outs = [(' "%s",' % out) for out in outs] - return """genrule( + """Returns a rule to copy a set of files.""" + cmds = [] + + # Copy files. + for src, out in zip(srcs, outs): + cmds.append('cp -f "%s" $(location %s)' % (src, out)) + outs = [(' "%s",' % out) for out in outs] + return """genrule( name = "%s", outs = [ %s @@ -1180,15 +1164,16 @@ def make_copy_files_rule(repository_ctx, name, srcs, outs): )""" % (name, "\n".join(outs), " && ".join(cmds)) def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir): - """Returns a rule to recursively copy a directory.""" - src_dir = _norm_path(src_dir) - out_dir = _norm_path(out_dir) - outs = _read_dir(repository_ctx, src_dir) - outs = [(' "%s",' % out.replace(src_dir, out_dir)) for out in outs] - # '@D' already contains the relative path for a single file, see - # http://docs.bazel.build/versions/master/be/make-variables.html#predefined_genrule_variables - out_dir = "$(@D)/%s" % out_dir if len(outs) > 1 else "$(@D)" - return """genrule( + """Returns a rule to recursively copy a directory.""" + src_dir = _norm_path(src_dir) + out_dir = _norm_path(out_dir) + outs = _read_dir(repository_ctx, src_dir) + outs = [(' "%s",' % out.replace(src_dir, out_dir)) for out in outs] + + # '@D' already contains the relative path for a single file, see + # http://docs.bazel.build/versions/master/be/make-variables.html#predefined_genrule_variables + out_dir = "$(@D)/%s" % out_dir if len(outs) > 1 else "$(@D)" + return """genrule( name = "%s", outs = [ %s @@ -1197,346 +1182,323 @@ def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir): )""" % (name, "\n".join(outs), src_dir, out_dir) def _read_dir(repository_ctx, src_dir): - """Returns a string with all files in a directory. - - Finds all files inside a directory, traversing subfolders and following - symlinks. The returned string contains the full path of all files - separated by line breaks. - """ - if _is_windows(repository_ctx): - src_dir = src_dir.replace("/", "\\") - find_result = _execute( - repository_ctx, - ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], - empty_stdout_fine=True, - ) - - # src_files will be used in genrule.outs where the paths must - # use forward slashes. - result = find_result.stdout.replace("\\", "/") - else: - find_result = _execute( - repository_ctx, - ["find", src_dir, "-follow", "-type", "f"], - empty_stdout_fine=True, - ) - result = find_result.stdout - return sorted(result.splitlines()) - + """Returns a string with all files in a directory. + + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine = True, + ) + + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + result = find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, + ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine = True, + ) + result = find_result.stdout + return sorted(result.splitlines()) def _flag_enabled(repository_ctx, flag_name): - if flag_name in repository_ctx.os.environ: - value = repository_ctx.os.environ[flag_name].strip() - return value == "1" - return False - + if flag_name in repository_ctx.os.environ: + value = repository_ctx.os.environ[flag_name].strip() + return value == "1" + return False def _use_cuda_clang(repository_ctx): - return _flag_enabled(repository_ctx, "TF_CUDA_CLANG") - + return _flag_enabled(repository_ctx, "TF_CUDA_CLANG") def _compute_cuda_extra_copts(repository_ctx, compute_capabilities): - if _use_cuda_clang(repository_ctx): - capability_flags = [ - "--cuda-gpu-arch=sm_" + cap.replace(".", "") - for cap in compute_capabilities - ] - else: - # Capabilities are handled in the "crosstool_wrapper_driver_is_not_gcc" for nvcc - # TODO(csigg): Make this consistent with cuda clang and pass to crosstool. - capability_flags = [] - return str(capability_flags) - + if _use_cuda_clang(repository_ctx): + capability_flags = [ + "--cuda-gpu-arch=sm_" + cap.replace(".", "") + for cap in compute_capabilities + ] + else: + # Capabilities are handled in the "crosstool_wrapper_driver_is_not_gcc" for nvcc + # TODO(csigg): Make this consistent with cuda clang and pass to crosstool. + capability_flags = [] + return str(capability_flags) def _create_local_cuda_repository(repository_ctx): - """Creates the repository containing files set up to build with CUDA.""" - cuda_config = _get_cuda_config(repository_ctx) + """Creates the repository containing files set up to build with CUDA.""" + cuda_config = _get_cuda_config(repository_ctx) - cuda_include_path = _find_cuda_include_path(repository_ctx, cuda_config) - cudnn_header_dir = _find_cudnn_header_dir( - repository_ctx, - cuda_config.cudnn_install_basedir, - ) - cupti_header_dir = _find_cupti_header_dir(repository_ctx, cuda_config) - nvvm_libdevice_dir = _find_nvvm_libdevice_dir(repository_ctx, cuda_config) - - # Create genrule to copy files from the installed CUDA toolkit into execroot. - copy_rules = [ - make_copy_dir_rule( - repository_ctx, - name = "cuda-include", - src_dir = cuda_include_path, - out_dir = "cuda/include", - ), - make_copy_dir_rule( - repository_ctx, - name = "cuda-nvvm", - src_dir = nvvm_libdevice_dir, - out_dir = "cuda/nvvm/libdevice", - ), - make_copy_dir_rule( - repository_ctx, - name = "cuda-extras", - src_dir = cupti_header_dir, - out_dir = "cuda/extras/CUPTI/include", - ), - ] - - cuda_libs = _find_libs(repository_ctx, cuda_config) - cuda_lib_srcs = [] - cuda_lib_outs = [] - for path in cuda_libs.values(): - cuda_lib_srcs.append(str(path)) - cuda_lib_outs.append("cuda/lib/" + path.basename) - copy_rules.append(make_copy_files_rule( - repository_ctx, - name = "cuda-lib", - srcs = cuda_lib_srcs, - outs = cuda_lib_outs, - )) - - copy_rules.append(make_copy_dir_rule( - repository_ctx, - name = "cuda-bin", - src_dir = cuda_config.cuda_toolkit_path + "/bin", - out_dir = "cuda/bin" - )) - - # Copy cudnn.h if cuDNN was not installed to CUDA_TOOLKIT_PATH. - included_files = _read_dir(repository_ctx, cuda_include_path) - if not any([file.endswith("cudnn.h") for file in included_files]): + cuda_include_path = _find_cuda_include_path(repository_ctx, cuda_config) + cudnn_header_dir = _find_cudnn_header_dir( + repository_ctx, + cuda_config.cudnn_install_basedir, + ) + cupti_header_dir = _find_cupti_header_dir(repository_ctx, cuda_config) + nvvm_libdevice_dir = _find_nvvm_libdevice_dir(repository_ctx, cuda_config) + + # Create genrule to copy files from the installed CUDA toolkit into execroot. + copy_rules = [ + make_copy_dir_rule( + repository_ctx, + name = "cuda-include", + src_dir = cuda_include_path, + out_dir = "cuda/include", + ), + make_copy_dir_rule( + repository_ctx, + name = "cuda-nvvm", + src_dir = nvvm_libdevice_dir, + out_dir = "cuda/nvvm/libdevice", + ), + make_copy_dir_rule( + repository_ctx, + name = "cuda-extras", + src_dir = cupti_header_dir, + out_dir = "cuda/extras/CUPTI/include", + ), + ] + + cuda_libs = _find_libs(repository_ctx, cuda_config) + cuda_lib_srcs = [] + cuda_lib_outs = [] + for path in cuda_libs.values(): + cuda_lib_srcs.append(str(path)) + cuda_lib_outs.append("cuda/lib/" + path.basename) copy_rules.append(make_copy_files_rule( repository_ctx, - name = "cudnn-include", - srcs = [cudnn_header_dir + "/cudnn.h"], - outs = ["cuda/include/cudnn.h"], + name = "cuda-lib", + srcs = cuda_lib_srcs, + outs = cuda_lib_outs, )) - else: - copy_rules.append("filegroup(name = 'cudnn-include')\n") - - # Set up BUILD file for cuda/ - _tpl( - repository_ctx, - "cuda:build_defs.bzl", - { - "%{cuda_is_configured}": - "True", - "%{cuda_extra_copts}": - _compute_cuda_extra_copts( - repository_ctx, - cuda_config.compute_capabilities, - ), - }, - ) - _tpl( - repository_ctx, - "cuda:BUILD.windows" if _is_windows(repository_ctx) else "cuda:BUILD", - { - "%{cuda_driver_lib}": - cuda_libs["cuda"].basename, - "%{cudart_static_lib}": - cuda_libs["cudart_static"].basename, - "%{cudart_static_linkopt}": - _cudart_static_linkopt(cuda_config.cpu_value,), - "%{cudart_lib}": - cuda_libs["cudart"].basename, - "%{cublas_lib}": - cuda_libs["cublas"].basename, - "%{cusolver_lib}": - cuda_libs["cusolver"].basename, - "%{cudnn_lib}": - cuda_libs["cudnn"].basename, - "%{cufft_lib}": - cuda_libs["cufft"].basename, - "%{curand_lib}": - cuda_libs["curand"].basename, - "%{cupti_lib}": - cuda_libs["cupti"].basename, - "%{copy_rules}": - "\n".join(copy_rules), - "%{cuda_headers}": ('":cuda-include",\n' + ' ":cudnn-include",' - ), - }, - "cuda/BUILD", - ) - is_cuda_clang = _use_cuda_clang(repository_ctx) + copy_rules.append(make_copy_dir_rule( + repository_ctx, + name = "cuda-bin", + src_dir = cuda_config.cuda_toolkit_path + "/bin", + out_dir = "cuda/bin", + )) - should_download_clang = is_cuda_clang and _flag_enabled( - repository_ctx, - _TF_DOWNLOAD_CLANG, - ) - if should_download_clang: - download_clang(repository_ctx, "crosstool/extra_tools") - - # Set up crosstool/ - cc = find_cc(repository_ctx) - cc_fullpath = cc if not should_download_clang else "crosstool/" + cc - - host_compiler_includes = _host_compiler_includes(repository_ctx, cc_fullpath) - cuda_defines = {} - # Bazel sets '-B/usr/bin' flag to workaround build errors on RHEL (see - # https://github.com/bazelbuild/bazel/issues/760). - # However, this stops our custom clang toolchain from picking the provided - # LLD linker, so we're only adding '-B/usr/bin' when using non-downloaded - # toolchain. - # TODO: when bazel stops adding '-B/usr/bin' by default, remove this - # flag from the CROSSTOOL completely (see - # https://github.com/bazelbuild/bazel/issues/5634) - if should_download_clang: - cuda_defines["%{linker_bin_path_flag}"] = "" - else: - cuda_defines["%{linker_bin_path_flag}"] = 'flag: "-B/usr/bin"' - - if is_cuda_clang: - cuda_defines["%{host_compiler_path}"] = str(cc) - cuda_defines["%{host_compiler_warnings}"] = """ - # Some parts of the codebase set -Werror and hit this warning, so - # switch it off for now. - flag: "-Wno-invalid-partial-specialization" - """ - cuda_defines["%{host_compiler_includes}"] = host_compiler_includes - cuda_defines["%{extra_no_canonical_prefixes_flags}"] = "" - _tpl(repository_ctx, "crosstool:BUILD", { - "%{linker_files}": ":empty", - "%{win_linker_files}": ":empty" - }) - repository_ctx.file( - "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc", "") - repository_ctx.file("crosstool/windows/msvc_wrapper_for_nvcc.py", "") - else: - cuda_defines[ - "%{host_compiler_path}"] = "clang/bin/crosstool_wrapper_driver_is_not_gcc" - cuda_defines["%{host_compiler_warnings}"] = "" - - # nvcc has the system include paths built in and will automatically - # search them; we cannot work around that, so we add the relevant cuda - # system paths to the allowed compiler specific include paths. - cuda_defines["%{host_compiler_includes}"] = ( - host_compiler_includes + "\n" + _cuda_include_path( - repository_ctx, cuda_config) + - "\n cxx_builtin_include_directory: \"%s\"" % cupti_header_dir + - "\n cxx_builtin_include_directory: \"%s\"" % cudnn_header_dir) - - # For gcc, do not canonicalize system header paths; some versions of gcc - # pick the shortest possible path for system includes when creating the - # .d file - given that includes that are prefixed with "../" multiple - # time quickly grow longer than the root of the tree, this can lead to - # bazel's header check failing. - cuda_defines["%{extra_no_canonical_prefixes_flags}"] = ( - "flag: \"-fno-canonical-system-headers\"") - nvcc_path = str( - repository_ctx.path("%s/bin/nvcc%s" % ( - cuda_config.cuda_toolkit_path, - ".exe" if _is_windows(repository_ctx) else "", - ))) + # Copy cudnn.h if cuDNN was not installed to CUDA_TOOLKIT_PATH. + included_files = _read_dir(repository_ctx, cuda_include_path) + if not any([file.endswith("cudnn.h") for file in included_files]): + copy_rules.append(make_copy_files_rule( + repository_ctx, + name = "cudnn-include", + srcs = [cudnn_header_dir + "/cudnn.h"], + outs = ["cuda/include/cudnn.h"], + )) + else: + copy_rules.append("filegroup(name = 'cudnn-include')\n") + + # Set up BUILD file for cuda/ _tpl( repository_ctx, - "crosstool:BUILD", + "cuda:build_defs.bzl", { - "%{linker_files}": ":crosstool_wrapper_driver_is_not_gcc", - "%{win_linker_files}": ":windows_msvc_wrapper_files", + "%{cuda_is_configured}": "True", + "%{cuda_extra_copts}": _compute_cuda_extra_copts( + repository_ctx, + cuda_config.compute_capabilities, + ), }, ) - wrapper_defines = { - "%{cpu_compiler}": - str(cc), - "%{cuda_version}": - cuda_config.cuda_version, - "%{nvcc_path}": - nvcc_path, - "%{gcc_host_compiler_path}": - str(cc), - "%{cuda_compute_capabilities}": - ", ".join( - ["\"%s\"" % c for c in cuda_config.compute_capabilities],), - "%{nvcc_tmp_dir}": - _get_nvcc_tmp_dir_for_windows(repository_ctx), - } _tpl( repository_ctx, - "crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc", - wrapper_defines, + "cuda:BUILD.windows" if _is_windows(repository_ctx) else "cuda:BUILD", + { + "%{cuda_driver_lib}": cuda_libs["cuda"].basename, + "%{cudart_static_lib}": cuda_libs["cudart_static"].basename, + "%{cudart_static_linkopt}": _cudart_static_linkopt(cuda_config.cpu_value), + "%{cudart_lib}": cuda_libs["cudart"].basename, + "%{cublas_lib}": cuda_libs["cublas"].basename, + "%{cusolver_lib}": cuda_libs["cusolver"].basename, + "%{cudnn_lib}": cuda_libs["cudnn"].basename, + "%{cufft_lib}": cuda_libs["cufft"].basename, + "%{curand_lib}": cuda_libs["curand"].basename, + "%{cupti_lib}": cuda_libs["cupti"].basename, + "%{copy_rules}": "\n".join(copy_rules), + "%{cuda_headers}": ( + '":cuda-include",\n' + ' ":cudnn-include",' + ), + }, + "cuda/BUILD", ) - _tpl( + + is_cuda_clang = _use_cuda_clang(repository_ctx) + + should_download_clang = is_cuda_clang and _flag_enabled( repository_ctx, - "crosstool:windows/msvc_wrapper_for_nvcc.py", - wrapper_defines, + _TF_DOWNLOAD_CLANG, ) + if should_download_clang: + download_clang(repository_ctx, "crosstool/extra_tools") + + # Set up crosstool/ + cc = find_cc(repository_ctx) + cc_fullpath = cc if not should_download_clang else "crosstool/" + cc + + host_compiler_includes = _host_compiler_includes(repository_ctx, cc_fullpath) + cuda_defines = {} + + # Bazel sets '-B/usr/bin' flag to workaround build errors on RHEL (see + # https://github.com/bazelbuild/bazel/issues/760). + # However, this stops our custom clang toolchain from picking the provided + # LLD linker, so we're only adding '-B/usr/bin' when using non-downloaded + # toolchain. + # TODO: when bazel stops adding '-B/usr/bin' by default, remove this + # flag from the CROSSTOOL completely (see + # https://github.com/bazelbuild/bazel/issues/5634) + if should_download_clang: + cuda_defines["%{linker_bin_path_flag}"] = "" + else: + cuda_defines["%{linker_bin_path_flag}"] = 'flag: "-B/usr/bin"' + + if is_cuda_clang: + cuda_defines["%{host_compiler_path}"] = str(cc) + cuda_defines["%{host_compiler_warnings}"] = """ + # Some parts of the codebase set -Werror and hit this warning, so + # switch it off for now. + flag: "-Wno-invalid-partial-specialization" + """ + cuda_defines["%{host_compiler_includes}"] = host_compiler_includes + cuda_defines["%{extra_no_canonical_prefixes_flags}"] = "" + _tpl(repository_ctx, "crosstool:BUILD", { + "%{linker_files}": ":empty", + "%{win_linker_files}": ":empty", + }) + repository_ctx.file( + "crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc", + "", + ) + repository_ctx.file("crosstool/windows/msvc_wrapper_for_nvcc.py", "") + else: + cuda_defines["%{host_compiler_path}"] = "clang/bin/crosstool_wrapper_driver_is_not_gcc" + cuda_defines["%{host_compiler_warnings}"] = "" + + # nvcc has the system include paths built in and will automatically + # search them; we cannot work around that, so we add the relevant cuda + # system paths to the allowed compiler specific include paths. + cuda_defines["%{host_compiler_includes}"] = ( + host_compiler_includes + "\n" + _cuda_include_path( + repository_ctx, + cuda_config, + ) + + "\n cxx_builtin_include_directory: \"%s\"" % cupti_header_dir + + "\n cxx_builtin_include_directory: \"%s\"" % cudnn_header_dir + ) + + # For gcc, do not canonicalize system header paths; some versions of gcc + # pick the shortest possible path for system includes when creating the + # .d file - given that includes that are prefixed with "../" multiple + # time quickly grow longer than the root of the tree, this can lead to + # bazel's header check failing. + cuda_defines["%{extra_no_canonical_prefixes_flags}"] = ( + "flag: \"-fno-canonical-system-headers\"" + ) + nvcc_path = str( + repository_ctx.path("%s/bin/nvcc%s" % ( + cuda_config.cuda_toolkit_path, + ".exe" if _is_windows(repository_ctx) else "", + )), + ) + _tpl( + repository_ctx, + "crosstool:BUILD", + { + "%{linker_files}": ":crosstool_wrapper_driver_is_not_gcc", + "%{win_linker_files}": ":windows_msvc_wrapper_files", + }, + ) + wrapper_defines = { + "%{cpu_compiler}": str(cc), + "%{cuda_version}": cuda_config.cuda_version, + "%{nvcc_path}": nvcc_path, + "%{gcc_host_compiler_path}": str(cc), + "%{cuda_compute_capabilities}": ", ".join( + ["\"%s\"" % c for c in cuda_config.compute_capabilities], + ), + "%{nvcc_tmp_dir}": _get_nvcc_tmp_dir_for_windows(repository_ctx), + } + _tpl( + repository_ctx, + "crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc", + wrapper_defines, + ) + _tpl( + repository_ctx, + "crosstool:windows/msvc_wrapper_for_nvcc.py", + wrapper_defines, + ) - _tpl( - repository_ctx, - "crosstool:CROSSTOOL", - cuda_defines + _get_win_cuda_defines(repository_ctx), - out="crosstool/CROSSTOOL", - ) - - # Set up cuda_config.h, which is used by - # tensorflow/stream_executor/dso_loader.cc. - _tpl( - repository_ctx, - "cuda:cuda_config.h", - { - "%{cuda_version}": - cuda_config.cuda_version, - "%{cudnn_version}": - cuda_config.cudnn_version, - "%{cuda_compute_capabilities}": - ",".join([ - "CudaVersion(\"%s\")" % c - for c in cuda_config.compute_capabilities - ],), - "%{cuda_toolkit_path}": - cuda_config.cuda_toolkit_path, - }, - "cuda/cuda/cuda_config.h", - ) + _tpl( + repository_ctx, + "crosstool:CROSSTOOL", + cuda_defines + _get_win_cuda_defines(repository_ctx), + out = "crosstool/CROSSTOOL", + ) + # Set up cuda_config.h, which is used by + # tensorflow/stream_executor/dso_loader.cc. + _tpl( + repository_ctx, + "cuda:cuda_config.h", + { + "%{cuda_version}": cuda_config.cuda_version, + "%{cudnn_version}": cuda_config.cudnn_version, + "%{cuda_compute_capabilities}": ",".join([ + "CudaVersion(\"%s\")" % c + for c in cuda_config.compute_capabilities + ]), + "%{cuda_toolkit_path}": cuda_config.cuda_toolkit_path, + }, + "cuda/cuda/cuda_config.h", + ) def _create_remote_cuda_repository(repository_ctx, remote_config_repo): - """Creates pointers to a remotely configured repo set up to build with CUDA.""" - _tpl( - repository_ctx, - "cuda:build_defs.bzl", - { - "%{cuda_is_configured}": - "True", - "%{cuda_extra_copts}": - _compute_cuda_extra_copts( - repository_ctx, - compute_capabilities(repository_ctx), - ), - }, - ) - repository_ctx.template( - "cuda/BUILD", - Label(remote_config_repo + "/cuda:BUILD"), - {}, - ) - repository_ctx.template( - "cuda/build_defs.bzl", - Label(remote_config_repo + "/cuda:build_defs.bzl"), - {}, - ) - repository_ctx.template( - "cuda/cuda/cuda_config.h", - Label(remote_config_repo + "/cuda:cuda/cuda_config.h"), - {}, - ) - - -def _cuda_autoconf_impl(repository_ctx): - """Implementation of the cuda_autoconf repository rule.""" - if not enable_cuda(repository_ctx): - _create_dummy_repository(repository_ctx) - elif _TF_CUDA_CONFIG_REPO in repository_ctx.os.environ: - _create_remote_cuda_repository( + """Creates pointers to a remotely configured repo set up to build with CUDA.""" + _tpl( repository_ctx, - repository_ctx.os.environ[_TF_CUDA_CONFIG_REPO], + "cuda:build_defs.bzl", + { + "%{cuda_is_configured}": "True", + "%{cuda_extra_copts}": _compute_cuda_extra_copts( + repository_ctx, + compute_capabilities(repository_ctx), + ), + }, + ) + repository_ctx.template( + "cuda/BUILD", + Label(remote_config_repo + "/cuda:BUILD"), + {}, + ) + repository_ctx.template( + "cuda/build_defs.bzl", + Label(remote_config_repo + "/cuda:build_defs.bzl"), + {}, + ) + repository_ctx.template( + "cuda/cuda/cuda_config.h", + Label(remote_config_repo + "/cuda:cuda/cuda_config.h"), + {}, ) - else: - _create_local_cuda_repository(repository_ctx) +def _cuda_autoconf_impl(repository_ctx): + """Implementation of the cuda_autoconf repository rule.""" + if not enable_cuda(repository_ctx): + _create_dummy_repository(repository_ctx) + elif _TF_CUDA_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_cuda_repository( + repository_ctx, + repository_ctx.os.environ[_TF_CUDA_CONFIG_REPO], + ) + else: + _create_local_cuda_repository(repository_ctx) cuda_configure = repository_rule( implementation = _cuda_autoconf_impl, diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl index fa6b942164556f..a874f1426c61de 100644 --- a/third_party/gpus/rocm_configure.bzl +++ b/third_party/gpus/rocm_configure.bzl @@ -242,11 +242,16 @@ def _hipcc_env(repository_ctx): A string containing environment variables for hipcc. """ hipcc_env = "" - for name in ["HIP_CLANG_PATH", "DEVICE_LIB_PATH", "HIP_VDI_HOME",\ - "HIPCC_VERBOSE", "HIPCC_COMPILE_FLAGS_APPEND"]: + for name in [ + "HIP_CLANG_PATH", + "DEVICE_LIB_PATH", + "HIP_VDI_HOME", + "HIPCC_VERBOSE", + "HIPCC_COMPILE_FLAGS_APPEND", + ]: if name in repository_ctx.os.environ: hipcc_env = hipcc_env + " " + name + "=\"" + \ - repository_ctx.os.environ[name].strip() + "\";" + repository_ctx.os.environ[name].strip() + "\";" return hipcc_env.strip() def _crosstool_verbose(repository_ctx): @@ -636,7 +641,6 @@ def _create_local_rocm_repository(repository_ctx): outs = rocm_lib_outs, )) - # Set up BUILD file for rocm/ _tpl( repository_ctx, diff --git a/third_party/nccl/system.BUILD.tpl b/third_party/nccl/system.BUILD.tpl index a07f54955fc5aa..970dddb1178460 100644 --- a/third_party/nccl/system.BUILD.tpl +++ b/third_party/nccl/system.BUILD.tpl @@ -1,26 +1,25 @@ filegroup( - name = "LICENSE", - visibility = ["//visibility:public"], + name = "LICENSE", + visibility = ["//visibility:public"], ) cc_library( - name = "nccl", - srcs = ["libnccl.so.%{version}"], - hdrs = ["nccl.h"], - include_prefix = "third_party/nccl", - deps = [ - "@local_config_cuda//cuda:cuda_headers", - ], - visibility = ["//visibility:public"], + name = "nccl", + srcs = ["libnccl.so.%{version}"], + hdrs = ["nccl.h"], + include_prefix = "third_party/nccl", + visibility = ["//visibility:public"], + deps = [ + "@local_config_cuda//cuda:cuda_headers", + ], ) genrule( - name = "nccl-files", - outs = [ - "libnccl.so.%{version}", - "nccl.h", - ], - cmd = """cp "%{hdr_path}/nccl.h" "$(@D)/nccl.h" && + name = "nccl-files", + outs = [ + "libnccl.so.%{version}", + "nccl.h", + ], + cmd = """cp "%{hdr_path}/nccl.h" "$(@D)/nccl.h" && cp "%{install_path}/libnccl.so.%{version}" "$(@D)/libnccl.so.%{version}" """, ) - diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 9a7581c246d9d4..d1b1e5949211cc 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -11,300 +11,337 @@ _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" _TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO" - -def _tpl(repository_ctx, tpl, substitutions={}, out=None): - if not out: - out = tpl - repository_ctx.template( - out, - Label("//third_party/py:%s.tpl" % tpl), - substitutions) - +def _tpl(repository_ctx, tpl, substitutions = {}, out = None): + if not out: + out = tpl + repository_ctx.template( + out, + Label("//third_party/py:%s.tpl" % tpl), + substitutions, + ) def _fail(msg): - """Output failure message when auto configuration fails.""" - red = "\033[0;31m" - no_color = "\033[0m" - fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) - + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) def _is_windows(repository_ctx): - """Returns true if the host operating system is windows.""" - os_name = repository_ctx.os.name.lower() - if os_name.find("windows") != -1: - return True - return False - - -def _execute(repository_ctx, cmdline, error_msg=None, error_details=None, - empty_stdout_fine=False): - """Executes an arbitrary shell command. - - Args: - repository_ctx: the repository_ctx object - cmdline: list of strings, the command to execute - error_msg: string, a summary of the error if the command fails - error_details: string, details about the error or steps to fix it - empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise - it's an error - Return: - the result of repository_ctx.execute(cmdline) - """ - result = repository_ctx.execute(cmdline) - if result.stderr or not (empty_stdout_fine or result.stdout): - _fail("\n".join([ - error_msg.strip() if error_msg else "Repository command failed", - result.stderr.strip(), - error_details if error_details else ""])) - return result - + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + if os_name.find("windows") != -1: + return True + return False + +def _execute( + repository_ctx, + cmdline, + error_msg = None, + error_details = None, + empty_stdout_fine = False): + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else "", + ])) + return result def _read_dir(repository_ctx, src_dir): - """Returns a string with all files in a directory. - - Finds all files inside a directory, traversing subfolders and following - symlinks. The returned string contains the full path of all files - separated by line breaks. - """ - if _is_windows(repository_ctx): - src_dir = src_dir.replace("/", "\\") - find_result = _execute( - repository_ctx, ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], - empty_stdout_fine=True) - # src_files will be used in genrule.outs where the paths must - # use forward slashes. - result = find_result.stdout.replace("\\", "/") - else: - find_result = _execute( - repository_ctx, ["find", src_dir, "-follow", "-type", "f"], - empty_stdout_fine=True) - result = find_result.stdout - return result - + """Returns a string with all files in a directory. + + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine = True, + ) + + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + result = find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, + ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine = True, + ) + result = find_result.stdout + return result def _genrule(src_dir, genrule_name, command, outs): - """Returns a string with a genrule. - - Genrule executes the given command and produces the given outputs. - """ - return ( - 'genrule(\n' + - ' name = "' + - genrule_name + '",\n' + - ' outs = [\n' + - outs + - '\n ],\n' + - ' cmd = """\n' + - command + - '\n """,\n' + - ')\n' - ) - + """Returns a string with a genrule. + + Genrule executes the given command and produces the given outputs. + """ + return ( + "genrule(\n" + + ' name = "' + + genrule_name + '",\n' + + " outs = [\n" + + outs + + "\n ],\n" + + ' cmd = """\n' + + command + + '\n """,\n' + + ")\n" + ) def _norm_path(path): - """Returns a path with '/' and remove the trailing slash.""" - path = path.replace("\\", "/") - if path[-1] == "/": - path = path[:-1] - return path - - -def _symlink_genrule_for_dir(repository_ctx, src_dir, dest_dir, genrule_name, - src_files = [], dest_files = []): - """Returns a genrule to symlink(or copy if on Windows) a set of files. - - If src_dir is passed, files will be read from the given directory; otherwise - we assume files are in src_files and dest_files - """ - if src_dir != None: - src_dir = _norm_path(src_dir) - dest_dir = _norm_path(dest_dir) - files = '\n'.join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) - # Create a list with the src_dir stripped to use for outputs. - dest_files = files.replace(src_dir, '').splitlines() - src_files = files.splitlines() - command = [] - outs = [] - for i in range(len(dest_files)): - if dest_files[i] != "": - # If we have only one file to link we do not want to use the dest_dir, as - # $(@D) will include the full path to the file. - dest = '$(@D)/' + dest_dir + dest_files[i] if len(dest_files) != 1 else '$(@D)/' + dest_files[i] - # Copy the headers to create a sandboxable setup. - cmd = 'cp -f' - command.append(cmd + ' "%s" "%s"' % (src_files[i] , dest)) - outs.append(' "' + dest_dir + dest_files[i] + '",') - genrule = _genrule(src_dir, genrule_name, " && ".join(command), - "\n".join(outs)) - return genrule - + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + +def _symlink_genrule_for_dir( + repository_ctx, + src_dir, + dest_dir, + genrule_name, + src_files = [], + dest_files = []): + """Returns a genrule to symlink(or copy if on Windows) a set of files. + + If src_dir is passed, files will be read from the given directory; otherwise + we assume files are in src_files and dest_files + """ + if src_dir != None: + src_dir = _norm_path(src_dir) + dest_dir = _norm_path(dest_dir) + files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) + + # Create a list with the src_dir stripped to use for outputs. + dest_files = files.replace(src_dir, "").splitlines() + src_files = files.splitlines() + command = [] + outs = [] + for i in range(len(dest_files)): + if dest_files[i] != "": + # If we have only one file to link we do not want to use the dest_dir, as + # $(@D) will include the full path to the file. + dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i] + + # Copy the headers to create a sandboxable setup. + cmd = "cp -f" + command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) + outs.append(' "' + dest_dir + dest_files[i] + '",') + genrule = _genrule( + src_dir, + genrule_name, + " && ".join(command), + "\n".join(outs), + ) + return genrule def _get_python_bin(repository_ctx): - """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_path = repository_ctx.which("python") - if python_bin_path != None: - return str(python_bin_path) - _fail("Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or --define " + - "%s='/something/else'.\nPATH=%s" % ( - _PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) - + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = repository_ctx.which("python") + if python_bin_path != None: + return str(python_bin_path) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % ( + _PYTHON_BIN_PATH, + repository_ctx.os.environ.get("PATH", ""), + )) def _get_bash_bin(repository_ctx): - """Gets the bash bin path.""" - bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) - if bash_bin != None: - return bash_bin - else: - bash_bin_path = repository_ctx.which("bash") - if bash_bin_path != None: - return str(bash_bin_path) + """Gets the bash bin path.""" + bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) + if bash_bin != None: + return bash_bin else: - _fail("Cannot find bash in PATH, please make sure " + - "bash is installed and add its directory in PATH, or --define " + - "%s='/path/to/bash'.\nPATH=%s" % ( - _BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) - + bash_bin_path = repository_ctx.which("bash") + if bash_bin_path != None: + return str(bash_bin_path) + else: + _fail("Cannot find bash in PATH, please make sure " + + "bash is installed and add its directory in PATH, or --define " + + "%s='/path/to/bash'.\nPATH=%s" % ( + _BAZEL_SH, + repository_ctx.os.environ.get("PATH", ""), + )) def _get_python_lib(repository_ctx, python_bin): - """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) - if python_lib != None: - return python_lib - print_lib = ("<=1:\n" + - " print(paths[0])\n" + - "END") - cmd = '%s - %s' % (python_bin, print_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - return result.stdout.strip('\n') - + """Gets the python lib path.""" + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib != None: + return python_lib + print_lib = ("<=1:\n" + + " print(paths[0])\n" + + "END") + cmd = "%s - %s" % (python_bin, print_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + return result.stdout.strip("\n") def _check_python_lib(repository_ctx, python_lib): - """Checks the python lib path.""" - cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - if result.return_code == 1: - _fail("Invalid python library path: %s" % python_lib) - + """Checks the python lib path.""" + cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("Invalid python library path: %s" % python_lib) def _check_python_bin(repository_ctx, python_bin): - """Checks the python bin path.""" - cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - if result.return_code == 1: - _fail("--define %s='%s' is not executable. Is it the python binary?" % ( - _PYTHON_BIN_PATH, python_bin)) - + """Checks the python bin path.""" + cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("--define %s='%s' is not executable. Is it the python binary?" % ( + _PYTHON_BIN_PATH, + python_bin, + )) def _get_python_include(repository_ctx, python_bin): - """Gets the python include path.""" - result = _execute( - repository_ctx, - [python_bin, "-c", - 'from __future__ import print_function;' + - 'from distutils import sysconfig;' + - 'print(sysconfig.get_python_inc())'], - error_msg="Problem getting python include path.", - error_details=("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) " + - "Is distutils installed?")) - return result.stdout.splitlines()[0] - + """Gets the python include path.""" + result = _execute( + repository_ctx, + [ + python_bin, + "-c", + "from __future__ import print_function;" + + "from distutils import sysconfig;" + + "print(sysconfig.get_python_inc())", + ], + error_msg = "Problem getting python include path.", + error_details = ("Is the Python binary path set up right? " + + "(See ./configure or " + _PYTHON_BIN_PATH + ".) " + + "Is distutils installed?"), + ) + return result.stdout.splitlines()[0] def _get_python_import_lib_name(repository_ctx, python_bin): - """Get Python import library name (pythonXY.lib) on Windows.""" - result = _execute( - repository_ctx, - [python_bin, "-c", - 'import sys;' + - 'print("python" + str(sys.version_info[0]) + ' + - ' str(sys.version_info[1]) + ".lib")'], - error_msg="Problem getting python import library.", - error_details=("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) - return result.stdout.splitlines()[0] - + """Get Python import library name (pythonXY.lib) on Windows.""" + result = _execute( + repository_ctx, + [ + python_bin, + "-c", + "import sys;" + + 'print("python" + str(sys.version_info[0]) + ' + + ' str(sys.version_info[1]) + ".lib")', + ], + error_msg = "Problem getting python import library.", + error_details = ("Is the Python binary path set up right? " + + "(See ./configure or " + _PYTHON_BIN_PATH + ".) "), + ) + return result.stdout.splitlines()[0] def _get_numpy_include(repository_ctx, python_bin): - """Gets the numpy include path.""" - return _execute(repository_ctx, - [python_bin, "-c", - 'from __future__ import print_function;' + - 'import numpy;' + - ' print(numpy.get_include());'], - error_msg="Problem getting numpy include path.", - error_details="Is numpy installed?").stdout.splitlines()[0] - + """Gets the numpy include path.""" + return _execute( + repository_ctx, + [ + python_bin, + "-c", + "from __future__ import print_function;" + + "import numpy;" + + " print(numpy.get_include());", + ], + error_msg = "Problem getting numpy include path.", + error_details = "Is numpy installed?", + ).stdout.splitlines()[0] def _create_local_python_repository(repository_ctx): - """Creates the repository containing files set up to build with Python.""" - python_bin = _get_python_bin(repository_ctx) - _check_python_bin(repository_ctx, python_bin) - python_lib = _get_python_lib(repository_ctx, python_bin) - _check_python_lib(repository_ctx, python_lib) - python_include = _get_python_include(repository_ctx, python_bin) - numpy_include = _get_numpy_include(repository_ctx, python_bin) + '/numpy' - python_include_rule = _symlink_genrule_for_dir( - repository_ctx, python_include, 'python_include', 'python_include') - python_import_lib_genrule = "" - # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib - # See https://docs.python.org/3/extending/windows.html - if _is_windows(repository_ctx): - python_include = _norm_path(python_include) - python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin) - python_import_lib_src = python_include.rsplit('/', 1)[0] + "/libs/" + python_import_lib_name - python_import_lib_genrule = _symlink_genrule_for_dir( - repository_ctx, None, '', 'python_import_lib', - [python_import_lib_src], [python_import_lib_name]) - numpy_include_rule = _symlink_genrule_for_dir( - repository_ctx, numpy_include, 'numpy_include/numpy', 'numpy_include') - _tpl(repository_ctx, "BUILD", { - "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, - "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, - "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, - }) - + """Creates the repository containing files set up to build with Python.""" + python_bin = _get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) + python_lib = _get_python_lib(repository_ctx, python_bin) + _check_python_lib(repository_ctx, python_lib) + python_include = _get_python_include(repository_ctx, python_bin) + numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy" + python_include_rule = _symlink_genrule_for_dir( + repository_ctx, + python_include, + "python_include", + "python_include", + ) + python_import_lib_genrule = "" + + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + if _is_windows(repository_ctx): + python_include = _norm_path(python_include) + python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin) + python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name + python_import_lib_genrule = _symlink_genrule_for_dir( + repository_ctx, + None, + "", + "python_import_lib", + [python_import_lib_src], + [python_import_lib_name], + ) + numpy_include_rule = _symlink_genrule_for_dir( + repository_ctx, + numpy_include, + "numpy_include/numpy", + "numpy_include", + ) + _tpl(repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, + "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, + "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, + }) def _create_remote_python_repository(repository_ctx, remote_config_repo): - """Creates pointers to a remotely configured repo set up to build with Python. - """ - repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {}) - + """Creates pointers to a remotely configured repo set up to build with Python. + """ + repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {}) def _python_autoconf_impl(repository_ctx): - """Implementation of the python_autoconf repository rule.""" - if _TF_PYTHON_CONFIG_REPO in repository_ctx.os.environ: - _create_remote_python_repository(repository_ctx, - repository_ctx.os.environ[_TF_PYTHON_CONFIG_REPO]) - else: - _create_local_python_repository(repository_ctx) - + """Implementation of the python_autoconf repository rule.""" + if _TF_PYTHON_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_python_repository( + repository_ctx, + repository_ctx.os.environ[_TF_PYTHON_CONFIG_REPO], + ) + else: + _create_local_python_repository(repository_ctx) python_configure = repository_rule( implementation = _python_autoconf_impl, diff --git a/third_party/sycl/sycl/build_defs.bzl.tpl b/third_party/sycl/sycl/build_defs.bzl.tpl index 33386f8957c821..a726c8d953cb56 100755 --- a/third_party/sycl/sycl/build_defs.bzl.tpl +++ b/third_party/sycl/sycl/build_defs.bzl.tpl @@ -11,7 +11,7 @@ def if_sycl(if_true, if_false = []): return select({ "@local_config_sycl//sycl:using_sycl_ccpp": if_true, "@local_config_sycl//sycl:using_sycl_trisycl": if_true[0:1], - "//conditions:default": if_false + "//conditions:default": if_false, }) def if_ccpp(if_true, if_false = []): @@ -24,5 +24,5 @@ def if_ccpp(if_true, if_false = []): return select({ "@local_config_sycl//sycl:using_sycl_ccpp": if_true, "@local_config_sycl//sycl:using_sycl_trisycl": if_false, - "//conditions:default": if_false + "//conditions:default": if_false, }) diff --git a/third_party/sycl/sycl_configure.bzl b/third_party/sycl/sycl_configure.bzl index 5b9d0eb383d1b0..deba6c4116e763 100644 --- a/third_party/sycl/sycl_configure.bzl +++ b/third_party/sycl/sycl_configure.bzl @@ -11,122 +11,124 @@ """ _HOST_CXX_COMPILER = "HOST_CXX_COMPILER" -_HOST_C_COMPILER= "HOST_C_COMPILER" +_HOST_C_COMPILER = "HOST_C_COMPILER" _COMPUTECPP_TOOLKIT_PATH = "COMPUTECPP_TOOLKIT_PATH" _TRISYCL_INCLUDE_DIR = "TRISYCL_INCLUDE_DIR" _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" def _enable_sycl(repository_ctx): - if "TF_NEED_OPENCL_SYCL" in repository_ctx.os.environ: - enable_sycl = repository_ctx.os.environ["TF_NEED_OPENCL_SYCL"].strip() - return enable_sycl == "1" - return False + if "TF_NEED_OPENCL_SYCL" in repository_ctx.os.environ: + enable_sycl = repository_ctx.os.environ["TF_NEED_OPENCL_SYCL"].strip() + return enable_sycl == "1" + return False def _enable_compute_cpp(repository_ctx): - return _COMPUTECPP_TOOLKIT_PATH in repository_ctx.os.environ + return _COMPUTECPP_TOOLKIT_PATH in repository_ctx.os.environ def auto_configure_fail(msg): - """Output failure message when auto configuration fails.""" - red = "\033[0;31m" - no_color = "\033[0m" - fail("\n%sAuto-Configuration Error:%s %s\n" % (red, no_color, msg)) + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("\n%sAuto-Configuration Error:%s %s\n" % (red, no_color, msg)) + # END cc_configure common functions (see TODO above). def find_c(repository_ctx): - """Find host C compiler.""" - c_name = "gcc" - if _HOST_C_COMPILER in repository_ctx.os.environ: - c_name = repository_ctx.os.environ[_HOST_C_COMPILER].strip() - if c_name.startswith("/"): - return c_name - c = repository_ctx.which(c_name) - if c == None: - fail("Cannot find C compiler, please correct your path.") - return c + """Find host C compiler.""" + c_name = "gcc" + if _HOST_C_COMPILER in repository_ctx.os.environ: + c_name = repository_ctx.os.environ[_HOST_C_COMPILER].strip() + if c_name.startswith("/"): + return c_name + c = repository_ctx.which(c_name) + if c == None: + fail("Cannot find C compiler, please correct your path.") + return c def find_cc(repository_ctx): - """Find host C++ compiler.""" - cc_name = "g++" - if _HOST_CXX_COMPILER in repository_ctx.os.environ: - cc_name = repository_ctx.os.environ[_HOST_CXX_COMPILER].strip() - if cc_name.startswith("/"): - return cc_name - cc = repository_ctx.which(cc_name) - if cc == None: - fail("Cannot find C++ compiler, please correct your path.") - return cc + """Find host C++ compiler.""" + cc_name = "g++" + if _HOST_CXX_COMPILER in repository_ctx.os.environ: + cc_name = repository_ctx.os.environ[_HOST_CXX_COMPILER].strip() + if cc_name.startswith("/"): + return cc_name + cc = repository_ctx.which(cc_name) + if cc == None: + fail("Cannot find C++ compiler, please correct your path.") + return cc def find_computecpp_root(repository_ctx): - """Find ComputeCpp compiler.""" - sycl_name = "" - if _COMPUTECPP_TOOLKIT_PATH in repository_ctx.os.environ: - sycl_name = repository_ctx.os.environ[_COMPUTECPP_TOOLKIT_PATH].strip() - if sycl_name.startswith("/"): - return sycl_name - fail("Cannot find SYCL compiler, please correct your path") + """Find ComputeCpp compiler.""" + sycl_name = "" + if _COMPUTECPP_TOOLKIT_PATH in repository_ctx.os.environ: + sycl_name = repository_ctx.os.environ[_COMPUTECPP_TOOLKIT_PATH].strip() + if sycl_name.startswith("/"): + return sycl_name + fail("Cannot find SYCL compiler, please correct your path") def find_trisycl_include_dir(repository_ctx): - """Find triSYCL include directory. """ - if _TRISYCL_INCLUDE_DIR in repository_ctx.os.environ: - sycl_name = repository_ctx.os.environ[_TRISYCL_INCLUDE_DIR].strip() - if sycl_name.startswith("/"): - return sycl_name - fail( "Cannot find triSYCL include directory, please correct your path") + """Find triSYCL include directory. """ + if _TRISYCL_INCLUDE_DIR in repository_ctx.os.environ: + sycl_name = repository_ctx.os.environ[_TRISYCL_INCLUDE_DIR].strip() + if sycl_name.startswith("/"): + return sycl_name + fail("Cannot find triSYCL include directory, please correct your path") def find_python_lib(repository_ctx): - """Returns python path.""" - if _PYTHON_LIB_PATH in repository_ctx.os.environ: - return repository_ctx.os.environ[_PYTHON_LIB_PATH].strip() - fail("Environment variable PYTHON_LIB_PATH was not specified re-run ./configure") - + """Returns python path.""" + if _PYTHON_LIB_PATH in repository_ctx.os.environ: + return repository_ctx.os.environ[_PYTHON_LIB_PATH].strip() + fail("Environment variable PYTHON_LIB_PATH was not specified re-run ./configure") def _check_lib(repository_ctx, toolkit_path, lib): - """Checks if lib exists under sycl_toolkit_path or fail if it doesn't. + """Checks if lib exists under sycl_toolkit_path or fail if it doesn't. - Args: - repository_ctx: The repository context. - toolkit_path: The toolkit directory containing the libraries. - ib: The library to look for under toolkit_path. - """ - lib_path = toolkit_path + "/" + lib - if not repository_ctx.path(lib_path).exists: - auto_configure_fail("Cannot find %s" % lib_path) + Args: + repository_ctx: The repository context. + toolkit_path: The toolkit directory containing the libraries. + ib: The library to look for under toolkit_path. + """ + lib_path = toolkit_path + "/" + lib + if not repository_ctx.path(lib_path).exists: + auto_configure_fail("Cannot find %s" % lib_path) def _check_dir(repository_ctx, directory): - """Checks whether the directory exists and fail if it does not. + """Checks whether the directory exists and fail if it does not. - Args: - repository_ctx: The repository context. - directory: The directory to check the existence of. - """ - if not repository_ctx.path(directory).exists: - auto_configure_fail("Cannot find dir: %s" % directory) + Args: + repository_ctx: The repository context. + directory: The directory to check the existence of. + """ + if not repository_ctx.path(directory).exists: + auto_configure_fail("Cannot find dir: %s" % directory) def _symlink_dir(repository_ctx, src_dir, dest_dir): - """Symlinks all the files in a directory. - - Args: - repository_ctx: The repository context. - src_dir: The source directory. - dest_dir: The destination directory to create the symlinks in. - """ - files = repository_ctx.path(src_dir).readdir() - for src_file in files: - repository_ctx.symlink(src_file, dest_dir + "/" + src_file.basename) - -def _tpl(repository_ctx, tpl, substitutions={}, out=None): - if not out: - out = tpl.replace(":", "/") - repository_ctx.template( - out, - Label("//third_party/sycl/%s.tpl" % tpl), - substitutions) + """Symlinks all the files in a directory. + + Args: + repository_ctx: The repository context. + src_dir: The source directory. + dest_dir: The destination directory to create the symlinks in. + """ + files = repository_ctx.path(src_dir).readdir() + for src_file in files: + repository_ctx.symlink(src_file, dest_dir + "/" + src_file.basename) + +def _tpl(repository_ctx, tpl, substitutions = {}, out = None): + if not out: + out = tpl.replace(":", "/") + repository_ctx.template( + out, + Label("//third_party/sycl/%s.tpl" % tpl), + substitutions, + ) def _file(repository_ctx, label): - repository_ctx.template( - label.replace(":", "/"), - Label("//third_party/sycl/%s" % label), - {}) + repository_ctx.template( + label.replace(":", "/"), + Label("//third_party/sycl/%s" % label), + {}, + ) _DUMMY_CROSSTOOL_BZL_FILE = """ def error_sycl_disabled(): @@ -147,7 +149,6 @@ def error_sycl_disabled(): ) """ - _DUMMY_CROSSTOOL_BUILD_FILE = """ load("//crosstool:error_sycl_disabled.bzl", "error_sycl_disabled") @@ -155,87 +156,97 @@ error_sycl_disabled() """ def _create_dummy_repository(repository_ctx): - # Set up BUILD file for sycl/. - _tpl(repository_ctx, "sycl:build_defs.bzl") - _tpl(repository_ctx, "sycl:BUILD") - _file(repository_ctx, "sycl:LICENSE.text") - _tpl(repository_ctx, "sycl:platform.bzl") - - # Create dummy files for the SYCL toolkit since they are still required by - # tensorflow/sycl/platform/default/build_config:sycl. - repository_ctx.file("sycl/include/sycl.hpp", "") - repository_ctx.file("sycl/lib/libComputeCpp.so", "") - - # If sycl_configure is not configured to build with SYCL support, and the user - # attempts to build with --config=sycl, add a dummy build rule to intercept - # this and fail with an actionable error message. - repository_ctx.file("crosstool/error_sycl_disabled.bzl", - _DUMMY_CROSSTOOL_BZL_FILE) - repository_ctx.file("crosstool/BUILD", _DUMMY_CROSSTOOL_BUILD_FILE) - - -def _sycl_autoconf_imp(repository_ctx): - """Implementation of the sycl_autoconf rule.""" - if not _enable_sycl(repository_ctx): - _create_dummy_repository(repository_ctx) - else: - # copy template files + # Set up BUILD file for sycl/. _tpl(repository_ctx, "sycl:build_defs.bzl") _tpl(repository_ctx, "sycl:BUILD") - _tpl(repository_ctx, "sycl:platform.bzl") - _tpl(repository_ctx, "crosstool:BUILD") _file(repository_ctx, "sycl:LICENSE.text") + _tpl(repository_ctx, "sycl:platform.bzl") - if _enable_compute_cpp(repository_ctx): - _tpl(repository_ctx, "crosstool:computecpp", - { - "%{host_cxx_compiler}" : find_cc(repository_ctx), - "%{host_c_compiler}" : find_c(repository_ctx) - }) - - computecpp_root = find_computecpp_root(repository_ctx); - _check_dir(repository_ctx, computecpp_root) - - _tpl(repository_ctx, "crosstool:CROSSTOOL", - { - "%{sycl_include_dir}" : computecpp_root, - "%{sycl_impl}" : "computecpp", - "%{c++_std}" : "-std=c++11", - "%{python_lib_path}" : find_python_lib(repository_ctx), - }) - - # symlink libraries - _check_lib(repository_ctx, computecpp_root+"/lib", "libComputeCpp.so" ) - _symlink_dir(repository_ctx, computecpp_root + "/lib", "sycl/lib") - _symlink_dir(repository_ctx, computecpp_root + "/include", "sycl/include") - _symlink_dir(repository_ctx, computecpp_root + "/bin", "sycl/bin") - else: - - trisycl_include_dir = find_trisycl_include_dir(repository_ctx); - _check_dir(repository_ctx, trisycl_include_dir) - - _tpl(repository_ctx, "crosstool:trisycl", - { - "%{host_cxx_compiler}" : find_cc(repository_ctx), - "%{host_c_compiler}" : find_c(repository_ctx), - "%{trisycl_include_dir}" : trisycl_include_dir - }) - - - _tpl(repository_ctx, "crosstool:CROSSTOOL", - { - "%{sycl_include_dir}" : trisycl_include_dir, - "%{sycl_impl}" : "trisycl", - "%{c++_std}" : "-std=c++1y", - "%{python_lib_path}" : find_python_lib(repository_ctx), - }) + # Create dummy files for the SYCL toolkit since they are still required by + # tensorflow/sycl/platform/default/build_config:sycl. + repository_ctx.file("sycl/include/sycl.hpp", "") + repository_ctx.file("sycl/lib/libComputeCpp.so", "") - _symlink_dir(repository_ctx, trisycl_include_dir, "sycl/include") + # If sycl_configure is not configured to build with SYCL support, and the user + # attempts to build with --config=sycl, add a dummy build rule to intercept + # this and fail with an actionable error message. + repository_ctx.file( + "crosstool/error_sycl_disabled.bzl", + _DUMMY_CROSSTOOL_BZL_FILE, + ) + repository_ctx.file("crosstool/BUILD", _DUMMY_CROSSTOOL_BUILD_FILE) +def _sycl_autoconf_imp(repository_ctx): + """Implementation of the sycl_autoconf rule.""" + if not _enable_sycl(repository_ctx): + _create_dummy_repository(repository_ctx) + else: + # copy template files + _tpl(repository_ctx, "sycl:build_defs.bzl") + _tpl(repository_ctx, "sycl:BUILD") + _tpl(repository_ctx, "sycl:platform.bzl") + _tpl(repository_ctx, "crosstool:BUILD") + _file(repository_ctx, "sycl:LICENSE.text") + + if _enable_compute_cpp(repository_ctx): + _tpl( + repository_ctx, + "crosstool:computecpp", + { + "%{host_cxx_compiler}": find_cc(repository_ctx), + "%{host_c_compiler}": find_c(repository_ctx), + }, + ) + + computecpp_root = find_computecpp_root(repository_ctx) + _check_dir(repository_ctx, computecpp_root) + + _tpl( + repository_ctx, + "crosstool:CROSSTOOL", + { + "%{sycl_include_dir}": computecpp_root, + "%{sycl_impl}": "computecpp", + "%{c++_std}": "-std=c++11", + "%{python_lib_path}": find_python_lib(repository_ctx), + }, + ) + + # symlink libraries + _check_lib(repository_ctx, computecpp_root + "/lib", "libComputeCpp.so") + _symlink_dir(repository_ctx, computecpp_root + "/lib", "sycl/lib") + _symlink_dir(repository_ctx, computecpp_root + "/include", "sycl/include") + _symlink_dir(repository_ctx, computecpp_root + "/bin", "sycl/bin") + else: + trisycl_include_dir = find_trisycl_include_dir(repository_ctx) + _check_dir(repository_ctx, trisycl_include_dir) + + _tpl( + repository_ctx, + "crosstool:trisycl", + { + "%{host_cxx_compiler}": find_cc(repository_ctx), + "%{host_c_compiler}": find_c(repository_ctx), + "%{trisycl_include_dir}": trisycl_include_dir, + }, + ) + + _tpl( + repository_ctx, + "crosstool:CROSSTOOL", + { + "%{sycl_include_dir}": trisycl_include_dir, + "%{sycl_impl}": "trisycl", + "%{c++_std}": "-std=c++1y", + "%{python_lib_path}": find_python_lib(repository_ctx), + }, + ) + + _symlink_dir(repository_ctx, trisycl_include_dir, "sycl/include") sycl_configure = repository_rule( - implementation = _sycl_autoconf_imp, - local = True, + implementation = _sycl_autoconf_imp, + local = True, ) """Detects and configures the SYCL toolchain. diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl index c6de25b33e3fab..1d019a1b4f531d 100644 --- a/third_party/tensorrt/tensorrt_configure.bzl +++ b/third_party/tensorrt/tensorrt_configure.bzl @@ -10,13 +10,13 @@ load( "//third_party/gpus:cuda_configure.bzl", "auto_configure_fail", - "get_cpu_value", "find_cuda_define", "find_lib", + "get_cpu_value", "lib_name", - "matches_version", "make_copy_dir_rule", "make_copy_files_rule", + "matches_version", ) _TENSORRT_INSTALL_PATH = "TENSORRT_INSTALL_PATH" @@ -30,185 +30,200 @@ _DEFINE_TENSORRT_SONAME_MAJOR = "#define NV_TENSORRT_SONAME_MAJOR" _DEFINE_TENSORRT_SONAME_MINOR = "#define NV_TENSORRT_SONAME_MINOR" _DEFINE_TENSORRT_SONAME_PATCH = "#define NV_TENSORRT_SONAME_PATCH" - def _headers_exist(repository_ctx, path): - """Returns whether all TensorRT header files could be found in 'path'. - - Args: - repository_ctx: The repository context. - path: The TensorRT include path to check. + """Returns whether all TensorRT header files could be found in 'path'. - Returns: - True if all TensorRT header files can be found in the path. - """ - for h in _TF_TENSORRT_HEADERS: - if not repository_ctx.path("%s/%s" % (path, h)).exists: - return False - return True + Args: + repository_ctx: The repository context. + path: The TensorRT include path to check. + Returns: + True if all TensorRT header files can be found in the path. + """ + for h in _TF_TENSORRT_HEADERS: + if not repository_ctx.path("%s/%s" % (path, h)).exists: + return False + return True def _find_trt_header_dir(repository_ctx, trt_install_path): - """Returns the path to the directory containing headers of TensorRT. - - Args: - repository_ctx: The repository context. - trt_install_path: The TensorRT library install directory. - - Returns: - The path of the directory containing the TensorRT header. - """ - if trt_install_path == "/usr/lib/x86_64-linux-gnu": - path = "/usr/include/x86_64-linux-gnu" - if _headers_exist(repository_ctx, path): - return path - if trt_install_path == "/usr/lib/aarch64-linux-gnu": - path = "/usr/include/aarch64-linux-gnu" + """Returns the path to the directory containing headers of TensorRT. + + Args: + repository_ctx: The repository context. + trt_install_path: The TensorRT library install directory. + + Returns: + The path of the directory containing the TensorRT header. + """ + if trt_install_path == "/usr/lib/x86_64-linux-gnu": + path = "/usr/include/x86_64-linux-gnu" + if _headers_exist(repository_ctx, path): + return path + if trt_install_path == "/usr/lib/aarch64-linux-gnu": + path = "/usr/include/aarch64-linux-gnu" + if _headers_exist(repository_ctx, path): + return path + path = str(repository_ctx.path("%s/../include" % trt_install_path).realpath) if _headers_exist(repository_ctx, path): - return path - path = str(repository_ctx.path("%s/../include" % trt_install_path).realpath) - if _headers_exist(repository_ctx, path): - return path - auto_configure_fail( - "Cannot find NvInfer.h with TensorRT install path %s" % trt_install_path) - - -def _trt_lib_version(repository_ctx, trt_install_path): - """Detects the library (e.g. libnvinfer) version of TensorRT. - - Args: - repository_ctx: The repository context. - trt_install_path: The TensorRT library install directory. - - Returns: - A string containing the library version of TensorRT. - """ - trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path) - major_version = find_cuda_define(repository_ctx, trt_header_dir, "NvInfer.h", - _DEFINE_TENSORRT_SONAME_MAJOR) - minor_version = find_cuda_define(repository_ctx, trt_header_dir, "NvInfer.h", - _DEFINE_TENSORRT_SONAME_MINOR) - patch_version = find_cuda_define(repository_ctx, trt_header_dir, "NvInfer.h", - _DEFINE_TENSORRT_SONAME_PATCH) - full_version = "%s.%s.%s" % (major_version, minor_version, patch_version) - environ_version = repository_ctx.os.environ[_TF_TENSORRT_VERSION].strip() - if not matches_version(environ_version, full_version): + return path auto_configure_fail( - ("TensorRT library version detected from %s/%s (%s) does not match " + - "TF_TENSORRT_VERSION (%s). To fix this rerun configure again.") % - (trt_header_dir, "NvInfer.h", full_version, environ_version)) - # Only use the major version to match the SONAME of the library. - return major_version + "Cannot find NvInfer.h with TensorRT install path %s" % trt_install_path, + ) +def _trt_lib_version(repository_ctx, trt_install_path): + """Detects the library (e.g. libnvinfer) version of TensorRT. + + Args: + repository_ctx: The repository context. + trt_install_path: The TensorRT library install directory. + + Returns: + A string containing the library version of TensorRT. + """ + trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path) + major_version = find_cuda_define( + repository_ctx, + trt_header_dir, + "NvInfer.h", + _DEFINE_TENSORRT_SONAME_MAJOR, + ) + minor_version = find_cuda_define( + repository_ctx, + trt_header_dir, + "NvInfer.h", + _DEFINE_TENSORRT_SONAME_MINOR, + ) + patch_version = find_cuda_define( + repository_ctx, + trt_header_dir, + "NvInfer.h", + _DEFINE_TENSORRT_SONAME_PATCH, + ) + full_version = "%s.%s.%s" % (major_version, minor_version, patch_version) + environ_version = repository_ctx.os.environ[_TF_TENSORRT_VERSION].strip() + if not matches_version(environ_version, full_version): + auto_configure_fail( + ("TensorRT library version detected from %s/%s (%s) does not match " + + "TF_TENSORRT_VERSION (%s). To fix this rerun configure again.") % + (trt_header_dir, "NvInfer.h", full_version, environ_version), + ) + + # Only use the major version to match the SONAME of the library. + return major_version def _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version): - """Finds the given TensorRT library on the system. - - Adapted from code contributed by Sami Kama (https://github.com/samikama). - - Args: - repository_ctx: The repository context. - trt_install_path: The TensorRT library installation directory. - trt_lib_version: The version of TensorRT library files as returned - by _trt_lib_version. - - Returns: - The path to the library. - """ - result = {} - for lib in _TF_TENSORRT_LIBS: - file_name = lib_name("nvinfer", cpu_value, trt_lib_version) - path = find_lib(repository_ctx, ["%s/%s" % (trt_install_path, file_name)]) - result[file_name] = path - return result - + """Finds the given TensorRT library on the system. + + Adapted from code contributed by Sami Kama (https://github.com/samikama). + + Args: + repository_ctx: The repository context. + trt_install_path: The TensorRT library installation directory. + trt_lib_version: The version of TensorRT library files as returned + by _trt_lib_version. + + Returns: + The path to the library. + """ + result = {} + for lib in _TF_TENSORRT_LIBS: + file_name = lib_name("nvinfer", cpu_value, trt_lib_version) + path = find_lib(repository_ctx, ["%s/%s" % (trt_install_path, file_name)]) + result[file_name] = path + return result def _tpl(repository_ctx, tpl, substitutions): - repository_ctx.template(tpl, Label("//third_party/tensorrt:%s.tpl" % tpl), - substitutions) - + repository_ctx.template( + tpl, + Label("//third_party/tensorrt:%s.tpl" % tpl), + substitutions, + ) def _create_dummy_repository(repository_ctx): - """Create a dummy TensorRT repository.""" - _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"}) + """Create a dummy TensorRT repository.""" + _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"}) - _tpl(repository_ctx, "BUILD", { - "%{tensorrt_genrules}": "", - "%{tensorrt_headers}": "[]", - "%{tensorrt_libs}": "[]" - }) + _tpl(repository_ctx, "BUILD", { + "%{tensorrt_genrules}": "", + "%{tensorrt_headers}": "[]", + "%{tensorrt_libs}": "[]", + }) def _tensorrt_configure_impl(repository_ctx): - """Implementation of the tensorrt_configure repository rule.""" - if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ: - # Forward to the pre-configured remote repository. - remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO] - repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {}) - repository_ctx.template( - "build_defs.bzl", - Label(remote_config_repo + ":build_defs.bzl"), - {}, + """Implementation of the tensorrt_configure repository rule.""" + if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ: + # Forward to the pre-configured remote repository. + remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO] + repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {}) + repository_ctx.template( + "build_defs.bzl", + Label(remote_config_repo + ":build_defs.bzl"), + {}, + ) + return + + if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ: + _create_dummy_repository(repository_ctx) + return + + cpu_value = get_cpu_value(repository_ctx) + if (cpu_value != "Linux"): + auto_configure_fail("TensorRT is supported only on Linux.") + if _TF_TENSORRT_VERSION not in repository_ctx.os.environ: + auto_configure_fail("TensorRT library (libnvinfer) version is not set.") + trt_install_path = repository_ctx.os.environ[_TENSORRT_INSTALL_PATH].strip() + if not repository_ctx.path(trt_install_path).exists: + auto_configure_fail( + "Cannot find TensorRT install path %s." % trt_install_path, + ) + + # Copy the library files. + trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path) + trt_libs = _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version) + trt_lib_srcs = [] + trt_lib_outs = [] + for path in trt_libs.values(): + trt_lib_srcs.append(str(path)) + trt_lib_outs.append("tensorrt/lib/" + path.basename) + copy_rules = [make_copy_files_rule( + repository_ctx, + name = "tensorrt_lib", + srcs = trt_lib_srcs, + outs = trt_lib_outs, + )] + + # Copy the header files header files. + trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path) + trt_header_srcs = [ + "%s/%s" % (trt_header_dir, header) + for header in _TF_TENSORRT_HEADERS + ] + trt_header_outs = [ + "tensorrt/include/" + header + for header in _TF_TENSORRT_HEADERS + ] + copy_rules.append( + make_copy_files_rule( + repository_ctx, + name = "tensorrt_include", + srcs = trt_header_srcs, + outs = trt_header_outs, + ), ) - return - - if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ: - _create_dummy_repository(repository_ctx) - return - - cpu_value = get_cpu_value(repository_ctx) - if (cpu_value != "Linux"): - auto_configure_fail("TensorRT is supported only on Linux.") - if _TF_TENSORRT_VERSION not in repository_ctx.os.environ: - auto_configure_fail("TensorRT library (libnvinfer) version is not set.") - trt_install_path = repository_ctx.os.environ[_TENSORRT_INSTALL_PATH].strip() - if not repository_ctx.path(trt_install_path).exists: - auto_configure_fail( - "Cannot find TensorRT install path %s." % trt_install_path) - - # Copy the library files. - trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path) - trt_libs = _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version) - trt_lib_srcs = [] - trt_lib_outs = [] - for path in trt_libs.values(): - trt_lib_srcs.append(str(path)) - trt_lib_outs.append("tensorrt/lib/" + path.basename) - copy_rules = [make_copy_files_rule( - repository_ctx, - name = "tensorrt_lib", - srcs = trt_lib_srcs, - outs = trt_lib_outs, - )] - - # Copy the header files header files. - trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path) - trt_header_srcs = [ - "%s/%s" % (trt_header_dir, header) for header in _TF_TENSORRT_HEADERS - ] - trt_header_outs = [ - "tensorrt/include/" + header for header in _TF_TENSORRT_HEADERS - ] - copy_rules.append( - make_copy_files_rule( - repository_ctx, - name = "tensorrt_include", - srcs = trt_header_srcs, - outs = trt_header_outs, - )) - - # Set up config file. - _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"}) - - # Set up BUILD file. - _tpl(repository_ctx, "BUILD", { - "%{copy_rules}": "\n".join(copy_rules), - "%{tensorrt_headers}": '":tensorrt_include"', - "%{tensorrt_libs}": str(trt_lib_outs), - }) + # Set up config file. + _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"}) + + # Set up BUILD file. + _tpl(repository_ctx, "BUILD", { + "%{copy_rules}": "\n".join(copy_rules), + "%{tensorrt_headers}": '":tensorrt_include"', + "%{tensorrt_libs}": str(trt_lib_outs), + }) tensorrt_configure = repository_rule( - implementation=_tensorrt_configure_impl, - environ=[ + implementation = _tensorrt_configure_impl, + environ = [ _TENSORRT_INSTALL_PATH, _TF_TENSORRT_VERSION, ], diff --git a/third_party/toolchains/clang6/repo.bzl b/third_party/toolchains/clang6/repo.bzl index b81f44506f382a..e4b6422c96d749 100644 --- a/third_party/toolchains/clang6/repo.bzl +++ b/third_party/toolchains/clang6/repo.bzl @@ -1,30 +1,37 @@ """Repository rule for Debian 8 Jessie Clang-6.0 portable Linux builds.""" def _clang6_configure(ctx): - # TODO(jart): It'd probably be better to use Bazel's struct.to_proto() - # method to generate a gigantic CROSSTOOL file that allows - # Clang to support everything. - ctx.symlink( - ctx.os.environ.get('TF_LLVM_PATH', - '/usr/lib/llvm-6.0'), - 'clang6/llvm') - ctx.symlink( - ctx.os.environ.get('STRIP', '/usr/bin/strip'), - 'clang6/sbin/strip') - ctx.symlink( - ctx.os.environ.get('OBJDUMP', '/usr/bin/objdump'), - 'clang6/sbin/objdump') - ctx.symlink(ctx.attr._build, 'clang6/BUILD') - ctx.template('clang6/CROSSTOOL', ctx.attr._crosstool, { - '%package(@local_config_clang6//clang6)%': str(ctx.path('clang6')), - }) + # TODO(jart): It'd probably be better to use Bazel's struct.to_proto() + # method to generate a gigantic CROSSTOOL file that allows + # Clang to support everything. + ctx.symlink( + ctx.os.environ.get( + "TF_LLVM_PATH", + "/usr/lib/llvm-6.0", + ), + "clang6/llvm", + ) + ctx.symlink( + ctx.os.environ.get("STRIP", "/usr/bin/strip"), + "clang6/sbin/strip", + ) + ctx.symlink( + ctx.os.environ.get("OBJDUMP", "/usr/bin/objdump"), + "clang6/sbin/objdump", + ) + ctx.symlink(ctx.attr._build, "clang6/BUILD") + ctx.template("clang6/CROSSTOOL", ctx.attr._crosstool, { + "%package(@local_config_clang6//clang6)%": str(ctx.path("clang6")), + }) clang6_configure = repository_rule( implementation = _clang6_configure, attrs = { - '_build': attr.label( - default=str(Label('//third_party/toolchains/clang6:clang.BUILD'))), - '_crosstool': attr.label( - default=str(Label('//third_party/toolchains/clang6:CROSSTOOL.tpl'))), + "_build": attr.label( + default = str(Label("//third_party/toolchains/clang6:clang.BUILD")), + ), + "_crosstool": attr.label( + default = str(Label("//third_party/toolchains/clang6:CROSSTOOL.tpl")), + ), }, ) diff --git a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl index ab6eac115ce552..d675e95f70fd31 100644 --- a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl +++ b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl @@ -1,38 +1,38 @@ # -*- Python -*- """Repository rule for arm compiler autoconfiguration.""" -def _tpl(repository_ctx, tpl, substitutions={}, out=None): - if not out: - out = tpl - repository_ctx.template( - out, - Label("//third_party/toolchains/cpus/arm:%s.tpl" % tpl), - substitutions) - +def _tpl(repository_ctx, tpl, substitutions = {}, out = None): + if not out: + out = tpl + repository_ctx.template( + out, + Label("//third_party/toolchains/cpus/arm:%s.tpl" % tpl), + substitutions, + ) def _arm_compiler_configure_impl(repository_ctx): - # We need to find a cross-compilation include directory for Python, so look - # for an environment variable. Be warned, this crosstool template is only - # regenerated on the first run of Bazel, so if you change the variable after - # it may not be reflected in later builds. Doing a shutdown and clean of Bazel - # doesn't fix this, you'll need to delete the generated file at something like: - # external/local_config_arm_compiler/CROSSTOOL in your Bazel install. - if "CROSSTOOL_PYTHON_INCLUDE_PATH" in repository_ctx.os.environ: - python_include_path = repository_ctx.os.environ["CROSSTOOL_PYTHON_INCLUDE_PATH"] - else: - python_include_path = "/usr/include/python2.7" - _tpl(repository_ctx, "CROSSTOOL", { - "%{ARM_COMPILER_PATH}%": str(repository_ctx.path( - repository_ctx.attr.remote_config_repo)), - "%{PYTHON_INCLUDE_PATH}%": python_include_path, - }) - repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") - + # We need to find a cross-compilation include directory for Python, so look + # for an environment variable. Be warned, this crosstool template is only + # regenerated on the first run of Bazel, so if you change the variable after + # it may not be reflected in later builds. Doing a shutdown and clean of Bazel + # doesn't fix this, you'll need to delete the generated file at something like: + # external/local_config_arm_compiler/CROSSTOOL in your Bazel install. + if "CROSSTOOL_PYTHON_INCLUDE_PATH" in repository_ctx.os.environ: + python_include_path = repository_ctx.os.environ["CROSSTOOL_PYTHON_INCLUDE_PATH"] + else: + python_include_path = "/usr/include/python2.7" + _tpl(repository_ctx, "CROSSTOOL", { + "%{ARM_COMPILER_PATH}%": str(repository_ctx.path( + repository_ctx.attr.remote_config_repo, + )), + "%{PYTHON_INCLUDE_PATH}%": python_include_path, + }) + repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") arm_compiler_configure = repository_rule( implementation = _arm_compiler_configure_impl, attrs = { - "remote_config_repo": attr.string(mandatory = False, default =""), + "remote_config_repo": attr.string(mandatory = False, default = ""), "build_file": attr.label(), }, ) diff --git a/third_party/toolchains/preconfig/generate/archives.bzl b/third_party/toolchains/preconfig/generate/archives.bzl index d20432e1de0a8e..a26d2c623c9b05 100644 --- a/third_party/toolchains/preconfig/generate/archives.bzl +++ b/third_party/toolchains/preconfig/generate/archives.bzl @@ -2,11 +2,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def bazel_toolchains_archive(): http_archive( - name = "bazel_toolchains", - sha256 = "109a99384f9d08f9e75136d218ebaebc68cc810c56897aea2224c57932052d30", - strip_prefix = "bazel-toolchains-94d31935a2c94fe7e7c7379a0f3393e181928ff7", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz", - "https://github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz", - ], + name = "bazel_toolchains", + sha256 = "109a99384f9d08f9e75136d218ebaebc68cc810c56897aea2224c57932052d30", + strip_prefix = "bazel-toolchains-94d31935a2c94fe7e7c7379a0f3393e181928ff7", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/94d31935a2c94fe7e7c7379a0f3393e181928ff7.tar.gz", + ], ) diff --git a/third_party/toolchains/preconfig/generate/generate.bzl b/third_party/toolchains/preconfig/generate/generate.bzl index 40e0957cf2e2e3..475db0fe139873 100644 --- a/third_party/toolchains/preconfig/generate/generate.bzl +++ b/third_party/toolchains/preconfig/generate/generate.bzl @@ -27,6 +27,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None, if cuda_version != None: base = "@cuda%s-cudnn%s-ubuntu14.04//image" % (cuda_version, cudnn_version) + # The cuda toolchain currently contains its own C++ toolchain definition, # so we do not fetch local_config_cc. config_repos = [ @@ -42,7 +43,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None, "TF_CUDNN_VERSION": cudnn_version, "TF_CUDA_VERSION": cuda_version, "CUDNN_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu", - "TF_NEED_TENSORRT" : "1", + "TF_NEED_TENSORRT": "1", "TF_TENSORRT_VERSION": tensorrt_version, "TENSORRT_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu", "GCC_HOST_COMPILER_PATH": compiler if compiler != "clang" else "",