From b1c7b61b6def14ef926651e8a98e849329cbc641 Mon Sep 17 00:00:00 2001 From: Junrou Nishida Date: Sun, 20 Mar 2022 15:16:22 +0900 Subject: [PATCH] build: bazel 5.0.0 (#496) * build: use bazel 5.0.0 * build: configure Android at runtime * build: pass api_level from build.py * build: remove maven and unity dependencies * build: use remote JDK * build: remove JDK from Docker containers * build: JDK is used internally * wip: fix assembly file compilation error * wip: workaround for C compilation errors * allow >= 4.2.1 --- .bazelversion | 2 +- WORKSPACE | 81 ++---------------- build.py | 6 ++ docker/windows/x86_64/Dockerfile | 8 +- .../java/com/github/homuler/mediapipe/BUILD | 5 +- .../com/google/mediapipe/mediapipe_aar.bzl | 64 ++++---------- third_party/BUILD | 70 ---------------- third_party/android_configure.bzl | 83 +++++++++++++++++++ third_party/bazel_android_fixes.diff | 65 ++++++++++++++- 9 files changed, 176 insertions(+), 208 deletions(-) create mode 100644 third_party/android_configure.bzl diff --git a/.bazelversion b/.bazelversion index fae6e3d04..0062ac971 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -4.2.1 +5.0.0 diff --git a/WORKSPACE b/WORKSPACE index 086815c65..9a916c1d5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,7 +18,7 @@ bazel_skylib_workspace() load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "3.7.2") +versions.check(minimum_bazel_version = "4.2.1") http_archive( name = "rules_pkg", @@ -51,12 +51,6 @@ load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") emsdk_emscripten_deps(emscripten_version = "2.0.22") -new_local_repository( - name = "unity", - build_file = "@//third_party:unity.BUILD", - path = "/path/to/unity/2020.3.30f1", -) - # mediapipe http_archive( name = "com_google_mediapipe", @@ -323,18 +317,11 @@ http_archive( urls = ["https://github.com/nothings/stb/archive/b42009b3b9d4ca35bc703f5310eedc74f584be58.tar.gz"], ) -# You may run setup_android.sh to install Android SDK and NDK. -android_ndk_repository( - name = "androidndk", - # If you need to support older versions of Android, please specify the API Level. - # Otherwise, some symbols in libmediapipe_jni.so cannot be resolved and `DllNotFoundException` will be thrown. +load("//third_party:android_configure.bzl", "android_configure") +android_configure(name = "local_config_android") - # api_level = 21, -) - -android_sdk_repository( - name = "androidsdk", -) +load("@local_config_android//:android_configure.bzl", "android_workspace") +android_workspace() # iOS basic build deps. @@ -391,64 +378,6 @@ http_archive( url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", ) -# Maven dependencies. - -RULES_JVM_EXTERNAL_TAG = "4.0" - -RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169" - -http_archive( - name = "rules_jvm_external", - sha256 = RULES_JVM_EXTERNAL_SHA, - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:defs.bzl", "maven_install") - -# Important: there can only be one maven_install rule. Add new maven deps here. -maven_install( - artifacts = [ - "androidx.concurrent:concurrent-futures:1.0.0-alpha03", - "androidx.lifecycle:lifecycle-common:2.3.1", - "androidx.activity:activity:1.2.2", - "androidx.exifinterface:exifinterface:1.3.3", - "androidx.fragment:fragment:1.3.4", - "androidx.annotation:annotation:aar:1.1.0", - "androidx.appcompat:appcompat:aar:1.1.0-rc01", - "androidx.camera:camera-core:1.0.0-beta10", - "androidx.camera:camera-camera2:1.0.0-beta10", - "androidx.camera:camera-lifecycle:1.0.0-beta10", - "androidx.constraintlayout:constraintlayout:aar:1.1.3", - "androidx.core:core:aar:1.1.0-rc03", - "androidx.legacy:legacy-support-v4:aar:1.0.0", - "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", - "androidx.test.espresso:espresso-core:3.1.1", - "com.github.bumptech.glide:glide:4.11.0", - "com.google.android.material:material:aar:1.0.0-rc01", - "com.google.auto.value:auto-value:1.8.1", - "com.google.auto.value:auto-value-annotations:1.8.1", - "com.google.code.findbugs:jsr305:latest.release", - "com.google.android.datatransport:transport-api:3.0.0", - "com.google.android.datatransport:transport-backend-cct:3.1.0", - "com.google.android.datatransport:transport-runtime:3.1.0", - "com.google.flogger:flogger-system-backend:0.6", - "com.google.flogger:flogger:0.6", - "com.google.guava:guava:27.0.1-android", - "com.google.guava:listenablefuture:1.0", - "junit:junit:4.12", - "org.hamcrest:hamcrest-library:1.3", - ], - fetch_sources = True, - repositories = [ - "https://maven.google.com", - "https://dl.google.com/dl/android/maven2", - "https://repo1.maven.org/maven2", - "https://jcenter.bintray.com", - ], - version_conflict_policy = "pinned", -) - # Needed by TensorFlow http_archive( name = "io_bazel_rules_closure", diff --git a/build.py b/build.py index 8d4521e5f..e2fb9fae4 100644 --- a/build.py +++ b/build.py @@ -100,6 +100,7 @@ def __init__(self, command_args): self.system = platform.system() self.desktop = command_args.args.desktop self.android = command_args.args.android + self.android_ndk_api_level = command_args.args.android_ndk_api_level self.ios= command_args.args.ios self.resources = command_args.args.resources self.analyzers = command_args.args.analyzers @@ -198,6 +199,9 @@ def _build_common_commands(self): commands += ['build', '-c', self.compilation_mode] commands += self._build_linkopt() + if self.android_ndk_api_level: + commands += ['--action_env', f'ANDROID_NDK_API_LEVEL="{self.android_ndk_api_level}"'] + if self._is_windows(): python_bin_path = sys.executable.replace('\\', '//') commands += ['--action_env', f'PYTHON_BIN_PATH="{python_bin_path}"'] @@ -282,6 +286,7 @@ def _build_android_commands(self): commands = self._build_common_commands() commands.append(f'--config=android_{self.android}') + commands.append(f'--java_runtime_version=remotejdk_11') commands.append('//mediapipe_api/java/com/github/homuler/mediapipe:mediapipe_android') return commands @@ -410,6 +415,7 @@ def __init__(self): build_command_parser = subparsers.add_parser('build', help='Build and install native libraries') build_command_parser.add_argument('--desktop', choices=['cpu', 'gpu']) build_command_parser.add_argument('--android', choices=['armv7', 'arm64', 'fat']) + build_command_parser.add_argument('--android_ndk_api_level', type=int, choices=range(16, 31)) build_command_parser.add_argument('--ios', choices=['arm64']) build_command_parser.add_argument('--resources', action=argparse.BooleanOptionalAction, default=True) build_command_parser.add_argument('--analyzers', action=argparse.BooleanOptionalAction, default=False, help='Install Roslyn Analyzers') diff --git a/docker/windows/x86_64/Dockerfile b/docker/windows/x86_64/Dockerfile index a17c8cc49..16dba335b 100644 --- a/docker/windows/x86_64/Dockerfile +++ b/docker/windows/x86_64/Dockerfile @@ -1,7 +1,6 @@ # escape=` ARG NODE_VERSION=16.13.0 -ARG BAZEL_VERSION=4.2.1 ARG PYTHON_VERSION=3.9.9 ARG ANDROID_SDK_VERSION=30 ARG ANDROID_NDK_VERSION=21.4.7075529 @@ -106,24 +105,21 @@ RUN powershell -command "for($i=0;$i -lt 30;$i++) { $response += """y`n""" }; $r FROM builder ARG ANDROID_NDK_VERSION -ARG BAZEL_VERSION ENV ANDROID_HOME C:\Android ENV ANDROID_NDK_HOME ${ANDROID_HOME}\ndk\${ANDROID_NDK_VERSION} -ENV JAVA_HOME=C:\Java\jdk-11.0.13+8 ENV PYTHON_INSTALL_PATH=C:\Python ENV PYTHON_BIN_PATH=${PYTHON_INSTALL_PATH}\python.exe # Install Bazel and NuGet -RUN setx path "C:\bin;%PYTHON_INSTALL_PATH%;%PYTHON_INSTALL_PATH%\Scripts;%JAVA_HOME%\bin;%PATH%" && ` +RUN setx path "C:\bin;%PYTHON_INSTALL_PATH%;%PYTHON_INSTALL_PATH%\Scripts;%PATH%" && ` mkdir C:\bin && ` - curl -L https://github.com/bazelbuild/bazel/releases/download/%BAZEL_VERSION%/bazel-%BAZEL_VERSION%-windows-x86_64.exe -o C:\bin\bazel.exe && ` + curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-windows-amd64.exe -o C:\bin\bazel.exe && ` curl -L https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -o C:\bin\nuget.exe COPY --from=python C:\Python C:\Python COPY --from=opencv C:\opencv C:\opencv COPY --from=android C:\Android C:\Android -COPY --from=android C:\Java C:\Java ENV WS_LONG_PATHS_VERSION 3.4.1 ENV WS_LONG_PATHS_BASE ndk-wsls-${WS_LONG_PATHS_VERSION} diff --git a/mediapipe_api/java/com/github/homuler/mediapipe/BUILD b/mediapipe_api/java/com/github/homuler/mediapipe/BUILD index 229aecb95..8b03e2183 100644 --- a/mediapipe_api/java/com/github/homuler/mediapipe/BUILD +++ b/mediapipe_api/java/com/github/homuler/mediapipe/BUILD @@ -9,9 +9,7 @@ load("//mediapipe_api/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_a # A sample aar to extend UnityPlayerActivity mediapipe_aar( name = "mediapipe_android", - srcs = [ - # "MediaPipeUnityPlayerActivity.java", - ], + srcs = [], assets = [ # If preferable, model files can be included to aar ], @@ -19,7 +17,6 @@ mediapipe_aar( "//mediapipe_api:mediapipe_c", ], deps = [ - # "@unity//:activity", "@opencv", ], ) diff --git a/mediapipe_api/java/com/google/mediapipe/mediapipe_aar.bzl b/mediapipe_api/java/com/google/mediapipe/mediapipe_aar.bzl index 8ab8a7513..853002833 100644 --- a/mediapipe_api/java/com/google/mediapipe/mediapipe_aar.bzl +++ b/mediapipe_api/java/com/google/mediapipe/mediapipe_aar.bzl @@ -1,26 +1,11 @@ -# Copyright 2019-2020 The MediaPipe Authors. +# Copyright (c) 2019 Adam Michael +# Copyright (c) 2021 homuler # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. -# CHANGES: -# -# - remove the usage comment -# - package name can be specified -# - add `srcs` option -# - add `deps` option -# - remove `calculators` option -# - not to generate *Proto.java -# - .so files are placed under lib/ +# Based on https://github.com/aj-michael/aar_with_jni/blob/b284343108b1e4cbfdbbf155bbcfa0b06e878f79/aar_with_jni.bzl """AAR Generator @@ -29,7 +14,7 @@ Macro to generate AAR, including libmediapipe_jni.so load("@build_bazel_rules_android//android:rules.bzl", "android_binary", "android_library") -def mediapipe_aar(name, package = "com.github.homuler.mediapipe", srcs = [], deps = [], jni_deps = [], assets = [], assets_dir = ""): +def mediapipe_aar(name, package = "com.github.homuler.mediapipe", srcs = [], deps = [], jni_deps = [], assets = [], assets_dir = "", target_sdk_version = 27, min_sdk_version = 21): """Generate MediaPipeUnityPlugin AAR. Args: @@ -40,14 +25,14 @@ def mediapipe_aar(name, package = "com.github.homuler.mediapipe", srcs = [], dep jni_deps: additional dependencies that will be linked to libmediapipe_jni.so assets: additional assets to be included into the archive. assets_dir: path where the assets will the packaged + target_sdk_version: AAR's target SDK version + min_sdk_version: AAR's min SDK version """ native.cc_binary( name = "libmediapipe_jni.so", linkshared = 1, linkstatic = 1, - deps = [ - "@com_google_mediapipe//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ] + jni_deps, + deps = jni_deps, ) native.cc_library( @@ -65,41 +50,20 @@ cat > $(OUTS) < + android:minSdkVersion="{}" + android:targetSdkVersion="{}" /> EOF -""".format(package), +""".format(package, min_sdk_version, target_sdk_version), ) android_library( name = name + "_android_lib", - srcs = [ - "@com_google_mediapipe//mediapipe/java/com/google/mediapipe/components:java_src", - "@com_google_mediapipe//mediapipe/java/com/google/mediapipe/framework:java_src", - "@com_google_mediapipe//mediapipe/java/com/google/mediapipe/glutil:java_src", - ] + srcs, + srcs = srcs, manifest = "AndroidManifest.xml", - proguard_specs = ["@com_google_mediapipe//mediapipe/java/com/google/mediapipe/framework:proguard.pgcfg"], deps = [ ":" + name + "_mediapipe_jni_lib", - "@com_google_mediapipe//mediapipe/framework:calculator_java_proto_lite", - "@com_google_mediapipe//mediapipe/framework:calculator_profile_java_proto_lite", - "@com_google_mediapipe//mediapipe/framework/tool:calculator_graph_template_java_proto_lite", - "@com_google_protobuf//:protobuf_javalite", - "//third_party:androidx_annotation", - "//third_party:androidx_appcompat", - "//third_party:androidx_core", - "//third_party:androidx_legacy_support_v4", - "//third_party:camerax_core", - "//third_party:camerax_camera2", - "//third_party:camerax_lifecycle", - "@maven//:com_google_code_findbugs_jsr305", - "@maven//:com_google_flogger_flogger", - "@maven//:com_google_flogger_flogger_system_backend", - "@maven//:com_google_guava_guava", - "@maven//:androidx_lifecycle_lifecycle_common", ] + deps, assets = assets, assets_dir = assets_dir, diff --git a/third_party/BUILD b/third_party/BUILD index ff34798a6..8fb6f6ad3 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -70,73 +70,3 @@ cc_library( ], }), ) - -android_library( - name = "androidx_annotation", - exports = [ - "@maven//:androidx_annotation_annotation", - ], -) - -android_library( - name = "androidx_appcompat", - exports = [ - "@maven//:androidx_appcompat_appcompat", - ], -) - -android_library( - name = "androidx_constraint_layout", - exports = [ - "@maven//:androidx_constraintlayout_constraintlayout", - ], -) - -android_library( - name = "androidx_core", - exports = [ - "@maven//:androidx_core_core", - ], -) - -android_library( - name = "androidx_legacy_support_v4", - exports = [ - "@maven//:androidx_legacy_legacy_support_v4", - ], -) - -android_library( - name = "androidx_material", - exports = [ - "@maven//:com_google_android_material_material", - ], -) - -android_library( - name = "androidx_recyclerview", - exports = [ - "@maven//:androidx_recyclerview_recyclerview", - ], -) - -android_library( - name = "camerax_camera2", - exports = [ - "@maven//:androidx_camera_camera_camera2", - ], -) - -android_library( - name = "camerax_core", - exports = [ - "@maven//:androidx_camera_camera_core", - ], -) - -android_library( - name = "camerax_lifecycle", - exports = [ - "@maven//:androidx_camera_camera_lifecycle", - ], -) diff --git a/third_party/android_configure.bzl b/third_party/android_configure.bzl new file mode 100644 index 000000000..f8e2271f4 --- /dev/null +++ b/third_party/android_configure.bzl @@ -0,0 +1,83 @@ +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Based on https://github.com/gnossen/grpc/blob/19fe1e777c9de7cfa42b04bbe764856f265b6110/third_party/android/android_configure.bzl +# and https://github.com/tensorflow/tensorflow/blob/c858694b46eb777087b57a721f0cbb7e271e4b9e/third_party/android/android_configure.bzl + +# Changes +# - determine ANDROID_NDK_API_LEVEL dynamically +# - not to set `path` explicitly + +"""Repository rule for Android SDK and NDK autoconfiguration. +This rule is a no-op unless the required android environment variables are set. +""" + +# Workaround for https://github.com/bazelbuild/bazel/issues/14260 + +_ANDROID_NDK_HOME = "ANDROID_NDK_HOME" +_ANDROID_NDK_API_LEVEL = "ANDROID_NDK_API_LEVEL" +_ANDROID_SDK_HOME = "ANDROID_HOME" + +def _android_autoconf_impl(repository_ctx): + sdk_home = repository_ctx.os.environ.get(_ANDROID_SDK_HOME) + ndk_home = repository_ctx.os.environ.get(_ANDROID_NDK_HOME) + ndk_api_level = repository_ctx.os.environ.get(_ANDROID_NDK_API_LEVEL) + + # version 31.0.0 won't work https://stackoverflow.com/a/68036845 + sdk_rule = "" + if sdk_home: + sdk_rule = """native.android_sdk_repository(name="androidsdk")""" + + # Note that Bazel does not support NDK 22 yet. + ndk_rule = "" + if ndk_home: + if not ndk_api_level: + ndk_rule = """native.android_ndk_repository(name="androidndk")""" + else: + ndk_rule = """ + native.android_ndk_repository( + name="androidndk", + api_level={}, + ) +""".format(ndk_api_level) + + if ndk_rule == "" and sdk_rule == "": + sdk_rule = "pass" + + repository_ctx.file("BUILD.bazel", "") + repository_ctx.file("android_configure.bzl", """ +def android_workspace(): + {} + {} + """.format(sdk_rule, ndk_rule)) + +android_configure = repository_rule( + implementation = _android_autoconf_impl, + environ = [ + _ANDROID_NDK_HOME, + _ANDROID_NDK_API_LEVEL, + _ANDROID_SDK_HOME, + ], +) diff --git a/third_party/bazel_android_fixes.diff b/third_party/bazel_android_fixes.diff index f54ebf10c..78647f7f3 100644 --- a/third_party/bazel_android_fixes.diff +++ b/third_party/bazel_android_fixes.diff @@ -1,5 +1,5 @@ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/android_ndk_cc_toolchain_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/android_ndk_cc_toolchain_template.txt -index d622f0c962..4ffc0134b7 100644 +index 6e6c5c9592..e201f38565 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/android_ndk_cc_toolchain_template.txt +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/android_ndk_cc_toolchain_template.txt @@ -14,7 +14,7 @@ cc_toolchain( @@ -11,3 +11,66 @@ index d622f0c962..4ffc0134b7 100644 toolchain_identifier = "%toolchainName%", toolchain_config = ":%toolchainName%-config", ) +diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java +index 5140b4afdd..3c207997ef 100644 +--- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java ++++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java +@@ -31,7 +31,7 @@ public final class CppFileTypes { + // supported with clang. Bazel is not officially supporting these targets, and the extensions are + // listed only as long as they work with the existing C++ actions. + public static final FileType CPP_SOURCE = +- FileType.of(".cc", ".cpp", ".cxx", ".c++", ".C", ".cu", ".cl"); ++ FileType.of(".cc", ".cpp", ".cxx", ".c++", ".cu", ".cl"); + public static final FileType C_SOURCE = FileType.of(".c"); + public static final FileType OBJC_SOURCE = FileType.of(".m"); + public static final FileType OBJCPP_SOURCE = FileType.of(".mm"); +@@ -91,15 +91,46 @@ public final class CppFileTypes { + } + }; + +- public static final FileType ASSEMBLER_WITH_C_PREPROCESSOR = FileType.of(".S"); +- public static final FileType PIC_ASSEMBLER = FileType.of(".pic.s"); ++ // FileType is extended to use case-sensitive comparison also on Windows ++ public static final FileType ASSEMBLER_WITH_C_PREPROCESSOR = ++ new FileType() { ++ final String ext = ".S"; ++ ++ @Override ++ public boolean apply(String path) { ++ return path.endsWith(ext); ++ } ++ ++ @Override ++ public ImmutableList getExtensions() { ++ return ImmutableList.of(ext); ++ } ++ }; ++ ++ // FileType is extended to use case-sensitive comparison also on Windows ++ public static final FileType PIC_ASSEMBLER = ++ new FileType() { ++ final String ext = ".pic.s"; ++ ++ @Override ++ public boolean apply(String path) { ++ return OS.endsWith(path, ext) && path.endsWith(".s"); ++ } ++ ++ @Override ++ public ImmutableList getExtensions() { ++ return ImmutableList.of(ext); ++ } ++ }; ++ ++ // FileType is extended to use case-sensitive comparison also on Windows + public static final FileType ASSEMBLER = + new FileType() { + final String ext = ".s"; + + @Override + public boolean apply(String path) { +- return (OS.endsWith(path, ext) && !PIC_ASSEMBLER.matches(path)) ++ return (path.endsWith(ext) && !PIC_ASSEMBLER.matches(path)) + || OS.endsWith(path, ".asm"); + } +