build!: update OpenCV version to 4.10.0 (#1295)

* build: build OpenCV 4 for Linux

* build: enable optflow

* build: OpenCV 4 for macOS

* build: OpenCV 4 for iOS

* build: OpenCV 4.5.5 -> 4.10.0

* fix: OpenCV lib path on Windows

* build: link self-built OpenCV framework by default

.github/workflows/build-package.yml

@@ -32,6 +32,6 @@ jobs:
       linuxBuildArgs: ${{ inputs.linuxBuildArgs || '--desktop gpu --opencv cmake' }}
       androidBuildArgs: ${{ inputs.androidBuildArgs || '--android arm64 --android_ndk_api_level 21' }}
       macosBuildArgs: ${{ inputs.macosBuildArgs || '--desktop cpu --opencv cmake --macos_universal' }}
-      iosBuildArgs: ${{ inputs.iosBuildArgs || '--ios arm64' }}
+      iosBuildArgs: ${{ inputs.iosBuildArgs || '--ios arm64 --opencv cmake' }}
       windowsBuildArgs: ${{ inputs.windowsBuildArgs || '--desktop cpu --opencv cmake' }}
     secrets: inherit

.github/workflows/package.yml

@@ -22,7 +22,7 @@ on:
         default: '--desktop cpu --opencv cmake --macos_universal'
       iosBuildArgs:
         type: string
-        default: '--ios arm64'
+        default: '--ios arm64 --opencv cmake'
       windowsBuildArgs:
         type: string
         default: '--desktop cpu --opencv cmake'

.github/workflows/release-packages.yml

@@ -14,7 +14,7 @@ jobs:
       linuxBuildArgs: '--desktop gpu --opencv cmake'
       androidBuildArgs: '--android arm64 --android_ndk_api_level 21'
       macosBuildArgs: '--desktop cpu --opencv cmake --macos_universal'
-      iosBuildArgs: '--ios arm64'
+      iosBuildArgs: '--ios arm64 --opencv cmake'
       windowsBuildArgs: '--desktop cpu --opencv cmake'
     secrets: inherit
 
@@ -26,6 +26,6 @@ jobs:
       linuxBuildArgs: '--linkopt="-s" --desktop gpu --opencv cmake'
       androidBuildArgs: '--linkopt="-s" --android arm64 --android_ndk_api_level 21'
       macosBuildArgs: '--linkopt="-s" --desktop cpu --opencv cmake --macos_universal'
-      iosBuildArgs: '--linkopt="-s" --ios arm64'
+      iosBuildArgs: '--linkopt="-s" --ios arm64 --opencv cmake'
       windowsBuildArgs: '--desktop cpu --opencv cmake'
     secrets: inherit

WORKSPACE

@@ -455,10 +455,24 @@ http_archive(
 
 http_archive(
     name = "opencv",
+    patch_args = [
+        "-p1",
+    ],
+    patches = [
+        "@//third_party:opencv_patch.diff",
+    ],
     build_file = "@//third_party:opencv.BUILD",
-    sha256 = "5e37b791b2fe42ed39b52d9955920b951ee42d5da95f79fbc9765a08ef733399",
-    strip_prefix = "opencv-3.4.16",
-    urls = ["https://github.com/opencv/opencv/archive/3.4.16.tar.gz"],
+    sha256 = "b2171af5be6b26f7a06b1229948bbb2bdaa74fcf5cd097e0af6378fce50a6eb9",
+    strip_prefix = "opencv-4.10.0",
+    urls = ["https://github.com/opencv/opencv/archive/4.10.0.tar.gz"],
+)
+
+http_archive(
+    name = "opencv_contrib",
+    build_file = "@//third_party:opencv_contrib.BUILD",
+    sha256 = "65597f8fb8dc2b876c1b45b928bbcc5f772ddbaf97539bf1b737623d0604cba1",
+    strip_prefix = "opencv_contrib-4.10.0",
+    urls = ["https://github.com/opencv/opencv_contrib/archive/4.10.0.tar.gz"],
 )
 
 new_local_repository(
@@ -475,13 +489,13 @@ new_local_repository(
 
 new_local_repository(
     name = "macos_opencv",
-    build_file = "@mediapipe//third_party:opencv_macos.BUILD",
+    build_file = "@//third_party:opencv_macos.BUILD",
     path = "/usr/local",
 )
 
 new_local_repository(
     name = "macos_arm64_opencv",
-    build_file = "@mediapipe//third_party:opencv_macos.BUILD",
+    build_file = "@//third_party:opencv_macos.BUILD",
     path = "/opt/homebrew",
 )
 
@@ -517,20 +531,22 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
-# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
+# Building an opencv.xcframework from the OpenCV 4.10.0 sources is necessary for
 # MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
 # `opencv.xcframework` archive has not been released and it is recommended to
 # build the same from source using a script provided in OpenCV 4.5.0 upwards.
-# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
-# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
-# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
-# Task libraries are built.
 http_archive(
     name = "ios_opencv_source",
-    build_file = "@mediapipe//third_party:opencv_ios_source.BUILD",
-    sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
-    type = "zip",
-    url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
+    build_file = "@//third_party:opencv_ios_source.BUILD",
+    patch_args = [
+        "-p1",
+    ],
+    patches = [
+        "@//third_party:opencv_ios_patch.diff",
+    ],
+    sha256 = "b2171af5be6b26f7a06b1229948bbb2bdaa74fcf5cd097e0af6378fce50a6eb9",
+    strip_prefix = "opencv-4.10.0",
+    urls = ["https://github.com/opencv/opencv/archive/4.10.0.tar.gz"],
 )
 
 http_archive(

build.py

@@ -298,6 +298,7 @@ def _build_ios_commands(self):
 
     commands = self._build_common_commands()
     commands += [f'--config=ios_{self.command_args.ios}']
+    commands += self._build_opencv_switch()
 
     commands.append('//mediapipe_api/objc:MediaPipeUnity')
     return commands

docker/windows/x86_64/Dockerfile

@@ -70,7 +70,7 @@ RUN curl -L --retry 4 --connect-timeout 10 https://www.python.org/ftp/python/%PY
 FROM base as opencv
 
 # Install OpenCV
-RUN curl -L --retry 4 --connect-timeout 10 https://github.com/opencv/opencv/releases/download/3.4.16/opencv-3.4.16-vc14_vc15.exe -o C:\TEMP\opencv-installer.exe && `
+RUN curl -L --retry 4 --connect-timeout 10 https://github.com/opencv/opencv/releases/download/4.10.0/opencv-4.10.0-windows.exe -o C:\TEMP\opencv-installer.exe && `
     start /wait C:\TEMP\opencv-installer.exe -gm2 -y -oC:\ && `
     del C:\TEMP\opencv-installer.exe
 

mediapipe_api/objc/BUILD

@@ -20,8 +20,10 @@ objc_library(
     ],
     deps = [
         "//mediapipe_api:mediapipe_c",
-        "@ios_opencv//:OpencvFramework",
-    ],
+    ] + select({
+        "@opencv//:opencv_ios_source_build": ["@ios_opencv_source//:opencv_xcframework"],
+        "//conditions:default": ["@ios_opencv//:OpencvFramework"],
+    }),
 )
 
 ios_framework(

third_party/mediapipe_workaround.diff

@@ -1,3 +1,16 @@
+diff --git a/mediapipe/framework/port/opencv_core_inc.h b/mediapipe/framework/port/opencv_core_inc.h
+index 12862472..1a409417 100644
+--- a/mediapipe/framework/port/opencv_core_inc.h
++++ b/mediapipe/framework/port/opencv_core_inc.h
+@@ -20,7 +20,7 @@
+ #ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
+ #include <opencv2/core/core.hpp>
+ #else
+-#if CV_VERSION_MAJOR == 3
++#if CV_VERSION_MAJOR == 3 || CV_VERSION_MAJOR == 4
+ #include <opencv2/cvconfig.h>
+ #endif
+
 diff --git a/mediapipe/tasks/cc/core/task_api_factory.h b/mediapipe/tasks/cc/core/task_api_factory.h
 index a11a23fc..dbb5fe6c 100644
 --- a/mediapipe/tasks/cc/core/task_api_factory.h

third_party/opencv.BUILD

@@ -38,6 +38,16 @@ config_setting(
     },
 )
 
+config_setting(
+    name = "opencv_ios_source_build",
+    flag_values = {
+        ":switch": "cmake",
+    },
+    values = {
+        "apple_platform_type": "ios",
+    },
+)
+
 selects.config_setting_group(
     name = "cmake_static_win",
     match_all = ["@bazel_tools//src/conditions:windows", ":cmake_static"],
@@ -62,6 +72,7 @@ alias(
     name = "opencv",
     actual = select({
         ":cmake_static": ":opencv_cmake",
+        ":opencv_ios_source_build": "@ios_opencv_source//:opencv",
         "//conditions:default": ":opencv_binary",
     }),
 )
@@ -86,6 +97,7 @@ OPENCV_MODULES = [
     "calib3d",
     "features2d",
     "highgui",
+    "optflow",
     "video",
     "videoio",
     "imgcodecs",
@@ -96,6 +108,7 @@ OPENCV_MODULES = [
 OPENCV_3RDPARTY_LIBS = [
     "IlmImf",
     "libjpeg-turbo",
+    "libopenjp2",
     "libpng",
     "libtiff",
     "zlib",
@@ -111,11 +124,13 @@ COMMON_CACHE_ENTRIES = {
     "BUILD_opencv_apps": "OFF",
     "BUILD_opencv_python": "OFF",
     "BUILD_opencv_world": "ON",
+    "BUILD_opencv_optflow": "ON",
     "BUILD_EXAMPLES": "OFF",
     "BUILD_PERF_TESTS": "OFF",
     "BUILD_TESTS": "OFF",
     "BUILD_JPEG": "ON",
     "BUILD_OPENEXR": "ON",
+    "BUILD_OPENJPEG": "ON",
     "BUILD_PNG": "ON",
     "BUILD_TIFF": "ON",
     "BUILD_ZLIB": "ON",
@@ -137,6 +152,7 @@ COMMON_CACHE_ENTRIES = {
     "BUILD_SHARED_LIBS": "OFF",
     "OPENCV_SKIP_PYTHON_LOADER": "ON",
     "OPENCV_SKIP_VISIBILITY_HIDDEN": "ON",
+    "OPENCV_EXTRA_MODULES_PATH": "$$EXT_BUILD_ROOT$$/external/opencv_contrib/modules",
 }
 
 CACHE_ENTRIES = COMMON_CACHE_ENTRIES | select({
@@ -171,6 +187,9 @@ CACHE_ENTRIES = COMMON_CACHE_ENTRIES | select({
 
 cmake(
     name = "opencv_cmake",
+    data = [
+        "@opencv_contrib//:all",
+    ],
     build_args = [
         "--verbose",
         "--parallel",
@@ -189,16 +208,23 @@ cmake(
         "//conditions:default": [],
     }),
     lib_source = "@opencv//:all",
-    out_lib_dir = ".",
+    out_include_dir = select({
+        "@bazel_tools//src/conditions:windows": "include",
+        "//conditions:default": "include/opencv4",
+    }),
+    out_lib_dir = select({
+        "@bazel_tools//src/conditions:windows": "x64/vc17",
+        "//conditions:default": ".", # need to include lib/ and share/OpenCV/3rdparty/lib when building static libs
+    }),
     out_static_libs = select({
-        ":dbg_cmake_static_win": ["staticlib/opencv_world3416d.lib"],
-        ":cmake_static_win": ["staticlib/opencv_world3416.lib"],
+        ":dbg_cmake_static_win": ["staticlib/opencv_world4100d.lib"],
+        ":cmake_static_win": ["staticlib/opencv_world4100.lib"],
         "//conditions:default": ["lib/libopencv_world.a"],
     }) + select({
         ":dbg_cmake_static_win": ["staticlib/%sd.lib" % lib for lib in OPENCV_3RDPARTY_LIBS],
         ":cmake_static_win": ["staticlib/%s.lib" % lib for lib in OPENCV_3RDPARTY_LIBS],
-        "@cpuinfo//:macos_arm64": ["share/OpenCV/3rdparty/lib/lib%s.a" % lib for lib in OPENCV_3RDPARTY_LIBS_M1],
-        "//conditions:default": ["share/OpenCV/3rdparty/lib/lib%s.a" % lib for lib in OPENCV_3RDPARTY_LIBS],
+        "@cpuinfo//:macos_arm64": ["lib/opencv4/3rdparty/lib%s.a" % lib for lib in OPENCV_3RDPARTY_LIBS_M1],
+        "//conditions:default": ["lib/opencv4/3rdparty/lib%s.a" % lib for lib in OPENCV_3RDPARTY_LIBS],
     }),
     out_shared_libs =  [],
     linkopts = select({

third_party/opencv_contrib.BUILD

@@ -0,0 +1,5 @@
+filegroup(
+  name = "all",
+  srcs = glob(["**"]),
+  visibility = ["//visibility:public"],
+)

third_party/opencv_ios_patch.diff

@@ -0,0 +1,13 @@
+diff --git a/modules/objc/generator/gen_objc.py b/modules/objc/generator/gen_objc.py
+index 58b7dd555c..6a90ebbc72 100755
+--- a/modules/objc/generator/gen_objc.py
++++ b/modules/objc/generator/gen_objc.py
+@@ -1661,7 +1661,7 @@ if __name__ == "__main__":
+                h_files += [os.path.join(root, filename) for filename in fnmatch.filter(filenames, '*.h')]
+                hpp_files += [os.path.join(root, filename) for filename in fnmatch.filter(filenames, '*.hpp')]
+             srcfiles = h_files + hpp_files
+-            srcfiles = [f for f in srcfiles if not re_bad.search(f.replace('\\', '/'))]
++            srcfiles = [f for f in srcfiles if not re_bad.search(f.removeprefix(module_location).replace('\\', '/'))]
+         logging.info("\nFiles (%d):\n%s", len(srcfiles), pformat(srcfiles))
+
+         common_headers_fname = os.path.join(misc_location, 'filelist_common')