hooks: update gribapi hook for compatibility with eccodes v2.37.0

Starting with `eccodes` v2.37.0, upstream provides binary wheels
with bundled copy of the `eccodes` shared library. Adjust the
hook for `gribapi` (part of `eccodes` dist) to account for that.
Also handle the issue with circular imports by importing `eccodes`
before `gribapi.bindings` when trying to obtain the path to the
shared library.

.github/workflows/pr-test.yml

@@ -111,8 +111,6 @@ jobs:
           # These are dependencies of gmsh
           sudo apt-get install -y libglu1 libgl1 libxrender1 libxcursor1 libxft2 \
             libxinerama1 libgomp1
-          # This one is required by eccodes
-          sudo apt-get install -y libeccodes0
 
       - name: Install brew dependencies
         if: startsWith(matrix.os, 'macos')
@@ -125,7 +123,8 @@ jobs:
           brew install libdiscid
           # Install lsl library for pylsl
           brew install labstreaminglayer/tap/lsl
-          # This one is required by eccodes
+          # This one is required by eccodes (binary wheels with bundled eccodes
+          # library are provided only for macOS 13+).
           brew install eccodes
 
       - name: Install dependencies

_pyinstaller_hooks_contrib/stdhooks/hook-gribapi.py

@@ -11,16 +11,39 @@
 # ------------------------------------------------------------------
 
 import os
+import pathlib
 
-from PyInstaller.utils.hooks import collect_data_files, get_module_attribute, logger
+from PyInstaller import isolated
+from PyInstaller.utils.hooks import collect_data_files, logger
 
 # Collect the headers (eccodes.h, gribapi.h) that are bundled with the package.
 datas = collect_data_files('gribapi')
 
-# Collect the eccodes shared library
+# Collect the eccodes shared library. Starting with eccodes 2.37.0, binary wheels with bundled shared library are
+# provided for linux and macOS.
+
+
+# NOTE: custom isolated function is used here instead of `get_module_attribute('gribapi.bindings', 'library_path')`
+# hook utility function because with eccodes 2.37.0, `eccodes` needs to be imported before `gribapi` to avoid circular
+# imports... Also, this way, we can obtain the root directory of eccodes package at the same time.
+@isolated.decorate
+def get_eccodes_library_path():
+    import eccodes
+    import gribapi.bindings
+
+    return (
+        # Path to eccodes shared library used by the gribapi bindings.
+        str(gribapi.bindings.library_path),
+        # Path to eccodes package (implicitly assumed to be next to the gribapi package, since they are part of the
+        # same eccodes dist).
+        str(eccodes.__path__[0]),
+    )
+
+
 binaries = []
+
 try:
-    library_path = get_module_attribute('gribapi.bindings', 'library_path')
+    library_path, package_path = get_eccodes_library_path()
 except Exception:
     logger.warning("hook-gribapi: failed to query gribapi.bindings.library_path!", exc_info=True)
     library_path = None
@@ -38,5 +61,23 @@
     logger.warning("hook-gribapi: could not determine path to eccodes shared library!")
 
 if library_path:
-    logger.debug("hook-gribapi: collecting eccodes shared library: %r", library_path)
-    binaries.append((library_path, '.'))
+    # If we are collecting eccodes shared library that is bundled with eccodes >= 2.37.0 binary wheel, attempt to
+    # preserve its parent directory layout. This ensures that the library is found at run-time, but implicitly requires
+    # PyInstaller 6.x, whose binary dependency analysis (that might also pick up this shared library) also preserves the
+    # parent directory layout of discovered shared libraries. With PyInstaller 5.x, this will result in duplication
+    # because binary dependency analysis collects into top-level application directory, but that copy will not be
+    # discovered at run-time, so duplication is unavoidable.
+    library_parent_path = pathlib.PurePath(library_path).parent
+    package_parent_path = pathlib.PurePath(package_path).parent
+
+    if package_parent_path in library_parent_path.parents:
+        # Should end up being `eccodes.libs` on Linux, and `eccodes/.dylib` on macOS).
+        dest_dir = str(library_parent_path.relative_to(package_parent_path))
+    else:
+        # External copy; collect into top-level application directory.
+        dest_dir = '.'
+
+    logger.info(
+        "hook-gribapi: collecting eccodes shared library %r to destination directory %r", library_path, dest_dir
+    )
+    binaries.append((library_path, dest_dir))

news/799.update.rst

@@ -0,0 +1,3 @@
+Update ``gribapi`` hook for compatibility with ``eccodes`` v2.37.0,
+to account for possibility of bundles ``eccodes`` shared library, which
+is provided by newly-introduced binary wheels for Linux and macOS 13+.

requirements-test-libraries.txt

@@ -226,7 +226,8 @@ pysaml2==7.3.0; python_version < '3.9'
 # ------------------- Platform (OS) specifics
 
 # eccodes package requires the eccodes shared library provided by the environment (linux distribution, homebrew, or Anaconda).
-eccodes==1.7.1; sys_platform == 'darwin' or sys_platform == 'linux'
+# Starting with v2.37.0, binary wheels with bundled shared library are provided for linux and macOS 13+.
+eccodes==2.37.0; sys_platform == 'darwin' or sys_platform == 'linux'
 
 # dbus-fast has pre-built wheels only for Linux; and D-Bus is available only there, anyway.
 dbus-fast==2.24.2; sys_platform == 'linux'

tests/test_libraries.py

@@ -2034,24 +2034,44 @@ def test_schwifty(pyi_builder):
     """)
 
 
-@importorskip('gribapi')
+@importorskip('eccodes')
 def test_eccodes_gribapi(pyi_builder):
     pyi_builder.test_source("""
         import sys
         import os
+        import pathlib
+
+        # With eccodes 2.37.0, eccodes needs to be imported before gribapi to avoid circular imports.
+        import eccodes
 
         # Basic import test
         import gribapi
 
         # Ensure that the eccodes shared library is bundled with the frozen application.
         import gribapi.bindings
 
-        lib_filename = os.path.join(
-            sys._MEIPASS,
-            os.path.basename(gribapi.bindings.library_path),
-        )
+        print(f"gribapi.bindings.library_path={gribapi.bindings.library_path}")
 
-        assert os.path.isfile(lib_filename), f"Shared library {lib_filename!s} not found!"
+        library_path = gribapi.bindings.library_path
+        if os.path.basename(library_path) == library_path:
+            # Only library basename is given - assume this is a system-wide copy that was collected
+            # into top-level application directory and loaded via `findlibs.find()`/`ctypes`.
+            expected_library_file = os.path.join(
+                sys._MEIPASS,
+                library_path,
+            )
+            if not os.path.isfile(expected_library_file):
+                raise RuntimeError(f"Shared library {expected_library_file!s} not found!")
+        else:
+            # Absolute path; check that it is rooted in top-level application directory. This covers all valid locations
+            # as per https://github.com/ecmwf/eccodes-python/blob/2.37.0/gribapi/bindings.py#L61-L64,
+            #  - sys._MEIPASS/eccodes
+            #  - sys._MEIPASS/eccodes.libs
+            #  - sys._MEIPASS/eccodes/.dylibs
+            if pathlib.PurePath(sys._MEIPASS) not in pathlib.PurePath(library_path).parents:
+                raise RuntimeError(
+                    f"Shared library path {library_path} is not rooted in top-level application directory!"
+                )
     """)