Adapt to latest FMS update and have an option to install DA-utils

pycodestyle.cfg

@@ -8,3 +8,4 @@
 max-line-length = 100
 indent-size = 4
 statistics = True
+exclude = ._*

requirements.txt

@@ -1,3 +1,3 @@
 PyYAML>=5.4
-pycodestyle>=2.8.0
+pycodestyle>=2.8.0,<2.12.0
 requests>=2.23.0

src/jedi_bundle/__init__.py

@@ -9,4 +9,4 @@
 build_directory = os.path.dirname(__file__)
 
 # Set the version for jedi_bundle
-__version__ = '1.0.20'
+__version__ = '1.0.25'

src/jedi_bundle/clone_jedi_bundle.py

@@ -10,13 +10,13 @@
 
 
 import copy
-import re
 import os
+import re
 
 from jedi_bundle.config.config import return_config_path
 from jedi_bundle.utils.config import config_get
 from jedi_bundle.utils.file_system import check_for_executable
-from jedi_bundle.utils.git import get_url_and_branch, clone_git_repo
+from jedi_bundle.utils.git import get_url_and_branch, clone_git_repo, clone_git_file
 from jedi_bundle.utils.yaml import load_yaml
 
 
@@ -226,6 +226,17 @@ def clone_jedi(logger, clone_config):
         if repo == 'jedicmake':
             logger.info(f'Skipping explicit clone of \'{repo}\' since it\'s usually a module. ' +
                         f'If it\'s not a module it will be cloned at configure time.')
+        elif repo == 'fv3':
+            logger.info('Cloning fv3-interface.cmake from the jedi-bundle repo for fv3')
+            found, url_tmp, branch_tmp, \
+                is_tag, is_commit = get_url_and_branch(logger, github_orgs, 'jedi-bundle',
+                                                       default_branch, user_branch,
+                                                       False, False)
+
+            clone_git_file(logger, url_tmp, ['fv3-interface.cmake'], path_to_source, depth=1)
+            logger.info('Cloning fv3.')
+            clone_git_repo(logger, url, branch,
+                           os.path.join(path_to_source, repo), is_tag, is_commit)
         else:
             logger.info(f'Cloning \'{repo}\'.')
             clone_git_repo(logger, url, branch,
@@ -295,6 +306,11 @@ def clone_jedi(logger, clone_config):
                     output_file_open.write(jedi_cmake_line + '\n')
 
             else:
+                # Add include(fv3-interface.cmake) line if repo is fv3
+                if repo == 'fv3':
+                    output_file_open.write(' include(fv3-interface.cmake )\n')
+                    output_file_open.write(f' list( APPEND CMAKE_INSTALL_RPATH '
+                                           '${CMAKE_CURRENT_BINARY_DIR}/fv3 )\n')
 
                 output_file_open.write(package_line + '\n')
                 if cmake != '':

src/jedi_bundle/config/bundles/build-order.yaml

@@ -10,6 +10,13 @@
 - oops:
     default_branch: develop
 
+# Variable changes
+- gsw:
+    repo_url_name: GSW-Fortran
+    default_branch: develop
+- vader:
+    default_branch: develop
+
 # Background error models
 - gsibec:
     repo_url_name: GSIbec
@@ -28,9 +35,6 @@
     default_branch: develop
 - geos-aero:
     default_branch: develop
-- gsw:
-    repo_url_name: GSW-Fortran
-    default_branch: develop
 - ropp-ufo:
     default_branch: develop
     repo_url_name: ropp-test
@@ -45,22 +49,11 @@
 - ufo:
     default_branch: develop
 
-# Variable changes
-- vader:
-    default_branch: develop
-
-# Soca and fv3
-- fms:
-    repo_url_name: FMS
-    default_branch: release-stable
 
 # Soca
 - icepack:
     repo_url_name: Icepack
     default_branch: feature/ecbuild-new
-- mom6:
-    repo_url_name: MOM6
-    default_branch: main-ecbuild
 - soca:
     default_branch: develop
 
@@ -77,3 +70,10 @@
     default_branch: develop
 - fv3-jedi:
     default_branch: develop
+- jedi-bundle:
+    default_branch: develop
+
+# Other external packages
+- da-utils:
+    repo_url_name: DA-utils
+    default_branch: develop

src/jedi_bundle/config/bundles/fv3-jedi.yaml

@@ -11,7 +11,6 @@ required_repos:
   - crtm
   - ufo
   - vader
-  - fms
   - fv3
   - fv3-jedi-lm
   - femps

src/jedi_bundle/config/bundles/soca.yaml

@@ -11,7 +11,5 @@ required_repos:
   - vader
   - ufo
   - vader
-  - fms
-  - mom6
   - soca
   - ioda-data

src/jedi_bundle/config/platforms/nccs_discover.yaml

@@ -24,7 +24,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
     configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.6.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np"
   intel-geos:
     init:
@@ -42,7 +43,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
       - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake
     configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.6.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np"
   gnu:
@@ -61,7 +63,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
     configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"
   gnu-geos:
     init:
@@ -79,6 +82,7 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
       - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake
     configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"

src/jedi_bundle/config/platforms/nccs_discover_sles15.yaml

@@ -23,7 +23,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
     configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.10.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np"
   intel-geos:
     init:
@@ -40,7 +41,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
       - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake
     configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.10.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np"
   gnu:
@@ -58,7 +60,8 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
     configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"
   gnu-geos:
     init:
@@ -75,6 +78,7 @@ modules:
       - module load jedi-fv3-env
       - module load soca-env
       - module load gmao-swell-env/1.0.0
-      - module unload gsibec crtm
+      - module unload gsibec crtm fms
+      - module load fms/2023.04
       - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake
     configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n"

src/jedi_bundle/utils/git.py

@@ -149,6 +149,35 @@ def get_url_and_branch(logger, github_orgs, repo_url_name, default_branch,
 
 # --------------------------------------------------------------------------------------------------
 
+def clone_git_file(
+    logger: Logger,
+    url: str,
+    files: list,
+    src_dir: str,
+    depth: int = 1,
+) -> None:
+
+    target = os.path.join(src_dir, 'tmp_bundle')
+    if os.path.exists(target):
+        subprocess_run(logger, ["rm", "-rf", target], cwd=src_dir)
+
+    # Clone the temporary repository
+    # Depth allows for shallow cloning and saves time and space!
+    subprocess_run(logger, ["git", "clone",
+                            "--depth", str(depth),
+                            url,
+                            target])
+
+    # Copy the file(s)
+    for file in files:
+        git_file = os.path.join(target, file)
+        subprocess_run(logger, ["cp", git_file, src_dir])
+
+    # Remove the temporary directory
+    subprocess_run(logger, ["rm", "-rf", target], cwd=src_dir)
+
+
+# --------------------------------------------------------------------------------------------------
 
 def clone_git_repo(logger, url, branch, target, is_tag, is_commit):