From 5e76d2cc950ecbf72009f1cb66b7a94041cc8e64 Mon Sep 17 00:00:00 2001 From: dooruk Date: Tue, 16 Jul 2024 16:00:20 -0400 Subject: [PATCH 01/10] version bump --- src/jedi_bundle/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jedi_bundle/__init__.py b/src/jedi_bundle/__init__.py index c07aeb3..fc46297 100644 --- a/src/jedi_bundle/__init__.py +++ b/src/jedi_bundle/__init__.py @@ -9,4 +9,4 @@ build_directory = os.path.dirname(__file__) # Set the version for jedi_bundle -__version__ = '1.0.20' +__version__ = '1.0.25' From cd7439c1c29c97fdaf708003825b9176446c61e0 Mon Sep 17 00:00:00 2001 From: dooruk Date: Tue, 16 Jul 2024 16:01:29 -0400 Subject: [PATCH 02/10] config updates for fms change and including DA-utils as an option --- src/jedi_bundle/config/bundles/build-order.yaml | 15 +++++++-------- src/jedi_bundle/config/bundles/fv3-jedi.yaml | 2 +- src/jedi_bundle/config/bundles/soca.yaml | 2 -- .../config/platforms/nccs_discover.yaml | 12 ++++++++---- .../config/platforms/nccs_discover_sles15.yaml | 12 ++++++++---- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/jedi_bundle/config/bundles/build-order.yaml b/src/jedi_bundle/config/bundles/build-order.yaml index 15a9e59..67baf26 100644 --- a/src/jedi_bundle/config/bundles/build-order.yaml +++ b/src/jedi_bundle/config/bundles/build-order.yaml @@ -49,18 +49,10 @@ - vader: default_branch: develop -# Soca and fv3 -- fms: - repo_url_name: FMS - default_branch: release-stable - # Soca - icepack: repo_url_name: Icepack default_branch: feature/ecbuild-new -- mom6: - repo_url_name: MOM6 - default_branch: main-ecbuild - soca: default_branch: develop @@ -77,3 +69,10 @@ default_branch: develop - fv3-jedi: default_branch: develop +- jedi-bundle: + default_branch: develop + +# Other external packages +- da-utils: + repo_url_name: DA-utils + default_branch: develop diff --git a/src/jedi_bundle/config/bundles/fv3-jedi.yaml b/src/jedi_bundle/config/bundles/fv3-jedi.yaml index 6e4af83..6b58389 100644 --- a/src/jedi_bundle/config/bundles/fv3-jedi.yaml +++ b/src/jedi_bundle/config/bundles/fv3-jedi.yaml @@ -11,7 +11,7 @@ required_repos: - crtm - ufo - vader - - fms + - jedi-bundle - fv3 - fv3-jedi-lm - femps diff --git a/src/jedi_bundle/config/bundles/soca.yaml b/src/jedi_bundle/config/bundles/soca.yaml index 9a5c289..53452c8 100644 --- a/src/jedi_bundle/config/bundles/soca.yaml +++ b/src/jedi_bundle/config/bundles/soca.yaml @@ -11,7 +11,5 @@ required_repos: - vader - ufo - vader - - fms - - mom6 - soca - ioda-data diff --git a/src/jedi_bundle/config/platforms/nccs_discover.yaml b/src/jedi_bundle/config/platforms/nccs_discover.yaml index 833dcde..8dcf85a 100644 --- a/src/jedi_bundle/config/platforms/nccs_discover.yaml +++ b/src/jedi_bundle/config/platforms/nccs_discover.yaml @@ -24,7 +24,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.6.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np" intel-geos: init: @@ -42,7 +43,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.6.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np" gnu: @@ -61,7 +63,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" gnu-geos: init: @@ -79,6 +82,7 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" diff --git a/src/jedi_bundle/config/platforms/nccs_discover_sles15.yaml b/src/jedi_bundle/config/platforms/nccs_discover_sles15.yaml index aa46451..d7493e5 100644 --- a/src/jedi_bundle/config/platforms/nccs_discover_sles15.yaml +++ b/src/jedi_bundle/config/platforms/nccs_discover_sles15.yaml @@ -23,7 +23,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.10.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np" intel-geos: init: @@ -40,7 +41,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake configure: -DMPIEXEC_EXECUTABLE="/usr/local/intel/oneapi/2021/mpi/2021.10.0/bin/mpirun" -DMPIEXEC_NUMPROC_FLAG="-np" gnu: @@ -58,7 +60,8 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" gnu-geos: init: @@ -75,6 +78,7 @@ modules: - module load jedi-fv3-env - module load soca-env - module load gmao-swell-env/1.0.0 - - module unload gsibec crtm + - module unload gsibec crtm fms + - module load fms/2023.04 - module load esmf python py-pyyaml py-numpy pflogger fargparse zlib-ng cmake configure: -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" From 500acdcd70cdfbd4ddae3ceff9a3ac85048dc961 Mon Sep 17 00:00:00 2001 From: dooruk Date: Tue, 16 Jul 2024 16:01:56 -0400 Subject: [PATCH 03/10] special handling for fv3-interface file --- src/jedi_bundle/clone_jedi_bundle.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py index e1ac3be..9202faf 100644 --- a/src/jedi_bundle/clone_jedi_bundle.py +++ b/src/jedi_bundle/clone_jedi_bundle.py @@ -10,8 +10,8 @@ import copy -import re import os +import re from jedi_bundle.config.config import return_config_path from jedi_bundle.utils.config import config_get @@ -295,6 +295,9 @@ def clone_jedi(logger, clone_config): output_file_open.write(jedi_cmake_line + '\n') else: + # Add include(fv3-interface.cmake) line if repo is fv3 + if repo == 'fv3': + output_file_open.write(' include(jedi-bundle/fv3-interface.cmake )\n') output_file_open.write(package_line + '\n') if cmake != '': From 6533c252837a19ce089641f5e077b271fb1f1eec Mon Sep 17 00:00:00 2001 From: dooruk Date: Tue, 16 Jul 2024 16:03:37 -0400 Subject: [PATCH 04/10] ignore temp files --- pycodestyle.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/pycodestyle.cfg b/pycodestyle.cfg index 25b0307..122297f 100644 --- a/pycodestyle.cfg +++ b/pycodestyle.cfg @@ -8,3 +8,4 @@ max-line-length = 100 indent-size = 4 statistics = True +exclude = ._* From 08e4b944d9f982218424ac9c0e32ff0df065654a Mon Sep 17 00:00:00 2001 From: dooruk Date: Fri, 19 Jul 2024 16:49:14 -0400 Subject: [PATCH 05/10] saber depends on vader --- src/jedi_bundle/config/bundles/build-order.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/jedi_bundle/config/bundles/build-order.yaml b/src/jedi_bundle/config/bundles/build-order.yaml index 67baf26..d8030fc 100644 --- a/src/jedi_bundle/config/bundles/build-order.yaml +++ b/src/jedi_bundle/config/bundles/build-order.yaml @@ -10,6 +10,10 @@ - oops: default_branch: develop +# Variable changes +- vader: + default_branch: develop + # Background error models - gsibec: repo_url_name: GSIbec @@ -45,9 +49,6 @@ - ufo: default_branch: develop -# Variable changes -- vader: - default_branch: develop # Soca - icepack: From 27ca30563e2445a45d59d4ae016b45565cf3a765 Mon Sep 17 00:00:00 2001 From: dooruk Date: Fri, 19 Jul 2024 16:49:59 -0400 Subject: [PATCH 06/10] make sure cmake includes the fv3 folder --- src/jedi_bundle/clone_jedi_bundle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py index 9202faf..79865a8 100644 --- a/src/jedi_bundle/clone_jedi_bundle.py +++ b/src/jedi_bundle/clone_jedi_bundle.py @@ -298,6 +298,8 @@ def clone_jedi(logger, clone_config): # Add include(fv3-interface.cmake) line if repo is fv3 if repo == 'fv3': output_file_open.write(' include(jedi-bundle/fv3-interface.cmake )\n') + output_file_open.write(f' list( APPEND CMAKE_BUILD_RPATH ' + '${CMAKE_CURRENT_BINARY_DIR}/fv3 )\n') output_file_open.write(package_line + '\n') if cmake != '': From e85f803ff67da7df07a262764f2e0271ee6ec8ea Mon Sep 17 00:00:00 2001 From: dooruk Date: Mon, 22 Jul 2024 11:20:00 -0400 Subject: [PATCH 07/10] set upper limit for pycodestyle --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b974c71..d34761c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ PyYAML>=5.4 -pycodestyle>=2.8.0 +pycodestyle>=2.8.0,<2.12.0 requests>=2.23.0 From 9fea23bec55019049139b28e974d784a17564bff Mon Sep 17 00:00:00 2001 From: dooruk Date: Mon, 22 Jul 2024 11:20:38 -0400 Subject: [PATCH 08/10] add method to clone single file from a repo --- src/jedi_bundle/clone_jedi_bundle.py | 14 ++++++++-- src/jedi_bundle/config/bundles/fv3-jedi.yaml | 1 - src/jedi_bundle/utils/git.py | 29 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py index 79865a8..8370d1f 100644 --- a/src/jedi_bundle/clone_jedi_bundle.py +++ b/src/jedi_bundle/clone_jedi_bundle.py @@ -16,7 +16,7 @@ from jedi_bundle.config.config import return_config_path from jedi_bundle.utils.config import config_get from jedi_bundle.utils.file_system import check_for_executable -from jedi_bundle.utils.git import get_url_and_branch, clone_git_repo +from jedi_bundle.utils.git import get_url_and_branch, clone_git_repo, clone_git_file from jedi_bundle.utils.yaml import load_yaml @@ -226,6 +226,14 @@ def clone_jedi(logger, clone_config): if repo == 'jedicmake': logger.info(f'Skipping explicit clone of \'{repo}\' since it\'s usually a module. ' + f'If it\'s not a module it will be cloned at configure time.') + elif repo == 'fv3': + logger.info('Cloning fv3-interface.cmake from the jedi-bundle repo for fv3') + found, url, branch, \ + is_tag, is_commit = get_url_and_branch(logger, github_orgs, 'jedi-bundle', + default_branch, user_branch, + False, False) + + clone_git_file(logger, url, ['fv3-interface.cmake'], path_to_source, depth=1) else: logger.info(f'Cloning \'{repo}\'.') clone_git_repo(logger, url, branch, @@ -297,8 +305,8 @@ def clone_jedi(logger, clone_config): else: # Add include(fv3-interface.cmake) line if repo is fv3 if repo == 'fv3': - output_file_open.write(' include(jedi-bundle/fv3-interface.cmake )\n') - output_file_open.write(f' list( APPEND CMAKE_BUILD_RPATH ' + output_file_open.write(' include(fv3-interface.cmake )\n') + output_file_open.write(f' list( APPEND CMAKE_INSTALL_RPATH ' '${CMAKE_CURRENT_BINARY_DIR}/fv3 )\n') output_file_open.write(package_line + '\n') diff --git a/src/jedi_bundle/config/bundles/fv3-jedi.yaml b/src/jedi_bundle/config/bundles/fv3-jedi.yaml index 6b58389..b9b5b81 100644 --- a/src/jedi_bundle/config/bundles/fv3-jedi.yaml +++ b/src/jedi_bundle/config/bundles/fv3-jedi.yaml @@ -11,7 +11,6 @@ required_repos: - crtm - ufo - vader - - jedi-bundle - fv3 - fv3-jedi-lm - femps diff --git a/src/jedi_bundle/utils/git.py b/src/jedi_bundle/utils/git.py index 9019ffc..a697166 100644 --- a/src/jedi_bundle/utils/git.py +++ b/src/jedi_bundle/utils/git.py @@ -149,6 +149,35 @@ def get_url_and_branch(logger, github_orgs, repo_url_name, default_branch, # -------------------------------------------------------------------------------------------------- +def clone_git_file( + logger: Logger, + url: str, + files: list, + src_dir: str, + depth: int = 1, +) -> None: + + target = os.path.join(src_dir, 'tmp_bundle') + if os.path.exists(target): + subprocess_run(logger, ["rm", "-rf", target], cwd=src_dir) + + # Clone the temporary repository + # Depth allows for shallow cloning and saves time and space! + subprocess_run(logger, ["git", "clone", + "--depth", str(depth), + url, + target]) + + # Copy the file(s) + for file in files: + git_file = os.path.join(target, file) + subprocess_run(logger, ["cp", git_file, src_dir]) + + # Remove the temporary directory + subprocess_run(logger, ["rm", "-rf", target], cwd=src_dir) + + +# -------------------------------------------------------------------------------------------------- def clone_git_repo(logger, url, branch, target, is_tag, is_commit): From a2a25be42fcada1aba92a3a6161ae67677f20c4a Mon Sep 17 00:00:00 2001 From: dooruk Date: Mon, 22 Jul 2024 11:29:41 -0400 Subject: [PATCH 09/10] clone fv3 repo in conditional --- src/jedi_bundle/clone_jedi_bundle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py index 8370d1f..65f6a2e 100644 --- a/src/jedi_bundle/clone_jedi_bundle.py +++ b/src/jedi_bundle/clone_jedi_bundle.py @@ -234,6 +234,9 @@ def clone_jedi(logger, clone_config): False, False) clone_git_file(logger, url, ['fv3-interface.cmake'], path_to_source, depth=1) + logger.info('Cloning fv3.') + clone_git_repo(logger, url, branch, + os.path.join(path_to_source, repo), is_tag, is_commit) else: logger.info(f'Cloning \'{repo}\'.') clone_git_repo(logger, url, branch, From 133bfa38c3a30ed2a443df53ce3ce3d652402264 Mon Sep 17 00:00:00 2001 From: dooruk Date: Tue, 30 Jul 2024 11:10:53 -0400 Subject: [PATCH 10/10] GSW before VADER --- src/jedi_bundle/clone_jedi_bundle.py | 4 ++-- src/jedi_bundle/config/bundles/build-order.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py index 65f6a2e..bf8f3b6 100644 --- a/src/jedi_bundle/clone_jedi_bundle.py +++ b/src/jedi_bundle/clone_jedi_bundle.py @@ -228,12 +228,12 @@ def clone_jedi(logger, clone_config): f'If it\'s not a module it will be cloned at configure time.') elif repo == 'fv3': logger.info('Cloning fv3-interface.cmake from the jedi-bundle repo for fv3') - found, url, branch, \ + found, url_tmp, branch_tmp, \ is_tag, is_commit = get_url_and_branch(logger, github_orgs, 'jedi-bundle', default_branch, user_branch, False, False) - clone_git_file(logger, url, ['fv3-interface.cmake'], path_to_source, depth=1) + clone_git_file(logger, url_tmp, ['fv3-interface.cmake'], path_to_source, depth=1) logger.info('Cloning fv3.') clone_git_repo(logger, url, branch, os.path.join(path_to_source, repo), is_tag, is_commit) diff --git a/src/jedi_bundle/config/bundles/build-order.yaml b/src/jedi_bundle/config/bundles/build-order.yaml index d8030fc..d392ab3 100644 --- a/src/jedi_bundle/config/bundles/build-order.yaml +++ b/src/jedi_bundle/config/bundles/build-order.yaml @@ -11,6 +11,9 @@ default_branch: develop # Variable changes +- gsw: + repo_url_name: GSW-Fortran + default_branch: develop - vader: default_branch: develop @@ -32,9 +35,6 @@ default_branch: develop - geos-aero: default_branch: develop -- gsw: - repo_url_name: GSW-Fortran - default_branch: develop - ropp-ufo: default_branch: develop repo_url_name: ropp-test