Use submodules rather than ecbuild_bundle git (NOAA-EMC#811)

This PR creates git submodules for each external JEDI library rather than using `ecbuild_bundle` to get the library source code through git. There is also a new set of scripts to automate the update process of the submodules (both an update and a git add, in separate scripts). Peppered through here are also some minor changes for CI testing that are needed due to this change + the global-workflow using submodules. --------- Co-authored-by: Guillaume Vernieres <[email protected]> Co-authored-by: RussTreadon-NOAA <[email protected]> Co-authored-by: RussTreadon-NOAA <[email protected]>
RussTreadon-NOAA · Jan 16, 2024 · 639ef16 · 639ef16
1 parent cc4c940
commit 639ef16
Show file tree

Hide file tree

Showing 58 changed files with 436 additions and 314 deletions.
diff --git a/.gitignore b/.gitignore
@@ -146,33 +146,40 @@ build*/
 install*/
 
 # Ignore the following bundle repositories
-ecbuild/
-atlas/
-crtm/
-eckit/
-fckit/
-femps/
-fms/
-fv3/
-fv3-jedi/
-fv3-jedi-data/
-fv3-jedi-lm/
-gsibec/
-gsw/
-icepack/
-/ioda/
-ioda-data/
-iodaconv/
-jedicmake/
-land-imsproc/
-land-jediincr/
-mom6/
-oops/
-saber/
-saber-data/
-/soca/
-test-data-release/
-ufo/
-ufo-data/
-vader/
+/ecbuild
+/atlas
+/crtm
+/eckit
+/fckit
+/femps
+/fms
+/fv3
+/fv3-jedi
+/fv3-jedi-data
+/fv3-jedi-lm
+/gsibec
+/gsw
+/icepack
+/ioda
+/ioda-data
+/iodaconv
+/jedicmake
+/land-imsproc
+/land-jediincr
+/mom6
+/oops
+/saber
+/saber-data
+/soca
+/test-data-release
+/ufo
+/ufo-data
+/vader
 /gdas-utils
+/sorc/fv3-jedi-data
+/sorc/ufo-data
+/sorc/ioda-data
+/sorc/test-data-release
+/sorc/gdas
+/sorc/gdas-utils
+
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,78 @@
+[submodule "sorc/jedicmake"]
+  path = sorc/jedicmake
+  url = https://github.com/jcsda/jedi-cmake.git
+[submodule "sorc/gsibec"]
+  path = sorc/gsibec
+  url = https://github.com/GEOS-ESM/GSIbec.git
+  branch = develop
+[submodule "sorc/gsw"]
+  path = sorc/gsw
+  url = https://github.com/jcsda-internal/GSW-Fortran.git
+  branch = develop
+[submodule "sorc/oops"]
+  path = sorc/oops
+  url = https://github.com/jcsda/oops.git
+  branch = develop
+[submodule "sorc/vader"]
+  path = sorc/vader
+  url = https://github.com/jcsda/vader.git
+  branch = develop
+[submodule "sorc/saber"]
+  path = sorc/saber
+  url = https://github.com/jcsda/saber.git
+  branch = develop
+[submodule "sorc/ioda"]
+  path = sorc/ioda
+  url = https://github.com/jcsda/ioda.git
+  branch = develop
+[submodule "sorc/ufo"]
+  path = sorc/ufo
+  url = https://github.com/jcsda/ufo.git
+  branch = develop
+[submodule "sorc/fv3"]
+  path = sorc/fv3
+  url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git
+  branch = release-stable
+[submodule "sorc/femps"]
+  path = sorc/femps
+  url = https://github.com/jcsda/femps.git
+  branch = develop
+[submodule "sorc/fv3-jedi-lm"]
+  path = sorc/fv3-jedi-lm
+  url = https://github.com/jcsda/fv3-jedi-linearmodel.git
+  branch = develop
+[submodule "sorc/fv3-jedi"]
+  path = sorc/fv3-jedi
+  url = https://github.com/jcsda/fv3-jedi.git
+  branch = develop
+[submodule "sorc/icepack"]
+  path = sorc/icepack
+  url = https://github.com/JCSDA-internal/Icepack.git
+  branch = feature/ecbuild-new
+[submodule "sorc/mom6"]
+  path = sorc/mom6
+  url = https://github.com/jcsda-internal/MOM6.git
+  branch = main-ecbuild
+[submodule "sorc/soca"]
+  path = sorc/soca
+  url = https://github.com/jcsda-internal/soca.git
+  branch = develop
+[submodule "sorc/iodaconv"]
+  path = sorc/iodaconv
+  url = https://github.com/JCSDA-internal/ioda-converters.git
+  branch = develop
+[submodule "sorc/land-imsproc"]
+  path = sorc/land-imsproc
+  url = https://github.com/NOAA-PSL/land-IMS_proc.git
+  branch = develop
+[submodule "sorc/land-jediincr"]
+  path = sorc/land-jediincr
+  url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git
+  branch = develop
+[submodule "sorc/crtm"]
+  path = sorc/crtm
+  url = https://github.com/jcsda/crtm.git
+[submodule "sorc/fms"]
+	path = sorc/fms
+	url = https://github.com/jcsda/fms.git
+  branch = release-stable
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,5 @@
 # ------------------------------------------------------------------------- #
-# Application for all the components needed for the GDAS system             #
+# Application for the GDAS system             #
 # ------------------------------------------------------------------------- #
 
 # Check for minimim cmake requirement
@@ -36,135 +36,8 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON)
 option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
 option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)
 
-# Initialize bundle
-# -----------------
-ecbuild_bundle_initialize()
-
-# Build bundle source code.
-if(BUILD_GDASBUNDLE)
-
-# jedi-cmake
-  ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" TAG 1.4.0 RECURSIVE )
-  include( jedicmake/cmake/Functions/git_functions.cmake )
-
-# ECMWF libraries
-  option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-  option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
-  option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
-
-# turn off optional OOPS toy models
-  option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF )
-  option( ENABLE_QG_MODEL "Build QG toy model" OFF )
-
-  ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
-  ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
-  ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 )
-
-# External (required) observation operators
-  option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF
-  ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git"   TAG v2.4.1-jedi.1 )
-
-# Build GSI-B
-  option(BUILD_GSIBEC "Build GSI-B" OFF)
-  if(BUILD_GSIBEC)
-    ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.1.2 )
-  endif()
-
-# Gibbs seawater
-  ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop )
-
-# Core JEDI repositories
-  ecbuild_bundle( PROJECT oops  GIT "https://github.com/jcsda/oops.git"  BRANCH develop)
-  ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop )
-  ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop )
-  option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT ioda  GIT "https://github.com/jcsda/ioda.git" BRANCH develop  )
-  option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT ufo   GIT "https://github.com/jcsda/ufo.git" BRANCH develop )
-
-# FMS and FV3 dynamical core
-  ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable )
-  ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable )
-
-# fv3-jedi and associated repositories
-  ecbuild_bundle( PROJECT femps       GIT "https://github.com/jcsda/femps.git"                BRANCH develop )
-  ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop )
-  option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT fv3-jedi    GIT "https://github.com/jcsda/fv3-jedi.git"             BRANCH develop )
-
-# SOCA associated repositories
-# TODO: Move the Icepack fork to EMC github
-  set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library")
-  if ( BUILD_ICEPACK )
-    ecbuild_bundle( PROJECT icepack GIT "https://github.com/JCSDA-internal/Icepack.git" BRANCH feature/ecbuild-new )
-  endif()
-  ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE )
-  ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop )
-
-  # Build JEDI/DA or other peripherals
-  ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" )
-
-# Build IODA converters
-  option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON)
-  if(BUILD_IODA_CONVERTERS)
-    ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop )
-  endif()
-
-# Land associated repositories
-  ecbuild_bundle( PROJECT land-imsproc  GIT "https://github.com/NOAA-PSL/land-IMS_proc.git"  TAG 6373819 )
-  ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git"  TAG 2923344)
-
-# ioda, ufo, fv3-jedi, and saber test data
-#---------------------------------
-if(CLONE_JCSDADATA)
-
-  # If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
-  # If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
-  # IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
-  # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
-  # in a local directory, ioda-data repo will not be cloned
-
-    find_branch_name(REPO_DIR_NAME ioda)
-  # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored
-  # in a local directory, ioda-data repo will not be cloned
-    if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
-      ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop )
-
-      # If IODA's current branch is available in ioda-data repo, that branch will be checked out
-      branch_checkout (REPO_DIR_NAME ioda-data
-                      BRANCH ${GIT_BRANCH_FUNC} )
-    endif()
-
-  # same procedure for ufo-data
-    find_branch_name(REPO_DIR_NAME ufo)
-    if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
-      ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop )
-
-      # If UFO's current branch is available in ioda-data repo, that branch will be checked out
-      branch_checkout (REPO_DIR_NAME ufo-data
-                      BRANCH ${GIT_BRANCH_FUNC} )
-    endif()
-
-  # same procedure for fv3-jedi-data
-    find_branch_name(REPO_DIR_NAME fv3-jedi)
-    if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC )
-      ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop )
-
-      # If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out
-      branch_checkout (REPO_DIR_NAME fv3-jedi-data
-                      BRANCH ${GIT_BRANCH_FUNC} )
-    endif()
-
-  endif(CLONE_JCSDADATA)
-
-endif(BUILD_GDASBUNDLE)
-
 # Install utility scripts.
 add_subdirectory(ush)
 
 # Include testing.
 add_subdirectory(test)
-
-# Finalize bundle
-# ---------------
-ecbuild_bundle_finalize()
diff --git a/build.sh b/build.sh
@@ -113,7 +113,7 @@ echo "Configuring ..."
 set -x
 cmake \
   ${CMAKE_OPTS:-} \
-  $dir_root
+  $dir_root/sorc
 set +x
 
 # Build

diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh
@@ -64,19 +64,10 @@ cd $stableroot/$datestr/global-workflow/sorc/gdas.cd
 git checkout develop
 git pull
 
-# ==============================================================================
-# run ecbuild to get the repos cloned
-mkdir -p build
-
-cd build
-ecbuild ../
-cd ..
-rm -rf build
-
 # ==============================================================================
 # update the hashes to the most recent
 gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd
-$my_dir/stable_mark.sh $gdasdir
+$my_dir/../ush/submodules/update_develop.sh $gdasdir
 
 # ==============================================================================
 # run the automated testing
@@ -85,40 +76,33 @@ ci_status=$?
 total=0
 if [ $ci_status -eq 0 ]; then
   cd $gdasdir
-  # copy the CMakeLists file for safe keeping
-  cp $gdasdir/CMakeLists.txt $gdasdir/CMakeLists.txt.new
-  total=$(($total+$?))
-  if [ $total -ne 0 ]; then
-    echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
-  fi
   # checkout feature/stable-nightly
   git stash
   total=$(($total+$?))
   if [ $total -ne 0 ]; then
-    echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
+    echo "Unable to git stash" >> $stableroot/$datestr/output
   fi
   git checkout feature/stable-nightly
   total=$(($total+$?))
   if [ $total -ne 0 ]; then
-    echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output
+    echo "Unable to checkout feature/stable-nightly" >> $stableroot/$datestr/output
   fi
   # merge in develop
   git merge develop
   total=$(($total+$?))
   if [ $total -ne 0 ]; then
     echo "Unable to merge develop" >> $stableroot/$datestr/output
   fi
-  # force move the copy to the original path of CMakeLists.txt
-  /bin/mv -f $gdasdir/CMakeLists.txt.new $gdasdir/CMakeLists.txt
+  # add in submodules
+  git stash pop
   total=$(($total+$?))
   if [ $total -ne 0 ]; then
-    echo "Unable to mv CMakeLists" >> $stableroot/$datestr/output
+    echo "Unable to git stash pop" >> $stableroot/$datestr/output
   fi
-  # commit this change and push
-  git add CMakeLists.txt
+  $my_dir/../ush/submodules/add_submodules.sh $gdasdir
   total=$(($total+$?))
   if [ $total -ne 0 ]; then
-    echo "Unable to add CMakeLists to commit" >> $stableroot/$datestr/output
+    echo "Unable to add updated submodules to commit" >> $stableroot/$datestr/output
   fi
   git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr"
   total=$(($total+$?))
-Original file line number
+Diff line change
@@ Expand Up / @@ -113,7 +113,7 @@ echo "Configuring ..." @@
     set -x
     cmake \
       ${CMAKE_OPTS:-} \
-      $dir_root
+      $dir_root/sorc
     set +x
     # Build
@@ Expand Down @@