User-defined "pinned versions" for JEDI repos (#38)

* adding pinned version handling * coding norms * commit clone and check fix * adding github action * action name * debugging github action * debugging * adding back pinned versions * debugging actions * chdir issue on action * clean up * more clean up * Update README.md * modifications based on comments * yaml fix * testing github action changes * github action changes * adding to tests * github action fix * github action fix * adding comment * changes based on comments * jedi_bundle's first type hint Co-authored-by: Alexey Shiklomanov <[email protected]> * import typing * Remove unnecessary `import typing` --------- Co-authored-by: Alexey Shiklomanov <[email protected]>
GEOS-ESM · Jun 28, 2024 · ba8cb07 · ba8cb07
1 parent ed1aeed
commit ba8cb07
Show file tree

Hide file tree

Showing 9 changed files with 158 additions and 47 deletions.
diff --git a/.github/workflows/build_with_pinned_versions.yml b/.github/workflows/build_with_pinned_versions.yml
@@ -0,0 +1,41 @@
+name: JEDI Bundle Pinned Version Test
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+
+jobs:
+  buildbundle:
+    name: Run jedi_bundle with pinned_versions
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+
+    # Setup Python
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    # Clone the code repo
+    - name: Clone code repo
+      uses: actions/checkout@v4
+      with:
+        lfs: true
+
+    - name: Install JEDI bundle and dependencies
+      run: python -m pip install --use-deprecated=legacy-resolver -r requirements.txt --user .
+    - name: Put executables in the path
+      run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    # Run an application test
+    - name: Run JEDI Bundle application test
+      run: jedi_bundle --pinned_versions
+
+    # Check if build.yaml has pinned_versions
+    - name: Check for pinned_versions
+      run: python src/jedi_bundle/tests/test_pinned_versions.py ${GITHUB_WORKSPACE}/build.yaml
diff --git a/.github/workflows/clone_a_jedi_bundle.yml b/.github/workflows/clone_a_jedi_bundle.yml
@@ -14,29 +14,19 @@ jobs:
 
     # Setup Python
     - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
 
-    # Update conda
-    - name: Update conda
-      run: conda update -n base -c defaults conda
-
-    # Install pip
-    - name: Install pip
-      run: conda install pip
-
     # Clone the code repo
     - name: Clone code repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         lfs: true
 
     # Install package
-    - name: Upgrade pip
-      run: $CONDA/bin/pip3 install --upgrade pip
     - name: Install JEDI bundle and dependencies
-      run: $CONDA/bin/pip3 install --use-deprecated=legacy-resolver -r requirements.txt --user .
+      run: python -m pip install --use-deprecated=legacy-resolver -r requirements.txt --user .
     - name: Put executables in the path
       run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 

diff --git a/README.md b/README.md
@@ -5,6 +5,7 @@
 | Test      | Status  |
 | --------- | --------|
 | Clone a JEDI bundle test | ![Status](https://github.com/GEOS-ESM/jedi_bundle/actions/workflows/clone_a_jedi_bundle.yml/badge.svg) |
+| Clone with pinned version test | ![Status](https://github.com/GEOS-ESM/jedi_bundle/actions/workflows/clone_with_pinned_version.yml/badge.svg) |
 | Python coding norms | ![Status](https://github.com/GEOS-ESM/jedi_bundle/actions/workflows/python_coding_norms.yml/badge.svg) |
 | YAML coding norms | ![Status](https://github.com/GEOS-ESM/jedi_bundle/actions/workflows/yaml_coding_norms.yml/badge.svg) |
 

diff --git a/src/jedi_bundle/bin/jedi_bundle.py b/src/jedi_bundle/bin/jedi_bundle.py
@@ -74,6 +74,8 @@ def execute_tasks(tasks, config_dict):
 
     # Run the build stages
     if 'all' in tasks or 'clone' in tasks:
+        if 'pinned_versions' in config_dict:
+            clone_dict['pinned_versions'] = config_dict['pinned_versions']
         clone_jedi(logger, clone_dict)
     if 'all' in tasks or 'configure' in tasks:
         configure_jedi(logger, configure_dict)
@@ -105,6 +107,8 @@ def jedi_bundle():
                              '  jedi_bundle all build.yaml              (All tasks) \n' +
                              '  jedi_bundle Clone build.yaml            (Clone task) \n'
                              '  jedi_bundle Clone Configure build.yaml  (Clone & Configure task)\n')
+    parser.add_argument('--pinned_versions', '-pv', action='store_true',
+                        help='Invoke flag to use pinned versions instead of defaults')
 
     # Write the welcome message
     write_welcome_message()
@@ -115,6 +119,7 @@ def jedi_bundle():
     # Parse input string
     args = parser.parse_args()
     tasks_and_config = args.tasks_and_config
+    pinned_versions = args.pinned_versions
 
     # If there are no arguments create build.yaml and exit
     if tasks_and_config == []:
@@ -153,6 +158,12 @@ def jedi_bundle():
 
         internal_config_dict['configure_options']['path_to_build'] = build_dir
 
+        # Add pinned_versions to build config if applicable
+        if pinned_versions:
+            pinned_config_file = os.path.join(return_config_path(), 'pinned_versions.yaml')
+            pinned_versions_dict = load_yaml(logger, pinned_config_file)
+            internal_config_dict['pinned_versions'] = pinned_versions_dict
+
         # Set path to new file and remove if existing
         config_file = os.path.join(os.getcwd(), 'build.yaml')
         prompt_and_remove_file(logger, config_file)

diff --git a/src/jedi_bundle/clone_jedi_bundle.py b/src/jedi_bundle/clone_jedi_bundle.py
@@ -33,6 +33,10 @@ def clone_jedi(logger, clone_config):
     path_to_source = config_get(logger, clone_config, 'path_to_source')
     extra_repos = config_get(logger, clone_config, 'extra_repos')
     crtm_tag_or_branch = config_get(logger, clone_config, 'crtm_tag_or_branch', 'v2.4-jedi.2')
+    if 'pinned_versions' in clone_config:
+        pinned_versions = config_get(logger, clone_config, 'pinned_versions')
+    else:
+        pinned_versions = None
 
     # Check for needed executables
     # ----------------------------
@@ -135,6 +139,7 @@ def clone_jedi(logger, clone_config):
     cmakelists_list = []
     recursive_list = []
     is_tag_list = []
+    is_commit_list = []
 
     optional_repos_not_found = []
 
@@ -145,14 +150,28 @@ def clone_jedi(logger, clone_config):
         # Extract repo information
         repo_dict = build_order_dict[repo]
         repo_url_name = config_get(logger, repo_dict, 'repo_url_name', repo)
-        default_branch = config_get(logger, repo_dict, 'default_branch')
         cmakelists = config_get(logger, repo_dict, 'cmakelists', '')
         recursive = config_get(logger, repo_dict, 'recursive', False)
+        default_branch = config_get(logger, repo_dict, 'default_branch')
         is_tag_in = config_get(logger, repo_dict, 'tag', False)
-
-        found, url, branch, is_tag = get_url_and_branch(logger, github_orgs, repo_url_name,
-                                                        default_branch, user_branch, is_tag_in)
-
+        is_commit_in = config_get(logger, repo_dict, 'commit', False)
+
+        # Extract information from pinned_versions if applicable
+        if pinned_versions:
+            for d in pinned_versions:
+                if repo in d:
+                    if 'branch' in d[repo]:
+                        default_branch = d[repo]['branch']
+                    if 'tag' in d[repo]:
+                        is_tag_in = d[repo]['tag']
+                    if 'commit' in d[repo]:
+                        is_commit_in = d[repo]['commit']
+                    break
+
+        found, url, branch, \
+            is_tag, is_commit = get_url_and_branch(logger, github_orgs, repo_url_name,
+                                                   default_branch, user_branch,
+                                                   is_tag_in, is_commit_in)
         if found:
 
             # List for writing CMakeLists.txt
@@ -162,6 +181,7 @@ def clone_jedi(logger, clone_config):
             cmakelists_list.append(cmakelists)
             recursive_list.append(recursive)
             is_tag_list.append(is_tag)
+            is_commit_list.append(is_commit)
 
         else:
 
@@ -180,10 +200,13 @@ def clone_jedi(logger, clone_config):
     logger.info(f'Repository clone summary:')
     logger.info(f'-------------------------')
 
-    for repo, url, branch, is_tag in zip(repo_list, url_list, branch_list, is_tag_list):
+    for repo, url, branch, is_tag, is_commit in zip(repo_list, url_list, branch_list,
+                                                    is_tag_list, is_commit_list):
         branch_or_tag = 'Branch'
         if is_tag:
             branch_or_tag = 'Tag'
+        if is_commit:
+            branch_or_tag = 'Commit'
         logger.info(f'{branch_or_tag.ljust(6)} {branch.ljust(branch_len)} of ' +
                     f'{repo.ljust(repo_len)} will be cloned from {url.ljust(url_len)}')
 
@@ -196,15 +219,17 @@ def clone_jedi(logger, clone_config):
 
     # Do the cloning
     # --------------
-    for repo, url, branch, is_tag in zip(repo_list, url_list, branch_list, is_tag_list):
+    for repo, url, branch, is_tag, is_commit in zip(repo_list, url_list, branch_list,
+                                                    is_tag_list, is_commit_list):
 
         # Special treatment for jedicmake since it is typically part of spack.
         if repo == 'jedicmake':
             logger.info(f'Skipping explicit clone of \'{repo}\' since it\'s usually a module. ' +
                         f'If it\'s not a module it will be cloned at configure time.')
         else:
             logger.info(f'Cloning \'{repo}\'.')
-            clone_git_repo(logger, url, branch, os.path.join(path_to_source, repo), is_tag)
+            clone_git_repo(logger, url, branch,
+                           os.path.join(path_to_source, repo), is_tag, is_commit)
 
     # Create CMakeLists.txt file
     # --------------------------
@@ -221,18 +246,13 @@ def clone_jedi(logger, clone_config):
     url_len = len(max(url_list, key=len))+2  # Plus 2 because of the quotes
     branch_len = len(max(branch_list, key=len))
 
-    # Remove file if it exists
-    if os.path.exists(output_file):
-        os.remove(output_file)
-
-    with open(output_file, 'a') as output_file_open:
+    with open(output_file, 'w') as output_file_open:
         for cmake_header_line in cmake_header_lines:
             output_file_open.write(cmake_header_line + '\n')
 
-        for repo, url, branch, cmake, recursive, is_tag in zip(repo_list, url_list, branch_list,
-                                                               cmakelists_list, recursive_list,
-                                                               is_tag_list):
-
+        for repo, url, branch, cmake, recursive, \
+            is_tag, is_commit in zip(repo_list, url_list, branch_list, cmakelists_list,
+                                     recursive_list, is_tag_list, is_commit_list):
             urlq = f'\"{url}\"'
 
             # Default cloning options
@@ -245,6 +265,11 @@ def clone_jedi(logger, clone_config):
                 branch_or_tag = 'TAG'
                 update = ''
 
+            # if commit, proceed as you would with branch but turn off update
+            if is_commit:
+                branch_or_tag = 'BRANCH'
+                update = ''
+
             # Add recursive if needed
             if recursive:
                 recursive_clone = 'RECURSIVE'

diff --git a/src/jedi_bundle/config/pinned_versions.yaml b/src/jedi_bundle/config/pinned_versions.yaml
@@ -0,0 +1,3 @@
+- ufo:
+    branch: 64655cf61a9d22c88c0f20e740554ca5225790bc
+    commit: True
diff --git a/src/jedi_bundle/tests/test_pinned_versions.py b/src/jedi_bundle/tests/test_pinned_versions.py
@@ -0,0 +1,15 @@
+import sys
+from jedi_bundle.utils.logger import Logger
+from jedi_bundle.utils.yaml import load_yaml
+
+'''
+    Script to search for 'pinned_versions' keyword in build.yaml
+    during github action test.
+'''
+
+logger = Logger('Pinned Versions Test')
+build_yaml_path = sys.argv[1]
+build_dict = load_yaml(logger, build_yaml_path)
+
+if 'pinned_versions' not in build_dict:
+    raise Exception('Pinned versions not found in build_dict')
diff --git a/src/jedi_bundle/utils/file_system.py b/src/jedi_bundle/utils/file_system.py
@@ -63,10 +63,9 @@ def check_for_executable(logger, executable):
 # --------------------------------------------------------------------------------------------------
 
 
-def subprocess_run(logger, command, abort_on_fail=True):
+def subprocess_run(logger, command, abort_on_fail=True, **kwargs):
 
-    # Prepare command
-    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
 
     # Run process
     output, error = p.communicate()