Compare VSC setup to EESSI develop branch

eessi_bot_event_handler.py

@@ -453,10 +453,11 @@ def handle_bot_command_build(self, event_info, bot_command):
         repo_name = event_info['raw_request_body']['repository']['full_name']
         pr_number = event_info['raw_request_body']['issue']['number']
         pr = gh.get_repo(repo_name).get_pull(pr_number)
+        github_token = self.gh._Github__requester._Requester__auth.token
         build_msg = ''
         if check_build_permission(pr, event_info):
             # use filter from command
-            submitted_jobs = submit_build_jobs(pr, event_info, bot_command.action_filters)
+            submitted_jobs = submit_build_jobs(pr, event_info, bot_command.action_filters, github_token)
             if submitted_jobs is None or len(submitted_jobs) == 0:
                 build_msg = "\n  - no jobs were submitted"
             else:

eessi_bot_job_manager.py

@@ -123,12 +123,14 @@ def get_current_jobs(self):
         for line in lines:
             job = line.rstrip().split()
             if len(job) >= 9:
-                job_id = job[0]
-                state = job[4]
+                job_id = job[1]
+                cluster = job[0]
+                state = job[5]
                 current_jobs[job_id] = {
                     "jobid": job_id,
                     "state": state,
-                    "reason": job[8],
+                    "cluster": cluster,
+                    "reason": job[9],
                 }
                 if state in bad_state_messages:
                     log("Job {} in state {}: {}".format(job_id, state, bad_state_messages[state]))
@@ -258,10 +260,12 @@ def process_new_job(self, new_job):
                 is not a bot job
         """
         job_id = new_job["jobid"]
+        cluster = new_job["cluster"]
 
-        scontrol_cmd = "%s --oneliner show jobid %s" % (
+        scontrol_cmd = "%s --oneliner show jobid %s --clusters=%s" % (
             self.scontrol_command,
             job_id,
+            cluster
         )
         scontrol_output, scontrol_err, scontrol_exitcode = run_cmd(
             scontrol_cmd,
@@ -304,9 +308,10 @@ def process_new_job(self, new_job):
             )
             os.symlink(match.group(1), symlink_source)
 
-            release_cmd = "%s release %s" % (
+            release_cmd = "%s release %s --clusters=%s" % (
                 self.scontrol_command,
                 job_id,
+                cluster
             )
 
             release_output, release_err, release_exitcode = run_cmd(

scripts/bot-build.slurm

@@ -23,7 +23,7 @@
 #    for example, repos.cfg and configuration file bundles for repositories
 
 echo "Starting bot-build.slurm"
-BOT_BUILD_SCRIPT=bot/build.sh
+BOT_BUILD_SCRIPT=bot/test-build-vsc.sh
 if [ -f ${BOT_BUILD_SCRIPT} ]; then
     echo "${BOT_BUILD_SCRIPT} script found in '${PWD}', so running it!"
     ${BOT_BUILD_SCRIPT}
@@ -32,7 +32,7 @@ else
     exit 1
 fi
 echo "bot/build.sh finished"
-CHECK_BUILD_SCRIPT=bot/check-build.sh
+CHECK_BUILD_SCRIPT=bot/test-check-build-vsc.sh
 if [ -f ${CHECK_BUILD_SCRIPT} ]; then
     echo "${CHECK_BUILD_SCRIPT} script found in '${PWD}', so running it!"
     ${CHECK_BUILD_SCRIPT}

tasks/build.py

@@ -344,7 +344,7 @@ def create_pr_dir(pr, cfg, event_info):
     return year_month, pr_id, run_dir
 
 
-def download_pr(repo_name, branch_name, pr, arch_job_dir):
+def download_pr(repo_name, branch_name, pr, arch_job_dir, token):
     """
     Download pull request to job working directory
 
@@ -365,7 +365,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir):
     # - 'git checkout' base branch of pull request
     # - 'curl' diff for pull request
     # - 'git apply' diff file
-    git_clone_cmd = ' '.join(['git clone', f'https://github.com/{repo_name}', arch_job_dir])
+    git_clone_cmd = ' '.join(['git clone', f'git@github.com:{repo_name}.git', arch_job_dir])
     log(f'cloning with command {git_clone_cmd}')
     clone_output, clone_error, clone_exit_code = run_cmd(
         git_clone_cmd, "Clone repo", arch_job_dir, raise_on_error=False
@@ -386,7 +386,7 @@ def download_pr(repo_name, branch_name, pr, arch_job_dir):
         error_stage = ERROR_GIT_CHECKOUT
         return checkout_output, checkout_err, checkout_exit_code, error_stage
 
-    curl_cmd = f'curl -L https://github.com/{repo_name}/pull/{pr.number}.diff > {pr.number}.diff'
+    curl_cmd = f'curl -H "Accept: application/vnd.github.diff" -H "Authorization: Bearer {token}"   -H "X-GitHub-Api-Version: 2022-11-28" -L https://api.github.com/repos/{repo_name}/pulls/{pr.number} > {pr.number}.diff'
     log(f'curl with command {curl_cmd}')
     curl_output, curl_error, curl_exit_code = run_cmd(
         curl_cmd, "Obtain patch", arch_job_dir, raise_on_error=False
@@ -481,7 +481,7 @@ def apply_cvmfs_customizations(cvmfs_customizations, arch_job_dir):
             #      for now, only existing mappings may be customized
 
 
-def prepare_jobs(pr, cfg, event_info, action_filter):
+def prepare_jobs(pr, cfg, event_info, action_filter, token):
     """
     Prepare all jobs whose context matches the given filter. Preparation includes
     creating a working directory for a job, downloading the pull request into
@@ -549,7 +549,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
 
             # TODO optimisation? download once, copy and cleanup initial copy?
             download_pr_output, download_pr_error, download_pr_exit_code, error_stage = download_pr(
-                base_repo_name, base_branch_name, pr, job_dir
+                base_repo_name, base_branch_name, pr, job_dir, token
                 )
             comment_download_pr(base_repo_name, pr, download_pr_exit_code, download_pr_error, error_stage)
             # prepare job configuration file 'job.cfg' in directory <job_dir>/cfg
@@ -768,7 +768,7 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink):
         return None
 
 
-def submit_build_jobs(pr, event_info, action_filter):
+def submit_build_jobs(pr, event_info, action_filter, token):
     """
     Create build jobs for a pull request by preparing jobs which match the given
     filters, submitting them, adding comments to the pull request on GitHub and
@@ -790,7 +790,7 @@ def submit_build_jobs(pr, event_info, action_filter):
     app_name = cfg[GITHUB].get(APP_NAME)
 
     # setup job directories (one per element in product of architecture x repositories)
-    jobs = prepare_jobs(pr, cfg, event_info, action_filter)
+    jobs = prepare_jobs(pr, cfg, event_info, action_filter, token)
 
     # return if there are no jobs to be submitted
     if not jobs: