add support for specifying bucket name as mapping with repo id as key

tasks/build.py

@@ -47,6 +47,7 @@
 INITIAL_COMMENT = "initial_comment"
 JOBS_BASE_DIR = "jobs_base_dir"
 JOB_ARCHITECTURE = "architecture"
+JOB_CFG_FILENAME = "job.cfg"
 JOB_CONTAINER = "container"
 JOB_LOCAL_TMP = "local_tmp"
 JOB_HTTPS_PROXY = "https_proxy"
@@ -64,7 +65,6 @@
 LOCAL_TMP = "local_tmp"
 NO_BUILD_PERMISSION_COMMENT = "no_build_permission_comment"
 REPOS_CFG_DIR = "repos_cfg_dir"
-REPOS_ID = "repo_id"
 REPOS_REPO_NAME = "repo_name"
 REPOS_REPO_VERSION = "repo_version"
 REPOS_CONFIG_BUNDLE = "config_bundle"
@@ -198,8 +198,8 @@ def get_repo_cfg(cfg):
         (dict): dictionary containing repository settings as follows
            - {REPOS_CFG_DIR: path to repository config directory as defined in 'app.cfg'}
            - {REPO_TARGET_MAP: json of REPO_TARGET_MAP value as defined in 'app.cfg'}
-           - for all sections [REPO_ID] defined in REPOS_CFG_DIR/repos.cfg add a
-             mapping {REPO_ID: dictionary containing settings of that section}
+           - for all sections [JOB_REPO_ID] defined in REPOS_CFG_DIR/repos.cfg add a
+             mapping {JOB_REPO_ID: dictionary containing settings of that section}
     """
     fn = sys._getframe().f_code.co_name
 
@@ -489,7 +489,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     """
     fn = sys._getframe().f_code.co_name
 
-    jobcfg_dir = os.path.join(job_dir, 'cfg')
+    jobcfg_dir = os.path.join(job_dir, CFG_DIRNAME)
     # create ini file job.cfg with entries:
     # [site_config]
     # local_tmp = LOCAL_TMP_VALUE
@@ -498,7 +498,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     #
     # [repository]
     # repos_cfg_dir = JOB_CFG_DIR
-    # repo_id = REPO_ID
+    # repo_id = JOB_REPO_ID
     # container = CONTAINER
     # repo_name = REPO_NAME
     # repo_version = REPO_VERSION
@@ -555,7 +555,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir,
     # make sure that <jobcfg_dir> exists
     os.makedirs(jobcfg_dir, exist_ok=True)
 
-    jobcfg_file = os.path.join(jobcfg_dir, 'job.cfg')
+    jobcfg_file = os.path.join(jobcfg_dir, JOB_CFG_FILENAME)
     with open(jobcfg_file, "w") as jcf:
         job_cfg.write(jcf)
 

tasks/deploy.py

@@ -22,6 +22,7 @@
 
 # Local application imports (anything from EESSI/eessi-bot-software-layer)
 from connections import github
+from tasks.build import CFG_DIRNAME, JOB_CFG_FILENAME, JOB_REPO_ID, JOB_REPOSITORY
 from tasks.build import get_build_env_cfg
 from tools import config, pr_comments, run_cmd
 
@@ -255,7 +256,28 @@ def upload_tarball(job_dir, build_target, timestamp, repo_name, pr_number):
     deploycfg = cfg[DEPLOYCFG]
     tarball_upload_script = deploycfg.get(TARBALL_UPLOAD_SCRIPT)
     endpoint_url = deploycfg.get(ENDPOINT_URL) or ''
-    bucket_name = deploycfg.get(BUCKET_NAME)
+    bucket_spec = deploycfg.get(BUCKET_NAME)
+
+    jobcfg_path = os.path.join(job_dir, CFG_DIRNAME, JOB_CFG_FILENAME)
+    jobcfg = config.read_config(jobcfg_path)
+    target_repo_id = jobcfg[JOB_REPOSITORY][JOB_REPO_ID]
+
+    if isinstance(bucket_spec, str):
+        bucket_name = bucket_spec
+        log(f"Using specified bucket: {bucket_name}")
+    elif isinstance(bucket_spec, dict):
+        # bucket spec may be a mapping of target repo id to bucket name
+        bucket_name = bucket_spec.get(target_repo_id)
+        if bucket_name is None:
+            update_pr_comment(tarball, repo_name, pr_number, "not uploaded",
+                              f"failed (no bucket specified for {target_repo_id})")
+            return
+        else:
+            log(f"Using bucket for {target_repo_id}: {bucket_name}")
+    else:
+        update_pr_comment(tarball, repo_name, pr_number, "not uploaded",
+                          f"failed (incorrect bucket spec: {bucket_spec})")
+        return
 
     # run 'eessi-upload-to-staging {abs_path}'
     # (1) construct command line