From 5d52e841c21a5e0171a10fecba5422a7841781cd Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Mon, 18 Apr 2022 14:21:35 -0400 Subject: [PATCH 1/2] switch to multiprocessing module that provides exit codes; JIRA: PGAPX-1005 --- scripts/pgap.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/pgap.py b/scripts/pgap.py index fa6d035..5ee72fe 100755 --- a/scripts/pgap.py +++ b/scripts/pgap.py @@ -15,6 +15,7 @@ import contextlib import glob import json +import multiprocessing as mp import os import platform import queue @@ -22,7 +23,6 @@ import shutil import subprocess import tarfile -import threading import time import tempfile import xml @@ -655,17 +655,22 @@ def str2sec(s): def update(self): self.update_self() threads = list() - docker_thread = threading.Thread(target = self.install_docker) + docker_thread = mp.Process(target = self.install_docker) docker_thread.start() threads.append(docker_thread) # precreate the directory where the tarfile will be unloaded. os.makedirs(f"{self.install_dir}/input-{self.use_version}/uniColl_path", exist_ok=True) self.install_data(threads) - genomes_thread = threading.Thread(target = self.install_test_genomes) + genomes_thread = mp.Process(target = self.install_test_genomes) genomes_thread.start() threads.append(genomes_thread) + global_exit_value = 0 for thread in threads: thread.join() + if thread.exitcode != 0: + global_exit_value = thread.exitcode + if global_exit_value != 0: + raise Exception('installation of at least some of reference data components or docker image failed') self.write_version() def install_docker(self): @@ -723,7 +728,7 @@ def install_data(self, threads): else: remote_path = f"https://s3.amazonaws.com/pgap/input-{self.use_version}.{suffix}{package}.tgz" if not os.path.isfile(guard_file): - url_thread = threading.Thread(target = install_url, args=(remote_path, self.install_dir, self.args.quiet, self.args.teamcity, guard_file, )) + url_thread = mp.Process(target = install_url, args=(remote_path, self.install_dir, self.args.quiet, self.args.teamcity, guard_file, )) url_thread.start() threads.append(url_thread) else: From 505b91e41741ccbcd5ebd2b6a09a3be604f9ece3 Mon Sep 17 00:00:00 2001 From: "Badretdin, Azat" Date: Tue, 19 Apr 2022 06:35:53 -0400 Subject: [PATCH 2/2] added descriptive names for parallel installation processes and more detailed error diagnostics for the case of failures during installation; JIRA: PGAPX-1005 --- scripts/pgap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/pgap.py b/scripts/pgap.py index 5ee72fe..5628f47 100755 --- a/scripts/pgap.py +++ b/scripts/pgap.py @@ -655,13 +655,13 @@ def str2sec(s): def update(self): self.update_self() threads = list() - docker_thread = mp.Process(target = self.install_docker) + docker_thread = mp.Process(target = self.install_docker, name='docker image pull') docker_thread.start() threads.append(docker_thread) # precreate the directory where the tarfile will be unloaded. os.makedirs(f"{self.install_dir}/input-{self.use_version}/uniColl_path", exist_ok=True) self.install_data(threads) - genomes_thread = mp.Process(target = self.install_test_genomes) + genomes_thread = mp.Process(target = self.install_test_genomes, name='test genomes installation') genomes_thread.start() threads.append(genomes_thread) global_exit_value = 0 @@ -670,7 +670,7 @@ def update(self): if thread.exitcode != 0: global_exit_value = thread.exitcode if global_exit_value != 0: - raise Exception('installation of at least some of reference data components or docker image failed') + raise Exception(f'installation of some or all of components failed. Please remove {self.data_path}, {self.install_dir}/test_genomes, {self.test_genomes_path} directories and try again.') self.write_version() def install_docker(self): @@ -728,7 +728,7 @@ def install_data(self, threads): else: remote_path = f"https://s3.amazonaws.com/pgap/input-{self.use_version}.{suffix}{package}.tgz" if not os.path.isfile(guard_file): - url_thread = mp.Process(target = install_url, args=(remote_path, self.install_dir, self.args.quiet, self.args.teamcity, guard_file, )) + url_thread = mp.Process(target = install_url, name='{package} installation',args=(remote_path, self.install_dir, self.args.quiet, self.args.teamcity, guard_file, )) url_thread.start() threads.append(url_thread) else: