From d0cd9618ed6285a880c6de8c3028418030d71cf0 Mon Sep 17 00:00:00 2001 From: brcopeland Date: Mon, 1 Aug 2016 15:00:41 -0400 Subject: [PATCH 1/2] Added --dont-remove-tmp-dir parameter to SGEJobTask. --- luigi/contrib/sge.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/luigi/contrib/sge.py b/luigi/contrib/sge.py index dd217eeae1..ff0a430a08 100755 --- a/luigi/contrib/sge.py +++ b/luigi/contrib/sge.py @@ -170,6 +170,7 @@ class SGEJobTask(luigi.Task): by StarCluster - run_locally: Run locally instead of on the cluster. - poll_time: the length of time to wait in order to poll qstat + - dont_remove_tmp_dir: Instead of deleting the temporary directory, keep it. """ @@ -177,11 +178,14 @@ class SGEJobTask(luigi.Task): shared_tmp_dir = luigi.Parameter(default='/home', significant=False) parallel_env = luigi.Parameter(default='orte', significant=False) run_locally = luigi.BoolParameter( - default=False, significant=False, + significant=False, description="run locally instead of on the cluster") poll_time = luigi.IntParameter( significant=False, default=POLL_TIME, description="specify the wait time to poll qstat for the job status") + dont_remove_tmp_dir = luigi.BoolParameter( + significant=False, + description="don't delete the temporary directory used (for debugging)") def _fetch_task_failures(self): if not os.path.exists(self.errfile): @@ -271,7 +275,8 @@ def _run_job(self): self._track_job() # Now delete the temporaries, if they're there. - if self.tmp_dir and os.path.exists(self.tmp_dir): + if (self.tmp_dir and os.path.exists(self.tmp_dir) and + not self.dont_remove_tmp_dir): logger.info('Removing temporary directory %s' % self.tmp_dir) subprocess.call(["rm", "-rf", self.tmp_dir]) From 44dcd299afca1cb2985db0783ca21b0e765f78ab Mon Sep 17 00:00:00 2001 From: brcopeland Date: Mon, 1 Aug 2016 19:57:28 -0400 Subject: [PATCH 2/2] Changed line formatting to match luigi style. --- luigi/contrib/sge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/luigi/contrib/sge.py b/luigi/contrib/sge.py index ff0a430a08..78fa28b13a 100755 --- a/luigi/contrib/sge.py +++ b/luigi/contrib/sge.py @@ -275,8 +275,7 @@ def _run_job(self): self._track_job() # Now delete the temporaries, if they're there. - if (self.tmp_dir and os.path.exists(self.tmp_dir) and - not self.dont_remove_tmp_dir): + if (self.tmp_dir and os.path.exists(self.tmp_dir) and not self.dont_remove_tmp_dir): logger.info('Removing temporary directory %s' % self.tmp_dir) subprocess.call(["rm", "-rf", self.tmp_dir])