Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added --dont-remove-tmp-dir parameter to SGEJobTask. #1798

Merged
merged 2 commits into from
Aug 4, 2016
Merged

Added --dont-remove-tmp-dir parameter to SGEJobTask. #1798

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0cd961
Added --dont-remove-tmp-dir parameter to SGEJobTask.
Aug 1, 2016
44dcd29
Changed line formatting to match luigi style.
Aug 1, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions luigi/contrib/sge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,18 +170,22 @@ class SGEJobTask(luigi.Task):
by StarCluster
- run_locally: Run locally instead of on the cluster.
- poll_time: the length of time to wait in order to poll qstat
- dont_remove_tmp_dir: Instead of deleting the temporary directory, keep it.

"""

n_cpu = luigi.IntParameter(default=2, significant=False)
shared_tmp_dir = luigi.Parameter(default='/home', significant=False)
parallel_env = luigi.Parameter(default='orte', significant=False)
run_locally = luigi.BoolParameter(
default=False, significant=False,
significant=False,
description="run locally instead of on the cluster")
poll_time = luigi.IntParameter(
significant=False, default=POLL_TIME,
description="specify the wait time to poll qstat for the job status")
dont_remove_tmp_dir = luigi.BoolParameter(
significant=False,
description="don't delete the temporary directory used (for debugging)")

def _fetch_task_failures(self):
if not os.path.exists(self.errfile):
Expand Down Expand Up @@ -271,7 +275,7 @@ def _run_job(self):
self._track_job()

# Now delete the temporaries, if they're there.
if self.tmp_dir and os.path.exists(self.tmp_dir):
if (self.tmp_dir and os.path.exists(self.tmp_dir) and not self.dont_remove_tmp_dir):
logger.info('Removing temporary directory %s' % self.tmp_dir)
subprocess.call(["rm", "-rf", self.tmp_dir])

Expand Down