diff --git a/kive/container/forms.py b/kive/container/forms.py index da8e83845..376d6cb36 100644 --- a/kive/container/forms.py +++ b/kive/container/forms.py @@ -27,10 +27,14 @@ class Meta(object): class ContainerForm(PermissionsForm): + parent = forms.ModelChoiceField( + help_text=Container.parent.field.help_text, + queryset=Container.objects.filter(file_type=Container.SIMG)) + class Meta(object): model = Container fields = ['file', 'parent', 'tag', 'description', 'permissions'] - widgets = dict(description=forms.Textarea(attrs=dict(cols=50, rows=10))) # FIXME figure out a widget for parent + widgets = dict(description=forms.Textarea(attrs=dict(cols=50, rows=10))) def __init__(self, *args, **kwargs): super(ContainerForm, self).__init__(*args, **kwargs) diff --git a/kive/container/management/commands/runcontainer.py b/kive/container/management/commands/runcontainer.py index f3cc92354..60e07f3d0 100644 --- a/kive/container/management/commands/runcontainer.py +++ b/kive/container/management/commands/runcontainer.py @@ -250,6 +250,9 @@ def run_pipeline(self, # - inputs (a list of (step_num, dataset_name) pairs) # - outputs (a list of dataset_names) executable = os.path.join(internal_binary_dir, step["driver"]) + driver_external_path = os.path.join(extracted_archive_dir, + step["driver"]) + os.chmod(driver_external_path, 0o777) input_paths = [] for input_dict in step["inputs"]: source_step = input_dict["source_step"] diff --git a/kive/container/migrations/0017_container_details.py b/kive/container/migrations/0017_container_details.py new file mode 100644 index 000000000..3150884a4 --- /dev/null +++ b/kive/container/migrations/0017_container_details.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2019-02-08 23:01 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('container', '0016_container_file_help_text'), + ] + + operations = [ + migrations.AlterField( + model_name='container', + name='file_type', + field=models.CharField(choices=[('SIMG', 'Singularity'), ('ZIP', 'Zip'), ('TAR', 'Tar')], default='SIMG', max_length=20), + ), + migrations.AlterField( + model_name='container', + name='parent', + field=models.ForeignKey(blank=True, help_text='Singularity container that an archive container runs in', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='container.Container'), + ), + ] diff --git a/kive/container/models.py b/kive/container/models.py index a43566a56..da0ded1e1 100644 --- a/kive/container/models.py +++ b/kive/container/models.py @@ -167,10 +167,12 @@ class Container(AccessControl): default=SIMG, max_length=20) - parent = models.ForeignKey("Container", - related_name="children", - null=True, - blank=True) + parent = models.ForeignKey( + "Container", + related_name="children", + null=True, + blank=True, + help_text='Singularity container that an archive container runs in') tag = models.CharField('Tag', help_text='Git tag or revision name', @@ -348,7 +350,8 @@ def get_content(self): return content def write_content(self, content): - pipeline_json = json.dumps(content['pipeline']) + pipeline = content['pipeline'] + pipeline_json = json.dumps(pipeline) with self.open_content('a') as archive: file_names = set(entry.name for entry in archive.infolist() @@ -358,6 +361,21 @@ def write_content(self, content): if file_name not in file_names: archive.write(file_name, pipeline_json) break + # Totally basic validation for now. + is_valid = min(len(pipeline['inputs']), + len(pipeline['steps']), + len(pipeline['outputs'])) > 0 + if is_valid: + self.apps.all().delete() + default_config = pipeline['default_config'] + app = self.apps.create(memory=default_config['memory'], + threads=default_config['threads']) + input_names = ' '.join(entry['dataset_name'] + for entry in pipeline['inputs']) + output_names = ' '.join(entry['dataset_name'] + for entry in pipeline['outputs']) + app.write_inputs(input_names) + app.write_outputs(output_names) def get_absolute_url(self): return reverse('container_update', kwargs=dict(pk=self.pk)) diff --git a/kive/container/tests.py b/kive/container/tests.py index e8a4f9ebe..66f3b7a8a 100644 --- a/kive/container/tests.py +++ b/kive/container/tests.py @@ -213,11 +213,13 @@ def test_write_tar_content(self): inputs=[], steps=[], outputs=[])) + expected_apps_count = 0 # Pipeline is incomplete, so no app created. container.write_content(expected_content) content = container.get_content() self.assertEqual(expected_content, content) + self.assertEqual(expected_apps_count, container.apps.count()) def test_rewrite_content(self): user = User.objects.first() @@ -241,6 +243,40 @@ def test_rewrite_content(self): self.assertEqual(expected_content, content) + def test_write_content_and_app(self): + user = User.objects.first() + family = ContainerFamily.objects.create(user=user) + container = Container.objects.create(family=family, user=user) + self.create_tar_content(container) + container.save() + expected_content = dict( + files=["bar.txt", "foo.txt"], + pipeline=dict(default_config=dict(memory=200, + threads=2), + inputs=[dict(dataset_name='in1')], + steps=[dict(driver='foo.txt', + inputs=[dict(dataset_name="in1", + source_step=0, + source_dataset_name="in1")], + outputs=["out1"])], + outputs=[dict(dataset_name="out1", + source_step=1, + source_dataset_name="out1")])) + expected_apps_count = 1 + expected_memory = 200 + expected_threads = 2 + expected_inputs = "in1" + expected_outputs = "out1" + + container.write_content(expected_content) + + self.assertEqual(expected_apps_count, container.apps.count()) + app = container.apps.first() + self.assertEqual(expected_memory, app.memory) + self.assertEqual(expected_threads, app.threads) + self.assertEqual(expected_inputs, app.inputs) + self.assertEqual(expected_outputs, app.outputs) + def test_extract_zip(self): run = ContainerRun() run.create_sandbox(prefix='test_extract_zip') @@ -1059,7 +1095,6 @@ def test_run_archive(self): tar_data = BytesIO() with TarFile(fileobj=tar_data, mode='w') as t: tar_info = TarInfo('greetings.py') - tar_info.mode = 0o777 tar_info.size = len(script_text) t.addfile(tar_info, BytesIO(script_text)) tar_data.seek(0) @@ -1122,7 +1157,6 @@ def test_step_stdout(self): tar_data = BytesIO() with TarFile(fileobj=tar_data, mode='w') as t: tar_info = TarInfo('greetings.py') - tar_info.mode = 0o777 tar_info.size = len(script_text) t.addfile(tar_info, BytesIO(script_text)) tar_data.seek(0) @@ -1201,7 +1235,6 @@ def test_run_multistep_archive(self): script_text = f.read() script_text = b'#!/usr/bin/env python\n' + script_text tar_info = TarInfo(script_name) - tar_info.mode = 0o777 tar_info.size = len(script_text) t.addfile(tar_info, BytesIO(script_text)) tar_data.seek(0) diff --git a/samplecode/singularity/greetings.py b/samplecode/singularity/greetings.py index cbfc75a8d..6ebdf8a83 100644 --- a/samplecode/singularity/greetings.py +++ b/samplecode/singularity/greetings.py @@ -6,7 +6,7 @@ def parse_args(): parser = ArgumentParser() parser.add_argument('names_csv', type=FileType()) - parser.add_argument('greetings_csv', type=FileType('wb')) + parser.add_argument('greetings_csv', type=FileType('w')) return parser.parse_args()