Create app for valid pipelines, as part of #751.

Also filter parent containers.
Make driver scripts executable.

kive/container/forms.py

@@ -27,10 +27,14 @@ class Meta(object):
 
 
 class ContainerForm(PermissionsForm):
+    parent = forms.ModelChoiceField(
+       help_text=Container.parent.field.help_text,
+       queryset=Container.objects.filter(file_type=Container.SIMG))
+
     class Meta(object):
         model = Container
         fields = ['file', 'parent', 'tag', 'description', 'permissions']
-        widgets = dict(description=forms.Textarea(attrs=dict(cols=50, rows=10))) # FIXME figure out a widget for parent
+        widgets = dict(description=forms.Textarea(attrs=dict(cols=50, rows=10)))
 
     def __init__(self, *args, **kwargs):
         super(ContainerForm, self).__init__(*args, **kwargs)

kive/container/management/commands/runcontainer.py

@@ -250,6 +250,9 @@ def run_pipeline(self,
             # - inputs (a list of (step_num, dataset_name) pairs)
             # - outputs (a list of dataset_names)
             executable = os.path.join(internal_binary_dir, step["driver"])
+            driver_external_path = os.path.join(extracted_archive_dir,
+                                                step["driver"])
+            os.chmod(driver_external_path, 0o777)
             input_paths = []
             for input_dict in step["inputs"]:
                 source_step = input_dict["source_step"]

kive/container/migrations/0017_container_details.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-02-08 23:01
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0016_container_file_help_text'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='container',
+            name='file_type',
+            field=models.CharField(choices=[('SIMG', 'Singularity'), ('ZIP', 'Zip'), ('TAR', 'Tar')], default='SIMG', max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='container',
+            name='parent',
+            field=models.ForeignKey(blank=True, help_text='Singularity container that an archive container runs in', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='container.Container'),
+        ),
+    ]

kive/container/models.py

@@ -167,10 +167,12 @@ class Container(AccessControl):
         default=SIMG,
         max_length=20)
 
-    parent = models.ForeignKey("Container",
-                               related_name="children",
-                               null=True,
-                               blank=True)
+    parent = models.ForeignKey(
+        "Container",
+        related_name="children",
+        null=True,
+        blank=True,
+        help_text='Singularity container that an archive container runs in')
 
     tag = models.CharField('Tag',
                            help_text='Git tag or revision name',
@@ -348,7 +350,8 @@ def get_content(self):
             return content
 
     def write_content(self, content):
-        pipeline_json = json.dumps(content['pipeline'])
+        pipeline = content['pipeline']
+        pipeline_json = json.dumps(pipeline)
         with self.open_content('a') as archive:
             file_names = set(entry.name
                              for entry in archive.infolist()
@@ -358,6 +361,21 @@ def write_content(self, content):
                 if file_name not in file_names:
                     archive.write(file_name, pipeline_json)
                     break
+        # Totally basic validation for now.
+        is_valid = min(len(pipeline['inputs']),
+                       len(pipeline['steps']),
+                       len(pipeline['outputs'])) > 0
+        if is_valid:
+            self.apps.all().delete()
+            default_config = pipeline['default_config']
+            app = self.apps.create(memory=default_config['memory'],
+                                   threads=default_config['threads'])
+            input_names = ' '.join(entry['dataset_name']
+                                   for entry in pipeline['inputs'])
+            output_names = ' '.join(entry['dataset_name']
+                                    for entry in pipeline['outputs'])
+            app.write_inputs(input_names)
+            app.write_outputs(output_names)
 
     def get_absolute_url(self):
         return reverse('container_update', kwargs=dict(pk=self.pk))

kive/container/tests.py

@@ -213,11 +213,13 @@ def test_write_tar_content(self):
                                               inputs=[],
                                               steps=[],
                                               outputs=[]))
+        expected_apps_count = 0  # Pipeline is incomplete, so no app created.
 
         container.write_content(expected_content)
         content = container.get_content()
 
         self.assertEqual(expected_content, content)
+        self.assertEqual(expected_apps_count, container.apps.count())
 
     def test_rewrite_content(self):
         user = User.objects.first()
@@ -241,6 +243,40 @@ def test_rewrite_content(self):
 
         self.assertEqual(expected_content, content)
 
+    def test_write_content_and_app(self):
+        user = User.objects.first()
+        family = ContainerFamily.objects.create(user=user)
+        container = Container.objects.create(family=family, user=user)
+        self.create_tar_content(container)
+        container.save()
+        expected_content = dict(
+            files=["bar.txt", "foo.txt"],
+            pipeline=dict(default_config=dict(memory=200,
+                                              threads=2),
+                          inputs=[dict(dataset_name='in1')],
+                          steps=[dict(driver='foo.txt',
+                                      inputs=[dict(dataset_name="in1",
+                                                   source_step=0,
+                                                   source_dataset_name="in1")],
+                                      outputs=["out1"])],
+                          outputs=[dict(dataset_name="out1",
+                                        source_step=1,
+                                        source_dataset_name="out1")]))
+        expected_apps_count = 1
+        expected_memory = 200
+        expected_threads = 2
+        expected_inputs = "in1"
+        expected_outputs = "out1"
+
+        container.write_content(expected_content)
+
+        self.assertEqual(expected_apps_count, container.apps.count())
+        app = container.apps.first()
+        self.assertEqual(expected_memory, app.memory)
+        self.assertEqual(expected_threads, app.threads)
+        self.assertEqual(expected_inputs, app.inputs)
+        self.assertEqual(expected_outputs, app.outputs)
+
     def test_extract_zip(self):
         run = ContainerRun()
         run.create_sandbox(prefix='test_extract_zip')
@@ -1059,7 +1095,6 @@ def test_run_archive(self):
         tar_data = BytesIO()
         with TarFile(fileobj=tar_data, mode='w') as t:
             tar_info = TarInfo('greetings.py')
-            tar_info.mode = 0o777
             tar_info.size = len(script_text)
             t.addfile(tar_info, BytesIO(script_text))
         tar_data.seek(0)
@@ -1122,7 +1157,6 @@ def test_step_stdout(self):
         tar_data = BytesIO()
         with TarFile(fileobj=tar_data, mode='w') as t:
             tar_info = TarInfo('greetings.py')
-            tar_info.mode = 0o777
             tar_info.size = len(script_text)
             t.addfile(tar_info, BytesIO(script_text))
         tar_data.seek(0)
@@ -1201,7 +1235,6 @@ def test_run_multistep_archive(self):
                     script_text = f.read()
                     script_text = b'#!/usr/bin/env python\n' + script_text
                 tar_info = TarInfo(script_name)
-                tar_info.mode = 0o777
                 tar_info.size = len(script_text)
                 t.addfile(tar_info, BytesIO(script_text))
         tar_data.seek(0)

samplecode/singularity/greetings.py

@@ -6,7 +6,7 @@
 def parse_args():
     parser = ArgumentParser()
     parser.add_argument('names_csv', type=FileType())
-    parser.add_argument('greetings_csv', type=FileType('wb'))
+    parser.add_argument('greetings_csv', type=FileType('w'))
 
     return parser.parse_args()