Test multiple pipeline steps, as part of #751.

Also configure the vagrant user to be able to run tests.

kive/container/tests.py

@@ -1069,20 +1069,20 @@ def test_run_archive(self):
             parent=old_container,
             family=old_container.family,
             user=old_container.user,
-            tag='zip_test',
+            tag='tar_test',
             file_type=Container.TAR)
         container.file.save('test_howdy.tar', ContentFile(tar_data.getvalue()))
         container.write_content(content)
         run.app = container.apps.create(memory=200, threads=1)
         run.app.write_inputs('names_csv')
         run.app.write_outputs('greetings_csv')
         run.save()
-        expected_greetings = '''\
-greeting\r
-"Howdy, Alice"\r
-"Howdy, Bob"\r
-"Howdy, Carolyn"\r
-"Howdy, Darius"\r
+        expected_greetings = b'''\
+greeting
+"Howdy, Alice"
+"Howdy, Bob"
+"Howdy, Carolyn"
+"Howdy, Darius"
 '''
         expected_stderr = """\
 ========
@@ -1102,6 +1102,94 @@ def test_run_archive(self):
         greetings = output_dataset.dataset_file.read()
         self.assertEqual(expected_greetings, greetings)
 
+    def test_run_multistep_archive(self):
+        pairs_text = """\
+x,y
+0,1
+1,1
+1,2
+2,3
+"""
+        expected_summary = b"""\
+sum,product,bigger
+1,0,sum
+2,1,sum
+3,2,sum
+5,6,product
+"""
+        source_path = os.path.abspath(os.path.join(__file__,
+                                                   '..',
+                                                   '..',
+                                                   '..',
+                                                   'samplecode',
+                                                   'singularity'))
+        content = dict(pipeline=dict(
+            inputs=[dict(dataset_name="pairs_csv")],
+            steps=[dict(driver="sums_and_products.py",
+                        inputs=[dict(dataset_name="pairs_csv",
+                                     source_step=0,
+                                     source_dataset_name="pairs_csv")],
+                        outputs=["sums_csv"]),
+                   dict(driver="sum_summary.py",
+                        inputs=[dict(dataset_name="sums_csv",
+                                     source_step=1,
+                                     source_dataset_name="sums_csv")],
+                        outputs=["summary_csv"])],
+            outputs=[dict(dataset_name="summary_csv",
+                          source_step=2,
+                          source_dataset_name="summary_csv")]))
+        tar_data = BytesIO()
+        with TarFile(fileobj=tar_data, mode='w') as t:
+            for script_name in ('sums_and_products.py', 'sum_summary.py'):
+                with open(os.path.join(source_path, script_name), 'rb') as f:
+                    script_text = f.read()
+                    script_text = b'#!/usr/bin/env python\n' + script_text
+                tar_info = TarInfo(script_name)
+                tar_info.mode = 0o777
+                tar_info.size = len(script_text)
+                t.addfile(tar_info, BytesIO(script_text))
+        tar_data.seek(0)
+        run = ContainerRun.objects.get(name='fixture run')
+        old_container = run.app.container
+        container = Container.objects.create(
+            parent=old_container,
+            family=old_container.family,
+            user=old_container.user,
+            tag='multi_test',
+            file_type=Container.TAR)
+        container.file.save('test_multi.tar', ContentFile(tar_data.getvalue()))
+        container.write_content(content)
+        run.app = container.apps.create(memory=200, threads=1)
+        run.app.write_inputs('pairs_csv')
+        run.app.write_outputs('summary_csv')
+        pairs_dataset = Dataset.objects.create(user=run.user, name='pairs.csv')
+        pairs_dataset.dataset_file.save('pairs.csv', ContentFile(pairs_text))
+        run_input = run.datasets.get()
+        run_input.dataset = pairs_dataset
+        run_input.argument = run.app.arguments.get(type=ContainerArgument.INPUT)
+        run_input.save()
+        run.save()
+        expected_stderr = """\
+========
+Processing step 1: sums_and_products.py
+========
+========
+Processing step 2: sum_summary.py
+========
+"""
+
+        call_command('runcontainer', str(run.id))
+
+        run.refresh_from_db()
+
+        stderr_log = run.logs.get(type=ContainerLog.STDERR)
+        self.assertEqual(expected_stderr, stderr_log.read(1000))
+        self.assertEqual(ContainerRun.COMPLETE, run.state)
+        output_dataset = run.datasets.get(
+            argument__type=ContainerArgument.OUTPUT).dataset
+        summary = output_dataset.dataset_file.read()
+        self.assertEqual(expected_summary, summary)
+
     def test_already_started(self):
         """ Pretend that another instance of the command already started. """
         run = ContainerRun.objects.get(name='fixture run')

samplecode/singularity/greetings.py

@@ -1,3 +1,4 @@
+import os
 from argparse import ArgumentParser, FileType
 from csv import DictReader, DictWriter
 
@@ -13,7 +14,9 @@ def parse_args():
 def main():
     args = parse_args()
     reader = DictReader(args.names_csv)
-    writer = DictWriter(args.greetings_csv, ['greeting'])
+    writer = DictWriter(args.greetings_csv,
+                        ['greeting'],
+                        lineterminator=os.linesep)
     writer.writeheader()
     for row in reader:
         writer.writerow(dict(greeting='Hello, ' + row['name']))

samplecode/singularity/sum_summary.py

@@ -0,0 +1,23 @@
+from argparse import FileType, ArgumentParser
+import csv
+import os
+
+parser = ArgumentParser(
+    description="Checks which is bigger: sum or product.")
+parser.add_argument("sums_csv",
+                    type=FileType('rU'),
+                    help="CSV containing (sum,product) pairs")
+parser.add_argument("summary_csv",
+                    type=FileType('wb'),
+                    help="CSV containing (sum,product,bigger) rows")
+args = parser.parse_args()
+
+reader = csv.DictReader(args.sums_csv)
+writer = csv.DictWriter(args.summary_csv,
+                        ['sum', 'product', 'bigger'],
+                        lineterminator=os.linesep)
+writer.writeheader()
+
+for row in reader:
+    row['bigger'] = 'sum' if row['product'] < row['sum'] else 'product'
+    writer.writerow(row)

vagrant/centos_dependencies.bash

@@ -8,6 +8,16 @@ cd /root
 echo ========== Installing PostgreSQL ==========
 yum install -q -y postgresql-server postgresql-contrib
 postgresql-setup initdb
+
+# Order matters for access rules.
+sudo sed -i '0,/^local/s/^local/local all kive      peer map=vagrantkive\n&/' \
+    /var/lib/pgsql/data/pg_hba.conf
+
+echo "
+# MAPNAME       SYSTEM-USERNAME         PG-USERNAME
+vagrantkive     vagrant                 kive
+vagrantkive     kive                    kive
+" >> /var/lib/pgsql/data/pg_ident.conf
 systemctl enable postgresql
 systemctl start postgresql
 
@@ -114,3 +124,7 @@ sed -e 's/Listen 80$/Listen 8080/' \
     -e 's/Group apache$/Group kive/' -i /etc/httpd/conf/httpd.conf
 systemctl enable httpd
 systemctl start httpd
+
+echo ========= Configuring vagrant user ===========
+cat /usr/local/share/Kive/vagrant_ubuntu/envvars.conf >> ~vagrant/.bash_profile
+echo ". /opt/venv_kive/bin/activate" >> ~vagrant/.bash_profile

vagrant_ubuntu/dbcreate.sh

@@ -26,16 +26,3 @@ sudo chown -R kive:kive /var/kive
 sudo -u kive /opt/venv_kive/bin/python ../kive/manage.py migrate
 
 echo "Created kive database."
-
-# Enable the vagrant user to access Postgres as the `kive` user.
-mv /var/lib/pgsql/data/pg_ident.conf /var/lib/pgsql/data/pg_ident.conf.orig
-mv /var/lib/pgsql/data/pg_hba.conf /var/lib/pgsql/data/pg_hba.conf.orig
-cp /usr/local/share/Kive/vagrant/pg_ident.conf /var/lib/pgsql/data/pg_ident.conf
-cp /usr/local/share/Kive/vagrant/pg_hba.conf /var/lib/pgsql/data/pg_hba.conf
-chown postgres:postgres /var/lib/pgsql/data/pg_ident.conf
-chown postgres:postgres /var/lib/pgsql/data/pg_hba.conf
-chmod 600 /var/lib/pgsql/data/pg_ident.conf
-chmod 600 /var/lib/pgsql/data/pg_hba.conf
-systemctl reload postgresql
-
-echo "The default vagrant user can now access Postgres as kive."